Merge branch 'master' into wip/tests

.travis.yml

@@ -0,0 +1,39 @@
+language: go
+sudo: false
+matrix:
+  include:
+    - go: 1.x
+      env: LATEST=true
+    - go: 1.9
+    - go: 1.8
+
+before_install:
+  - go get github.com/mitchellh/gox # go tool for cross compiling
+  - go get github.com/inconshreveable/mousetrap # needed for windows builds
+
+install:
+  - go get github.com/RadhiFadlillah/shiori
+
+script:
+  - go get -t -v ./...
+  - diff -u <(echo -n) <(gofmt -d .)
+  - go vet $(go list ./... | grep -v /vendor/)
+  - go test -v -race ./...
+  # only build binaries from the latest Go release.
+  - if [ "${LATEST}" = "true" ]; then gox -os="linux darwin windows" -arch="amd64" -output "shiori_{{.OS}}_{{.Arch}}" -ldflags "-X main.Rev=`git rev-parse --short HEAD`" -verbose ./...; fi
+
+deploy:
+  provider: releases
+  skip_cleanup: true
+  api_key:
+    secure: $TOKEN
+  file:
+  - shiori_windows_amd64.exe
+  - shiori_darwin_amd64
+  - shiori_linux_amd64
+  on:
+    tags: true
+    branches:
+      only:
+        - master
+    condition: $LATEST = true

README.md

@@ -1,5 +1,8 @@
 # Shiori
 
+[![Travis CI](https://travis-ci.org/RadhiFadlillah/shiori.svg?branch=master)](https://travis-ci.org/RadhiFadlillah/shiori)
+[![Go Report Card](https://goreportcard.com/badge/github.com/radhifadlillah/shiori)](https://goreportcard.com/report/github.com/radhifadlillah/shiori)
+
 Shiori is a simple bookmarks manager written in Go language. Intended as a simple clone of [Pocket](https://getpocket.com//). You can use it as command line application or as web application. This application is distributed as a single binary, which means it can be installed and used easily.
 
 ![Screenshot](https://raw.githubusercontent.com/RadhiFadlillah/shiori/master/screenshot.png)
@@ -9,6 +12,7 @@ Shiori is a simple bookmarks manager written in Go language. Intended as a simpl
 - [Features](#features)
 - [Installation](#installation)
 - [Usage](#usage)
+- [Advanced](#advanced)
 - [Examples](#examples)
 - [License](#license)
 
@@ -57,6 +61,10 @@ Flags:
 Use "shiori [command] --help" for more information about a command.
 ```
 
+### Advanced
+
+By default, `shiori` will create database in the location where you run it. For example, if you run `shiori`. To set the database to a specific location, you can set the environment variable `ENV_SHIORI_DB` to your desired path. 
+
 ## Examples
 
 1. Save new bookmark with tags "nature" and "climate-change".

cmd/add.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"html/template"
+	nurl "net/url"
 	"strings"
 	"time"
 
@@ -59,6 +60,12 @@ func addBookmark(base model.Bookmark, offline bool) (book model.Bookmark, err er
 	// Prepare initial result
 	book = base
 
+	// Clear UTM parameters from URL
+	book.URL, err = clearUTMParams(book.URL)
+	if err != nil {
+		return book, err
+	}
+
 	// Fetch data from internet
 	if !offline {
 		article, err := readability.Parse(book.URL, 10*time.Second)
@@ -94,3 +101,22 @@ func addBookmark(base model.Bookmark, offline bool) (book model.Bookmark, err er
 func normalizeSpace(str string) string {
 	return strings.Join(strings.Fields(str), " ")
 }
+
+func clearUTMParams(uri string) (string, error) {
+	tempURL, err := nurl.Parse(uri)
+	if err != nil {
+		return "", err
+	}
+
+	newQuery := nurl.Values{}
+	for key, value := range tempURL.Query() {
+		if strings.HasPrefix(key, "utm_") {
+			continue
+		}
+
+		newQuery[key] = value
+	}
+
+	tempURL.RawQuery = newQuery.Encode()
+	return tempURL.String(), nil
+}

cmd/delete.go

@@ -2,7 +2,6 @@ package cmd
 
 import (
 	"fmt"
-	"os"
 
 	"github.com/spf13/cobra"
 )
@@ -32,16 +31,9 @@ var (
 			}
 
 			// Delete bookmarks from database
-			oldIndices, newIndices, err := DB.DeleteBookmarks(args...)
+			err := DB.DeleteBookmarks(args...)
 			if err != nil {
 				cError.Println(err)
-				os.Exit(1)
-			}
-
-			fmt.Println("Bookmarks has been deleted")
-			for i, oldIndex := range oldIndices {
-				newIndex := newIndices[i]
-				fmt.Printf("Index %d moved to %d\n", oldIndex, newIndex)
 			}
 		},
 	}

cmd/import.go

@@ -41,15 +41,6 @@ func init() {
 	rootCmd.AddCommand(importCmd)
 }
 
-func printTagName(s *goquery.Selection) string {
-	tags := []string{}
-	for _, nd := range s.Nodes {
-		tags = append(tags, nd.Data)
-	}
-
-	return strings.Join(tags, ",")
-}
-
 func importBookmarks(pth string, generateTag bool) error {
 	// Open file
 	srcFile, err := os.Open(pth)
@@ -74,10 +65,19 @@ func importBookmarks(pth string, generateTag bool) error {
 		// Get metadata
 		title := a.Text()
 		url, _ := a.Attr("href")
+		strTags, _ := a.Attr("tags")
 		strModified, _ := a.Attr("last_modified")
 		intModified, _ := strconv.ParseInt(strModified, 10, 64)
 		modified := time.Unix(intModified, 0)
 
+		// Get bookmark tags
+		tags := []model.Tag{}
+		for _, strTag := range strings.Split(strTags, ",") {
+			if strTag != "" {
+				tags = append(tags, model.Tag{Name: strTag})
+			}
+		}
+
 		// Get bookmark excerpt
 		excerpt := ""
 		if dd := dt.Next(); dd.Is("dd") {
@@ -85,6 +85,7 @@ func importBookmarks(pth string, generateTag bool) error {
 		}
 
 		// Get category name for this bookmark
+		// and add it as tags (if necessary)
 		category := ""
 		if dtCategory := dl.Prev(); dtCategory.Is("h3") {
 			category = dtCategory.Text()
@@ -93,9 +94,8 @@ func importBookmarks(pth string, generateTag bool) error {
 			category = strings.Replace(category, " ", "-", -1)
 		}
 
-		tags := []model.Tag{}
 		if category != "" && generateTag {
-			tags = []model.Tag{{Name: category}}
+			tags = append(tags, model.Tag{Name: category})
 		}
 
 		// Add item to list

cmd/open.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"fmt"
 	"os/exec"
+	"runtime"
 	"strings"
 
 	"github.com/spf13/cobra"
@@ -67,7 +68,7 @@ func openBookmarks(args ...string) {
 
 	// Open in browser
 	for _, book := range bookmarks {
-		exec.Command("xdg-open", book.URL).Run()
+		err = openBrowser(book.URL)
 		if err != nil {
 			cError.Printf("Failed to open %s: %v\n", book.URL, err)
 		}
@@ -110,3 +111,20 @@ func openBookmarksCache(trimSpace bool, args ...string) {
 		fmt.Println()
 	}
 }
+
+// openBrowser tries to open the URL in a browser,
+// and returns whether it succeed in doing so.
+func openBrowser(url string) error {
+	var args []string
+	switch runtime.GOOS {
+	case "darwin":
+		args = []string{"open"}
+	case "windows":
+		args = []string{"cmd", "/c", "start"}
+	default:
+		args = []string{"xdg-open"}
+	}
+
+	cmd := exec.Command(args[0], append(args[1:], url)...)
+	return cmd.Run()
+}

cmd/print.go

@@ -3,7 +3,6 @@ package cmd
 import (
 	"encoding/json"
 	"fmt"
-	"os"
 	"strings"
 
 	"github.com/RadhiFadlillah/shiori/model"
@@ -26,7 +25,7 @@ var (
 			bookmarks, err := DB.GetBookmarks(false, args...)
 			if err != nil {
 				cError.Println(err)
-				os.Exit(1)
+				return
 			}
 
 			if len(bookmarks) == 0 {
@@ -36,7 +35,7 @@ var (
 					cError.Println("No bookmarks saved yet")
 				}
 
-				os.Exit(1)
+				return
 			}
 
 			// Print data
@@ -44,7 +43,7 @@ var (
 				bt, err := json.MarshalIndent(&bookmarks, "", "    ")
 				if err != nil {
 					cError.Println(err)
-					os.Exit(1)
+					return
 				}
 				fmt.Println(string(bt))
 			} else if indexOnly {

cmd/root.go

@@ -2,7 +2,6 @@ package cmd
 
 import (
 	"fmt"
-	"os"
 
 	"github.com/RadhiFadlillah/shiori/database"
 	"github.com/spf13/cobra"
@@ -23,6 +22,5 @@ var (
 func Execute() {
 	if err := rootCmd.Execute(); err != nil {
 		fmt.Println(err)
-		os.Exit(1)
 	}
 }

cmd/search.go

@@ -3,7 +3,6 @@ package cmd
 import (
 	"encoding/json"
 	"fmt"
-	"os"
 
 	"github.com/spf13/cobra"
 )
@@ -34,12 +33,12 @@ var (
 			bookmarks, err := DB.SearchBookmarks(false, keyword, tags...)
 			if err != nil {
 				cError.Println(err)
-				os.Exit(1)
+				return
 			}
 
 			if len(bookmarks) == 0 {
 				cError.Println("No matching bookmarks found")
-				os.Exit(1)
+				return
 			}
 
 			// Print data
@@ -47,7 +46,7 @@ var (
 				bt, err := json.MarshalIndent(&bookmarks, "", "    ")
 				if err != nil {
 					cError.Println(err)
-					os.Exit(1)
+					return
 				}
 				fmt.Println(string(bt))
 			} else if indexOnly {

cmd/serve.go

@@ -266,7 +266,7 @@ func apiDeleteBookmarks(w http.ResponseWriter, r *http.Request, ps httprouter.Pa
 	checkError(err)
 
 	// Delete bookmarks
-	_, _, err = DB.DeleteBookmarks(request...)
+	err = DB.DeleteBookmarks(request...)
 	checkError(err)
 
 	fmt.Fprint(w, request)

cmd/update.go

@@ -82,6 +82,13 @@ func init() {
 
 func updateBookmarks(indices []string, url, title, excerpt string, tags []string, offline bool) ([]model.Bookmark, error) {
 	mutex := sync.Mutex{}
+
+	// Clear UTM parameters from URL
+	url, err := clearUTMParams(url)
+	if err != nil {
+		return []model.Bookmark{}, err
+	}
+
 	// Read bookmarks from database
 	bookmarks, err := DB.GetBookmarks(true, indices...)
 	if err != nil {

cmd/utils.go

@@ -1,9 +1,10 @@
 package cmd
 
 import (
+	"os"
+
 	"github.com/fatih/color"
 	"golang.org/x/crypto/ssh/terminal"
-	"os"
 )
 
 var (

database/database.go

@@ -2,6 +2,7 @@ package database
 
 import (
 	"database/sql"
+
 	"github.com/RadhiFadlillah/shiori/model"
 )
 
@@ -14,7 +15,7 @@ type Database interface {
 	GetBookmarks(withContent bool, indices ...string) ([]model.Bookmark, error)
 
 	// DeleteBookmarks removes all record with matching indices from database.
-	DeleteBookmarks(indices ...string) ([]int, []int, error)
+	DeleteBookmarks(indices ...string) error
 
 	// SearchBookmarks search bookmarks by the keyword or tags.
 	SearchBookmarks(orderLatest bool, keyword string, tags ...string) ([]model.Bookmark, error)

database/sqlite.go

@@ -3,13 +3,13 @@ package database
 import (
 	"database/sql"
 	"fmt"
-	"github.com/RadhiFadlillah/shiori/model"
-	"github.com/jmoiron/sqlx"
-	"golang.org/x/crypto/bcrypt"
-	"sort"
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/RadhiFadlillah/shiori/model"
+	"github.com/jmoiron/sqlx"
+	"golang.org/x/crypto/bcrypt"
 )
 
 // SQLiteDatabase is implementation of Database interface for connecting to SQLite3 database.
@@ -18,10 +18,10 @@ type SQLiteDatabase struct {
 }
 
 // OpenSQLiteDatabase creates and open connection to new SQLite3 database.
-func OpenSQLiteDatabase(dbFile string) (*SQLiteDatabase, error) {
+func OpenSQLiteDatabase(databasePath string) (*SQLiteDatabase, error) {
 	// Open database and start transaction
 	var err error
-	db := sqlx.MustConnect("sqlite3", dbFile)
+	db := sqlx.MustConnect("sqlite3", databasePath)
 	tx := db.MustBegin()
 
 	// Make sure to rollback if panic ever happened
@@ -262,7 +262,7 @@ func (db *SQLiteDatabase) GetBookmarks(withContent bool, indices ...string) ([]m
 }
 
 // DeleteBookmarks removes all record with matching indices from database.
-func (db *SQLiteDatabase) DeleteBookmarks(indices ...string) (oldIndices, newIndices []int, err error) {
+func (db *SQLiteDatabase) DeleteBookmarks(indices ...string) (err error) {
 	// Convert list of index to int
 	listIndex := []int{}
 	errInvalidIndex := fmt.Errorf("Index is not valid")
@@ -271,13 +271,13 @@ func (db *SQLiteDatabase) DeleteBookmarks(indices ...string) (oldIndices, newInd
 		if strings.Contains(strIndex, "-") {
 			parts := strings.Split(strIndex, "-")
 			if len(parts) != 2 {
-				return nil, nil, errInvalidIndex
+				return errInvalidIndex
 			}
 
 			minIndex, errMin := strconv.Atoi(parts[0])
 			maxIndex, errMax := strconv.Atoi(parts[1])
 			if errMin != nil || errMax != nil || minIndex < 1 || minIndex > maxIndex {
-				return nil, nil, errInvalidIndex
+				return errInvalidIndex
 			}
 
 			for i := minIndex; i <= maxIndex; i++ {
@@ -286,16 +286,13 @@ func (db *SQLiteDatabase) DeleteBookmarks(indices ...string) (oldIndices, newInd
 		} else {
 			index, err := strconv.Atoi(strIndex)
 			if err != nil || index < 1 {
-				return nil, nil, errInvalidIndex
+				return errInvalidIndex
 			}
 
 			listIndex = append(listIndex, index)
 		}
 	}
 
-	// Sort the index
-	sort.Ints(listIndex)
-
 	// Create args and where clause
 	args := []interface{}{}
 	whereClause := " WHERE 1"
@@ -314,7 +311,7 @@ func (db *SQLiteDatabase) DeleteBookmarks(indices ...string) (oldIndices, newInd
 	// Begin transaction
 	tx, err := db.Beginx()
 	if err != nil {
-		return nil, nil, errInvalidIndex
+		return errInvalidIndex
 	}
 
 	// Make sure to rollback if panic ever happened
@@ -323,8 +320,6 @@ func (db *SQLiteDatabase) DeleteBookmarks(indices ...string) (oldIndices, newInd
 			panicErr, _ := r.(error)
 			tx.Rollback()
 
-			oldIndices = nil
-			newIndices = nil
 			err = panicErr
 		}
 	}()
@@ -337,56 +332,11 @@ func (db *SQLiteDatabase) DeleteBookmarks(indices ...string) (oldIndices, newInd
 	tx.MustExec("DELETE FROM bookmark_tag "+whereTagClause, args...)
 	tx.MustExec("DELETE FROM bookmark_content "+whereContentClause, args...)
 
-	// Prepare statement for updating index
-	stmtGetMaxID, err := tx.Preparex(`SELECT IFNULL(MAX(id), 0) FROM bookmark`)
-	checkError(err)
-
-	stmtUpdateBookmark, err := tx.Preparex(`UPDATE bookmark SET id = ? WHERE id = ?`)
-	checkError(err)
-
-	stmtUpdateBookmarkTag, err := tx.Preparex(`UPDATE bookmark_tag SET bookmark_id = ? WHERE bookmark_id = ?`)
-	checkError(err)
-
-	stmtUpdateBookmarkContent, err := tx.Preparex(`UPDATE bookmark_content SET docid = ? WHERE docid = ?`)
-	checkError(err)
-
-	// Get list of removed indices
-	maxIndex := 0
-	err = stmtGetMaxID.Get(&maxIndex)
-	checkError(err)
-
-	removedIndices := []int{}
-	err = tx.Select(&removedIndices,
-		`WITH cnt(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM cnt LIMIT ?)
-		SELECT x FROM cnt WHERE x NOT IN (SELECT id FROM bookmark)`,
-		maxIndex)
-	checkError(err)
-
-	// Fill removed indices
-	newIndices = []int{}
-	oldIndices = []int{}
-	for _, removedIndex := range removedIndices {
-		oldIndex := 0
-		err = stmtGetMaxID.Get(&oldIndex)
-		checkError(err)
-
-		if oldIndex <= removedIndex {
-			break
-		}
-
-		stmtUpdateBookmark.MustExec(removedIndex, oldIndex)
-		stmtUpdateBookmarkTag.MustExec(removedIndex, oldIndex)
-		stmtUpdateBookmarkContent.MustExec(removedIndex, oldIndex)
-
-		newIndices = append(newIndices, removedIndex)
-		oldIndices = append(oldIndices, oldIndex)
-	}
-
 	// Commit transaction
 	err = tx.Commit()
 	checkError(err)
 
-	return oldIndices, newIndices, err
+	return err
 }
 
 // SearchBookmarks search bookmarks by the keyword or tags.

main.go

@@ -2,13 +2,20 @@
 package main
 
 import (
+	"os"
+
 	"github.com/RadhiFadlillah/shiori/cmd"
 	db "github.com/RadhiFadlillah/shiori/database"
 	_ "github.com/mattn/go-sqlite3"
 )
 
 func main() {
-	sqliteDB, err := db.OpenSQLiteDatabase("shiori.db")
+	databasePath := "shiori.db"
+	if value, found := os.LookupEnv("ENV_SHIORI_DB"); found {
+		databasePath = value
+	}
+
+	sqliteDB, err := db.OpenSQLiteDatabase(databasePath)
 	checkError(err)
 
 	cmd.DB = sqliteDB