Download thumbnail image to local disk

.gitignore

@@ -7,4 +7,5 @@ sample.txt
 
 # Exclude generated file
 shiori*
-*.db
+*.db
+thumb/

cmd/root.go

@@ -8,7 +8,7 @@ import (
 )
 
 // NewShioriCmd creates new command for shiori
-func NewShioriCmd(db dt.Database) *cobra.Command {
+func NewShioriCmd(db dt.Database, dataDir string) *cobra.Command {
 	// Create handler
 	hdl := cmdHandler{db: db}
 
@@ -89,7 +89,7 @@ func NewShioriCmd(db dt.Database) *cobra.Command {
 
 	// Create sub command that has its own sub command
 	accountCmd := account.NewAccountCmd(db)
-	serveCmd := serve.NewServeCmd(db)
+	serveCmd := serve.NewServeCmd(db, dataDir)
 
 	// Set sub command flags
 	addCmd.Flags().StringP("title", "i", "", "Custom title for this bookmark.")

cmd/serve/root.go

@@ -13,9 +13,9 @@ import (
 )
 
 // NewServeCmd creates new command for serving web page
-func NewServeCmd(db dt.Database) *cobra.Command {
+func NewServeCmd(db dt.Database, dataDir string) *cobra.Command {
 	// Create handler
-	hdl, err := newWebHandler(db)
+	hdl, err := newWebHandler(db, dataDir)
 	checkError(err)
 
 	// Create root command
@@ -39,6 +39,7 @@ func NewServeCmd(db dt.Database) *cobra.Command {
 			router.GET("/", hdl.serveIndexPage)
 			router.GET("/login", hdl.serveLoginPage)
 			router.GET("/bookmark/:id", hdl.serveBookmarkCache)
+			router.GET("/thumb/:id", hdl.serveThumbnailImage)
 
 			router.POST("/api/login", hdl.apiLogin)
 			router.GET("/api/bookmarks", hdl.apiGetBookmarks)

cmd/serve/web-handler-api.go

@@ -3,8 +3,11 @@ package serve
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
 	nurl "net/url"
+	"os"
+	fp "path/filepath"
 	"strings"
 	"time"
 
@@ -97,12 +100,17 @@ func (h *webHandler) apiInsertBookmark(w http.ResponseWriter, r *http.Request, p
 	err = json.NewDecoder(r.Body).Decode(&book)
 	checkError(err)
 
+	// Get new bookmark id
+	book.ID, err = h.db.GetNewID("bookmark")
+	checkError(err)
+
 	// Fetch data from internet
 	article, err := readability.Parse(book.URL, 20*time.Second)
 	checkError(err)
 
 	book.URL = article.URL
-	book.ImageURL = article.Meta.Image
+	book.Title = article.Meta.Title
+	book.Excerpt = article.Meta.Excerpt
 	book.Author = article.Meta.Author
 	book.MinReadTime = article.Meta.MinReadTime
 	book.MaxReadTime = article.Meta.MaxReadTime
@@ -114,8 +122,15 @@ func (h *webHandler) apiInsertBookmark(w http.ResponseWriter, r *http.Request, p
 		book.Title = book.URL
 	}
 
-	// Save to database
-	book.ID, err = h.db.CreateBookmark(book)
+	// Save bookmark image to local disk
+	imgPath := fp.Join(h.dataDir, "thumb", fmt.Sprintf("%d", book.ID))
+	err = downloadFile(article.Meta.Image, imgPath, 20*time.Second)
+	if err == nil {
+		book.ImageURL = fmt.Sprintf("/thumb/%d", book.ID)
+	}
+
+	// Save bookmark to database
+	_, err = h.db.CreateBookmark(book)
 	checkError(err)
 
 	// Return new saved result
@@ -143,9 +158,6 @@ func (h *webHandler) apiUpdateBookmark(w http.ResponseWriter, r *http.Request, p
 		panic(fmt.Errorf("URL is not valid"))
 	}
 
-	// Clear UTM parameters from URL
-	request.URL = clearUTMParams(parsedURL)
-
 	// Get existing bookmark from database
 	bookmarks, err := h.db.GetBookmarks(true, fmt.Sprintf("%d", request.ID))
 	checkError(err)
@@ -249,16 +261,33 @@ func (h *webHandler) apiDeleteBookmark(w http.ResponseWriter, r *http.Request, p
 	fmt.Fprint(w, 1)
 }
 
-func clearUTMParams(url *nurl.URL) string {
-	newQuery := nurl.Values{}
-	for key, value := range url.Query() {
-		if strings.HasPrefix(key, "utm_") {
-			continue
-		}
+func downloadFile(url, dstPath string, timeout time.Duration) error {
+	// Fetch data from URL
+	client := &http.Client{Timeout: timeout}
+	resp, err := client.Get(url)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
 
-		newQuery[key] = value
+	// Make sure destination directory exist
+	err = os.MkdirAll(fp.Dir(dstPath), os.ModePerm)
+	if err != nil {
+		return err
 	}
 
-	url.RawQuery = newQuery.Encode()
-	return url.String()
+	// Create destination file
+	dst, err := os.Create(dstPath)
+	if err != nil {
+		return err
+	}
+	defer dst.Close()
+
+	// Write response body to the file
+	_, err = io.Copy(dst, resp.Body)
+	if err != nil {
+		return err
+	}
+
+	return nil
 }

cmd/serve/web-handler-ui.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"mime"
 	"net/http"
+	"os"
 	fp "path/filepath"
 
 	"github.com/julienschmidt/httprouter"
@@ -60,6 +61,31 @@ func (h *webHandler) serveBookmarkCache(w http.ResponseWriter, r *http.Request,
 	checkError(err)
 }
 
+// serveThumbnailImage is handler for GET /thumb/:id
+func (h *webHandler) serveThumbnailImage(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+	// Get bookmark ID from URL
+	id := ps.ByName("id")
+
+	// Open image
+	imgPath := fp.Join(h.dataDir, "thumb", id)
+	img, err := os.Open(imgPath)
+	checkError(err)
+	defer img.Close()
+
+	// Get image type from its 512 first bytes
+	buffer := make([]byte, 512)
+	_, err = img.Read(buffer)
+	checkError(err)
+
+	mimeType := http.DetectContentType(buffer)
+	w.Header().Set("Content-Type", mimeType)
+
+	// Serve image
+	img.Seek(0, 0)
+	_, err = io.Copy(w, img)
+	checkError(err)
+}
+
 func serveFile(w http.ResponseWriter, path string) error {
 	// Open file
 	src, err := assets.Open(path)

cmd/serve/web-handler.go

@@ -15,12 +15,13 @@ import (
 // webHandler is handler for every API and routes to web page
 type webHandler struct {
 	db       dt.Database
+	dataDir  string
 	jwtKey   []byte
 	tplCache *template.Template
 }
 
 // newWebHandler returns new webHandler
-func newWebHandler(db dt.Database) (*webHandler, error) {
+func newWebHandler(db dt.Database, dataDir string) (*webHandler, error) {
 	// Create JWT key
 	jwtKey := make([]byte, 32)
 	_, err := rand.Read(jwtKey)
@@ -45,6 +46,7 @@ func newWebHandler(db dt.Database) (*webHandler, error) {
 	// Create handler
 	handler := &webHandler{
 		db:       db,
+		dataDir:  dataDir,
 		jwtKey:   jwtKey,
 		tplCache: tplCache,
 	}

database/database.go

@@ -14,9 +14,12 @@ type Database interface {
 	// GetBookmarks fetch list of bookmarks based on submitted indices.
 	GetBookmarks(withContent bool, indices ...string) ([]model.Bookmark, error)
 
-	//GetTags fetch list of tags and their frequency
+	// GetTags fetch list of tags and their frequency
 	GetTags() ([]model.Tag, error)
 
+	//GetNewID get new id for specified table
+	GetNewID(table string) (int64, error)
+
 	// DeleteBookmarks removes all record with matching indices from database.
 	DeleteBookmarks(indices ...string) error
 

database/sqlite.go

@@ -89,6 +89,14 @@ func (db *SQLiteDatabase) CreateBookmark(bookmark model.Bookmark) (bookmarkID in
 		return -1, fmt.Errorf("Title must not be empty")
 	}
 
+	// Set default ID and modified time
+	if bookmark.ID == 0 {
+		bookmark.ID, err = db.GetNewID("bookmark")
+		if err != nil {
+			return -1, err
+		}
+	}
+
 	if bookmark.Modified == "" {
 		bookmark.Modified = time.Now().UTC().Format("2006-01-02 15:04:05")
 	}
@@ -111,10 +119,11 @@ func (db *SQLiteDatabase) CreateBookmark(bookmark model.Bookmark) (bookmarkID in
 	}()
 
 	// Save article to database
-	res := tx.MustExec(`INSERT INTO bookmark (
-		url, title, image_url, excerpt, author, 
+	tx.MustExec(`INSERT INTO bookmark (
+		id, url, title, image_url, excerpt, author, 
 		min_read_time, max_read_time, modified) 
-		VALUES(?, ?, ?, ?, ?, ?, ?, ?)`,
+		VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+		bookmark.ID,
 		bookmark.URL,
 		bookmark.Title,
 		bookmark.ImageURL,
@@ -124,14 +133,10 @@ func (db *SQLiteDatabase) CreateBookmark(bookmark model.Bookmark) (bookmarkID in
 		bookmark.MaxReadTime,
 		bookmark.Modified)
 
-	// Get last inserted ID
-	bookmarkID, err = res.LastInsertId()
-	checkError(err)
-
 	// Save bookmark content
 	tx.MustExec(`INSERT INTO bookmark_content 
 		(docid, title, content, html) VALUES (?, ?, ?, ?)`,
-		bookmarkID, bookmark.Title, bookmark.Content, bookmark.HTML)
+		bookmark.ID, bookmark.Title, bookmark.Content, bookmark.HTML)
 
 	// Save tags
 	stmtGetTag, err := tx.Preparex(`SELECT id FROM tag WHERE name = ?`)
@@ -157,13 +162,14 @@ func (db *SQLiteDatabase) CreateBookmark(bookmark model.Bookmark) (bookmarkID in
 			checkError(err)
 		}
 
-		stmtInsertBookmarkTag.Exec(tagID, bookmarkID)
+		stmtInsertBookmarkTag.Exec(tagID, bookmark.ID)
 	}
 
 	// Commit transaction
 	err = tx.Commit()
 	checkError(err)
 
+	bookmarkID = bookmark.ID
 	return bookmarkID, err
 }
 
@@ -542,6 +548,19 @@ func (db *SQLiteDatabase) GetTags() ([]model.Tag, error) {
 	return tags, nil
 }
 
+// GetNewID creates new ID for specified table
+func (db *SQLiteDatabase) GetNewID(table string) (int64, error) {
+	var tableID int64
+	query := fmt.Sprintf(`SELECT IFNULL(MAX(id) + 1, 1) FROM %s`, table)
+
+	err := db.Get(&tableID, query)
+	if err != nil && err != sql.ErrNoRows {
+		return -1, err
+	}
+
+	return tableID, nil
+}
+
 // ErrInvalidIndex is returned is an index is not valid
 var ErrInvalidIndex = errors.New("Index is not valid")
 

main.go

@@ -3,21 +3,24 @@
 package main
 
 import (
+	fp "path/filepath"
+
 	"github.com/RadhiFadlillah/shiori/cmd"
 	dt "github.com/RadhiFadlillah/shiori/database"
 	_ "github.com/mattn/go-sqlite3"
 	"github.com/sirupsen/logrus"
 )
 
-var dbPath = "shiori.db"
+var dataDir = "."
 
 func main() {
 	// Open database
+	dbPath := fp.Join(dataDir, "shiori.db")
 	sqliteDB, err := dt.OpenSQLiteDatabase(dbPath)
 	checkError(err)
 
 	// Start cmd
-	shioriCmd := cmd.NewShioriCmd(sqliteDB)
+	shioriCmd := cmd.NewShioriCmd(sqliteDB, dataDir)
 	if err := shioriCmd.Execute(); err != nil {
 		logrus.Fatalln(err)
 	}

path-generator.go

@@ -4,38 +4,32 @@ package main
 
 import (
 	"os"
-	fp "path/filepath"
 
 	apppaths "github.com/muesli/go-app-paths"
 )
 
 func init() {
-	// Set database path
-	dbPath = createDatabasePath()
+	// Get data directory
+	dataDir = getDataDirectory()
 
 	// Make sure directory exist
-	os.MkdirAll(fp.Dir(dbPath), os.ModePerm)
+	os.MkdirAll(dataDir, os.ModePerm)
 }
 
-func createDatabasePath() string {
+func getDataDirectory() string {
 	// Try to look at environment variables
-	dbPath, found := os.LookupEnv("ENV_SHIORI_DB")
+	dataDir, found := os.LookupEnv("ENV_SHIORI_DIR")
 	if found {
-		// If ENV_SHIORI_DB is directory, append "shiori.db" as filename
-		if f1, err := os.Stat(dbPath); err == nil && f1.IsDir() {
-			dbPath = fp.Join(dbPath, "shiori.db")
-		}
-
-		return dbPath
+		return dataDir
 	}
 
 	// Try to use platform specific app path
 	userScope := apppaths.NewScope(apppaths.User, "shiori", "shiori")
 	dataDir, err := userScope.DataDir()
 	if err == nil {
-		return fp.Join(dataDir, "shiori.db")
+		return dataDir
 	}
 
-	// When all fail, create database in working directory
-	return "shiori.db"
+	// When all fail, use current working directory
+	return "."
 }

view/index.html

@@ -95,7 +95,7 @@
         var token = Cookies.get('token'),
             rest = axios.create();
 
-        rest.defaults.timeout = 15000;
+        rest.defaults.timeout = 60000;
         rest.defaults.headers.common['Authorization'] = 'Bearer ' + token;
 
         // Register Vue component