diff --git a/go.mod b/go.mod index 2b2b003..219395f 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.12 require ( github.com/fatih/color v1.7.0 - github.com/go-shiori/go-readability v0.0.0-20190521101123-866575e3f1b6 + github.com/go-shiori/go-readability v0.0.0-20190522013032-128e0c654d14 github.com/go-sql-driver/mysql v1.4.1 // indirect github.com/jmoiron/sqlx v1.2.0 github.com/lib/pq v1.1.1 // indirect diff --git a/go.sum b/go.sum index 7500bd0..b476e03 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/go-shiori/go-readability v0.0.0-20190521101123-866575e3f1b6 h1:lp+AH6pCPsOKf9M2Az76JsxUneHweAsOlq0GMtOf6OY= -github.com/go-shiori/go-readability v0.0.0-20190521101123-866575e3f1b6/go.mod h1:1tFV9uTM/xnAKQw5EgPs+ip50udKhCjaP0nYdkSDXcU= +github.com/go-shiori/go-readability v0.0.0-20190522013032-128e0c654d14 h1:rvu9FluelHm3ykyizaBwhQMAxsDeg164iNEoMoZo7cI= +github.com/go-shiori/go-readability v0.0.0-20190522013032-128e0c654d14/go.mod h1:1tFV9uTM/xnAKQw5EgPs+ip50udKhCjaP0nYdkSDXcU= github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= diff --git a/internal/cmd/add.go b/internal/cmd/add.go index f96d41e..1ad80b7 100644 --- a/internal/cmd/add.go +++ b/internal/cmd/add.go @@ -53,6 +53,13 @@ func addHandler(cmd *cobra.Command, args []string) { Excerpt: normalizeSpace(excerpt), } + // Create bookmark ID + book.ID, err = DB.CreateNewID("bookmark") + if err != nil { + cError.Printf("Failed to create ID: %v\n", err) + return + } + // Set bookmark tags book.Tags = make([]model.Tag, len(tags)) for i, tag := range tags { @@ -63,6 +70,8 @@ func addHandler(cmd *cobra.Command, args []string) { var imageURL string if !offline { + cInfo.Println("Downloading article") + article, err := readability.FromURL(book.URL, time.Minute) if err != nil { cError.Printf("Failed to download article: %v\n", err) @@ -96,9 +105,9 @@ func addHandler(cmd *cobra.Command, args []string) { } // Save bookmark to database - book.ID, err = DB.InsertBookmark(book) + _, err = DB.SaveBookmarks(book) if err != nil { - cError.Printf("Failed to insert bookmark: %v\n", err) + cError.Printf("Failed to save bookmark: %v\n", err) return } @@ -113,5 +122,7 @@ func addHandler(cmd *cobra.Command, args []string) { } } + // Print added bookmark + fmt.Println() printBookmarks(book) } diff --git a/internal/cmd/update.go b/internal/cmd/update.go index 7ecedf6..c29f5e1 100644 --- a/internal/cmd/update.go +++ b/internal/cmd/update.go @@ -1,6 +1,18 @@ package cmd -import "github.com/spf13/cobra" +import ( + "fmt" + nurl "net/url" + fp "path/filepath" + "strings" + "sync" + "time" + + "github.com/go-shiori/go-readability" + "github.com/go-shiori/shiori/internal/database" + "github.com/go-shiori/shiori/internal/model" + "github.com/spf13/cobra" +) func updateCmd() *cobra.Command { cmd := &cobra.Command{ @@ -13,6 +25,7 @@ func updateCmd() *cobra.Command { "- If indices are passed without any flags (--url, --title, --tag and --excerpt), read the URLs from DB and update titles from web.\n" + "- If --url is passed (and --title is omitted), update the title from web using the URL. While using this flag, update only accept EXACTLY one index.\n" + "While updating bookmark's tags, you can use - to remove tag (e.g. -nature to remove nature tag from this bookmark).", + Run: updateHandler, } cmd.Flags().StringP("url", "u", "", "New URL for this bookmark.") @@ -25,3 +38,214 @@ func updateCmd() *cobra.Command { return cmd } + +func updateHandler(cmd *cobra.Command, args []string) { + // Parse flags + url, _ := cmd.Flags().GetString("url") + title, _ := cmd.Flags().GetString("title") + excerpt, _ := cmd.Flags().GetString("excerpt") + tags, _ := cmd.Flags().GetStringSlice("tags") + offline, _ := cmd.Flags().GetBool("offline") + skipConfirm, _ := cmd.Flags().GetBool("yes") + dontOverwrite := cmd.Flags().Changed("dont-overwrite") + + // If no arguments (i.e all bookmarks going to be updated), confirm to user + if len(args) == 0 && !skipConfirm { + confirmUpdate := "" + fmt.Print("Update ALL bookmarks? (y/N): ") + fmt.Scanln(&confirmUpdate) + + if confirmUpdate != "y" { + fmt.Println("No bookmarks updated") + return + } + } + + // Convert args to ids + ids, err := parseStrIndices(args) + if err != nil { + cError.Printf("Failed to parse args: %v\n", err) + return + } + + // Clean up new parameter from flags + title = normalizeSpace(title) + excerpt = normalizeSpace(excerpt) + + if cmd.Flags().Changed("url") { + // Clean up URL by removing its fragment and UTM parameters + tmp, err := nurl.Parse(url) + if err != nil || tmp.Scheme == "" || tmp.Hostname() == "" { + cError.Println("URL is not valid") + return + } + + tmp.Fragment = "" + clearUTMParams(tmp) + url = tmp.String() + + // Since user uses custom URL, make sure there is only one ID to update + if len(ids) != 1 { + cError.Println("Update only accepts one index while using --url flag") + return + } + } + + // Fetch bookmarks from database + filterOptions := database.GetBookmarksOptions{ + IDs: ids, + } + + bookmarks, err := DB.GetBookmarks(filterOptions) + if err != nil { + cError.Printf("Failed to get bookmarks: %v\n", err) + return + } + + if len(bookmarks) == 0 { + cError.Println("No matching index found") + return + } + + // If it's not offline mode, fetch data from internet + if !offline { + mx := sync.RWMutex{} + wg := sync.WaitGroup{} + semaphore := make(chan struct{}, 10) + + for i, book := range bookmarks { + wg.Add(1) + + // If used, use submitted URL + if url != "" { + book.URL = url + } + + go func(i int, book model.Bookmark, nData int) { + // Make sure to finish the WG + defer wg.Done() + + // Register goroutine to semaphore + semaphore <- struct{}{} + defer func() { + <-semaphore + }() + + // Download article + cInfo.Printf("[ %d / %d ] Downloading %s\n", i+1, nData, book.URL) + + article, err := readability.FromURL(book.URL, time.Minute) + if err != nil { + cError.Printf("[ %d / %d ] Failed to download article: %v\n", i+1, nData, err) + return + } + + book.Author = article.Byline + book.Content = article.TextContent + book.HTML = article.Content + + if !dontOverwrite { + book.Title = article.Title + book.Excerpt = article.Excerpt + } + + // Get image URL and save it to local disk + var imageURL string + if article.Image != "" { + imageURL = article.Image + } else if article.Favicon != "" { + imageURL = article.Favicon + } + + if imageURL != "" { + imgPath := fp.Join(DataDir, "thumb", fmt.Sprintf("%d", book.ID)) + + err = downloadFile(imageURL, imgPath, time.Minute) + if err != nil { + cError.Printf("Failed to download image: %v\n", err) + return + } + } + + // Save parse result to bookmark + mx.Lock() + bookmarks[i] = book + mx.Unlock() + }(i, book, len(bookmarks)) + } + + // Wait until all download finished + wg.Wait() + } + + // Map which tags is new or deleted from flag --tags + addedTags := make(map[string]struct{}) + deletedTags := make(map[string]struct{}) + for _, tag := range tags { + tagName := strings.ToLower(tag) + tagName = strings.TrimSpace(tagName) + + if strings.HasPrefix(tagName, "-") { + tagName = strings.TrimPrefix(tagName, "-") + deletedTags[tagName] = struct{}{} + } else { + addedTags[tagName] = struct{}{} + } + } + + // Attach user submitted value to the bookmarks + for i, book := range bookmarks { + // If user submit his own title or excerpt, use it + if title != "" { + book.Title = title + } + + if excerpt != "" { + book.Excerpt = excerpt + } + + // Make sure title is not empty + if book.Title == "" { + book.Title = book.URL + } + + // Generate new tags + tmpAddedTags := make(map[string]struct{}) + for key, value := range addedTags { + tmpAddedTags[key] = value + } + + newTags := []model.Tag{} + for _, tag := range book.Tags { + if _, isDeleted := deletedTags[tag.Name]; isDeleted { + tag.Deleted = true + } + + if _, alreadyExist := addedTags[tag.Name]; alreadyExist { + delete(tmpAddedTags, tag.Name) + } + + newTags = append(newTags, tag) + } + + for tag := range tmpAddedTags { + newTags = append(newTags, model.Tag{Name: tag}) + } + + book.Tags = newTags + + // Set bookmark's new data + bookmarks[i] = book + } + + // Save bookmarks to database + bookmarks, err = DB.SaveBookmarks(bookmarks...) + if err != nil { + cError.Printf("Failed to save bookmark: %v\n", err) + return + } + + // Print updated bookmarks + fmt.Println() + printBookmarks(bookmarks...) +} diff --git a/internal/cmd/utils.go b/internal/cmd/utils.go index 19ae134..0ebe79c 100644 --- a/internal/cmd/utils.go +++ b/internal/cmd/utils.go @@ -23,10 +23,13 @@ var ( cTitle = color.New(color.FgHiGreen).Add(color.Bold) cReadTime = color.New(color.FgHiMagenta) cURL = color.New(color.FgHiYellow) - cError = color.New(color.FgHiRed) cExcerpt = color.New(color.FgHiWhite) cTag = color.New(color.FgHiBlue) + cInfo = color.New(color.FgHiCyan) + cError = color.New(color.FgHiRed) + cWarning = color.New(color.FgHiYellow) + errInvalidIndex = errors.New("Index is not valid") ) diff --git a/internal/database/database.go b/internal/database/database.go index 8770daf..62ea301 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -17,8 +17,8 @@ type GetBookmarksOptions struct { // DB is interface for accessing and manipulating data in database. type DB interface { - // InsertBookmark inserts new bookmark to database. - InsertBookmark(bookmark model.Bookmark) (int, error) + // SaveBookmarks saves bookmarks data to database. + SaveBookmarks(bookmarks ...model.Bookmark) ([]model.Bookmark, error) // GetBookmarks fetch list of bookmarks based on submitted options. GetBookmarks(opts GetBookmarksOptions) ([]model.Bookmark, error) diff --git a/internal/database/sqlite.go b/internal/database/sqlite.go index 08baf64..b8ef0d1 100644 --- a/internal/database/sqlite.go +++ b/internal/database/sqlite.go @@ -77,36 +77,13 @@ func OpenSQLiteDatabase(databasePath string) (*SQLiteDatabase, error) { return &SQLiteDatabase{*db}, err } -// InsertBookmark saves new bookmark to database. -// Returns new ID and error message if any happened. -func (db *SQLiteDatabase) InsertBookmark(bookmark model.Bookmark) (bookmarkID int, err error) { - // Check URL and title - if bookmark.URL == "" { - return -1, fmt.Errorf("URL must not be empty") - } - - if bookmark.Title == "" { - return -1, fmt.Errorf("title must not be empty") - } - - // Create ID (if needed) and modified time - if bookmark.ID != 0 { - bookmarkID = bookmark.ID - } else { - bookmarkID, err = db.CreateNewID("bookmark") - if err != nil { - return -1, err - } - } - - if bookmark.Modified == "" { - bookmark.Modified = time.Now().UTC().Format("2006-01-02 15:04:05") - } - - // Begin transaction +// SaveBookmarks saves new or updated bookmarks to database. +// Returns the saved ID and error message if any happened. +func (db *SQLiteDatabase) SaveBookmarks(bookmarks ...model.Bookmark) (result []model.Bookmark, err error) { + // Prepare transaction tx, err := db.Beginx() if err != nil { - return -1, err + return []model.Bookmark{}, err } // Make sure to rollback if panic ever happened @@ -115,65 +92,108 @@ func (db *SQLiteDatabase) InsertBookmark(bookmark model.Bookmark) (bookmarkID in panicErr, _ := r.(error) tx.Rollback() - bookmarkID = -1 + result = []model.Bookmark{} err = panicErr } }() - // Save article to database - tx.MustExec(`INSERT INTO bookmark ( - id, url, title, excerpt, author, modified) - VALUES(?, ?, ?, ?, ?, ?)`, - bookmarkID, - bookmark.URL, - bookmark.Title, - bookmark.Excerpt, - bookmark.Author, - bookmark.Modified) + // Prepare statement + stmtInsertBook, _ := tx.Preparex(`INSERT INTO bookmark + (id, url, title, excerpt, author, modified) + VALUES(?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + url = ?, title = ?, excerpt = ?, author = ?, modified = ?`) - // Save bookmark content - tx.MustExec(`INSERT INTO bookmark_content - (docid, title, content, html) VALUES (?, ?, ?, ?)`, - bookmarkID, - bookmark.Title, - bookmark.Content, - bookmark.HTML) + stmtInsertBookContent, _ := tx.Preparex(`INSERT OR IGNORE INTO bookmark_content + (docid, title, content, html) + VALUES (?, ?, ?, ?)`) - // Save tags - stmtGetTag, err := tx.Preparex(`SELECT id FROM tag WHERE name = ?`) - checkError(err) + stmtUpdateBookContent, _ := tx.Preparex(`UPDATE bookmark_content SET + title = ?, content = ?, html = ? + WHERE docid = ?`) - stmtInsertTag, err := tx.Preparex(`INSERT INTO tag (name) VALUES (?)`) - checkError(err) + stmtGetTag, _ := tx.Preparex(`SELECT id FROM tag WHERE name = ?`) - stmtInsertBookmarkTag, err := tx.Preparex(`INSERT OR IGNORE INTO bookmark_tag + stmtInsertTag, _ := tx.Preparex(`INSERT INTO tag (name) VALUES (?)`) + + stmtInsertBookTag, _ := tx.Preparex(`INSERT OR IGNORE INTO bookmark_tag (tag_id, bookmark_id) VALUES (?, ?)`) - checkError(err) - for _, tag := range bookmark.Tags { - tagName := strings.ToLower(tag.Name) - tagName = strings.TrimSpace(tagName) + stmtDeleteBookTag, _ := tx.Preparex(`DELETE FROM bookmark_tag + WHERE bookmark_id = ? AND tag_id = ?`) - tagID := -1 - err = stmtGetTag.Get(&tagID, tagName) - checkError(err) + // Prepare modified time + modifiedTime := time.Now().UTC().Format("2006-01-02 15:04:05") - if tagID == -1 { - res := stmtInsertTag.MustExec(tagName) - tagID64, err := res.LastInsertId() - checkError(err) - - tagID = int(tagID64) + // Execute statements + result = []model.Bookmark{} + for _, book := range bookmarks { + // Check ID, URL and title + if book.ID == 0 { + panic(fmt.Errorf("ID must not be empty")) } - stmtInsertBookmarkTag.Exec(tagID, bookmarkID) + if book.URL == "" { + panic(fmt.Errorf("URL must not be empty")) + } + + if book.Title == "" { + panic(fmt.Errorf("title must not be empty")) + } + + // Set modified time + book.Modified = modifiedTime + + // Save bookmark + stmtInsertBook.MustExec(book.ID, + book.URL, book.Title, book.Excerpt, book.Author, book.Modified, + book.URL, book.Title, book.Excerpt, book.Author, book.Modified) + + stmtUpdateBookContent.MustExec(book.Title, book.Content, book.HTML, book.ID) + stmtInsertBookContent.MustExec(book.ID, book.Title, book.Content, book.HTML) + + // Save book tags + newTags := []model.Tag{} + for _, tag := range book.Tags { + // If it's deleted tag, delete and continue + if tag.Deleted { + stmtDeleteBookTag.MustExec(book.ID, tag.ID) + continue + } + + // Normalize tag name + tagName := strings.ToLower(tag.Name) + tagName = strings.Join(strings.Fields(tagName), " ") + + // If tag doesn't have any ID, fetch it from database + if tag.ID == 0 { + err = stmtGetTag.Get(&tag.ID, tagName) + checkError(err) + + // If tag doesn't exist in database, save it + if tag.ID == 0 { + res := stmtInsertTag.MustExec(tagName) + tagID64, err := res.LastInsertId() + checkError(err) + + tag.ID = int(tagID64) + } + + stmtInsertBookTag.Exec(tag.ID, book.ID) + } + + newTags = append(newTags, tag) + } + + book.Tags = newTags + result = append(result, book) } // Commit transaction err = tx.Commit() checkError(err) - return bookmarkID, err + return result, err } // GetBookmarks fetch list of bookmarks based on submitted ids.