From 659a3291d8436f1647a25e9d678a3653d2a3d21d Mon Sep 17 00:00:00 2001 From: Radhi Fadlillah Date: Tue, 21 May 2019 23:24:11 +0700 Subject: [PATCH] Implement logic for add and print cmd --- .gitignore | 5 +- go.mod | 7 + go.sum | 30 ++++ internal/cmd/add.go | 97 ++++++++++++ internal/cmd/print.go | 77 +++++++++- internal/cmd/root.go | 17 ++- internal/cmd/search.go | 22 --- internal/cmd/utils.go | 168 +++++++++++++++++++++ internal/database/database.go | 29 ++++ internal/database/sqlite.go | 270 ++++++++++++++++++++++++++++++++++ internal/model/model.go | 25 ++-- main.go | 23 ++- 12 files changed, 728 insertions(+), 42 deletions(-) delete mode 100644 internal/cmd/search.go create mode 100644 internal/cmd/utils.go diff --git a/.gitignore b/.gitignore index eef2841..5bd2f33 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,7 @@ *.toml # Exclude executable file -/shiori* \ No newline at end of file +/shiori* + +# Exclude development data +/dev-data \ No newline at end of file diff --git a/go.mod b/go.mod index 1021a91..2b2b003 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,14 @@ module github.com/go-shiori/shiori go 1.12 require ( + github.com/fatih/color v1.7.0 + github.com/go-shiori/go-readability v0.0.0-20190521101123-866575e3f1b6 + github.com/go-sql-driver/mysql v1.4.1 // indirect github.com/jmoiron/sqlx v1.2.0 + github.com/lib/pq v1.1.1 // indirect + github.com/mattn/go-colorable v0.1.1 // indirect + github.com/mattn/go-isatty v0.0.7 // indirect + github.com/mattn/go-sqlite3 v1.10.0 github.com/sirupsen/logrus v1.4.2 github.com/spf13/cobra v0.0.4 google.golang.org/appengine v1.6.0 // indirect diff --git a/go.sum b/go.sum index 4107dab..7500bd0 100644 --- a/go.sum +++ b/go.sum @@ -4,11 +4,18 @@ github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-shiori/go-readability v0.0.0-20190521101123-866575e3f1b6 h1:lp+AH6pCPsOKf9M2Az76JsxUneHweAsOlq0GMtOf6OY= +github.com/go-shiori/go-readability v0.0.0-20190521101123-866575e3f1b6/go.mod h1:1tFV9uTM/xnAKQw5EgPs+ip50udKhCjaP0nYdkSDXcU= github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= @@ -17,17 +24,30 @@ github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4= +github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o= +github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= @@ -38,16 +58,26 @@ github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb6 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190520210107-018c4d40a106 h1:EZofHp/BzEf3j39/+7CX1JvH0WaPG+ikBrqAdAPf+GM= +golang.org/x/net v0.0.0-20190520210107-018c4d40a106/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190520201301-c432e742b0af h1:NXfmMfXz6JqGfG3ikSxcz2N93j6DgScr19Oo2uwFu88= +golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= google.golang.org/appengine v1.6.0 h1:Tfd7cKwKbFRsI8RMAD3oqqw7JPFRrvFlOsfbgVkjOOw= google.golang.org/appengine v1.6.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= diff --git a/internal/cmd/add.go b/internal/cmd/add.go index d271616..f96d41e 100644 --- a/internal/cmd/add.go +++ b/internal/cmd/add.go @@ -1,6 +1,14 @@ package cmd import ( + "fmt" + nurl "net/url" + fp "path/filepath" + "strings" + "time" + + "github.com/go-shiori/go-readability" + "github.com/go-shiori/shiori/internal/model" "github.com/spf13/cobra" ) @@ -9,6 +17,7 @@ func addCmd() *cobra.Command { Use: "add url", Short: "Bookmark the specified URL", Args: cobra.ExactArgs(1), + Run: addHandler, } cmd.Flags().StringP("title", "i", "", "Custom title for this bookmark.") @@ -18,3 +27,91 @@ func addCmd() *cobra.Command { return cmd } + +func addHandler(cmd *cobra.Command, args []string) { + // Read flag and arguments + url := args[0] + title, _ := cmd.Flags().GetString("title") + excerpt, _ := cmd.Flags().GetString("excerpt") + tags, _ := cmd.Flags().GetStringSlice("tags") + offline, _ := cmd.Flags().GetBool("offline") + + // Clean up URL by removing its fragment and UTM parameters + tmp, err := nurl.Parse(url) + if err != nil || tmp.Scheme == "" || tmp.Hostname() == "" { + cError.Println("URL is not valid") + return + } + + tmp.Fragment = "" + clearUTMParams(tmp) + + // Create bookmark item + book := model.Bookmark{ + URL: tmp.String(), + Title: normalizeSpace(title), + Excerpt: normalizeSpace(excerpt), + } + + // Set bookmark tags + book.Tags = make([]model.Tag, len(tags)) + for i, tag := range tags { + book.Tags[i].Name = strings.TrimSpace(tag) + } + + // If it's not offline mode, fetch data from internet + var imageURL string + + if !offline { + article, err := readability.FromURL(book.URL, time.Minute) + if err != nil { + cError.Printf("Failed to download article: %v\n", err) + return + } + + book.Author = article.Byline + book.Content = article.TextContent + book.HTML = article.Content + + // If title and excerpt doesnt have submitted value, use from article + if book.Title == "" { + book.Title = article.Title + } + + if book.Excerpt == "" { + book.Excerpt = article.Excerpt + } + + // Get image URL + if article.Image != "" { + imageURL = article.Image + } else if article.Favicon != "" { + imageURL = article.Favicon + } + } + + // Make sure title is not empty + if book.Title == "" { + book.Title = book.URL + } + + // Save bookmark to database + book.ID, err = DB.InsertBookmark(book) + if err != nil { + cError.Printf("Failed to insert bookmark: %v\n", err) + return + } + + // Save article image to local disk + if imageURL != "" { + imgPath := fp.Join(DataDir, "thumb", fmt.Sprintf("%d", book.ID)) + + err = downloadFile(imageURL, imgPath, time.Minute) + if err != nil { + cError.Printf("Failed to download image: %v\n", err) + return + } + } + + printBookmarks(book) +} diff --git a/internal/cmd/print.go b/internal/cmd/print.go index 1e1c36d..a2a2e1a 100644 --- a/internal/cmd/print.go +++ b/internal/cmd/print.go @@ -1,6 +1,12 @@ package cmd -import "github.com/spf13/cobra" +import ( + "encoding/json" + "fmt" + + "github.com/go-shiori/shiori/internal/database" + "github.com/spf13/cobra" +) func printCmd() *cobra.Command { cmd := &cobra.Command{ @@ -11,10 +17,79 @@ func printCmd() *cobra.Command { "hyphenated range (e.g. 100-200) or both (e.g. 1-3 7 9). " + "If no arguments, all records with actual index from database are shown.", Aliases: []string{"list", "ls"}, + Run: printHandler, } cmd.Flags().BoolP("json", "j", false, "Output data in JSON format") + cmd.Flags().BoolP("latest", "l", false, "Sort bookmark by latest instead of ID") cmd.Flags().BoolP("index-only", "i", false, "Only print the index of bookmarks") + cmd.Flags().StringP("search", "s", "", "Search bookmark with specified keyword") + cmd.Flags().StringSliceP("tags", "t", []string{}, "Print bookmarks with matching tag(s)") return cmd } + +func printHandler(cmd *cobra.Command, args []string) { + // Read flags + tags, _ := cmd.Flags().GetStringSlice("tags") + keyword, _ := cmd.Flags().GetString("search") + useJSON, _ := cmd.Flags().GetBool("json") + indexOnly, _ := cmd.Flags().GetBool("index-only") + orderLatest, _ := cmd.Flags().GetBool("latest") + + // Convert args to ids + ids, err := parseStrIndices(args) + if err != nil { + cError.Printf("Failed to parse args: %v\n", err) + return + } + + // Read bookmarks from database + searchOptions := database.GetBookmarksOptions{ + IDs: ids, + Tags: tags, + Keyword: keyword, + OrderLatest: orderLatest, + } + + bookmarks, err := DB.GetBookmarks(searchOptions) + if err != nil { + cError.Printf("Failed to get bookmarks: %v\n", err) + return + } + + if len(bookmarks) == 0 { + switch { + case len(ids) > 0: + cError.Println("No matching index found") + case keyword != "", len(tags) > 0: + cError.Println("No matching bookmarks found") + default: + cError.Println("No bookmarks saved yet") + } + return + } + + // Print data + if useJSON { + bt, err := json.MarshalIndent(&bookmarks, "", " ") + if err != nil { + cError.Println(err) + return + } + + fmt.Println(string(bt)) + return + } + + if indexOnly { + for _, bookmark := range bookmarks { + fmt.Printf("%d ", bookmark.ID) + } + + fmt.Println() + return + } + + printBookmarks(bookmarks...) +} diff --git a/internal/cmd/root.go b/internal/cmd/root.go index 14ba4ab..e10a3bb 100644 --- a/internal/cmd/root.go +++ b/internal/cmd/root.go @@ -1,9 +1,23 @@ package cmd import ( + "net/http" + "time" + + "github.com/go-shiori/shiori/internal/database" "github.com/spf13/cobra" ) +var ( + // DB is database that used by cmd + DB database.DB + + // DataDir is directory for downloaded data + DataDir string + + httpClient = &http.Client{Timeout: time.Minute} +) + // ShioriCmd returns the root command for shiori func ShioriCmd() *cobra.Command { rootCmd := &cobra.Command{ @@ -12,10 +26,8 @@ func ShioriCmd() *cobra.Command { } rootCmd.AddCommand( - accountCmd(), addCmd(), printCmd(), - searchCmd(), updateCmd(), deleteCmd(), openCmd(), @@ -23,6 +35,7 @@ func ShioriCmd() *cobra.Command { exportCmd(), pocketCmd(), serveCmd(), + accountCmd(), ) return rootCmd diff --git a/internal/cmd/search.go b/internal/cmd/search.go deleted file mode 100644 index 44a33c3..0000000 --- a/internal/cmd/search.go +++ /dev/null @@ -1,22 +0,0 @@ -package cmd - -import "github.com/spf13/cobra" - -func searchCmd() *cobra.Command { - cmd := &cobra.Command{ - Use: "search keyword", - Short: "Search bookmarks by submitted keyword", - Long: "Search bookmarks by looking for matching keyword in bookmark's title and content. " + - "If no keyword submitted, print all saved bookmarks. " + - "Search results will be different depending on DBMS that used by shiori :\n" + - "- sqlite3, search works using fts4 method: https://www.sqlite.org/fts3.html.\n" + - "- mysql or mariadb, search works using natural language mode: https://dev.mysql.com/doc/refman/5.5/en/fulltext-natural-language.html.", - Args: cobra.MaximumNArgs(1), - } - - cmd.Flags().BoolP("json", "j", false, "Output data in JSON format") - cmd.Flags().BoolP("index-only", "i", false, "Only print the index of bookmarks") - cmd.Flags().StringSliceP("tags", "t", []string{}, "Search bookmarks with specified tag(s)") - - return cmd -} diff --git a/internal/cmd/utils.go b/internal/cmd/utils.go new file mode 100644 index 0000000..19ae134 --- /dev/null +++ b/internal/cmd/utils.go @@ -0,0 +1,168 @@ +package cmd + +import ( + "errors" + "fmt" + "io" + "mime" + "net/http" + nurl "net/url" + "os" + fp "path/filepath" + "strconv" + "strings" + "time" + + "github.com/fatih/color" + "github.com/go-shiori/shiori/internal/model" +) + +var ( + cIndex = color.New(color.FgHiCyan) + cSymbol = color.New(color.FgHiMagenta) + cTitle = color.New(color.FgHiGreen).Add(color.Bold) + cReadTime = color.New(color.FgHiMagenta) + cURL = color.New(color.FgHiYellow) + cError = color.New(color.FgHiRed) + cExcerpt = color.New(color.FgHiWhite) + cTag = color.New(color.FgHiBlue) + + errInvalidIndex = errors.New("Index is not valid") +) + +func normalizeSpace(str string) string { + return strings.Join(strings.Fields(str), " ") +} + +func isURLValid(s string) bool { + tmp, err := nurl.Parse(s) + return err == nil && tmp.Scheme != "" && tmp.Hostname() != "" +} + +func clearUTMParams(url *nurl.URL) { + queries := url.Query() + + for key := range queries { + if strings.HasPrefix(key, "utm_") { + queries.Del(key) + } + } + + url.RawQuery = queries.Encode() +} + +func downloadFile(url, dstPath string, timeout time.Duration) error { + // Fetch data from URL + client := &http.Client{Timeout: timeout} + resp, err := client.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + // Make sure destination directory exist + err = os.MkdirAll(fp.Dir(dstPath), os.ModePerm) + if err != nil { + return err + } + + // If destination path doesn't have extension, create it + if fp.Ext(dstPath) == "" { + cp := resp.Header.Get("Content-Type") + exts, err := mime.ExtensionsByType(cp) + if err != nil { + return fmt.Errorf("failed to create extension: %v", err) + } + + if len(exts) == 0 { + return fmt.Errorf("unknown content type") + } + + dstPath += exts[0] + } + + // Create destination file + dst, err := os.Create(dstPath) + if err != nil { + return err + } + defer dst.Close() + + // Write response body to the file + _, err = io.Copy(dst, resp.Body) + if err != nil { + return err + } + + return nil +} + +func printBookmarks(bookmarks ...model.Bookmark) { + for _, bookmark := range bookmarks { + // Create bookmark index + strBookmarkIndex := fmt.Sprintf("%d. ", bookmark.ID) + strSpace := strings.Repeat(" ", len(strBookmarkIndex)) + + // Print bookmark title + cIndex.Print(strBookmarkIndex) + cTitle.Println(bookmark.Title) + + // Print bookmark URL + cSymbol.Print(strSpace + "> ") + cURL.Println(bookmark.URL) + + // Print bookmark excerpt + if bookmark.Excerpt != "" { + cSymbol.Print(strSpace + "+ ") + cExcerpt.Println(bookmark.Excerpt) + } + + // Print bookmark tags + if len(bookmark.Tags) > 0 { + cSymbol.Print(strSpace + "# ") + for i, tag := range bookmark.Tags { + if i == len(bookmark.Tags)-1 { + cTag.Println(tag.Name) + } else { + cTag.Print(tag.Name + ", ") + } + } + } + + // Append new line + fmt.Println() + } +} + +// parseStrIndices converts a list of indices to their integer values +func parseStrIndices(indices []string) ([]int, error) { + var listIndex []int + for _, strIndex := range indices { + if !strings.Contains(strIndex, "-") { + index, err := strconv.Atoi(strIndex) + if err != nil || index < 1 { + return nil, errInvalidIndex + } + + listIndex = append(listIndex, index) + continue + } + + parts := strings.Split(strIndex, "-") + if len(parts) != 2 { + return nil, errInvalidIndex + } + + minIndex, errMin := strconv.Atoi(parts[0]) + maxIndex, errMax := strconv.Atoi(parts[1]) + if errMin != nil || errMax != nil || minIndex < 1 || minIndex > maxIndex { + return nil, errInvalidIndex + } + + for i := minIndex; i <= maxIndex; i++ { + listIndex = append(listIndex, i) + } + } + + return listIndex, nil +} diff --git a/internal/database/database.go b/internal/database/database.go index a7b306c..49151c2 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -1,5 +1,34 @@ package database +import ( + "database/sql" + + "github.com/go-shiori/shiori/internal/model" +) + +// GetBookmarksOptions is options for fetching bookmarks from database. +type GetBookmarksOptions struct { + IDs []int + Tags []string + Keyword string + WithContent bool + OrderLatest bool +} + // DB is interface for accessing and manipulating data in database. type DB interface { + // InsertBookmark inserts new bookmark to database. + InsertBookmark(bookmark model.Bookmark) (int, error) + + // GetBookmarks fetch list of bookmarks based on submitted options. + GetBookmarks(opts GetBookmarksOptions) ([]model.Bookmark, error) + + // CreateNewID creates new id for specified table. + CreateNewID(table string) (int, error) +} + +func checkError(err error) { + if err != nil && err != sql.ErrNoRows { + panic(err) + } } diff --git a/internal/database/sqlite.go b/internal/database/sqlite.go index 4b9b2de..c740b2b 100644 --- a/internal/database/sqlite.go +++ b/internal/database/sqlite.go @@ -1,6 +1,12 @@ package database import ( + "database/sql" + "fmt" + "strings" + "time" + + "github.com/go-shiori/shiori/internal/model" "github.com/jmoiron/sqlx" ) @@ -9,3 +15,267 @@ import ( type SQLiteDatabase struct { sqlx.DB } + +// OpenSQLiteDatabase creates and open connection to new SQLite3 database. +func OpenSQLiteDatabase(databasePath string) (*SQLiteDatabase, error) { + // Open database and start transaction + var err error + db := sqlx.MustConnect("sqlite3", databasePath) + + tx, err := db.Beginx() + if err != nil { + return nil, err + } + + // Make sure to rollback if panic ever happened + defer func() { + if r := recover(); r != nil { + panicErr, _ := r.(error) + tx.Rollback() + + db = nil + err = panicErr + } + }() + + // Create tables + tx.MustExec(`CREATE TABLE IF NOT EXISTS account( + id INTEGER NOT NULL, + username TEXT NOT NULL, + password TEXT NOT NULL, + CONSTRAINT account_PK PRIMARY KEY(id), + CONSTRAINT account_username_UNIQUE UNIQUE(username))`) + + tx.MustExec(`CREATE TABLE IF NOT EXISTS bookmark( + id INTEGER NOT NULL, + url TEXT NOT NULL, + title TEXT NOT NULL, + excerpt TEXT NOT NULL DEFAULT "", + author TEXT NOT NULL DEFAULT "", + modified TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT bookmark_PK PRIMARY KEY(id), + CONSTRAINT bookmark_url_UNIQUE UNIQUE(url))`) + + tx.MustExec(`CREATE TABLE IF NOT EXISTS tag( + id INTEGER NOT NULL, + name TEXT NOT NULL, + CONSTRAINT tag_PK PRIMARY KEY(id), + CONSTRAINT tag_name_UNIQUE UNIQUE(name))`) + + tx.MustExec(`CREATE TABLE IF NOT EXISTS bookmark_tag( + bookmark_id INTEGER NOT NULL, + tag_id INTEGER NOT NULL, + CONSTRAINT bookmark_tag_PK PRIMARY KEY(bookmark_id, tag_id), + CONSTRAINT bookmark_id_FK FOREIGN KEY(bookmark_id) REFERENCES bookmark(id), + CONSTRAINT tag_id_FK FOREIGN KEY(tag_id) REFERENCES tag(id))`) + + tx.MustExec(`CREATE VIRTUAL TABLE IF NOT EXISTS bookmark_content USING fts4(title, content, html)`) + + err = tx.Commit() + checkError(err) + + return &SQLiteDatabase{*db}, err +} + +// InsertBookmark saves new bookmark to database. +// Returns new ID and error message if any happened. +func (db *SQLiteDatabase) InsertBookmark(bookmark model.Bookmark) (bookmarkID int, err error) { + // Check URL and title + if bookmark.URL == "" { + return -1, fmt.Errorf("URL must not be empty") + } + + if bookmark.Title == "" { + return -1, fmt.Errorf("title must not be empty") + } + + // Create ID (if needed) and modified time + if bookmark.ID != 0 { + bookmarkID = bookmark.ID + } else { + bookmarkID, err = db.CreateNewID("bookmark") + if err != nil { + return -1, err + } + } + + if bookmark.Modified == "" { + bookmark.Modified = time.Now().UTC().Format("2006-01-02 15:04:05") + } + + // Begin transaction + tx, err := db.Beginx() + if err != nil { + return -1, err + } + + // Make sure to rollback if panic ever happened + defer func() { + if r := recover(); r != nil { + panicErr, _ := r.(error) + tx.Rollback() + + bookmarkID = -1 + err = panicErr + } + }() + + // Save article to database + tx.MustExec(`INSERT INTO bookmark ( + id, url, title, excerpt, author, modified) + VALUES(?, ?, ?, ?, ?, ?)`, + bookmarkID, + bookmark.URL, + bookmark.Title, + bookmark.Excerpt, + bookmark.Author, + bookmark.Modified) + + // Save bookmark content + tx.MustExec(`INSERT INTO bookmark_content + (docid, title, content, html) VALUES (?, ?, ?, ?)`, + bookmarkID, + bookmark.Title, + bookmark.Content, + bookmark.HTML) + + // Save tags + stmtGetTag, err := tx.Preparex(`SELECT id FROM tag WHERE name = ?`) + checkError(err) + + stmtInsertTag, err := tx.Preparex(`INSERT INTO tag (name) VALUES (?)`) + checkError(err) + + stmtInsertBookmarkTag, err := tx.Preparex(`INSERT OR IGNORE INTO bookmark_tag + (tag_id, bookmark_id) VALUES (?, ?)`) + checkError(err) + + for _, tag := range bookmark.Tags { + tagName := strings.ToLower(tag.Name) + tagName = strings.TrimSpace(tagName) + + tagID := -1 + err = stmtGetTag.Get(&tagID, tagName) + checkError(err) + + if tagID == -1 { + res := stmtInsertTag.MustExec(tagName) + tagID64, err := res.LastInsertId() + checkError(err) + + tagID = int(tagID64) + } + + stmtInsertBookmarkTag.Exec(tagID, bookmarkID) + } + + // Commit transaction + err = tx.Commit() + checkError(err) + + return bookmarkID, err +} + +// GetBookmarks fetch list of bookmarks based on submitted ids. +func (db *SQLiteDatabase) GetBookmarks(opts GetBookmarksOptions) ([]model.Bookmark, error) { + // Create initial query + columns := []string{ + `b.id`, + `b.url`, + `b.title`, + `b.excerpt`, + `b.author`, + `b.modified`, + `bc.content <> "" has_content`} + + if opts.WithContent { + columns = append(columns, `bc.content`, `bc.html`) + } + + query := `SELECT ` + strings.Join(columns, ",") + ` + FROM bookmark b + LEFT JOIN bookmark_content bc ON bc.docid = b.id + WHERE 1` + + // Add where clause + args := []interface{}{} + + if len(opts.IDs) > 0 { + query += ` AND b.id IN (?)` + args = append(args, opts.IDs) + } + + if opts.Keyword != "" { + query += ` AND (b.url LIKE ? OR b.id IN ( + SELECT docid id + FROM bookmark_content + WHERE title MATCH ? OR content MATCH ?))` + + args = append(args, + "%"+opts.Keyword+"%", + opts.Keyword, + opts.Keyword) + } + + if len(opts.Tags) > 0 { + query += ` AND b.id IN ( + SELECT bookmark_id FROM bookmark_tag + WHERE tag_id IN (SELECT id FROM tag WHERE name IN (?)))` + + args = append(args, opts.Tags) + } + + // Add order clause + if opts.OrderLatest { + query += ` ORDER BY b.modified DESC` + } + + // Expand query, because some of the args might be an array + query, args, err := sqlx.In(query, args...) + if err != nil { + return nil, fmt.Errorf("failed to expand query: %v", err) + } + + // Fetch bookmarks + bookmarks := []model.Bookmark{} + err = db.Select(&bookmarks, query, args...) + if err != nil && err != sql.ErrNoRows { + return nil, fmt.Errorf("failed to fetch data: %v", err) + } + + // Fetch tags for each bookmarks + stmtGetTags, err := db.Preparex(`SELECT t.id, t.name + FROM bookmark_tag bt + LEFT JOIN tag t ON bt.tag_id = t.id + WHERE bt.bookmark_id = ? + ORDER BY t.name`) + if err != nil { + return nil, fmt.Errorf("failed to prepare tag query: %v", err) + } + defer stmtGetTags.Close() + + for i, book := range bookmarks { + book.Tags = []model.Tag{} + err = stmtGetTags.Select(&book.Tags, book.ID) + if err != nil && err != sql.ErrNoRows { + return nil, fmt.Errorf("failed to fetch tags: %v", err) + } + + bookmarks[i] = book + } + + return bookmarks, nil +} + +// CreateNewID creates new ID for specified table +func (db *SQLiteDatabase) CreateNewID(table string) (int, error) { + var tableID int + query := fmt.Sprintf(`SELECT IFNULL(MAX(id) + 1, 1) FROM %s`, table) + + err := db.Get(&tableID, query) + if err != nil && err != sql.ErrNoRows { + return -1, err + } + + return tableID, nil +} diff --git a/internal/model/model.go b/internal/model/model.go index 1cab8fa..1174fe0 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -4,25 +4,22 @@ package model type Tag struct { ID int `db:"id" json:"id"` Name string `db:"name" json:"name"` - NBookmarks int `db:"n_bookmarks" json:"nBookmarks"` + NBookmarks int `db:"n_bookmarks" json:"nBookmarks,omitempty"` Deleted bool `json:"-"` } // Bookmark is the record for an URL. type Bookmark struct { - ID int `db:"id" json:"id"` - URL string `db:"url" json:"url"` - Title string `db:"title" json:"title"` - ImageURL string `db:"image_url" json:"imageURL"` - Excerpt string `db:"excerpt" json:"excerpt"` - Author string `db:"author" json:"author"` - MinReadTime int `db:"min_read_time" json:"minReadTime"` - MaxReadTime int `db:"max_read_time" json:"maxReadTime"` - Modified string `db:"modified" json:"modified"` - Content string `db:"content" json:"-"` - HTML string `db:"html" json:"html,omitempty"` - HasContent bool `db:"has_content" json:"hasContent"` - Tags []Tag `json:"tags"` + ID int `db:"id" json:"id"` + URL string `db:"url" json:"url"` + Title string `db:"title" json:"title"` + Excerpt string `db:"excerpt" json:"excerpt"` + Author string `db:"author" json:"author"` + Modified string `db:"modified" json:"modified"` + Content string `db:"content" json:"-"` + HTML string `db:"html" json:"html,omitempty"` + HasContent bool `db:"has_content" json:"hasContent"` + Tags []Tag `json:"tags"` } // Account is person that allowed to access web interface. diff --git a/main.go b/main.go index d26bb9c..ac68370 100644 --- a/main.go +++ b/main.go @@ -1,13 +1,32 @@ package main import ( + "os" + fp "path/filepath" + "github.com/go-shiori/shiori/internal/cmd" + "github.com/go-shiori/shiori/internal/database" + _ "github.com/mattn/go-sqlite3" "github.com/sirupsen/logrus" ) +var dataDir = "dev-data" + func main() { - shioriCmd := cmd.ShioriCmd() - if err := shioriCmd.Execute(); err != nil { + // Make sure data dir exists + os.MkdirAll(dataDir, os.ModePerm) + + // Open database + dbPath := fp.Join(dataDir, "shiori.db") + sqliteDB, err := database.OpenSQLiteDatabase(dbPath) + if err != nil { + logrus.Fatalln(err) + } + + // Execute cmd + cmd.DB = sqliteDB + cmd.DataDir = dataDir + if err := cmd.ShioriCmd().Execute(); err != nil { logrus.Fatalln(err) } }