From a88a25c41a8d6ff3a7234db5bfb69f81c3d0c2f2 Mon Sep 17 00:00:00 2001 From: Radhi Fadlillah Date: Mon, 10 Jun 2019 15:49:54 +0700 Subject: [PATCH] Make no-archival flags works --- internal/cmd/add.go | 34 +++++++++++++++++++++------------- internal/cmd/update.go | 36 +++++++++++++++++++++--------------- 2 files changed, 42 insertions(+), 28 deletions(-) diff --git a/internal/cmd/add.go b/internal/cmd/add.go index 8bf6f590..5aa5bb59 100644 --- a/internal/cmd/add.go +++ b/internal/cmd/add.go @@ -29,6 +29,7 @@ func addCmd() *cobra.Command { cmd.Flags().StringP("excerpt", "e", "", "Custom excerpt for this bookmark") cmd.Flags().StringSliceP("tags", "t", []string{}, "Comma-separated tags for this bookmark") cmd.Flags().BoolP("offline", "o", false, "Save bookmark without fetching data from internet") + cmd.Flags().BoolP("no-archival", "a", false, "Save bookmark without creating offline archive") cmd.Flags().Bool("log-archival", false, "Log the archival process") return cmd @@ -41,6 +42,7 @@ func addHandler(cmd *cobra.Command, args []string) { excerpt, _ := cmd.Flags().GetString("excerpt") tags, _ := cmd.Flags().GetStringSlice("tags") offline, _ := cmd.Flags().GetBool("offline") + noArchival, _ := cmd.Flags().GetBool("no-archival") logArchival, _ := cmd.Flags().GetBool("log-archival") // Clean up URL by removing its fragment and UTM parameters @@ -97,24 +99,30 @@ func addHandler(cmd *cobra.Command, args []string) { defer resp.Body.Close() // Save as archive - buffer := bytes.NewBuffer(nil) + var readabilityInput io.Reader = resp.Body - archivePath := fp.Join(DataDir, "archive", fmt.Sprintf("%d", book.ID)) - archivalRequest := warc.ArchivalRequest{ - URL: url, - Reader: io.TeeReader(resp.Body, buffer), - ContentType: resp.Header.Get("Content-Type"), - LogEnabled: logArchival, - } + if !noArchival { + buffer := bytes.NewBuffer(nil) - err = warc.NewArchive(archivalRequest, archivePath) - if err != nil { - cError.Printf("Failed to create archive: %v\n", err) - return + archivePath := fp.Join(DataDir, "archive", fmt.Sprintf("%d", book.ID)) + archivalRequest := warc.ArchivalRequest{ + URL: url, + Reader: io.TeeReader(resp.Body, buffer), + ContentType: resp.Header.Get("Content-Type"), + LogEnabled: logArchival, + } + + err = warc.NewArchive(archivalRequest, archivePath) + if err != nil { + cError.Printf("Failed to create archive: %v\n", err) + return + } + + readabilityInput = buffer } // Parse article - article, err := readability.FromReader(buffer, url) + article, err := readability.FromReader(readabilityInput, url) if err != nil { cError.Printf("Failed to parse article: %v\n", err) return diff --git a/internal/cmd/update.go b/internal/cmd/update.go index 2564496e..436ebfa7 100644 --- a/internal/cmd/update.go +++ b/internal/cmd/update.go @@ -183,27 +183,33 @@ func updateHandler(cmd *cobra.Command, args []string) { defer resp.Body.Close() // Save as archive, make sure to delete the old one first - buffer := bytes.NewBuffer(nil) + var readabilityInput io.Reader = resp.Body - archivePath := fp.Join(DataDir, "archive", fmt.Sprintf("%d", book.ID)) - os.Remove(archivePath) + if !noArchival { + buffer := bytes.NewBuffer(nil) - archivalRequest := warc.ArchivalRequest{ - URL: book.URL, - Reader: io.TeeReader(resp.Body, buffer), - ContentType: resp.Header.Get("Content-Type"), - LogEnabled: logArchival, - } + archivePath := fp.Join(DataDir, "archive", fmt.Sprintf("%d", book.ID)) + os.Remove(archivePath) - err = warc.NewArchive(archivalRequest, archivePath) - if err != nil { - chProblem <- book.ID - chMessage <- fmt.Errorf("Failed to create archive %s: %v", book.URL, err) - return + archivalRequest := warc.ArchivalRequest{ + URL: book.URL, + Reader: io.TeeReader(resp.Body, buffer), + ContentType: resp.Header.Get("Content-Type"), + LogEnabled: logArchival, + } + + err = warc.NewArchive(archivalRequest, archivePath) + if err != nil { + chProblem <- book.ID + chMessage <- fmt.Errorf("Failed to create archive %s: %v", book.URL, err) + return + } + + readabilityInput = buffer } // Parse article - article, err := readability.FromReader(buffer, book.URL) + article, err := readability.FromReader(readabilityInput, book.URL) if err != nil { chProblem <- book.ID chMessage <- fmt.Errorf("Failed to parse %s: %v", book.URL, err)