fix: import should respect add_date or last_modified fields (#490)

* fix: handle last_modified and add_date in imports

* refactor: moved date format to a constant

* fix: use provided modified date when set

internal/cmd/export.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/go-shiori/shiori/internal/database"
+	"github.com/go-shiori/shiori/internal/model"
 	"github.com/spf13/cobra"
 )
 
@@ -59,7 +60,7 @@ func exportHandler(cmd *cobra.Command, args []string) {
 
 	for _, book := range bookmarks {
 		// Create Unix timestamp for bookmark
-		modifiedTime, err := time.Parse("2006-01-02 15:04:05", book.Modified)
+		modifiedTime, err := time.Parse(model.DatabaseDateFormat, book.Modified)
 		if err != nil {
 			modifiedTime = time.Now()
 		}

internal/cmd/import.go

@@ -5,7 +5,9 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"strconv"
 	"strings"
+	"time"
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/go-shiori/shiori/internal/core"
@@ -75,8 +77,24 @@ func importHandler(cmd *cobra.Command, args []string) {
 		url, _ := a.Attr("href")
 		strTags, _ := a.Attr("tags")
 
+		dateStr, fieldExists := a.Attr("last_modified")
+		if !fieldExists {
+			dateStr, _ = a.Attr("add_date")
+		}
+
+		// Using now as default date in case no last_modified nor add_date are present
+		modifiedDate := time.Now()
+		if dateStr != "" {
+			modifiedTsInt, err := strconv.Atoi(dateStr)
+			if err != nil {
+				cError.Printf("Skip %s: date field is not valid: %s", url, err)
+				return
+			}
+
+			modifiedDate = time.Unix(int64(modifiedTsInt), 0)
+		}
+
 		// Clean up URL
-		var err error
 		url, err = core.RemoveUTMParams(url)
 		if err != nil {
 			cError.Printf("Skip %s: URL is not valid\n", url)
@@ -123,10 +141,11 @@ func importHandler(cmd *cobra.Command, args []string) {
 
 		// Add item to list
 		bookmark := model.Bookmark{
-			ID:    bookID,
-			URL:   url,
-			Title: title,
-			Tags:  tags,
+			ID:       bookID,
+			URL:      url,
+			Title:    title,
+			Tags:     tags,
+			Modified: modifiedDate.Format(model.DatabaseDateFormat),
 		}
 
 		bookID++

internal/cmd/pocket.go

@@ -102,7 +102,7 @@ func pocketHandler(cmd *cobra.Command, args []string) {
 			ID:       bookID,
 			URL:      url,
 			Title:    title,
-			Modified: modified.Format("2006-01-02 15:04:05"),
+			Modified: modified.Format(model.DatabaseDateFormat),
 			Tags:     tags,
 		}
 

internal/database/mysql.go

@@ -108,7 +108,7 @@ func (db *MySQLDatabase) SaveBookmarks(ctx context.Context, bookmarks ...model.B
 		}
 
 		// Prepare modified time
-		modifiedTime := time.Now().UTC().Format("2006-01-02 15:04:05")
+		modifiedTime := time.Now().UTC().Format(model.DatabaseDateFormat)
 
 		// Execute statements
 
@@ -127,7 +127,9 @@ func (db *MySQLDatabase) SaveBookmarks(ctx context.Context, bookmarks ...model.B
 			}
 
 			// Set modified time
-			book.Modified = modifiedTime
+			if book.Modified == "" {
+				book.Modified = modifiedTime
+			}
 
 			// Save bookmark
 			_, err := stmtInsertBook.ExecContext(ctx, book.ID,

internal/database/pg.go

@@ -108,7 +108,7 @@ func (db *PGDatabase) SaveBookmarks(ctx context.Context, bookmarks ...model.Book
 		}
 
 		// Prepare modified time
-		modifiedTime := time.Now().UTC().Format("2006-01-02 15:04:05")
+		modifiedTime := time.Now().UTC().Format(model.DatabaseDateFormat)
 
 		// Execute statements
 		result = []model.Bookmark{}
@@ -123,7 +123,9 @@ func (db *PGDatabase) SaveBookmarks(ctx context.Context, bookmarks ...model.Book
 			}
 
 			// Set modified time
-			book.Modified = modifiedTime
+			if book.Modified == "" {
+				book.Modified = modifiedTime
+			}
 
 			// Save bookmark
 			err := stmtInsertBook.QueryRowContext(ctx,

internal/database/sqlite.go

@@ -125,7 +125,7 @@ func (db *SQLiteDatabase) SaveBookmarks(ctx context.Context, bookmarks ...model.
 		}
 
 		// Prepare modified time
-		modifiedTime := time.Now().UTC().Format("2006-01-02 15:04:05")
+		modifiedTime := time.Now().UTC().Format(model.DatabaseDateFormat)
 
 		// Execute statements
 
@@ -144,7 +144,9 @@ func (db *SQLiteDatabase) SaveBookmarks(ctx context.Context, bookmarks ...model.
 			}
 
 			// Set modified time
-			book.Modified = modifiedTime
+			if book.Modified == "" {
+				book.Modified = modifiedTime
+			}
 
 			// Save bookmark
 			hasContent := book.Content != ""
@@ -234,7 +236,7 @@ func (db *SQLiteDatabase) SaveBookmarks(ctx context.Context, bookmarks ...model.
 // GetBookmarks fetch list of bookmarks based on submitted options.
 func (db *SQLiteDatabase) GetBookmarks(ctx context.Context, opts GetBookmarksOptions) ([]model.Bookmark, error) {
 	// Create initial query
-	query := `SELECT 
+	query := `SELECT
 		b.id,
 		b.url,
 		b.title,

internal/model/const.go

@@ -0,0 +1,3 @@
+package model
+
+const DatabaseDateFormat = "2006-01-02 15:04:05"