go-shiori/shiori
1
1
Fork 0
mirror of https://github.com/go-shiori/shiori.git synced 2024-09-20 06:56:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: Ensure bookmark files are correctly downloaded before deleting current ones (#683)

Browse source 
* generate ebook get dstPath

* Archive and ebook can recover if download faild

* recover thumb if download faild

* thumb image create just if image processing is sucssesful

* create epub in tmp if it sucssesful copy to destination

* archive file create in tmp if it successful move to destination

* move to destination as function

* update ebook download api and remove .epub from file name

* report faild item to user

* not show dialog if error not happen

* update thumbnail based on last status of bookmark fix #524

* better warning massage

Co-authored-by: Felipe Martin <812088+fmartingr@users.noreply.github.com>

* tmpFile without .epub

* MoveToDestination change to MoveFileToDestination

* return .epub

* log if downloadBookImage return error

* fix bug remove imgPath just if download last image be unsuccessful

* update old unit test

* add processing.go unit test

* small massage for report failded item to the user

* add some more unit test and samplefile

* use sample image in unit test

* use local sample file and unit test not need internet connection anymore

* update error to user and log that too

* add more comment

* update comment

* change variable name parentDir to dstDir

* more simpler error handling

* remove unneeded defer

* remvoe unneeded epubWriter.Close()

* more readable unit test in processing

* more readable unit test for ebooks

* delete all defer os.RemoveAll from unit tests

* Better comment

Co-authored-by: Felipe Martin <812088+fmartingr@users.noreply.github.com>

* Better Error output

Co-authored-by: Felipe Martin <812088+fmartingr@users.noreply.github.com>

* fix err.String() method

---------

Co-authored-by: Felipe Martin <812088+fmartingr@users.noreply.github.com>
This commit is contained in:
Monirzadeh 2023-08-20 16:20:13 +00:00 committed by GitHub
parent 8b015a3850
commit f4817cb9c3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 665 additions and 250 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
internal/core/ebook.go
View file

@ -16,7 +16,10 @@ import (
"github.com/pkg/errors"
)
func GenerateEbook(req ProcessRequest) (book model.Bookmark, err error) {
// GenerateEbook receives a `ProcessRequest` and generates an ebook file in the destination path specified.
// The destination path `dstPath` should include file name with ".epub" extension
// The bookmark model will be used to update the UI based on whether this function is successful or not.
func GenerateEbook(req ProcessRequest, dstPath string) (book model.Bookmark, err error) {
// variable for store generated html code
var html string
@ -27,6 +30,7 @@ func GenerateEbook(req ProcessRequest) (book model.Bookmark, err error) {
return book, errors.New("bookmark ID is not valid")
}
// get current state of bookmark
// cheak archive and thumb
strID := strconv.Itoa(book.ID)
@ -40,35 +44,23 @@ func GenerateEbook(req ProcessRequest) (book model.Bookmark, err error) {
if _, err := os.Stat(archivePath); err == nil {
book.HasArchive = true
}
ebookfile := fp.Join(req.DataDir, "ebook", fmt.Sprintf("%d.epub", book.ID))
// if epub exist finish prosess else continue
if _, err := os.Stat(ebookfile); err == nil {
book.HasEbook = true
return book, nil
}
// this function create ebook from reader mode of bookmark so
// we can't create ebook from PDF so we return error here if bookmark is a pdf
contentType := req.ContentType
if strings.Contains(contentType, "application/pdf") {
return book, errors.New("can't create ebook for pdf")
}
ebookDir := fp.Join(req.DataDir, "ebook")
// check if directory not exsist create that
if _, err := os.Stat(ebookDir); os.IsNotExist(err) {
err := os.MkdirAll(ebookDir, model.DataDirPerm)
// create temporary epub file
tmpFile, err := os.CreateTemp("", "ebook")
if err != nil {
return book, errors.Wrap(err, "can't create ebook directory")
return book, errors.Wrap(err, "can't create temporary EPUB file")
}
}
// create epub file
epubFile, err := os.Create(ebookfile)
if err != nil {
return book, errors.Wrap(err, "can't create ebook")
}
defer epubFile.Close()
defer os.Remove(tmpFile.Name())
// Create zip archive
epubWriter := zip.NewWriter(epubFile)
defer epubWriter.Close()
epubWriter := zip.NewWriter(tmpFile)
// Create the mimetype file
mimetypeWriter, err := epubWriter.Create("mimetype")
@ -223,6 +215,27 @@ img {
if err != nil {
return book, errors.Wrap(err, "can't write into content.html")
}
// close epub and tmpFile
err = epubWriter.Close()
if err != nil {
return book, errors.Wrap(err, "failed to close EPUB writer")
}
err = tmpFile.Close()
if err != nil {
return book, errors.Wrap(err, "failed to close temporary EPUB file")
}
// open temporary file again
tmpFile, err = os.Open(tmpFile.Name())
if err != nil {
return book, errors.Wrap(err, "can't open temporary EPUB file")
}
defer tmpFile.Close()
// if everitings go well we start move ebook to dstPath
err = MoveFileToDestination(dstPath, tmpFile)
if err != nil {
return book, errors.Wrap(err, "failed move ebook to destination")
}
book.HasEbook = true
return book, nil
}

188
internal/core/ebook_test.go
View file

@ -1,7 +1,6 @@
package core_test
import (
"errors"
"fmt"
"os"
fp "path/filepath"
@ -12,10 +11,12 @@ import (
"github.com/stretchr/testify/assert"
)
func TestGenerateEbook_ValidBookmarkID_ReturnsBookmarkWithHasEbookTrue(t *testing.T) {
func TestGenerateEbook(t *testing.T) {
t.Run("Successful ebook generate", func(t *testing.T) {
t.Run("valid bookmarkId that return HasEbook true", func(t *testing.T) {
// test cae
tempDir := t.TempDir()
defer os.RemoveAll(tempDir)
dstDir := t.TempDir()
mockRequest := core.ProcessRequest{
Bookmark: model.Bookmark{
@ -24,79 +25,25 @@ func TestGenerateEbook_ValidBookmarkID_ReturnsBookmarkWithHasEbookTrue(t *testin
HTML: "<html><body>Example HTML</body></html>",
HasEbook: false,
},
DataDir: tempDir,
DataDir: dstDir,
ContentType: "text/html",
}
bookmark, err := core.GenerateEbook(mockRequest)
bookmark, err := core.GenerateEbook(mockRequest, fp.Join(tempDir, "1"))
assert.True(t, bookmark.HasEbook)
assert.NoError(t, err)
}
func TestGenerateEbook_InvalidBookmarkID_ReturnsError(t *testing.T) {
})
t.Run("ebook generate with valid BookmarkID EbookExist ImagePathExist ReturnWithHasEbookTrue", func(t *testing.T) {
tempDir := t.TempDir()
defer os.RemoveAll(tempDir)
mockRequest := core.ProcessRequest{
Bookmark: model.Bookmark{
ID: 0,
HasEbook: false,
},
DataDir: tempDir,
ContentType: "text/html",
}
bookmark, err := core.GenerateEbook(mockRequest)
assert.Equal(t, model.Bookmark{
ID: 0,
HasEbook: false,
}, bookmark)
assert.Error(t, err)
}
func TestGenerateEbook_ValidBookmarkID_EbookExist_EbookExist_ReturnWithHasEbookTrue(t *testing.T) {
tempDir := t.TempDir()
defer os.RemoveAll(tempDir)
dstDir := t.TempDir()
mockRequest := core.ProcessRequest{
Bookmark: model.Bookmark{
ID: 1,
HasEbook: false,
},
DataDir: tempDir,
ContentType: "text/html",
}
// Create the ebook directory
ebookDir := fp.Join(mockRequest.DataDir, "ebook")
err := os.MkdirAll(ebookDir, os.ModePerm)
if err != nil {
t.Fatal(err)
}
// Create the ebook file
ebookfile := fp.Join(mockRequest.DataDir, "ebook", fmt.Sprintf("%d.epub", mockRequest.Bookmark.ID))
file, err := os.Create(ebookfile)
if err != nil {
t.Fatal(err)
}
defer file.Close()
bookmark, err := core.GenerateEbook(mockRequest)
assert.True(t, bookmark.HasEbook)
assert.NoError(t, err)
}
func TestGenerateEbook_ValidBookmarkID_EbookExist_ImagePathExist_ReturnWithHasEbookTrue(t *testing.T) {
tempDir := t.TempDir()
defer os.RemoveAll(tempDir)
mockRequest := core.ProcessRequest{
Bookmark: model.Bookmark{
ID: 1,
HasEbook: false,
},
DataDir: tempDir,
DataDir: dstDir,
ContentType: "text/html",
}
// Create the image directory
@ -113,25 +60,25 @@ func TestGenerateEbook_ValidBookmarkID_EbookExist_ImagePathExist_ReturnWithHasEb
}
defer file.Close()
bookmark, err := core.GenerateEbook(mockRequest)
bookmark, err := core.GenerateEbook(mockRequest, fp.Join(tempDir, "1"))
expectedimagePath := "/bookmark/1/thumb"
if expectedimagePath != bookmark.ImageURL {
t.Errorf("Expected imageURL %s, but got %s", bookmark.ImageURL, expectedimagePath)
}
assert.True(t, bookmark.HasEbook)
assert.NoError(t, err)
}
})
t.Run("generate ebook valid BookmarkID EbookExist Returnh HasArchive True", func(t *testing.T) {
func TestGenerateEbook_ValidBookmarkID_EbookExist_ReturnWithHasArchiveTrue(t *testing.T) {
tempDir := t.TempDir()
defer os.RemoveAll(tempDir)
dstDir := t.TempDir()
mockRequest := core.ProcessRequest{
Bookmark: model.Bookmark{
ID: 1,
HasEbook: false,
},
DataDir: tempDir,
DataDir: dstDir,
ContentType: "text/html",
}
// Create the archive directory
@ -148,14 +95,64 @@ func TestGenerateEbook_ValidBookmarkID_EbookExist_ReturnWithHasArchiveTrue(t *te
}
defer file.Close()
bookmark, err := core.GenerateEbook(mockRequest)
bookmark, err := core.GenerateEbook(mockRequest, fp.Join(tempDir, "1"))
assert.True(t, bookmark.HasArchive)
assert.NoError(t, err)
})
})
t.Run("specific ebook generate case", func(t *testing.T) {
t.Run("unvalid bookmarkId that return Error", func(t *testing.T) {
tempDir := t.TempDir()
mockRequest := core.ProcessRequest{
Bookmark: model.Bookmark{
ID: 0,
HasEbook: false,
},
DataDir: tempDir,
ContentType: "text/html",
}
func TestGenerateEbook_ValidBookmarkID_RetuenError_PDF(t *testing.T) {
bookmark, err := core.GenerateEbook(mockRequest, tempDir)
assert.Equal(t, model.Bookmark{
ID: 0,
HasEbook: false,
}, bookmark)
assert.Error(t, err)
})
t.Run("ebook exist return HasEbook true", func(t *testing.T) {
tempDir := t.TempDir()
dstDir := t.TempDir()
mockRequest := core.ProcessRequest{
Bookmark: model.Bookmark{
ID: 1,
HasEbook: false,
},
DataDir: dstDir,
ContentType: "text/html",
}
// Create the ebook directory
ebookDir := fp.Join(mockRequest.DataDir, "ebook")
err := os.MkdirAll(ebookDir, os.ModePerm)
if err != nil {
t.Fatal(err)
}
// Create the ebook file
ebookfile := fp.Join(mockRequest.DataDir, "ebook", fmt.Sprintf("%d.epub", mockRequest.Bookmark.ID))
file, err := os.Create(ebookfile)
if err != nil {
t.Fatal(err)
}
defer file.Close()
bookmark, err := core.GenerateEbook(mockRequest, fp.Join(tempDir, "1"))
assert.True(t, bookmark.HasEbook)
assert.NoError(t, err)
})
t.Run("generate ebook valid BookmarkID RetuenError for PDF file", func(t *testing.T) {
tempDir := t.TempDir()
defer os.RemoveAll(tempDir)
mockRequest := core.ProcessRequest{
Bookmark: model.Bookmark{
@ -166,54 +163,13 @@ func TestGenerateEbook_ValidBookmarkID_RetuenError_PDF(t *testing.T) {
ContentType: "application/pdf",
}
bookmark, err := core.GenerateEbook(mockRequest)
bookmark, err := core.GenerateEbook(mockRequest, tempDir)
assert.False(t, bookmark.HasEbook)
assert.Error(t, err)
assert.Contains(t, err.Error(), "can't create ebook for pdf")
}
func TestGenerateEbook_CreateEbookDirectoryNotWritable(t *testing.T) {
// Create a temporary directory to use as the parent directory
parentDir := t.TempDir()
// Create a child directory with read-only permissions
ebookDir := fp.Join(parentDir, "ebook")
err := os.Mkdir(ebookDir, 0444)
if err != nil {
t.Fatalf("could not create ebook directory: %s", err)
}
mockRequest := core.ProcessRequest{
Bookmark: model.Bookmark{
ID: 1,
HasEbook: false,
},
DataDir: ebookDir,
ContentType: "text/html",
}
// Call GenerateEbook to create the ebook directory
bookmark, err := core.GenerateEbook(mockRequest)
if err == nil {
t.Fatal("GenerateEbook succeeded even though MkdirAll should have failed")
}
if !errors.Is(err, os.ErrPermission) {
t.Fatalf("unexpected error: expected os.ErrPermission, got %v", err)
}
// Check if the ebook directory still exists and has read-only permissions
info, err := os.Stat(ebookDir)
if err != nil {
t.Fatalf("could not retrieve ebook directory info: %s", err)
}
if !info.IsDir() {
t.Errorf("ebook directory is not a directory")
}
if info.Mode().Perm() != 0444 {
t.Errorf("ebook directory has incorrect permissions: expected 0444, got %o", info.Mode().Perm())
}
assert.False(t, bookmark.HasEbook)
})
})
}
// Add more unit tests for other scenarios that missing specialy

106
internal/core/processing.go
View file

@ -8,10 +8,10 @@ import (
"image/draw"
"image/jpeg"
"io"
"log"
"math"
"net/url"
"os"
"path"
fp "path/filepath"
"strconv"
"strings"
@ -37,6 +37,8 @@ type ProcessRequest struct {
LogArchival bool
}
var ErrNoSupportedImageType = errors.New("unsupported image type")
// ProcessBookmark process the bookmark and archive it if needed.
// Return three values, is error fatal, and error value.
func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool, err error) {
@ -66,13 +68,15 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool,
}
// If this is HTML, parse for readable content
strID := strconv.Itoa(book.ID)
imgPath := fp.Join(req.DataDir, "thumb", strID)
var imageURLs []string
if strings.Contains(contentType, "text/html") {
isReadable := readability.Check(readabilityCheckInput)
nurl, err := url.Parse(book.URL)
if err != nil {
return book, true, fmt.Errorf("Failed to parse url: %v", err)
return book, true, fmt.Errorf("failed to parse url: %v", err)
}
article, err := readability.FromReader(readabilityInput, nurl)
@ -101,6 +105,8 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool,
// Get image URL
if article.Image != "" {
imageURLs = append(imageURLs, article.Image)
} else {
os.Remove(imgPath)
}
if article.Favicon != "" {
@ -115,26 +121,32 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool,
}
// Save article image to local disk
strID := strconv.Itoa(book.ID)
imgPath := fp.Join(req.DataDir, "thumb", strID)
for _, imageURL := range imageURLs {
err = downloadBookImage(imageURL, imgPath)
for i, imageURL := range imageURLs {
err = DownloadBookImage(imageURL, imgPath)
if err != nil && errors.Is(err, ErrNoSupportedImageType) {
log.Printf("%s: %s", err, imageURL)
if i == len(imageURLs)-1 {
os.Remove(imgPath)
}
}
if err != nil {
log.Printf("File download not successful for image URL: %s", imageURL)
continue
}
if err == nil {
book.ImageURL = path.Join("/", "bookmark", strID, "thumb")
book.ImageURL = fp.Join("/", "bookmark", strID, "thumb")
break
}
}
// If needed, create ebook as well
if book.CreateEbook {
ebookPath := fp.Join(req.DataDir, "ebook", fmt.Sprintf("%d.epub", book.ID))
os.Remove(ebookPath)
ebookPath := fp.Join(req.DataDir, "ebook", strID+".epub")
if strings.Contains(contentType, "application/pdf") {
return book, false, errors.Wrap(err, "can't create ebook from pdf")
} else {
_, err = GenerateEbook(req)
_, err = GenerateEbook(req, ebookPath)
if err != nil {
return book, true, errors.Wrap(err, "failed to create ebook")
}
@ -144,8 +156,11 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool,
// If needed, create offline archive as well
if book.CreateArchive {
archivePath := fp.Join(req.DataDir, "archive", fmt.Sprintf("%d", book.ID))
os.Remove(archivePath)
tmpFile, err := os.CreateTemp("", "archive")
if err != nil {
return book, false, fmt.Errorf("failed to create temp archive: %v", err)
}
defer os.Remove(tmpFile.Name())
archivalRequest := warc.ArchivalRequest{
URL: book.URL,
@ -155,18 +170,27 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool,
LogEnabled: req.LogArchival,
}
err = warc.NewArchive(archivalRequest, archivePath)
err = warc.NewArchive(archivalRequest, tmpFile.Name())
if err != nil {
defer os.Remove(tmpFile.Name())
return book, false, fmt.Errorf("failed to create archive: %v", err)
}
// Prepare destination file.
dstPath := fp.Join(req.DataDir, "archive", fmt.Sprintf("%d", book.ID))
err = MoveFileToDestination(dstPath, tmpFile)
if err != nil {
return book, false, fmt.Errorf("failed move archive to destination `: %v", err)
}
book.HasArchive = true
}
return book, false, nil
}
func downloadBookImage(url, dstPath string) error {
func DownloadBookImage(url, dstPath string) error {
// Fetch data from URL
resp, err := httpClient.Get(url)
if err != nil {
@ -180,22 +204,16 @@ func downloadBookImage(url, dstPath string) error {
!strings.Contains(cp, "image/pjpeg") &&
!strings.Contains(cp, "image/jpg") &&
!strings.Contains(cp, "image/png") {
return fmt.Errorf("%s is not a supported image", url)
return ErrNoSupportedImageType
}
// At this point, the download has finished successfully.
// Prepare destination file.
err = os.MkdirAll(fp.Dir(dstPath), model.DataDirPerm)
// Create tmpFile
tmpFile, err := os.CreateTemp("", "image")
if err != nil {
return fmt.Errorf("failed to create image dir: %v", err)
return fmt.Errorf("failed to create temporary image file: %v", err)
}
dstFile, err := os.Create(dstPath)
if err != nil {
return fmt.Errorf("failed to create image file: %v", err)
}
defer dstFile.Close()
defer os.Remove(tmpFile.Name())
// Parse image and process it.
// If image is smaller than 600x400 or its ratio is less than 4:3, resize.
@ -211,7 +229,7 @@ func downloadBookImage(url, dstPath string) error {
imgRatio := float64(imgWidth) / float64(imgHeight)
if imgWidth >= 600 && imgHeight >= 400 && imgRatio > 1.3 {
err = jpeg.Encode(dstFile, img, nil)
err = jpeg.Encode(tmpFile, img, nil)
} else {
// Create background
bg := image.NewNRGBA(imgRect)
@ -236,12 +254,44 @@ func downloadBookImage(url, dstPath string) error {
draw.Draw(bg, bgRect, fg, fgPosition, draw.Over)
// Save to file
err = jpeg.Encode(dstFile, bg, nil)
err = jpeg.Encode(tmpFile, bg, nil)
}
if err != nil {
return fmt.Errorf("failed to save image %s: %v", url, err)
}
err = MoveFileToDestination(dstPath, tmpFile)
if err != nil {
return err
}
return nil
}
// dstPath requires the filename
func MoveFileToDestination(dstPath string, tmpFile *os.File) error {
// Prepare destination file.
err := os.MkdirAll(fp.Dir(dstPath), model.DataDirPerm)
if err != nil {
return fmt.Errorf("failed to create destination dir: %v", err)
}
dstFile, err := os.Create(dstPath)
if err != nil {
return fmt.Errorf("failed to create destination file: %v", err)
}
defer dstFile.Close()
// Copy temporary file to destination
_, err = tmpFile.Seek(0, io.SeekStart)
if err != nil {
return fmt.Errorf("failed to rewind temporary file: %v", err)
}
_, err = io.Copy(dstFile, tmpFile)
if err != nil {
return fmt.Errorf("failed to copy file to the destination")
}
return nil
}

349
internal/core/processing_test.go Normal file
View file

@ -0,0 +1,349 @@
package core_test
import (
"bytes"
"net/http"
"net/http/httptest"
"os"
fp "path/filepath"
"testing"
"github.com/go-shiori/shiori/internal/core"
"github.com/go-shiori/shiori/internal/model"
"github.com/stretchr/testify/assert"
)
func TestMoveFileToDestination(t *testing.T) {
t.Run("create fails", func(t *testing.T) {
t.Run("directory create fails", func(t *testing.T) {
// test if create dir fails
tmpFile, err := os.CreateTemp("", "image")
assert.NoError(t, err)
defer os.Remove(tmpFile.Name())
err = core.MoveFileToDestination("/destination/test", tmpFile)
assert.Error(t, err)
assert.Contains(t, err.Error(), "failed to create destination dir")
})
t.Run("file create fails", func(t *testing.T) {
// if create file failed
tmpFile, err := os.CreateTemp("", "image")
assert.NoError(t, err)
defer os.Remove(tmpFile.Name())
// Create a destination directory
dstDir := t.TempDir()
assert.NoError(t, err)
defer os.Remove(dstDir)
// Set destination path to an invalid file name to force os.Create to fail
dstPath := fp.Join(dstDir, "\000invalid\000")
err = core.MoveFileToDestination(dstPath, tmpFile)
assert.Error(t, err)
assert.Contains(t, err.Error(), "failed to create destination file")
})
})
}
func TestDownloadBookImage(t *testing.T) {
t.Run("Download Images", func(t *testing.T) {
t.Run("fails", func(t *testing.T) {
// images is too small with unsupported format with a valid URL
imageURL := "https://github.com/go-shiori/shiori/blob/master/internal/view/assets/res/apple-touch-icon-152x152.png"
tempDir := t.TempDir()
dstPath := fp.Join(tempDir, "1")
defer os.Remove(dstPath)
// Act
err := core.DownloadBookImage(imageURL, dstPath)
// Assert
assert.EqualError(t, err, "unsupported image type")
assert.NoFileExists(t, dstPath)
})
t.Run("sucssesful downlosd image", func(t *testing.T) {
// Arrange
imageURL := "https://raw.githubusercontent.com/go-shiori/shiori/master/docs/readme/cover.png"
tempDir := t.TempDir()
dstPath := fp.Join(tempDir, "1")
defer os.Remove(dstPath)
// Act
err := core.DownloadBookImage(imageURL, dstPath)
// Assert
assert.NoError(t, err)
assert.FileExists(t, dstPath)
})
t.Run("sucssesful downlosd medium size image", func(t *testing.T) {
// create a file server handler for the 'testdata' directory
fs := http.FileServer(http.Dir("../../testdata/"))
// start a test server with the file server handler
server := httptest.NewServer(fs)
defer server.Close()
// Arrange
imageURL := server.URL + "/medium_image.png"
tempDir := t.TempDir()
dstPath := fp.Join(tempDir, "1")
defer os.Remove(dstPath)
// Act
err := core.DownloadBookImage(imageURL, dstPath)
// Assert
assert.NoError(t, err)
assert.FileExists(t, dstPath)
})
})
}
func TestProcessBookmark(t *testing.T) {
t.Run("ProcessRequest with sucssesful result", func(t *testing.T) {
t.Run("Normal without image", func(t *testing.T) {
bookmark := model.Bookmark{
ID: 1,
URL: "https://example.com",
Title: "Example",
Excerpt: "This is an example article",
CreateEbook: true,
CreateArchive: true,
}
content := bytes.NewBufferString("<html><head></head><body><p>This is an example article</p></body></html>")
request := core.ProcessRequest{
Bookmark: bookmark,
Content: content,
ContentType: "text/html",
DataDir: "/tmp",
KeepTitle: true,
KeepExcerpt: true,
}
expected, _, _ := core.ProcessBookmark(request)
if expected.ID != bookmark.ID {
t.Errorf("Unexpected ID: got %v, want %v", expected.ID, bookmark.ID)
}
if expected.URL != bookmark.URL {
t.Errorf("Unexpected URL: got %v, want %v", expected.URL, bookmark.URL)
}
if expected.Title != bookmark.Title {
t.Errorf("Unexpected Title: got %v, want %v", expected.Title, bookmark.Title)
}
if expected.Excerpt != bookmark.Excerpt {
t.Errorf("Unexpected Excerpt: got %v, want %v", expected.Excerpt, bookmark.Excerpt)
}
})
t.Run("Normal with multipleimage", func(t *testing.T) {
html := `html<html>
<head>
<meta property="og:image" content="http://example.com/image1.jpg">
<meta property="og:image" content="http://example.com/image2.jpg">
<link rel="icon" type="image/png" href="http://example.com/favicon.png">
</head>
<body>
<p>This is an example article</p>
</body>
</html>`
bookmark := model.Bookmark{
ID: 1,
URL: "https://example.com",
Title: "Example",
Excerpt: "This is an example article",
CreateEbook: true,
CreateArchive: true,
}
content := bytes.NewBufferString(html)
request := core.ProcessRequest{
Bookmark: bookmark,
Content: content,
ContentType: "text/html",
DataDir: "/tmp",
KeepTitle: true,
KeepExcerpt: true,
}
expected, _, _ := core.ProcessBookmark(request)
if expected.ID != bookmark.ID {
t.Errorf("Unexpected ID: got %v, want %v", expected.ID, bookmark.ID)
}
if expected.URL != bookmark.URL {
t.Errorf("Unexpected URL: got %v, want %v", expected.URL, bookmark.URL)
}
if expected.Title != bookmark.Title {
t.Errorf("Unexpected Title: got %v, want %v", expected.Title, bookmark.Title)
}
if expected.Excerpt != bookmark.Excerpt {
t.Errorf("Unexpected Excerpt: got %v, want %v", expected.Excerpt, bookmark.Excerpt)
}
})
t.Run("ProcessRequest sucssesful with multipleimage included favicon and Thumbnail ", func(t *testing.T) {
// create a file server handler for the 'testdata' directory
fs := http.FileServer(http.Dir("../../testdata/"))
// start a test server with the file server handler
server := httptest.NewServer(fs)
defer server.Close()
html := `html<html>
<head>
<meta property="og:image" content="http://example.com/image1.jpg">
<meta property="og:image" content="` + server.URL + `/big_image.png">
<link rel="icon" type="image/svg" href="` + server.URL + `/favicon.svg">
</head>
<body>
<p>This is an example article</p>
</body>
</html>`
bookmark := model.Bookmark{
ID: 1,
URL: "https://example.com",
Title: "Example",
Excerpt: "This is an example article",
CreateEbook: true,
CreateArchive: true,
}
content := bytes.NewBufferString(html)
request := core.ProcessRequest{
Bookmark: bookmark,
Content: content,
ContentType: "text/html",
DataDir: "/tmp",
KeepTitle: true,
KeepExcerpt: true,
}
expected, _, _ := core.ProcessBookmark(request)
if expected.ID != bookmark.ID {
t.Errorf("Unexpected ID: got %v, want %v", expected.ID, bookmark.ID)
}
if expected.URL != bookmark.URL {
t.Errorf("Unexpected URL: got %v, want %v", expected.URL, bookmark.URL)
}
if expected.Title != bookmark.Title {
t.Errorf("Unexpected Title: got %v, want %v", expected.Title, bookmark.Title)
}
if expected.Excerpt != bookmark.Excerpt {
t.Errorf("Unexpected Excerpt: got %v, want %v", expected.Excerpt, bookmark.Excerpt)
}
})
t.Run("ProcessRequest sucssesful with empty title ", func(t *testing.T) {
bookmark := model.Bookmark{
ID: 1,
URL: "https://example.com",
Title: "",
Excerpt: "This is an example article",
CreateEbook: true,
CreateArchive: true,
}
content := bytes.NewBufferString("<html><head></head><body><p>This is an example article</p></body></html>")
request := core.ProcessRequest{
Bookmark: bookmark,
Content: content,
ContentType: "text/html",
DataDir: "/tmp",
KeepTitle: true,
KeepExcerpt: true,
}
expected, _, _ := core.ProcessBookmark(request)
if expected.ID != bookmark.ID {
t.Errorf("Unexpected ID: got %v, want %v", expected.ID, bookmark.ID)
}
if expected.URL != bookmark.URL {
t.Errorf("Unexpected URL: got %v, want %v", expected.URL, bookmark.URL)
}
if expected.Title != bookmark.URL {
t.Errorf("Unexpected Title: got %v, want %v", expected.Title, bookmark.Title)
}
if expected.Excerpt != bookmark.Excerpt {
t.Errorf("Unexpected Excerpt: got %v, want %v", expected.Excerpt, bookmark.Excerpt)
}
})
t.Run("ProcessRequest sucssesful with empty Excerpt", func(t *testing.T) {
bookmark := model.Bookmark{
ID: 1,
URL: "https://example.com",
Title: "",
Excerpt: "This is an example article",
CreateEbook: true,
CreateArchive: true,
}
content := bytes.NewBufferString("<html><head></head><body><p>This is an example article</p></body></html>")
request := core.ProcessRequest{
Bookmark: bookmark,
Content: content,
ContentType: "text/html",
DataDir: "/tmp",
KeepTitle: true,
KeepExcerpt: false,
}
expected, _, _ := core.ProcessBookmark(request)
if expected.ID != bookmark.ID {
t.Errorf("Unexpected ID: got %v, want %v", expected.ID, bookmark.ID)
}
if expected.URL != bookmark.URL {
t.Errorf("Unexpected URL: got %v, want %v", expected.URL, bookmark.URL)
}
if expected.Title != bookmark.URL {
t.Errorf("Unexpected Title: got %v, want %v", expected.Title, bookmark.Title)
}
if expected.Excerpt != bookmark.Excerpt {
t.Errorf("Unexpected Excerpt: got %v, want %v", expected.Excerpt, bookmark.Excerpt)
}
})
t.Run("Specific case", func(t *testing.T) {
t.Run("ProcessRequest with ID zero", func(t *testing.T) {
bookmark := model.Bookmark{
ID: 0,
URL: "https://example.com",
Title: "Example",
Excerpt: "This is an example article",
CreateEbook: true,
CreateArchive: true,
}
content := bytes.NewBufferString("<html><head></head><body><p>This is an example article</p></body></html>")
request := core.ProcessRequest{
Bookmark: bookmark,
Content: content,
ContentType: "text/html",
DataDir: "/tmp",
KeepTitle: true,
KeepExcerpt: true,
}
_, isFatal, err := core.ProcessBookmark(request)
assert.Error(t, err)
assert.Contains(t, err.Error(), "bookmark ID is not valid")
assert.True(t, isFatal)
})
t.Run("ProcessRequest that content type not zero", func(t *testing.T) {
bookmark := model.Bookmark{
ID: 1,
URL: "https://example.com",
Title: "Example",
Excerpt: "This is an example article",
CreateEbook: true,
CreateArchive: true,
}
content := bytes.NewBufferString("<html><head></head><body><p>This is an example article</p></body></html>")
request := core.ProcessRequest{
Bookmark: bookmark,
Content: content,
ContentType: "application/pdf",
DataDir: "/tmp",
KeepTitle: true,
KeepExcerpt: true,
}
_, _, err := core.ProcessBookmark(request)
assert.NoError(t, err)
})
})
})
}

21
internal/view/assets/js/page/home.js
View file

@ -703,10 +703,31 @@ export default {
this.dialog.loading = false;
this.dialog.visible = false;
let faildedUpdateArchives = [];
let faildedCreateEbook = [];
json.forEach(book => {
var item = items.find(el => el.id === book.id);
this.bookmarks.splice(item.index, 1, book);
if (data.createArchive && !book.hasArchive){
faildedUpdateArchives.push(book.id);
console.error("can't update archive for bookmark id", book.id)
}
if (data.createEbook && !book.hasEbook){
faildedCreateEbook.push(book.id);
console.error("can't update ebook for bookmark id:", book.id)
}
});
if(faildedCreateEbook.length > 0 || faildedUpdateArchives.length > 0){
this.showDialog({
title: `Bookmarks Id that Update Action Faild`,
content: `Not all bookmarks could have their contents updated, but no files were overwritten.`,
mainText: "OK",
mainClick: () => {
this.dialog.visible = false;
},
})
}
}).catch(err => {
this.selection = [];
this.editMode = false;

22
internal/webserver/handler-api.go
View file

@ -434,13 +434,33 @@ func (h *Handler) ApiDownloadEbook(w http.ResponseWriter, r *http.Request, ps ht
ContentType: contentType,
}
book, err = core.GenerateEbook(request)
// if file exist book return avilable file
strID := strconv.Itoa(book.ID)
ebookPath := fp.Join(request.DataDir, "ebook", strID+".epub")
_, err = os.Stat(ebookPath)
if err == nil {
// file already exists, return the existing file
imagePath := fp.Join(request.DataDir, "thumb", fmt.Sprintf("%d", book.ID))
archivePath := fp.Join(request.DataDir, "archive", fmt.Sprintf("%d", book.ID))
if _, err := os.Stat(imagePath); err == nil {
book.ImageURL = fp.Join("/", "bookmark", strID, "thumb")
}
if _, err := os.Stat(archivePath); err == nil {
book.HasArchive = true
}
book.HasEbook = true
} else {
// generate ebook file
book, err = core.GenerateEbook(request, ebookPath)
content.Close()
if err != nil {
chProblem <- book.ID
return
}
}
// Update list of bookmarks
mx.Lock()

2
internal/webserver/handler-ui.go
View file

@ -48,7 +48,7 @@ func (h *Handler) ServeBookmarkContent(w http.ResponseWriter, r *http.Request, p
}
}
// Check if it has archive.
// Check if it has ebook.
ebookPath := fp.Join(h.DataDir, "ebook", strID+".epub")
if fileExists(ebookPath) {
bookmark.HasEbook = true

BIN
testdata/big_image.png vendored Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 16 KiB

BIN
testdata/favicon.png vendored Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 534 B

6
testdata/favicon.svg vendored Normal file
View file

@ -0,0 +1,6 @@
<?xml version="1.0" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 20010904//EN"
"http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd">
<svg width="32" height="32">
<circle cx="16" cy="16" r="16"/>
</svg>
Side by side

After

Width:  |  Height:  |  Size: 223 B

BIN
testdata/medium_image.png vendored Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 3.3 KiB