mirror of
https://github.com/go-shiori/shiori.git
synced 2024-09-20 15:06:14 +08:00
Initial Ebook Support (#623)
* generate ebook
* serve ebook file
* Update ebook.go not download same image twice anymore
* download ebook request api/ui part
* fix typo
* add stylesheet
* update hasEbook status
* download link update after ebook generate
update bookmark command in ui
* download ebook with bookmark title
* Apply suggestions from code review for better error handling
Co-authored-by: Felipe Martin <812088+fmartingr@users.noreply.github.com>
* Update internal/view/js/page/home.js fix typo
Co-authored-by: Felipe Martin <812088+fmartingr@users.noreply.github.com>
* import error lib and retuen missing error
* move ebook download action to update cache
* replace io/ioutil with io
* add missing error handling
* update Archive now always update ebook
* replace panic error with 404
* remove ebook with delete action
* add download ebook link to content page
* remove tags that not work correctly right now
* if file is pdf not generate ebook
* update style.css
* Revert "update style.css"
This reverts commit 519e10d6ce.
* remove download limit for api
* fix missing fmt.Errorf and change to errors.Wrap
* fix double panic
* return 404 if bookmark not exist
* change function name to GenerateEbook
* not isFatalErr anymore
* add unit test
* remove uneeded field for unit test
---------
Co-authored-by: Felipe Martin <812088+fmartingr@users.noreply.github.com>
This commit is contained in:
Monirzadeh
GitHub
parent
ec86febdaa
commit
249f4b89c8
252
internal/core/ebook.go
Normal file
252
internal/core/ebook.go
Normal file
|
|
@ -0,0 +1,252 @@
|
|||
package core
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
fp "path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/go-shiori/shiori/internal/model"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
func GenerateEbook(req ProcessRequest) (book model.Bookmark, err error) {
|
||||
// variable for store generated html code
|
||||
var html string
|
||||
|
||||
book = req.Bookmark
|
||||
|
||||
// Make sure bookmark ID is defined
|
||||
if book.ID == 0 {
|
||||
return book, errors.New("bookmark ID is not valid")
|
||||
}
|
||||
|
||||
// cheak archive and thumb
|
||||
strID := strconv.Itoa(book.ID)
|
||||
|
||||
imagePath := fp.Join(req.DataDir, "thumb", fmt.Sprintf("%d", book.ID))
|
||||
archivePath := fp.Join(req.DataDir, "archive", fmt.Sprintf("%d", book.ID))
|
||||
|
||||
if _, err := os.Stat(imagePath); err == nil {
|
||||
book.ImageURL = fp.Join("/", "bookmark", strID, "thumb")
|
||||
}
|
||||
|
||||
if _, err := os.Stat(archivePath); err == nil {
|
||||
book.HasArchive = true
|
||||
}
|
||||
ebookfile := fp.Join(req.DataDir, "ebook", fmt.Sprintf("%d.epub", book.ID))
|
||||
// if epub exist finish prosess else continue
|
||||
if _, err := os.Stat(ebookfile); err == nil {
|
||||
book.HasEbook = true
|
||||
return book, nil
|
||||
}
|
||||
contentType := req.ContentType
|
||||
if strings.Contains(contentType, "application/pdf") {
|
||||
return book, errors.New("can't create ebook for pdf")
|
||||
}
|
||||
|
||||
ebookDir := fp.Join(req.DataDir, "ebook")
|
||||
// check if directory not exsist create that
|
||||
if _, err := os.Stat(ebookDir); os.IsNotExist(err) {
|
||||
err := os.MkdirAll(ebookDir, model.DataDirPerm)
|
||||
if err != nil {
|
||||
return book, errors.Wrap(err, "can't create ebook directory")
|
||||
}
|
||||
}
|
||||
// create epub file
|
||||
epubFile, err := os.Create(ebookfile)
|
||||
if err != nil {
|
||||
return book, errors.Wrap(err, "can't create ebook")
|
||||
}
|
||||
defer epubFile.Close()
|
||||
|
||||
// Create zip archive
|
||||
epubWriter := zip.NewWriter(epubFile)
|
||||
defer epubWriter.Close()
|
||||
|
||||
// Create the mimetype file
|
||||
mimetypeWriter, err := epubWriter.Create("mimetype")
|
||||
if err != nil {
|
||||
return book, errors.Wrap(err, "can't create mimetype")
|
||||
}
|
||||
_, err = mimetypeWriter.Write([]byte("application/epub+zip"))
|
||||
if err != nil {
|
||||
return book, errors.Wrap(err, "can't write into mimetype file")
|
||||
}
|
||||
|
||||
// Create the container.xml file
|
||||
containerWriter, err := epubWriter.Create("META-INF/container.xml")
|
||||
if err != nil {
|
||||
return book, errors.Wrap(err, "can't create container.xml")
|
||||
}
|
||||
|
||||
_, err = containerWriter.Write([]byte(`<?xml version="1.0" encoding="UTF-8"?>
|
||||
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
|
||||
<rootfiles>
|
||||
<rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/>
|
||||
</rootfiles>
|
||||
</container>`))
|
||||
if err != nil {
|
||||
return book, errors.Wrap(err, "can't write into container.xml file")
|
||||
}
|
||||
|
||||
contentOpfWriter, err := epubWriter.Create("OEBPS/content.opf")
|
||||
if err != nil {
|
||||
return book, errors.Wrap(err, "can't create content.opf")
|
||||
}
|
||||
_, err = contentOpfWriter.Write([]byte(`<?xml version="1.0" encoding="UTF-8"?>
|
||||
<package xmlns="http://www.idpf.org/2007/opf" version="2.0" unique-identifier="BookId">
|
||||
<metadata>
|
||||
<dc:title>` + book.Title + `</dc:title>
|
||||
</metadata>
|
||||
<manifest>
|
||||
<item id="ncx" href="toc.ncx" media-type="application/x-dtbncx+xml"/>
|
||||
<item id="content" href="content.html" media-type="application/xhtml+xml"/>
|
||||
<item id="id" href="../style.css" media-type="text/css"/>
|
||||
</manifest>
|
||||
<spine toc="ncx">
|
||||
<itemref idref="content"/>
|
||||
</spine>
|
||||
</package>`))
|
||||
if err != nil {
|
||||
return book, errors.Wrap(err, "can't write into container.opf file")
|
||||
}
|
||||
|
||||
// Create the style.css file
|
||||
styleWriter, err := epubWriter.Create("style.css")
|
||||
if err != nil {
|
||||
return book, errors.Wrap(err, "can't create content.xml")
|
||||
}
|
||||
_, err = styleWriter.Write([]byte(`content {
|
||||
display: block;
|
||||
font-size: 1em;
|
||||
line-height: 1.2;
|
||||
padding-left: 0;
|
||||
padding-right: 0;
|
||||
text-align: justify;
|
||||
margin: 0 5pt
|
||||
}
|
||||
img {
|
||||
margin: auto;
|
||||
display: block;
|
||||
}`))
|
||||
if err != nil {
|
||||
return book, errors.Wrap(err, "can't write into style.css file")
|
||||
}
|
||||
// Create the toc.ncx file
|
||||
tocNcxWriter, err := epubWriter.Create("OEBPS/toc.ncx")
|
||||
if err != nil {
|
||||
return book, errors.Wrap(err, "can't create toc.ncx")
|
||||
}
|
||||
_, err = tocNcxWriter.Write([]byte(`<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE ncx PUBLIC "-//NISO//DTD ncx 2005-1//EN"
|
||||
"http://www.daisy.org/z3986/2005/ncx-2005-1.dtd">
|
||||
<ncx xmlns="http://www.daisy.org/z3986/2005/ncx/" version="2005-1">
|
||||
<head>
|
||||
<meta name="dtb:uid" content="urn:uuid:12345678-1234-5678-1234-567812345678"/>
|
||||
<meta name="dtb:depth" content="1"/>
|
||||
<meta name="dtb:totalPageCount" content="0"/>
|
||||
<meta name="dtb:maxPageNumber" content="0"/>
|
||||
</head>
|
||||
<docTitle>
|
||||
<text>` + book.Title + `</text>
|
||||
</docTitle>
|
||||
<navMap>
|
||||
<navPoint id="navPoint-1" playOrder="1">
|
||||
<navLabel>
|
||||
<text >` + book.Title + `</text>
|
||||
</navLabel>
|
||||
<content src="content.html"/>
|
||||
</navPoint>
|
||||
</navMap>
|
||||
</ncx>`))
|
||||
if err != nil {
|
||||
return book, errors.Wrap(err, "can't write into toc.ncx file")
|
||||
}
|
||||
|
||||
// get list of images tag in html
|
||||
imageList, _ := GetImages(book.HTML)
|
||||
imgRegex := regexp.MustCompile(`<img.*?src="([^"]*)".*?>`)
|
||||
|
||||
// Create a set to store unique image URLs
|
||||
imageSet := make(map[string]bool)
|
||||
|
||||
// Download image in html file and generate new html
|
||||
html = book.HTML
|
||||
for _, match := range imgRegex.FindAllStringSubmatch(book.HTML, -1) {
|
||||
imageURL := match[1]
|
||||
if _, ok := imageList[imageURL]; ok && !imageSet[imageURL] {
|
||||
// Add the image URL to the set
|
||||
imageSet[imageURL] = true
|
||||
|
||||
// Download the image
|
||||
resp, err := http.Get(imageURL)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Get the image data
|
||||
imageData, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return book, errors.Wrap(err, "can't get image from the internet")
|
||||
}
|
||||
|
||||
fileName := fp.Base(imageURL)
|
||||
filePath := "images/" + fileName
|
||||
imageWriter, err := epubWriter.Create(filePath)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// Write the image to the file
|
||||
_, err = imageWriter.Write(imageData)
|
||||
if err != nil {
|
||||
return book, errors.Wrap(err, "can't create image file")
|
||||
}
|
||||
// Replace the image tag with the new downloaded image
|
||||
html = strings.ReplaceAll(html, match[0], fmt.Sprintf(`<img src="../%s"/>`, filePath))
|
||||
}
|
||||
}
|
||||
// Create the content.html file
|
||||
contentHtmlWriter, err := epubWriter.Create("OEBPS/content.html")
|
||||
if err != nil {
|
||||
return book, errors.Wrap(err, "can't create content.xml")
|
||||
}
|
||||
_, err = contentHtmlWriter.Write([]byte("<?xml version='1.0' encoding='utf-8'?>\n<html xmlns=\"http://www.w3.org/1999/xhtml\">\n<head>\n\t<title>" + book.Title + "</title>\n\t<link href=\"../style.css\" rel=\"stylesheet\" type=\"text/css\"/>\n</head>\n<body>\n\t<h1 dir=\"auto\">" + book.Title + "</h1>" + "\n<content dir=\"auto\">\n" + html + "\n</content>" + "\n</body></html>"))
|
||||
if err != nil {
|
||||
return book, errors.Wrap(err, "can't write into content.html")
|
||||
}
|
||||
book.HasEbook = true
|
||||
return book, nil
|
||||
}
|
||||
|
||||
// function get html and return list of image url inside html file
|
||||
func GetImages(html string) (map[string]string, error) {
|
||||
// Regular expression to match image tags and their URLs
|
||||
imageTagRegex := regexp.MustCompile(`<img.*?src="(.*?)".*?>`)
|
||||
|
||||
// Find all matches in the HTML string
|
||||
imageTagMatches := imageTagRegex.FindAllStringSubmatch(html, -1)
|
||||
// Create a dictionary to store the image URLs
|
||||
images := make(map[string]string)
|
||||
|
||||
// Check if there are any matches
|
||||
if len(imageTagMatches) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Loop through all the matches and add them to the dictionary
|
||||
for _, match := range imageTagMatches {
|
||||
imageURL := match[1]
|
||||
images[imageURL] = match[0]
|
||||
}
|
||||
|
||||
return images, nil
|
||||
}
|
||||
288
internal/core/ebook_test.go
Normal file
288
internal/core/ebook_test.go
Normal file
|
|
@ -0,0 +1,288 @@
|
|||
package core_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
fp "path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/go-shiori/shiori/internal/core"
|
||||
"github.com/go-shiori/shiori/internal/model"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestGenerateEbook_ValidBookmarkID_ReturnsBookmarkWithHasEbookTrue(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
mockRequest := core.ProcessRequest{
|
||||
Bookmark: model.Bookmark{
|
||||
ID: 1,
|
||||
Title: "Example Bookmark",
|
||||
HTML: "<html><body>Example HTML</body></html>",
|
||||
HasEbook: false,
|
||||
},
|
||||
DataDir: tempDir,
|
||||
ContentType: "text/html",
|
||||
}
|
||||
|
||||
bookmark, err := core.GenerateEbook(mockRequest)
|
||||
|
||||
assert.True(t, bookmark.HasEbook)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestGenerateEbook_InvalidBookmarkID_ReturnsError(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
defer os.RemoveAll(tempDir)
|
||||
mockRequest := core.ProcessRequest{
|
||||
Bookmark: model.Bookmark{
|
||||
ID: 0,
|
||||
HasEbook: false,
|
||||
},
|
||||
DataDir: tempDir,
|
||||
ContentType: "text/html",
|
||||
}
|
||||
|
||||
bookmark, err := core.GenerateEbook(mockRequest)
|
||||
|
||||
assert.Equal(t, model.Bookmark{
|
||||
ID: 0,
|
||||
HasEbook: false,
|
||||
}, bookmark)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestGenerateEbook_ValidBookmarkID_EbookExist_EbookExist_ReturnWithHasEbookTrue(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
mockRequest := core.ProcessRequest{
|
||||
Bookmark: model.Bookmark{
|
||||
ID: 1,
|
||||
HasEbook: false,
|
||||
},
|
||||
DataDir: tempDir,
|
||||
ContentType: "text/html",
|
||||
}
|
||||
// Create the ebook directory
|
||||
ebookDir := fp.Join(mockRequest.DataDir, "ebook")
|
||||
err := os.MkdirAll(ebookDir, os.ModePerm)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Create the ebook file
|
||||
ebookfile := fp.Join(mockRequest.DataDir, "ebook", fmt.Sprintf("%d.epub", mockRequest.Bookmark.ID))
|
||||
file, err := os.Create(ebookfile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
bookmark, err := core.GenerateEbook(mockRequest)
|
||||
|
||||
assert.True(t, bookmark.HasEbook)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestGenerateEbook_ValidBookmarkID_EbookExist_ImagePathExist_ReturnWithHasEbookTrue(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
mockRequest := core.ProcessRequest{
|
||||
Bookmark: model.Bookmark{
|
||||
ID: 1,
|
||||
HasEbook: false,
|
||||
},
|
||||
DataDir: tempDir,
|
||||
ContentType: "text/html",
|
||||
}
|
||||
// Create the image directory
|
||||
imageDir := fp.Join(mockRequest.DataDir, "thumb")
|
||||
err := os.MkdirAll(imageDir, os.ModePerm)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Create the image file
|
||||
imagePath := fp.Join(mockRequest.DataDir, "thumb", fmt.Sprintf("%d", mockRequest.Bookmark.ID))
|
||||
file, err := os.Create(imagePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
bookmark, err := core.GenerateEbook(mockRequest)
|
||||
expectedimagePath := "/bookmark/1/thumb"
|
||||
if expectedimagePath != bookmark.ImageURL {
|
||||
t.Errorf("Expected imageURL %s, but got %s", bookmark.ImageURL, expectedimagePath)
|
||||
}
|
||||
assert.True(t, bookmark.HasEbook)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestGenerateEbook_ValidBookmarkID_EbookExist_ReturnWithHasArchiveTrue(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
mockRequest := core.ProcessRequest{
|
||||
Bookmark: model.Bookmark{
|
||||
ID: 1,
|
||||
HasEbook: false,
|
||||
},
|
||||
DataDir: tempDir,
|
||||
ContentType: "text/html",
|
||||
}
|
||||
// Create the archive directory
|
||||
archiveDir := fp.Join(mockRequest.DataDir, "archive")
|
||||
err := os.MkdirAll(archiveDir, os.ModePerm)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Create the archive file
|
||||
archivePath := fp.Join(mockRequest.DataDir, "archive", fmt.Sprintf("%d", mockRequest.Bookmark.ID))
|
||||
file, err := os.Create(archivePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
bookmark, err := core.GenerateEbook(mockRequest)
|
||||
assert.True(t, bookmark.HasArchive)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestGenerateEbook_ValidBookmarkID_RetuenError_PDF(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
mockRequest := core.ProcessRequest{
|
||||
Bookmark: model.Bookmark{
|
||||
ID: 1,
|
||||
HasEbook: false,
|
||||
},
|
||||
DataDir: tempDir,
|
||||
ContentType: "application/pdf",
|
||||
}
|
||||
|
||||
bookmark, err := core.GenerateEbook(mockRequest)
|
||||
|
||||
assert.False(t, bookmark.HasEbook)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "can't create ebook for pdf")
|
||||
}
|
||||
|
||||
func TestGenerateEbook_CreateEbookDirectoryNotWritable(t *testing.T) {
|
||||
// Create a temporary directory to use as the parent directory
|
||||
parentDir := t.TempDir()
|
||||
|
||||
// Create a child directory with read-only permissions
|
||||
ebookDir := fp.Join(parentDir, "ebook")
|
||||
err := os.Mkdir(ebookDir, 0444)
|
||||
if err != nil {
|
||||
t.Fatalf("could not create ebook directory: %s", err)
|
||||
}
|
||||
|
||||
mockRequest := core.ProcessRequest{
|
||||
Bookmark: model.Bookmark{
|
||||
ID: 1,
|
||||
HasEbook: false,
|
||||
},
|
||||
DataDir: ebookDir,
|
||||
ContentType: "text/html",
|
||||
}
|
||||
|
||||
// Call GenerateEbook to create the ebook directory
|
||||
bookmark, err := core.GenerateEbook(mockRequest)
|
||||
if err == nil {
|
||||
t.Fatal("GenerateEbook succeeded even though MkdirAll should have failed")
|
||||
}
|
||||
if !errors.Is(err, os.ErrPermission) {
|
||||
t.Fatalf("unexpected error: expected os.ErrPermission, got %v", err)
|
||||
}
|
||||
|
||||
// Check if the ebook directory still exists and has read-only permissions
|
||||
info, err := os.Stat(ebookDir)
|
||||
if err != nil {
|
||||
t.Fatalf("could not retrieve ebook directory info: %s", err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
t.Errorf("ebook directory is not a directory")
|
||||
}
|
||||
if info.Mode().Perm() != 0444 {
|
||||
t.Errorf("ebook directory has incorrect permissions: expected 0444, got %o", info.Mode().Perm())
|
||||
}
|
||||
assert.False(t, bookmark.HasEbook)
|
||||
}
|
||||
|
||||
// Add more unit tests for other scenarios that missing specialy
|
||||
// can't create ebook directory and can't write situatuin
|
||||
// writing inside zip file
|
||||
// html variable that not export and image download loop
|
||||
|
||||
func TestGetImages(t *testing.T) {
|
||||
// Test case 1: HTML with no image tags
|
||||
html1 := `<html><body><h1>Hello, World!</h1></body></html>`
|
||||
expected1 := make(map[string]string)
|
||||
result1, err1 := core.GetImages(html1)
|
||||
if err1 != nil {
|
||||
t.Errorf("Unexpected error: %v", err1)
|
||||
}
|
||||
if len(result1) != len(expected1) {
|
||||
t.Errorf("Expected %d images, but got %d", len(expected1), len(result1))
|
||||
}
|
||||
|
||||
// Test case 2: HTML with one image tag
|
||||
html2 := `<html><body><img src="image1.jpg"></body></html>`
|
||||
expected2 := map[string]string{"image1.jpg": "<img src=\"image1.jpg\">"}
|
||||
result2, err2 := core.GetImages(html2)
|
||||
if err2 != nil {
|
||||
t.Errorf("Unexpected error: %v", err2)
|
||||
}
|
||||
if len(result2) != len(expected2) {
|
||||
t.Errorf("Expected %d images, but got %d", len(expected2), len(result2))
|
||||
}
|
||||
for key, value := range expected2 {
|
||||
if result2[key] != value {
|
||||
t.Errorf("Expected image URL %s with tag %s, but got %s", key, value, result2[key])
|
||||
}
|
||||
}
|
||||
|
||||
// Test case 3: HTML with multiple image tags
|
||||
html3 := `<html><body><img src="image1.jpg"><img src="image2.jpg"></body></html>`
|
||||
expected3 := map[string]string{
|
||||
"image1.jpg": "<img src=\"image1.jpg\">",
|
||||
"image2.jpg": "<img src=\"image2.jpg\">",
|
||||
}
|
||||
result3, err3 := core.GetImages(html3)
|
||||
if err3 != nil {
|
||||
t.Errorf("Unexpected error: %v", err3)
|
||||
}
|
||||
if len(result3) != len(expected3) {
|
||||
t.Errorf("Expected %d images, but got %d", len(expected3), len(result3))
|
||||
}
|
||||
for key, value := range expected3 {
|
||||
if result3[key] != value {
|
||||
t.Errorf("Expected image URL %s with tag %s, but got %s", key, value, result3[key])
|
||||
}
|
||||
}
|
||||
// Test case 4: HTML with multiple image tags with duplicayr
|
||||
html4 := `<html><body><img src="image1.jpg"><img src="image2.jpg"><img src="image2.jpg"></body></html>`
|
||||
expected4 := map[string]string{
|
||||
"image1.jpg": "<img src=\"image1.jpg\">",
|
||||
"image2.jpg": "<img src=\"image2.jpg\">",
|
||||
}
|
||||
result4, err4 := core.GetImages(html4)
|
||||
if err4 != nil {
|
||||
t.Errorf("Unexpected error: %v", err4)
|
||||
}
|
||||
if len(result4) != len(expected4) {
|
||||
t.Errorf("Expected %d images, but got %d", len(expected4), len(result4))
|
||||
}
|
||||
for key, value := range expected4 {
|
||||
if result4[key] != value {
|
||||
t.Errorf("Expected image URL %s with tag %s, but got %s", key, value, result4[key])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -20,6 +20,7 @@ import (
|
|||
"github.com/go-shiori/go-readability"
|
||||
"github.com/go-shiori/shiori/internal/model"
|
||||
"github.com/go-shiori/warc"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
// Add support for png
|
||||
_ "image/png"
|
||||
|
|
@ -125,6 +126,22 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool,
|
|||
}
|
||||
}
|
||||
|
||||
// If needed, create ebook as well
|
||||
if book.CreateEbook {
|
||||
ebookPath := fp.Join(req.DataDir, "ebook", fmt.Sprintf("%d.epub", book.ID))
|
||||
os.Remove(ebookPath)
|
||||
|
||||
if strings.Contains(contentType, "application/pdf") {
|
||||
return book, false, errors.Wrap(err, "can't create ebook from pdf")
|
||||
} else {
|
||||
_, err = GenerateEbook(req)
|
||||
if err != nil {
|
||||
return book, true, errors.Wrap(err, "failed to create ebook")
|
||||
}
|
||||
book.HasEbook = true
|
||||
}
|
||||
}
|
||||
|
||||
// If needed, create offline archive as well
|
||||
if book.CreateArchive {
|
||||
archivePath := fp.Join(req.DataDir, "archive", fmt.Sprintf("%d", book.ID))
|
||||
|
|
|
|||
|
|
@ -22,8 +22,10 @@ type Bookmark struct {
|
|||
ImageURL string `db:"image_url" json:"imageURL"`
|
||||
HasContent bool `db:"has_content" json:"hasContent"`
|
||||
HasArchive bool `json:"hasArchive"`
|
||||
HasEbook bool `json:"hasEbook"`
|
||||
Tags []Tag `json:"tags"`
|
||||
CreateArchive bool `json:"createArchive"`
|
||||
CreateEbook bool `json:"createEbook"`
|
||||
}
|
||||
|
||||
// Account is person that allowed to access web interface.
|
||||
|
|
|
|||
|
|
@ -33,6 +33,9 @@
|
|||
$$if .Book.HasArchive$$
|
||||
<a href="bookmark/$$.Book.ID$$/archive">View Archive</a>
|
||||
$$end$$
|
||||
$$if .Book.HasEbook$$
|
||||
<a href="bookmark/$$.Book.ID$$/ebook" download="$$.Book.Title$$.epub">Download Ebook</a>
|
||||
$$end$$
|
||||
</div>
|
||||
</div>
|
||||
<div id="content" dir="auto" v-pre>
|
||||
|
|
|
|||
|
|
@ -32,6 +32,9 @@ var template = `
|
|||
<a title="Update archive" @click="updateBookmark">
|
||||
<i class="fas fa-fw fa-cloud-download-alt"></i>
|
||||
</a>
|
||||
<a v-if="hasEbook" title="Download book" @click="downloadebook">
|
||||
<i class="fas fa-fw fa-book"></i>
|
||||
</a>
|
||||
</template>
|
||||
</div>
|
||||
</div>`;
|
||||
|
|
@ -47,6 +50,7 @@ export default {
|
|||
imageURL: String,
|
||||
hasContent: Boolean,
|
||||
hasArchive: Boolean,
|
||||
hasEbook: Boolean,
|
||||
index: Number,
|
||||
showId: Boolean,
|
||||
editMode: Boolean,
|
||||
|
|
@ -72,6 +76,13 @@ export default {
|
|||
return this.url;
|
||||
}
|
||||
},
|
||||
ebookURL() {
|
||||
if (this.hasEbook) {
|
||||
return new URL(`bookmark/${this.id}/ebook`, document.baseURI);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
},
|
||||
hostnameURL() {
|
||||
var url = new URL(this.url);
|
||||
return url.hostname.replace(/^www\./, "");
|
||||
|
|
@ -112,6 +123,14 @@ export default {
|
|||
},
|
||||
updateBookmark() {
|
||||
this.$emit("update", this.eventItem);
|
||||
}
|
||||
},
|
||||
downloadebook() {
|
||||
const id = this.id;
|
||||
const ebook_url = new URL(`bookmark/${id}/ebook`, document.baseURI);
|
||||
const downloadLink = document.createElement("a");
|
||||
downloadLink.href = ebook_url.toString();
|
||||
downloadLink.download = `${this.title}.epub`;
|
||||
downloadLink.click();
|
||||
},
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ export default {
|
|||
|
||||
keepMetadata: false,
|
||||
useArchive: false,
|
||||
createEbook: false,
|
||||
makePublic: false,
|
||||
};
|
||||
}
|
||||
|
|
@ -111,4 +112,4 @@ export default {
|
|||
});
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,6 +26,9 @@ var template = `
|
|||
<a title="Update archives" @click="showDialogUpdateCache(selection)">
|
||||
<i class="fas fa-fw fa-cloud-download-alt"></i>
|
||||
</a>
|
||||
<a title="Download ebooks" @click="ebookGenerate(selection)">
|
||||
<i class="fas fa-fw fa-book"></i>
|
||||
</a>
|
||||
<a title="Cancel" @click="toggleEditMode">
|
||||
<i class="fas fa-fw fa-times"></i>
|
||||
</a>
|
||||
|
|
@ -47,6 +50,7 @@ var template = `
|
|||
:imageURL="book.imageURL"
|
||||
:hasContent="book.hasContent"
|
||||
:hasArchive="book.hasArchive"
|
||||
:hasEbook="book.hasEbook"
|
||||
:tags="book.tags"
|
||||
:index="index"
|
||||
:key="book.id"
|
||||
|
|
@ -61,6 +65,7 @@ var template = `
|
|||
@tag-clicked="bookmarkTagClicked"
|
||||
@edit="showDialogEdit"
|
||||
@delete="showDialogDelete"
|
||||
@generate-ebook="ebookGenerate"
|
||||
@update="showDialogUpdateCache">
|
||||
</bookmark-item>
|
||||
<pagination-box v-if="maxPage > 1"
|
||||
|
|
@ -583,6 +588,61 @@ export default {
|
|||
}
|
||||
});
|
||||
},
|
||||
ebookGenerate(items) {
|
||||
// Check and filter items
|
||||
if (typeof items !== "object") return;
|
||||
if (!Array.isArray(items)) items = [items];
|
||||
|
||||
items = items.filter(item => {
|
||||
var id = (typeof item.id === "number") ? item.id : 0,
|
||||
index = (typeof item.index === "number") ? item.index : -1;
|
||||
|
||||
return id > 0 && index > -1;
|
||||
});
|
||||
|
||||
if (items.length === 0) return;
|
||||
|
||||
// define variable and send request
|
||||
var ids = items.map(item => item.id);
|
||||
var data = {
|
||||
ids: ids,
|
||||
};
|
||||
this.loading = true;
|
||||
fetch(new URL("api/ebook", document.baseURI), {
|
||||
method: "put",
|
||||
body: JSON.stringify(data),
|
||||
headers: { "Content-Type": "application/json" },
|
||||
}).then(response => {
|
||||
if (!response.ok) throw response;
|
||||
return response.json();
|
||||
}).then(json => {
|
||||
this.selection = [];
|
||||
this.editMode = false;
|
||||
json.forEach(book => {
|
||||
// download ebooks
|
||||
const id = book.id;
|
||||
if (book.hasEbook){
|
||||
const ebook_url = new URL(`bookmark/${id}/ebook`, document.baseURI);
|
||||
const downloadLink = document.createElement("a");
|
||||
downloadLink.href = ebook_url.toString();
|
||||
downloadLink.download = `${book.title}.epub`;
|
||||
downloadLink.click();
|
||||
}
|
||||
|
||||
var item = items.find(el => el.id === book.id);
|
||||
this.bookmarks.splice(item.index, 1, book);
|
||||
});
|
||||
}).catch(err => {
|
||||
this.selection = [];
|
||||
this.editMode = false;
|
||||
this.getErrorMessage(err).then(msg => {
|
||||
this.showErrorDialog(msg);
|
||||
})
|
||||
})
|
||||
.finally(() => {
|
||||
this.loading = false;
|
||||
});
|
||||
},
|
||||
showDialogUpdateCache(items) {
|
||||
// Check and filter items
|
||||
if (typeof items !== "object") return;
|
||||
|
|
@ -613,7 +673,12 @@ export default {
|
|||
label: "Update archive as well",
|
||||
type: "check",
|
||||
value: this.appOptions.useArchive,
|
||||
}],
|
||||
}, {
|
||||
name: "createEbook",
|
||||
label: "Update Ebook as well",
|
||||
type: "check",
|
||||
value: this.appOptions.createEbook,
|
||||
}],
|
||||
mainText: "Yes",
|
||||
secondText: "No",
|
||||
mainClick: (data) => {
|
||||
|
|
@ -621,6 +686,7 @@ export default {
|
|||
ids: ids,
|
||||
createArchive: data.createArchive,
|
||||
keepMetadata: data.keepMetadata,
|
||||
createEbook: data.createEbook,
|
||||
};
|
||||
|
||||
this.dialog.loading = true;
|
||||
|
|
@ -834,4 +900,4 @@ export default {
|
|||
|
||||
this.loadData(false, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -203,6 +203,7 @@ func (h *handler) apiGetBookmarks(w http.ResponseWriter, r *http.Request, ps htt
|
|||
strID := strconv.Itoa(bookmarks[i].ID)
|
||||
imgPath := fp.Join(h.DataDir, "thumb", strID)
|
||||
archivePath := fp.Join(h.DataDir, "archive", strID)
|
||||
ebookPath := fp.Join(h.DataDir, "ebook", strID+".epub")
|
||||
|
||||
if fileExists(imgPath) {
|
||||
bookmarks[i].ImageURL = path.Join(h.RootPath, "bookmark", strID, "thumb")
|
||||
|
|
@ -211,6 +212,9 @@ func (h *handler) apiGetBookmarks(w http.ResponseWriter, r *http.Request, ps htt
|
|||
if fileExists(archivePath) {
|
||||
bookmarks[i].HasArchive = true
|
||||
}
|
||||
if fileExists(ebookPath) {
|
||||
bookmarks[i].HasEbook = true
|
||||
}
|
||||
}
|
||||
|
||||
// Return JSON response
|
||||
|
|
@ -373,9 +377,11 @@ func (h *handler) apiDeleteBookmark(w http.ResponseWriter, r *http.Request, ps h
|
|||
strID := strconv.Itoa(id)
|
||||
imgPath := fp.Join(h.DataDir, "thumb", strID)
|
||||
archivePath := fp.Join(h.DataDir, "archive", strID)
|
||||
ebookPath := fp.Join(h.DataDir, "ebook", strID+".epub")
|
||||
|
||||
os.Remove(imgPath)
|
||||
os.Remove(archivePath)
|
||||
os.Remove(ebookPath)
|
||||
}
|
||||
|
||||
fmt.Fprint(w, 1)
|
||||
|
|
@ -458,6 +464,104 @@ func (h *handler) apiUpdateBookmark(w http.ResponseWriter, r *http.Request, ps h
|
|||
checkError(err)
|
||||
}
|
||||
|
||||
// apiDownloadEbook is handler for PUT /api/ebook
|
||||
func (h *handler) apiDownloadEbook(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
|
||||
ctx := r.Context()
|
||||
|
||||
// Make sure session still valid
|
||||
err := h.validateSession(r)
|
||||
checkError(err)
|
||||
|
||||
// Decode request
|
||||
request := struct {
|
||||
IDs []int `json:"ids"`
|
||||
}{}
|
||||
err = json.NewDecoder(r.Body).Decode(&request)
|
||||
checkError(err)
|
||||
|
||||
// Get existing bookmark from database
|
||||
filter := database.GetBookmarksOptions{
|
||||
IDs: request.IDs,
|
||||
WithContent: true,
|
||||
}
|
||||
|
||||
bookmarks, err := h.DB.GetBookmarks(ctx, filter)
|
||||
checkError(err)
|
||||
if len(bookmarks) == 0 {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// Fetch data from internet
|
||||
mx := sync.RWMutex{}
|
||||
wg := sync.WaitGroup{}
|
||||
chDone := make(chan struct{})
|
||||
chProblem := make(chan int, 10)
|
||||
semaphore := make(chan struct{}, 10)
|
||||
|
||||
for i, book := range bookmarks {
|
||||
wg.Add(1)
|
||||
|
||||
go func(i int, book model.Bookmark) {
|
||||
// Make sure to finish the WG
|
||||
defer wg.Done()
|
||||
|
||||
// Register goroutine to semaphore
|
||||
semaphore <- struct{}{}
|
||||
defer func() {
|
||||
<-semaphore
|
||||
}()
|
||||
|
||||
// Download data from internet
|
||||
content, contentType, err := core.DownloadBookmark(book.URL)
|
||||
if err != nil {
|
||||
chProblem <- book.ID
|
||||
return
|
||||
}
|
||||
|
||||
request := core.ProcessRequest{
|
||||
DataDir: h.DataDir,
|
||||
Bookmark: book,
|
||||
Content: content,
|
||||
ContentType: contentType,
|
||||
}
|
||||
|
||||
book, err = core.GenerateEbook(request)
|
||||
content.Close()
|
||||
|
||||
if err != nil {
|
||||
chProblem <- book.ID
|
||||
return
|
||||
}
|
||||
|
||||
// Update list of bookmarks
|
||||
mx.Lock()
|
||||
bookmarks[i] = book
|
||||
mx.Unlock()
|
||||
}(i, book)
|
||||
}
|
||||
// Receive all problematic bookmarks
|
||||
idWithProblems := []int{}
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-chDone:
|
||||
return
|
||||
case id := <-chProblem:
|
||||
idWithProblems = append(idWithProblems, id)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Wait until all download finished
|
||||
wg.Wait()
|
||||
close(chDone)
|
||||
|
||||
w.Header().Set("Content-Type", "application1/json")
|
||||
err = json.NewEncoder(w).Encode(&bookmarks)
|
||||
checkError(err)
|
||||
}
|
||||
|
||||
// apiUpdateCache is handler for PUT /api/cache
|
||||
func (h *handler) apiUpdateCache(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
|
||||
ctx := r.Context()
|
||||
|
|
@ -471,6 +575,7 @@ func (h *handler) apiUpdateCache(w http.ResponseWriter, r *http.Request, ps http
|
|||
IDs []int `json:"ids"`
|
||||
KeepMetadata bool `json:"keepMetadata"`
|
||||
CreateArchive bool `json:"createArchive"`
|
||||
CreateEbook bool `json:"createEbook"`
|
||||
}{}
|
||||
|
||||
err = json.NewDecoder(r.Body).Decode(&request)
|
||||
|
|
@ -506,8 +611,9 @@ func (h *handler) apiUpdateCache(w http.ResponseWriter, r *http.Request, ps http
|
|||
for i, book := range bookmarks {
|
||||
wg.Add(1)
|
||||
|
||||
// Mark whether book will be archived
|
||||
// Mark whether book will be archived or ebook generate request
|
||||
book.CreateArchive = request.CreateArchive
|
||||
book.CreateEbook = request.CreateEbook
|
||||
|
||||
go func(i int, book model.Bookmark, keepMetadata bool) {
|
||||
// Make sure to finish the WG
|
||||
|
|
|
|||
|
|
@ -119,6 +119,10 @@ func (h *handler) serveBookmarkContent(w http.ResponseWriter, r *http.Request, p
|
|||
}
|
||||
|
||||
// Check if it has archive.
|
||||
ebookPath := fp.Join(h.DataDir, "ebook", strID+".epub")
|
||||
if fileExists(ebookPath) {
|
||||
bookmark.HasEbook = true
|
||||
}
|
||||
archivePath := fp.Join(h.DataDir, "archive", strID)
|
||||
if fileExists(archivePath) {
|
||||
bookmark.HasArchive = true
|
||||
|
|
@ -356,3 +360,67 @@ func (h *handler) serveBookmarkArchive(w http.ResponseWriter, r *http.Request, p
|
|||
log.Printf("error writting response: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
// serveEbook is handler for GET /bookmark/:id/ebook
|
||||
func (h *handler) serveBookmarkEbook(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
|
||||
ctx := r.Context()
|
||||
|
||||
// Get bookmark ID from URL
|
||||
strID := ps.ByName("id")
|
||||
id, err := strconv.Atoi(strID)
|
||||
checkError(err)
|
||||
// Get bookmark in database
|
||||
bookmark, exist, err := h.DB.GetBookmark(ctx, id, "")
|
||||
checkError(err)
|
||||
|
||||
if !exist {
|
||||
http.Error(w, "bookmark not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
// If it's not public, make sure session still valid
|
||||
if bookmark.Public != 1 {
|
||||
err = h.validateSession(r)
|
||||
if err != nil {
|
||||
newPath := path.Join(h.RootPath, "/login")
|
||||
redirectURL := createRedirectURL(newPath, r.URL.String())
|
||||
redirectPage(w, r, redirectURL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Check if it has ebook.
|
||||
ebookPath := fp.Join(h.DataDir, "ebook", strID+".epub")
|
||||
if !fileExists(ebookPath) {
|
||||
http.Error(w, "ebook not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
epub, err := os.Open(ebookPath)
|
||||
if err != nil {
|
||||
http.Error(w, "Internal server error", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
defer epub.Close()
|
||||
// Set content type
|
||||
w.Header().Set("Content-Type", "application/epub+zip")
|
||||
// Set cache value
|
||||
info, err := epub.Stat()
|
||||
if err != nil {
|
||||
http.Error(w, "Internal server error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
etag := fmt.Sprintf(`W/"%x-%x"`, info.ModTime().Unix(), info.Size())
|
||||
w.Header().Set("ETag", etag)
|
||||
w.Header().Set("Cache-Control", "max-age=86400")
|
||||
|
||||
// Serve epub file
|
||||
if _, err := epub.Seek(0, 0); err != nil {
|
||||
log.Printf("error during epub seek: %s", err)
|
||||
}
|
||||
_, err = io.Copy(w, epub)
|
||||
if err != nil {
|
||||
http.Error(w, "Internal server error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -183,6 +183,7 @@ func ServeApp(cfg Config) error {
|
|||
router.GET(jp("/login"), withLogging(hdl.serveLoginPage))
|
||||
router.GET(jp("/bookmark/:id/thumb"), withLogging(hdl.serveThumbnailImage))
|
||||
router.GET(jp("/bookmark/:id/content"), withLogging(hdl.serveBookmarkContent))
|
||||
router.GET(jp("/bookmark/:id/ebook"), withLogging(hdl.serveBookmarkEbook))
|
||||
router.GET(jp("/bookmark/:id/archive/*filepath"), withLogging(hdl.serveBookmarkArchive))
|
||||
|
||||
router.POST(jp("/api/login"), withLogging(hdl.apiLogin))
|
||||
|
|
@ -194,6 +195,7 @@ func ServeApp(cfg Config) error {
|
|||
router.DELETE(jp("/api/bookmarks"), withLogging(hdl.apiDeleteBookmark))
|
||||
router.PUT(jp("/api/bookmarks"), withLogging(hdl.apiUpdateBookmark))
|
||||
router.PUT(jp("/api/cache"), withLogging(hdl.apiUpdateCache))
|
||||
router.PUT(jp("/api/ebook"), withLogging(hdl.apiDownloadEbook))
|
||||
router.PUT(jp("/api/bookmarks/tags"), withLogging(hdl.apiUpdateBookmarkTags))
|
||||
router.POST(jp("/api/bookmarks/ext"), withLogging(hdl.apiInsertViaExtension))
|
||||
router.DELETE(jp("/api/bookmarks/ext"), withLogging(hdl.apiDeleteViaExtension))
|
||||
|
|
|
|||
Loading…
Reference in a new issue