From 249f4b89c878e0badbff8470d36ec90434e4c9d9 Mon Sep 17 00:00:00 2001 From: Monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sun, 9 Jul 2023 09:59:32 +0330 Subject: [PATCH] Initial Ebook Support (#623) * generate ebook * serve ebook file * Update ebook.go not download same image twice anymore * download ebook request api/ui part * fix typo * add stylesheet * update hasEbook status * download link update after ebook generate update bookmark command in ui * download ebook with bookmark title * Apply suggestions from code review for better error handling Co-authored-by: Felipe Martin <812088+fmartingr@users.noreply.github.com> * Update internal/view/js/page/home.js fix typo Co-authored-by: Felipe Martin <812088+fmartingr@users.noreply.github.com> * import error lib and retuen missing error * move ebook download action to update cache * replace io/ioutil with io * add missing error handling * update Archive now always update ebook * replace panic error with 404 * remove ebook with delete action * add download ebook link to content page * remove tags that not work correctly right now * if file is pdf not generate ebook * update style.css * Revert "update style.css" This reverts commit 519e10d6cebe24206ce72d6f843c37a8d4ac1915. * remove download limit for api * fix missing fmt.Errorf and change to errors.Wrap * fix double panic * return 404 if bookmark not exist * change function name to GenerateEbook * not isFatalErr anymore * add unit test * remove uneeded field for unit test --------- Co-authored-by: Felipe Martin <812088+fmartingr@users.noreply.github.com> --- internal/core/ebook.go | 252 ++++++++++++++++++++++ internal/core/ebook_test.go | 288 +++++++++++++++++++++++++ internal/core/processing.go | 17 ++ internal/model/model.go | 2 + internal/view/content.html | 3 + internal/view/js/component/bookmark.js | 21 +- internal/view/js/page/base.js | 3 +- internal/view/js/page/home.js | 70 +++++- internal/webserver/handler-api.go | 108 +++++++++- internal/webserver/handler-ui.go | 68 ++++++ internal/webserver/server.go | 2 + 11 files changed, 829 insertions(+), 5 deletions(-) create mode 100644 internal/core/ebook.go create mode 100644 internal/core/ebook_test.go diff --git a/internal/core/ebook.go b/internal/core/ebook.go new file mode 100644 index 0000000..73767b8 --- /dev/null +++ b/internal/core/ebook.go @@ -0,0 +1,252 @@ +package core + +import ( + "archive/zip" + "fmt" + "io" + "log" + "net/http" + "os" + fp "path/filepath" + "regexp" + "strconv" + "strings" + + "github.com/go-shiori/shiori/internal/model" + "github.com/pkg/errors" +) + +func GenerateEbook(req ProcessRequest) (book model.Bookmark, err error) { + // variable for store generated html code + var html string + + book = req.Bookmark + + // Make sure bookmark ID is defined + if book.ID == 0 { + return book, errors.New("bookmark ID is not valid") + } + + // cheak archive and thumb + strID := strconv.Itoa(book.ID) + + imagePath := fp.Join(req.DataDir, "thumb", fmt.Sprintf("%d", book.ID)) + archivePath := fp.Join(req.DataDir, "archive", fmt.Sprintf("%d", book.ID)) + + if _, err := os.Stat(imagePath); err == nil { + book.ImageURL = fp.Join("/", "bookmark", strID, "thumb") + } + + if _, err := os.Stat(archivePath); err == nil { + book.HasArchive = true + } + ebookfile := fp.Join(req.DataDir, "ebook", fmt.Sprintf("%d.epub", book.ID)) + // if epub exist finish prosess else continue + if _, err := os.Stat(ebookfile); err == nil { + book.HasEbook = true + return book, nil + } + contentType := req.ContentType + if strings.Contains(contentType, "application/pdf") { + return book, errors.New("can't create ebook for pdf") + } + + ebookDir := fp.Join(req.DataDir, "ebook") + // check if directory not exsist create that + if _, err := os.Stat(ebookDir); os.IsNotExist(err) { + err := os.MkdirAll(ebookDir, model.DataDirPerm) + if err != nil { + return book, errors.Wrap(err, "can't create ebook directory") + } + } + // create epub file + epubFile, err := os.Create(ebookfile) + if err != nil { + return book, errors.Wrap(err, "can't create ebook") + } + defer epubFile.Close() + + // Create zip archive + epubWriter := zip.NewWriter(epubFile) + defer epubWriter.Close() + + // Create the mimetype file + mimetypeWriter, err := epubWriter.Create("mimetype") + if err != nil { + return book, errors.Wrap(err, "can't create mimetype") + } + _, err = mimetypeWriter.Write([]byte("application/epub+zip")) + if err != nil { + return book, errors.Wrap(err, "can't write into mimetype file") + } + + // Create the container.xml file + containerWriter, err := epubWriter.Create("META-INF/container.xml") + if err != nil { + return book, errors.Wrap(err, "can't create container.xml") + } + + _, err = containerWriter.Write([]byte(` + + + + +`)) + if err != nil { + return book, errors.Wrap(err, "can't write into container.xml file") + } + + contentOpfWriter, err := epubWriter.Create("OEBPS/content.opf") + if err != nil { + return book, errors.Wrap(err, "can't create content.opf") + } + _, err = contentOpfWriter.Write([]byte(` + + + ` + book.Title + ` + + + + + + + + + +`)) + if err != nil { + return book, errors.Wrap(err, "can't write into container.opf file") + } + + // Create the style.css file + styleWriter, err := epubWriter.Create("style.css") + if err != nil { + return book, errors.Wrap(err, "can't create content.xml") + } + _, err = styleWriter.Write([]byte(`content { + display: block; + font-size: 1em; + line-height: 1.2; + padding-left: 0; + padding-right: 0; + text-align: justify; + margin: 0 5pt +} +img { + margin: auto; + display: block; +}`)) + if err != nil { + return book, errors.Wrap(err, "can't write into style.css file") + } + // Create the toc.ncx file + tocNcxWriter, err := epubWriter.Create("OEBPS/toc.ncx") + if err != nil { + return book, errors.Wrap(err, "can't create toc.ncx") + } + _, err = tocNcxWriter.Write([]byte(` + + + + + + + + + + ` + book.Title + ` + + + + + ` + book.Title + ` + + + + +`)) + if err != nil { + return book, errors.Wrap(err, "can't write into toc.ncx file") + } + + // get list of images tag in html + imageList, _ := GetImages(book.HTML) + imgRegex := regexp.MustCompile(``) + + // Create a set to store unique image URLs + imageSet := make(map[string]bool) + + // Download image in html file and generate new html + html = book.HTML + for _, match := range imgRegex.FindAllStringSubmatch(book.HTML, -1) { + imageURL := match[1] + if _, ok := imageList[imageURL]; ok && !imageSet[imageURL] { + // Add the image URL to the set + imageSet[imageURL] = true + + // Download the image + resp, err := http.Get(imageURL) + if err != nil { + log.Fatal(err) + } + defer resp.Body.Close() + + // Get the image data + imageData, err := io.ReadAll(resp.Body) + if err != nil { + return book, errors.Wrap(err, "can't get image from the internet") + } + + fileName := fp.Base(imageURL) + filePath := "images/" + fileName + imageWriter, err := epubWriter.Create(filePath) + if err != nil { + log.Fatal(err) + } + + // Write the image to the file + _, err = imageWriter.Write(imageData) + if err != nil { + return book, errors.Wrap(err, "can't create image file") + } + // Replace the image tag with the new downloaded image + html = strings.ReplaceAll(html, match[0], fmt.Sprintf(``, filePath)) + } + } + // Create the content.html file + contentHtmlWriter, err := epubWriter.Create("OEBPS/content.html") + if err != nil { + return book, errors.Wrap(err, "can't create content.xml") + } + _, err = contentHtmlWriter.Write([]byte("\n\n\n\t" + book.Title + "\n\t\n\n\n\t

" + book.Title + "

" + "\n\n" + html + "\n" + "\n")) + if err != nil { + return book, errors.Wrap(err, "can't write into content.html") + } + book.HasEbook = true + return book, nil +} + +// function get html and return list of image url inside html file +func GetImages(html string) (map[string]string, error) { + // Regular expression to match image tags and their URLs + imageTagRegex := regexp.MustCompile(``) + + // Find all matches in the HTML string + imageTagMatches := imageTagRegex.FindAllStringSubmatch(html, -1) + // Create a dictionary to store the image URLs + images := make(map[string]string) + + // Check if there are any matches + if len(imageTagMatches) == 0 { + return nil, nil + } + + // Loop through all the matches and add them to the dictionary + for _, match := range imageTagMatches { + imageURL := match[1] + images[imageURL] = match[0] + } + + return images, nil +} diff --git a/internal/core/ebook_test.go b/internal/core/ebook_test.go new file mode 100644 index 0000000..77c7bf1 --- /dev/null +++ b/internal/core/ebook_test.go @@ -0,0 +1,288 @@ +package core_test + +import ( + "errors" + "fmt" + "os" + fp "path/filepath" + "testing" + + "github.com/go-shiori/shiori/internal/core" + "github.com/go-shiori/shiori/internal/model" + "github.com/stretchr/testify/assert" +) + +func TestGenerateEbook_ValidBookmarkID_ReturnsBookmarkWithHasEbookTrue(t *testing.T) { + tempDir := t.TempDir() + + defer os.RemoveAll(tempDir) + + mockRequest := core.ProcessRequest{ + Bookmark: model.Bookmark{ + ID: 1, + Title: "Example Bookmark", + HTML: "Example HTML", + HasEbook: false, + }, + DataDir: tempDir, + ContentType: "text/html", + } + + bookmark, err := core.GenerateEbook(mockRequest) + + assert.True(t, bookmark.HasEbook) + assert.NoError(t, err) +} + +func TestGenerateEbook_InvalidBookmarkID_ReturnsError(t *testing.T) { + tempDir := t.TempDir() + defer os.RemoveAll(tempDir) + mockRequest := core.ProcessRequest{ + Bookmark: model.Bookmark{ + ID: 0, + HasEbook: false, + }, + DataDir: tempDir, + ContentType: "text/html", + } + + bookmark, err := core.GenerateEbook(mockRequest) + + assert.Equal(t, model.Bookmark{ + ID: 0, + HasEbook: false, + }, bookmark) + assert.Error(t, err) +} + +func TestGenerateEbook_ValidBookmarkID_EbookExist_EbookExist_ReturnWithHasEbookTrue(t *testing.T) { + tempDir := t.TempDir() + defer os.RemoveAll(tempDir) + + mockRequest := core.ProcessRequest{ + Bookmark: model.Bookmark{ + ID: 1, + HasEbook: false, + }, + DataDir: tempDir, + ContentType: "text/html", + } + // Create the ebook directory + ebookDir := fp.Join(mockRequest.DataDir, "ebook") + err := os.MkdirAll(ebookDir, os.ModePerm) + if err != nil { + t.Fatal(err) + } + // Create the ebook file + ebookfile := fp.Join(mockRequest.DataDir, "ebook", fmt.Sprintf("%d.epub", mockRequest.Bookmark.ID)) + file, err := os.Create(ebookfile) + if err != nil { + t.Fatal(err) + } + defer file.Close() + + bookmark, err := core.GenerateEbook(mockRequest) + + assert.True(t, bookmark.HasEbook) + assert.NoError(t, err) +} + +func TestGenerateEbook_ValidBookmarkID_EbookExist_ImagePathExist_ReturnWithHasEbookTrue(t *testing.T) { + tempDir := t.TempDir() + defer os.RemoveAll(tempDir) + + mockRequest := core.ProcessRequest{ + Bookmark: model.Bookmark{ + ID: 1, + HasEbook: false, + }, + DataDir: tempDir, + ContentType: "text/html", + } + // Create the image directory + imageDir := fp.Join(mockRequest.DataDir, "thumb") + err := os.MkdirAll(imageDir, os.ModePerm) + if err != nil { + t.Fatal(err) + } + // Create the image file + imagePath := fp.Join(mockRequest.DataDir, "thumb", fmt.Sprintf("%d", mockRequest.Bookmark.ID)) + file, err := os.Create(imagePath) + if err != nil { + t.Fatal(err) + } + defer file.Close() + + bookmark, err := core.GenerateEbook(mockRequest) + expectedimagePath := "/bookmark/1/thumb" + if expectedimagePath != bookmark.ImageURL { + t.Errorf("Expected imageURL %s, but got %s", bookmark.ImageURL, expectedimagePath) + } + assert.True(t, bookmark.HasEbook) + assert.NoError(t, err) +} + +func TestGenerateEbook_ValidBookmarkID_EbookExist_ReturnWithHasArchiveTrue(t *testing.T) { + tempDir := t.TempDir() + defer os.RemoveAll(tempDir) + + mockRequest := core.ProcessRequest{ + Bookmark: model.Bookmark{ + ID: 1, + HasEbook: false, + }, + DataDir: tempDir, + ContentType: "text/html", + } + // Create the archive directory + archiveDir := fp.Join(mockRequest.DataDir, "archive") + err := os.MkdirAll(archiveDir, os.ModePerm) + if err != nil { + t.Fatal(err) + } + // Create the archive file + archivePath := fp.Join(mockRequest.DataDir, "archive", fmt.Sprintf("%d", mockRequest.Bookmark.ID)) + file, err := os.Create(archivePath) + if err != nil { + t.Fatal(err) + } + defer file.Close() + + bookmark, err := core.GenerateEbook(mockRequest) + assert.True(t, bookmark.HasArchive) + assert.NoError(t, err) +} + +func TestGenerateEbook_ValidBookmarkID_RetuenError_PDF(t *testing.T) { + tempDir := t.TempDir() + defer os.RemoveAll(tempDir) + + mockRequest := core.ProcessRequest{ + Bookmark: model.Bookmark{ + ID: 1, + HasEbook: false, + }, + DataDir: tempDir, + ContentType: "application/pdf", + } + + bookmark, err := core.GenerateEbook(mockRequest) + + assert.False(t, bookmark.HasEbook) + assert.Error(t, err) + assert.Contains(t, err.Error(), "can't create ebook for pdf") +} + +func TestGenerateEbook_CreateEbookDirectoryNotWritable(t *testing.T) { + // Create a temporary directory to use as the parent directory + parentDir := t.TempDir() + + // Create a child directory with read-only permissions + ebookDir := fp.Join(parentDir, "ebook") + err := os.Mkdir(ebookDir, 0444) + if err != nil { + t.Fatalf("could not create ebook directory: %s", err) + } + + mockRequest := core.ProcessRequest{ + Bookmark: model.Bookmark{ + ID: 1, + HasEbook: false, + }, + DataDir: ebookDir, + ContentType: "text/html", + } + + // Call GenerateEbook to create the ebook directory + bookmark, err := core.GenerateEbook(mockRequest) + if err == nil { + t.Fatal("GenerateEbook succeeded even though MkdirAll should have failed") + } + if !errors.Is(err, os.ErrPermission) { + t.Fatalf("unexpected error: expected os.ErrPermission, got %v", err) + } + + // Check if the ebook directory still exists and has read-only permissions + info, err := os.Stat(ebookDir) + if err != nil { + t.Fatalf("could not retrieve ebook directory info: %s", err) + } + if !info.IsDir() { + t.Errorf("ebook directory is not a directory") + } + if info.Mode().Perm() != 0444 { + t.Errorf("ebook directory has incorrect permissions: expected 0444, got %o", info.Mode().Perm()) + } + assert.False(t, bookmark.HasEbook) +} + +// Add more unit tests for other scenarios that missing specialy +// can't create ebook directory and can't write situatuin +// writing inside zip file +// html variable that not export and image download loop + +func TestGetImages(t *testing.T) { + // Test case 1: HTML with no image tags + html1 := `

Hello, World!

` + expected1 := make(map[string]string) + result1, err1 := core.GetImages(html1) + if err1 != nil { + t.Errorf("Unexpected error: %v", err1) + } + if len(result1) != len(expected1) { + t.Errorf("Expected %d images, but got %d", len(expected1), len(result1)) + } + + // Test case 2: HTML with one image tag + html2 := `` + expected2 := map[string]string{"image1.jpg": ""} + result2, err2 := core.GetImages(html2) + if err2 != nil { + t.Errorf("Unexpected error: %v", err2) + } + if len(result2) != len(expected2) { + t.Errorf("Expected %d images, but got %d", len(expected2), len(result2)) + } + for key, value := range expected2 { + if result2[key] != value { + t.Errorf("Expected image URL %s with tag %s, but got %s", key, value, result2[key]) + } + } + + // Test case 3: HTML with multiple image tags + html3 := `` + expected3 := map[string]string{ + "image1.jpg": "", + "image2.jpg": "", + } + result3, err3 := core.GetImages(html3) + if err3 != nil { + t.Errorf("Unexpected error: %v", err3) + } + if len(result3) != len(expected3) { + t.Errorf("Expected %d images, but got %d", len(expected3), len(result3)) + } + for key, value := range expected3 { + if result3[key] != value { + t.Errorf("Expected image URL %s with tag %s, but got %s", key, value, result3[key]) + } + } + // Test case 4: HTML with multiple image tags with duplicayr + html4 := `` + expected4 := map[string]string{ + "image1.jpg": "", + "image2.jpg": "", + } + result4, err4 := core.GetImages(html4) + if err4 != nil { + t.Errorf("Unexpected error: %v", err4) + } + if len(result4) != len(expected4) { + t.Errorf("Expected %d images, but got %d", len(expected4), len(result4)) + } + for key, value := range expected4 { + if result4[key] != value { + t.Errorf("Expected image URL %s with tag %s, but got %s", key, value, result4[key]) + } + } +} diff --git a/internal/core/processing.go b/internal/core/processing.go index 74a9b4b..12a2c10 100644 --- a/internal/core/processing.go +++ b/internal/core/processing.go @@ -20,6 +20,7 @@ import ( "github.com/go-shiori/go-readability" "github.com/go-shiori/shiori/internal/model" "github.com/go-shiori/warc" + "github.com/pkg/errors" // Add support for png _ "image/png" @@ -125,6 +126,22 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool, } } + // If needed, create ebook as well + if book.CreateEbook { + ebookPath := fp.Join(req.DataDir, "ebook", fmt.Sprintf("%d.epub", book.ID)) + os.Remove(ebookPath) + + if strings.Contains(contentType, "application/pdf") { + return book, false, errors.Wrap(err, "can't create ebook from pdf") + } else { + _, err = GenerateEbook(req) + if err != nil { + return book, true, errors.Wrap(err, "failed to create ebook") + } + book.HasEbook = true + } + } + // If needed, create offline archive as well if book.CreateArchive { archivePath := fp.Join(req.DataDir, "archive", fmt.Sprintf("%d", book.ID)) diff --git a/internal/model/model.go b/internal/model/model.go index 96f214f..8b65d5d 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -22,8 +22,10 @@ type Bookmark struct { ImageURL string `db:"image_url" json:"imageURL"` HasContent bool `db:"has_content" json:"hasContent"` HasArchive bool `json:"hasArchive"` + HasEbook bool `json:"hasEbook"` Tags []Tag `json:"tags"` CreateArchive bool `json:"createArchive"` + CreateEbook bool `json:"createEbook"` } // Account is person that allowed to access web interface. diff --git a/internal/view/content.html b/internal/view/content.html index 6489ea6..9a9b89e 100644 --- a/internal/view/content.html +++ b/internal/view/content.html @@ -33,6 +33,9 @@ $$if .Book.HasArchive$$ View Archive $$end$$ + $$if .Book.HasEbook$$ + Download Ebook + $$end$$
diff --git a/internal/view/js/component/bookmark.js b/internal/view/js/component/bookmark.js index d2e3815..fd2f287 100644 --- a/internal/view/js/component/bookmark.js +++ b/internal/view/js/component/bookmark.js @@ -32,6 +32,9 @@ var template = ` + + +
`; @@ -47,6 +50,7 @@ export default { imageURL: String, hasContent: Boolean, hasArchive: Boolean, + hasEbook: Boolean, index: Number, showId: Boolean, editMode: Boolean, @@ -72,6 +76,13 @@ export default { return this.url; } }, + ebookURL() { + if (this.hasEbook) { + return new URL(`bookmark/${this.id}/ebook`, document.baseURI); + } else { + return null; + } + }, hostnameURL() { var url = new URL(this.url); return url.hostname.replace(/^www\./, ""); @@ -112,6 +123,14 @@ export default { }, updateBookmark() { this.$emit("update", this.eventItem); - } + }, + downloadebook() { + const id = this.id; + const ebook_url = new URL(`bookmark/${id}/ebook`, document.baseURI); + const downloadLink = document.createElement("a"); + downloadLink.href = ebook_url.toString(); + downloadLink.download = `${this.title}.epub`; + downloadLink.click(); + }, } } diff --git a/internal/view/js/page/base.js b/internal/view/js/page/base.js index 3d357a7..b50ebae 100644 --- a/internal/view/js/page/base.js +++ b/internal/view/js/page/base.js @@ -22,6 +22,7 @@ export default { keepMetadata: false, useArchive: false, + createEbook: false, makePublic: false, }; } @@ -111,4 +112,4 @@ export default { }); }, } -} \ No newline at end of file +} diff --git a/internal/view/js/page/home.js b/internal/view/js/page/home.js index cd52298..85bb80b 100644 --- a/internal/view/js/page/home.js +++ b/internal/view/js/page/home.js @@ -26,6 +26,9 @@ var template = ` + + + @@ -47,6 +50,7 @@ var template = ` :imageURL="book.imageURL" :hasContent="book.hasContent" :hasArchive="book.hasArchive" + :hasEbook="book.hasEbook" :tags="book.tags" :index="index" :key="book.id" @@ -61,6 +65,7 @@ var template = ` @tag-clicked="bookmarkTagClicked" @edit="showDialogEdit" @delete="showDialogDelete" + @generate-ebook="ebookGenerate" @update="showDialogUpdateCache"> { + var id = (typeof item.id === "number") ? item.id : 0, + index = (typeof item.index === "number") ? item.index : -1; + + return id > 0 && index > -1; + }); + + if (items.length === 0) return; + + // define variable and send request + var ids = items.map(item => item.id); + var data = { + ids: ids, + }; + this.loading = true; + fetch(new URL("api/ebook", document.baseURI), { + method: "put", + body: JSON.stringify(data), + headers: { "Content-Type": "application/json" }, + }).then(response => { + if (!response.ok) throw response; + return response.json(); + }).then(json => { + this.selection = []; + this.editMode = false; + json.forEach(book => { + // download ebooks + const id = book.id; + if (book.hasEbook){ + const ebook_url = new URL(`bookmark/${id}/ebook`, document.baseURI); + const downloadLink = document.createElement("a"); + downloadLink.href = ebook_url.toString(); + downloadLink.download = `${book.title}.epub`; + downloadLink.click(); + } + + var item = items.find(el => el.id === book.id); + this.bookmarks.splice(item.index, 1, book); + }); + }).catch(err => { + this.selection = []; + this.editMode = false; + this.getErrorMessage(err).then(msg => { + this.showErrorDialog(msg); + }) + }) + .finally(() => { + this.loading = false; + }); + }, showDialogUpdateCache(items) { // Check and filter items if (typeof items !== "object") return; @@ -613,7 +673,12 @@ export default { label: "Update archive as well", type: "check", value: this.appOptions.useArchive, - }], + }, { + name: "createEbook", + label: "Update Ebook as well", + type: "check", + value: this.appOptions.createEbook, + }], mainText: "Yes", secondText: "No", mainClick: (data) => { @@ -621,6 +686,7 @@ export default { ids: ids, createArchive: data.createArchive, keepMetadata: data.keepMetadata, + createEbook: data.createEbook, }; this.dialog.loading = true; @@ -834,4 +900,4 @@ export default { this.loadData(false, true); } -} \ No newline at end of file +} diff --git a/internal/webserver/handler-api.go b/internal/webserver/handler-api.go index bbecd52..1af30d7 100644 --- a/internal/webserver/handler-api.go +++ b/internal/webserver/handler-api.go @@ -203,6 +203,7 @@ func (h *handler) apiGetBookmarks(w http.ResponseWriter, r *http.Request, ps htt strID := strconv.Itoa(bookmarks[i].ID) imgPath := fp.Join(h.DataDir, "thumb", strID) archivePath := fp.Join(h.DataDir, "archive", strID) + ebookPath := fp.Join(h.DataDir, "ebook", strID+".epub") if fileExists(imgPath) { bookmarks[i].ImageURL = path.Join(h.RootPath, "bookmark", strID, "thumb") @@ -211,6 +212,9 @@ func (h *handler) apiGetBookmarks(w http.ResponseWriter, r *http.Request, ps htt if fileExists(archivePath) { bookmarks[i].HasArchive = true } + if fileExists(ebookPath) { + bookmarks[i].HasEbook = true + } } // Return JSON response @@ -373,9 +377,11 @@ func (h *handler) apiDeleteBookmark(w http.ResponseWriter, r *http.Request, ps h strID := strconv.Itoa(id) imgPath := fp.Join(h.DataDir, "thumb", strID) archivePath := fp.Join(h.DataDir, "archive", strID) + ebookPath := fp.Join(h.DataDir, "ebook", strID+".epub") os.Remove(imgPath) os.Remove(archivePath) + os.Remove(ebookPath) } fmt.Fprint(w, 1) @@ -458,6 +464,104 @@ func (h *handler) apiUpdateBookmark(w http.ResponseWriter, r *http.Request, ps h checkError(err) } +// apiDownloadEbook is handler for PUT /api/ebook +func (h *handler) apiDownloadEbook(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + ctx := r.Context() + + // Make sure session still valid + err := h.validateSession(r) + checkError(err) + + // Decode request + request := struct { + IDs []int `json:"ids"` + }{} + err = json.NewDecoder(r.Body).Decode(&request) + checkError(err) + + // Get existing bookmark from database + filter := database.GetBookmarksOptions{ + IDs: request.IDs, + WithContent: true, + } + + bookmarks, err := h.DB.GetBookmarks(ctx, filter) + checkError(err) + if len(bookmarks) == 0 { + http.NotFound(w, r) + return + } + + // Fetch data from internet + mx := sync.RWMutex{} + wg := sync.WaitGroup{} + chDone := make(chan struct{}) + chProblem := make(chan int, 10) + semaphore := make(chan struct{}, 10) + + for i, book := range bookmarks { + wg.Add(1) + + go func(i int, book model.Bookmark) { + // Make sure to finish the WG + defer wg.Done() + + // Register goroutine to semaphore + semaphore <- struct{}{} + defer func() { + <-semaphore + }() + + // Download data from internet + content, contentType, err := core.DownloadBookmark(book.URL) + if err != nil { + chProblem <- book.ID + return + } + + request := core.ProcessRequest{ + DataDir: h.DataDir, + Bookmark: book, + Content: content, + ContentType: contentType, + } + + book, err = core.GenerateEbook(request) + content.Close() + + if err != nil { + chProblem <- book.ID + return + } + + // Update list of bookmarks + mx.Lock() + bookmarks[i] = book + mx.Unlock() + }(i, book) + } + // Receive all problematic bookmarks + idWithProblems := []int{} + go func() { + for { + select { + case <-chDone: + return + case id := <-chProblem: + idWithProblems = append(idWithProblems, id) + } + } + }() + + // Wait until all download finished + wg.Wait() + close(chDone) + + w.Header().Set("Content-Type", "application1/json") + err = json.NewEncoder(w).Encode(&bookmarks) + checkError(err) +} + // apiUpdateCache is handler for PUT /api/cache func (h *handler) apiUpdateCache(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { ctx := r.Context() @@ -471,6 +575,7 @@ func (h *handler) apiUpdateCache(w http.ResponseWriter, r *http.Request, ps http IDs []int `json:"ids"` KeepMetadata bool `json:"keepMetadata"` CreateArchive bool `json:"createArchive"` + CreateEbook bool `json:"createEbook"` }{} err = json.NewDecoder(r.Body).Decode(&request) @@ -506,8 +611,9 @@ func (h *handler) apiUpdateCache(w http.ResponseWriter, r *http.Request, ps http for i, book := range bookmarks { wg.Add(1) - // Mark whether book will be archived + // Mark whether book will be archived or ebook generate request book.CreateArchive = request.CreateArchive + book.CreateEbook = request.CreateEbook go func(i int, book model.Bookmark, keepMetadata bool) { // Make sure to finish the WG diff --git a/internal/webserver/handler-ui.go b/internal/webserver/handler-ui.go index 9c7214d..554288d 100644 --- a/internal/webserver/handler-ui.go +++ b/internal/webserver/handler-ui.go @@ -119,6 +119,10 @@ func (h *handler) serveBookmarkContent(w http.ResponseWriter, r *http.Request, p } // Check if it has archive. + ebookPath := fp.Join(h.DataDir, "ebook", strID+".epub") + if fileExists(ebookPath) { + bookmark.HasEbook = true + } archivePath := fp.Join(h.DataDir, "archive", strID) if fileExists(archivePath) { bookmark.HasArchive = true @@ -356,3 +360,67 @@ func (h *handler) serveBookmarkArchive(w http.ResponseWriter, r *http.Request, p log.Printf("error writting response: %s", err) } } + +// serveEbook is handler for GET /bookmark/:id/ebook +func (h *handler) serveBookmarkEbook(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + ctx := r.Context() + + // Get bookmark ID from URL + strID := ps.ByName("id") + id, err := strconv.Atoi(strID) + checkError(err) + // Get bookmark in database + bookmark, exist, err := h.DB.GetBookmark(ctx, id, "") + checkError(err) + + if !exist { + http.Error(w, "bookmark not found", http.StatusNotFound) + return + } + + // If it's not public, make sure session still valid + if bookmark.Public != 1 { + err = h.validateSession(r) + if err != nil { + newPath := path.Join(h.RootPath, "/login") + redirectURL := createRedirectURL(newPath, r.URL.String()) + redirectPage(w, r, redirectURL) + return + } + } + + // Check if it has ebook. + ebookPath := fp.Join(h.DataDir, "ebook", strID+".epub") + if !fileExists(ebookPath) { + http.Error(w, "ebook not found", http.StatusNotFound) + return + } + + epub, err := os.Open(ebookPath) + if err != nil { + http.Error(w, "Internal server error", http.StatusNotFound) + return + } + defer epub.Close() + // Set content type + w.Header().Set("Content-Type", "application/epub+zip") + // Set cache value + info, err := epub.Stat() + if err != nil { + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + etag := fmt.Sprintf(`W/"%x-%x"`, info.ModTime().Unix(), info.Size()) + w.Header().Set("ETag", etag) + w.Header().Set("Cache-Control", "max-age=86400") + + // Serve epub file + if _, err := epub.Seek(0, 0); err != nil { + log.Printf("error during epub seek: %s", err) + } + _, err = io.Copy(w, epub) + if err != nil { + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } +} diff --git a/internal/webserver/server.go b/internal/webserver/server.go index 26ad02b..bf7e74b 100644 --- a/internal/webserver/server.go +++ b/internal/webserver/server.go @@ -183,6 +183,7 @@ func ServeApp(cfg Config) error { router.GET(jp("/login"), withLogging(hdl.serveLoginPage)) router.GET(jp("/bookmark/:id/thumb"), withLogging(hdl.serveThumbnailImage)) router.GET(jp("/bookmark/:id/content"), withLogging(hdl.serveBookmarkContent)) + router.GET(jp("/bookmark/:id/ebook"), withLogging(hdl.serveBookmarkEbook)) router.GET(jp("/bookmark/:id/archive/*filepath"), withLogging(hdl.serveBookmarkArchive)) router.POST(jp("/api/login"), withLogging(hdl.apiLogin)) @@ -194,6 +195,7 @@ func ServeApp(cfg Config) error { router.DELETE(jp("/api/bookmarks"), withLogging(hdl.apiDeleteBookmark)) router.PUT(jp("/api/bookmarks"), withLogging(hdl.apiUpdateBookmark)) router.PUT(jp("/api/cache"), withLogging(hdl.apiUpdateCache)) + router.PUT(jp("/api/ebook"), withLogging(hdl.apiDownloadEbook)) router.PUT(jp("/api/bookmarks/tags"), withLogging(hdl.apiUpdateBookmarkTags)) router.POST(jp("/api/bookmarks/ext"), withLogging(hdl.apiInsertViaExtension)) router.DELETE(jp("/api/bookmarks/ext"), withLogging(hdl.apiDeleteViaExtension))