refactor: migrate bookmark static pages to new http server (#775)

* migrate bookmark content route to new http server

* new archive page

* remove unused go generate comment

* database mock

* utils cleanup

* unused var

* domains refactor and tests

* fixed secret key type

* redirect to login on ui errors

* fixed archive folder with storage domain

* webroot documentation

* some bookmark route tests

* fixed error in bookmark domain for non existant bookmarks

* centralice errors

* add coverage data to unittests

* added tests, refactor storage to use afero

* removed mock to avoid increasing complexity

* using deps to copy files around

* remove config usage (to deps)

* remove handler-ui file
This commit is contained in:
Felipe Martin 2023-12-28 18:18:32 +01:00 committed by GitHub
parent fe6a306e9e
commit cc7c75116d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
65 changed files with 1340 additions and 937 deletions

View file

@ -94,3 +94,8 @@ build: clean
coverage:
$(GO) test $(GO_TEST_FLAGS) -coverprofile=coverage.txt ./...
$(GO) tool cover -html=coverage.txt
## Run generate accross the project
.PHONY: generated
generate:
$(GO) generate ./...

View file

@ -1,35 +1,53 @@
Content
---
# Configuration
<!-- TOC -->
- [Content](#content)
- [Data Directory](#data-directory)
- [Webroot](#webroot)
- [Nginx](#nginx)
- [Database](#database)
- [MySQL](#mysql)
- [PostgreSQL](#postgresql)
- [MySQL](#mysql)
- [PostgreSQL](#postgresql)
<!-- /TOC -->
Data Directory
---
## Data Directory
Shiori is designed to work out of the box, but you can change where it stores your bookmarks if you need to.
By default, Shiori saves your bookmarks in one of the following directories:
| Platform | Directory |
|----------|--------------------------------------------------------------|
| Platform | Directory |
| -------- | ------------------------------------------------------------ |
| Linux | `${XDG_DATA_HOME}/shiori` (default: `~/.local/share/shiori`) |
| macOS | `~/Library/Application Support/shiori` |
| macOS | `~/Library/Application Support/shiori` |
| Windows | `%LOCALAPPDATA%/shiori` |
If you pass the flag `--portable` to Shiori, your data will be stored in the `shiori-data` subdirectory alongside the shiori executable.
To specify a custom path, set the `SHIORI_DIR` environment variable.
Database
---
## Webroot
If you want to serve Shiori behind a reverse proxy, you can set the `SHIORI_WEBROOT` environment variable to the path where Shiori is served, e.g. `/shiori`.
Keep in mind this configuration wont make Shiori accessible from `/shiori` path so you need to setup your reverse proxy accordingly so it can strip the webroot path.
We provide some examples for popular reverse proxies below. Please follow your reverse proxy documentation in order to setup it properly.
### Nginx
Fox nginx, you can use the following configuration as a example. The important part **is the trailing slash in `proxy_pass` directive**:
```nginx
location /shiori {
proxy_pass http://localhost:8080/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
}
```
## Database
Shiori uses an SQLite3 database stored in the above data directory by default. If you prefer, you can also use MySQL or PostgreSQL database by setting it in environment variables.

2
go.mod
View file

@ -7,7 +7,6 @@ require (
github.com/PuerkitoBio/goquery v1.8.1
github.com/disintegration/imaging v1.6.2
github.com/fatih/color v1.16.0
github.com/gin-contrib/gzip v0.0.6
github.com/gin-contrib/requestid v0.0.6
github.com/gin-contrib/static v0.0.1
github.com/gin-gonic/gin v1.9.1
@ -27,6 +26,7 @@ require (
github.com/sethvargo/go-envconfig v0.9.0
github.com/shurcooL/vfsgen v0.0.0-20230704071429-0000e147ea92
github.com/sirupsen/logrus v1.9.3
github.com/spf13/afero v1.11.0
github.com/spf13/cobra v1.8.0
github.com/stretchr/testify v1.8.4
github.com/swaggo/files v1.0.1

2
go.sum
View file

@ -212,6 +212,8 @@ github.com/shurcooL/vfsgen v0.0.0-20230704071429-0000e147ea92/go.mod h1:7/OT02F6
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=

View file

@ -45,7 +45,7 @@ func addHandler(cmd *cobra.Command, args []string) {
excerpt = normalizeSpace(excerpt)
// Create bookmark item
book := model.Bookmark{
book := model.BookmarkDTO{
URL: url,
Title: title,
Excerpt: excerpt,
@ -101,7 +101,7 @@ func addHandler(cmd *cobra.Command, args []string) {
KeepExcerpt: excerpt != "",
}
book, isFatalErr, err = core.ProcessBookmark(request)
book, isFatalErr, err = core.ProcessBookmark(deps, request)
content.Close()
if err != nil {

View file

@ -76,7 +76,7 @@ func checkHandler(cmd *cobra.Command, args []string) {
for i, book := range bookmarks {
wg.Add(1)
go func(i int, book model.Bookmark) {
go func(i int, book model.BookmarkDTO) {
// Make sure to finish the WG
defer wg.Done()

View file

@ -52,7 +52,7 @@ func importHandler(cmd *cobra.Command, args []string) {
defer srcFile.Close()
// Parse bookmark's file
bookmarks := []model.Bookmark{}
bookmarks := []model.BookmarkDTO{}
mapURL := make(map[string]struct{})
doc, err := goquery.NewDocumentFromReader(srcFile)
@ -135,7 +135,7 @@ func importHandler(cmd *cobra.Command, args []string) {
}
// Add item to list
bookmark := model.Bookmark{
bookmark := model.BookmarkDTO{
URL: url,
Title: title,
Tags: tags,

View file

@ -36,7 +36,7 @@ func pocketHandler(cmd *cobra.Command, args []string) {
defer srcFile.Close()
// Parse pocket's file
bookmarks := []model.Bookmark{}
bookmarks := []model.BookmarkDTO{}
mapURL := make(map[string]struct{})
doc, err := goquery.NewDocumentFromReader(srcFile)
@ -93,7 +93,7 @@ func pocketHandler(cmd *cobra.Command, args []string) {
}
// Add item to list
bookmark := model.Bookmark{
bookmark := model.BookmarkDTO{
URL: url,
Title: title,
Modified: modified.Format(model.DatabaseDateFormat),

View file

@ -8,9 +8,11 @@ import (
"github.com/go-shiori/shiori/internal/config"
"github.com/go-shiori/shiori/internal/database"
"github.com/go-shiori/shiori/internal/dependencies"
"github.com/go-shiori/shiori/internal/domains"
"github.com/go-shiori/shiori/internal/model"
"github.com/sirupsen/logrus"
"github.com/spf13/afero"
"github.com/spf13/cobra"
"golang.org/x/net/context"
)
@ -47,7 +49,7 @@ func ShioriCmd() *cobra.Command {
return rootCmd
}
func initShiori(ctx context.Context, cmd *cobra.Command) (*config.Config, *config.Dependencies) {
func initShiori(ctx context.Context, cmd *cobra.Command) (*config.Config, *dependencies.Dependencies) {
logger := logrus.New()
portableMode, _ := cmd.Flags().GetBool("portable")
@ -96,9 +98,11 @@ func initShiori(ctx context.Context, cmd *cobra.Command) (*config.Config, *confi
logger.Warn("Development mode is ENABLED, this will enable some helpers for local development, unsuitable for production environments")
}
dependencies := config.NewDependencies(logger, db, cfg)
dependencies.Domains.Auth = domains.NewAccountsDomain(logger, cfg.Http.SecretKey, db)
dependencies.Domains.Archiver = domains.NewArchiverDomain(logger, cfg.Storage.DataDir)
dependencies := dependencies.NewDependencies(logger, db, cfg)
dependencies.Domains.Auth = domains.NewAccountsDomain(dependencies)
dependencies.Domains.Archiver = domains.NewArchiverDomain(dependencies)
dependencies.Domains.Bookmarks = domains.NewBookmarksDomain(dependencies)
dependencies.Domains.Storage = domains.NewStorageDomain(dependencies, afero.NewBasePathFs(afero.NewOsFs(), cfg.Storage.DataDir))
// Workaround: Get accounts to make sure at least one is present in the database.
// If there's no accounts in the database, create the shiori/gopher account the legacy api

View file

@ -37,7 +37,7 @@ func newServerCommandHandler() func(cmd *cobra.Command, args []string) {
rootPath, _ := cmd.Flags().GetString("webroot")
accessLog, _ := cmd.Flags().GetBool("access-log")
serveWebUI, _ := cmd.Flags().GetBool("serve-web-ui")
secretKey, _ := cmd.Flags().GetString("secret-key")
secretKey, _ := cmd.Flags().GetBytesHex("secret-key")
cfg, dependencies := initShiori(ctx, cmd)
@ -66,7 +66,10 @@ func newServerCommandHandler() func(cmd *cobra.Command, args []string) {
dependencies.Log.Infof("Starting Shiori v%s", model.BuildVersion)
server := http.NewHttpServer(dependencies.Log).Setup(cfg, dependencies)
server, err := http.NewHttpServer(dependencies.Log).Setup(cfg, dependencies)
if err != nil {
dependencies.Log.WithError(err).Fatal("error setting up server")
}
if err := server.Start(ctx); err != nil {
dependencies.Log.WithError(err).Fatal("error starting server")

View file

@ -147,7 +147,7 @@ func updateHandler(cmd *cobra.Command, args []string) {
book.URL = url
}
go func(i int, book model.Bookmark) {
go func(i int, book model.BookmarkDTO) {
// Make sure to finish the WG
defer wg.Done()
@ -175,7 +175,7 @@ func updateHandler(cmd *cobra.Command, args []string) {
LogArchival: logArchival,
}
book, _, err = core.ProcessBookmark(request)
book, _, err = core.ProcessBookmark(deps, request)
content.Close()
if err != nil {

View file

@ -41,7 +41,7 @@ func isURLValid(s string) bool {
return err == nil && tmp.Scheme != "" && tmp.Hostname() != ""
}
func printBookmarks(bookmarks ...model.Bookmark) {
func printBookmarks(bookmarks ...model.BookmarkDTO) {
for _, bookmark := range bookmarks {
// Create bookmark index
strBookmarkIndex := fmt.Sprintf("%d. ", bookmark.ID)

View file

@ -50,7 +50,7 @@ type HttpConfig struct {
RootPath string `env:"HTTP_ROOT_PATH,default=/"`
AccessLog bool `env:"HTTP_ACCESS_LOG,default=True"`
ServeWebUI bool `env:"HTTP_SERVE_WEB_UI,default=True"`
SecretKey string `env:"HTTP_SECRET_KEY"`
SecretKey []byte `env:"HTTP_SECRET_KEY"`
// Fiber Specific
BodyLimit int `env:"HTTP_BODY_LIMIT,default=1024"`
ReadTimeout time.Duration `env:"HTTP_READ_TIMEOUT,default=10s"`
@ -82,13 +82,13 @@ type Config struct {
// SetDefaults sets the default values for the configuration
func (c *HttpConfig) SetDefaults(logger *logrus.Logger) {
// Set a random secret key if not set
if c.SecretKey == "" {
if len(c.SecretKey) == 0 {
logger.Warn("SHIORI_HTTP_SECRET_KEY is not set, using random value. This means that all sessions will be invalidated on server restart.")
randomUUID, err := uuid.NewV4()
if err != nil {
logger.WithError(err).Fatal("couldn't generate a random UUID")
}
c.SecretKey = randomUUID.String()
c.SecretKey = []byte(randomUUID.String())
}
}

View file

@ -1,25 +0,0 @@
package config
import (
"github.com/go-shiori/shiori/internal/database"
"github.com/go-shiori/shiori/internal/domains"
"github.com/sirupsen/logrus"
)
type Dependencies struct {
Log *logrus.Logger
Database database.DB
Config *Config
Domains struct {
Auth domains.AccountsDomain
Archiver domains.ArchiverDomain
}
}
func NewDependencies(log *logrus.Logger, db database.DB, cfg *Config) *Dependencies {
return &Dependencies{
Log: log,
Config: cfg,
Database: db,
}
}

View file

@ -1,13 +1,13 @@
package core
import (
"fmt"
"os"
fp "path/filepath"
"strconv"
"strings"
epub "github.com/go-shiori/go-epub"
"github.com/go-shiori/shiori/internal/dependencies"
"github.com/go-shiori/shiori/internal/model"
"github.com/pkg/errors"
)
@ -15,8 +15,7 @@ import (
// GenerateEbook receives a `ProcessRequest` and generates an ebook file in the destination path specified.
// The destination path `dstPath` should include file name with ".epub" extension
// The bookmark model will be used to update the UI based on whether this function is successful or not.
func GenerateEbook(req ProcessRequest, dstPath string) (book model.Bookmark, err error) {
func GenerateEbook(deps *dependencies.Dependencies, req ProcessRequest, dstPath string) (book model.BookmarkDTO, err error) {
book = req.Bookmark
// Make sure bookmark ID is defined
@ -27,14 +26,14 @@ func GenerateEbook(req ProcessRequest, dstPath string) (book model.Bookmark, err
// Get current state of bookmark cheak archive and thumb
strID := strconv.Itoa(book.ID)
imagePath := fp.Join(req.DataDir, "thumb", fmt.Sprintf("%d", book.ID))
archivePath := fp.Join(req.DataDir, "archive", fmt.Sprintf("%d", book.ID))
bookmarkThumbnailPath := model.GetThumbnailPath(&book)
bookmarkArchivePath := model.GetArchivePath(&book)
if _, err := os.Stat(imagePath); err == nil {
if deps.Domains.Storage.FileExists(bookmarkThumbnailPath) {
book.ImageURL = fp.Join("/", "bookmark", strID, "thumb")
}
if _, err := os.Stat(archivePath); err == nil {
if deps.Domains.Storage.FileExists(bookmarkArchivePath) {
book.HasArchive = true
}
@ -77,7 +76,7 @@ func GenerateEbook(req ProcessRequest, dstPath string) (book model.Bookmark, err
defer tmpFile.Close()
// If everything go well we move ebook to dstPath
err = MoveFileToDestination(dstPath, tmpFile)
err = deps.Domains.Storage.WriteFile(dstPath, tmpFile)
if err != nil {
return book, errors.Wrap(err, "failed move ebook to destination")
}

View file

@ -1,25 +1,34 @@
package core_test
import (
"fmt"
"context"
"os"
fp "path/filepath"
"testing"
"github.com/go-shiori/shiori/internal/core"
"github.com/go-shiori/shiori/internal/domains"
"github.com/go-shiori/shiori/internal/model"
"github.com/go-shiori/shiori/internal/testutil"
"github.com/sirupsen/logrus"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
)
func TestGenerateEbook(t *testing.T) {
logger := logrus.New()
_, deps := testutil.GetTestConfigurationAndDependencies(t, context.TODO(), logger)
t.Run("Successful ebook generate", func(t *testing.T) {
t.Run("valid bookmarkId that return HasEbook true", func(t *testing.T) {
// test cae
tempDir := t.TempDir()
dstDir := t.TempDir()
deps.Domains.Storage = domains.NewStorageDomain(deps, afero.NewBasePathFs(afero.NewOsFs(), dstDir))
mockRequest := core.ProcessRequest{
Bookmark: model.Bookmark{
Bookmark: model.BookmarkDTO{
ID: 1,
Title: "Example Bookmark",
HTML: "<html><body>Example HTML</body></html>",
@ -29,73 +38,65 @@ func TestGenerateEbook(t *testing.T) {
ContentType: "text/html",
}
bookmark, err := core.GenerateEbook(mockRequest, fp.Join(tempDir, "1"))
bookmark, err := core.GenerateEbook(deps, mockRequest, fp.Join(tempDir, "1"))
assert.True(t, bookmark.HasEbook)
assert.NoError(t, err)
})
t.Run("ebook generate with valid BookmarkID EbookExist ImagePathExist ReturnWithHasEbookTrue", func(t *testing.T) {
tempDir := t.TempDir()
dstDir := t.TempDir()
deps.Domains.Storage = domains.NewStorageDomain(deps, afero.NewBasePathFs(afero.NewOsFs(), dstDir))
bookmark := model.BookmarkDTO{
ID: 2,
HasEbook: false,
}
mockRequest := core.ProcessRequest{
Bookmark: model.Bookmark{
ID: 1,
HasEbook: false,
},
Bookmark: bookmark,
DataDir: dstDir,
ContentType: "text/html",
}
// Create the image directory
imageDir := fp.Join(mockRequest.DataDir, "thumb")
err := os.MkdirAll(imageDir, os.ModePerm)
if err != nil {
t.Fatal(err)
}
// Create the image file
imagePath := fp.Join(mockRequest.DataDir, "thumb", fmt.Sprintf("%d", mockRequest.Bookmark.ID))
file, err := os.Create(imagePath)
// Create the thumbnail file
imagePath := model.GetThumbnailPath(&bookmark)
deps.Domains.Storage.FS().MkdirAll(fp.Dir(imagePath), os.ModePerm)
file, err := deps.Domains.Storage.FS().Create(imagePath)
if err != nil {
t.Fatal(err)
}
defer file.Close()
bookmark, err := core.GenerateEbook(mockRequest, fp.Join(tempDir, "1"))
expectedimagePath := "/bookmark/1/thumb"
if expectedimagePath != bookmark.ImageURL {
t.Errorf("Expected imageURL %s, but got %s", bookmark.ImageURL, expectedimagePath)
}
assert.True(t, bookmark.HasEbook)
bookmark, err = core.GenerateEbook(deps, mockRequest, model.GetEbookPath(&bookmark))
expectedImagePath := "/bookmark/2/thumb"
assert.NoError(t, err)
assert.True(t, bookmark.HasEbook)
assert.Equalf(t, expectedImagePath, bookmark.ImageURL, "Expected imageURL %s, but got %s", expectedImagePath, bookmark.ImageURL)
})
t.Run("generate ebook valid BookmarkID EbookExist Returnh HasArchive True", func(t *testing.T) {
t.Run("generate ebook valid BookmarkID EbookExist ReturnHasArchiveTrue", func(t *testing.T) {
tempDir := t.TempDir()
dstDir := t.TempDir()
deps.Domains.Storage = domains.NewStorageDomain(deps, afero.NewBasePathFs(afero.NewOsFs(), dstDir))
bookmark := model.BookmarkDTO{
ID: 3,
HasEbook: false,
}
mockRequest := core.ProcessRequest{
Bookmark: model.Bookmark{
ID: 1,
HasEbook: false,
},
Bookmark: bookmark,
DataDir: dstDir,
ContentType: "text/html",
}
// Create the archive directory
archiveDir := fp.Join(mockRequest.DataDir, "archive")
err := os.MkdirAll(archiveDir, os.ModePerm)
if err != nil {
t.Fatal(err)
}
// Create the archive file
archivePath := fp.Join(mockRequest.DataDir, "archive", fmt.Sprintf("%d", mockRequest.Bookmark.ID))
file, err := os.Create(archivePath)
archivePath := model.GetArchivePath(&bookmark)
deps.Domains.Storage.FS().MkdirAll(fp.Dir(archivePath), os.ModePerm)
file, err := deps.Domains.Storage.FS().Create(archivePath)
if err != nil {
t.Fatal(err)
}
defer file.Close()
bookmark, err := core.GenerateEbook(mockRequest, fp.Join(tempDir, "1"))
bookmark, err = core.GenerateEbook(deps, mockRequest, fp.Join(tempDir, "1"))
assert.True(t, bookmark.HasArchive)
assert.NoError(t, err)
})
@ -104,7 +105,7 @@ func TestGenerateEbook(t *testing.T) {
t.Run("invalid bookmarkId that return Error", func(t *testing.T) {
tempDir := t.TempDir()
mockRequest := core.ProcessRequest{
Bookmark: model.Bookmark{
Bookmark: model.BookmarkDTO{
ID: 0,
HasEbook: false,
},
@ -112,41 +113,38 @@ func TestGenerateEbook(t *testing.T) {
ContentType: "text/html",
}
bookmark, err := core.GenerateEbook(mockRequest, tempDir)
bookmark, err := core.GenerateEbook(deps, mockRequest, tempDir)
assert.Equal(t, model.Bookmark{
assert.Equal(t, model.BookmarkDTO{
ID: 0,
HasEbook: false,
}, bookmark)
assert.Error(t, err)
})
t.Run("ebook exist return HasEbook true", func(t *testing.T) {
tempDir := t.TempDir()
dstDir := t.TempDir()
deps.Domains.Storage = domains.NewStorageDomain(deps, afero.NewBasePathFs(afero.NewOsFs(), dstDir))
bookmark := model.BookmarkDTO{
ID: 1,
HasEbook: false,
}
mockRequest := core.ProcessRequest{
Bookmark: model.Bookmark{
ID: 1,
HasEbook: false,
},
Bookmark: bookmark,
DataDir: dstDir,
ContentType: "text/html",
}
// Create the ebook directory
ebookDir := fp.Join(mockRequest.DataDir, "ebook")
err := os.MkdirAll(ebookDir, os.ModePerm)
if err != nil {
t.Fatal(err)
}
// Create the ebook file
ebookfile := fp.Join(mockRequest.DataDir, "ebook", fmt.Sprintf("%d.epub", mockRequest.Bookmark.ID))
file, err := os.Create(ebookfile)
ebookFilePath := model.GetEbookPath(&bookmark)
deps.Domains.Storage.FS().MkdirAll(fp.Dir(ebookFilePath), os.ModePerm)
file, err := deps.Domains.Storage.FS().Create(ebookFilePath)
if err != nil {
t.Fatal(err)
}
defer file.Close()
bookmark, err := core.GenerateEbook(mockRequest, fp.Join(tempDir, "1"))
bookmark, err = core.GenerateEbook(deps, mockRequest, ebookFilePath)
assert.True(t, bookmark.HasEbook)
assert.NoError(t, err)
@ -155,7 +153,7 @@ func TestGenerateEbook(t *testing.T) {
tempDir := t.TempDir()
mockRequest := core.ProcessRequest{
Bookmark: model.Bookmark{
Bookmark: model.BookmarkDTO{
ID: 1,
HasEbook: false,
},
@ -163,7 +161,7 @@ func TestGenerateEbook(t *testing.T) {
ContentType: "application/pdf",
}
bookmark, err := core.GenerateEbook(mockRequest, tempDir)
bookmark, err := core.GenerateEbook(deps, mockRequest, tempDir)
assert.False(t, bookmark.HasEbook)
assert.Error(t, err)

View file

@ -18,6 +18,7 @@ import (
"github.com/disintegration/imaging"
"github.com/go-shiori/go-readability"
"github.com/go-shiori/shiori/internal/dependencies"
"github.com/go-shiori/shiori/internal/model"
"github.com/go-shiori/warc"
"github.com/pkg/errors"
@ -30,7 +31,7 @@ import (
// ProcessRequest is the request for processing bookmark.
type ProcessRequest struct {
DataDir string
Bookmark model.Bookmark
Bookmark model.BookmarkDTO
Content io.Reader
ContentType string
KeepTitle bool
@ -42,7 +43,7 @@ var ErrNoSupportedImageType = errors.New("unsupported image type")
// ProcessBookmark process the bookmark and archive it if needed.
// Return three values, is error fatal, and error value.
func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool, err error) {
func ProcessBookmark(deps *dependencies.Dependencies, req ProcessRequest) (book model.BookmarkDTO, isFatalErr bool, err error) {
book = req.Bookmark
contentType := req.ContentType
@ -70,7 +71,7 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool,
// If this is HTML, parse for readable content
strID := strconv.Itoa(book.ID)
imgPath := fp.Join(req.DataDir, "thumb", strID)
imgPath := model.GetThumbnailPath(&book)
var imageURLs []string
if strings.Contains(contentType, "text/html") {
isReadable := readability.Check(readabilityCheckInput)
@ -107,7 +108,7 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool,
if article.Image != "" {
imageURLs = append(imageURLs, article.Image)
} else {
os.Remove(imgPath)
deps.Domains.Storage.FS().Remove(imgPath)
}
if article.Favicon != "" {
@ -123,11 +124,11 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool,
// Save article image to local disk
for i, imageURL := range imageURLs {
err = DownloadBookImage(imageURL, imgPath)
err = DownloadBookImage(deps, imageURL, imgPath)
if err != nil && errors.Is(err, ErrNoSupportedImageType) {
log.Printf("%s: %s", err, imageURL)
if i == len(imageURLs)-1 {
os.Remove(imgPath)
deps.Domains.Storage.FS().Remove(imgPath)
}
}
if err != nil {
@ -142,13 +143,13 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool,
// If needed, create ebook as well
if book.CreateEbook {
ebookPath := fp.Join(req.DataDir, "ebook", strID+".epub")
ebookPath := model.GetEbookPath(&book)
req.Bookmark = book
if strings.Contains(contentType, "application/pdf") {
return book, false, errors.Wrap(err, "can't create ebook from pdf")
} else {
_, err = GenerateEbook(req, ebookPath)
_, err = GenerateEbook(deps, req, ebookPath)
if err != nil {
return book, true, errors.Wrap(err, "failed to create ebook")
}
@ -162,7 +163,7 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool,
if err != nil {
return book, false, fmt.Errorf("failed to create temp archive: %v", err)
}
defer os.Remove(tmpFile.Name())
defer deps.Domains.Storage.FS().Remove(tmpFile.Name())
archivalRequest := warc.ArchivalRequest{
URL: book.URL,
@ -178,10 +179,8 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool,
return book, false, fmt.Errorf("failed to create archive: %v", err)
}
// Prepare destination file.
dstPath := fp.Join(req.DataDir, "archive", fmt.Sprintf("%d", book.ID))
err = MoveFileToDestination(dstPath, tmpFile)
dstPath := model.GetArchivePath(&book)
err = deps.Domains.Storage.WriteFile(dstPath, tmpFile)
if err != nil {
return book, false, fmt.Errorf("failed move archive to destination `: %v", err)
}
@ -192,7 +191,7 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool,
return book, false, nil
}
func DownloadBookImage(url, dstPath string) error {
func DownloadBookImage(deps *dependencies.Dependencies, url, dstPath string) error {
// Fetch data from URL
resp, err := httpClient.Get(url)
if err != nil {
@ -264,37 +263,10 @@ func DownloadBookImage(url, dstPath string) error {
return fmt.Errorf("failed to save image %s: %v", url, err)
}
err = MoveFileToDestination(dstPath, tmpFile)
err = deps.Domains.Storage.WriteFile(dstPath, tmpFile)
if err != nil {
return err
}
return nil
}
// dstPath requires the filename
func MoveFileToDestination(dstPath string, tmpFile *os.File) error {
// Prepare destination file.
err := os.MkdirAll(fp.Dir(dstPath), model.DataDirPerm)
if err != nil {
return fmt.Errorf("failed to create destination dir: %v", err)
}
dstFile, err := os.Create(dstPath)
if err != nil {
return fmt.Errorf("failed to create destination file: %v", err)
}
defer dstFile.Close()
// Copy temporary file to destination
_, err = tmpFile.Seek(0, io.SeekStart)
if err != nil {
return fmt.Errorf("failed to rewind temporary file: %v", err)
}
_, err = io.Copy(dstFile, tmpFile)
if err != nil {
return fmt.Errorf("failed to copy file to the destination")
}
return nil
}

View file

@ -2,6 +2,7 @@ package core_test
import (
"bytes"
"context"
"net/http"
"net/http/httptest"
"os"
@ -10,43 +11,15 @@ import (
"github.com/go-shiori/shiori/internal/core"
"github.com/go-shiori/shiori/internal/model"
"github.com/go-shiori/shiori/internal/testutil"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
)
func TestMoveFileToDestination(t *testing.T) {
t.Run("create fails", func(t *testing.T) {
t.Run("directory create fails", func(t *testing.T) {
// test if create dir fails
tmpFile, err := os.CreateTemp("", "image")
assert.NoError(t, err)
defer os.Remove(tmpFile.Name())
err = core.MoveFileToDestination("/destination/test", tmpFile)
assert.Error(t, err)
assert.Contains(t, err.Error(), "failed to create destination dir")
})
t.Run("file create fails", func(t *testing.T) {
// if create file failed
tmpFile, err := os.CreateTemp("", "image")
assert.NoError(t, err)
defer os.Remove(tmpFile.Name())
// Create a destination directory
dstDir := t.TempDir()
assert.NoError(t, err)
defer os.Remove(dstDir)
// Set destination path to an invalid file name to force os.Create to fail
dstPath := fp.Join(dstDir, "\000invalid\000")
err = core.MoveFileToDestination(dstPath, tmpFile)
assert.Error(t, err)
assert.Contains(t, err.Error(), "failed to create destination file")
})
})
}
func TestDownloadBookImage(t *testing.T) {
logger := logrus.New()
_, deps := testutil.GetTestConfigurationAndDependencies(t, context.TODO(), logger)
t.Run("Download Images", func(t *testing.T) {
t.Run("fails", func(t *testing.T) {
// images is too small with unsupported format with a valid URL
@ -56,13 +29,13 @@ func TestDownloadBookImage(t *testing.T) {
defer os.Remove(dstPath)
// Act
err := core.DownloadBookImage(imageURL, dstPath)
err := core.DownloadBookImage(deps, imageURL, dstPath)
// Assert
assert.EqualError(t, err, "unsupported image type")
assert.NoFileExists(t, dstPath)
assert.False(t, deps.Domains.Storage.FileExists(dstPath))
})
t.Run("sucssesful downlosd image", func(t *testing.T) {
t.Run("successful download image", func(t *testing.T) {
// Arrange
imageURL := "https://raw.githubusercontent.com/go-shiori/shiori/master/docs/readme/cover.png"
tempDir := t.TempDir()
@ -70,13 +43,13 @@ func TestDownloadBookImage(t *testing.T) {
defer os.Remove(dstPath)
// Act
err := core.DownloadBookImage(imageURL, dstPath)
err := core.DownloadBookImage(deps, imageURL, dstPath)
// Assert
assert.NoError(t, err)
assert.FileExists(t, dstPath)
assert.True(t, deps.Domains.Storage.FileExists(dstPath))
})
t.Run("sucssesful downlosd medium size image", func(t *testing.T) {
t.Run("successful download medium size image", func(t *testing.T) {
// create a file server handler for the 'testdata' directory
fs := http.FileServer(http.Dir("../../testdata/"))
@ -91,20 +64,22 @@ func TestDownloadBookImage(t *testing.T) {
defer os.Remove(dstPath)
// Act
err := core.DownloadBookImage(imageURL, dstPath)
err := core.DownloadBookImage(deps, imageURL, dstPath)
// Assert
assert.NoError(t, err)
assert.FileExists(t, dstPath)
assert.True(t, deps.Domains.Storage.FileExists(dstPath))
})
})
}
func TestProcessBookmark(t *testing.T) {
logger := logrus.New()
_, deps := testutil.GetTestConfigurationAndDependencies(t, context.TODO(), logger)
t.Run("ProcessRequest with sucssesful result", func(t *testing.T) {
t.Run("Normal without image", func(t *testing.T) {
bookmark := model.Bookmark{
bookmark := model.BookmarkDTO{
ID: 1,
URL: "https://example.com",
Title: "Example",
@ -121,7 +96,7 @@ func TestProcessBookmark(t *testing.T) {
KeepTitle: true,
KeepExcerpt: true,
}
expected, _, _ := core.ProcessBookmark(request)
expected, _, _ := core.ProcessBookmark(deps, request)
if expected.ID != bookmark.ID {
t.Errorf("Unexpected ID: got %v, want %v", expected.ID, bookmark.ID)
@ -148,7 +123,7 @@ func TestProcessBookmark(t *testing.T) {
<p>This is an example article</p>
</body>
</html>`
bookmark := model.Bookmark{
bookmark := model.BookmarkDTO{
ID: 1,
URL: "https://example.com",
Title: "Example",
@ -165,7 +140,7 @@ func TestProcessBookmark(t *testing.T) {
KeepTitle: true,
KeepExcerpt: true,
}
expected, _, _ := core.ProcessBookmark(request)
expected, _, _ := core.ProcessBookmark(deps, request)
if expected.ID != bookmark.ID {
t.Errorf("Unexpected ID: got %v, want %v", expected.ID, bookmark.ID)
@ -198,7 +173,7 @@ func TestProcessBookmark(t *testing.T) {
<p>This is an example article</p>
</body>
</html>`
bookmark := model.Bookmark{
bookmark := model.BookmarkDTO{
ID: 1,
URL: "https://example.com",
Title: "Example",
@ -215,7 +190,7 @@ func TestProcessBookmark(t *testing.T) {
KeepTitle: true,
KeepExcerpt: true,
}
expected, _, _ := core.ProcessBookmark(request)
expected, _, _ := core.ProcessBookmark(deps, request)
if expected.ID != bookmark.ID {
t.Errorf("Unexpected ID: got %v, want %v", expected.ID, bookmark.ID)
@ -231,7 +206,7 @@ func TestProcessBookmark(t *testing.T) {
}
})
t.Run("ProcessRequest sucssesful with empty title ", func(t *testing.T) {
bookmark := model.Bookmark{
bookmark := model.BookmarkDTO{
ID: 1,
URL: "https://example.com",
Title: "",
@ -248,7 +223,7 @@ func TestProcessBookmark(t *testing.T) {
KeepTitle: true,
KeepExcerpt: true,
}
expected, _, _ := core.ProcessBookmark(request)
expected, _, _ := core.ProcessBookmark(deps, request)
if expected.ID != bookmark.ID {
t.Errorf("Unexpected ID: got %v, want %v", expected.ID, bookmark.ID)
@ -264,7 +239,7 @@ func TestProcessBookmark(t *testing.T) {
}
})
t.Run("ProcessRequest sucssesful with empty Excerpt", func(t *testing.T) {
bookmark := model.Bookmark{
bookmark := model.BookmarkDTO{
ID: 1,
URL: "https://example.com",
Title: "",
@ -281,7 +256,7 @@ func TestProcessBookmark(t *testing.T) {
KeepTitle: true,
KeepExcerpt: false,
}
expected, _, _ := core.ProcessBookmark(request)
expected, _, _ := core.ProcessBookmark(deps, request)
if expected.ID != bookmark.ID {
t.Errorf("Unexpected ID: got %v, want %v", expected.ID, bookmark.ID)
@ -299,7 +274,7 @@ func TestProcessBookmark(t *testing.T) {
t.Run("Specific case", func(t *testing.T) {
t.Run("ProcessRequest with ID zero", func(t *testing.T) {
bookmark := model.Bookmark{
bookmark := model.BookmarkDTO{
ID: 0,
URL: "https://example.com",
Title: "Example",
@ -316,7 +291,7 @@ func TestProcessBookmark(t *testing.T) {
KeepTitle: true,
KeepExcerpt: true,
}
_, isFatal, err := core.ProcessBookmark(request)
_, isFatal, err := core.ProcessBookmark(deps, request)
assert.Error(t, err)
assert.Contains(t, err.Error(), "bookmark ID is not valid")
assert.True(t, isFatal)
@ -324,7 +299,7 @@ func TestProcessBookmark(t *testing.T) {
t.Run("ProcessRequest that content type not zero", func(t *testing.T) {
bookmark := model.Bookmark{
bookmark := model.BookmarkDTO{
ID: 1,
URL: "https://example.com",
Title: "Example",
@ -341,7 +316,7 @@ func TestProcessBookmark(t *testing.T) {
KeepTitle: true,
KeepExcerpt: true,
}
_, _, err := core.ProcessBookmark(request)
_, _, err := core.ProcessBookmark(deps, request)
assert.NoError(t, err)
})
})

View file

@ -72,10 +72,10 @@ type DB interface {
Migrate() error
// SaveBookmarks saves bookmarks data to database.
SaveBookmarks(ctx context.Context, create bool, bookmarks ...model.Bookmark) ([]model.Bookmark, error)
SaveBookmarks(ctx context.Context, create bool, bookmarks ...model.BookmarkDTO) ([]model.BookmarkDTO, error)
// GetBookmarks fetch list of bookmarks based on submitted options.
GetBookmarks(ctx context.Context, opts GetBookmarksOptions) ([]model.Bookmark, error)
GetBookmarks(ctx context.Context, opts GetBookmarksOptions) ([]model.BookmarkDTO, error)
// GetBookmarksCount get count of bookmarks in database.
GetBookmarksCount(ctx context.Context, opts GetBookmarksOptions) (int, error)
@ -84,7 +84,7 @@ type DB interface {
DeleteBookmarks(ctx context.Context, ids ...int) error
// GetBookmark fetches bookmark based on its ID or URL.
GetBookmark(ctx context.Context, id int, url string) (model.Bookmark, bool, error)
GetBookmark(ctx context.Context, id int, url string) (model.BookmarkDTO, bool, error)
// SaveAccount saves new account in database
SaveAccount(ctx context.Context, a model.Account) error

View file

@ -45,7 +45,7 @@ func testDatabase(t *testing.T, dbFactory testDatabaseFactory) {
func testBookmarkAutoIncrement(t *testing.T, db DB) {
ctx := context.TODO()
book := model.Bookmark{
book := model.BookmarkDTO{
URL: "https://github.com/go-shiori/shiori",
Title: "shiori",
}
@ -54,7 +54,7 @@ func testBookmarkAutoIncrement(t *testing.T, db DB) {
assert.NoError(t, err, "Save bookmarks must not fail")
assert.Equal(t, 1, result[0].ID, "Saved bookmark must have ID %d", 1)
book = model.Bookmark{
book = model.BookmarkDTO{
URL: "https://github.com/go-shiori/obelisk",
Title: "obelisk",
}
@ -67,7 +67,7 @@ func testBookmarkAutoIncrement(t *testing.T, db DB) {
func testCreateBookmark(t *testing.T, db DB) {
ctx := context.TODO()
book := model.Bookmark{
book := model.BookmarkDTO{
URL: "https://github.com/go-shiori/obelisk",
Title: "shiori",
}
@ -81,7 +81,7 @@ func testCreateBookmark(t *testing.T, db DB) {
func testCreateBookmarkWithContent(t *testing.T, db DB) {
ctx := context.TODO()
book := model.Bookmark{
book := model.BookmarkDTO{
URL: "https://github.com/go-shiori/obelisk",
Title: "shiori",
Content: "Some content",
@ -106,7 +106,7 @@ func testCreateBookmarkWithContent(t *testing.T, db DB) {
func testCreateBookmarkWithTag(t *testing.T, db DB) {
ctx := context.TODO()
book := model.Bookmark{
book := model.BookmarkDTO{
URL: "https://github.com/go-shiori/obelisk",
Title: "shiori",
Tags: []model.Tag{
@ -126,7 +126,7 @@ func testCreateBookmarkWithTag(t *testing.T, db DB) {
func testCreateBookmarkTwice(t *testing.T, db DB) {
ctx := context.TODO()
book := model.Bookmark{
book := model.BookmarkDTO{
URL: "https://github.com/go-shiori/shiori",
Title: "shiori",
}
@ -144,7 +144,7 @@ func testCreateBookmarkTwice(t *testing.T, db DB) {
func testCreateTwoDifferentBookmarks(t *testing.T, db DB) {
ctx := context.TODO()
book := model.Bookmark{
book := model.BookmarkDTO{
URL: "https://github.com/go-shiori/shiori",
Title: "shiori",
}
@ -152,7 +152,7 @@ func testCreateTwoDifferentBookmarks(t *testing.T, db DB) {
_, err := db.SaveBookmarks(ctx, true, book)
assert.NoError(t, err, "Save first bookmark must not fail")
book = model.Bookmark{
book = model.BookmarkDTO{
URL: "https://github.com/go-shiori/go-readability",
Title: "go-readability",
}
@ -163,7 +163,7 @@ func testCreateTwoDifferentBookmarks(t *testing.T, db DB) {
func testUpdateBookmark(t *testing.T, db DB) {
ctx := context.TODO()
book := model.Bookmark{
book := model.BookmarkDTO{
URL: "https://github.com/go-shiori/shiori",
Title: "shiori",
}
@ -184,7 +184,7 @@ func testUpdateBookmark(t *testing.T, db DB) {
func testUpdateBookmarkWithContent(t *testing.T, db DB) {
ctx := context.TODO()
book := model.Bookmark{
book := model.BookmarkDTO{
URL: "https://github.com/go-shiori/obelisk",
Title: "shiori",
Content: "Some content",
@ -216,7 +216,7 @@ func testUpdateBookmarkWithContent(t *testing.T, db DB) {
func testGetBookmark(t *testing.T, db DB) {
ctx := context.TODO()
book := model.Bookmark{
book := model.BookmarkDTO{
URL: "https://github.com/go-shiori/shiori",
Title: "shiori",
}
@ -237,13 +237,13 @@ func testGetBookmarkNotExistent(t *testing.T, db DB) {
savedBookmark, exists, err := db.GetBookmark(ctx, 1, "")
assert.NoError(t, err, "Get bookmark should not fail")
assert.False(t, exists, "Bookmark should not exist")
assert.Equal(t, model.Bookmark{}, savedBookmark)
assert.Equal(t, model.BookmarkDTO{}, savedBookmark)
}
func testGetBookmarks(t *testing.T, db DB) {
ctx := context.TODO()
book := model.Bookmark{
book := model.BookmarkDTO{
URL: "https://github.com/go-shiori/shiori",
Title: "shiori",
}
@ -266,7 +266,7 @@ func testGetBookmarksWithSQLCharacters(t *testing.T, db DB) {
ctx := context.TODO()
// _ := 0
book := model.Bookmark{
book := model.BookmarkDTO{
URL: "https://github.com/go-shiori/shiori",
Title: "shiori",
}
@ -296,7 +296,7 @@ func testGetBookmarksCount(t *testing.T, db DB) {
ctx := context.TODO()
expectedCount := 1
book := model.Bookmark{
book := model.BookmarkDTO{
URL: "https://github.com/go-shiori/shiori",
Title: "shiori",
}

View file

@ -67,8 +67,8 @@ func (db *MySQLDatabase) Migrate() error {
// SaveBookmarks saves new or updated bookmarks to database.
// Returns the saved ID and error message if any happened.
func (db *MySQLDatabase) SaveBookmarks(ctx context.Context, create bool, bookmarks ...model.Bookmark) ([]model.Bookmark, error) {
var result []model.Bookmark
func (db *MySQLDatabase) SaveBookmarks(ctx context.Context, create bool, bookmarks ...model.BookmarkDTO) ([]model.BookmarkDTO, error) {
var result []model.BookmarkDTO
if err := db.withTx(ctx, func(tx *sqlx.Tx) error {
// Prepare statement
@ -218,7 +218,7 @@ func (db *MySQLDatabase) SaveBookmarks(ctx context.Context, create bool, bookmar
}
// GetBookmarks fetch list of bookmarks based on submitted options.
func (db *MySQLDatabase) GetBookmarks(ctx context.Context, opts GetBookmarksOptions) ([]model.Bookmark, error) {
func (db *MySQLDatabase) GetBookmarks(ctx context.Context, opts GetBookmarksOptions) ([]model.BookmarkDTO, error) {
// Create initial query
columns := []string{
`id`,
@ -330,7 +330,7 @@ func (db *MySQLDatabase) GetBookmarks(ctx context.Context, opts GetBookmarksOpti
}
// Fetch bookmarks
bookmarks := []model.Bookmark{}
bookmarks := []model.BookmarkDTO{}
err = db.Select(&bookmarks, query, args...)
if err != nil && err != sql.ErrNoRows {
return nil, errors.WithStack(err)
@ -502,7 +502,7 @@ func (db *MySQLDatabase) DeleteBookmarks(ctx context.Context, ids ...int) (err e
// GetBookmark fetches bookmark based on its ID or URL.
// Returns the bookmark and boolean whether it's exist or not.
func (db *MySQLDatabase) GetBookmark(ctx context.Context, id int, url string) (model.Bookmark, bool, error) {
func (db *MySQLDatabase) GetBookmark(ctx context.Context, id int, url string) (model.BookmarkDTO, bool, error) {
args := []interface{}{id}
query := `SELECT
id, url, title, excerpt, author, public,
@ -514,7 +514,7 @@ func (db *MySQLDatabase) GetBookmark(ctx context.Context, id int, url string) (m
args = append(args, url)
}
book := model.Bookmark{}
book := model.BookmarkDTO{}
if err := db.GetContext(ctx, &book, query, args...); err != nil && err != sql.ErrNoRows {
return book, false, errors.WithStack(err)
}

View file

@ -68,8 +68,8 @@ func (db *PGDatabase) Migrate() error {
// SaveBookmarks saves new or updated bookmarks to database.
// Returns the saved ID and error message if any happened.
func (db *PGDatabase) SaveBookmarks(ctx context.Context, create bool, bookmarks ...model.Bookmark) (result []model.Bookmark, err error) {
result = []model.Bookmark{}
func (db *PGDatabase) SaveBookmarks(ctx context.Context, create bool, bookmarks ...model.BookmarkDTO) (result []model.BookmarkDTO, err error) {
result = []model.BookmarkDTO{}
if err := db.withTx(ctx, func(tx *sqlx.Tx) error {
// Prepare statement
stmtInsertBook, err := tx.Preparex(`INSERT INTO bookmark
@ -120,7 +120,7 @@ func (db *PGDatabase) SaveBookmarks(ctx context.Context, create bool, bookmarks
modifiedTime := time.Now().UTC().Format(model.DatabaseDateFormat)
// Execute statements
result = []model.Bookmark{}
result = []model.BookmarkDTO{}
for _, book := range bookmarks {
// URL and title
if book.URL == "" {
@ -206,7 +206,7 @@ func (db *PGDatabase) SaveBookmarks(ctx context.Context, create bool, bookmarks
}
// GetBookmarks fetch list of bookmarks based on submitted options.
func (db *PGDatabase) GetBookmarks(ctx context.Context, opts GetBookmarksOptions) ([]model.Bookmark, error) {
func (db *PGDatabase) GetBookmarks(ctx context.Context, opts GetBookmarksOptions) ([]model.BookmarkDTO, error) {
// Create initial query
columns := []string{
`id`,
@ -325,7 +325,7 @@ func (db *PGDatabase) GetBookmarks(ctx context.Context, opts GetBookmarksOptions
query = db.Rebind(query)
// Fetch bookmarks
bookmarks := []model.Bookmark{}
bookmarks := []model.BookmarkDTO{}
err = db.SelectContext(ctx, &bookmarks, query, args...)
if err != nil && err != sql.ErrNoRows {
return nil, fmt.Errorf("failed to fetch data: %v", err)
@ -511,7 +511,7 @@ func (db *PGDatabase) DeleteBookmarks(ctx context.Context, ids ...int) (err erro
// GetBookmark fetches bookmark based on its ID or URL.
// Returns the bookmark and boolean whether it's exist or not.
func (db *PGDatabase) GetBookmark(ctx context.Context, id int, url string) (model.Bookmark, bool, error) {
func (db *PGDatabase) GetBookmark(ctx context.Context, id int, url string) (model.BookmarkDTO, bool, error) {
args := []interface{}{id}
query := `SELECT
id, url, title, excerpt, author, public,
@ -523,7 +523,7 @@ func (db *PGDatabase) GetBookmark(ctx context.Context, id int, url string) (mode
args = append(args, url)
}
book := model.Bookmark{}
book := model.BookmarkDTO{}
if err := db.GetContext(ctx, &book, query, args...); err != nil && err != sql.ErrNoRows {
return book, false, errors.WithStack(err)
}

View file

@ -76,8 +76,8 @@ func (db *SQLiteDatabase) Migrate() error {
// SaveBookmarks saves new or updated bookmarks to database.
// Returns the saved ID and error message if any happened.
func (db *SQLiteDatabase) SaveBookmarks(ctx context.Context, create bool, bookmarks ...model.Bookmark) ([]model.Bookmark, error) {
var result []model.Bookmark
func (db *SQLiteDatabase) SaveBookmarks(ctx context.Context, create bool, bookmarks ...model.BookmarkDTO) ([]model.BookmarkDTO, error) {
var result []model.BookmarkDTO
if err := db.withTx(ctx, func(tx *sqlx.Tx) error {
// Prepare statement
@ -245,7 +245,7 @@ func (db *SQLiteDatabase) SaveBookmarks(ctx context.Context, create bool, bookma
}
// GetBookmarks fetch list of bookmarks based on submitted options.
func (db *SQLiteDatabase) GetBookmarks(ctx context.Context, opts GetBookmarksOptions) ([]model.Bookmark, error) {
func (db *SQLiteDatabase) GetBookmarks(ctx context.Context, opts GetBookmarksOptions) ([]model.BookmarkDTO, error) {
// Create initial query
query := `SELECT
b.id,
@ -362,7 +362,7 @@ func (db *SQLiteDatabase) GetBookmarks(ctx context.Context, opts GetBookmarksOpt
}
// Fetch bookmarks
bookmarks := []model.Bookmark{}
bookmarks := []model.BookmarkDTO{}
err = db.SelectContext(ctx, &bookmarks, query, args...)
if err != nil && err != sql.ErrNoRows {
return nil, errors.WithStack(err)
@ -622,7 +622,7 @@ func (db *SQLiteDatabase) DeleteBookmarks(ctx context.Context, ids ...int) error
// GetBookmark fetches bookmark based on its ID or URL.
// Returns the bookmark and boolean whether it's exist or not.
func (db *SQLiteDatabase) GetBookmark(ctx context.Context, id int, url string) (model.Bookmark, bool, error) {
func (db *SQLiteDatabase) GetBookmark(ctx context.Context, id int, url string) (model.BookmarkDTO, bool, error) {
args := []interface{}{id}
query := `SELECT
b.id, b.url, b.title, b.excerpt, b.author, b.public, b.modified,
@ -636,7 +636,7 @@ func (db *SQLiteDatabase) GetBookmark(ctx context.Context, id int, url string) (
args = append(args, url)
}
book := model.Bookmark{}
book := model.BookmarkDTO{}
if err := db.GetContext(ctx, &book, query, args...); err != nil && err != sql.ErrNoRows {
return book, false, errors.WithStack(err)
}

View file

@ -50,7 +50,7 @@ func testSqliteGetBookmarksWithDash(t *testing.T) {
db, err := sqliteTestDatabaseFactory(ctx)
assert.NoError(t, err)
book := model.Bookmark{
book := model.BookmarkDTO{
URL: "https://github.com/go-shiori/shiori",
Title: "shiori",
}
@ -58,7 +58,7 @@ func testSqliteGetBookmarksWithDash(t *testing.T) {
_, err = db.SaveBookmarks(ctx, true, book)
assert.NoError(t, err, "Save bookmarks must not fail")
book = model.Bookmark{
book = model.BookmarkDTO{
URL: "https://github.com/jamiehannaford/what-happens-when-k8s",
Title: "what-happens-when-k8s",
}

View file

@ -0,0 +1,31 @@
package dependencies
import (
"github.com/go-shiori/shiori/internal/config"
"github.com/go-shiori/shiori/internal/database"
"github.com/go-shiori/shiori/internal/model"
"github.com/sirupsen/logrus"
)
type Domains struct {
Archiver model.ArchiverDomain
Auth model.AccountsDomain
Bookmarks model.BookmarksDomain
Storage model.StorageDomain
}
type Dependencies struct {
Log *logrus.Logger
Database database.DB
Config *config.Config
Domains *Domains
}
func NewDependencies(log *logrus.Logger, db database.DB, cfg *config.Config) *Dependencies {
return &Dependencies{
Log: log,
Config: cfg,
Database: db,
Domains: &Domains{},
}
}

View file

@ -5,18 +5,15 @@ import (
"fmt"
"time"
"github.com/go-shiori/shiori/internal/database"
"github.com/go-shiori/shiori/internal/dependencies"
"github.com/go-shiori/shiori/internal/model"
"github.com/golang-jwt/jwt/v5"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/crypto/bcrypt"
)
type AccountsDomain struct {
logger *logrus.Logger
db database.DB
secret []byte
deps *dependencies.Dependencies
}
type JWTClaim struct {
@ -32,7 +29,7 @@ func (d *AccountsDomain) CheckToken(ctx context.Context, userJWT string) (*model
return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"])
}
return d.secret, nil
return d.deps.Config.Http.SecretKey, nil
})
if err != nil {
return nil, errors.Wrap(err, "error parsing token")
@ -52,7 +49,7 @@ func (d *AccountsDomain) CheckToken(ctx context.Context, userJWT string) (*model
}
func (d *AccountsDomain) GetAccountFromCredentials(ctx context.Context, username, password string) (*model.Account, error) {
account, _, err := d.db.GetAccount(ctx, username)
account, _, err := d.deps.Database.GetAccount(ctx, username)
if err != nil {
return nil, fmt.Errorf("username and password do not match")
}
@ -72,18 +69,16 @@ func (d *AccountsDomain) CreateTokenForAccount(account *model.Account, expiratio
token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
t, err := token.SignedString(d.secret)
t, err := token.SignedString(d.deps.Config.Http.SecretKey)
if err != nil {
d.logger.WithError(err).Error("error signing token")
d.deps.Log.WithError(err).Error("error signing token")
}
return t, err
}
func NewAccountsDomain(logger *logrus.Logger, secretKey string, db database.DB) AccountsDomain {
return AccountsDomain{
logger: logger,
db: db,
secret: []byte(secretKey),
func NewAccountsDomain(deps *dependencies.Dependencies) *AccountsDomain {
return &AccountsDomain{
deps: deps,
}
}

View file

@ -2,34 +2,32 @@ package domains
import (
"fmt"
"os"
"path/filepath"
"strconv"
"github.com/go-shiori/shiori/internal/core"
"github.com/go-shiori/shiori/internal/dependencies"
"github.com/go-shiori/shiori/internal/model"
"github.com/sirupsen/logrus"
"github.com/go-shiori/warc"
)
type ArchiverDomain struct {
dataDir string
logger *logrus.Logger
deps *dependencies.Dependencies
}
func (d *ArchiverDomain) DownloadBookmarkArchive(book model.Bookmark) (*model.Bookmark, error) {
func (d *ArchiverDomain) DownloadBookmarkArchive(book model.BookmarkDTO) (*model.BookmarkDTO, error) {
content, contentType, err := core.DownloadBookmark(book.URL)
if err != nil {
return nil, fmt.Errorf("error downloading url: %s", err)
}
processRequest := core.ProcessRequest{
DataDir: d.dataDir,
DataDir: d.deps.Config.Storage.DataDir,
Bookmark: book,
Content: content,
ContentType: contentType,
}
result, isFatalErr, err := core.ProcessBookmark(processRequest)
result, isFatalErr, err := core.ProcessBookmark(d.deps, processRequest)
content.Close()
if err != nil && isFatalErr {
@ -39,20 +37,19 @@ func (d *ArchiverDomain) DownloadBookmarkArchive(book model.Bookmark) (*model.Bo
return &result, nil
}
func (d *ArchiverDomain) GetBookmarkArchive(book model.Bookmark) error {
archivePath := filepath.Join(d.dataDir, "archive", strconv.Itoa(book.ID))
func (d *ArchiverDomain) GetBookmarkArchive(book *model.BookmarkDTO) (*warc.Archive, error) {
archivePath := model.GetArchivePath(book)
info, err := os.Stat(archivePath)
if !os.IsNotExist(err) && !info.IsDir() {
return fmt.Errorf("archive not found")
if !d.deps.Domains.Storage.FileExists(archivePath) {
return nil, fmt.Errorf("archive for bookmark %d doesn't exist", book.ID)
}
return nil
// FIXME: This only works in local filesystem
return warc.Open(filepath.Join(d.deps.Config.Storage.DataDir, archivePath))
}
func NewArchiverDomain(logger *logrus.Logger, dataDir string) ArchiverDomain {
return ArchiverDomain{
dataDir: dataDir,
logger: logger,
func NewArchiverDomain(deps *dependencies.Dependencies) *ArchiverDomain {
return &ArchiverDomain{
deps: deps,
}
}

View file

@ -0,0 +1,51 @@
package domains
import (
"context"
"fmt"
"github.com/go-shiori/shiori/internal/dependencies"
"github.com/go-shiori/shiori/internal/model"
)
type BookmarksDomain struct {
deps *dependencies.Dependencies
}
func (d *BookmarksDomain) HasEbook(b *model.BookmarkDTO) bool {
ebookPath := model.GetEbookPath(b)
return d.deps.Domains.Storage.FileExists(ebookPath)
}
func (d *BookmarksDomain) HasArchive(b *model.BookmarkDTO) bool {
archivePath := model.GetArchivePath(b)
return d.deps.Domains.Storage.FileExists(archivePath)
}
func (d *BookmarksDomain) HasThumbnail(b *model.BookmarkDTO) bool {
thumbnailPath := model.GetThumbnailPath(b)
return d.deps.Domains.Storage.FileExists(thumbnailPath)
}
func (d *BookmarksDomain) GetBookmark(ctx context.Context, id model.DBID) (*model.BookmarkDTO, error) {
bookmark, exists, err := d.deps.Database.GetBookmark(ctx, int(id), "")
if err != nil {
return nil, fmt.Errorf("failed to get bookmark: %w", err)
}
if !exists {
return nil, model.ErrBookmarkNotFound
}
// Check if it has ebook and archive.
bookmark.HasEbook = d.HasEbook(&bookmark)
bookmark.HasArchive = d.HasArchive(&bookmark)
return &bookmark, nil
}
func NewBookmarksDomain(deps *dependencies.Dependencies) *BookmarksDomain {
return &BookmarksDomain{
deps: deps,
}
}

View file

@ -0,0 +1,82 @@
package domains_test
import (
"context"
"testing"
"github.com/go-shiori/shiori/internal/config"
"github.com/go-shiori/shiori/internal/database"
"github.com/go-shiori/shiori/internal/dependencies"
"github.com/go-shiori/shiori/internal/domains"
"github.com/go-shiori/shiori/internal/model"
"github.com/go-shiori/shiori/internal/testutil"
"github.com/sirupsen/logrus"
"github.com/spf13/afero"
"github.com/stretchr/testify/require"
)
func TestBookmarkDomain(t *testing.T) {
fs := afero.NewMemMapFs()
db, err := database.OpenSQLiteDatabase(context.TODO(), ":memory:")
require.NoError(t, err)
require.NoError(t, db.Migrate())
deps := &dependencies.Dependencies{
Database: db,
Config: config.ParseServerConfiguration(context.TODO(), logrus.New()),
Log: logrus.New(),
Domains: &dependencies.Domains{},
}
deps.Domains.Storage = domains.NewStorageDomain(deps, fs)
fs.MkdirAll("thumb", 0755)
fs.Create("thumb/1")
fs.MkdirAll("ebook", 0755)
fs.Create("ebook/1.epub")
fs.MkdirAll("archive", 0755)
// TODO: write a valid archive file
fs.Create("archive/1")
domain := domains.NewBookmarksDomain(deps)
t.Run("HasEbook", func(t *testing.T) {
t.Run("Yes", func(t *testing.T) {
require.True(t, domain.HasEbook(&model.BookmarkDTO{ID: 1}))
})
t.Run("No", func(t *testing.T) {
require.False(t, domain.HasEbook(&model.BookmarkDTO{ID: 2}))
})
})
t.Run("HasArchive", func(t *testing.T) {
t.Run("Yes", func(t *testing.T) {
require.True(t, domain.HasArchive(&model.BookmarkDTO{ID: 1}))
})
t.Run("No", func(t *testing.T) {
require.False(t, domain.HasArchive(&model.BookmarkDTO{ID: 2}))
})
})
t.Run("HasThumbnail", func(t *testing.T) {
t.Run("Yes", func(t *testing.T) {
require.True(t, domain.HasThumbnail(&model.BookmarkDTO{ID: 1}))
})
t.Run("No", func(t *testing.T) {
require.False(t, domain.HasThumbnail(&model.BookmarkDTO{ID: 2}))
})
})
t.Run("GetBookmark", func(t *testing.T) {
t.Run("Success", func(t *testing.T) {
_, err := deps.Database.SaveBookmarks(context.TODO(), true, *testutil.GetValidBookmark())
require.NoError(t, err)
bookmark, err := domain.GetBookmark(context.Background(), 1)
require.NoError(t, err)
require.Equal(t, 1, bookmark.ID)
// Check DTO attributes
require.True(t, bookmark.HasEbook)
require.True(t, bookmark.HasArchive)
})
})
}

View file

@ -0,0 +1,99 @@
package domains
import (
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"github.com/go-shiori/shiori/internal/dependencies"
"github.com/go-shiori/shiori/internal/model"
"github.com/spf13/afero"
)
type StorageDomain struct {
deps *dependencies.Dependencies
fs afero.Fs
}
func NewStorageDomain(deps *dependencies.Dependencies, fs afero.Fs) *StorageDomain {
return &StorageDomain{
deps: deps,
fs: fs,
}
}
// Stat returns the FileInfo structure describing file.
func (d *StorageDomain) Stat(name string) (fs.FileInfo, error) {
return d.fs.Stat(name)
}
// FS returns the filesystem used by this domain.
func (d *StorageDomain) FS() afero.Fs {
return d.fs
}
// FileExists checks if a file exists in storage.
func (d *StorageDomain) FileExists(name string) bool {
info, err := d.Stat(name)
return err == nil && !info.IsDir()
}
// DirExists checks if a directory exists in storage.
func (d *StorageDomain) DirExists(name string) bool {
info, err := d.Stat(name)
return err == nil && info.IsDir()
}
// WriteData writes bytes data to a file in storage.
// CAUTION: This function will overwrite existing file.
func (d *StorageDomain) WriteData(dst string, data []byte) error {
// Create directory if not exist
dir := filepath.Dir(dst)
if !d.DirExists(dir) {
err := d.fs.MkdirAll(dir, os.ModePerm)
if err != nil {
return err
}
}
// Create file
file, err := d.fs.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.ModePerm)
if err != nil {
return err
}
defer file.Close()
// Write data
_, err = file.Write(data)
return err
}
// WriteFile writes a file to storage.
func (d *StorageDomain) WriteFile(dst string, tmpFile *os.File) error {
if dst != "" && !d.DirExists(dst) {
err := d.fs.MkdirAll(filepath.Dir(dst), model.DataDirPerm)
if err != nil {
return fmt.Errorf("failed to create destination dir: %v", err)
}
}
dstFile, err := d.fs.Create(dst)
if err != nil {
return fmt.Errorf("failed to create destination file: %v", err)
}
defer dstFile.Close()
_, err = tmpFile.Seek(0, io.SeekStart)
if err != nil {
return fmt.Errorf("failed to rewind temporary file: %v", err)
}
_, err = io.Copy(dstFile, tmpFile)
if err != nil {
return fmt.Errorf("failed to copy file to the destination")
}
return nil
}

View file

@ -0,0 +1,105 @@
package domains_test
import (
"context"
"os"
"testing"
"github.com/go-shiori/shiori/internal/config"
"github.com/go-shiori/shiori/internal/dependencies"
"github.com/go-shiori/shiori/internal/domains"
"github.com/sirupsen/logrus"
"github.com/spf13/afero"
"github.com/stretchr/testify/require"
)
func TestDirExists(t *testing.T) {
fs := afero.NewMemMapFs()
fs.MkdirAll("foo", 0755)
domain := domains.NewStorageDomain(
&dependencies.Dependencies{
Config: config.ParseServerConfiguration(context.TODO(), logrus.New()),
Log: logrus.New(),
},
fs,
)
require.True(t, domain.DirExists("foo"))
require.False(t, domain.DirExists("foo/file"))
require.False(t, domain.DirExists("bar"))
}
func TestFileExists(t *testing.T) {
fs := afero.NewMemMapFs()
fs.MkdirAll("foo", 0755)
fs.Create("foo/file")
domain := domains.NewStorageDomain(
&dependencies.Dependencies{
Config: config.ParseServerConfiguration(context.TODO(), logrus.New()),
Log: logrus.New(),
},
fs,
)
require.True(t, domain.FileExists("foo/file"))
require.False(t, domain.FileExists("bar"))
}
func TestWriteFile(t *testing.T) {
fs := afero.NewMemMapFs()
domain := domains.NewStorageDomain(
&dependencies.Dependencies{
Config: config.ParseServerConfiguration(context.TODO(), logrus.New()),
Log: logrus.New(),
},
fs,
)
err := domain.WriteData("foo/file.ext", []byte("foo"))
require.NoError(t, err)
require.True(t, domain.FileExists("foo/file.ext"))
require.True(t, domain.DirExists("foo"))
handler, err := domain.FS().Open("foo/file.ext")
require.NoError(t, err)
defer handler.Close()
data, err := afero.ReadAll(handler)
require.NoError(t, err)
require.Equal(t, "foo", string(data))
}
func TestSaveFile(t *testing.T) {
fs := afero.NewMemMapFs()
domain := domains.NewStorageDomain(
&dependencies.Dependencies{
Config: config.ParseServerConfiguration(context.TODO(), logrus.New()),
Log: logrus.New(),
},
fs,
)
tempFile, err := os.CreateTemp("", "")
require.NoError(t, err)
defer os.Remove(tempFile.Name())
_, err = tempFile.WriteString("foo")
require.NoError(t, err)
err = domain.WriteFile("foo/file.ext", tempFile)
require.NoError(t, err)
require.True(t, domain.FileExists("foo/file.ext"))
require.True(t, domain.DirExists("foo"))
handler, err := domain.FS().Open("foo/file.ext")
require.NoError(t, err)
defer handler.Close()
data, err := afero.ReadAll(handler)
require.NoError(t, err)
require.Equal(t, "foo", string(data))
}

View file

@ -5,7 +5,7 @@ import (
"strings"
"github.com/gin-gonic/gin"
"github.com/go-shiori/shiori/internal/config"
"github.com/go-shiori/shiori/internal/dependencies"
"github.com/go-shiori/shiori/internal/http/context"
"github.com/go-shiori/shiori/internal/http/response"
"github.com/go-shiori/shiori/internal/model"
@ -14,7 +14,7 @@ import (
// AuthMiddleware provides basic authentication capabilities to all routes underneath
// its usage, only allowing authenticated users access and set a custom local context
// `account` with the account model for the logged in user.
func AuthMiddleware(deps *config.Dependencies) gin.HandlerFunc {
func AuthMiddleware(deps *dependencies.Dependencies) gin.HandlerFunc {
return func(c *gin.Context) {
token := getTokenFromHeader(c)
if token == "" {

View file

@ -0,0 +1,42 @@
package response
import (
"fmt"
"io"
"net/http"
"github.com/gin-gonic/gin"
"github.com/go-shiori/shiori/internal/model"
)
// SendFile sends file to client with caching header
func SendFile(c *gin.Context, storageDomain model.StorageDomain, path string) {
c.Header("Cache-Control", "public, max-age=86400")
if !storageDomain.FileExists(path) {
c.AbortWithStatus(http.StatusNotFound)
return
}
info, err := storageDomain.Stat(path)
if err != nil {
c.AbortWithStatus(http.StatusInternalServerError)
return
}
c.Header("ETag", fmt.Sprintf("W/%x-%x", info.ModTime().Unix(), info.Size()))
// TODO: Find a better way to send the file to the client from the FS, probably making a
// conversion between afero.Fs and http.FileSystem to use c.FileFromFS.
fileContent, err := storageDomain.FS().Open(path)
if err != nil {
c.AbortWithStatus(http.StatusInternalServerError)
return
}
_, err = io.Copy(c.Writer, fileContent)
if err != nil {
c.AbortWithStatus(http.StatusInternalServerError)
return
}
}

View file

@ -32,3 +32,12 @@ func SendErrorWithParams(ctx *gin.Context, statusCode int, data interface{}, err
func SendInternalServerError(ctx *gin.Context) {
SendError(ctx, http.StatusInternalServerError, internalServerErrorMessage)
}
// SendNotFound directly sends a not found response
func RedirectToLogin(ctx *gin.Context, dst string) {
ctx.Redirect(http.StatusFound, "/login?dst="+dst)
}
func NotFound(ctx *gin.Context) {
ctx.AbortWithStatus(http.StatusNotFound)
}

View file

@ -2,7 +2,7 @@ package api_v1
import (
"github.com/gin-gonic/gin"
"github.com/go-shiori/shiori/internal/config"
"github.com/go-shiori/shiori/internal/dependencies"
"github.com/go-shiori/shiori/internal/http/middleware"
"github.com/go-shiori/shiori/internal/model"
"github.com/sirupsen/logrus"
@ -10,7 +10,7 @@ import (
type APIRoutes struct {
logger *logrus.Logger
deps *config.Dependencies
deps *dependencies.Dependencies
loginHandler model.LegacyLoginHandler
}
@ -31,7 +31,7 @@ func (s *APIRoutes) handle(g *gin.RouterGroup, path string, routes model.Routes)
routes.Setup(group)
}
func NewAPIRoutes(logger *logrus.Logger, deps *config.Dependencies, loginHandler model.LegacyLoginHandler) *APIRoutes {
func NewAPIRoutes(logger *logrus.Logger, deps *dependencies.Dependencies, loginHandler model.LegacyLoginHandler) *APIRoutes {
return &APIRoutes{
logger: logger,
deps: deps,

View file

@ -6,7 +6,7 @@ import (
"time"
"github.com/gin-gonic/gin"
"github.com/go-shiori/shiori/internal/config"
"github.com/go-shiori/shiori/internal/dependencies"
"github.com/go-shiori/shiori/internal/http/context"
"github.com/go-shiori/shiori/internal/http/response"
"github.com/go-shiori/shiori/internal/model"
@ -15,7 +15,7 @@ import (
type AuthAPIRoutes struct {
logger *logrus.Logger
deps *config.Dependencies
deps *dependencies.Dependencies
legacyLoginHandler model.LegacyLoginHandler
}
@ -189,7 +189,7 @@ func (r *AuthAPIRoutes) settingsHandler(c *gin.Context) {
response.Send(c, http.StatusOK, ctx.GetAccount())
}
func NewAuthAPIRoutes(logger *logrus.Logger, deps *config.Dependencies, loginHandler model.LegacyLoginHandler) *AuthAPIRoutes {
func NewAuthAPIRoutes(logger *logrus.Logger, deps *dependencies.Dependencies, loginHandler model.LegacyLoginHandler) *AuthAPIRoutes {
return &AuthAPIRoutes{
logger: logger,
deps: deps,

View file

@ -10,9 +10,9 @@ import (
"sync"
"github.com/gin-gonic/gin"
"github.com/go-shiori/shiori/internal/config"
"github.com/go-shiori/shiori/internal/core"
"github.com/go-shiori/shiori/internal/database"
"github.com/go-shiori/shiori/internal/dependencies"
"github.com/go-shiori/shiori/internal/http/context"
"github.com/go-shiori/shiori/internal/http/response"
"github.com/go-shiori/shiori/internal/model"
@ -21,7 +21,7 @@ import (
type BookmarksAPIRoutes struct {
logger *logrus.Logger
deps *config.Dependencies
deps *dependencies.Dependencies
}
func (r *BookmarksAPIRoutes) Setup(g *gin.RouterGroup) model.Routes {
@ -32,6 +32,13 @@ func (r *BookmarksAPIRoutes) Setup(g *gin.RouterGroup) model.Routes {
return r
}
func NewBookmarksPIRoutes(logger *logrus.Logger, deps *dependencies.Dependencies) *BookmarksAPIRoutes {
return &BookmarksAPIRoutes{
logger: logger,
deps: deps,
}
}
type updateCachePayload struct {
Ids []int `json:"ids" validate:"required"`
KeepMetadata bool `json:"keep_metadata"`
@ -73,8 +80,8 @@ type apiCreateBookmarkPayload struct {
Async bool `json:"async"`
}
func (payload *apiCreateBookmarkPayload) ToBookmark() (*model.Bookmark, error) {
bookmark := &model.Bookmark{
func (payload *apiCreateBookmarkPayload) ToBookmark() (*model.BookmarkDTO, error) {
bookmark := &model.BookmarkDTO{
URL: payload.URL,
Title: payload.Title,
Excerpt: payload.Excerpt,
@ -188,13 +195,6 @@ func (r *BookmarksAPIRoutes) deleteHandler(c *gin.Context) {
response.Send(c, 200, "Bookmark deleted")
}
func NewBookmarksPIRoutes(logger *logrus.Logger, deps *config.Dependencies) *BookmarksAPIRoutes {
return &BookmarksAPIRoutes{
logger: logger,
deps: deps,
}
}
// updateCache godoc
//
// @Summary Update Cache and Ebook on server.
@ -212,10 +212,6 @@ func (r *BookmarksAPIRoutes) updateCache(c *gin.Context) {
return
}
// Get server config
logger := logrus.New()
cfg := config.ParseServerConfiguration(ctx, logger)
var payload updateCachePayload
if err := c.ShouldBindJSON(&payload); err != nil {
response.SendInternalServerError(c)
@ -261,7 +257,7 @@ func (r *BookmarksAPIRoutes) updateCache(c *gin.Context) {
book.CreateArchive = payload.CreateArchive
book.CreateEbook = payload.CreateEbook
go func(i int, book model.Bookmark, keep_metadata bool) {
go func(i int, book model.BookmarkDTO, keep_metadata bool) {
// Make sure to finish the WG
defer wg.Done()
@ -279,7 +275,7 @@ func (r *BookmarksAPIRoutes) updateCache(c *gin.Context) {
}
request := core.ProcessRequest{
DataDir: cfg.Storage.DataDir,
DataDir: r.deps.Config.Storage.DataDir,
Bookmark: book,
Content: content,
ContentType: contentType,
@ -297,7 +293,7 @@ func (r *BookmarksAPIRoutes) updateCache(c *gin.Context) {
}
}
book, _, err = core.ProcessBookmark(request)
book, _, err = core.ProcessBookmark(r.deps, request)
content.Close()
if err != nil {

View file

@ -4,7 +4,7 @@ import (
"net/http"
"github.com/gin-gonic/gin"
"github.com/go-shiori/shiori/internal/config"
"github.com/go-shiori/shiori/internal/dependencies"
"github.com/go-shiori/shiori/internal/http/response"
"github.com/go-shiori/shiori/internal/model"
"github.com/sirupsen/logrus"
@ -12,7 +12,7 @@ import (
type TagsAPIRoutes struct {
logger *logrus.Logger
deps *config.Dependencies
deps *dependencies.Dependencies
}
func (r *TagsAPIRoutes) Setup(g *gin.RouterGroup) model.Routes {
@ -62,7 +62,7 @@ func (r *TagsAPIRoutes) createHandler(c *gin.Context) {
response.Send(c, http.StatusCreated, nil)
}
func NewTagsPIRoutes(logger *logrus.Logger, deps *config.Dependencies) *TagsAPIRoutes {
func NewTagsPIRoutes(logger *logrus.Logger, deps *dependencies.Dependencies) *TagsAPIRoutes {
return &TagsAPIRoutes{
logger: logger,
deps: deps,

View file

@ -1,118 +1,189 @@
package routes
import (
"fmt"
"html/template"
"net/http"
"strconv"
fp "path/filepath"
"strings"
"github.com/gin-gonic/gin"
"github.com/go-shiori/shiori/internal/config"
"github.com/go-shiori/shiori/internal/dependencies"
"github.com/go-shiori/shiori/internal/http/context"
"github.com/go-shiori/shiori/internal/http/response"
"github.com/go-shiori/shiori/internal/model"
ws "github.com/go-shiori/shiori/internal/webserver"
"github.com/gofrs/uuid/v5"
"github.com/sirupsen/logrus"
)
type BookmarkRoutes struct {
logger *logrus.Logger
deps *config.Dependencies
deps *dependencies.Dependencies
}
func (r *BookmarkRoutes) Setup(group *gin.RouterGroup) model.Routes {
//group.GET("/:id/archive", r.bookmarkArchiveHandler)
//group.GET("/:id/content", r.bookmarkContentHandler)
group.GET("/:id/ebook", r.bookmarkEbookHandler)
return r
}
// func (r *BookmarkRoutes) bookmarkContentHandler(c *gin.Context) {
// ctx := context.NewContextFromGin(c)
// bookmarkIDParam, present := c.Params.Get("id")
// if !present {
// response.SendError(c, 400, "Invalid bookmark ID")
// return
// }
// bookmarkID, err := strconv.Atoi(bookmarkIDParam)
// if err != nil {
// r.logger.WithError(err).Error("error parsing bookmark ID parameter")
// response.SendInternalServerError(c)
// return
// }
// if bookmarkID == 0 {
// response.SendError(c, 404, nil)
// return
// }
// bookmark, found, err := r.deps.Database.GetBookmark(c, bookmarkID, "")
// if err != nil || !found {
// response.SendError(c, 404, nil)
// return
// }
// if bookmark.Public != 1 && !ctx.UserIsLogged() {
// response.SendError(c, http.StatusForbidden, nil)
// return
// }
// response.Send(c, 200, bookmark.Content)
// }
// func (r *BookmarkRoutes) bookmarkArchiveHandler(c *gin.Context) {}
func NewBookmarkRoutes(logger *logrus.Logger, deps *config.Dependencies) *BookmarkRoutes {
func NewBookmarkRoutes(logger *logrus.Logger, deps *dependencies.Dependencies) *BookmarkRoutes {
return &BookmarkRoutes{
logger: logger,
deps: deps,
}
}
func (r *BookmarkRoutes) bookmarkEbookHandler(c *gin.Context) {
ctx := context.NewContextFromGin(c)
func (r *BookmarkRoutes) Setup(group *gin.RouterGroup) model.Routes {
group.GET("/:id/archive", r.bookmarkArchiveHandler)
group.GET("/:id/archive/file/*filepath", r.bookmarkArchiveFileHandler)
group.GET("/:id/content", r.bookmarkContentHandler)
group.GET("/:id/thumb", r.bookmarkThumbnailHandler)
group.GET("/:id/ebook", r.bookmarkEbookHandler)
// Get server config
logger := logrus.New()
cfg := config.ParseServerConfiguration(ctx, logger)
DataDir := cfg.Storage.DataDir
return r
}
func (r *BookmarkRoutes) getBookmark(c *context.Context) (*model.BookmarkDTO, error) {
bookmarkIDParam, present := c.Params.Get("id")
if !present {
response.SendError(c, http.StatusBadRequest, "Invalid bookmark ID")
return
response.SendError(c.Context, http.StatusBadRequest, "Invalid bookmark ID")
return nil, model.ErrBookmarkInvalidID
}
bookmarkID, err := strconv.Atoi(bookmarkIDParam)
if err != nil {
r.logger.WithError(err).Error("error parsing bookmark ID parameter")
response.SendInternalServerError(c.Context)
return nil, err
}
if bookmarkID == 0 {
response.SendError(c.Context, http.StatusNotFound, nil)
return nil, model.ErrBookmarkNotFound
}
bookmark, err := r.deps.Domains.Bookmarks.GetBookmark(c.Context, model.DBID(bookmarkID))
if err != nil {
response.SendError(c.Context, http.StatusNotFound, nil)
return nil, model.ErrBookmarkNotFound
}
if bookmark.Public != 1 && !c.UserIsLogged() {
response.RedirectToLogin(c.Context, c.Request.URL.String())
return nil, model.ErrUnauthorized
}
return bookmark, nil
}
func (r *BookmarkRoutes) bookmarkContentHandler(c *gin.Context) {
ctx := context.NewContextFromGin(c)
bookmark, err := r.getBookmark(ctx)
if err != nil {
return
}
ctx.HTML(http.StatusOK, "content.html", gin.H{
"RootPath": r.deps.Config.Http.RootPath,
"Version": model.BuildVersion,
"Book": bookmark,
"HTML": template.HTML(bookmark.HTML),
})
}
func (r *BookmarkRoutes) bookmarkArchiveHandler(c *gin.Context) {
ctx := context.NewContextFromGin(c)
bookmark, err := r.getBookmark(ctx)
if err != nil {
return
}
if !r.deps.Domains.Bookmarks.HasArchive(bookmark) {
response.NotFound(c)
return
}
c.HTML(http.StatusOK, "archive.html", gin.H{
"RootPath": r.deps.Config.Http.RootPath,
"Version": model.BuildVersion,
"Book": bookmark,
})
}
func (r *BookmarkRoutes) bookmarkArchiveFileHandler(c *gin.Context) {
ctx := context.NewContextFromGin(c)
bookmark, err := r.getBookmark(ctx)
if err != nil {
return
}
if !r.deps.Domains.Bookmarks.HasArchive(bookmark) {
response.NotFound(c)
return
}
resourcePath, _ := c.Params.Get("filepath")
resourcePath = strings.TrimPrefix(resourcePath, "/")
archive, err := r.deps.Domains.Archiver.GetBookmarkArchive(bookmark)
if err != nil {
r.logger.WithError(err).Error("error opening archive")
response.SendInternalServerError(c)
return
}
defer archive.Close()
if !archive.HasResource(resourcePath) {
response.NotFound(c)
return
}
content, resourceContentType, err := archive.Read(resourcePath)
if err != nil {
r.logger.WithError(err).Error("error reading archive file")
response.SendInternalServerError(c)
return
}
if bookmarkID == 0 {
response.SendError(c, http.StatusNotFound, nil)
return
}
// Generate weak ETAG
shioriUUID := uuid.NewV5(uuid.NamespaceURL, model.ShioriURLNamespace)
c.Header("Etag", fmt.Sprintf("W/%s", uuid.NewV5(shioriUUID, fmt.Sprintf("%x-%x-%x", bookmark.ID, resourcePath, len(content)))))
c.Header("Cache-Control", "max-age=31536000")
bookmark, found, err := r.deps.Database.GetBookmark(c, bookmarkID, "")
if err != nil || !found {
response.SendError(c, http.StatusNotFound, nil)
return
}
if bookmark.Public != 1 && !ctx.UserIsLogged() {
response.SendError(c, http.StatusUnauthorized, nil)
return
}
ebookPath := fp.Join(DataDir, "ebook", bookmarkIDParam+".epub")
if !ws.FileExists(ebookPath) {
response.SendError(c, http.StatusNotFound, nil)
return
}
filename := bookmark.Title + ".epub"
c.FileAttachment(ebookPath, filename)
c.Header("Content-Encoding", "gzip")
c.Data(http.StatusOK, resourceContentType, content)
}
func (r *BookmarkRoutes) bookmarkThumbnailHandler(c *gin.Context) {
ctx := context.NewContextFromGin(c)
bookmark, err := r.getBookmark(ctx)
if err != nil {
return
}
if !r.deps.Domains.Bookmarks.HasThumbnail(bookmark) {
response.NotFound(c)
return
}
response.SendFile(c, r.deps.Domains.Storage, model.GetThumbnailPath(bookmark))
}
func (r *BookmarkRoutes) bookmarkEbookHandler(c *gin.Context) {
ctx := context.NewContextFromGin(c)
bookmark, err := r.getBookmark(ctx)
if err != nil {
return
}
ebookPath := model.GetEbookPath(bookmark)
if !r.deps.Domains.Storage.FileExists(ebookPath) {
response.SendError(c, http.StatusNotFound, nil)
return
}
// TODO: Potentially improve this
c.Header("Content-Disposition", `attachment; filename="`+bookmark.Title+`.epub"`)
response.SendFile(c, r.deps.Domains.Storage, model.GetEbookPath(bookmark))
}

View file

@ -0,0 +1,223 @@
package routes
import (
"context"
"net/http"
"net/http/httptest"
"strconv"
"testing"
"github.com/gin-gonic/gin"
sctx "github.com/go-shiori/shiori/internal/http/context"
"github.com/go-shiori/shiori/internal/http/templates"
"github.com/go-shiori/shiori/internal/model"
"github.com/go-shiori/shiori/internal/testutil"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/require"
)
func TestBookmarkRoutesGetBookmark(t *testing.T) {
logger := logrus.New()
_, deps := testutil.GetTestConfigurationAndDependencies(t, context.TODO(), logger)
g := gin.Default()
templates.SetupTemplates(g)
w := httptest.NewRecorder()
// Create a private and a public bookmark to use in tests
publicBookmark := testutil.GetValidBookmark()
publicBookmark.Public = 1
bookmarks, err := deps.Database.SaveBookmarks(context.TODO(), true, []model.BookmarkDTO{
*testutil.GetValidBookmark(),
*publicBookmark,
}...)
require.NoError(t, err)
router := NewBookmarkRoutes(logger, deps)
t.Run("bookmark ID is not present", func(t *testing.T) {
gctx := gin.CreateTestContextOnly(w, g)
c := sctx.NewContextFromGin(gctx)
_, err := router.getBookmark(c)
require.Error(t, err)
require.Equal(t, http.StatusBadRequest, c.Writer.Status())
})
t.Run("bookmark ID is not parsable number", func(t *testing.T) {
gctx := gin.CreateTestContextOnly(w, g)
c := sctx.NewContextFromGin(gctx)
c.Params = append(c.Params, gin.Param{Key: "id", Value: "not a number"})
_, err := router.getBookmark(c)
require.Error(t, err)
require.Equal(t, http.StatusInternalServerError, c.Writer.Status())
})
t.Run("bookmark ID does not exist", func(t *testing.T) {
gctx := gin.CreateTestContextOnly(w, g)
c := sctx.NewContextFromGin(gctx)
c.Params = append(c.Params, gin.Param{Key: "id", Value: "99"})
bookmark, err := router.getBookmark(c)
require.Equal(t, http.StatusNotFound, c.Writer.Status())
require.Nil(t, bookmark)
require.Error(t, err)
})
t.Run("bookmark ID exists but user is not logged in", func(t *testing.T) {
gctx := gin.CreateTestContextOnly(w, g)
c := sctx.NewContextFromGin(gctx)
c.Request = httptest.NewRequest(http.MethodGet, "/bookmark/1", nil)
c.Params = append(c.Params, gin.Param{Key: "id", Value: "1"})
bookmark, err := router.getBookmark(c)
require.Equal(t, http.StatusFound, c.Writer.Status())
require.Nil(t, bookmark)
require.Error(t, err)
})
t.Run("bookmark ID exists and its public and user is not logged in", func(t *testing.T) {
gctx := gin.CreateTestContextOnly(w, g)
c := sctx.NewContextFromGin(gctx)
c.Request = httptest.NewRequest(http.MethodGet, "/bookmark/"+strconv.Itoa(bookmarks[0].ID), nil)
c.Params = append(c.Params, gin.Param{Key: "id", Value: strconv.Itoa(bookmarks[1].ID)})
bookmark, err := router.getBookmark(c)
require.Equal(t, http.StatusOK, c.Writer.Status())
require.NotNil(t, bookmark)
require.NoError(t, err)
})
t.Run("bookmark ID exists and user is logged in", func(t *testing.T) {
g := gin.Default()
templates.SetupTemplates(g)
gctx := gin.CreateTestContextOnly(w, g)
c := sctx.NewContextFromGin(gctx)
c.Set(model.ContextAccountKey, &model.Account{})
c.Request = httptest.NewRequest(http.MethodGet, "/bookmark/"+strconv.Itoa(bookmarks[0].ID), nil)
c.Params = append(c.Params, gin.Param{Key: "id", Value: strconv.Itoa(bookmarks[0].ID)})
bookmark, err := router.getBookmark(c)
require.Equal(t, http.StatusOK, c.Writer.Status())
require.NotNil(t, bookmark)
require.NoError(t, err)
})
}
func TestBookmarkContentHandler(t *testing.T) {
logger := logrus.New()
_, deps := testutil.GetTestConfigurationAndDependencies(t, context.Background(), logger)
bookmark := testutil.GetValidBookmark()
bookmark.HTML = "<html><body><h1>Bookmark HTML content</h1></body></html>"
boomkarks, err := deps.Database.SaveBookmarks(context.TODO(), true, *bookmark)
require.NoError(t, err)
bookmark = &boomkarks[0]
t.Run("not logged in", func(t *testing.T) {
g := gin.Default()
router := NewBookmarkRoutes(logger, deps)
router.Setup(g.Group("/"))
w := httptest.NewRecorder()
path := "/" + strconv.Itoa(bookmark.ID) + "/content"
req, _ := http.NewRequest("GET", path, nil)
g.ServeHTTP(w, req)
require.Equal(t, http.StatusFound, w.Code)
require.Equal(t, "/login?dst="+path, w.Header().Get("Location"))
})
t.Run("get existing bookmark content", func(t *testing.T) {
g := gin.Default()
templates.SetupTemplates(g)
g.Use(func(c *gin.Context) {
c.Set(model.ContextAccountKey, "test")
})
router := NewBookmarkRoutes(logger, deps)
router.Setup(g.Group("/"))
w := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "/"+strconv.Itoa(bookmark.ID)+"/content", nil)
g.ServeHTTP(w, req)
t.Log(w.Header().Get("Location"))
require.Equal(t, 200, w.Code)
require.Contains(t, w.Body.String(), bookmark.HTML)
})
}
func TestBookmarkFileHandlers(t *testing.T) {
logger := logrus.New()
_, deps := testutil.GetTestConfigurationAndDependencies(t, context.Background(), logger)
bookmark := testutil.GetValidBookmark()
bookmark.HTML = "<html><body><h1>Bookmark HTML content</h1></body></html>"
bookmark.HasArchive = true
bookmark.CreateArchive = true
bookmark.CreateEbook = true
bookmarks, err := deps.Database.SaveBookmarks(context.TODO(), true, *bookmark)
require.NoError(t, err)
bookmark, err = deps.Domains.Archiver.DownloadBookmarkArchive(bookmarks[0])
require.NoError(t, err)
bookmarks, err = deps.Database.SaveBookmarks(context.TODO(), false, *bookmark)
require.NoError(t, err)
bookmark = &bookmarks[0]
g := gin.Default()
templates.SetupTemplates(g)
g.Use(func(c *gin.Context) {
c.Set(model.ContextAccountKey, "test")
})
router := NewBookmarkRoutes(logger, deps)
router.Setup(g.Group("/"))
t.Run("get existing bookmark archive", func(t *testing.T) {
w := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "/"+strconv.Itoa(bookmark.ID)+"/archive", nil)
g.ServeHTTP(w, req)
require.Contains(t, w.Body.String(), "iframe")
require.Equal(t, 200, w.Code)
})
t.Run("get existing bookmark thumbnail", func(t *testing.T) {
w := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "/"+strconv.Itoa(bookmark.ID)+"/thumb", nil)
g.ServeHTTP(w, req)
require.Equal(t, 200, w.Code)
})
t.Run("bookmark without archive", func(t *testing.T) {
bookmark := testutil.GetValidBookmark()
bookmarks, err := deps.Database.SaveBookmarks(context.TODO(), true, *bookmark)
require.NoError(t, err)
w := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "/"+strconv.Itoa(bookmarks[0].ID)+"/archive", nil)
g.ServeHTTP(w, req)
require.Equal(t, http.StatusNotFound, w.Code)
})
t.Run("get existing bookmark archive file", func(t *testing.T) {
w := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "/"+strconv.Itoa(bookmark.ID)+"/archive/file/", nil)
g.ServeHTTP(w, req)
require.Equal(t, 200, w.Code)
})
t.Run("bookmark with ebook", func(t *testing.T) {
w := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "/"+strconv.Itoa(bookmarks[0].ID)+"/ebook", nil)
g.ServeHTTP(w, req)
require.Equal(t, http.StatusOK, w.Code)
})
t.Run("bookmark without ebook", func(t *testing.T) {
bookmark := testutil.GetValidBookmark()
bookmarks, err := deps.Database.SaveBookmarks(context.TODO(), true, *bookmark)
require.NoError(t, err)
w := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "/"+strconv.Itoa(bookmarks[0].ID)+"/ebook", nil)
g.ServeHTTP(w, req)
require.Equal(t, http.StatusNotFound, w.Code)
})
}

View file

@ -2,11 +2,9 @@ package routes
import (
"embed"
"html/template"
"net/http"
"path/filepath"
"github.com/gin-contrib/gzip"
"github.com/gin-contrib/static"
"github.com/gin-gonic/gin"
"github.com/go-shiori/shiori/internal/config"
@ -48,21 +46,8 @@ type FrontendRoutes struct {
cfg *config.Config
}
func (r *FrontendRoutes) loadTemplates(e *gin.Engine) {
tmpl, err := template.New("html").Delims("$$", "$$").ParseFS(views.Templates, "*.html")
if err != nil {
r.logger.WithError(err).Error("Failed to parse templates")
return
}
e.SetHTMLTemplate(tmpl)
}
func (r *FrontendRoutes) Setup(e *gin.Engine) {
group := e.Group("/")
e.Delims("$$", "$$")
r.loadTemplates(e)
// e.LoadHTMLGlob("internal/view/*.html")
group.Use(gzip.Gzip(gzip.DefaultCompression))
group.GET("/login", func(ctx *gin.Context) {
ctx.HTML(http.StatusOK, "login.html", gin.H{
"RootPath": r.cfg.Http.RootPath,

View file

@ -7,6 +7,7 @@ import (
"testing"
"github.com/gin-gonic/gin"
"github.com/go-shiori/shiori/internal/http/templates"
"github.com/go-shiori/shiori/internal/testutil"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/require"
@ -18,6 +19,8 @@ func TestFrontendRoutes(t *testing.T) {
cfg, _ := testutil.GetTestConfigurationAndDependencies(t, context.Background(), logger)
g := gin.Default()
templates.SetupTemplates(g)
router := NewFrontendRoutes(logger, cfg)
router.Setup(g)

View file

@ -6,6 +6,7 @@ import (
"github.com/gin-gonic/gin"
"github.com/go-shiori/shiori/internal/config"
"github.com/go-shiori/shiori/internal/dependencies"
"github.com/go-shiori/shiori/internal/model"
"github.com/go-shiori/shiori/internal/webserver"
"github.com/gofrs/uuid/v5"
@ -17,7 +18,7 @@ import (
type LegacyAPIRoutes struct {
logger *logrus.Logger
cfg *config.Config
deps *config.Dependencies
deps *dependencies.Dependencies
legacyHandler *webserver.Handler
}
@ -80,12 +81,12 @@ func (r *LegacyAPIRoutes) Setup(g *gin.Engine) {
legacyGroup.POST("/api/logout", r.handle(r.legacyHandler.ApiLogout))
// router.GET(jp("/bookmark/:id/thumb"), withLogging(hdl.serveThumbnailImage))
legacyGroup.GET("/bookmark/:id/thumb", r.handle(r.legacyHandler.ServeThumbnailImage))
// legacyGroup.GET("/bookmark/:id/thumb", r.handle(r.legacyHandler.ServeThumbnailImage))
// router.GET(jp("/bookmark/:id/content"), withLogging(hdl.serveBookmarkContent))
legacyGroup.GET("/bookmark/:id/content", r.handle(r.legacyHandler.ServeBookmarkContent))
// legacyGroup.GET("/bookmark/:id/content", r.handle(r.legacyHandler.ServeBookmarkContent))
// router.GET(jp("/bookmark/:id/archive/*filepath"), withLogging(hdl.serveBookmarkArchive))
legacyGroup.GET("/bookmark/:id/archive/*filepath", r.handle(r.legacyHandler.ServeBookmarkArchive))
// legacyGroup.GET("/bookmark/:id/archive/", r.handle(r.legacyHandler.ServeBookmarkArchive))
// legacyGroup.GET("/legacy/:id/archive/", r.handle(r.legacyHandler.ServeBookmarkArchive))
// legacyGroup.GET("/legacy/:id/archive/*filepath", r.handle(r.legacyHandler.ServeBookmarkArchive))
// router.GET(jp("/api/tags"), withLogging(hdl.apiGetTags))
legacyGroup.GET("/api/tags", r.handle(r.legacyHandler.ApiGetTags))
@ -116,7 +117,7 @@ func (r *LegacyAPIRoutes) Setup(g *gin.Engine) {
legacyGroup.DELETE("/api/accounts", r.handle(r.legacyHandler.ApiDeleteAccount))
}
func NewLegacyAPIRoutes(logger *logrus.Logger, deps *config.Dependencies, cfg *config.Config) *LegacyAPIRoutes {
func NewLegacyAPIRoutes(logger *logrus.Logger, deps *dependencies.Dependencies, cfg *config.Config) *LegacyAPIRoutes {
return &LegacyAPIRoutes{
logger: logger,
cfg: cfg,

View file

@ -11,9 +11,11 @@ import (
"github.com/gin-contrib/requestid"
"github.com/gin-gonic/gin"
"github.com/go-shiori/shiori/internal/config"
"github.com/go-shiori/shiori/internal/dependencies"
"github.com/go-shiori/shiori/internal/http/middleware"
"github.com/go-shiori/shiori/internal/http/routes"
api_v1 "github.com/go-shiori/shiori/internal/http/routes/api/v1"
"github.com/go-shiori/shiori/internal/http/templates"
"github.com/go-shiori/shiori/internal/model"
"github.com/sirupsen/logrus"
ginlogrus "github.com/toorop/gin-logrus"
@ -25,13 +27,17 @@ type HttpServer struct {
logger *logrus.Logger
}
func (s *HttpServer) Setup(cfg *config.Config, deps *config.Dependencies) *HttpServer {
func (s *HttpServer) Setup(cfg *config.Config, deps *dependencies.Dependencies) (*HttpServer, error) {
if !cfg.Development {
gin.SetMode(gin.ReleaseMode)
}
s.engine = gin.New()
templates.SetupTemplates(s.engine)
// s.engine.Use(gzip.Gzip(gzip.DefaultCompression))
s.engine.Use(requestid.New())
if cfg.Http.AccessLog {
@ -60,7 +66,7 @@ func (s *HttpServer) Setup(cfg *config.Config, deps *config.Dependencies) *HttpS
s.http.Handler = s.engine
s.http.Addr = fmt.Sprintf("%s%d", cfg.Http.Address, cfg.Http.Port)
return s
return s, nil
}
func (s *HttpServer) handle(path string, routes model.Routes) {

View file

@ -0,0 +1,25 @@
package templates
import (
"fmt"
"html/template"
"github.com/gin-gonic/gin"
views "github.com/go-shiori/shiori/internal/view"
)
const (
leftTemplateDelim = "$$"
rightTemplateDelim = "$$"
)
// SetupTemplates sets up the templates for the webserver.
func SetupTemplates(engine *gin.Engine) error {
engine.Delims(leftTemplateDelim, rightTemplateDelim)
tmpl, err := template.New("html").Delims(leftTemplateDelim, rightTemplateDelim).ParseFS(views.Templates, "*.html")
if err != nil {
return fmt.Errorf("failed to parse templates: %w", err)
}
engine.SetHTMLTemplate(tmpl)
return nil
}

View file

@ -27,6 +27,23 @@ type UserConfig struct {
MakePublic bool `json:"MakePublic"`
}
func (c *UserConfig) Scan(value interface{}) error {
switch v := value.(type) {
case []byte:
json.Unmarshal(v, &c)
return nil
case string:
json.Unmarshal([]byte(v), &c)
return nil
default:
return fmt.Errorf("unsupported type: %T", v)
}
}
func (c UserConfig) Value() (driver.Value, error) {
return json.Marshal(c)
}
// ToDTO converts Account to AccountDTO.
func (a Account) ToDTO() AccountDTO {
return AccountDTO{
@ -44,20 +61,3 @@ type AccountDTO struct {
Owner bool `json:"owner"`
Config UserConfig `json:"config"`
}
func (c *UserConfig) Scan(value interface{}) error {
switch v := value.(type) {
case []byte:
json.Unmarshal(v, &c)
return nil
case string:
json.Unmarshal([]byte(v), &c)
return nil
default:
return fmt.Errorf("unsupported type: %T", v)
}
}
func (c UserConfig) Value() (driver.Value, error) {
return json.Marshal(c)
}

View file

@ -0,0 +1,42 @@
package model
import (
"path/filepath"
"strconv"
)
// BookmarkDTO is the bookmark object representation in database and the data transfer object
// at the same time, pending a refactor to two separate object to represent each role.
type BookmarkDTO struct {
ID int `db:"id" json:"id"`
URL string `db:"url" json:"url"`
Title string `db:"title" json:"title"`
Excerpt string `db:"excerpt" json:"excerpt"`
Author string `db:"author" json:"author"`
Public int `db:"public" json:"public"`
Modified string `db:"modified" json:"modified"`
Content string `db:"content" json:"-"`
HTML string `db:"html" json:"html,omitempty"`
ImageURL string `db:"image_url" json:"imageURL"`
HasContent bool `db:"has_content" json:"hasContent"`
Tags []Tag `json:"tags"`
HasArchive bool `json:"hasArchive"`
HasEbook bool `json:"hasEbook"`
CreateArchive bool `json:"create_archive"` // TODO: migrate outside the DTO
CreateEbook bool `json:"create_ebook"` // TODO: migrate outside the DTO
}
// GetTumnbailPath returns the relative path to the thumbnail of a bookmark in the filesystem
func GetThumbnailPath(bookmark *BookmarkDTO) string {
return filepath.Join("thumb", strconv.Itoa(bookmark.ID))
}
// GetEbookPath returns the relative path to the ebook of a bookmark in the filesystem
func GetEbookPath(bookmark *BookmarkDTO) string {
return filepath.Join("ebook", strconv.Itoa(bookmark.ID)+".epub")
}
// GetArchivePath returns the relative path to the archive of a bookmark in the filesystem
func GetArchivePath(bookmark *BookmarkDTO) string {
return filepath.Join("archive", strconv.Itoa(bookmark.ID))
}

3
internal/model/db.go Normal file
View file

@ -0,0 +1,3 @@
package model
type DBID int

38
internal/model/domains.go Normal file
View file

@ -0,0 +1,38 @@
package model
import (
"context"
"io/fs"
"os"
"time"
"github.com/go-shiori/warc"
"github.com/spf13/afero"
)
type BookmarksDomain interface {
HasEbook(b *BookmarkDTO) bool
HasArchive(b *BookmarkDTO) bool
HasThumbnail(b *BookmarkDTO) bool
GetBookmark(ctx context.Context, id DBID) (*BookmarkDTO, error)
}
type AccountsDomain interface {
CheckToken(ctx context.Context, userJWT string) (*Account, error)
GetAccountFromCredentials(ctx context.Context, username, password string) (*Account, error)
CreateTokenForAccount(account *Account, expiration time.Time) (string, error)
}
type ArchiverDomain interface {
DownloadBookmarkArchive(book BookmarkDTO) (*BookmarkDTO, error)
GetBookmarkArchive(book *BookmarkDTO) (*warc.Archive, error)
}
type StorageDomain interface {
Stat(name string) (fs.FileInfo, error)
FS() afero.Fs
FileExists(path string) bool
DirExists(path string) bool
WriteData(dst string, data []byte) error
WriteFile(dst string, src *os.File) error
}

9
internal/model/errors.go Normal file
View file

@ -0,0 +1,9 @@
package model
import "errors"
var (
ErrBookmarkNotFound = errors.New("bookmark not found")
ErrBookmarkInvalidID = errors.New("invalid bookmark ID")
ErrUnauthorized = errors.New("unauthorized user")
)

View file

@ -6,3 +6,8 @@ var (
BuildCommit = "none"
BuildDate = "unknown"
)
const (
// ShioriNamespace
ShioriURLNamespace = "https://github.com/go-shiori/shiori"
)

View file

@ -1,29 +0,0 @@
package model
// Tag is the tag for a bookmark.
type Tag struct {
ID int `db:"id" json:"id"`
Name string `db:"name" json:"name"`
NBookmarks int `db:"n_bookmarks" json:"nBookmarks,omitempty"`
Deleted bool `json:"-"`
}
// Bookmark is the record for an URL.
type Bookmark struct {
ID int `db:"id" json:"id"`
URL string `db:"url" json:"url"`
Title string `db:"title" json:"title"`
Excerpt string `db:"excerpt" json:"excerpt"`
Author string `db:"author" json:"author"`
Public int `db:"public" json:"public"`
Modified string `db:"modified" json:"modified"`
Content string `db:"content" json:"-"`
HTML string `db:"html" json:"html,omitempty"`
ImageURL string `db:"image_url" json:"imageURL"`
HasContent bool `db:"has_content" json:"hasContent"`
HasArchive bool `json:"hasArchive"`
HasEbook bool `json:"hasEbook"`
Tags []Tag `json:"tags"`
CreateArchive bool `json:"create_archive"`
CreateEbook bool `json:"create_ebook"`
}

9
internal/model/tag.go Normal file
View file

@ -0,0 +1,9 @@
package model
// Tag is the tag for a bookmark.
type Tag struct {
ID int `db:"id" json:"id"`
Name string `db:"name" json:"name"`
NBookmarks int `db:"n_bookmarks" json:"nBookmarks,omitempty"`
Deleted bool `json:"-"`
}

View file

@ -7,17 +7,23 @@ import (
"github.com/go-shiori/shiori/internal/config"
"github.com/go-shiori/shiori/internal/database"
"github.com/go-shiori/shiori/internal/dependencies"
"github.com/go-shiori/shiori/internal/domains"
"github.com/go-shiori/shiori/internal/model"
"github.com/gofrs/uuid/v5"
"github.com/sirupsen/logrus"
"github.com/spf13/afero"
"github.com/stretchr/testify/require"
)
func GetTestConfigurationAndDependencies(t *testing.T, ctx context.Context, logger *logrus.Logger) (*config.Config, *config.Dependencies) {
func GetTestConfigurationAndDependencies(t *testing.T, ctx context.Context, logger *logrus.Logger) (*config.Config, *dependencies.Dependencies) {
t.Helper()
tmp, err := os.CreateTemp("", "")
require.NoError(t, err)
cfg := config.ParseServerConfiguration(ctx, logger)
cfg.Http.SecretKey = "test"
cfg.Http.SecretKey = []byte("test")
tempDir, err := os.MkdirTemp("", "")
require.NoError(t, err)
@ -28,10 +34,20 @@ func GetTestConfigurationAndDependencies(t *testing.T, ctx context.Context, logg
cfg.Storage.DataDir = tempDir
deps := config.NewDependencies(logger, db, cfg)
deps := dependencies.NewDependencies(logger, db, cfg)
deps.Database = db
deps.Domains.Auth = domains.NewAccountsDomain(logger, cfg.Http.SecretKey, db)
deps.Domains.Archiver = domains.NewArchiverDomain(logger, cfg.Storage.DataDir)
deps.Domains.Auth = domains.NewAccountsDomain(deps)
deps.Domains.Archiver = domains.NewArchiverDomain(deps)
deps.Domains.Bookmarks = domains.NewBookmarksDomain(deps)
deps.Domains.Storage = domains.NewStorageDomain(deps, afero.NewBasePathFs(afero.NewOsFs(), cfg.Storage.DataDir))
return cfg, deps
}
func GetValidBookmark() *model.BookmarkDTO {
uuidV4, _ := uuid.NewV4()
return &model.BookmarkDTO{
URL: "https://github.com/go-shiori/shiori#" + uuidV4.String(),
Title: "Shiori repository",
}
}

View file

@ -0,0 +1,23 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>$$.Book.Title$$</title>
<link rel="stylesheet" href="$$.RootPath$$assets/css/archive.css" />
</head>
<body class="archive">
<div id="shiori-archive-header" class="header">
<p id="shiori-logo"><span></span>shiori</p>
<div class="spacer"></div>
<a href="$$.URL$$" target="_blank">View Original</a>
$$if .Book.HasContent$$
<a href="/bookmark/$$.Book.ID$$/content">View Readable</a>
$$end$$
</div>
<iframe src="/bookmark/$$.Book.ID$$/archive/file/" frameborder="0"></iframe>
</body>
</html>

View file

@ -1 +1 @@
:root{--main:#f44336;--border:#e5e5e5;--colorLink:#999;--archiveHeaderBg:rgba(255, 255, 255, 0.95)}@media (prefers-color-scheme:dark){:root{--border:#191919;--archiveHeaderBg:rgba(41, 41, 41, 0.95)}}#shiori-archive-header{top:0;left:0;right:0;height:60px;box-sizing:border-box;position:fixed;padding:0 16px;display:flex;flex-flow:row wrap;align-items:center;font-size:16px;border-bottom:1px solid var(--border);background-color:var(--archiveHeaderBg);z-index:9999999999}#shiori-archive-header *{border-width:0;box-sizing:border-box;font-family:"Source Sans Pro",sans-serif;margin:0;padding:0}#shiori-archive-header>:not(:last-child){margin-right:8px}#shiori-archive-header>.spacer{flex:1}#shiori-archive-header #shiori-logo{font-size:2em;font-weight:100;color:var(--main)}#shiori-archive-header #shiori-logo span{margin-right:8px}#shiori-archive-header a{display:block;color:var(--colorLink);text-decoration:underline}#shiori-archive-header a:focus,#shiori-archive-header a:hover{color:var(--main)}@media (max-width:600px){#shiori-archive-header{font-size:14px;height:50px}#shiori-archive-header #shiori-logo{font-size:1.5em}}body.shiori-archive-content{margin-top:60px!important}@media (max-width:600px){body.shiori-archive-content{margin-top:50px!important}}
:root{--main:#f44336;--border:#e5e5e5;--colorLink:#999;--archiveHeaderBg:rgba(255, 255, 255, 0.95)}@media (prefers-color-scheme:dark){:root{--border:#191919;--archiveHeaderBg:rgba(41, 41, 41, 0.95)}}body{padding:0;margin:0}*{box-sizing:border-box}body.archive{display:grid;grid-template-rows:minmax(1px,auto) 1fr;height:100vh;width:100vw}body.archive .header{display:flex;flex-flow:row wrap;height:60px;box-sizing:border-box;padding:0 16px;align-items:center;font-size:16px;border-bottom:1px solid var(--border);background-color:var(--archiveHeaderBg)}body.archive .header *{border-width:0;box-sizing:border-box;font-family:"Source Sans Pro",sans-serif;margin:0;padding:0}body.archive .header>:not(:last-child){margin-right:8px}body.archive .header>.spacer{flex:1}body.archive .header #shiori-logo{font-size:2em;font-weight:100;color:var(--main)}body.archive .header #shiori-logo span{margin-right:8px}body.archive .header a{display:block;color:var(--colorLink);text-decoration:underline}body.archive .header a:focus,body.archive .header a:hover{color:var(--main)}@media (max-width:600px){body.archive .header{font-size:14px;height:50px}body.archive .header #shiori-logo{font-size:1.5em}}body.archive iframe{width:100%;height:100%}

View file

@ -14,73 +14,81 @@
@screen-sm-max: 600px;
@screen-sm-header-height: 50px;
#shiori-archive-header {
top: 0;
left: 0;
right: 0;
height: @header-height;
body {
padding: 0;
margin: 0;
}
* {
box-sizing: border-box;
position: fixed;
padding: 0 16px;
display: flex;
flex-flow: row wrap;
align-items: center;
font-size: 16px;
border-bottom: 1px solid var(--border);
background-color: var(--archiveHeaderBg);
z-index: 9999999999;
}
* {
border-width: 0;
body.archive {
display: grid;
grid-template-rows: minmax(1px, auto) 1fr;
height: 100vh;
width: 100vw;
.header {
display: flex;
flex-flow: row wrap;
height: @header-height;
box-sizing: border-box;
font-family: "Source Sans Pro", sans-serif;
margin: 0;
padding: 0;
}
padding: 0 16px;
align-items: center;
font-size: 16px;
border-bottom: 1px solid var(--border);
background-color: var(--archiveHeaderBg);
> *:not(:last-child) {
margin-right: 8px;
}
* {
border-width: 0;
box-sizing: border-box;
font-family: "Source Sans Pro", sans-serif;
margin: 0;
padding: 0;
}
> .spacer {
flex: 1;
}
#shiori-logo {
font-size: 2em;
font-weight: 100;
color: var(--main);
span {
> *:not(:last-child) {
margin-right: 8px;
}
}
a {
display: block;
color: var(--colorLink);
text-decoration: underline;
&:hover,
&:focus {
color: var(--main);
> .spacer {
flex: 1;
}
}
@media (max-width: @screen-sm-max) {
font-size: 14px;
height: @screen-sm-header-height;
#shiori-logo {
font-size: 1.5em;
font-size: 2em;
font-weight: 100;
color: var(--main);
span {
margin-right: 8px;
}
}
a {
display: block;
color: var(--colorLink);
text-decoration: underline;
&:hover,
&:focus {
color: var(--main);
}
}
@media (max-width: @screen-sm-max) {
font-size: 14px;
height: @screen-sm-header-height;
#shiori-logo {
font-size: 1.5em;
}
}
}
}
body.shiori-archive-content {
margin-top: @header-height !important;
@media (max-width: @screen-sm-max) {
margin-top: @screen-sm-header-height !important;
iframe {
width: 100%;
height: 100%;
}
}

View file

@ -36,7 +36,7 @@
</div>
</div>
<div id="content" dir="auto" v-pre>
$$html .Book.HTML$$
$$.HTML$$
</div>
</div>

View file

@ -25,7 +25,7 @@ func (h *Handler) ApiInsertViaExtension(w http.ResponseWriter, r *http.Request,
checkError(err)
// Decode request
request := model.Bookmark{}
request := model.BookmarkDTO{}
err = json.NewDecoder(r.Body).Decode(&request)
checkError(err)
@ -96,7 +96,7 @@ func (h *Handler) ApiInsertViaExtension(w http.ResponseWriter, r *http.Request,
}
var isFatalErr bool
book, isFatalErr, err = core.ProcessBookmark(request)
book, isFatalErr, err = core.ProcessBookmark(h.dependencies, request)
if tmp, ok := contentBuffer.(io.ReadCloser); ok {
tmp.Close()
@ -126,7 +126,7 @@ func (h *Handler) ApiDeleteViaExtension(w http.ResponseWriter, r *http.Request,
checkError(err)
// Decode request
request := model.Bookmark{}
request := model.BookmarkDTO{}
err = json.NewDecoder(r.Body).Decode(&request)
checkError(err)

View file

@ -15,12 +15,13 @@ import (
"github.com/go-shiori/shiori/internal/core"
"github.com/go-shiori/shiori/internal/database"
"github.com/go-shiori/shiori/internal/dependencies"
"github.com/go-shiori/shiori/internal/model"
"github.com/julienschmidt/httprouter"
"golang.org/x/crypto/bcrypt"
)
func downloadBookmarkContent(book *model.Bookmark, dataDir string, request *http.Request, keepTitle, keepExcerpt bool) (*model.Bookmark, error) {
func downloadBookmarkContent(deps *dependencies.Dependencies, book *model.BookmarkDTO, dataDir string, request *http.Request, keepTitle, keepExcerpt bool) (*model.BookmarkDTO, error) {
content, contentType, err := core.DownloadBookmark(book.URL)
if err != nil {
return nil, fmt.Errorf("error downloading url: %s", err)
@ -35,7 +36,7 @@ func downloadBookmarkContent(book *model.Bookmark, dataDir string, request *http
KeepExcerpt: keepExcerpt,
}
result, isFatalErr, err := core.ProcessBookmark(processRequest)
result, isFatalErr, err := core.ProcessBookmark(deps, processRequest)
content.Close()
if err != nil && isFatalErr {
@ -207,7 +208,7 @@ func (h *Handler) ApiInsertBookmark(w http.ResponseWriter, r *http.Request, ps h
err = json.NewDecoder(r.Body).Decode(&payload)
checkError(err)
book := &model.Bookmark{
book := &model.BookmarkDTO{
URL: payload.URL,
Title: payload.Title,
Excerpt: payload.Excerpt,
@ -239,7 +240,7 @@ func (h *Handler) ApiInsertBookmark(w http.ResponseWriter, r *http.Request, ps h
if payload.Async {
go func() {
bookmark, err := downloadBookmarkContent(book, h.DataDir, r, userHasDefinedTitle, book.Excerpt != "")
bookmark, err := downloadBookmarkContent(h.dependencies, book, h.DataDir, r, userHasDefinedTitle, book.Excerpt != "")
if err != nil {
log.Printf("error downloading boorkmark: %s", err)
return
@ -251,7 +252,7 @@ func (h *Handler) ApiInsertBookmark(w http.ResponseWriter, r *http.Request, ps h
} else {
// Workaround. Download content after saving the bookmark so we have the proper database
// id already set in the object regardless of the database engine.
book, err = downloadBookmarkContent(book, h.DataDir, r, userHasDefinedTitle, book.Excerpt != "")
book, err = downloadBookmarkContent(h.dependencies, book, h.DataDir, r, userHasDefinedTitle, book.Excerpt != "")
if err != nil {
log.Printf("error downloading boorkmark: %s", err)
} else if _, err := h.DB.SaveBookmarks(ctx, false, *book); err != nil {
@ -306,7 +307,7 @@ func (h *Handler) ApiUpdateBookmark(w http.ResponseWriter, r *http.Request, ps h
checkError(err)
// Decode request
request := model.Bookmark{}
request := model.BookmarkDTO{}
err = json.NewDecoder(r.Body).Decode(&request)
checkError(err)

View file

@ -1,355 +0,0 @@
package webserver
import (
"bytes"
"compress/gzip"
"fmt"
"io"
"log"
"net/http"
"os"
"path"
fp "path/filepath"
"strconv"
"strings"
"github.com/PuerkitoBio/goquery"
"github.com/go-shiori/warc"
"github.com/julienschmidt/httprouter"
"github.com/go-shiori/shiori/internal/model"
)
// ServeBookmarkContent is handler for GET /bookmark/:id/content
func (h *Handler) ServeBookmarkContent(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
ctx := r.Context()
// Get bookmark ID from URL
strID := ps.ByName("id")
id, err := strconv.Atoi(strID)
checkError(err)
// Get bookmark in database
bookmark, exist, err := h.DB.GetBookmark(ctx, id, "")
checkError(err)
if !exist {
panic(fmt.Errorf("bookmark not found"))
}
// If it's not public, make sure session still valid
if bookmark.Public != 1 {
err = h.validateSession(r)
if err != nil {
newPath := path.Join(h.RootPath, "/login")
redirectURL := createRedirectURL(newPath, r.URL.String())
redirectPage(w, r, redirectURL)
return
}
}
// Check if it has ebook.
ebookPath := fp.Join(h.DataDir, "ebook", strID+".epub")
if FileExists(ebookPath) {
bookmark.HasEbook = true
}
archivePath := fp.Join(h.DataDir, "archive", strID)
if FileExists(archivePath) {
bookmark.HasArchive = true
// Open archive, look in cache first
var archive *warc.Archive
cacheData, found := h.ArchiveCache.Get(strID)
if found {
archive = cacheData.(*warc.Archive)
} else {
archivePath := fp.Join(h.DataDir, "archive", strID)
archive, err = warc.Open(archivePath)
checkError(err)
h.ArchiveCache.Set(strID, archive, 0)
}
// Find all image and convert its source to use the archive URL.
createArchivalURL := func(archivalName string) string {
archivalURL := *r.URL
archivalURL.Path = path.Join(h.RootPath, "bookmark", strID, "archive", archivalName)
return archivalURL.String()
}
buffer := strings.NewReader(bookmark.HTML)
doc, err := goquery.NewDocumentFromReader(buffer)
checkError(err)
doc.Find("img, picture, figure, source").Each(func(_ int, node *goquery.Selection) {
// Get the needed attributes
src, _ := node.Attr("src")
strSrcSets, _ := node.Attr("srcset")
// Convert `src` attributes
if src != "" {
archivalName := getArchivalName(src)
if archivalName != "" && archive.HasResource(archivalName) {
node.SetAttr("src", createArchivalURL(archivalName))
}
}
// Split srcset by comma, then process it like any URLs
srcSets := strings.Split(strSrcSets, ",")
for i, srcSet := range srcSets {
srcSet = strings.TrimSpace(srcSet)
parts := strings.SplitN(srcSet, " ", 2)
if parts[0] == "" {
continue
}
archivalName := getArchivalName(parts[0])
if archivalName != "" && archive.HasResource(archivalName) {
archivalURL := createArchivalURL(archivalName)
srcSets[i] = strings.Replace(srcSets[i], parts[0], archivalURL, 1)
}
}
if len(srcSets) > 0 {
node.SetAttr("srcset", strings.Join(srcSets, ","))
}
})
bookmark.HTML, err = goquery.OuterHtml(doc.Selection)
checkError(err)
}
// Execute template
if developmentMode {
if err := h.PrepareTemplates(); err != nil {
log.Printf("error during template preparation: %s", err)
}
}
tplData := struct {
RootPath string
Book model.Bookmark
}{h.RootPath, bookmark}
err = h.templates["content"].Execute(w, &tplData)
checkError(err)
}
// ServeThumbnailImage is handler for GET /bookmark/:id/thumb
func (h *Handler) ServeThumbnailImage(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
// Get bookmark ID from URL
ctx := r.Context()
// Get bookmark ID from URL
strID := ps.ByName("id")
// Get bookmark from database
id, err := strconv.Atoi(strID)
checkError(err)
bookmark, exist, err := h.DB.GetBookmark(ctx, id, "")
checkError(err)
if !exist {
log.Println("error: bookmark not found")
return
}
// If it's not public, make sure session still valid
if bookmark.Public != 1 {
err = h.validateSession(r)
if err != nil {
newPath := path.Join(h.RootPath, "/login")
redirectURL := createRedirectURL(newPath, r.URL.String())
redirectPage(w, r, redirectURL)
return
}
}
// Open image
imgPath := fp.Join(h.DataDir, "thumb", strID)
img, err := os.Open(imgPath)
checkError(err)
defer img.Close()
// Get image type from its 512 first bytes
buffer := make([]byte, 512)
_, err = img.Read(buffer)
checkError(err)
mimeType := http.DetectContentType(buffer)
w.Header().Set("Content-Type", mimeType)
// Set cache value
info, err := img.Stat()
checkError(err)
etag := fmt.Sprintf(`W/"%x-%x"`, info.ModTime().Unix(), info.Size())
w.Header().Set("ETag", etag)
w.Header().Set("Cache-Control", "max-age=86400")
// Serve image
if _, err := img.Seek(0, 0); err != nil {
log.Printf("error during image seek: %s", err)
}
_, err = io.Copy(w, img)
checkError(err)
}
// ServeBookmarkArchive is handler for GET /bookmark/:id/archive/*filepath
func (h *Handler) ServeBookmarkArchive(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
ctx := r.Context()
// Get parameter from URL
strID := ps.ByName("id")
resourcePath := ps.ByName("filepath")
resourcePath = strings.TrimPrefix(resourcePath, "/")
// Get bookmark from database
id, err := strconv.Atoi(strID)
checkError(err)
bookmark, exist, err := h.DB.GetBookmark(ctx, id, "")
checkError(err)
if !exist {
panic(fmt.Errorf("bookmark not found"))
}
// If it's not public, make sure session still valid
if bookmark.Public != 1 {
err = h.validateSession(r)
if err != nil {
newPath := path.Join(h.RootPath, "/login")
redirectURL := createRedirectURL(newPath, r.URL.String())
redirectPage(w, r, redirectURL)
return
}
}
// Open archive, look in cache first
var archive *warc.Archive
cacheData, found := h.ArchiveCache.Get(strID)
if found {
archive = cacheData.(*warc.Archive)
} else {
archivePath := fp.Join(h.DataDir, "archive", strID)
archive, err = warc.Open(archivePath)
checkError(err)
h.ArchiveCache.Set(strID, archive, 0)
}
content, contentType, err := archive.Read(resourcePath)
checkError(err)
// Set response header
w.Header().Set("Content-Encoding", "gzip")
w.Header().Set("Content-Type", contentType)
// If this is HTML and root, inject shiori header
if strings.Contains(strings.ToLower(contentType), "text/html") && resourcePath == "" {
// Extract gzip
buffer := bytes.NewBuffer(content)
gzipReader, err := gzip.NewReader(buffer)
checkError(err)
// Parse gzipped content
doc, err := goquery.NewDocumentFromReader(gzipReader)
checkError(err)
// Add Shiori overlay
tplOutput := bytes.NewBuffer(nil)
err = h.templates["archive"].Execute(tplOutput, &bookmark)
checkError(err)
archiveCSSPath := path.Join(h.RootPath, "/assets/css/archive.css")
docHead := doc.Find("head")
docHead.PrependHtml(`<meta charset="UTF-8">`)
docHead.AppendHtml(`<link href="` + archiveCSSPath + `" rel="stylesheet">`)
doc.Find("body").PrependHtml(tplOutput.String())
doc.Find("body").AddClass("shiori-archive-content")
// Revert back to HTML
outerHTML, err := goquery.OuterHtml(doc.Selection)
checkError(err)
// Gzip it again and send to response writer
gzipWriter := gzip.NewWriter(w)
if _, err := gzipWriter.Write([]byte(outerHTML)); err != nil {
log.Printf("error writing gzip file: %s", err)
}
gzipWriter.Flush()
return
}
// Serve content
if _, err := w.Write(content); err != nil {
log.Printf("error writing response: %s", err)
}
}
// serveEbook is handler for GET /bookmark/:id/ebook
func (h *Handler) ServeBookmarkEbook(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
ctx := r.Context()
// Get bookmark ID from URL
strID := ps.ByName("id")
id, err := strconv.Atoi(strID)
checkError(err)
// Get bookmark in database
bookmark, exist, err := h.DB.GetBookmark(ctx, id, "")
checkError(err)
if !exist {
http.Error(w, "bookmark not found", http.StatusNotFound)
return
}
// If it's not public, make sure session still valid
if bookmark.Public != 1 {
err = h.validateSession(r)
if err != nil {
newPath := path.Join(h.RootPath, "/login")
redirectURL := createRedirectURL(newPath, r.URL.String())
redirectPage(w, r, redirectURL)
return
}
}
// Check if it has ebook.
ebookPath := fp.Join(h.DataDir, "ebook", strID+".epub")
if !FileExists(ebookPath) {
http.Error(w, "ebook not found", http.StatusNotFound)
return
}
epub, err := os.Open(ebookPath)
if err != nil {
http.Error(w, "Internal server error", http.StatusNotFound)
return
}
defer epub.Close()
// Set content type
w.Header().Set("Content-Type", "application/epub+zip")
// Set cache value
info, err := epub.Stat()
if err != nil {
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
etag := fmt.Sprintf(`W/"%x-%x"`, info.ModTime().Unix(), info.Size())
w.Header().Set("ETag", etag)
w.Header().Set("Cache-Control", "max-age=86400")
// Serve epub file
if _, err := epub.Seek(0, 0); err != nil {
log.Printf("error during epub seek: %s", err)
}
_, err = io.Copy(w, epub)
if err != nil {
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
}

View file

@ -6,15 +6,13 @@ import (
"net/http"
"strings"
"github.com/go-shiori/shiori/internal/config"
"github.com/go-shiori/shiori/internal/database"
"github.com/go-shiori/shiori/internal/dependencies"
"github.com/go-shiori/shiori/internal/model"
cch "github.com/patrickmn/go-cache"
"github.com/sirupsen/logrus"
)
var developmentMode = false
// Handler is Handler for serving the web interface.
type Handler struct {
DB database.DB
@ -25,7 +23,7 @@ type Handler struct {
ArchiveCache *cch.Cache
Log bool
dependencies *config.Dependencies
dependencies *dependencies.Dependencies
templates map[string]*template.Template
}

View file

@ -3,8 +3,8 @@ package webserver
import (
"time"
"github.com/go-shiori/shiori/internal/config"
"github.com/go-shiori/shiori/internal/database"
"github.com/go-shiori/shiori/internal/dependencies"
cch "github.com/patrickmn/go-cache"
)
@ -18,7 +18,7 @@ type Config struct {
Log bool
}
func GetLegacyHandler(cfg Config, dependencies *config.Dependencies) *Handler {
func GetLegacyHandler(cfg Config, dependencies *dependencies.Dependencies) *Handler {
return &Handler{
DB: cfg.DB,
DataDir: cfg.DataDir,

View file

@ -7,13 +7,9 @@ import (
"net/http"
nurl "net/url"
"os"
"regexp"
"strings"
"syscall"
)
var rxRepeatedStrip = regexp.MustCompile(`(?i)-+`)
func createRedirectURL(newPath, previousPath string) string {
urlQueries := nurl.Values{}
urlQueries.Set("dst", previousPath)
@ -53,37 +49,6 @@ func createTemplate(filename string, funcMap template.FuncMap) (*template.Templa
return template.New(filename).Delims("$$", "$$").Funcs(funcMap).Parse(string(srcContent))
}
// getArchivalName converts an URL into an archival name.
func getArchivalName(src string) string {
archivalURL := src
// Some URL have its query or path escaped, e.g. Wikipedia and Dev.to.
// For example, Wikipedia's stylesheet looks like this :
// load.php?lang=en&modules=ext.3d.styles%7Cext.cite.styles%7Cext.uls.interlanguage
// However, when browser download it, it will be registered as unescaped query :
// load.php?lang=en&modules=ext.3d.styles|ext.cite.styles|ext.uls.interlanguage
// So, for archival URL, we need to unescape the query and path first.
tmp, err := nurl.Parse(src)
if err == nil {
unescapedQuery, _ := nurl.QueryUnescape(tmp.RawQuery)
if unescapedQuery != "" {
tmp.RawQuery = unescapedQuery
}
archivalURL = tmp.String()
archivalURL = strings.Replace(archivalURL, tmp.EscapedPath(), tmp.Path, 1)
}
archivalURL = strings.ReplaceAll(archivalURL, "://", "/")
archivalURL = strings.ReplaceAll(archivalURL, "?", "-")
archivalURL = strings.ReplaceAll(archivalURL, "#", "-")
archivalURL = strings.ReplaceAll(archivalURL, "/", "-")
archivalURL = strings.ReplaceAll(archivalURL, " ", "-")
archivalURL = rxRepeatedStrip.ReplaceAllString(archivalURL, "-")
return archivalURL
}
func checkError(err error) {
if err == nil {
return

View file

@ -1,5 +1,3 @@
//go:generate go run assets-generator.go
package main
import (