diff --git a/internal/database/migrations/20230817172319_init.sql b/internal/database/migrations/20230817172319_init.sql index e6a1296..d4cb0a6 100644 --- a/internal/database/migrations/20230817172319_init.sql +++ b/internal/database/migrations/20230817172319_init.sql @@ -6,8 +6,6 @@ CREATE EXTENSION IF NOT EXISTS btree_gin; CREATE SCHEMA IF NOT EXISTS teldrive; -CREATE COLLATION IF NOT EXISTS numeric (PROVIDER = ICU, LOCALE = 'en@colnumeric=yes'); - -- +goose StatementBegin CREATE OR REPLACE diff --git a/internal/database/migrations/20231102165658_tables.sql b/internal/database/migrations/20231102165658_tables.sql index 38bf088..cd35327 100644 --- a/internal/database/migrations/20231102165658_tables.sql +++ b/internal/database/migrations/20231102165658_tables.sql @@ -68,10 +68,7 @@ CREATE TABLE IF NOT EXISTS teldrive.sessions ( FOREIGN KEY (user_id) REFERENCES teldrive.users(user_id) ); -CREATE INDEX IF NOT EXISTS name_numeric_idx ON teldrive.files USING btree (name COLLATE "numeric" NULLS FIRST); -CREATE INDEX IF NOT EXISTS name_search_idx ON teldrive.files USING gin (teldrive.get_tsvector(name), updated_at); CREATE INDEX IF NOT EXISTS parent_idx ON teldrive.files USING btree (parent_id); -CREATE INDEX IF NOT EXISTS parent_name_numeric_idx ON teldrive.files USING btree (parent_id, name COLLATE "numeric" DESC); CREATE INDEX IF NOT EXISTS path_idx ON teldrive.files USING btree (path); CREATE INDEX IF NOT EXISTS starred_updated_at_idx ON teldrive.files USING btree (starred, updated_at DESC); CREATE INDEX IF NOT EXISTS status_idx ON teldrive.files USING btree (status); @@ -79,12 +76,7 @@ CREATE UNIQUE INDEX IF NOT EXISTS unique_file ON teldrive.files USING btree (nam CREATE INDEX IF NOT EXISTS user_id_idx ON teldrive.files USING btree (user_id); -- +goose Down - -DROP INDEX IF EXISTS name_numeric_idx ; -DROP INDEX IF EXISTS name_search_idx ; DROP INDEX IF EXISTS parent_idx ; -DROP INDEX IF EXISTS parent_name_numeric_idx ; -DROP INDEX IF EXISTS path_idx ; DROP INDEX IF EXISTS starred_updated_at_idx ; DROP INDEX IF EXISTS status_idx ; DROP INDEX IF EXISTS unique_file; diff --git a/internal/database/migrations/20240715001936_index.sql b/internal/database/migrations/20240715001936_index.sql new file mode 100644 index 0000000..8f5d419 --- /dev/null +++ b/internal/database/migrations/20240715001936_index.sql @@ -0,0 +1,66 @@ +-- +goose Up +-- +goose StatementBegin +DROP INDEX IF EXISTS teldrive.name_numeric_idx; +DROP INDEX IF EXISTS teldrive.parent_name_numeric_idx; +DROP INDEX IF EXISTS teldrive.path_idx; +DROP COLLATION IF EXISTS numeric; +CREATE INDEX IF NOT EXISTS name_idx ON teldrive.files (name); +CREATE INDEX IF NOT EXISTS updated_at_status_user_id_idx ON teldrive.files (updated_at DESC, user_id, status); +CREATE INDEX IF NOT EXISTS name_status_user_id_idx ON teldrive.files (name, user_id, status); +CREATE INDEX IF NOT EXISTS updated_at_idx ON teldrive.files (updated_at); + +CREATE OR REPLACE FUNCTION teldrive.get_file_from_path(full_path text, u_id bigint) + RETURNS SETOF teldrive.files + LANGUAGE plpgsql +AS $function$ +DECLARE + target_id text; +begin + + IF full_path = '/' then + full_path := ''; + END IF; + + WITH RECURSIVE dir_hierarchy AS ( + SELECT + root.id, + root.name, + root.parent_id, + 0 AS depth, + '' as path + FROM + teldrive.files as root + WHERE + root.parent_id = 'root' AND root.user_id = u_id + + UNION ALL + + SELECT + f.id, + f.name, + f.parent_id, + dh.depth + 1 AS depth, + dh.path || '/' || f.name + FROM + teldrive.files f + JOIN + dir_hierarchy dh ON dh.id = f.parent_id + WHERE f.type = 'folder' AND f.user_id = u_id + ) + + SELECT id into target_id FROM dir_hierarchy dh + WHERE dh.path = full_path + ORDER BY dh.depth DESC + LIMIT 1; + + IF target_id IS NULL THEN + RAISE EXCEPTION 'file not found for path: %', full_path; + END IF; + + RETURN QUERY select * from teldrive.files where id=target_id; + +END; +$function$ +; + +-- +goose StatementEnd diff --git a/pkg/controller/file.go b/pkg/controller/file.go index 2a333f3..eed0b96 100644 --- a/pkg/controller/file.go +++ b/pkg/controller/file.go @@ -63,10 +63,11 @@ func (fc *Controller) ListFiles(c *gin.Context) { userId, _ := auth.GetUser(c) fquery := schemas.FileQuery{ - PerPage: 500, - Order: "asc", - Sort: "name", - Op: "list", + Limit: 500, + Page: 1, + Order: "asc", + Sort: "name", + Op: "list", } if err := c.ShouldBindQuery(&fquery); err != nil { diff --git a/pkg/schemas/file.go b/pkg/schemas/file.go index feaae1e..4bb2256 100644 --- a/pkg/schemas/file.go +++ b/pkg/schemas/file.go @@ -10,20 +10,20 @@ type Part struct { } type FileQuery struct { - Name string `form:"name"` - Query string `form:"query"` - Type string `form:"type"` - Path string `form:"path"` - Op string `form:"op"` - DeepSearch bool `form:"deepSearch"` - Starred *bool `form:"starred"` - ParentID string `form:"parentId"` - Category string `form:"category"` - UpdatedAt string `form:"updatedAt"` - Sort string `form:"sort"` - Order string `form:"order"` - PerPage int `form:"perPage"` - NextPageToken string `form:"nextPageToken"` + Name string `form:"name"` + Query string `form:"query"` + Type string `form:"type"` + Path string `form:"path"` + Op string `form:"op"` + DeepSearch bool `form:"deepSearch"` + Starred *bool `form:"starred"` + ParentID string `form:"parentId"` + Category string `form:"category"` + UpdatedAt string `form:"updatedAt"` + Sort string `form:"sort"` + Order string `form:"order"` + Limit int `form:"limit"` + Page int `form:"page"` } type FileIn struct { @@ -50,6 +50,7 @@ type FileOut struct { ParentID string `json:"parentId,omitempty"` ParentPath string `json:"parentPath,omitempty"` UpdatedAt time.Time `json:"updatedAt,omitempty"` + Total int `json:"total,omitempty"` } type FileOutFull struct { @@ -70,9 +71,14 @@ type FileUpdate struct { Size *int64 `json:"size,omitempty"` } +type Meta struct { + Count int `json:"count,omitempty"` + TotalPages int `json:"totalPages,omitempty"` + CurrentPage int `json:"currentPage,omitempty"` +} type FileResponse struct { - Files []FileOut `json:"results"` - NextPageToken string `json:"nextPageToken,omitempty"` + Files []FileOut `json:"files"` + Meta Meta `json:"meta"` } type FileOperation struct { diff --git a/pkg/services/file.go b/pkg/services/file.go index 6ef7826..705c2b6 100644 --- a/pkg/services/file.go +++ b/pkg/services/file.go @@ -3,11 +3,11 @@ package services import ( "context" "crypto/rand" - "encoding/base64" "encoding/binary" "errors" "fmt" "io" + "math" "mime" "net/http" "strconv" @@ -202,37 +202,19 @@ func (fs *FileService) GetFileByID(id string) (*schemas.FileOutFull, *types.AppE func (fs *FileService) ListFiles(userId int64, fquery *schemas.FileQuery) (*schemas.FileResponse, *types.AppError) { - var parentID string - - if fquery.Path != "" && fquery.ParentID == "" { - parent, err := fs.getFileFromPath(fquery.Path, userId) - if err != nil { - return nil, &types.AppError{Error: err, Code: http.StatusNotFound} - } - parentID = parent.Id - } else if fquery.ParentID != "" { - parentID = fquery.ParentID - } - - query := fs.db.Limit(fquery.PerPage) - setOrderFilter(query, fquery) + query := fs.db.Where("user_id = ?", userId).Where("status = ?", "active") if fquery.Op == "list" { - filter := &models.File{UserID: userId, Status: "active", ParentID: parentID} - query.Order("type DESC").Order(getOrder(fquery)).Model(filter).Where(&filter) - - } else if fquery.Op == "find" { - if !fquery.DeepSearch && parentID != "" && (fquery.Name != "" || fquery.Query != "") { - query.Where("parent_id = ?", parentID) - fquery.Path = "" - } else if fquery.DeepSearch && parentID != "" && fquery.Query != "" { - query = fs.db.Clauses(exclause.With{Recursive: true, CTEs: []exclause.CTE{{Name: "subdirs", - Subquery: exclause.Subquery{DB: fs.db.Model(&models.File{Id: parentID}).Select("id", "parent_id").Clauses(exclause.NewUnion("ALL ?", - fs.db.Table("teldrive.files as f").Select("f.id", "f.parent_id"). - Joins("inner join subdirs ON f.parent_id = subdirs.id")))}}}}).Where("files.id in (select id from subdirs)") - fquery.Path = "" + if fquery.Path != "" && fquery.ParentID == "" { + query.Where("parent_id in (SELECT id FROM teldrive.get_file_from_path(?, ?))", fquery.Path, userId) + } + if fquery.ParentID != "" { + query.Where("parent_id = ?", fquery.ParentID) + } + } else if fquery.Op == "find" { + if fquery.DeepSearch && fquery.Query != "" && fquery.Path != "" { + query.Where("files.id in (select id from subdirs)") } - if fquery.UpdatedAt != "" { dateFilters := strings.Split(fquery.UpdatedAt, ",") for _, dateFilter := range dateFilters { @@ -261,7 +243,7 @@ func (fs *FileService) ListFiles(userId int64, fquery *schemas.FileQuery) (*sche } if fquery.Query != "" { - query.Where("name &@~ REGEXP_REPLACE(?, '[.,-_]', ' ', 'g')", fquery.Query) + query = query.Where("name &@~ REGEXP_REPLACE(?, '[.,-_]', ' ', 'g')", fquery.Query) } if fquery.Category != "" { @@ -272,7 +254,6 @@ func (fs *FileService) ListFiles(userId int64, fquery *schemas.FileQuery) (*sche } else { filterQuery = fs.db.Where("category = ?", categories[0]) } - if len(categories) > 1 { for _, category := range categories[1:] { if category == "folder" { @@ -283,37 +264,85 @@ func (fs *FileService) ListFiles(userId int64, fquery *schemas.FileQuery) (*sche } } query.Where(filterQuery) - } - filter := &models.File{UserID: userId, Status: "active"} - filter.Name = fquery.Name - filter.ParentID = fquery.ParentID - filter.Type = fquery.Type + if fquery.Name != "" { + query.Where("name = ?", fquery.Name) + } + if fquery.ParentID != "" { + query.Where("parent_id = ?", fquery.ParentID) + } + if fquery.Type != "" { + query.Where("type = ?", fquery.Type) + } if fquery.Starred != nil { - filter.Starred = *fquery.Starred + query.Where("starred = ?", *fquery.Starred) + } + } + + orderField := utils.CamelToSnake(fquery.Sort) + + var op string + + if fquery.Page == 1 { + if fquery.Order == "asc" { + op = ">=" + } else { + op = "<=" + } + } else { + if fquery.Order == "asc" { + op = ">" + } else { + op = "<" } - query.Order("type DESC").Order(getOrder(fquery)). - Model(&filter).Where(&filter) - - query.Limit(fquery.PerPage) - setOrderFilter(query, fquery) } + var fileQuery *gorm.DB + + if fquery.DeepSearch && fquery.Query != "" && fquery.Path != "" { + fileQuery = fs.db.Clauses(exclause.With{Recursive: true, CTEs: []exclause.CTE{{Name: "subdirs", + Subquery: exclause.Subquery{DB: fs.db.Model(&models.File{}).Select("id", "parent_id"). + Where("id in (SELECT id FROM teldrive.get_file_from_path(?, ?))", fquery.Path, userId). + Clauses(exclause.NewUnion("ALL ?", + fs.db.Table("teldrive.files as f").Select("f.id", "f.parent_id"). + Joins("inner join subdirs ON f.parent_id = subdirs.id")))}}}}) + } + + if fileQuery == nil { + fileQuery = fs.db + } + + fileQuery = fileQuery.Clauses(exclause.NewWith("ranked_scores", fs.db.Model(&models.File{}).Select(orderField, "count(*) OVER () as total", + fmt.Sprintf("ROW_NUMBER() OVER (ORDER BY %s %s) AS rank", orderField, strings.ToUpper(fquery.Order))).Where(query))). + Model(&models.File{}).Select("*", "(select total from ranked_scores limit 1) as total"). + Where(fmt.Sprintf("%s %s (SELECT %s FROM ranked_scores WHERE rank = ?)", orderField, op, orderField), + max((fquery.Page-1)*fquery.Limit, 1)). + Where(query).Order(getOrder(fquery)).Limit(fquery.Limit) + files := []schemas.FileOut{} - query.Scan(&files) - - token := "" - - if len(files) == fquery.PerPage { - lastItem := files[len(files)-1] - token = utils.GetField(&lastItem, utils.CamelToPascalCase(fquery.Sort)) - token = base64.StdEncoding.EncodeToString([]byte(token)) + if err := fileQuery.Scan(&files).Error; err != nil { + if strings.Contains(err.Error(), "file not found") { + return nil, &types.AppError{Error: database.ErrNotFound, Code: http.StatusNotFound} + } + return nil, &types.AppError{Error: err} } - res := &schemas.FileResponse{Files: files, NextPageToken: token} + count := 0 + + if len(files) > 0 { + count = files[0].Total + } + + for i := range files { + files[i].Total = 0 + } + + res := &schemas.FileResponse{Files: files, + Meta: schemas.Meta{Count: count, TotalPages: int(math.Ceil(float64(count) / float64(fquery.Limit))), + CurrentPage: fquery.Page}} return res, nil } @@ -753,21 +782,6 @@ func (fs *FileService) GetFileStream(c *gin.Context, download bool) { } } } -func setOrderFilter(query *gorm.DB, fquery *schemas.FileQuery) *gorm.DB { - if fquery.NextPageToken != "" { - sortColumn := utils.CamelToSnake(fquery.Sort) - - tokenValue, err := base64.StdEncoding.DecodeString(fquery.NextPageToken) - if err == nil { - if fquery.Order == "asc" { - return query.Where(fmt.Sprintf("%s > ?", sortColumn), string(tokenValue)) - } else { - return query.Where(fmt.Sprintf("%s < ?", sortColumn), string(tokenValue)) - } - } - } - return query -} func getOrder(fquery *schemas.FileQuery) clause.OrderByColumn { sortColumn := utils.CamelToSnake(fquery.Sort)