tgdrive/teldrive
1
1
Fork 0
mirror of https://github.com/tgdrive/teldrive.git synced 2025-09-05 22:14:30 +08:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: Implemented offset pagination by ranks

Browse source
This commit is contained in:
divyam234 2024-07-15 00:39:09 +05:30
parent d8846d65e4
commit c49b78b863
6 changed files with 173 additions and 96 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
internal/database/migrations/20230817172319_init.sql
View file

@ -6,8 +6,6 @@ CREATE EXTENSION IF NOT EXISTS btree_gin;
CREATE SCHEMA IF NOT EXISTS teldrive;
CREATE COLLATION IF NOT EXISTS numeric (PROVIDER = ICU, LOCALE = 'en@colnumeric=yes');
-- +goose StatementBegin
CREATE OR REPLACE

8
internal/database/migrations/20231102165658_tables.sql
View file

@ -68,10 +68,7 @@ CREATE TABLE IF NOT EXISTS teldrive.sessions (
FOREIGN KEY (user_id) REFERENCES teldrive.users(user_id)
);
CREATE INDEX IF NOT EXISTS name_numeric_idx ON teldrive.files USING btree (name COLLATE "numeric" NULLS FIRST);
CREATE INDEX IF NOT EXISTS name_search_idx ON teldrive.files USING gin (teldrive.get_tsvector(name), updated_at);
CREATE INDEX IF NOT EXISTS parent_idx ON teldrive.files USING btree (parent_id);
CREATE INDEX IF NOT EXISTS parent_name_numeric_idx ON teldrive.files USING btree (parent_id, name COLLATE "numeric" DESC);
CREATE INDEX IF NOT EXISTS path_idx ON teldrive.files USING btree (path);
CREATE INDEX IF NOT EXISTS starred_updated_at_idx ON teldrive.files USING btree (starred, updated_at DESC);
CREATE INDEX IF NOT EXISTS status_idx ON teldrive.files USING btree (status);
@ -79,12 +76,7 @@ CREATE UNIQUE INDEX IF NOT EXISTS unique_file ON teldrive.files USING btree (nam
CREATE INDEX IF NOT EXISTS user_id_idx ON teldrive.files USING btree (user_id);
-- +goose Down
DROP INDEX IF EXISTS name_numeric_idx ;
DROP INDEX IF EXISTS name_search_idx ;
DROP INDEX IF EXISTS parent_idx ;
DROP INDEX IF EXISTS parent_name_numeric_idx ;
DROP INDEX IF EXISTS path_idx ;
DROP INDEX IF EXISTS starred_updated_at_idx ;
DROP INDEX IF EXISTS status_idx ;
DROP INDEX IF EXISTS unique_file;

66
internal/database/migrations/20240715001936_index.sql Normal file
View file

@ -0,0 +1,66 @@
-- +goose Up
-- +goose StatementBegin
DROP INDEX IF EXISTS teldrive.name_numeric_idx;
DROP INDEX IF EXISTS teldrive.parent_name_numeric_idx;
DROP INDEX IF EXISTS teldrive.path_idx;
DROP COLLATION IF EXISTS numeric;
CREATE INDEX IF NOT EXISTS name_idx ON teldrive.files (name);
CREATE INDEX IF NOT EXISTS updated_at_status_user_id_idx ON teldrive.files (updated_at DESC, user_id, status);
CREATE INDEX IF NOT EXISTS name_status_user_id_idx ON teldrive.files (name, user_id, status);
CREATE INDEX IF NOT EXISTS updated_at_idx ON teldrive.files (updated_at);
CREATE OR REPLACE FUNCTION teldrive.get_file_from_path(full_path text, u_id bigint)
RETURNS SETOF teldrive.files
LANGUAGE plpgsql
AS $function$
DECLARE
target_id text;
begin
IF full_path = '/' then
full_path := '';
END IF;
WITH RECURSIVE dir_hierarchy AS (
SELECT
root.id,
root.name,
root.parent_id,
0 AS depth,
'' as path
FROM
teldrive.files as root
WHERE
root.parent_id = 'root' AND root.user_id = u_id
UNION ALL
SELECT
f.id,
f.name,
f.parent_id,
dh.depth + 1 AS depth,
dh.path || '/' || f.name
FROM
teldrive.files f
JOIN
dir_hierarchy dh ON dh.id = f.parent_id
WHERE f.type = 'folder' AND f.user_id = u_id
)
SELECT id into target_id FROM dir_hierarchy dh
WHERE dh.path = full_path
ORDER BY dh.depth DESC
LIMIT 1;
IF target_id IS NULL THEN
RAISE EXCEPTION 'file not found for path: %', full_path;
END IF;
RETURN QUERY select * from teldrive.files where id=target_id;
END;
$function$
;
-- +goose StatementEnd

9
pkg/controller/file.go
View file

@ -63,10 +63,11 @@ func (fc *Controller) ListFiles(c *gin.Context) {
userId, _ := auth.GetUser(c)
fquery := schemas.FileQuery{
PerPage: 500,
Order: "asc",
Sort: "name",
Op: "list",
Limit: 500,
Page: 1,
Order: "asc",
Sort: "name",
Op: "list",
}
if err := c.ShouldBindQuery(&fquery); err != nil {

38
pkg/schemas/file.go
View file

@ -10,20 +10,20 @@ type Part struct {
}
type FileQuery struct {
Name string `form:"name"`
Query string `form:"query"`
Type string `form:"type"`
Path string `form:"path"`
Op string `form:"op"`
DeepSearch bool `form:"deepSearch"`
Starred *bool `form:"starred"`
ParentID string `form:"parentId"`
Category string `form:"category"`
UpdatedAt string `form:"updatedAt"`
Sort string `form:"sort"`
Order string `form:"order"`
PerPage int `form:"perPage"`
NextPageToken string `form:"nextPageToken"`
Name string `form:"name"`
Query string `form:"query"`
Type string `form:"type"`
Path string `form:"path"`
Op string `form:"op"`
DeepSearch bool `form:"deepSearch"`
Starred *bool `form:"starred"`
ParentID string `form:"parentId"`
Category string `form:"category"`
UpdatedAt string `form:"updatedAt"`
Sort string `form:"sort"`
Order string `form:"order"`
Limit int `form:"limit"`
Page int `form:"page"`
}
type FileIn struct {
@ -50,6 +50,7 @@ type FileOut struct {
ParentID string `json:"parentId,omitempty"`
ParentPath string `json:"parentPath,omitempty"`
UpdatedAt time.Time `json:"updatedAt,omitempty"`
Total int `json:"total,omitempty"`
}
type FileOutFull struct {
@ -70,9 +71,14 @@ type FileUpdate struct {
Size *int64 `json:"size,omitempty"`
}
type Meta struct {
Count int `json:"count,omitempty"`
TotalPages int `json:"totalPages,omitempty"`
CurrentPage int `json:"currentPage,omitempty"`
}
type FileResponse struct {
Files []FileOut `json:"results"`
NextPageToken string `json:"nextPageToken,omitempty"`
Files []FileOut `json:"files"`
Meta Meta `json:"meta"`
}
type FileOperation struct {

146
pkg/services/file.go
View file

@ -3,11 +3,11 @@ package services
import (
"context"
"crypto/rand"
"encoding/base64"
"encoding/binary"
"errors"
"fmt"
"io"
"math"
"mime"
"net/http"
"strconv"
@ -202,37 +202,19 @@ func (fs *FileService) GetFileByID(id string) (*schemas.FileOutFull, *types.AppE
func (fs *FileService) ListFiles(userId int64, fquery *schemas.FileQuery) (*schemas.FileResponse, *types.AppError) {
var parentID string
if fquery.Path != "" && fquery.ParentID == "" {
parent, err := fs.getFileFromPath(fquery.Path, userId)
if err != nil {
return nil, &types.AppError{Error: err, Code: http.StatusNotFound}
}
parentID = parent.Id
} else if fquery.ParentID != "" {
parentID = fquery.ParentID
}
query := fs.db.Limit(fquery.PerPage)
setOrderFilter(query, fquery)
query := fs.db.Where("user_id = ?", userId).Where("status = ?", "active")
if fquery.Op == "list" {
filter := &models.File{UserID: userId, Status: "active", ParentID: parentID}
query.Order("type DESC").Order(getOrder(fquery)).Model(filter).Where(&filter)
} else if fquery.Op == "find" {
if !fquery.DeepSearch && parentID != "" && (fquery.Name != "" || fquery.Query != "") {
query.Where("parent_id = ?", parentID)
fquery.Path = ""
} else if fquery.DeepSearch && parentID != "" && fquery.Query != "" {
query = fs.db.Clauses(exclause.With{Recursive: true, CTEs: []exclause.CTE{{Name: "subdirs",
Subquery: exclause.Subquery{DB: fs.db.Model(&models.File{Id: parentID}).Select("id", "parent_id").Clauses(exclause.NewUnion("ALL ?",
fs.db.Table("teldrive.files as f").Select("f.id", "f.parent_id").
Joins("inner join subdirs ON f.parent_id = subdirs.id")))}}}}).Where("files.id in (select id from subdirs)")
fquery.Path = ""
if fquery.Path != "" && fquery.ParentID == "" {
query.Where("parent_id in (SELECT id FROM teldrive.get_file_from_path(?, ?))", fquery.Path, userId)
}
if fquery.ParentID != "" {
query.Where("parent_id = ?", fquery.ParentID)
}
} else if fquery.Op == "find" {
if fquery.DeepSearch && fquery.Query != "" && fquery.Path != "" {
query.Where("files.id in (select id from subdirs)")
}
if fquery.UpdatedAt != "" {
dateFilters := strings.Split(fquery.UpdatedAt, ",")
for _, dateFilter := range dateFilters {
@ -261,7 +243,7 @@ func (fs *FileService) ListFiles(userId int64, fquery *schemas.FileQuery) (*sche
}
if fquery.Query != "" {
query.Where("name &@~ REGEXP_REPLACE(?, '[.,-_]', ' ', 'g')", fquery.Query)
query = query.Where("name &@~ REGEXP_REPLACE(?, '[.,-_]', ' ', 'g')", fquery.Query)
}
if fquery.Category != "" {
@ -272,7 +254,6 @@ func (fs *FileService) ListFiles(userId int64, fquery *schemas.FileQuery) (*sche
} else {
filterQuery = fs.db.Where("category = ?", categories[0])
}
if len(categories) > 1 {
for _, category := range categories[1:] {
if category == "folder" {
@ -283,37 +264,85 @@ func (fs *FileService) ListFiles(userId int64, fquery *schemas.FileQuery) (*sche
}
}
query.Where(filterQuery)
}
filter := &models.File{UserID: userId, Status: "active"}
filter.Name = fquery.Name
filter.ParentID = fquery.ParentID
filter.Type = fquery.Type
if fquery.Name != "" {
query.Where("name = ?", fquery.Name)
}
if fquery.ParentID != "" {
query.Where("parent_id = ?", fquery.ParentID)
}
if fquery.Type != "" {
query.Where("type = ?", fquery.Type)
}
if fquery.Starred != nil {
filter.Starred = *fquery.Starred
query.Where("starred = ?", *fquery.Starred)
}
}
orderField := utils.CamelToSnake(fquery.Sort)
var op string
if fquery.Page == 1 {
if fquery.Order == "asc" {
op = ">="
} else {
op = "<="
}
} else {
if fquery.Order == "asc" {
op = ">"
} else {
op = "<"
}
query.Order("type DESC").Order(getOrder(fquery)).
Model(&filter).Where(&filter)
query.Limit(fquery.PerPage)
setOrderFilter(query, fquery)
}
var fileQuery *gorm.DB
if fquery.DeepSearch && fquery.Query != "" && fquery.Path != "" {
fileQuery = fs.db.Clauses(exclause.With{Recursive: true, CTEs: []exclause.CTE{{Name: "subdirs",
Subquery: exclause.Subquery{DB: fs.db.Model(&models.File{}).Select("id", "parent_id").
Where("id in (SELECT id FROM teldrive.get_file_from_path(?, ?))", fquery.Path, userId).
Clauses(exclause.NewUnion("ALL ?",
fs.db.Table("teldrive.files as f").Select("f.id", "f.parent_id").
Joins("inner join subdirs ON f.parent_id = subdirs.id")))}}}})
}
if fileQuery == nil {
fileQuery = fs.db
}
fileQuery = fileQuery.Clauses(exclause.NewWith("ranked_scores", fs.db.Model(&models.File{}).Select(orderField, "count(*) OVER () as total",
fmt.Sprintf("ROW_NUMBER() OVER (ORDER BY %s %s) AS rank", orderField, strings.ToUpper(fquery.Order))).Where(query))).
Model(&models.File{}).Select("*", "(select total from ranked_scores limit 1) as total").
Where(fmt.Sprintf("%s %s (SELECT %s FROM ranked_scores WHERE rank = ?)", orderField, op, orderField),
max((fquery.Page-1)*fquery.Limit, 1)).
Where(query).Order(getOrder(fquery)).Limit(fquery.Limit)
files := []schemas.FileOut{}
query.Scan(&files)
token := ""
if len(files) == fquery.PerPage {
lastItem := files[len(files)-1]
token = utils.GetField(&lastItem, utils.CamelToPascalCase(fquery.Sort))
token = base64.StdEncoding.EncodeToString([]byte(token))
if err := fileQuery.Scan(&files).Error; err != nil {
if strings.Contains(err.Error(), "file not found") {
return nil, &types.AppError{Error: database.ErrNotFound, Code: http.StatusNotFound}
}
return nil, &types.AppError{Error: err}
}
res := &schemas.FileResponse{Files: files, NextPageToken: token}
count := 0
if len(files) > 0 {
count = files[0].Total
}
for i := range files {
files[i].Total = 0
}
res := &schemas.FileResponse{Files: files,
Meta: schemas.Meta{Count: count, TotalPages: int(math.Ceil(float64(count) / float64(fquery.Limit))),
CurrentPage: fquery.Page}}
return res, nil
}
@ -753,21 +782,6 @@ func (fs *FileService) GetFileStream(c *gin.Context, download bool) {
}
}
}
func setOrderFilter(query *gorm.DB, fquery *schemas.FileQuery) *gorm.DB {
if fquery.NextPageToken != "" {
sortColumn := utils.CamelToSnake(fquery.Sort)
tokenValue, err := base64.StdEncoding.DecodeString(fquery.NextPageToken)
if err == nil {
if fquery.Order == "asc" {
return query.Where(fmt.Sprintf("%s > ?", sortColumn), string(tokenValue))
} else {
return query.Where(fmt.Sprintf("%s < ?", sortColumn), string(tokenValue))
}
}
}
return query
}
func getOrder(fquery *schemas.FileQuery) clause.OrderByColumn {
sortColumn := utils.CamelToSnake(fquery.Sort)