diff --git a/api/router.go b/api/router.go index 80c1028..7594345 100644 --- a/api/router.go +++ b/api/router.go @@ -11,8 +11,6 @@ func InitRouter() *gin.Engine { r := gin.Default() - r.Use(gin.Logger()) - r.Use(middleware.Cors()) c := controller.NewController() diff --git a/cmd/teldrive/main.go b/cmd/teldrive/main.go index 09a571f..5863f64 100644 --- a/cmd/teldrive/main.go +++ b/cmd/teldrive/main.go @@ -4,14 +4,12 @@ import ( "fmt" "mime" "path/filepath" - "time" "github.com/divyam234/teldrive/api" "github.com/divyam234/teldrive/internal/cron" "github.com/divyam234/teldrive/internal/logger" "github.com/divyam234/teldrive/internal/utils" "github.com/divyam234/teldrive/pkg/database" - "github.com/go-co-op/gocron" cnf "github.com/divyam234/teldrive/config" "github.com/divyam234/teldrive/internal/cache" @@ -30,13 +28,7 @@ func main() { cache.InitCache() - scheduler := gocron.NewScheduler(time.UTC) - - scheduler.Every(1).Hour().Do(cron.FilesDeleteJob) - - scheduler.Every(12).Hour().Do(cron.UploadCleanJob) - - scheduler.StartAsync() + cron.StartCronJobs() mime.AddExtensionType(".js", "application/javascript") diff --git a/go.mod b/go.mod index 1708403..02ef1e6 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,6 @@ require ( github.com/thoas/go-funk v0.9.3 go.etcd.io/bbolt v1.3.8 go.uber.org/zap v1.26.0 - golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb golang.org/x/time v0.5.0 gorm.io/driver/postgres v1.5.4 gorm.io/gorm v1.25.5 diff --git a/go.sum b/go.sum index 8b8c50d..178d2f1 100644 --- a/go.sum +++ b/go.sum @@ -284,8 +284,8 @@ golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb h1:mIKbk8weKhSeLH2GmUTrvx8CjkyJmnU1wFmg59CUjFA= -golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230116083435-1de6713980de h1:DBWn//IJw30uYCgERoxCg84hWtA97F4wMiKOIh00Uf0= +golang.org/x/exp v0.0.0-20230116083435-1de6713980de/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= diff --git a/internal/cron/cron.go b/internal/cron/cron.go index 62cf010..bc0278e 100644 --- a/internal/cron/cron.go +++ b/internal/cron/cron.go @@ -12,6 +12,7 @@ import ( "github.com/divyam234/teldrive/pkg/database" "github.com/divyam234/teldrive/pkg/models" "github.com/divyam234/teldrive/pkg/services" + "github.com/go-co-op/gocron" "github.com/gotd/td/tg" ) @@ -150,7 +151,7 @@ func cleanUploadsMessages(ctx context.Context, result UploadResult) error { return nil } -func FilesDeleteJob() { +func filesDeleteJob() { db := database.DB ctx, cancel := context.WithCancel(context.Background()) @@ -173,7 +174,7 @@ func FilesDeleteJob() { } } -func UploadCleanJob() { +func uploadCleanJob() { db := database.DB ctx, cancel := context.WithCancel(context.Background()) config := cnf.GetConfig() @@ -194,3 +195,19 @@ func UploadCleanJob() { cleanUploadsMessages(ctx, row) } } + +func folderSizeUpdate() { + database.DB.Exec("call teldrive.update_size();") +} + +func StartCronJobs() { + scheduler := gocron.NewScheduler(time.UTC) + + scheduler.Every(1).Hour().Do(filesDeleteJob) + + scheduler.Every(12).Hour().Do(uploadCleanJob) + + scheduler.Every(2).Hour().Do(folderSizeUpdate) + + scheduler.StartAsync() +} diff --git a/internal/utils/utils.go b/internal/utils/utils.go index aa3b3d4..4ad6f61 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -8,25 +8,9 @@ import ( "reflect" - "golang.org/x/exp/constraints" - "unicode" ) -func Max[T constraints.Ordered](a, b T) T { - if a > b { - return a - } - return b -} - -func Min[T constraints.Ordered](a, b T) T { - if a > b { - return b - } - return a -} - func CamelToPascalCase(input string) string { var result strings.Builder upperNext := true diff --git a/pkg/database/migrations/20231102165658_tables.sql b/pkg/database/migrations/20231102165658_tables.sql index bc8d9b7..0f5fe17 100644 --- a/pkg/database/migrations/20231102165658_tables.sql +++ b/pkg/database/migrations/20231102165658_tables.sql @@ -29,16 +29,15 @@ CREATE TABLE IF NOT EXISTS teldrive.users ( CREATE TABLE IF NOT EXISTS teldrive.uploads ( - id text NOT NULL DEFAULT teldrive.generate_uid(16) PRIMARY KEY, upload_id text NOT NULL, "name" text NOT NULL, user_id bigint, part_no int NOT NULL, part_id int NOT NULL, - total_parts int NOT NULL, channel_id bigint NOT NULL, "size" bigint NOT NULL, - created_at timestamp DEFAULT timezone('utc'::text, now()) + created_at timestamp DEFAULT timezone('utc'::text, now()), + PRIMARY KEY(upload_id, part_no), ); @@ -88,7 +87,7 @@ DROP INDEX IF EXISTS parent_name_numeric_idx ; DROP INDEX IF EXISTS path_idx ; DROP INDEX IF EXISTS starred_updated_at_idx ; DROP INDEX IF EXISTS status_idx ; -DROP UNIQUE IF EXISTS INDEX unique_file; +DROP INDEX IF EXISTS unique_file; DROP INDEX IF EXISTS user_id_idx; DROP TABLE IF EXISTS teldrive.files; diff --git a/pkg/database/migrations/20231104192800_functions.sql b/pkg/database/migrations/20231104192800_functions.sql index f486b19..f6edc9b 100644 --- a/pkg/database/migrations/20231104192800_functions.sql +++ b/pkg/database/migrations/20231104192800_functions.sql @@ -8,7 +8,7 @@ DECLARE BEGIN FOR rec IN SELECT id - FROM files + FROM teldrive.files WHERE type = 'folder' ORDER BY depth DESC LOOP @@ -26,43 +26,56 @@ END; $$; -CREATE OR REPLACE PROCEDURE teldrive.delete_files(IN file_ids TEXT[], IN op TEXT DEFAULT 'bulk') LANGUAGE PLPGSQL AS $$ +CREATE OR REPLACE PROCEDURE teldrive.delete_files( + IN file_ids TEXT[], + IN op TEXT DEFAULT 'bulk' +) LANGUAGE plpgsql +AS $$ DECLARE - rec RECORD; + rec RECORD; BEGIN - IF op = 'bulk' THEN - FOR rec IN - SELECT id, type - FROM teldrive.files - WHERE id = ANY (file_ids) - LOOP - IF rec.type = 'folder' THEN - CALL teldrive.delete_files(ARRAY[rec.id], 'single'); - END IF; + IF op = 'bulk' THEN + FOR rec IN + SELECT + id, + type + FROM + teldrive.files + WHERE + id = ANY (file_ids) + LOOP + IF rec.type = 'folder' THEN + CALL teldrive.delete_files(ARRAY[rec.id], 'single'); + DELETE FROM teldrive.files WHERE id = rec.id; + ELSE + UPDATE teldrive.files SET status = 'pending_deletion' WHERE id = rec.id; - DELETE FROM teldrive.files - WHERE id = rec.id; - END LOOP; - ELSE - FOR rec IN - SELECT id, type - FROM teldrive.files - WHERE parent_id = file_ids[1] - LOOP - IF rec.type = 'folder' THEN - CALL teldrive.delete_files(ARRAY[rec.id], 'single'); - END IF; - - DELETE FROM teldrive.files - WHERE id = rec.id; - END LOOP; - END IF; + END IF; + END LOOP; + ELSE + FOR rec IN + SELECT + id, + type + FROM + teldrive.files + WHERE + parent_id = file_ids[1] + LOOP + IF rec.type = 'folder' THEN + CALL teldrive.delete_files(ARRAY[rec.id], 'single'); + DELETE FROM teldrive.files WHERE id = rec.id; + ELSE + UPDATE teldrive.files SET status = 'pending_deletion' WHERE id = rec.id; + END IF; + END LOOP; + END IF; END; $$; CREATE OR REPLACE FUNCTION teldrive.create_directories( - IN tg_id BIGINT, + IN u_id BIGINT, IN long_path TEXT ) RETURNS SETOF teldrive.files AS $$ DECLARE @@ -80,7 +93,7 @@ BEGIN SELECT id INTO current_directory_id FROM teldrive.files - WHERE parent_id = 'root' AND user_id = tg_id; + WHERE parent_id = 'root' AND user_id = u_id; FOR directory_name IN SELECT unnest(path_parts) LOOP path_so_far := CONCAT(path_so_far, '/', directory_name); @@ -90,11 +103,11 @@ BEGIN FROM teldrive.files WHERE parent_id = current_directory_id AND "name" = directory_name - AND "user_id" = tg_id; + AND "user_id" = u_id; IF new_directory_id IS NULL THEN INSERT INTO teldrive.files ("name", "type", mime_type, parent_id, "user_id", starred, "depth", "path") - VALUES (directory_name, 'folder', 'teldrive/folder', current_directory_id, tg_id, false, depth_dir, path_so_far) + VALUES (directory_name, 'folder', 'drive/folder', current_directory_id, u_id, false, depth_dir, path_so_far) RETURNING id INTO new_directory_id; END IF; @@ -234,20 +247,21 @@ $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION teldrive.account_stats( IN u_id BIGINT -) RETURNS TABLE (total_size BIGINT, total_files BIGINT, ch_id BIGINT,ch_name TEXT ) AS $$ +) RETURNS TABLE (total_size BIGINT, total_files BIGINT, channel_id BIGINT, channel_name TEXT ) AS $$ DECLARE total_size BIGINT; total_files BIGINT; - ch_id BIGINT; - ch_name TEXT; + channel_id BIGINT; + channel_name TEXT; BEGIN - SELECT COUNT(*), SUM(size) into total_files,total_size FROM teldrive.files WHERE user_id=u_id AND type= 'file' and status='active'; - SELECT channel_id ,channel_name into ch_id,ch_name FROM teldrive.channels WHERE selected=TRUE AND user_id=u_id; - RETURN QUERY SELECT total_size,total_files,ch_id,ch_name; + SELECT COUNT(*), coalesce(SUM(size),0) into total_files,total_size FROM teldrive.files WHERE user_id=u_id AND type= 'file' and status='active'; + SELECT c.channel_id ,c.channel_name into channel_id, channel_name FROM teldrive.channels c WHERE selected=TRUE AND user_id=u_id; + RETURN QUERY SELECT total_size,total_files,channel_id,channel_name; END; $$ LANGUAGE plpgsql; + -- +goose StatementEnd -- +goose Down diff --git a/pkg/httputil/error.go b/pkg/httputil/error.go index 3029e92..717382a 100644 --- a/pkg/httputil/error.go +++ b/pkg/httputil/error.go @@ -11,6 +11,6 @@ func NewError(ctx *gin.Context, status int, err error) { } type HTTPError struct { - Code int `json:"code" example:"400"` - Message string `json:"message" example:"status bad request"` + Code int `json:"code"` + Message string `json:"message"` } diff --git a/pkg/mapper/mapper.go b/pkg/mapper/mapper.go index 21c5a11..f6d0af7 100644 --- a/pkg/mapper/mapper.go +++ b/pkg/mapper/mapper.go @@ -40,7 +40,6 @@ func ToFileOutFull(file models.File) *schemas.FileOutFull { func ToUploadOut(in *models.Upload) *schemas.UploadPartOut { out := &schemas.UploadPartOut{ - ID: in.ID, Name: in.Name, PartId: in.PartId, ChannelID: in.ChannelID, diff --git a/pkg/models/upload.go b/pkg/models/upload.go index 3949701..222b39c 100644 --- a/pkg/models/upload.go +++ b/pkg/models/upload.go @@ -5,14 +5,12 @@ import ( ) type Upload struct { - ID string `gorm:"type:text;primary_key;default:generate_uid(16)"` - UploadId string `gorm:"type:text"` - UserId int64 `gorm:"type:bigint"` - Name string `gorm:"type:text"` - PartNo int `gorm:"type:integer"` - TotalParts int `gorm:"type:integer"` - PartId int `gorm:"type:integer"` - ChannelID int64 `gorm:"type:bigint"` - Size int64 `gorm:"type:bigint"` - CreatedAt time.Time `gorm:"default:timezone('utc'::text, now())"` + UploadId string `gorm:"type:text"` + UserId int64 `gorm:"type:bigint"` + Name string `gorm:"type:text"` + PartNo int `gorm:"type:integer"` + PartId int `gorm:"type:integer"` + ChannelID int64 `gorm:"type:bigint"` + Size int64 `gorm:"type:bigint"` + CreatedAt time.Time `gorm:"default:timezone('utc'::text, now())"` } diff --git a/pkg/schemas/upload.go b/pkg/schemas/upload.go index 6f511b6..24379c9 100644 --- a/pkg/schemas/upload.go +++ b/pkg/schemas/upload.go @@ -1,10 +1,9 @@ package schemas type UploadQuery struct { - Filename string `form:"fileName"` - PartNo int `form:"partNo,omitempty"` - TotalParts int `form:"totalparts"` - ChannelID int64 `form:"channelId"` + Filename string `form:"fileName" binding:"required` + PartNo int `form:"partNo" binding:"required` + ChannelID int64 `form:"channelId" binding:"required` } type UploadPartOut struct { diff --git a/pkg/schemas/user.go b/pkg/schemas/user.go index ef23885..29c5968 100644 --- a/pkg/schemas/user.go +++ b/pkg/schemas/user.go @@ -1,13 +1,12 @@ package schemas -type AccountStats struct { - TotalSize int64 `json:"totalSize"` - TotalFiles int64 `json:"totalFiles"` - ChId int64 `json:"channelId,omitempty"` - ChName string `json:"channelName,omitempty"` -} - type Channel struct { ChannelID int64 `json:"channelId"` ChannelName string `json:"channelName"` } + +type AccountStats struct { + TotalSize int64 `json:"totalSize"` + TotalFiles int64 `json:"totalFiles"` + Channel +} diff --git a/pkg/services/file.go b/pkg/services/file.go index f48bf11..d4195d0 100644 --- a/pkg/services/file.go +++ b/pkg/services/file.go @@ -575,7 +575,7 @@ func (fs *FileService) GetFileStream(c *gin.Context) { channelUser = strconv.FormatInt(session.UserId, 10) } else { var index int - limit := utils.Min(len(tokens), config.BgBotsLimit) + limit := min(len(tokens), config.BgBotsLimit) tgc.StreamWorkers.Set(tokens[:limit], file.ChannelID) diff --git a/pkg/services/upload.go b/pkg/services/upload.go index 81408a2..3bd9589 100644 --- a/pkg/services/upload.go +++ b/pkg/services/upload.go @@ -65,14 +65,13 @@ func (us *UploadService) CreateUploadPart(c *gin.Context) (*schemas.UploadPartOu } partUpload := &models.Upload{ - Name: payload.Name, - UploadId: payload.UploadId, - PartId: payload.PartId, - ChannelID: payload.ChannelID, - Size: payload.Size, - PartNo: payload.PartNo, - TotalParts: 1, - UserId: userId, + Name: payload.Name, + UploadId: payload.UploadId, + PartId: payload.PartId, + ChannelID: payload.ChannelID, + Size: payload.Size, + PartNo: payload.PartNo, + UserId: userId, } if err := us.Db.Create(partUpload).Error; err != nil { @@ -97,7 +96,6 @@ func (us *UploadService) UploadFile(c *gin.Context) (*schemas.UploadPartOut, *ty ) uploadQuery.PartNo = 1 - uploadQuery.TotalParts = 1 if err := c.ShouldBindQuery(&uploadQuery); err != nil { return nil, &types.AppError{Error: err, Code: http.StatusBadRequest} @@ -191,14 +189,13 @@ func (us *UploadService) UploadFile(c *gin.Context) (*schemas.UploadPartOut, *ty } partUpload := &models.Upload{ - Name: fileName, - UploadId: uploadId, - PartId: message.ID, - ChannelID: channelId, - Size: fileSize, - PartNo: uploadQuery.PartNo, - TotalParts: uploadQuery.TotalParts, - UserId: userId, + Name: fileName, + UploadId: uploadId, + PartId: message.ID, + ChannelID: channelId, + Size: fileSize, + PartNo: uploadQuery.PartNo, + UserId: userId, } if err := us.Db.Create(partUpload).Error; err != nil {