mirror of
https://github.com/tgdrive/teldrive.git
synced 2025-09-05 05:54:55 +08:00
feat: add new check command to check integrity of file parts
This commit is contained in:
Bhunter
parent
a1875379a7
commit
0540ae1075
11 changed files with 498 additions and 49 deletions
413
cmd/check.go
Normal file
413
cmd/check.go
Normal file
|
|
@ -0,0 +1,413 @@
|
|||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/gotd/td/telegram"
|
||||
"github.com/gotd/td/telegram/query"
|
||||
"github.com/gotd/td/telegram/query/messages"
|
||||
"github.com/gotd/td/tg"
|
||||
"github.com/k0kubun/go-ansi"
|
||||
"github.com/manifoldco/promptui"
|
||||
"github.com/schollz/progressbar/v3"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/tgdrive/teldrive/internal/api"
|
||||
"github.com/tgdrive/teldrive/internal/config"
|
||||
"github.com/tgdrive/teldrive/internal/database"
|
||||
"github.com/tgdrive/teldrive/internal/logging"
|
||||
"github.com/tgdrive/teldrive/internal/tgc"
|
||||
"github.com/tgdrive/teldrive/internal/utils"
|
||||
"github.com/tgdrive/teldrive/pkg/models"
|
||||
"golang.org/x/term"
|
||||
"gorm.io/datatypes"
|
||||
)
|
||||
|
||||
type channel struct {
|
||||
tg.InputPeerChannel
|
||||
ChannelName string
|
||||
}
|
||||
|
||||
type file struct {
|
||||
ID string
|
||||
Name string
|
||||
Parts datatypes.JSONSlice[api.Part]
|
||||
}
|
||||
|
||||
type exportFile struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
type channelExport struct {
|
||||
ChannelID int64 `json:"channel_id"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
FileCount int `json:"file_count"`
|
||||
Files []exportFile `json:"files"`
|
||||
}
|
||||
|
||||
const dateLayout = "2006-01-02_15-04-05"
|
||||
|
||||
var termWidth = func() (width int, err error) {
|
||||
width, _, err = term.GetSize(int(os.Stdout.Fd()))
|
||||
if err == nil {
|
||||
return width, nil
|
||||
}
|
||||
|
||||
return 0, err
|
||||
}
|
||||
|
||||
func NewCheckCmd() *cobra.Command {
|
||||
var cfg config.ServerCmdConfig
|
||||
cmd := &cobra.Command{
|
||||
Use: "check",
|
||||
Short: "Check and purge incomplete files",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
runCheckCmd(cmd, &cfg)
|
||||
},
|
||||
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
loader := config.NewConfigLoader()
|
||||
if err := loader.InitializeConfig(cmd); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := loader.Load(&cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := checkRequiredCheckFlags(&cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
addChecktFlags(cmd, &cfg)
|
||||
return cmd
|
||||
}
|
||||
|
||||
func addChecktFlags(cmd *cobra.Command, cfg *config.ServerCmdConfig) {
|
||||
flags := cmd.Flags()
|
||||
config.AddCommonFlags(flags, cfg)
|
||||
flags.Bool("export", true, "Export incomplete files to json file")
|
||||
flags.Bool("clean", false, "Clean missing and orphan file parts")
|
||||
flags.String("user", "", "Telegram User Name")
|
||||
}
|
||||
|
||||
func checkRequiredCheckFlags(cfg *config.ServerCmdConfig) error {
|
||||
var missingFields []string
|
||||
|
||||
if cfg.DB.DataSource == "" {
|
||||
missingFields = append(missingFields, "db-data-source")
|
||||
}
|
||||
if len(missingFields) > 0 {
|
||||
return fmt.Errorf("required configuration values not set: %s", strings.Join(missingFields, ", "))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func selectUser(user string, users []models.User) (*models.User, error) {
|
||||
if user != "" {
|
||||
res := utils.Filter(users, func(u models.User) bool {
|
||||
return u.UserName == user
|
||||
})
|
||||
if len(res) == 0 {
|
||||
return nil, fmt.Errorf("invalid user name: %s", user)
|
||||
}
|
||||
return &res[0], nil
|
||||
}
|
||||
templates := &promptui.SelectTemplates{
|
||||
Label: "{{ . }}",
|
||||
Active: "{{ .UserName | cyan }}",
|
||||
Inactive: "{{ .UserName | white }}",
|
||||
Selected: "{{ .UserName | red | cyan }}",
|
||||
}
|
||||
|
||||
prompt := promptui.Select{
|
||||
Label: "Select User",
|
||||
Items: users,
|
||||
Templates: templates,
|
||||
Size: 50,
|
||||
}
|
||||
|
||||
index, _, err := prompt.Run()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &users[index], nil
|
||||
}
|
||||
|
||||
func runCheckCmd(cmd *cobra.Command, cfg *config.ServerCmdConfig) {
|
||||
|
||||
lg := logging.DefaultLogger().Sugar()
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT)
|
||||
|
||||
defer func() {
|
||||
stop()
|
||||
logging.DefaultLogger().Sync()
|
||||
}()
|
||||
|
||||
db, err := database.NewDatabase(&cfg.DB, lg)
|
||||
|
||||
if err != nil {
|
||||
lg.Fatalw("failed to create database", "err", err)
|
||||
}
|
||||
|
||||
users := []models.User{}
|
||||
if err := db.Model(&models.User{}).Find(&users).Error; err != nil {
|
||||
lg.Fatalw("failed to get users", "err", err)
|
||||
}
|
||||
userName, _ := cmd.Flags().GetString("user")
|
||||
user, err := selectUser(userName, users)
|
||||
if err != nil {
|
||||
lg.Fatalw("failed to select user", "err", err)
|
||||
}
|
||||
session := models.Session{}
|
||||
if err := db.Model(&models.Session{}).Where("user_id = ?", user.UserId).Order("created_at desc").First(&session).Error; err != nil {
|
||||
lg.Fatalw("failed to get session", "err", err)
|
||||
}
|
||||
channelIds := []int64{}
|
||||
|
||||
if err := db.Model(&models.Channel{}).Where("user_id = ?", user.UserId).Pluck("channel_id", &channelIds).Error; err != nil {
|
||||
lg.Fatalw("failed to get channels", "err", err)
|
||||
}
|
||||
if len(channelIds) == 0 {
|
||||
lg.Fatalw("no channels found")
|
||||
}
|
||||
|
||||
tgconfig := &config.TGConfig{
|
||||
RateLimit: true,
|
||||
RateBurst: 5,
|
||||
Rate: 100,
|
||||
}
|
||||
|
||||
middlewares := tgc.NewMiddleware(tgconfig, tgc.WithFloodWait(), tgc.WithRateLimit())
|
||||
|
||||
export, _ := cmd.Flags().GetBool("export")
|
||||
|
||||
clean, _ := cmd.Flags().GetBool("clean")
|
||||
|
||||
var channelExports []channelExport
|
||||
for _, id := range channelIds {
|
||||
lg.Info("Processing channel: ", id)
|
||||
const batchSize = 1000
|
||||
var offset int
|
||||
var files []file
|
||||
|
||||
for {
|
||||
var batch []file
|
||||
result := db.Model(&models.File{}).
|
||||
Offset(offset).
|
||||
Limit(batchSize).
|
||||
Where("user_id = ?", user.UserId).
|
||||
Where("channel_id = ?", id).
|
||||
Where("type = ?", "file").
|
||||
Scan(&batch)
|
||||
if result.Error != nil {
|
||||
lg.Errorw("failed to load files", "err", result.Error)
|
||||
break
|
||||
}
|
||||
|
||||
files = append(files, batch...)
|
||||
if len(batch) < batchSize {
|
||||
break
|
||||
}
|
||||
offset += batchSize
|
||||
}
|
||||
if len(files) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
lg.Infof("Channel %d: %d files found", id, len(files))
|
||||
|
||||
lg.Infof("Loading messages from telegram")
|
||||
|
||||
client, err := tgc.AuthClient(ctx, tgconfig, session.Session, middlewares...)
|
||||
|
||||
if err != nil {
|
||||
lg.Fatalw("failed to create client", "err", err)
|
||||
}
|
||||
|
||||
msgs, total, err := loadChannelMessages(ctx, client, id)
|
||||
if err != nil {
|
||||
lg.Fatalw("failed to load channel messages", "err", err)
|
||||
}
|
||||
if total == 0 && len(msgs) == 0 {
|
||||
lg.Infof("Channel %d: no messages found", id)
|
||||
continue
|
||||
}
|
||||
if len(msgs) < total {
|
||||
lg.Fatalf("Channel %d: found %d messages out of %d", id, len(msgs), total)
|
||||
continue
|
||||
}
|
||||
uploadPartIds := []int{}
|
||||
if err := db.Model(&models.Upload{}).Where("user_id = ?", user.UserId).Where("channel_id = ?", id).
|
||||
Pluck("part_id", &uploadPartIds).Error; err != nil {
|
||||
lg.Errorw("failed to get upload part ids", "err", err)
|
||||
}
|
||||
|
||||
uploadPartMap := make(map[int]bool)
|
||||
|
||||
for _, partID := range uploadPartIds {
|
||||
uploadPartMap[partID] = true
|
||||
}
|
||||
msgMap := make(map[int]bool)
|
||||
for _, m := range msgs {
|
||||
if m > 0 && !uploadPartMap[m] {
|
||||
msgMap[m] = true
|
||||
}
|
||||
}
|
||||
filesWithMissingParts := []file{}
|
||||
allPartIDs := make(map[int]bool)
|
||||
for _, f := range files {
|
||||
for _, p := range f.Parts {
|
||||
if p.ID == 0 {
|
||||
filesWithMissingParts = append(filesWithMissingParts, f)
|
||||
break
|
||||
}
|
||||
allPartIDs[p.ID] = true
|
||||
}
|
||||
}
|
||||
if len(allPartIDs) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, f := range files {
|
||||
for _, p := range f.Parts {
|
||||
if !msgMap[p.ID] {
|
||||
filesWithMissingParts = append(filesWithMissingParts, f)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
missingMsgIDs := []int{}
|
||||
for msgID := range msgMap {
|
||||
if !allPartIDs[msgID] {
|
||||
missingMsgIDs = append(missingMsgIDs, msgID)
|
||||
}
|
||||
}
|
||||
|
||||
if len(filesWithMissingParts) > 0 {
|
||||
lg.Infof("Channel %d: found %d files with missing parts", id, len(filesWithMissingParts))
|
||||
}
|
||||
|
||||
if export && len(filesWithMissingParts) > 0 {
|
||||
channelData := channelExport{
|
||||
ChannelID: id,
|
||||
Timestamp: time.Now().Format(time.RFC3339),
|
||||
FileCount: len(filesWithMissingParts),
|
||||
Files: make([]exportFile, 0, len(filesWithMissingParts)),
|
||||
}
|
||||
|
||||
for _, f := range filesWithMissingParts {
|
||||
channelData.Files = append(channelData.Files, exportFile{
|
||||
ID: f.ID,
|
||||
Name: f.Name,
|
||||
})
|
||||
}
|
||||
if clean {
|
||||
err = db.Exec("call teldrive.delete_files_bulk($1 , $2)",
|
||||
utils.Map(filesWithMissingParts, func(f file) string { return f.ID }), user.UserId).Error
|
||||
if err != nil {
|
||||
lg.Errorw("failed to delete files", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
channelExports = append(channelExports, channelData)
|
||||
}
|
||||
if clean && len(missingMsgIDs) > 0 {
|
||||
lg.Infof("Channel %d: cleaning %d orphan messages", id, len(missingMsgIDs))
|
||||
tgc.DeleteMessages(ctx, client, id, missingMsgIDs)
|
||||
}
|
||||
|
||||
}
|
||||
if len(channelExports) > 0 {
|
||||
jsonData, err := json.MarshalIndent(channelExports, "", " ")
|
||||
if err != nil {
|
||||
lg.Errorw("failed to marshal JSON", "err", err)
|
||||
return
|
||||
}
|
||||
err = os.WriteFile("missing_files.json", jsonData, 0644)
|
||||
if err != nil {
|
||||
lg.Errorw("failed to write JSON file", "err", err)
|
||||
return
|
||||
}
|
||||
lg.Infof("Exported data to missing_files.json")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func loadChannelMessages(ctx context.Context, client *telegram.Client, channelId int64) (msgs []int, total int, err error) {
|
||||
errChan := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
errChan <- tgc.RunWithAuth(ctx, client, "", func(ctx context.Context) error {
|
||||
channel, err := tgc.GetChannelById(ctx, client.API(), channelId)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
count := 0
|
||||
|
||||
q := query.NewQuery(client.API()).Messages().GetHistory(&tg.InputPeerChannel{
|
||||
ChannelID: channelId,
|
||||
AccessHash: channel.AccessHash,
|
||||
})
|
||||
|
||||
msgiter := messages.NewIterator(q, 100)
|
||||
|
||||
total, err = msgiter.Total(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get total messages: %w", err)
|
||||
}
|
||||
|
||||
width, err := termWidth()
|
||||
if err != nil {
|
||||
width = 50
|
||||
}
|
||||
bar := progressbar.NewOptions(
|
||||
total,
|
||||
progressbar.OptionSetWriter(ansi.NewAnsiStdout()),
|
||||
progressbar.OptionEnableColorCodes(true),
|
||||
progressbar.OptionThrottle(65*time.Millisecond),
|
||||
progressbar.OptionShowCount(),
|
||||
progressbar.OptionSetWidth(width/3),
|
||||
progressbar.OptionShowIts(),
|
||||
progressbar.OptionSetTheme(progressbar.Theme{
|
||||
Saucer: "[green]=[reset]",
|
||||
SaucerHead: "[green]>[reset]",
|
||||
SaucerPadding: " ",
|
||||
BarStart: "[",
|
||||
BarEnd: "]",
|
||||
}),
|
||||
)
|
||||
|
||||
defer bar.Clear()
|
||||
|
||||
for msgiter.Next(ctx) {
|
||||
msg := msgiter.Value()
|
||||
msgs = append(msgs, msg.Msg.GetID())
|
||||
count++
|
||||
bar.Set(count)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}()
|
||||
|
||||
select {
|
||||
case err = <-errChan:
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
case <-ctx.Done():
|
||||
fmt.Print("\r\033[K")
|
||||
err = ctx.Err()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
|
@ -12,6 +12,6 @@ func New() *cobra.Command {
|
|||
cmd.Help()
|
||||
},
|
||||
}
|
||||
cmd.AddCommand(NewRun(), NewVersion())
|
||||
cmd.AddCommand(NewRun(), NewCheckCmd(), NewVersion())
|
||||
return cmd
|
||||
}
|
||||
|
|
|
|||
11
cmd/run.go
11
cmd/run.go
|
|
@ -90,6 +90,17 @@ func addServerFlags(cmd *cobra.Command, cfg *config.ServerCmdConfig) {
|
|||
duration.DurationVar(flags, &cfg.JWT.SessionTime, "jwt-session-time", (30*24)*time.Hour, "JWT session duration")
|
||||
flags.StringSliceVar(&cfg.JWT.AllowedUsers, "jwt-allowed-users", []string{}, "Allowed users")
|
||||
|
||||
// Telegram config
|
||||
flags.StringVar(&cfg.TG.StorageFile, "tg-storage-file", "", "Sqlite Storage file path")
|
||||
flags.BoolVar(&cfg.TG.RateLimit, "tg-rate-limit", true, "Enable rate limiting for telegram client")
|
||||
flags.IntVar(&cfg.TG.RateBurst, "tg-rate-burst", 5, "Limiting burst for telegram client")
|
||||
flags.IntVar(&cfg.TG.Rate, "tg-rate", 100, "Limiting rate for telegram client")
|
||||
flags.StringVar(&cfg.TG.Proxy, "tg-proxy", "", "HTTP OR SOCKS5 proxy URL")
|
||||
flags.BoolVar(&cfg.TG.DisableStreamBots, "tg-disable-stream-bots", false, "Disable Stream bots")
|
||||
flags.BoolVar(&cfg.TG.Ntp, "tg-ntp", false, "Use NTP server time")
|
||||
flags.BoolVar(&cfg.TG.EnableLogging, "tg-enable-logging", false, "Enable telegram client logging")
|
||||
flags.Int64Var(&cfg.TG.PoolSize, "tg-pool-size", 8, "Telegram Session pool size")
|
||||
|
||||
// Telegram Uploads config
|
||||
flags.StringVar(&cfg.TG.Uploads.EncryptionKey, "tg-uploads-encryption-key", "", "Uploads encryption key")
|
||||
flags.IntVar(&cfg.TG.Uploads.Threads, "tg-uploads-threads", 8, "Uploads threads")
|
||||
|
|
|
|||
7
go.mod
7
go.mod
|
|
@ -16,13 +16,17 @@ require (
|
|||
github.com/gotd/contrib v0.21.0
|
||||
github.com/gotd/td v0.117.0
|
||||
github.com/iyear/connectproxy v0.1.1
|
||||
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213
|
||||
github.com/manifoldco/promptui v0.9.0
|
||||
github.com/ogen-go/ogen v1.8.1
|
||||
github.com/redis/go-redis/v9 v9.7.0
|
||||
github.com/schollz/progressbar/v3 v3.18.0
|
||||
github.com/spf13/cobra v1.8.1
|
||||
github.com/spf13/pflag v1.0.5
|
||||
github.com/spf13/viper v1.19.0
|
||||
github.com/vmihailenco/msgpack/v5 v5.4.1
|
||||
go.uber.org/zap v1.27.0
|
||||
golang.org/x/term v0.28.0
|
||||
golang.org/x/time v0.9.0
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1
|
||||
gorm.io/datatypes v1.2.5
|
||||
|
|
@ -34,6 +38,7 @@ require (
|
|||
filippo.io/edwards25519 v1.1.0 // indirect
|
||||
github.com/beevik/ntp v1.4.3 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
|
||||
github.com/coder/websocket v1.8.12 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
|
|
@ -52,10 +57,12 @@ require (
|
|||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-sqlite3 v1.14.24 // indirect
|
||||
github.com/mfridman/interpolate v0.0.2 // indirect
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/ncruces/go-strftime v0.1.9 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/robfig/cron/v3 v3.0.1 // indirect
|
||||
github.com/sagikazarmark/locafero v0.7.0 // indirect
|
||||
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
|
||||
|
|
|
|||
23
go.sum
23
go.sum
|
|
@ -21,6 +21,14 @@ github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyY
|
|||
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM=
|
||||
github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY=
|
||||
github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE=
|
||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8=
|
||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8=
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I=
|
||||
github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8=
|
||||
github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4=
|
||||
|
|
@ -135,6 +143,8 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
|
|||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
|
||||
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcMpsXCp3lJ03zYT1PkRd3kQGPn9GVg=
|
||||
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
|
||||
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
|
||||
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
|
|
@ -148,16 +158,22 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
|||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/magiconair/properties v1.8.9 h1:nWcCbLq1N2v/cpNsy5WvQ37Fb+YElfq20WJ/a8RkpQM=
|
||||
github.com/magiconair/properties v1.8.9/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
|
||||
github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYthEiA=
|
||||
github.com/manifoldco/promptui v0.9.0/go.mod h1:ka04sppxSGFAtxX0qhlYQjISsg9mR4GWtQEhdbn6Pgg=
|
||||
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
|
||||
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
|
||||
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM=
|
||||
github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=
|
||||
github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
|
||||
github.com/microsoft/go-mssqldb v1.8.0 h1:7cyZ/AT7ycDsEoWPIXibd+aVKFtteUNhDGf3aobP+tw=
|
||||
github.com/microsoft/go-mssqldb v1.8.0/go.mod h1:6znkekS3T2vp0waiMhen4GPU1BiAsrP+iXHcE7a7rFo=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
|
||||
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
|
||||
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
|
|
@ -188,6 +204,8 @@ github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa
|
|||
github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
||||
|
|
@ -199,6 +217,8 @@ github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsF
|
|||
github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k=
|
||||
github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
|
||||
github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
|
||||
github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA=
|
||||
github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
|
||||
github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
|
||||
github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
|
||||
github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE=
|
||||
|
|
@ -259,9 +279,12 @@ golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
|
|||
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
|
||||
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg=
|
||||
golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
|
||||
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY=
|
||||
|
|
|
|||
10
internal/cache/cache.go
vendored
10
internal/cache/cache.go
vendored
|
|
@ -149,6 +149,16 @@ func Fetch[T any](cache Cacher, key string, expiration time.Duration, fn func()
|
|||
return value, nil
|
||||
}
|
||||
|
||||
func FetchArg[T any, A any](
|
||||
cache Cacher,
|
||||
key string,
|
||||
expiration time.Duration,
|
||||
fn func(a A) (T, error), a A) (T, error) {
|
||||
return Fetch(cache, key, expiration, func() (T, error) {
|
||||
return fn(a)
|
||||
})
|
||||
}
|
||||
|
||||
func Key(args ...interface{}) string {
|
||||
parts := make([]string, len(args))
|
||||
for i, arg := range args {
|
||||
|
|
|
|||
|
|
@ -208,23 +208,4 @@ func AddCommonFlags(flags *pflag.FlagSet, config *ServerCmdConfig) {
|
|||
flags.IntVar(&config.DB.Pool.MaxIdleConnections, "db-pool-max-idle-connections", 25, "Database max idle connections")
|
||||
duration.DurationVar(flags, &config.DB.Pool.MaxLifetime, "db-pool-max-lifetime", 10*time.Minute, "Database max connection lifetime")
|
||||
|
||||
// Telegram config
|
||||
flags.IntVar(&config.TG.AppId, "tg-app-id", 0, "Telegram app ID")
|
||||
flags.StringVar(&config.TG.AppHash, "tg-app-hash", "", "Telegram app hash")
|
||||
flags.StringVar(&config.TG.StorageFile, "tg-storage-file", "", "Sqlite Storage file path")
|
||||
flags.BoolVar(&config.TG.RateLimit, "tg-rate-limit", true, "Enable rate limiting for telegram client")
|
||||
flags.IntVar(&config.TG.RateBurst, "tg-rate-burst", 5, "Limiting burst for telegram client")
|
||||
flags.IntVar(&config.TG.Rate, "tg-rate", 100, "Limiting rate for telegram client")
|
||||
flags.StringVar(&config.TG.DeviceModel, "tg-device-model",
|
||||
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/116.0", "Device model")
|
||||
flags.StringVar(&config.TG.SystemVersion, "tg-system-version", "Win32", "System version")
|
||||
flags.StringVar(&config.TG.AppVersion, "tg-app-version", "4.6.3 K", "App version")
|
||||
flags.StringVar(&config.TG.LangCode, "tg-lang-code", "en", "Language code")
|
||||
flags.StringVar(&config.TG.SystemLangCode, "tg-system-lang-code", "en-US", "System language code")
|
||||
flags.StringVar(&config.TG.LangPack, "tg-lang-pack", "webk", "Language pack")
|
||||
flags.StringVar(&config.TG.Proxy, "tg-proxy", "", "HTTP OR SOCKS5 proxy URL")
|
||||
flags.BoolVar(&config.TG.DisableStreamBots, "tg-disable-stream-bots", false, "Disable Stream bots")
|
||||
flags.BoolVar(&config.TG.Ntp, "tg-ntp", false, "Use NTP server time")
|
||||
flags.BoolVar(&config.TG.EnableLogging, "tg-enable-logging", false, "Enable telegram client logging")
|
||||
flags.Int64Var(&config.TG.PoolSize, "tg-pool-size", 8, "Telegram Session pool size")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,6 +26,20 @@ import (
|
|||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const (
|
||||
AppId = 2040
|
||||
AppHash = "b18441a1ff607e10a989891a5462e627"
|
||||
)
|
||||
|
||||
var Device = telegram.DeviceConfig{
|
||||
DeviceModel: "Desktop",
|
||||
SystemVersion: "Windows 11",
|
||||
AppVersion: "5.10.3 x64",
|
||||
LangCode: "en",
|
||||
SystemLangCode: "en-US",
|
||||
LangPack: "tdesktop",
|
||||
}
|
||||
|
||||
func newClient(ctx context.Context, config *config.TGConfig, handler telegram.UpdateHandler, storage session.Storage, middlewares ...telegram.Middleware) (*telegram.Client, error) {
|
||||
|
||||
var dialer dcs.DialFunc = proxy.Direct.DialContext
|
||||
|
|
@ -50,14 +64,7 @@ func newClient(ctx context.Context, config *config.TGConfig, handler telegram.Up
|
|||
ReconnectionBackoff: func() backoff.BackOff {
|
||||
return newBackoff(config.ReconnectTimeout)
|
||||
},
|
||||
Device: telegram.DeviceConfig{
|
||||
DeviceModel: config.DeviceModel,
|
||||
SystemVersion: config.SystemVersion,
|
||||
AppVersion: config.AppVersion,
|
||||
SystemLangCode: config.SystemLangCode,
|
||||
LangPack: config.LangPack,
|
||||
LangCode: config.LangCode,
|
||||
},
|
||||
Device: Device,
|
||||
SessionStorage: storage,
|
||||
RetryInterval: 2 * time.Second,
|
||||
MaxRetries: 10,
|
||||
|
|
@ -75,7 +82,7 @@ func newClient(ctx context.Context, config *config.TGConfig, handler telegram.Up
|
|||
|
||||
}
|
||||
|
||||
return telegram.NewClient(config.AppId, config.AppHash, opts), nil
|
||||
return telegram.NewClient(AppId, AppHash, opts), nil
|
||||
}
|
||||
|
||||
func NoAuthClient(ctx context.Context, config *config.TGConfig, handler telegram.UpdateHandler, storage session.Storage) (*telegram.Client, error) {
|
||||
|
|
|
|||
|
|
@ -7,11 +7,13 @@ import (
|
|||
"github.com/go-faster/errors"
|
||||
"github.com/gotd/td/telegram"
|
||||
"github.com/ogen-go/ogen/ogenerrors"
|
||||
"go.uber.org/zap"
|
||||
|
||||
ht "github.com/ogen-go/ogen/http"
|
||||
"github.com/tgdrive/teldrive/internal/api"
|
||||
"github.com/tgdrive/teldrive/internal/cache"
|
||||
"github.com/tgdrive/teldrive/internal/config"
|
||||
"github.com/tgdrive/teldrive/internal/logging"
|
||||
"github.com/tgdrive/teldrive/internal/tgc"
|
||||
"github.com/tgdrive/teldrive/internal/version"
|
||||
"gorm.io/gorm"
|
||||
|
|
@ -45,8 +47,14 @@ func (a *apiService) NewError(ctx context.Context, err error) *api.ErrorStatusCo
|
|||
code = ogenErr.Code()
|
||||
message = ogenErr.Error()
|
||||
case errors.As(err, &apiError):
|
||||
code = apiError.Code()
|
||||
message = apiError.Error()
|
||||
if apiError.code == 0 {
|
||||
code = http.StatusInternalServerError
|
||||
message = http.StatusText(code)
|
||||
} else {
|
||||
code = apiError.code
|
||||
message = apiError.Error()
|
||||
}
|
||||
logging.FromContext(ctx).Error("api error", zap.Error(apiError))
|
||||
}
|
||||
return &api.ErrorStatusCode{StatusCode: code, Response: api.Error{Code: code, Message: message}}
|
||||
}
|
||||
|
|
@ -108,13 +116,6 @@ func (a apiError) Error() string {
|
|||
return a.err.Error()
|
||||
}
|
||||
|
||||
func (a *apiError) Code() int {
|
||||
if a.code == 0 {
|
||||
return http.StatusInternalServerError
|
||||
}
|
||||
return a.code
|
||||
}
|
||||
|
||||
func (a *apiError) Unwrap() error {
|
||||
return a.err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -285,7 +285,7 @@ func (a *apiService) FilesCreate(ctx context.Context, fileIn *api.File) (*api.Fi
|
|||
if len(fileIn.Parts) > 0 {
|
||||
fileDB.Parts = datatypes.NewJSONSlice(mapParts(fileIn.Parts))
|
||||
}
|
||||
fileDB.Size = utils.Ptr(fileIn.Size.Or(0))
|
||||
fileDB.Size = utils.Ptr(fileIn.Size.Value)
|
||||
}
|
||||
fileDB.Name = fileIn.Name
|
||||
fileDB.Type = string(fileIn.Type)
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import (
|
|||
|
||||
"github.com/tgdrive/teldrive/internal/api"
|
||||
"github.com/tgdrive/teldrive/internal/appcontext"
|
||||
"github.com/tgdrive/teldrive/internal/cache"
|
||||
"github.com/tgdrive/teldrive/internal/database"
|
||||
"github.com/tgdrive/teldrive/pkg/mapper"
|
||||
"github.com/tgdrive/teldrive/pkg/models"
|
||||
|
|
@ -120,16 +121,10 @@ func (a *apiService) SharesListFiles(ctx context.Context, params api.SharesListF
|
|||
}
|
||||
func (a *apiService) validFileShare(r *http.Request, id string) (*fileShare, error) {
|
||||
|
||||
share := &fileShare{}
|
||||
share, err := cache.FetchArg(a.cache, cache.Key("shares", id), 0, a.shareGetById, id)
|
||||
|
||||
key := "shares:" + id
|
||||
|
||||
if err := a.cache.Get(key, share); err != nil {
|
||||
share, err = a.shareGetById(id)
|
||||
if err != nil {
|
||||
return nil, &apiError{err: err}
|
||||
}
|
||||
a.cache.Set(key, share, 0)
|
||||
if err != nil {
|
||||
return nil, &apiError{err: err}
|
||||
}
|
||||
|
||||
if share.Password != nil {
|
||||
|
|
@ -138,10 +133,11 @@ func (a *apiService) validFileShare(r *http.Request, id string) (*fileShare, err
|
|||
return nil, &apiError{err: ErrEmptyAuth, code: http.StatusUnauthorized}
|
||||
}
|
||||
bytes, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(authHeader, "Basic "))
|
||||
password := strings.Split(string(bytes), ":")[1]
|
||||
if err != nil {
|
||||
return nil, &apiError{err: err}
|
||||
}
|
||||
password := strings.Split(string(bytes), ":")[1]
|
||||
|
||||
if err := bcrypt.CompareHashAndPassword([]byte(*share.Password), []byte(password)); err != nil {
|
||||
return nil, &apiError{err: ErrInvalidPassword, code: http.StatusUnauthorized}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue