From 6499d7f462680c2d04342bfdb9d4ba66bc99a84b Mon Sep 17 00:00:00 2001 From: ssongliu <73214554+ssongliu@users.noreply.github.com> Date: Fri, 19 Jan 2024 16:48:41 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20OneDrive=20=E5=A2=9E=E5=8A=A0=20Token?= =?UTF-8?q?=20=E5=88=B7=E6=96=B0=E6=9C=BA=E5=88=B6=20(#3637)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/v1/backup.go | 11 + backend/app/service/backup.go | 124 +++++++---- backend/app/service/monitor.go | 2 +- backend/cron/cron.go | 6 + backend/global/global.go | 5 +- backend/init/migration/migrate.go | 1 + backend/init/migration/migrations/v_1_9.go | 61 ++++++ backend/router/ro_setting.go | 1 + .../utils/cloud_storage/client/onedrive.go | 204 ++++++++++-------- backend/utils/cloud_storage/client/s3.go | 7 + backend/utils/cloud_storage/client/sftp.go | 41 +--- frontend/src/api/modules/setting.ts | 3 + frontend/src/lang/modules/en.ts | 2 + frontend/src/lang/modules/tw.ts | 2 + frontend/src/lang/modules/zh.ts | 2 + .../views/setting/backup-account/index.vue | 33 ++- 16 files changed, 343 insertions(+), 162 deletions(-) diff --git a/backend/app/api/v1/backup.go b/backend/app/api/v1/backup.go index f2a7541eb..82bdb0d99 100644 --- a/backend/app/api/v1/backup.go +++ b/backend/app/api/v1/backup.go @@ -49,6 +49,17 @@ func (b *BaseApi) CreateBackup(c *gin.Context) { helper.SuccessWithData(c, nil) } +// @Tags Backup Account +// @Summary Refresh OneDrive token +// @Description 刷新 OneDrive token +// @Success 200 +// @Security ApiKeyAuth +// @Router /settings/backup/refresh/onedrive [post] +func (b *BaseApi) RefreshOneDriveToken(c *gin.Context) { + backupService.Run() + helper.SuccessWithData(c, nil) +} + // @Tags Backup Account // @Summary List buckets // @Description 获取 bucket 列表 diff --git a/backend/app/service/backup.go b/backend/app/service/backup.go index ad781c1b5..dbb69a6f7 100644 --- a/backend/app/service/backup.go +++ b/backend/app/service/backup.go @@ -6,12 +6,10 @@ import ( "encoding/base64" "encoding/json" "fmt" - "io" - "net/http" - "net/url" "os" "path" "strings" + "time" "github.com/1Panel-dev/1Panel/backend/app/dto" "github.com/1Panel-dev/1Panel/backend/app/model" @@ -19,6 +17,7 @@ import ( "github.com/1Panel-dev/1Panel/backend/constant" "github.com/1Panel-dev/1Panel/backend/global" "github.com/1Panel-dev/1Panel/backend/utils/cloud_storage" + "github.com/1Panel-dev/1Panel/backend/utils/cloud_storage/client" fileUtils "github.com/1Panel-dev/1Panel/backend/utils/files" "github.com/jinzhu/copier" "github.com/pkg/errors" @@ -55,6 +54,8 @@ type IBackupService interface { AppBackup(db dto.CommonBackup) error AppRecover(req dto.CommonRecover) error + + Run() } func NewIBackupService() IBackupService { @@ -205,6 +206,9 @@ func (u *BackupService) Create(req dto.BackupOperate) error { return buserr.WithMap("ErrBackupCheck", map[string]interface{}{"err": err.Error()}, err) } } + if backup.Type == constant.OneDrive { + StartRefreshOneDriveToken() + } if err := backupRepo.Create(&backup); err != nil { return err } @@ -232,6 +236,13 @@ func (u *BackupService) GetBuckets(backupDto dto.ForBuckets) ([]interface{}, err } func (u *BackupService) Delete(id uint) error { + backup, _ := backupRepo.Get(commonRepo.WithByID(id)) + if backup.ID == 0 { + return constant.ErrRecordNotFound + } + if backup.Type == constant.OneDrive { + global.Cron.Remove(global.OneDriveCronID) + } cronjobs, _ := cronjobRepo.List(cronjobRepo.WithByBackupID(id)) if len(cronjobs) != 0 { return buserr.New(constant.ErrBackupInUsed) @@ -387,6 +398,15 @@ func (u *BackupService) loadByType(accountType string, accounts []model.BackupAc if err := copier.Copy(&item, &account); err != nil { global.LOG.Errorf("copy backup account to dto backup info failed, err: %v", err) } + if account.Type == constant.OneDrive { + varMap := make(map[string]interface{}) + if err := json.Unmarshal([]byte(item.Vars), &varMap); err != nil { + return dto.BackupInfo{Type: accountType} + } + delete(varMap, "refresh_token") + itemVars, _ := json.Marshal(varMap) + item.Vars = string(itemVars) + } return item } } @@ -398,44 +418,23 @@ func (u *BackupService) loadAccessToken(backup *model.BackupAccount) error { if err := json.Unmarshal([]byte(backup.Vars), &varMap); err != nil { return fmt.Errorf("unmarshal backup vars failed, err: %v", err) } - - data := url.Values{} - data.Set("client_id", global.CONF.System.OneDriveID) - data.Set("client_secret", global.CONF.System.OneDriveSc) - data.Set("grant_type", "authorization_code") - data.Set("code", varMap["code"].(string)) - data.Set("redirect_uri", constant.OneDriveRedirectURI) - client := &http.Client{} - req, err := http.NewRequest("POST", "https://login.microsoftonline.com/common/oauth2/v2.0/token", strings.NewReader(data.Encode())) - if err != nil { - return fmt.Errorf("new http post client for access token failed, err: %v", err) - } - req.Header.Add("Content-Type", "application/x-www-form-urlencoded") - resp, err := client.Do(req) - if err != nil { - return fmt.Errorf("request for access token failed, err: %v", err) - } - delete(varMap, "code") - respBody, err := io.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("read data from response body failed, err: %v", err) - } - defer resp.Body.Close() - - token := map[string]interface{}{} - if err := json.Unmarshal(respBody, &token); err != nil { - return fmt.Errorf("unmarshal data from response body failed, err: %v", err) - } - accessToken, ok := token["refresh_token"].(string) + code, ok := varMap["code"] if !ok { - return errors.New("no such access token in response") + return errors.New("no such token in request, please retry!") } - - itemVars, err := json.Marshal(varMap) + token, refreshToken, err := client.RefreshToken("authorization_code", code.(string)) + if err != nil { + return err + } + backup.Credential = token + varMapItem := make(map[string]interface{}) + varMapItem["refresh_status"] = constant.StatusSuccess + varMapItem["refresh_time"] = time.Now().Format("2006-01-02 15:04:05") + varMapItem["refresh_token"] = refreshToken + itemVars, err := json.Marshal(varMapItem) if err != nil { return fmt.Errorf("json marshal var map failed, err: %v", err) } - backup.Credential = accessToken backup.Vars = string(itemVars) return nil } @@ -521,3 +520,56 @@ func (u *BackupService) checkBackupConn(backup *model.BackupAccount) (bool, erro targetPath := strings.TrimPrefix(path.Join(backup.BackupPath, "test/1panel"), "/") return client.Upload(fileItem, targetPath) } + +func StartRefreshOneDriveToken() { + service := NewIBackupService() + oneDriveCronID, err := global.Cron.AddJob("0 * * * *", service) + if err != nil { + global.LOG.Errorf("can not add OneDrive corn job: %s", err.Error()) + return + } + global.OneDriveCronID = oneDriveCronID +} + +func (u *BackupService) Run() { + var backupItem model.BackupAccount + _ = global.DB.Where("`type` = ?", "OneDrive").First(&backupItem) + if backupItem.ID == 0 { + return + } + if len(backupItem.Credential) == 0 { + global.LOG.Error("OneDrive configuration lacks token information, please rebind.") + return + } + global.LOG.Info("start to refresh token of OneDrive ...") + varMap := make(map[string]interface{}) + if err := json.Unmarshal([]byte(backupItem.Vars), &varMap); err != nil { + global.LOG.Errorf("Failed to refresh OneDrive token, please retry, err: %v", err) + return + } + refreshItem, ok := varMap["refresh_token"] + if !ok { + global.LOG.Error("Failed to refresh OneDrive token, please retry, err: no such refresh token") + return + } + + token, refreshToken, err := client.RefreshToken("refresh_token", refreshItem.(string)) + varMap["refresh_status"] = constant.StatusSuccess + varMap["refresh_time"] = time.Now().Format("2006-01-02 15:04:05") + if err != nil { + varMap["refresh_status"] = constant.StatusFailed + varMap["refresh_msg"] = err.Error() + global.LOG.Errorf("Failed to refresh OneDrive token, please retry, err: %v", err) + return + } + varMap["refresh_token"] = refreshToken + + varsItem, _ := json.Marshal(varMap) + _ = global.DB.Model(&model.BackupAccount{}). + Where("id = ?", backupItem.ID). + Updates(map[string]interface{}{ + "credential": token, + "vars": varsItem, + }).Error + global.LOG.Info("Successfully refreshed OneDrive token.") +} diff --git a/backend/app/service/monitor.go b/backend/app/service/monitor.go index 706bd2412..8f907a71f 100644 --- a/backend/app/service/monitor.go +++ b/backend/app/service/monitor.go @@ -211,6 +211,6 @@ func StartMonitor(removeBefore bool, interval string) error { go service.saveIODataToDB(ctx, float64(intervalItem)) go service.saveNetDataToDB(ctx, float64(intervalItem)) - global.MonitorCronID = int(monitorID) + global.MonitorCronID = monitorID return nil } diff --git a/backend/cron/cron.go b/backend/cron/cron.go index bbf4d712e..7d99e0c86 100644 --- a/backend/cron/cron.go +++ b/backend/cron/cron.go @@ -44,6 +44,12 @@ func Run() { if _, err := global.Cron.AddJob("@daily", job.NewAppStoreJob()); err != nil { global.LOG.Errorf("can not add appstore corn job: %s", err.Error()) } + + var backup model.BackupAccount + _ = global.DB.Where("type = ?", "OneDrive").Find(&backup).Error + if backup.ID != 0 { + service.StartRefreshOneDriveToken() + } global.Cron.Start() var cronJobs []model.Cronjob diff --git a/backend/global/global.go b/backend/global/global.go index 3c629c692..1ac9dff42 100644 --- a/backend/global/global.go +++ b/backend/global/global.go @@ -20,6 +20,7 @@ var ( CACHE *badger_db.Cache Viper *viper.Viper - Cron *cron.Cron - MonitorCronID int + Cron *cron.Cron + MonitorCronID cron.EntryID + OneDriveCronID cron.EntryID ) diff --git a/backend/init/migration/migrate.go b/backend/init/migration/migrate.go index 5d4a85221..9d02eae7d 100644 --- a/backend/init/migration/migrate.go +++ b/backend/init/migration/migrate.go @@ -66,6 +66,7 @@ func Init() { migrations.AddTableDatabasePostgresql, migrations.AddPostgresqlSuperUser, migrations.UpdateCronjobWithWebsite, + migrations.UpdateOneDriveToken, }) if err := m.Migrate(); err != nil { global.LOG.Error(err) diff --git a/backend/init/migration/migrations/v_1_9.go b/backend/init/migration/migrations/v_1_9.go index 3b06d7289..d725ac6f7 100644 --- a/backend/init/migration/migrations/v_1_9.go +++ b/backend/init/migration/migrations/v_1_9.go @@ -1,11 +1,17 @@ package migrations import ( + "encoding/base64" + "encoding/json" + "time" + "github.com/1Panel-dev/1Panel/backend/app/dto/request" "github.com/1Panel-dev/1Panel/backend/app/model" "github.com/1Panel-dev/1Panel/backend/app/repo" "github.com/1Panel-dev/1Panel/backend/app/service" + "github.com/1Panel-dev/1Panel/backend/constant" "github.com/1Panel-dev/1Panel/backend/global" + "github.com/1Panel-dev/1Panel/backend/utils/cloud_storage/client" "github.com/go-gormigrate/gormigrate/v2" "gorm.io/gorm" ) @@ -188,3 +194,58 @@ var UpdateCronjobWithWebsite = &gormigrate.Migration{ return nil }, } + +var UpdateOneDriveToken = &gormigrate.Migration{ + ID: "20240117-update-onedrive-token", + Migrate: func(tx *gorm.DB) error { + var ( + backup model.BackupAccount + clientSetting model.Setting + secretSetting model.Setting + ) + _ = tx.Where("type = ?", "OneDrive").First(&backup).Error + if backup.ID == 0 { + return nil + } + if len(backup.Credential) == 0 { + global.LOG.Error("OneDrive configuration lacks token information, please rebind.") + return nil + } + + _ = tx.Where("key = ?", "OneDriveID").First(&clientSetting).Error + if clientSetting.ID == 0 { + global.LOG.Error("system configuration lacks clientID information, please retry.") + return nil + } + _ = tx.Where("key = ?", "OneDriveSc").First(&secretSetting).Error + if secretSetting.ID == 0 { + global.LOG.Error("system configuration lacks clientID information, please retry.") + return nil + } + idItem, _ := base64.StdEncoding.DecodeString(clientSetting.Value) + global.CONF.System.OneDriveID = string(idItem) + scItem, _ := base64.StdEncoding.DecodeString(secretSetting.Value) + global.CONF.System.OneDriveSc = string(scItem) + + varMap := make(map[string]interface{}) + token, refreshToken, err := client.RefreshToken("refresh_token", backup.Credential) + varMap["refresh_status"] = constant.StatusSuccess + varMap["refresh_time"] = time.Now().Format("2006-01-02 15:04:05") + if err != nil { + varMap["refresh_msg"] = err.Error() + varMap["refresh_status"] = constant.StatusFailed + } + varMap["refresh_token"] = refreshToken + itemVars, _ := json.Marshal(varMap) + if err := tx.Model(&model.BackupAccount{}). + Where("id = ?", backup.ID). + Updates(map[string]interface{}{ + "credential": token, + "vars": string(itemVars), + }).Error; err != nil { + return err + } + + return nil + }, +} diff --git a/backend/router/ro_setting.go b/backend/router/ro_setting.go index 5990c454c..17a3dd41c 100644 --- a/backend/router/ro_setting.go +++ b/backend/router/ro_setting.go @@ -45,6 +45,7 @@ func (s *SettingRouter) InitRouter(Router *gin.RouterGroup) { settingRouter.GET("/backup/search", baseApi.ListBackup) settingRouter.GET("/backup/onedrive", baseApi.LoadOneDriveInfo) settingRouter.POST("/backup/backup", baseApi.Backup) + settingRouter.POST("/backup/refresh/onedrive", baseApi.RefreshOneDriveToken) settingRouter.POST("/backup/recover", baseApi.Recover) settingRouter.POST("/backup/recover/byupload", baseApi.RecoverByUpload) settingRouter.POST("/backup/search/files", baseApi.LoadFilesFromBackup) diff --git a/backend/utils/cloud_storage/client/onedrive.go b/backend/utils/cloud_storage/client/onedrive.go index 603010aa2..576310f46 100644 --- a/backend/utils/cloud_storage/client/onedrive.go +++ b/backend/utils/cloud_storage/client/onedrive.go @@ -4,7 +4,6 @@ import ( "bufio" "bytes" "context" - "crypto/tls" "encoding/json" "errors" "fmt" @@ -15,11 +14,10 @@ import ( "path" "strconv" "strings" - "time" - "github.com/1Panel-dev/1Panel/backend/app/model" "github.com/1Panel-dev/1Panel/backend/constant" "github.com/1Panel-dev/1Panel/backend/global" + "github.com/1Panel-dev/1Panel/backend/utils/files" odsdk "github.com/goh-chunlin/go-onedrive/onedrive" "golang.org/x/oauth2" ) @@ -31,15 +29,8 @@ type oneDriveClient struct { func NewOneDriveClient(vars map[string]interface{}) (*oneDriveClient, error) { token := loadParamFromVars("accessToken", true, vars) ctx := context.Background() - - newToken, err := refreshToken(token) - if err != nil { - return nil, err - } - _ = global.DB.Model(&model.Group{}).Where("type = ?", "OneDrive").Updates(map[string]interface{}{"credential": newToken}).Error - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: newToken}, + &oauth2.Token{AccessToken: token}, ) tc := oauth2.NewClient(ctx, ts) @@ -110,87 +101,24 @@ func (o oneDriveClient) Upload(src, target string) (bool, error) { } ctx := context.Background() - file, err := os.Open(src) + folderID, err := o.loadIDByPath(path.Dir(target)) if err != nil { return false, err } - defer file.Close() - fileInfo, err := file.Stat() + fileInfo, err := os.Stat(src) if err != nil { return false, err } if fileInfo.IsDir() { return false, errors.New("Only file is allowed to be uploaded here.") } - fileName := fileInfo.Name() - fileSize := fileInfo.Size() - - folderID, err := o.loadIDByPath(path.Dir(target)) - if err != nil { - return false, err + var isOk bool + if fileInfo.Size() > 4*1024*1024 { + isOk, err = o.upSmall(ctx, src, folderID, fileInfo.Size()) + } else { + isOk, err = o.upBig(ctx, src, folderID, fileInfo.Size()) } - apiURL := fmt.Sprintf("me/drive/items/%s:/%s:/createUploadSession", url.PathEscape(folderID), fileName) - sessionCreationRequestInside := NewUploadSessionCreationRequest{ - ConflictBehavior: "rename", - } - - sessionCreationRequest := struct { - Item NewUploadSessionCreationRequest `json:"item"` - DeferCommit bool `json:"deferCommit"` - }{sessionCreationRequestInside, false} - - sessionCreationReq, err := o.client.NewRequest("POST", apiURL, sessionCreationRequest) - if err != nil { - return false, err - } - - var sessionCreationResp *NewUploadSessionCreationResponse - err = o.client.Do(ctx, sessionCreationReq, false, &sessionCreationResp) - if err != nil { - return false, fmt.Errorf("session creation failed %w", err) - } - - fileSessionUploadUrl := sessionCreationResp.UploadURL - - sizePerSplit := int64(3200 * 1024) - buffer := make([]byte, 3200*1024) - splitCount := fileSize / sizePerSplit - if fileSize%sizePerSplit != 0 { - splitCount += 1 - } - bfReader := bufio.NewReader(file) - httpClient := http.Client{ - Timeout: time.Minute * 10, - Transport: &http.Transport{ - Proxy: http.ProxyFromEnvironment, - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - }, - } - for splitNow := int64(0); splitNow < splitCount; splitNow++ { - length, err := bfReader.Read(buffer) - if err != nil { - return false, err - } - if int64(length) < sizePerSplit { - bufferLast := buffer[:length] - buffer = bufferLast - } - sessionFileUploadReq, err := o.NewSessionFileUploadRequest(fileSessionUploadUrl, splitNow*sizePerSplit, fileSize, bytes.NewReader(buffer)) - if err != nil { - return false, err - } - res, err := httpClient.Do(sessionFileUploadReq) - if err != nil { - return false, err - } - if res.StatusCode != 201 && res.StatusCode != 202 && res.StatusCode != 200 { - data, _ := io.ReadAll(res.Body) - res.Body.Close() - return false, errors.New(string(data)) - } - } - - return true, nil + return isOk, err } func (o oneDriveClient) Download(src, target string) (bool, error) { @@ -264,38 +192,48 @@ func (o *oneDriveClient) loadIDByPath(path string) (string, error) { return driveItem.Id, nil } -func refreshToken(oldToken string) (string, error) { +func RefreshToken(grantType, grantCode string) (string, string, error) { data := url.Values{} data.Set("client_id", global.CONF.System.OneDriveID) data.Set("client_secret", global.CONF.System.OneDriveSc) data.Set("grant_type", "refresh_token") - data.Set("refresh_token", oldToken) + if grantType == "refresh_token" { + data.Set("grant_type", "refresh_token") + data.Set("refresh_token", grantCode) + } else { + data.Set("grant_type", "authorization_code") + data.Set("code", grantCode) + } data.Set("redirect_uri", constant.OneDriveRedirectURI) client := &http.Client{} req, err := http.NewRequest("POST", "https://login.microsoftonline.com/common/oauth2/v2.0/token", strings.NewReader(data.Encode())) if err != nil { - return "", fmt.Errorf("new http post client for access token failed, err: %v", err) + return "", "", fmt.Errorf("new http post client for access token failed, err: %v", err) } req.Header.Add("Content-Type", "application/x-www-form-urlencoded") resp, err := client.Do(req) if err != nil { - return "", fmt.Errorf("request for access token failed, err: %v", err) + return "", "", fmt.Errorf("request for access token failed, err: %v", err) } respBody, err := io.ReadAll(resp.Body) if err != nil { - return "", fmt.Errorf("read data from response body failed, err: %v", err) + return "", "", fmt.Errorf("read data from response body failed, err: %v", err) } defer resp.Body.Close() tokenMap := map[string]interface{}{} if err := json.Unmarshal(respBody, &tokenMap); err != nil { - return "", fmt.Errorf("unmarshal data from response body failed, err: %v", err) + return "", "", fmt.Errorf("unmarshal data from response body failed, err: %v", err) } accessToken, ok := tokenMap["access_token"].(string) if !ok { - return "", errors.New("no such access token in response") + return "", "", errors.New("no such access token in response") } - return accessToken, nil + refreshToken, ok := tokenMap["refresh_token"].(string) + if !ok { + return "", "", errors.New("no such access token in response") + } + return accessToken, refreshToken, nil } func (o *oneDriveClient) createFolder(parent string) error { @@ -355,3 +293,89 @@ func (o *oneDriveClient) NewSessionFileUploadRequest(absoluteUrl string, grandOf return req, err } + +func (o *oneDriveClient) upSmall(ctx context.Context, srcPath, folderID string, fileSize int64) (bool, error) { + file, err := os.Open(srcPath) + if err != nil { + return false, err + } + defer file.Close() + + buffer := make([]byte, fileSize) + _, _ = file.Read(buffer) + fileReader := bytes.NewReader(buffer) + apiURL := fmt.Sprintf("me/drive/items/%s:/%s:/content?@microsoft.graph.conflictBehavior=rename", url.PathEscape(folderID), path.Base(srcPath)) + + mimeType := files.GetMimeType(srcPath) + req, err := o.client.NewFileUploadRequest(apiURL, mimeType, fileReader) + if err != nil { + return false, err + } + var response *DriveItem + if err := o.client.Do(context.Background(), req, false, &response); err != nil { + return false, fmt.Errorf("do request for list failed, err: %v", err) + } + fmt.Println(response) + return true, nil +} + +func (o *oneDriveClient) upBig(ctx context.Context, srcPath, folderID string, fileSize int64) (bool, error) { + file, err := os.Open(srcPath) + if err != nil { + return false, err + } + defer file.Close() + + apiURL := fmt.Sprintf("me/drive/items/%s:/%s:/createUploadSession", url.PathEscape(folderID), path.Base(srcPath)) + sessionCreationRequestInside := NewUploadSessionCreationRequest{ + ConflictBehavior: "rename", + } + + sessionCreationRequest := struct { + Item NewUploadSessionCreationRequest `json:"item"` + DeferCommit bool `json:"deferCommit"` + }{sessionCreationRequestInside, false} + + sessionCreationReq, err := o.client.NewRequest("POST", apiURL, sessionCreationRequest) + if err != nil { + return false, err + } + + var sessionCreationResp *NewUploadSessionCreationResponse + err = o.client.Do(ctx, sessionCreationReq, false, &sessionCreationResp) + if err != nil { + return false, fmt.Errorf("session creation failed %w", err) + } + + fileSessionUploadUrl := sessionCreationResp.UploadURL + + sizePerSplit := int64(3200 * 1024) + buffer := make([]byte, 3200*1024) + splitCount := fileSize / sizePerSplit + if fileSize%sizePerSplit != 0 { + splitCount += 1 + } + bfReader := bufio.NewReader(file) + var fileUploadResp *UploadSessionUploadResponse + for splitNow := int64(0); splitNow < splitCount; splitNow++ { + length, err := bfReader.Read(buffer) + if err != nil { + return false, err + } + if int64(length) < sizePerSplit { + bufferLast := buffer[:length] + buffer = bufferLast + } + sessionFileUploadReq, err := o.NewSessionFileUploadRequest(fileSessionUploadUrl, splitNow*sizePerSplit, fileSize, bytes.NewReader(buffer)) + if err != nil { + return false, err + } + if err := o.client.Do(ctx, sessionFileUploadReq, false, &fileUploadResp); err != nil { + return false, err + } + } + if fileUploadResp.Id == "" { + return false, errors.New("something went wrong. file upload incomplete. consider upload the file in a step-by-step manner") + } + return true, nil +} diff --git a/backend/utils/cloud_storage/client/s3.go b/backend/utils/cloud_storage/client/s3.go index 0f6cb96c8..0a4ee58d3 100644 --- a/backend/utils/cloud_storage/client/s3.go +++ b/backend/utils/cloud_storage/client/s3.go @@ -97,6 +97,10 @@ func (s s3Client) Delete(path string) (bool, error) { } func (s s3Client) Upload(src, target string) (bool, error) { + fileInfo, err := os.Stat(src) + if err != nil { + return false, err + } file, err := os.Open(src) if err != nil { return false, err @@ -104,6 +108,9 @@ func (s s3Client) Upload(src, target string) (bool, error) { defer file.Close() uploader := s3manager.NewUploader(&s.Sess) + if fileInfo.Size() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { + uploader.PartSize = fileInfo.Size() / (s3manager.MaxUploadParts - 1) + } if _, err := uploader.Upload(&s3manager.UploadInput{ Bucket: aws.String(s.bucket), Key: aws.String(target), diff --git a/backend/utils/cloud_storage/client/sftp.go b/backend/utils/cloud_storage/client/sftp.go index 63af2744a..e6e3acbb6 100644 --- a/backend/utils/cloud_storage/client/sftp.go +++ b/backend/utils/cloud_storage/client/sftp.go @@ -1,8 +1,8 @@ package client import ( - "bufio" "fmt" + "io" "net" "os" "path" @@ -56,33 +56,14 @@ func (s sftpClient) Upload(src, target string) (bool, error) { } defer srcFile.Close() - targetFilePath := s.bucket + "/" + target - targetDir, _ := path.Split(targetFilePath) - if _, err = client.Stat(targetDir); err != nil { - if os.IsNotExist(err) { - if err = client.MkdirAll(targetDir); err != nil { - return false, err - } - } else { - return false, err - } - } - dstFile, err := client.Create(targetFilePath) + dstFile, err := client.Create(path.Join(s.bucket, target)) if err != nil { return false, err } defer dstFile.Close() - reader := bufio.NewReaderSize(srcFile, 128*1024*1024) - for { - chunk, err := reader.Peek(8 * 1024 * 1024) - if len(chunk) != 0 { - _, _ = dstFile.Write(chunk) - _, _ = reader.Discard(len(chunk)) - } - if err != nil { - break - } + if _, err := io.Copy(dstFile, srcFile); err != nil { + return false, err } return true, nil } @@ -103,6 +84,7 @@ func (s sftpClient) Download(src, target string) (bool, error) { } defer client.Close() defer sshClient.Close() + srcFile, err := client.Open(s.bucket + "/" + src) if err != nil { return false, err @@ -121,7 +103,7 @@ func (s sftpClient) Download(src, target string) (bool, error) { return true, err } -func (s sftpClient) Exist(path string) (bool, error) { +func (s sftpClient) Exist(filePath string) (bool, error) { sshClient, err := ssh.Dial("tcp", s.connInfo, s.config) if err != nil { return false, err @@ -133,7 +115,7 @@ func (s sftpClient) Exist(path string) (bool, error) { defer client.Close() defer sshClient.Close() - srcFile, err := client.Open(s.bucket + "/" + path) + srcFile, err := client.Open(path.Join(s.bucket, filePath)) if err != nil { if os.IsNotExist(err) { return false, nil @@ -145,7 +127,7 @@ func (s sftpClient) Exist(path string) (bool, error) { return true, err } -func (s sftpClient) Size(path string) (int64, error) { +func (s sftpClient) Size(filePath string) (int64, error) { sshClient, err := ssh.Dial("tcp", s.connInfo, s.config) if err != nil { return 0, err @@ -157,7 +139,7 @@ func (s sftpClient) Size(path string) (int64, error) { defer client.Close() defer sshClient.Close() - files, err := client.Stat(s.bucket + "/" + path) + files, err := client.Stat(path.Join(s.bucket, filePath)) if err != nil { return 0, err } @@ -176,8 +158,7 @@ func (s sftpClient) Delete(filePath string) (bool, error) { defer client.Close() defer sshClient.Close() - targetFilePath := s.bucket + "/" + filePath - if err := client.Remove(targetFilePath); err != nil { + if err := client.Remove(path.Join(s.bucket, filePath)); err != nil { return false, err } return true, nil @@ -195,7 +176,7 @@ func (s sftpClient) ListObjects(prefix string) ([]string, error) { defer client.Close() defer sshClient.Close() - files, err := client.ReadDir(s.bucket + "/" + prefix) + files, err := client.ReadDir(path.Join(s.bucket, prefix)) if err != nil { return nil, err } diff --git a/frontend/src/api/modules/setting.ts b/frontend/src/api/modules/setting.ts index c63fa6886..4385d24f4 100644 --- a/frontend/src/api/modules/setting.ts +++ b/frontend/src/api/modules/setting.ts @@ -84,6 +84,9 @@ export const handleRecover = (params: Backup.Recover) => { export const handleRecoverByUpload = (params: Backup.Recover) => { return http.post(`/settings/backup/recover/byupload`, params, TimeoutEnum.T_1D); }; +export const refreshOneDrive = () => { + return http.post(`/settings/backup/refresh/onedrive`, {}); +}; export const downloadBackupRecord = (params: Backup.RecordDownload) => { return http.post(`/settings/backup/record/download`, params, TimeoutEnum.T_10M); }; diff --git a/frontend/src/lang/modules/en.ts b/frontend/src/lang/modules/en.ts index e79443ac9..dd81a995b 100644 --- a/frontend/src/lang/modules/en.ts +++ b/frontend/src/lang/modules/en.ts @@ -1241,6 +1241,8 @@ const message = { SFTP: 'SFTP', WebDAV: 'WebDAV', OneDrive: 'Microsoft OneDrive', + refreshTime: 'Token Refresh Time', + refreshStatus: 'Token Refresh Status', backupDir: 'Backup dir', codeWarning: 'The current authorization code format is incorrect, please confirm again!', isCN: 'Domestic version (not supported at the moment)', diff --git a/frontend/src/lang/modules/tw.ts b/frontend/src/lang/modules/tw.ts index 4b7b1838d..5685d590d 100644 --- a/frontend/src/lang/modules/tw.ts +++ b/frontend/src/lang/modules/tw.ts @@ -1165,6 +1165,8 @@ const message = { SFTP: 'SFTP', WebDAV: 'WebDAV', OneDrive: '微軟 OneDrive', + refreshTime: '令牌刷新時間', + refreshStatus: '令牌刷新狀態', codeWarning: '當前授權碼格式錯誤，請重新確認！', backupDir: '備份路徑', isCN: '國內版 (暫不支持)', diff --git a/frontend/src/lang/modules/zh.ts b/frontend/src/lang/modules/zh.ts index cbceb668c..9ee113a82 100644 --- a/frontend/src/lang/modules/zh.ts +++ b/frontend/src/lang/modules/zh.ts @@ -1166,6 +1166,8 @@ const message = { SFTP: 'SFTP', WebDAV: 'WebDAV', OneDrive: '微软 OneDrive', + refreshTime: '令牌刷新时间', + refreshStatus: '令牌刷新状态', codeWarning: '当前授权码格式错误，请重新确认！', backupDir: '备份路径', isCN: '国内版 (暂不支持)', diff --git a/frontend/src/views/setting/backup-account/index.vue b/frontend/src/views/setting/backup-account/index.vue index 045fda09a..3937b9068 100644 --- a/frontend/src/views/setting/backup-account/index.vue +++ b/frontend/src/views/setting/backup-account/index.vue @@ -212,7 +212,7 @@ round plain :disabled="oneDriveData.id === 0" - @click="onOpenDialog('edit', 'SFTP', oneDriveData)" + @click="onOpenDialog('edit', 'OneDrive', oneDriveData)" > {{ $t('commons.button.edit') }} @@ -227,6 +227,26 @@ {{ oneDriveData.backupPath }} {{ $t('setting.unSetting') }} + + {{ oneDriveData.varsJson['refresh_time'] }} + + {{ $t('commons.button.refresh') }} + + + + + {{ $t('commons.status.success') }} + + + + {{ $t('commons.status.failed') }} + + + {{ dateFormat(0, 0, oneDriveData.createdAt) }} @@ -430,7 +450,7 @@ import { dateFormat } from '@/utils/util'; import { onMounted, ref } from 'vue'; import OpDialog from '@/components/del-dialog/index.vue'; -import { getBackupList, deleteBackup } from '@/api/modules/setting'; +import { getBackupList, deleteBackup, refreshOneDrive } from '@/api/modules/setting'; import localDialog from '@/views/setting/backup-account/local/index.vue'; import s3Dialog from '@/views/setting/backup-account/s3/index.vue'; import ossDialog from '@/views/setting/backup-account/oss/index.vue'; @@ -536,7 +556,9 @@ const oneDriveData = ref({ backupPath: '', vars: '', varsJson: { - redirectURI: '', + refresh_msg: '', + refresh_time: '', + refresh_status: '', }, createdAt: new Date(), }); @@ -680,6 +702,11 @@ const onOpenDialog = async ( } }; +const refreshToken = async () => { + await refreshOneDrive(); + search(); +}; + onMounted(() => { search(); });