From 39f334f3a2ceee2dbc8a34a91538d2dbd58be8fa Mon Sep 17 00:00:00 2001 From: ssongliu <73214554+ssongliu@users.noreply.github.com> Date: Tue, 26 Aug 2025 17:07:41 +0800 Subject: [PATCH] fix: Optimize backup logic for cronjob (#10145) --- agent/app/service/backup.go | 59 ++++++++++-- agent/app/service/backup_record.go | 5 +- agent/app/service/cronjob.go | 5 +- agent/app/service/cronjob_backup.go | 91 ++++++++++--------- agent/app/service/cronjob_helper.go | 43 +++------ agent/app/service/snapshot.go | 9 +- agent/app/service/snapshot_create.go | 40 +++----- agent/i18n/lang/en.yaml | 7 +- agent/i18n/lang/ja.yaml | 7 +- agent/i18n/lang/ko.yaml | 7 +- agent/i18n/lang/ms.yaml | 7 +- agent/i18n/lang/pt-BR.yaml | 7 +- agent/i18n/lang/ru.yaml | 7 +- agent/i18n/lang/tr.yaml | 5 +- agent/i18n/lang/zh-Hant.yaml | 7 +- agent/i18n/lang/zh.yaml | 3 +- frontend/src/components/task-list/index.vue | 2 +- frontend/src/views/ai/gpu/index.vue | 4 +- .../views/cronjob/cronjob/record/index.vue | 17 ++-- frontend/src/views/setting/snapshot/index.vue | 50 +++++----- 20 files changed, 199 insertions(+), 183 deletions(-) diff --git a/agent/app/service/backup.go b/agent/app/service/backup.go index b7e2d49bc..3b2533095 100644 --- a/agent/app/service/backup.go +++ b/agent/app/service/backup.go @@ -14,9 +14,11 @@ import ( "github.com/1Panel-dev/1Panel/agent/app/dto" "github.com/1Panel-dev/1Panel/agent/app/model" "github.com/1Panel-dev/1Panel/agent/app/repo" + "github.com/1Panel-dev/1Panel/agent/app/task" "github.com/1Panel-dev/1Panel/agent/buserr" "github.com/1Panel-dev/1Panel/agent/constant" "github.com/1Panel-dev/1Panel/agent/global" + "github.com/1Panel-dev/1Panel/agent/i18n" "github.com/1Panel-dev/1Panel/agent/utils/cloud_storage" "github.com/1Panel-dev/1Panel/agent/utils/cloud_storage/client" "github.com/1Panel-dev/1Panel/agent/utils/encrypt" @@ -393,9 +395,13 @@ type backupClientHelper struct { name string backupPath string client cloud_storage.CloudStorageClient + + isOk bool + hasBackuped bool + message string } -func NewBackupClientMap(ids []string) (map[string]backupClientHelper, error) { +func NewBackupClientMap(ids []string) map[string]backupClientHelper { var accounts []model.BackupAccount var idItems []uint for i := 0; i < len(ids); i++ { @@ -406,18 +412,59 @@ func NewBackupClientMap(ids []string) (map[string]backupClientHelper, error) { clientMap := make(map[string]backupClientHelper) for _, item := range accounts { backClient, err := newClient(&item, true) - if err != nil { - return nil, err - } - clientMap[fmt.Sprintf("%v", item.ID)] = backupClientHelper{ + itemHelper := backupClientHelper{ client: backClient, name: item.Name, backupPath: item.BackupPath, accountType: item.Type, id: item.ID, + isOk: err == nil, } + if err != nil { + itemHelper.message = err.Error() + } + clientMap[fmt.Sprintf("%v", item.ID)] = itemHelper } - return clientMap, nil + return clientMap +} + +func uploadWithMap(taskItem task.Task, accountMap map[string]backupClientHelper, src, dst, accountIDs string, downloadAccountID, retry uint) error { + accounts := strings.Split(accountIDs, ",") + for _, account := range accounts { + if len(account) == 0 { + continue + } + itemBackup, ok := accountMap[account] + if !ok { + continue + } + if itemBackup.hasBackuped { + continue + } + if !itemBackup.isOk { + taskItem.LogFailed(i18n.GetMsgWithDetail("LoadBackupFailed", itemBackup.message)) + continue + } + taskItem.LogStart(i18n.GetMsgWithMap("UploadFile", map[string]interface{}{ + "file": path.Join(itemBackup.backupPath, dst), + "backup": itemBackup.name, + })) + for i := 0; i < int(retry); i++ { + _, err := itemBackup.client.Upload(src, path.Join(itemBackup.backupPath, dst)) + taskItem.LogWithStatus(i18n.GetMsgByKey("Upload"), err) + if err != nil { + if account == fmt.Sprintf("%d", downloadAccountID) { + return err + } + } else { + break + } + } + itemBackup.hasBackuped = true + accountMap[account] = itemBackup + } + os.RemoveAll(src) + return nil } func newClient(account *model.BackupAccount, isEncrypt bool) (cloud_storage.CloudStorageClient, error) { diff --git a/agent/app/service/backup_record.go b/agent/app/service/backup_record.go index 1d5feb8e8..b66a01ea8 100644 --- a/agent/app/service/backup_record.go +++ b/agent/app/service/backup_record.go @@ -250,11 +250,8 @@ func (u *BackupRecordService) LoadRecordSize(req dto.SearchForSize) ([]dto.Recor recordIds = append(recordIds, fmt.Sprintf("%v", record.DownloadID)) } } - clientMap, err := NewBackupClientMap(recordIds) - if err != nil { - return nil, err - } + clientMap := NewBackupClientMap(recordIds) var datas []dto.RecordFileSize var wg sync.WaitGroup ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) diff --git a/agent/app/service/cronjob.go b/agent/app/service/cronjob.go index d4f287799..10e807615 100644 --- a/agent/app/service/cronjob.go +++ b/agent/app/service/cronjob.go @@ -497,10 +497,7 @@ func (u *CronjobService) CleanRecord(req dto.CronjobClean) error { } if req.CleanData { if hasBackup(cronjob.Type) { - accountMap, err := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ",")) - if err != nil { - return err - } + accountMap := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ",")) if !req.CleanRemoteData { for key := range accountMap { if key != constant.Local { diff --git a/agent/app/service/cronjob_backup.go b/agent/app/service/cronjob_backup.go index 45fdd4788..2670bb78b 100644 --- a/agent/app/service/cronjob_backup.go +++ b/agent/app/service/cronjob_backup.go @@ -29,9 +29,9 @@ func (u *CronjobService) handleApp(cronjob model.Cronjob, startTime time.Time, t if len(apps) == 0 { return errors.New("no such app in database!") } - accountMap, err := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ",")) - if err != nil { - return err + accountMap := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ",")) + if !accountMap[fmt.Sprintf("%d", cronjob.DownloadAccountID)].isOk { + return errors.New(i18n.GetMsgWithDetail("LoadBackupFailed", accountMap[fmt.Sprintf("%d", cronjob.DownloadAccountID)].message)) } for _, app := range apps { retry := 0 @@ -54,8 +54,10 @@ func (u *CronjobService) handleApp(cronjob model.Cronjob, startTime time.Time, t return nil } } - downloadPath, err := u.uploadCronjobBackFile(cronjob, task, accountMap, path.Join(backupDir, record.FileName)) - if err != nil { + + src := path.Join(backupDir, record.FileName) + dst := strings.TrimPrefix(src, global.Dir.LocalBackupDir+"/tmp/") + if err := uploadWithMap(*task, accountMap, src, dst, cronjob.SourceAccountIDs, cronjob.DownloadAccountID, cronjob.RetryTimes); err != nil { if retry < int(cronjob.RetryTimes) || !cronjob.IgnoreErr { retry++ return err @@ -63,7 +65,7 @@ func (u *CronjobService) handleApp(cronjob model.Cronjob, startTime time.Time, t task.Log(i18n.GetMsgWithDetail("IgnoreUploadErr", err.Error())) return nil } - record.FileDir = path.Dir(downloadPath) + record.FileDir = path.Dir(dst) if err := backupRepo.CreateRecord(&record); err != nil { global.LOG.Errorf("save backup record failed, err: %v", err) return err @@ -80,9 +82,9 @@ func (u *CronjobService) handleWebsite(cronjob model.Cronjob, startTime time.Tim if len(webs) == 0 { return errors.New("no such website in database!") } - accountMap, err := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ",")) - if err != nil { - return err + accountMap := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ",")) + if !accountMap[fmt.Sprintf("%d", cronjob.DownloadAccountID)].isOk { + return errors.New(i18n.GetMsgWithDetail("LoadBackupFailed", accountMap[fmt.Sprintf("%d", cronjob.DownloadAccountID)].message)) } for _, web := range webs { retry := 0 @@ -107,8 +109,9 @@ func (u *CronjobService) handleWebsite(cronjob model.Cronjob, startTime time.Tim } } - downloadPath, err := u.uploadCronjobBackFile(cronjob, task, accountMap, path.Join(backupDir, record.FileName)) - if err != nil { + src := path.Join(backupDir, record.FileName) + dst := strings.TrimPrefix(src, global.Dir.LocalBackupDir+"/tmp/") + if err := uploadWithMap(*task, accountMap, src, dst, cronjob.SourceAccountIDs, cronjob.DownloadAccountID, cronjob.RetryTimes); err != nil { if retry < int(cronjob.RetryTimes) || !cronjob.IgnoreErr { retry++ return err @@ -116,7 +119,7 @@ func (u *CronjobService) handleWebsite(cronjob model.Cronjob, startTime time.Tim task.Log(i18n.GetMsgWithDetail("IgnoreUploadErr", err.Error())) return nil } - record.FileDir = path.Dir(downloadPath) + record.FileDir = path.Dir(dst) if err := backupRepo.CreateRecord(&record); err != nil { global.LOG.Errorf("save backup record failed, err: %v", err) return err @@ -133,9 +136,9 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Ti if len(dbs) == 0 { return errors.New("no such db in database!") } - accountMap, err := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ",")) - if err != nil { - return err + accountMap := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ",")) + if !accountMap[fmt.Sprintf("%d", cronjob.DownloadAccountID)].isOk { + return errors.New(i18n.GetMsgWithDetail("LoadBackupFailed", accountMap[fmt.Sprintf("%d", cronjob.DownloadAccountID)].message)) } for _, dbInfo := range dbs { retry := 0 @@ -173,8 +176,9 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Ti } } - downloadPath, err := u.uploadCronjobBackFile(cronjob, task, accountMap, path.Join(backupDir, record.FileName)) - if err != nil { + src := path.Join(backupDir, record.FileName) + dst := strings.TrimPrefix(src, global.Dir.LocalBackupDir+"/tmp/") + if err := uploadWithMap(*task, accountMap, src, dst, cronjob.SourceAccountIDs, cronjob.DownloadAccountID, cronjob.RetryTimes); err != nil { if retry < int(cronjob.RetryTimes) || !cronjob.IgnoreErr { retry++ return err @@ -182,7 +186,7 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Ti task.Log(i18n.GetMsgWithDetail("IgnoreUploadErr", err.Error())) return nil } - record.FileDir = path.Dir(downloadPath) + record.FileDir = path.Dir(dst) if err := backupRepo.CreateRecord(&record); err != nil { global.LOG.Errorf("save backup record failed, err: %v", err) return err @@ -195,11 +199,11 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Ti } func (u *CronjobService) handleDirectory(cronjob model.Cronjob, startTime time.Time, taskItem *task.Task) error { + accountMap := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ",")) + if !accountMap[fmt.Sprintf("%d", cronjob.DownloadAccountID)].isOk { + return errors.New(i18n.GetMsgWithDetail("LoadBackupFailed", accountMap[fmt.Sprintf("%d", cronjob.DownloadAccountID)].message)) + } taskItem.AddSubTaskWithOps(task.GetTaskName(cronjob.SourceDir, task.TaskBackup, task.TaskScopeCronjob), func(task *task.Task) error { - accountMap, err := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ",")) - if err != nil { - return err - } fileName := fmt.Sprintf("%s.tar.gz", startTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(2)) if cronjob.IsDir || len(strings.Split(cronjob.SourceDir, ",")) == 1 { fileName = loadFileName(cronjob.SourceDir) @@ -226,15 +230,15 @@ func (u *CronjobService) handleDirectory(cronjob model.Cronjob, startTime time.T record.CronjobID = cronjob.ID record.Name = cronjob.Name record.DownloadAccountID, record.SourceAccountIDs = cronjob.DownloadAccountID, cronjob.SourceAccountIDs - downloadPath, err := u.uploadCronjobBackFile(cronjob, task, accountMap, path.Join(backupDir, fileName)) - if err != nil { - taskItem.LogFailedWithErr("Upload backup file", err) + + src := path.Join(backupDir, fileName) + dst := strings.TrimPrefix(src, global.Dir.LocalBackupDir+"/tmp/") + if err := uploadWithMap(*task, accountMap, src, dst, cronjob.SourceAccountIDs, cronjob.DownloadAccountID, cronjob.RetryTimes); err != nil { return err } - record.FileDir = path.Dir(downloadPath) + record.FileDir = path.Dir(dst) record.FileName = fileName if err := backupRepo.CreateRecord(&record); err != nil { - taskItem.LogFailedWithErr("Save record", err) return err } u.removeExpiredBackup(cronjob, accountMap, record) @@ -244,11 +248,11 @@ func (u *CronjobService) handleDirectory(cronjob model.Cronjob, startTime time.T } func (u *CronjobService) handleSystemLog(cronjob model.Cronjob, startTime time.Time, taskItem *task.Task) error { - taskItem.AddSubTaskWithOps(task.GetTaskName(i18n.GetMsgByKey("BackupSystemLog"), task.TaskBackup, task.TaskScopeCronjob), func(task *task.Task) error { - accountMap, err := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ",")) - if err != nil { - return err - } + accountMap := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ",")) + if !accountMap[fmt.Sprintf("%d", cronjob.DownloadAccountID)].isOk { + return errors.New(i18n.GetMsgWithDetail("LoadBackupFailed", accountMap[fmt.Sprintf("%d", cronjob.DownloadAccountID)].message)) + } + taskItem.AddSubTaskWithOps(task.GetTaskName(i18n.GetMsgByKey("SystemLog"), task.TaskBackup, task.TaskScopeCronjob), func(task *task.Task) error { nameItem := startTime.Format(constant.DateTimeSlimLayout) + common.RandStrAndNum(5) fileName := fmt.Sprintf("system_log_%s.tar.gz", nameItem) backupDir := path.Join(global.Dir.LocalBackupDir, "tmp/log", nameItem) @@ -261,15 +265,15 @@ func (u *CronjobService) handleSystemLog(cronjob model.Cronjob, startTime time.T record.CronjobID = cronjob.ID record.Name = cronjob.Name record.DownloadAccountID, record.SourceAccountIDs = cronjob.DownloadAccountID, cronjob.SourceAccountIDs - downloadPath, err := u.uploadCronjobBackFile(cronjob, task, accountMap, path.Join(path.Dir(backupDir), fileName)) - if err != nil { - taskItem.LogFailedWithErr("Upload backup file", err) + + src := path.Join(path.Dir(backupDir), fileName) + dst := strings.TrimPrefix(src, global.Dir.LocalBackupDir+"/tmp/") + if err := uploadWithMap(*task, accountMap, src, dst, cronjob.SourceAccountIDs, cronjob.DownloadAccountID, cronjob.RetryTimes); err != nil { return err } - record.FileDir = path.Dir(downloadPath) + record.FileDir = path.Dir(dst) record.FileName = fileName if err := backupRepo.CreateRecord(&record); err != nil { - taskItem.LogFailedWithErr("Save record", err) return err } u.removeExpiredBackup(cronjob, accountMap, record) @@ -279,11 +283,10 @@ func (u *CronjobService) handleSystemLog(cronjob model.Cronjob, startTime time.T } func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, jobRecord model.JobRecords, taskItem *task.Task) error { - accountMap, err := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ",")) - if err != nil { - return err + accountMap := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ",")) + if !accountMap[fmt.Sprintf("%d", cronjob.DownloadAccountID)].isOk { + return errors.New(i18n.GetMsgWithDetail("LoadBackupFailed", accountMap[fmt.Sprintf("%d", cronjob.DownloadAccountID)].message)) } - var record model.BackupRecord record.From = "cronjob" record.Type = "snapshot" @@ -461,6 +464,7 @@ func handleBackupLogs(taskItem *task.Task, targetDir, fileName string, secret st } } } + taskItem.Logf("%s Website logs...", i18n.GetMsgByKey("TaskBackup")) systemDir := path.Join(targetDir, "system") if _, err := os.Stat(systemDir); err != nil && os.IsNotExist(err) { @@ -469,7 +473,6 @@ func handleBackupLogs(taskItem *task.Task, targetDir, fileName string, secret st } } - taskItem.Logf("%s System logs...", i18n.GetMsgByKey("TaskBackup")) systemLogFiles, _ := os.ReadDir(global.Dir.LogDir) if len(systemLogFiles) != 0 { for i := 0; i < len(systemLogFiles); i++ { @@ -478,8 +481,8 @@ func handleBackupLogs(taskItem *task.Task, targetDir, fileName string, secret st } } } + taskItem.Logf("%s System logs...", i18n.GetMsgByKey("TaskBackup")) - taskItem.Logf("%s SSH logs...", i18n.GetMsgByKey("TaskBackup")) loginLogFiles, _ := os.ReadDir("/var/log") loginDir := path.Join(targetDir, "login") if _, err := os.Stat(loginDir); err != nil && os.IsNotExist(err) { @@ -494,7 +497,7 @@ func handleBackupLogs(taskItem *task.Task, targetDir, fileName string, secret st } } } - taskItem.Log("backup ssh log successful!") + taskItem.Logf("%s SSH logs...", i18n.GetMsgByKey("TaskBackup")) if err := fileOp.TarGzCompressPro(true, targetDir, path.Join(path.Dir(targetDir), fileName), secret, ""); err != nil { return err diff --git a/agent/app/service/cronjob_helper.go b/agent/app/service/cronjob_helper.go index fec679f07..95c3e0eec 100644 --- a/agent/app/service/cronjob_helper.go +++ b/agent/app/service/cronjob_helper.go @@ -10,6 +10,7 @@ import ( "time" "github.com/1Panel-dev/1Panel/agent/utils/alert_push" + "github.com/pkg/errors" "github.com/1Panel-dev/1Panel/agent/app/dto" "github.com/1Panel-dev/1Panel/agent/app/model" @@ -191,14 +192,14 @@ func (u *CronjobService) handleNtpSync(cronjob model.Cronjob, taskItem *task.Tas } func (u *CronjobService) handleCutWebsiteLog(cronjob *model.Cronjob, startTime time.Time, taskItem *task.Task) error { + clientMap := NewBackupClientMap([]string{fmt.Sprintf("%v", cronjob.DownloadAccountID)}) + if !clientMap[fmt.Sprintf("%d", cronjob.DownloadAccountID)].isOk { + return errors.New(i18n.GetMsgWithDetail("LoadBackupFailed", clientMap[fmt.Sprintf("%d", cronjob.DownloadAccountID)].message)) + } taskItem.AddSubTaskWithOps(i18n.GetWithName("CutWebsiteLog", cronjob.Name), func(t *task.Task) error { websites := loadWebsForJob(*cronjob) fileOp := files.NewFileOp() baseDir := GetOpenrestyDir(SitesRootDir) - clientMap, err := NewBackupClientMap([]string{fmt.Sprintf("%v", cronjob.DownloadAccountID)}) - if err != nil { - return fmt.Errorf("load local backup client failed, err: %v", err) - } for _, website := range websites { taskItem.Log(website.Alias) var record model.BackupRecord @@ -264,34 +265,6 @@ func (u *CronjobService) handleSystemClean(cronjob model.Cronjob, taskItem *task taskItem.AddSubTaskWithOps(i18n.GetMsgByKey("HandleSystemClean"), cleanTask, nil, int(cronjob.RetryTimes), time.Duration(cronjob.Timeout)*time.Second) } -func (u *CronjobService) uploadCronjobBackFile(cronjob model.Cronjob, task *task.Task, accountMap map[string]backupClientHelper, file string) (string, error) { - defer func() { - _ = os.Remove(file) - }() - var errItem error - accounts := strings.Split(cronjob.SourceAccountIDs, ",") - cloudSrc := strings.TrimPrefix(file, global.Dir.LocalBackupDir+"/tmp/") - for _, account := range accounts { - if len(account) != 0 { - task.LogStart(i18n.GetMsgWithMap("UploadFile", map[string]interface{}{ - "file": pathUtils.Join(accountMap[account].backupPath, cloudSrc), - "backup": accountMap[account].name, - })) - _, err := accountMap[account].client.Upload(file, pathUtils.Join(accountMap[account].backupPath, cloudSrc)) - task.LogWithStatus( - i18n.GetMsgWithMap("UploadFile", map[string]interface{}{ - "file": pathUtils.Join(accountMap[account].backupPath, cloudSrc), - "backup": accountMap[account].name, - }), err) - if err != nil { - errItem = err - continue - } - } - } - return cloudSrc, errItem -} - func (u *CronjobService) removeExpiredBackup(cronjob model.Cronjob, accountMap map[string]backupClientHelper, record model.BackupRecord) { var opts []repo.DBOption opts = append(opts, repo.WithByFrom("cronjob")) @@ -314,6 +287,9 @@ func (u *CronjobService) removeExpiredBackup(cronjob model.Cronjob, accountMap m if _, ok := accountMap[account]; !ok { continue } + if !accountMap[account].isOk { + continue + } _, _ = accountMap[account].client.Delete(pathUtils.Join(accountMap[account].backupPath, "system_snapshot", records[i].FileName)) } } @@ -324,6 +300,9 @@ func (u *CronjobService) removeExpiredBackup(cronjob model.Cronjob, accountMap m if _, ok := accountMap[account]; !ok { continue } + if !accountMap[account].isOk { + continue + } _, _ = accountMap[account].client.Delete(pathUtils.Join(accountMap[account].backupPath, records[i].FileDir, records[i].FileName)) } } diff --git a/agent/app/service/snapshot.go b/agent/app/service/snapshot.go index c5ea3ba3c..fd8fd0431 100644 --- a/agent/app/service/snapshot.go +++ b/agent/app/service/snapshot.go @@ -146,12 +146,13 @@ func (u *SnapshotService) Delete(req dto.SnapshotBatchDelete) error { snaps, _ := snapshotRepo.GetList(repo.WithByIDs(req.Ids)) for _, snap := range snaps { if req.DeleteWithFile { - accounts, err := NewBackupClientMap(strings.Split(snap.SourceAccountIDs, ",")) - if err != nil { - return err - } + accounts := NewBackupClientMap(strings.Split(snap.SourceAccountIDs, ",")) for _, item := range accounts { global.LOG.Debugf("remove snapshot file %s.tar.gz from %s", snap.Name, item.name) + if !item.isOk { + global.LOG.Errorf("remove snapshot file %s.tar.gz from %s failed, err: %s", snap.Name, item.name, item.message) + continue + } _, _ = item.client.Delete(path.Join(item.backupPath, "system_snapshot", snap.Name+".tar.gz")) } _ = backupRepo.DeleteRecord(context.Background(), repo.WithByType("snapshot"), backupRepo.WithByFileName(snap.Name+".tar.gz")) diff --git a/agent/app/service/snapshot_create.go b/agent/app/service/snapshot_create.go index c2c5f19c4..59ce66c56 100644 --- a/agent/app/service/snapshot_create.go +++ b/agent/app/service/snapshot_create.go @@ -93,10 +93,6 @@ func (u *SnapshotService) SnapshotReCreate(id uint) error { if err != nil { return err } - taskModel, err := taskRepo.GetFirst(taskRepo.WithResourceID(snap.ID), repo.WithByType(task.TaskScopeSnapshot)) - if err != nil { - return err - } var req dto.SnapshotCreate _ = copier.Copy(&req, snap) @@ -109,7 +105,7 @@ func (u *SnapshotService) SnapshotReCreate(id uint) error { if err := json.Unmarshal([]byte(snap.BackupData), &req.BackupData); err != nil { return err } - req.TaskID = taskModel.ID + req.TaskID = snap.TaskID taskItem, err := task.ReNewTaskWithOps(req.Name, task.TaskCreate, task.TaskScopeSnapshot, req.TaskID, req.ID) if err != nil { global.LOG.Errorf("new task for create snapshot failed, err: %v", err) @@ -117,7 +113,7 @@ func (u *SnapshotService) SnapshotReCreate(id uint) error { } _ = snapshotRepo.Update(req.ID, map[string]interface{}{"status": constant.StatusWaiting, "message": ""}) go func() { - _ = handleSnapshot(req, taskItem, 0, 3, 0) + _ = handleSnapshot(req, taskItem, 0, 3, req.Timeout) }() return nil @@ -197,7 +193,7 @@ func handleSnapshot(req dto.SnapshotCreate, taskItem *task.Task, jobID, retry, t taskItem.AddSubTaskWithAliasAndOps( "SnapUpload", func(t *task.Task) error { - return snapUpload(itemHelper, req.SourceAccountIDs, fmt.Sprintf("%s.tar.gz", rootDir)) + return snapUpload(itemHelper, req.SourceAccountIDs, req.DownloadAccountID, retry, fmt.Sprintf("%s.tar.gz", rootDir)) }, nil, int(retry), time.Duration(timeout)*time.Second, ) req.InterruptStep = "" @@ -427,7 +423,7 @@ func loadBackupExcludes(snap snapHelper, req []dto.DataTree) []string { if err := snap.snapAgentDB.Where("file_dir = ? AND file_name = ?", strings.TrimPrefix(path.Dir(item.Path), global.Dir.LocalBackupDir+"/"), path.Base(item.Path)).Delete(&model.BackupRecord{}).Error; err != nil { snap.Task.LogWithStatus("delete backup file from database", err) } - itemDir, _ := filepath.Rel(item.Path, global.Dir.LocalBackupDir) + itemDir, _ := filepath.Rel(global.Dir.LocalBackupDir, item.Path) excludes = append(excludes, itemDir) } else { excludes = append(excludes, loadBackupExcludes(snap, item.Children)...) @@ -440,7 +436,7 @@ func loadAppBackupExcludes(req []dto.DataTree) []string { for _, item := range req { if len(item.Children) == 0 { if !item.IsCheck { - itemDir, _ := filepath.Rel(item.Path, global.Dir.LocalBackupDir) + itemDir, _ := filepath.Rel(global.Dir.LocalBackupDir, item.Path) excludes = append(excludes, itemDir) } } else { @@ -502,7 +498,7 @@ func loadPanelExcludes(req []dto.DataTree) []string { for _, item := range req { if len(item.Children) == 0 { if !item.IsCheck { - itemDir, _ := filepath.Rel(item.Path, path.Join(global.Dir.BaseDir, "1panel")) + itemDir, _ := filepath.Rel(path.Join(global.Dir.BaseDir, "1panel"), item.Path) excludes = append(excludes, itemDir) } } else { @@ -536,26 +532,12 @@ func snapCompress(snap snapHelper, rootDir string, secret string) error { return nil } -func snapUpload(snap snapHelper, accounts string, file string) error { +func snapUpload(snap snapHelper, accounts string, downloadID, retry uint, file string) error { snap.Task.Log("---------------------- 8 / 8 ----------------------") snap.Task.LogStart(i18n.GetMsgByKey("SnapUpload")) - source := path.Join(global.Dir.LocalBackupDir, "tmp/system", path.Base(file)) - accountMap, err := NewBackupClientMap(strings.Split(accounts, ",")) - snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapLoadBackup"), err) - if err != nil { - return err - } - - targetAccounts := strings.Split(accounts, ",") - for _, item := range targetAccounts { - snap.Task.LogStart(i18n.GetWithName("SnapUploadTo", fmt.Sprintf("[%s] %s", accountMap[item].name, path.Join("system_snapshot", path.Base(file))))) - _, err := accountMap[item].client.Upload(source, path.Join(accountMap[item].backupPath, "system_snapshot", path.Base(file))) - snap.Task.LogWithStatus(i18n.GetWithName("SnapUploadRes", accountMap[item].name), err) - if err != nil { - return err - } - } - _ = os.Remove(source) - return nil + src := path.Join(global.Dir.LocalBackupDir, "tmp/system", path.Base(file)) + dst := path.Join("system_snapshot", path.Base(file)) + accountMap := NewBackupClientMap(strings.Split(accounts, ",")) + return uploadWithMap(snap.Task, accountMap, src, dst, accounts, downloadID, retry) } diff --git a/agent/i18n/lang/en.yaml b/agent/i18n/lang/en.yaml index 6781c8078..a86fb1aab 100644 --- a/agent/i18n/lang/en.yaml +++ b/agent/i18n/lang/en.yaml @@ -228,8 +228,10 @@ SystemLog: 'System Log' CutWebsiteLog: 'Rotate Website Log' FileOrDir: 'Directory / File' UploadFile: 'Uploading backup file {{ .file }} to {{ .backup }}' -IgnoreBackupErr: 'Backup failed, error: {{ .detail }}, ignoring this error...' -IgnoreUploadErr: 'Upload failed, error: {{ .detail }}, ignoring this error...' +Upload: "Upload" +IgnoreBackupErr: "Backup failed, error: {{ .detail }}, ignoring this error..." +IgnoreUploadErr: "Upload failed, error: {{ .detail }}, ignoring this error..." +LoadBackupFailed: "Failed to get backup account connection, error: {{ .detail }}" #toolbox ErrNotExistUser: 'The current user does not exist, please modify and try again!' @@ -376,7 +378,6 @@ SnapCompressFile: 'Compress snapshot file' SnapCheckCompress: 'Check snapshot compression file' SnapCompressSize: 'Snapshot file size {{ .name }}' SnapUpload: 'Upload snapshot file' -SnapLoadBackup: 'Get backup account information' SnapUploadTo: 'Upload snapshot file to {{ .name }}' SnapUploadRes: 'Upload snapshot file to {{ .name }}' diff --git a/agent/i18n/lang/ja.yaml b/agent/i18n/lang/ja.yaml index e5897e4fa..2656bf518 100644 --- a/agent/i18n/lang/ja.yaml +++ b/agent/i18n/lang/ja.yaml @@ -228,8 +228,10 @@ SystemLog: 'システムログ' CutWebsiteLog: 'ウェブサイトログのローテーション' FileOrDir: 'ディレクトリ / ファイル' UploadFile: 'バックアップファイル {{ .file }} を {{ .backup }} にアップロード中' -IgnoreBackupErr: 'バックアップ失敗、エラー：{{ .detail }}、このエラーを無視します...' -IgnoreUploadErr: 'アップロード失敗、エラー：{{ .detail }}、このエラーを無視します...' +Upload: "アップロード" +IgnoreBackupErr: "バックアップ失敗、エラー: {{ .detail }}、このエラーを無視します..." +IgnoreUploadErr: "アップロード失敗、エラー: {{ .detail }}、このエラーを無視します..." +LoadBackupFailed: "バックアップアカウントの接続取得に失敗、エラー: {{ .detail }}" #toolbox ErrNotExistUser: '現在のユーザーは存在しません。変更してもう一度お試しください。' @@ -376,7 +378,6 @@ SnapCompressFile: 'スナップショット ファイルを圧縮する' SnapCheckCompress: 'スナップショット圧縮ファイルをチェックする' SnapCompressSize: 'スナップショット ファイル サイズ {{ .name }}' SnapUpload: 'スナップショット ファイルをアップロード' -SnapLoadBackup: 'バックアップ アカウント情報を取得する' SnapUploadTo: 'スナップショット ファイルを {{ .name }} にアップロードします' SnapUploadRes: 'スナップショット ファイルを {{ .name }} にアップロードします' diff --git a/agent/i18n/lang/ko.yaml b/agent/i18n/lang/ko.yaml index 50ef72608..2fb5261d9 100644 --- a/agent/i18n/lang/ko.yaml +++ b/agent/i18n/lang/ko.yaml @@ -228,8 +228,10 @@ SystemLog: '시스템 로그' CutWebsiteLog: '웹사이트 로그 회전' FileOrDir: '디렉터리 / 파일' UploadFile: '백업 파일 {{ .file }} 을(를) {{ .backup }}(으)로 업로드 중' -IgnoreBackupErr: 'Sandaran gagal, ralat: {{ .detail }}, abaikan ralat ini...' -IgnoreUploadErr: 'Muat naik gagal, ralat: {{ .detail }}, abaikan ralat ini...' +Upload: "업로드" +IgnoreBackupErr: "백업 실패, 오류: {{ .detail }}, 이 오류를 무시합니다..." +IgnoreUploadErr: "업로드 실패, 오류: {{ .detail }}, 이 오류를 무시합니다..." +LoadBackupFailed: "백업 계정 연결 획득 실패, 오류: {{ .detail }}" #도구상자 ErrNotExistUser: '현재 사용자가 존재하지 않습니다. 수정한 후 다시 시도하세요!' @@ -376,7 +378,6 @@ SnapCompressFile: '스냅샷 파일 압축' SnapCheckCompress: '스냅샷 압축 파일 확인' SnapCompressSize: '스냅샷 파일 크기 {{ .name }}' SnapUpload: '스냅샷 파일 업로드' -SnapLoadBackup: '백업 계정 정보 가져오기' SnapUploadTo: '스냅샷 파일을 {{ .name }}에 업로드' SnapUploadRes: '스냅샷 파일을 {{ .name }}에 업로드' diff --git a/agent/i18n/lang/ms.yaml b/agent/i18n/lang/ms.yaml index d3cd27d19..cbe52a3d3 100644 --- a/agent/i18n/lang/ms.yaml +++ b/agent/i18n/lang/ms.yaml @@ -228,8 +228,10 @@ SystemLog: 'Log Sistem' CutWebsiteLog: 'Putar Log Laman Web' FileOrDir: 'Direktori / Fail' UploadFile: 'Muat naik fail sandaran {{ .file }} ke {{ .backup }}' -IgnoreBackupErr: 'Sandaran gagal, ralat: {{ .detail }}, abaikan ralat ini...' -IgnoreUploadErr: 'Muat naik gagal, ralat: {{ .detail }}, abaikan ralat ini...' +Upload: "Muat Naik" +IgnoreBackupErr: "Sandaran gagal, ralat: {{ .detail }}, abaikan ralat ini..." +IgnoreUploadErr: "Muat naik gagal, ralat: {{ .detail }}, abaikan ralat ini..." +LoadBackupFailed: "Gagal mendapatkan sambungan akaun sandaran, ralat: {{ .detail }}" #kotak alat ErrNotExistUser: 'Pengguna semasa tidak wujud, sila ubah suai dan cuba lagi!' @@ -376,7 +378,6 @@ SnapCompressFile: 'Mampatkan fail syot kilat' SnapCheckCompress: 'Semak fail mampatan syot kilat' SnapCompressSize: 'Saiz fail syot kilat {{ .name }}' SnapUpload: 'Muat naik fail syot kilat' -SnapLoadBackup: 'Dapatkan maklumat akaun sandaran' SnapUploadTo: 'Muat naik fail syot kilat ke {{ .name }}' SnapUploadRes: 'Muat naik fail syot kilat ke {{ .name }}' diff --git a/agent/i18n/lang/pt-BR.yaml b/agent/i18n/lang/pt-BR.yaml index 2d666fa03..cfe82f794 100644 --- a/agent/i18n/lang/pt-BR.yaml +++ b/agent/i18n/lang/pt-BR.yaml @@ -228,8 +228,10 @@ SystemLog: 'Log do Sistema' CutWebsiteLog: 'Rotacionar Log do Website' FileOrDir: 'Diretório / Arquivo' UploadFile: 'Enviando arquivo de backup {{ .file }} para {{ .backup }}' -IgnoreBackupErr: 'Backup falhou, erro: {{ .detail }}, ignorando este erro...' -IgnoreUploadErr: 'Upload falhou, erro: {{ .detail }}, ignorando este erro...' +Upload: "Enviar" +IgnoreBackupErr: "Backup falhou, erro: {{ .detail }}, ignorando este erro..." +IgnoreUploadErr: "Upload falhou, erro: {{ .detail }}, ignorando este erro..." +LoadBackupFailed: "Falha ao obter conexão da conta de backup, erro: {{ .detail }}" #caixa de ferramentas ErrNotExistUser: 'O usuário atual não existe, modifique e tente novamente!' @@ -376,7 +378,6 @@ SnapCompressFile: 'Compactar arquivo de instantâneo' SnapCheckCompress: 'Verificar arquivo de compactação de instantâneo' SnapCompressSize: 'Tamanho do arquivo de instantâneo {{ .name }}' SnapUpload: 'Carregar arquivo de instantâneo' -SnapLoadBackup: 'Obter informações da conta de backup' SnapUploadTo: 'Carregar arquivo de instantâneo para {{ .name }}' SnapUploadRes: 'Carregar arquivo de instantâneo para {{ .name }}' diff --git a/agent/i18n/lang/ru.yaml b/agent/i18n/lang/ru.yaml index 436106eb7..4e88eecbb 100644 --- a/agent/i18n/lang/ru.yaml +++ b/agent/i18n/lang/ru.yaml @@ -228,8 +228,10 @@ SystemLog: 'Системный лог' CutWebsiteLog: 'Ротация логов сайта' FileOrDir: 'Каталог / Файл' UploadFile: 'Загрузка файла резервной копии {{ .file }} в {{ .backup }}' -IgnoreBackupErr: 'Ошибка резервного копирования: {{ .detail }}, игнорируем эту ошибку...' -IgnoreUploadErr: 'Ошибка загрузки: {{ .detail }}, игнорируем эту ошибку...' +Upload: "Загрузить" +IgnoreBackupErr: "Ошибка резервного копирования: {{ .detail }}, игнорируем эту ошибку..." +IgnoreUploadErr: "Ошибка загрузки: {{ .detail }}, игнорируем эту ошибку..." +LoadBackupFailed: "Не удалось получить подключение к учетной записи резервной копии, ошибка: {{ .detail }}" #ящик для инструментов ErrNotExistUser: 'Текущий пользователь не существует, измените его и повторите попытку!' @@ -376,7 +378,6 @@ SnapCompressFile: 'Сжать файл моментального снимка' SnapCheckCompress: 'Проверить файл сжатия снимка' SnapCompressSize: 'Размер файла снимка {{ .name }}' SnapUpload: 'Загрузить файл снимка' -SnapLoadBackup: 'Получить резервную информацию об учетной записи' SnapUploadTo: 'Загрузить файл снимка в {{ .name }}' SnapUploadRes: 'Загрузить файл снимка в {{ .name }}' diff --git a/agent/i18n/lang/tr.yaml b/agent/i18n/lang/tr.yaml index 96a1a42e4..883aab0c1 100644 --- a/agent/i18n/lang/tr.yaml +++ b/agent/i18n/lang/tr.yaml @@ -229,6 +229,10 @@ SystemLog: 'Sistem Günlüğü' CutWebsiteLog: 'Web Sitesi Günlüğünü Döndür' FileOrDir: 'Dizin / Dosya' UploadFile: 'Yedekleme dosyası {{ .file }} {{ .backup }} konumuna yükleniyor' +Upload: "Yükle" +IgnoreBackupErr: "Yedekleme başarısız, hata: {{ .detail }}, bu hata yoksayılıyor..." +IgnoreUploadErr: "Yükleme başarısız, hata: {{ .detail }}, bu hata yoksayılıyor..." +LoadBackupFailed: "Yedek hesap bağlantısı alınamadı, hata: {{ .detail }}" #toolbox ErrNotExistUser: 'Mevcut kullanıcı mevcut değil, lütfen değiştirin ve tekrar deneyin!' @@ -375,7 +379,6 @@ SnapCompressFile: 'Anlık görüntü dosyasını sıkıştır' SnapCheckCompress: 'Anlık görüntü sıkıştırma dosyasını kontrol et' SnapCompressSize: 'Anlık görüntü dosya boyutu {{ .name }}' SnapUpload: 'Anlık görüntü dosyasını yükle' -SnapLoadBackup: 'Yedekleme hesabı bilgilerini al' SnapUploadTo: 'Anlık görüntü dosyasını {{ .name }} konumuna yükle' SnapUploadRes: 'Anlık görüntü dosyasını {{ .name }} konumuna yükle' diff --git a/agent/i18n/lang/zh-Hant.yaml b/agent/i18n/lang/zh-Hant.yaml index 339420e8a..4758ca367 100644 --- a/agent/i18n/lang/zh-Hant.yaml +++ b/agent/i18n/lang/zh-Hant.yaml @@ -227,8 +227,10 @@ SystemLog: '系統日誌' CutWebsiteLog: '切割網站日誌' FileOrDir: '目錄 / 檔案' UploadFile: '上傳備份文件 {{ .file }} 到 {{ .backup }}' -IgnoreBackupErr: '備份失敗，錯誤：{{ .detail }}，忽略本次錯誤...' -IgnoreUploadErr: '上傳失敗，錯誤：{{ .detail }}，忽略本次錯誤...' +Upload: "上傳" +IgnoreBackupErr: "備份失敗，錯誤：{{ .detail }}，忽略本次錯誤..." +IgnoreUploadErr: "上傳失敗，錯誤：{{ .detail }}，忽略本次錯誤..." +LoadBackupFailed: "獲取備份帳號連接失敗，錯誤：{{ .detail }}" #toolbox ErrNotExistUser: '目前使用者不存在，請修改後重試！' @@ -375,7 +377,6 @@ SnapCompressFile: '壓縮快照檔案' SnapCheckCompress: '檢查快照壓縮檔' SnapCompressSize: '快照檔案大小{{ .name }}' SnapUpload: '上傳快照檔案' -SnapLoadBackup: '取得備份帳號資訊' SnapUploadTo: '上傳快照檔案到{{ .name }}' SnapUploadRes: '上傳快照檔案到{{ .name }}' diff --git a/agent/i18n/lang/zh.yaml b/agent/i18n/lang/zh.yaml index 919947767..66bfd911a 100644 --- a/agent/i18n/lang/zh.yaml +++ b/agent/i18n/lang/zh.yaml @@ -227,8 +227,10 @@ SystemLog: "系统日志" CutWebsiteLog: "切割网站日志" FileOrDir: "目录 / 文件" UploadFile: "上传备份文件 {{ .file }} 到 {{ .backup }}" +Upload: "上传" IgnoreBackupErr: "备份失败，错误：{{ .detail }}，忽略本次错误..." IgnoreUploadErr: "上传失败，错误：{{ .detail }}，忽略本次错误..." +LoadBackupFailed: "获取备份账号连接失败，错误：{{ .detail }}" #toolbox ErrNotExistUser: "当前用户不存在，请修改后重试！" @@ -376,7 +378,6 @@ SnapCompressFile: "压缩快照文件" SnapCheckCompress: "检查快照压缩文件" SnapCompressSize: "快照文件大小 {{ .name }}" SnapUpload: "上传快照文件" -SnapLoadBackup: "获取备份账号信息" SnapUploadTo: "上传快照文件到 {{ .name }}" SnapUploadRes: "上传快照文件到 {{ .name }}" diff --git a/frontend/src/components/task-list/index.vue b/frontend/src/components/task-list/index.vue index d436ef521..6c034c438 100644 --- a/frontend/src/components/task-list/index.vue +++ b/frontend/src/components/task-list/index.vue @@ -17,7 +17,7 @@ - + @@ -181,7 +181,7 @@ - + diff --git a/frontend/src/views/cronjob/cronjob/record/index.vue b/frontend/src/views/cronjob/cronjob/record/index.vue index 793ae1547..57defa7de 100644 --- a/frontend/src/views/cronjob/cronjob/record/index.vue +++ b/frontend/src/views/cronjob/cronjob/record/index.vue @@ -156,14 +156,15 @@ {{ $t('commons.table.status') }} - - - - - - {{ currentRecord?.message }} + + + diff --git a/frontend/src/views/setting/snapshot/index.vue b/frontend/src/views/setting/snapshot/index.vue index 9c591a6cf..604f789c7 100644 --- a/frontend/src/views/setting/snapshot/index.vue +++ b/frontend/src/views/setting/snapshot/index.vue @@ -36,36 +36,34 @@ fix /> - +