1Panel-dev/1Panel
1
1
Fork 0
mirror of https://github.com/1Panel-dev/1Panel.git synced 2025-10-17 02:46:49 +08:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: Optimize backup logic for cronjob (#10145)

Browse source
This commit is contained in:
ssongliu 2025-08-26 17:07:41 +08:00 • committed by GitHub
parent ca84909716
commit 39f334f3a2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
20 changed files with 199 additions and 183 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
agent/app/service/backup.go
View file

@ -14,9 +14,11 @@ import (
"github.com/1Panel-dev/1Panel/agent/app/dto"
"github.com/1Panel-dev/1Panel/agent/app/model"
"github.com/1Panel-dev/1Panel/agent/app/repo"
"github.com/1Panel-dev/1Panel/agent/app/task"
"github.com/1Panel-dev/1Panel/agent/buserr"
"github.com/1Panel-dev/1Panel/agent/constant"
"github.com/1Panel-dev/1Panel/agent/global"
"github.com/1Panel-dev/1Panel/agent/i18n"
"github.com/1Panel-dev/1Panel/agent/utils/cloud_storage"
"github.com/1Panel-dev/1Panel/agent/utils/cloud_storage/client"
"github.com/1Panel-dev/1Panel/agent/utils/encrypt"
@ -393,9 +395,13 @@ type backupClientHelper struct {
name string
backupPath string
client cloud_storage.CloudStorageClient
isOk bool
hasBackuped bool
message string
}
func NewBackupClientMap(ids []string) (map[string]backupClientHelper, error) {
func NewBackupClientMap(ids []string) map[string]backupClientHelper {
var accounts []model.BackupAccount
var idItems []uint
for i := 0; i < len(ids); i++ {
@ -406,18 +412,59 @@ func NewBackupClientMap(ids []string) (map[string]backupClientHelper, error) {
clientMap := make(map[string]backupClientHelper)
for _, item := range accounts {
backClient, err := newClient(&item, true)
if err != nil {
return nil, err
}
clientMap[fmt.Sprintf("%v", item.ID)] = backupClientHelper{
itemHelper := backupClientHelper{
client: backClient,
name: item.Name,
backupPath: item.BackupPath,
accountType: item.Type,
id: item.ID,
isOk: err == nil,
}
if err != nil {
itemHelper.message = err.Error()
}
clientMap[fmt.Sprintf("%v", item.ID)] = itemHelper
}
return clientMap, nil
return clientMap
}
func uploadWithMap(taskItem task.Task, accountMap map[string]backupClientHelper, src, dst, accountIDs string, downloadAccountID, retry uint) error {
accounts := strings.Split(accountIDs, ",")
for _, account := range accounts {
if len(account) == 0 {
continue
}
itemBackup, ok := accountMap[account]
if !ok {
continue
}
if itemBackup.hasBackuped {
continue
}
if !itemBackup.isOk {
taskItem.LogFailed(i18n.GetMsgWithDetail("LoadBackupFailed", itemBackup.message))
continue
}
taskItem.LogStart(i18n.GetMsgWithMap("UploadFile", map[string]interface{}{
"file": path.Join(itemBackup.backupPath, dst),
"backup": itemBackup.name,
}))
for i := 0; i < int(retry); i++ {
_, err := itemBackup.client.Upload(src, path.Join(itemBackup.backupPath, dst))
taskItem.LogWithStatus(i18n.GetMsgByKey("Upload"), err)
if err != nil {
if account == fmt.Sprintf("%d", downloadAccountID) {
return err
}
} else {
break
}
}
itemBackup.hasBackuped = true
accountMap[account] = itemBackup
}
os.RemoveAll(src)
return nil
}
func newClient(account *model.BackupAccount, isEncrypt bool) (cloud_storage.CloudStorageClient, error) {

5
agent/app/service/backup_record.go
View file

@ -250,11 +250,8 @@ func (u *BackupRecordService) LoadRecordSize(req dto.SearchForSize) ([]dto.Recor
recordIds = append(recordIds, fmt.Sprintf("%v", record.DownloadID))
}
}
clientMap, err := NewBackupClientMap(recordIds)
if err != nil {
return nil, err
}
clientMap := NewBackupClientMap(recordIds)
var datas []dto.RecordFileSize
var wg sync.WaitGroup
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)

5
agent/app/service/cronjob.go
View file

@ -497,10 +497,7 @@ func (u *CronjobService) CleanRecord(req dto.CronjobClean) error {
}
if req.CleanData {
if hasBackup(cronjob.Type) {
accountMap, err := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ","))
if err != nil {
return err
}
accountMap := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ","))
if !req.CleanRemoteData {
for key := range accountMap {
if key != constant.Local {

91
agent/app/service/cronjob_backup.go
View file

@ -29,9 +29,9 @@ func (u *CronjobService) handleApp(cronjob model.Cronjob, startTime time.Time, t
if len(apps) == 0 {
return errors.New("no such app in database!")
}
accountMap, err := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ","))
if err != nil {
return err
accountMap := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ","))
if !accountMap[fmt.Sprintf("%d", cronjob.DownloadAccountID)].isOk {
return errors.New(i18n.GetMsgWithDetail("LoadBackupFailed", accountMap[fmt.Sprintf("%d", cronjob.DownloadAccountID)].message))
}
for _, app := range apps {
retry := 0
@ -54,8 +54,10 @@ func (u *CronjobService) handleApp(cronjob model.Cronjob, startTime time.Time, t
return nil
}
}
downloadPath, err := u.uploadCronjobBackFile(cronjob, task, accountMap, path.Join(backupDir, record.FileName))
if err != nil {
src := path.Join(backupDir, record.FileName)
dst := strings.TrimPrefix(src, global.Dir.LocalBackupDir+"/tmp/")
if err := uploadWithMap(*task, accountMap, src, dst, cronjob.SourceAccountIDs, cronjob.DownloadAccountID, cronjob.RetryTimes); err != nil {
if retry < int(cronjob.RetryTimes) || !cronjob.IgnoreErr {
retry++
return err
@ -63,7 +65,7 @@ func (u *CronjobService) handleApp(cronjob model.Cronjob, startTime time.Time, t
task.Log(i18n.GetMsgWithDetail("IgnoreUploadErr", err.Error()))
return nil
}
record.FileDir = path.Dir(downloadPath)
record.FileDir = path.Dir(dst)
if err := backupRepo.CreateRecord(&record); err != nil {
global.LOG.Errorf("save backup record failed, err: %v", err)
return err
@ -80,9 +82,9 @@ func (u *CronjobService) handleWebsite(cronjob model.Cronjob, startTime time.Tim
if len(webs) == 0 {
return errors.New("no such website in database!")
}
accountMap, err := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ","))
if err != nil {
return err
accountMap := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ","))
if !accountMap[fmt.Sprintf("%d", cronjob.DownloadAccountID)].isOk {
return errors.New(i18n.GetMsgWithDetail("LoadBackupFailed", accountMap[fmt.Sprintf("%d", cronjob.DownloadAccountID)].message))
}
for _, web := range webs {
retry := 0
@ -107,8 +109,9 @@ func (u *CronjobService) handleWebsite(cronjob model.Cronjob, startTime time.Tim
}
}
downloadPath, err := u.uploadCronjobBackFile(cronjob, task, accountMap, path.Join(backupDir, record.FileName))
if err != nil {
src := path.Join(backupDir, record.FileName)
dst := strings.TrimPrefix(src, global.Dir.LocalBackupDir+"/tmp/")
if err := uploadWithMap(*task, accountMap, src, dst, cronjob.SourceAccountIDs, cronjob.DownloadAccountID, cronjob.RetryTimes); err != nil {
if retry < int(cronjob.RetryTimes) || !cronjob.IgnoreErr {
retry++
return err
@ -116,7 +119,7 @@ func (u *CronjobService) handleWebsite(cronjob model.Cronjob, startTime time.Tim
task.Log(i18n.GetMsgWithDetail("IgnoreUploadErr", err.Error()))
return nil
}
record.FileDir = path.Dir(downloadPath)
record.FileDir = path.Dir(dst)
if err := backupRepo.CreateRecord(&record); err != nil {
global.LOG.Errorf("save backup record failed, err: %v", err)
return err
@ -133,9 +136,9 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Ti
if len(dbs) == 0 {
return errors.New("no such db in database!")
}
accountMap, err := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ","))
if err != nil {
return err
accountMap := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ","))
if !accountMap[fmt.Sprintf("%d", cronjob.DownloadAccountID)].isOk {
return errors.New(i18n.GetMsgWithDetail("LoadBackupFailed", accountMap[fmt.Sprintf("%d", cronjob.DownloadAccountID)].message))
}
for _, dbInfo := range dbs {
retry := 0
@ -173,8 +176,9 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Ti
}
}
downloadPath, err := u.uploadCronjobBackFile(cronjob, task, accountMap, path.Join(backupDir, record.FileName))
if err != nil {
src := path.Join(backupDir, record.FileName)
dst := strings.TrimPrefix(src, global.Dir.LocalBackupDir+"/tmp/")
if err := uploadWithMap(*task, accountMap, src, dst, cronjob.SourceAccountIDs, cronjob.DownloadAccountID, cronjob.RetryTimes); err != nil {
if retry < int(cronjob.RetryTimes) || !cronjob.IgnoreErr {
retry++
return err
@ -182,7 +186,7 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Ti
task.Log(i18n.GetMsgWithDetail("IgnoreUploadErr", err.Error()))
return nil
}
record.FileDir = path.Dir(downloadPath)
record.FileDir = path.Dir(dst)
if err := backupRepo.CreateRecord(&record); err != nil {
global.LOG.Errorf("save backup record failed, err: %v", err)
return err
@ -195,11 +199,11 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Ti
}
func (u *CronjobService) handleDirectory(cronjob model.Cronjob, startTime time.Time, taskItem *task.Task) error {
accountMap := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ","))
if !accountMap[fmt.Sprintf("%d", cronjob.DownloadAccountID)].isOk {
return errors.New(i18n.GetMsgWithDetail("LoadBackupFailed", accountMap[fmt.Sprintf("%d", cronjob.DownloadAccountID)].message))
}
taskItem.AddSubTaskWithOps(task.GetTaskName(cronjob.SourceDir, task.TaskBackup, task.TaskScopeCronjob), func(task *task.Task) error {
accountMap, err := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ","))
if err != nil {
return err
}
fileName := fmt.Sprintf("%s.tar.gz", startTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(2))
if cronjob.IsDir || len(strings.Split(cronjob.SourceDir, ",")) == 1 {
fileName = loadFileName(cronjob.SourceDir)
@ -226,15 +230,15 @@ func (u *CronjobService) handleDirectory(cronjob model.Cronjob, startTime time.T
record.CronjobID = cronjob.ID
record.Name = cronjob.Name
record.DownloadAccountID, record.SourceAccountIDs = cronjob.DownloadAccountID, cronjob.SourceAccountIDs
downloadPath, err := u.uploadCronjobBackFile(cronjob, task, accountMap, path.Join(backupDir, fileName))
if err != nil {
taskItem.LogFailedWithErr("Upload backup file", err)
src := path.Join(backupDir, fileName)
dst := strings.TrimPrefix(src, global.Dir.LocalBackupDir+"/tmp/")
if err := uploadWithMap(*task, accountMap, src, dst, cronjob.SourceAccountIDs, cronjob.DownloadAccountID, cronjob.RetryTimes); err != nil {
return err
}
record.FileDir = path.Dir(downloadPath)
record.FileDir = path.Dir(dst)
record.FileName = fileName
if err := backupRepo.CreateRecord(&record); err != nil {
taskItem.LogFailedWithErr("Save record", err)
return err
}
u.removeExpiredBackup(cronjob, accountMap, record)
@ -244,11 +248,11 @@ func (u *CronjobService) handleDirectory(cronjob model.Cronjob, startTime time.T
}
func (u *CronjobService) handleSystemLog(cronjob model.Cronjob, startTime time.Time, taskItem *task.Task) error {
taskItem.AddSubTaskWithOps(task.GetTaskName(i18n.GetMsgByKey("BackupSystemLog"), task.TaskBackup, task.TaskScopeCronjob), func(task *task.Task) error {
accountMap, err := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ","))
if err != nil {
return err
}
accountMap := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ","))
if !accountMap[fmt.Sprintf("%d", cronjob.DownloadAccountID)].isOk {
return errors.New(i18n.GetMsgWithDetail("LoadBackupFailed", accountMap[fmt.Sprintf("%d", cronjob.DownloadAccountID)].message))
}
taskItem.AddSubTaskWithOps(task.GetTaskName(i18n.GetMsgByKey("SystemLog"), task.TaskBackup, task.TaskScopeCronjob), func(task *task.Task) error {
nameItem := startTime.Format(constant.DateTimeSlimLayout) + common.RandStrAndNum(5)
fileName := fmt.Sprintf("system_log_%s.tar.gz", nameItem)
backupDir := path.Join(global.Dir.LocalBackupDir, "tmp/log", nameItem)
@ -261,15 +265,15 @@ func (u *CronjobService) handleSystemLog(cronjob model.Cronjob, startTime time.T
record.CronjobID = cronjob.ID
record.Name = cronjob.Name
record.DownloadAccountID, record.SourceAccountIDs = cronjob.DownloadAccountID, cronjob.SourceAccountIDs
downloadPath, err := u.uploadCronjobBackFile(cronjob, task, accountMap, path.Join(path.Dir(backupDir), fileName))
if err != nil {
taskItem.LogFailedWithErr("Upload backup file", err)
src := path.Join(path.Dir(backupDir), fileName)
dst := strings.TrimPrefix(src, global.Dir.LocalBackupDir+"/tmp/")
if err := uploadWithMap(*task, accountMap, src, dst, cronjob.SourceAccountIDs, cronjob.DownloadAccountID, cronjob.RetryTimes); err != nil {
return err
}
record.FileDir = path.Dir(downloadPath)
record.FileDir = path.Dir(dst)
record.FileName = fileName
if err := backupRepo.CreateRecord(&record); err != nil {
taskItem.LogFailedWithErr("Save record", err)
return err
}
u.removeExpiredBackup(cronjob, accountMap, record)
@ -279,11 +283,10 @@ func (u *CronjobService) handleSystemLog(cronjob model.Cronjob, startTime time.T
}
func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, jobRecord model.JobRecords, taskItem *task.Task) error {
accountMap, err := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ","))
if err != nil {
return err
accountMap := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ","))
if !accountMap[fmt.Sprintf("%d", cronjob.DownloadAccountID)].isOk {
return errors.New(i18n.GetMsgWithDetail("LoadBackupFailed", accountMap[fmt.Sprintf("%d", cronjob.DownloadAccountID)].message))
}
var record model.BackupRecord
record.From = "cronjob"
record.Type = "snapshot"
@ -461,6 +464,7 @@ func handleBackupLogs(taskItem *task.Task, targetDir, fileName string, secret st
}
}
}
taskItem.Logf("%s Website logs...", i18n.GetMsgByKey("TaskBackup"))
systemDir := path.Join(targetDir, "system")
if _, err := os.Stat(systemDir); err != nil && os.IsNotExist(err) {
@ -469,7 +473,6 @@ func handleBackupLogs(taskItem *task.Task, targetDir, fileName string, secret st
}
}
taskItem.Logf("%s System logs...", i18n.GetMsgByKey("TaskBackup"))
systemLogFiles, _ := os.ReadDir(global.Dir.LogDir)
if len(systemLogFiles) != 0 {
for i := 0; i < len(systemLogFiles); i++ {
@ -478,8 +481,8 @@ func handleBackupLogs(taskItem *task.Task, targetDir, fileName string, secret st
}
}
}
taskItem.Logf("%s System logs...", i18n.GetMsgByKey("TaskBackup"))
taskItem.Logf("%s SSH logs...", i18n.GetMsgByKey("TaskBackup"))
loginLogFiles, _ := os.ReadDir("/var/log")
loginDir := path.Join(targetDir, "login")
if _, err := os.Stat(loginDir); err != nil && os.IsNotExist(err) {
@ -494,7 +497,7 @@ func handleBackupLogs(taskItem *task.Task, targetDir, fileName string, secret st
}
}
}
taskItem.Log("backup ssh log successful!")
taskItem.Logf("%s SSH logs...", i18n.GetMsgByKey("TaskBackup"))
if err := fileOp.TarGzCompressPro(true, targetDir, path.Join(path.Dir(targetDir), fileName), secret, ""); err != nil {
return err

43
agent/app/service/cronjob_helper.go
View file

@ -10,6 +10,7 @@ import (
"time"
"github.com/1Panel-dev/1Panel/agent/utils/alert_push"
"github.com/pkg/errors"
"github.com/1Panel-dev/1Panel/agent/app/dto"
"github.com/1Panel-dev/1Panel/agent/app/model"
@ -191,14 +192,14 @@ func (u *CronjobService) handleNtpSync(cronjob model.Cronjob, taskItem *task.Tas
}
func (u *CronjobService) handleCutWebsiteLog(cronjob *model.Cronjob, startTime time.Time, taskItem *task.Task) error {
clientMap := NewBackupClientMap([]string{fmt.Sprintf("%v", cronjob.DownloadAccountID)})
if !clientMap[fmt.Sprintf("%d", cronjob.DownloadAccountID)].isOk {
return errors.New(i18n.GetMsgWithDetail("LoadBackupFailed", clientMap[fmt.Sprintf("%d", cronjob.DownloadAccountID)].message))
}
taskItem.AddSubTaskWithOps(i18n.GetWithName("CutWebsiteLog", cronjob.Name), func(t *task.Task) error {
websites := loadWebsForJob(*cronjob)
fileOp := files.NewFileOp()
baseDir := GetOpenrestyDir(SitesRootDir)
clientMap, err := NewBackupClientMap([]string{fmt.Sprintf("%v", cronjob.DownloadAccountID)})
if err != nil {
return fmt.Errorf("load local backup client failed, err: %v", err)
}
for _, website := range websites {
taskItem.Log(website.Alias)
var record model.BackupRecord
@ -264,34 +265,6 @@ func (u *CronjobService) handleSystemClean(cronjob model.Cronjob, taskItem *task
taskItem.AddSubTaskWithOps(i18n.GetMsgByKey("HandleSystemClean"), cleanTask, nil, int(cronjob.RetryTimes), time.Duration(cronjob.Timeout)*time.Second)
}
func (u *CronjobService) uploadCronjobBackFile(cronjob model.Cronjob, task *task.Task, accountMap map[string]backupClientHelper, file string) (string, error) {
defer func() {
_ = os.Remove(file)
}()
var errItem error
accounts := strings.Split(cronjob.SourceAccountIDs, ",")
cloudSrc := strings.TrimPrefix(file, global.Dir.LocalBackupDir+"/tmp/")
for _, account := range accounts {
if len(account) != 0 {
task.LogStart(i18n.GetMsgWithMap("UploadFile", map[string]interface{}{
"file": pathUtils.Join(accountMap[account].backupPath, cloudSrc),
"backup": accountMap[account].name,
}))
_, err := accountMap[account].client.Upload(file, pathUtils.Join(accountMap[account].backupPath, cloudSrc))
task.LogWithStatus(
i18n.GetMsgWithMap("UploadFile", map[string]interface{}{
"file": pathUtils.Join(accountMap[account].backupPath, cloudSrc),
"backup": accountMap[account].name,
}), err)
if err != nil {
errItem = err
continue
}
}
}
return cloudSrc, errItem
}
func (u *CronjobService) removeExpiredBackup(cronjob model.Cronjob, accountMap map[string]backupClientHelper, record model.BackupRecord) {
var opts []repo.DBOption
opts = append(opts, repo.WithByFrom("cronjob"))
@ -314,6 +287,9 @@ func (u *CronjobService) removeExpiredBackup(cronjob model.Cronjob, accountMap m
if _, ok := accountMap[account]; !ok {
continue
}
if !accountMap[account].isOk {
continue
}
_, _ = accountMap[account].client.Delete(pathUtils.Join(accountMap[account].backupPath, "system_snapshot", records[i].FileName))
}
}
@ -324,6 +300,9 @@ func (u *CronjobService) removeExpiredBackup(cronjob model.Cronjob, accountMap m
if _, ok := accountMap[account]; !ok {
continue
}
if !accountMap[account].isOk {
continue
}
_, _ = accountMap[account].client.Delete(pathUtils.Join(accountMap[account].backupPath, records[i].FileDir, records[i].FileName))
}
}

9
agent/app/service/snapshot.go
View file

@ -146,12 +146,13 @@ func (u *SnapshotService) Delete(req dto.SnapshotBatchDelete) error {
snaps, _ := snapshotRepo.GetList(repo.WithByIDs(req.Ids))
for _, snap := range snaps {
if req.DeleteWithFile {
accounts, err := NewBackupClientMap(strings.Split(snap.SourceAccountIDs, ","))
if err != nil {
return err
}
accounts := NewBackupClientMap(strings.Split(snap.SourceAccountIDs, ","))
for _, item := range accounts {
global.LOG.Debugf("remove snapshot file %s.tar.gz from %s", snap.Name, item.name)
if !item.isOk {
global.LOG.Errorf("remove snapshot file %s.tar.gz from %s failed, err: %s", snap.Name, item.name, item.message)
continue
}
_, _ = item.client.Delete(path.Join(item.backupPath, "system_snapshot", snap.Name+".tar.gz"))
}
_ = backupRepo.DeleteRecord(context.Background(), repo.WithByType("snapshot"), backupRepo.WithByFileName(snap.Name+".tar.gz"))

40
agent/app/service/snapshot_create.go
View file

@ -93,10 +93,6 @@ func (u *SnapshotService) SnapshotReCreate(id uint) error {
if err != nil {
return err
}
taskModel, err := taskRepo.GetFirst(taskRepo.WithResourceID(snap.ID), repo.WithByType(task.TaskScopeSnapshot))
if err != nil {
return err
}
var req dto.SnapshotCreate
_ = copier.Copy(&req, snap)
@ -109,7 +105,7 @@ func (u *SnapshotService) SnapshotReCreate(id uint) error {
if err := json.Unmarshal([]byte(snap.BackupData), &req.BackupData); err != nil {
return err
}
req.TaskID = taskModel.ID
req.TaskID = snap.TaskID
taskItem, err := task.ReNewTaskWithOps(req.Name, task.TaskCreate, task.TaskScopeSnapshot, req.TaskID, req.ID)
if err != nil {
global.LOG.Errorf("new task for create snapshot failed, err: %v", err)
@ -117,7 +113,7 @@ func (u *SnapshotService) SnapshotReCreate(id uint) error {
}
_ = snapshotRepo.Update(req.ID, map[string]interface{}{"status": constant.StatusWaiting, "message": ""})
go func() {
_ = handleSnapshot(req, taskItem, 0, 3, 0)
_ = handleSnapshot(req, taskItem, 0, 3, req.Timeout)
}()
return nil
@ -197,7 +193,7 @@ func handleSnapshot(req dto.SnapshotCreate, taskItem *task.Task, jobID, retry, t
taskItem.AddSubTaskWithAliasAndOps(
"SnapUpload",
func(t *task.Task) error {
return snapUpload(itemHelper, req.SourceAccountIDs, fmt.Sprintf("%s.tar.gz", rootDir))
return snapUpload(itemHelper, req.SourceAccountIDs, req.DownloadAccountID, retry, fmt.Sprintf("%s.tar.gz", rootDir))
}, nil, int(retry), time.Duration(timeout)*time.Second,
)
req.InterruptStep = ""
@ -427,7 +423,7 @@ func loadBackupExcludes(snap snapHelper, req []dto.DataTree) []string {
if err := snap.snapAgentDB.Where("file_dir = ? AND file_name = ?", strings.TrimPrefix(path.Dir(item.Path), global.Dir.LocalBackupDir+"/"), path.Base(item.Path)).Delete(&model.BackupRecord{}).Error; err != nil {
snap.Task.LogWithStatus("delete backup file from database", err)
}
itemDir, _ := filepath.Rel(item.Path, global.Dir.LocalBackupDir)
itemDir, _ := filepath.Rel(global.Dir.LocalBackupDir, item.Path)
excludes = append(excludes, itemDir)
} else {
excludes = append(excludes, loadBackupExcludes(snap, item.Children)...)
@ -440,7 +436,7 @@ func loadAppBackupExcludes(req []dto.DataTree) []string {
for _, item := range req {
if len(item.Children) == 0 {
if !item.IsCheck {
itemDir, _ := filepath.Rel(item.Path, global.Dir.LocalBackupDir)
itemDir, _ := filepath.Rel(global.Dir.LocalBackupDir, item.Path)
excludes = append(excludes, itemDir)
}
} else {
@ -502,7 +498,7 @@ func loadPanelExcludes(req []dto.DataTree) []string {
for _, item := range req {
if len(item.Children) == 0 {
if !item.IsCheck {
itemDir, _ := filepath.Rel(item.Path, path.Join(global.Dir.BaseDir, "1panel"))
itemDir, _ := filepath.Rel(path.Join(global.Dir.BaseDir, "1panel"), item.Path)
excludes = append(excludes, itemDir)
}
} else {
@ -536,26 +532,12 @@ func snapCompress(snap snapHelper, rootDir string, secret string) error {
return nil
}
func snapUpload(snap snapHelper, accounts string, file string) error {
func snapUpload(snap snapHelper, accounts string, downloadID, retry uint, file string) error {
snap.Task.Log("---------------------- 8 / 8 ----------------------")
snap.Task.LogStart(i18n.GetMsgByKey("SnapUpload"))
source := path.Join(global.Dir.LocalBackupDir, "tmp/system", path.Base(file))
accountMap, err := NewBackupClientMap(strings.Split(accounts, ","))
snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapLoadBackup"), err)
if err != nil {
return err
}
targetAccounts := strings.Split(accounts, ",")
for _, item := range targetAccounts {
snap.Task.LogStart(i18n.GetWithName("SnapUploadTo", fmt.Sprintf("[%s] %s", accountMap[item].name, path.Join("system_snapshot", path.Base(file)))))
_, err := accountMap[item].client.Upload(source, path.Join(accountMap[item].backupPath, "system_snapshot", path.Base(file)))
snap.Task.LogWithStatus(i18n.GetWithName("SnapUploadRes", accountMap[item].name), err)
if err != nil {
return err
}
}
_ = os.Remove(source)
return nil
src := path.Join(global.Dir.LocalBackupDir, "tmp/system", path.Base(file))
dst := path.Join("system_snapshot", path.Base(file))
accountMap := NewBackupClientMap(strings.Split(accounts, ","))
return uploadWithMap(snap.Task, accountMap, src, dst, accounts, downloadID, retry)
}

7
agent/i18n/lang/en.yaml
View file

@ -228,8 +228,10 @@ SystemLog: 'System Log'
CutWebsiteLog: 'Rotate Website Log'
FileOrDir: 'Directory / File'
UploadFile: 'Uploading backup file {{ .file }} to {{ .backup }}'
IgnoreBackupErr: 'Backup failed, error: {{ .detail }}, ignoring this error...'
IgnoreUploadErr: 'Upload failed, error: {{ .detail }}, ignoring this error...'
Upload: "Upload"
IgnoreBackupErr: "Backup failed, error: {{ .detail }}, ignoring this error..."
IgnoreUploadErr: "Upload failed, error: {{ .detail }}, ignoring this error..."
LoadBackupFailed: "Failed to get backup account connection, error: {{ .detail }}"
#toolbox
ErrNotExistUser: 'The current user does not exist, please modify and try again!'
@ -376,7 +378,6 @@ SnapCompressFile: 'Compress snapshot file'
SnapCheckCompress: 'Check snapshot compression file'
SnapCompressSize: 'Snapshot file size {{ .name }}'
SnapUpload: 'Upload snapshot file'
SnapLoadBackup: 'Get backup account information'
SnapUploadTo: 'Upload snapshot file to {{ .name }}'
SnapUploadRes: 'Upload snapshot file to {{ .name }}'

7
agent/i18n/lang/ja.yaml
View file

@ -228,8 +228,10 @@ SystemLog: 'システムログ'
CutWebsiteLog: 'ウェブサイトログのローテーション'
FileOrDir: 'ディレクトリ / ファイル'
UploadFile: 'バックアップファイル {{ .file }} を {{ .backup }} にアップロード中'
IgnoreBackupErr: 'バックアップ失敗、エラー：{{ .detail }}、このエラーを無視します...'
IgnoreUploadErr: 'アップロード失敗、エラー：{{ .detail }}、このエラーを無視します...'
Upload: "アップロード"
IgnoreBackupErr: "バックアップ失敗、エラー: {{ .detail }}、このエラーを無視します..."
IgnoreUploadErr: "アップロード失敗、エラー: {{ .detail }}、このエラーを無視します..."
LoadBackupFailed: "バックアップアカウントの接続取得に失敗、エラー: {{ .detail }}"
#toolbox
ErrNotExistUser: '現在のユーザーは存在しません。変更してもう一度お試しください。'
@ -376,7 +378,6 @@ SnapCompressFile: 'スナップショット ファイルを圧縮する'
SnapCheckCompress: 'スナップショット圧縮ファイルをチェックする'
SnapCompressSize: 'スナップショット ファイル サイズ {{ .name }}'
SnapUpload: 'スナップショット ファイルをアップロード'
SnapLoadBackup: 'バックアップ アカウント情報を取得する'
SnapUploadTo: 'スナップショット ファイルを {{ .name }} にアップロードします'
SnapUploadRes: 'スナップショット ファイルを {{ .name }} にアップロードします'

7
agent/i18n/lang/ko.yaml
View file

@ -228,8 +228,10 @@ SystemLog: '시스템 로그'
CutWebsiteLog: '웹사이트 로그 회전'
FileOrDir: '디렉터리 / 파일'
UploadFile: '백업 파일 {{ .file }} 을(를) {{ .backup }}(으)로 업로드 중'
IgnoreBackupErr: 'Sandaran gagal, ralat: {{ .detail }}, abaikan ralat ini...'
IgnoreUploadErr: 'Muat naik gagal, ralat: {{ .detail }}, abaikan ralat ini...'
Upload: "업로드"
IgnoreBackupErr: "백업 실패, 오류: {{ .detail }}, 이 오류를 무시합니다..."
IgnoreUploadErr: "업로드 실패, 오류: {{ .detail }}, 이 오류를 무시합니다..."
LoadBackupFailed: "백업 계정 연결 획득 실패, 오류: {{ .detail }}"
#도구상자
ErrNotExistUser: '현재 사용자가 존재하지 않습니다. 수정한 후 다시 시도하세요!'
@ -376,7 +378,6 @@ SnapCompressFile: '스냅샷 파일 압축'
SnapCheckCompress: '스냅샷 압축 파일 확인'
SnapCompressSize: '스냅샷 파일 크기 {{ .name }}'
SnapUpload: '스냅샷 파일 업로드'
SnapLoadBackup: '백업 계정 정보 가져오기'
SnapUploadTo: '스냅샷 파일을 {{ .name }}에 업로드'
SnapUploadRes: '스냅샷 파일을 {{ .name }}에 업로드'

7
agent/i18n/lang/ms.yaml
View file

@ -228,8 +228,10 @@ SystemLog: 'Log Sistem'
CutWebsiteLog: 'Putar Log Laman Web'
FileOrDir: 'Direktori / Fail'
UploadFile: 'Muat naik fail sandaran {{ .file }} ke {{ .backup }}'
IgnoreBackupErr: 'Sandaran gagal, ralat: {{ .detail }}, abaikan ralat ini...'
IgnoreUploadErr: 'Muat naik gagal, ralat: {{ .detail }}, abaikan ralat ini...'
Upload: "Muat Naik"
IgnoreBackupErr: "Sandaran gagal, ralat: {{ .detail }}, abaikan ralat ini..."
IgnoreUploadErr: "Muat naik gagal, ralat: {{ .detail }}, abaikan ralat ini..."
LoadBackupFailed: "Gagal mendapatkan sambungan akaun sandaran, ralat: {{ .detail }}"
#kotak alat
ErrNotExistUser: 'Pengguna semasa tidak wujud, sila ubah suai dan cuba lagi!'
@ -376,7 +378,6 @@ SnapCompressFile: 'Mampatkan fail syot kilat'
SnapCheckCompress: 'Semak fail mampatan syot kilat'
SnapCompressSize: 'Saiz fail syot kilat {{ .name }}'
SnapUpload: 'Muat naik fail syot kilat'
SnapLoadBackup: 'Dapatkan maklumat akaun sandaran'
SnapUploadTo: 'Muat naik fail syot kilat ke {{ .name }}'
SnapUploadRes: 'Muat naik fail syot kilat ke {{ .name }}'

7
agent/i18n/lang/pt-BR.yaml
View file

@ -228,8 +228,10 @@ SystemLog: 'Log do Sistema'
CutWebsiteLog: 'Rotacionar Log do Website'
FileOrDir: 'Diretório / Arquivo'
UploadFile: 'Enviando arquivo de backup {{ .file }} para {{ .backup }}'
IgnoreBackupErr: 'Backup falhou, erro: {{ .detail }}, ignorando este erro...'
IgnoreUploadErr: 'Upload falhou, erro: {{ .detail }}, ignorando este erro...'
Upload: "Enviar"
IgnoreBackupErr: "Backup falhou, erro: {{ .detail }}, ignorando este erro..."
IgnoreUploadErr: "Upload falhou, erro: {{ .detail }}, ignorando este erro..."
LoadBackupFailed: "Falha ao obter conexão da conta de backup, erro: {{ .detail }}"
#caixa de ferramentas
ErrNotExistUser: 'O usuário atual não existe, modifique e tente novamente!'
@ -376,7 +378,6 @@ SnapCompressFile: 'Compactar arquivo de instantâneo'
SnapCheckCompress: 'Verificar arquivo de compactação de instantâneo'
SnapCompressSize: 'Tamanho do arquivo de instantâneo {{ .name }}'
SnapUpload: 'Carregar arquivo de instantâneo'
SnapLoadBackup: 'Obter informações da conta de backup'
SnapUploadTo: 'Carregar arquivo de instantâneo para {{ .name }}'
SnapUploadRes: 'Carregar arquivo de instantâneo para {{ .name }}'

7
agent/i18n/lang/ru.yaml
View file

@ -228,8 +228,10 @@ SystemLog: 'Системный лог'
CutWebsiteLog: 'Ротация логов сайта'
FileOrDir: 'Каталог / Файл'
UploadFile: 'Загрузка файла резервной копии {{ .file }} в {{ .backup }}'
IgnoreBackupErr: 'Ошибка резервного копирования: {{ .detail }}, игнорируем эту ошибку...'
IgnoreUploadErr: 'Ошибка загрузки: {{ .detail }}, игнорируем эту ошибку...'
Upload: "Загрузить"
IgnoreBackupErr: "Ошибка резервного копирования: {{ .detail }}, игнорируем эту ошибку..."
IgnoreUploadErr: "Ошибка загрузки: {{ .detail }}, игнорируем эту ошибку..."
LoadBackupFailed: "Не удалось получить подключение к учетной записи резервной копии, ошибка: {{ .detail }}"
#ящик для инструментов
ErrNotExistUser: 'Текущий пользователь не существует, измените его и повторите попытку!'
@ -376,7 +378,6 @@ SnapCompressFile: 'Сжать файл моментального снимка'
SnapCheckCompress: 'Проверить файл сжатия снимка'
SnapCompressSize: 'Размер файла снимка {{ .name }}'
SnapUpload: 'Загрузить файл снимка'
SnapLoadBackup: 'Получить резервную информацию об учетной записи'
SnapUploadTo: 'Загрузить файл снимка в {{ .name }}'
SnapUploadRes: 'Загрузить файл снимка в {{ .name }}'

5
agent/i18n/lang/tr.yaml
View file

@ -229,6 +229,10 @@ SystemLog: 'Sistem Günlüğü'
CutWebsiteLog: 'Web Sitesi Günlüğünü Döndür'
FileOrDir: 'Dizin / Dosya'
UploadFile: 'Yedekleme dosyası {{ .file }} {{ .backup }} konumuna yükleniyor'
Upload: "Yükle"
IgnoreBackupErr: "Yedekleme başarısız, hata: {{ .detail }}, bu hata yoksayılıyor..."
IgnoreUploadErr: "Yükleme başarısız, hata: {{ .detail }}, bu hata yoksayılıyor..."
LoadBackupFailed: "Yedek hesap bağlantısı alınamadı, hata: {{ .detail }}"
#toolbox
ErrNotExistUser: 'Mevcut kullanıcı mevcut değil, lütfen değiştirin ve tekrar deneyin!'
@ -375,7 +379,6 @@ SnapCompressFile: 'Anlık görüntü dosyasını sıkıştır'
SnapCheckCompress: 'Anlık görüntü sıkıştırma dosyasını kontrol et'
SnapCompressSize: 'Anlık görüntü dosya boyutu {{ .name }}'
SnapUpload: 'Anlık görüntü dosyasını yükle'
SnapLoadBackup: 'Yedekleme hesabı bilgilerini al'
SnapUploadTo: 'Anlık görüntü dosyasını {{ .name }} konumuna yükle'
SnapUploadRes: 'Anlık görüntü dosyasını {{ .name }} konumuna yükle'

7
agent/i18n/lang/zh-Hant.yaml
View file

@ -227,8 +227,10 @@ SystemLog: '系統日誌'
CutWebsiteLog: '切割網站日誌'
FileOrDir: '目錄 / 檔案'
UploadFile: '上傳備份文件 {{ .file }} 到 {{ .backup }}'
IgnoreBackupErr: '備份失敗，錯誤：{{ .detail }}，忽略本次錯誤...'
IgnoreUploadErr: '上傳失敗，錯誤：{{ .detail }}，忽略本次錯誤...'
Upload: "上傳"
IgnoreBackupErr: "備份失敗，錯誤：{{ .detail }}，忽略本次錯誤..."
IgnoreUploadErr: "上傳失敗，錯誤：{{ .detail }}，忽略本次錯誤..."
LoadBackupFailed: "獲取備份帳號連接失敗，錯誤：{{ .detail }}"
#toolbox
ErrNotExistUser: '目前使用者不存在，請修改後重試！'
@ -375,7 +377,6 @@ SnapCompressFile: '壓縮快照檔案'
SnapCheckCompress: '檢查快照壓縮檔'
SnapCompressSize: '快照檔案大小{{ .name }}'
SnapUpload: '上傳快照檔案'
SnapLoadBackup: '取得備份帳號資訊'
SnapUploadTo: '上傳快照檔案到{{ .name }}'
SnapUploadRes: '上傳快照檔案到{{ .name }}'

3
agent/i18n/lang/zh.yaml
View file

@ -227,8 +227,10 @@ SystemLog: "系统日志"
CutWebsiteLog: "切割网站日志"
FileOrDir: "目录 / 文件"
UploadFile: "上传备份文件 {{ .file }} 到 {{ .backup }}"
Upload: "上传"
IgnoreBackupErr: "备份失败，错误：{{ .detail }}，忽略本次错误..."
IgnoreUploadErr: "上传失败，错误：{{ .detail }}，忽略本次错误..."
LoadBackupFailed: "获取备份账号连接失败，错误：{{ .detail }}"
#toolbox
ErrNotExistUser: "当前用户不存在，请修改后重试！"
@ -376,7 +378,6 @@ SnapCompressFile: "压缩快照文件"
SnapCheckCompress: "检查快照压缩文件"
SnapCompressSize: "快照文件大小 {{ .name }}"
SnapUpload: "上传快照文件"
SnapLoadBackup: "获取备份账号信息"
SnapUploadTo: "上传快照文件到 {{ .name }}"
SnapUploadRes: "上传快照文件到 {{ .name }}"

2
frontend/src/components/task-list/index.vue
View file

@ -17,7 +17,7 @@
<el-option :label="$t('logs.taskRunning')" value="Executing"></el-option>
</el-select>
<TableRefresh @search="search()" />
<TableSetting @search="search()" :rate="5" />
<TableSetting title="task-refresh" @search="search()" :rate="5" />
</template>
<template #main>
<ComplexTable :pagination-config="paginationConfig" :data="data" @search="search" :heightDiff="320">

4
frontend/src/views/ai/gpu/index.vue
View file

@ -20,7 +20,7 @@
<el-row>
<el-col :xs="24" :sm="16" :md="16" :lg="16" :xl="16" />
<el-col :xs="24" :sm="8" :md="8" :lg="8" :xl="8">
<TableSetting @search="refresh()" />
<TableSetting title="gpu-refresh" @search="refresh()" />
</el-col>
</el-row>
</template>
@ -181,7 +181,7 @@
<el-row>
<el-col :xs="24" :sm="16" :md="16" :lg="16" :xl="16" />
<el-col :xs="24" :sm="8" :md="8" :lg="8" :xl="8">
<TableSetting @search="refresh()" />
<TableSetting title="xpu-refresh" @search="refresh()" />
</el-col>
</el-row>
</template>

17
frontend/src/views/cronjob/cronjob/record/index.vue
View file

@ -156,14 +156,15 @@
<span class="status-label">{{ $t('commons.table.status') }}</span>
</template>
<Status :status="currentRecord?.status" />
</el-form-item>
</el-row>
<el-row v-if="currentRecord?.status === 'Failed'">
<el-form-item class="w-full">
<template #label>
<span class="status-label">{{ $t('commons.table.message') }}</span>
</template>
{{ currentRecord?.message }}
<el-tooltip :content="currentRecord?.message">
<el-button
class="mt-0.5"
type="danger"
v-if="currentRecord?.status === 'Failed'"
icon="Warning"
link
/>
</el-tooltip>
</el-form-item>
</el-row>
<el-row v-if="currentRecord?.taskID && currentRecord?.taskID != ''">

50
frontend/src/views/setting/snapshot/index.vue
View file

@ -36,36 +36,34 @@
fix
/>
<el-table-column prop="version" :label="$t('app.version')" />
<el-table-column :label="$t('setting.backupAccount')" min-width="80" prop="from">
<el-table-column :label="$t('setting.backupAccount')" :min-width="120" prop="from">
<template #default="{ row }">
<div class="flex items-center justify-start gap-0.5">
<div v-for="(item, index) of row.sourceAccounts" :key="index" class="mt-1">
<div v-if="row.expand || (!row.expand && index < 3)">
<span type="info">
<span>
{{ item === 'localhost' ? $t('setting.LOCAL') : item }}
</span>
<el-icon
v-if="item === row.downloadAccount"
size="12"
class="relative top-px left-1"
>
<Star />
</el-icon>
<div v-for="(item, index) of row.sourceAccounts" :key="index">
<div v-if="row.expand || (!row.expand && index < 3)">
<span type="info">
<span>
{{ item === 'localhost' ? $t('setting.LOCAL') : item }}
</span>
</div>
</div>
<div v-if="!row.expand && row.sourceAccounts?.length > 3">
<el-button type="primary" link @click="row.expand = true">
{{ $t('commons.button.expand') }}...
</el-button>
</div>
<div v-if="row.expand && row.sourceAccounts?.length > 3">
<el-button type="primary" link @click="row.expand = false">
{{ $t('commons.button.collapse') }}
</el-button>
<el-icon
v-if="item === row.downloadAccount"
size="12"
class="relative top-px left-1"
>
<Star />
</el-icon>
</span>
</div>
</div>
<div v-if="!row.expand && row.sourceAccounts?.length > 3">
<el-button type="primary" link @click="row.expand = true">
{{ $t('commons.button.expand') }}...
</el-button>
</div>
<div v-if="row.expand && row.sourceAccounts?.length > 3">
<el-button type="primary" link @click="row.expand = false">
{{ $t('commons.button.collapse') }}
</el-button>
</div>
</template>
</el-table-column>
<el-table-column :label="$t('file.size')" prop="size" min-width="60" show-overflow-tooltip>