1Panel-dev/1Panel
1
1
Fork 0
mirror of https://github.com/1Panel-dev/1Panel.git synced 2025-09-07 15:14:36 +08:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: Optimize backup logic for cronjob (#10163)

Browse source
This commit is contained in:
ssongliu 2025-08-27 14:35:28 +08:00 • committed by GitHub
parent abf3247c50
commit 5a34e2d380
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 52 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
agent/app/api/v2/snapshot.go
View file

@ -36,7 +36,7 @@ func (b *BaseApi) CreateSnapshot(c *gin.Context) {
return return
} }
if err := snapshotService.SnapshotCreate(nil, req, 0, 0, 0); err != nil { if err := snapshotService.SnapshotCreate(nil, req, 0, 3); err != nil {
helper.InternalServer(c, err) helper.InternalServer(c, err)
return return
} }

10
agent/app/service/backup.go
View file

@ -396,9 +396,9 @@ type backupClientHelper struct {
backupPath string backupPath string
client cloud_storage.CloudStorageClient client cloud_storage.CloudStorageClient
isOk bool isOk bool
hasBackuped bool hasBackup bool
message string message string
} }
func NewBackupClientMap(ids []string) map[string]backupClientHelper { func NewBackupClientMap(ids []string) map[string]backupClientHelper {
@ -438,7 +438,7 @@ func uploadWithMap(taskItem task.Task, accountMap map[string]backupClientHelper,
if !ok { if !ok {
continue continue
} }
if itemBackup.hasBackuped { if itemBackup.hasBackup {
continue continue
} }
if !itemBackup.isOk { if !itemBackup.isOk {
@ -460,7 +460,7 @@ func uploadWithMap(taskItem task.Task, accountMap map[string]backupClientHelper,
break break
} }
} }
itemBackup.hasBackuped = true itemBackup.hasBackup = true
accountMap[account] = itemBackup accountMap[account] = itemBackup
} }
os.RemoveAll(src) os.RemoveAll(src)

3
agent/app/service/backup_record.go
View file

@ -259,6 +259,9 @@ func (u *BackupRecordService) LoadRecordSize(req dto.SearchForSize) ([]dto.Recor
for i := 0; i < len(list); i++ { for i := 0; i < len(list); i++ {
datas = append(datas, dto.RecordFileSize{ID: list[i].ID}) datas = append(datas, dto.RecordFileSize{ID: list[i].ID})
if val, ok := clientMap[fmt.Sprintf("%v", list[i].DownloadID)]; ok { if val, ok := clientMap[fmt.Sprintf("%v", list[i].DownloadID)]; ok {
if !val.isOk {
continue
}
wg.Add(1) wg.Add(1)
go func(index int) { go func(index int) {
defer wg.Done() defer wg.Done()

26
agent/app/service/cronjob_backup.go
View file

@ -51,6 +51,7 @@ func (u *CronjobService) handleApp(cronjob model.Cronjob, startTime time.Time, t
return err return err
} else { } else {
task.Log(i18n.GetMsgWithDetail("IgnoreBackupErr", err.Error())) task.Log(i18n.GetMsgWithDetail("IgnoreBackupErr", err.Error()))
cleanAccountMap(accountMap)
return nil return nil
} }
} }
@ -63,6 +64,7 @@ func (u *CronjobService) handleApp(cronjob model.Cronjob, startTime time.Time, t
return err return err
} }
task.Log(i18n.GetMsgWithDetail("IgnoreUploadErr", err.Error())) task.Log(i18n.GetMsgWithDetail("IgnoreUploadErr", err.Error()))
cleanAccountMap(accountMap)
return nil return nil
} }
record.FileDir = path.Dir(dst) record.FileDir = path.Dir(dst)
@ -71,6 +73,7 @@ func (u *CronjobService) handleApp(cronjob model.Cronjob, startTime time.Time, t
return err return err
} }
u.removeExpiredBackup(cronjob, accountMap, record) u.removeExpiredBackup(cronjob, accountMap, record)
cleanAccountMap(accountMap)
return nil return nil
}, nil, int(cronjob.RetryTimes), time.Duration(cronjob.Timeout)*time.Second) }, nil, int(cronjob.RetryTimes), time.Duration(cronjob.Timeout)*time.Second)
} }
@ -105,6 +108,7 @@ func (u *CronjobService) handleWebsite(cronjob model.Cronjob, startTime time.Tim
return err return err
} else { } else {
task.Log(i18n.GetMsgWithDetail("IgnoreBackupErr", err.Error())) task.Log(i18n.GetMsgWithDetail("IgnoreBackupErr", err.Error()))
cleanAccountMap(accountMap)
return nil return nil
} }
} }
@ -117,6 +121,7 @@ func (u *CronjobService) handleWebsite(cronjob model.Cronjob, startTime time.Tim
return err return err
} }
task.Log(i18n.GetMsgWithDetail("IgnoreUploadErr", err.Error())) task.Log(i18n.GetMsgWithDetail("IgnoreUploadErr", err.Error()))
cleanAccountMap(accountMap)
return nil return nil
} }
record.FileDir = path.Dir(dst) record.FileDir = path.Dir(dst)
@ -125,6 +130,7 @@ func (u *CronjobService) handleWebsite(cronjob model.Cronjob, startTime time.Tim
return err return err
} }
u.removeExpiredBackup(cronjob, accountMap, record) u.removeExpiredBackup(cronjob, accountMap, record)
cleanAccountMap(accountMap)
return nil return nil
}, nil, int(cronjob.RetryTimes), time.Duration(cronjob.Timeout)*time.Second) }, nil, int(cronjob.RetryTimes), time.Duration(cronjob.Timeout)*time.Second)
} }
@ -161,6 +167,7 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Ti
return err return err
} else { } else {
task.Log(i18n.GetMsgWithDetail("IgnoreBackupErr", err.Error())) task.Log(i18n.GetMsgWithDetail("IgnoreBackupErr", err.Error()))
cleanAccountMap(accountMap)
return nil return nil
} }
} }
@ -171,6 +178,7 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Ti
return err return err
} else { } else {
task.Log(i18n.GetMsgWithDetail("IgnoreBackupErr", err.Error())) task.Log(i18n.GetMsgWithDetail("IgnoreBackupErr", err.Error()))
cleanAccountMap(accountMap)
return nil return nil
} }
} }
@ -184,6 +192,7 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Ti
return err return err
} }
task.Log(i18n.GetMsgWithDetail("IgnoreUploadErr", err.Error())) task.Log(i18n.GetMsgWithDetail("IgnoreUploadErr", err.Error()))
cleanAccountMap(accountMap)
return nil return nil
} }
record.FileDir = path.Dir(dst) record.FileDir = path.Dir(dst)
@ -192,6 +201,7 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Ti
return err return err
} }
u.removeExpiredBackup(cronjob, accountMap, record) u.removeExpiredBackup(cronjob, accountMap, record)
cleanAccountMap(accountMap)
return nil return nil
}, nil, int(cronjob.RetryTimes), time.Duration(cronjob.Timeout)*time.Second) }, nil, int(cronjob.RetryTimes), time.Duration(cronjob.Timeout)*time.Second)
} }
@ -306,9 +316,10 @@ func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, jobRecord model.J
return err return err
} }
req := dto.SnapshotCreate{ req := dto.SnapshotCreate{
Name: fmt.Sprintf("snapshot-1panel-%s-%s-linux-%s-%s", scope, versionItem.Value, loadOs(), jobRecord.StartTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5)), Name: fmt.Sprintf("snapshot-1panel-%s-%s-linux-%s-%s", scope, versionItem.Value, loadOs(), jobRecord.StartTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5)),
Secret: cronjob.Secret, Secret: cronjob.Secret,
TaskID: jobRecord.TaskID, TaskID: jobRecord.TaskID,
Timeout: cronjob.Timeout,
SourceAccountIDs: record.SourceAccountIDs, SourceAccountIDs: record.SourceAccountIDs,
DownloadAccountID: cronjob.DownloadAccountID, DownloadAccountID: cronjob.DownloadAccountID,
@ -324,7 +335,7 @@ func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, jobRecord model.J
IgnoreFiles: strings.Split(cronjob.ExclusionRules, ","), IgnoreFiles: strings.Split(cronjob.ExclusionRules, ","),
} }
if err := NewISnapshotService().SnapshotCreate(taskItem, req, jobRecord.ID, cronjob.RetryTimes, cronjob.Timeout); err != nil { if err := NewISnapshotService().SnapshotCreate(taskItem, req, jobRecord.ID, cronjob.RetryTimes); err != nil {
return err return err
} }
record.FileName = req.Name + ".tar.gz" record.FileName = req.Name + ".tar.gz"
@ -579,3 +590,10 @@ func simplifiedFileName(name string) string {
name = strings.ReplaceAll(name, "|", "_") name = strings.ReplaceAll(name, "|", "_")
return name return name
} }
func cleanAccountMap(accountMap map[string]backupClientHelper) {
for key, val := range accountMap {
val.hasBackup = false
accountMap[key] = val
}
}

34
agent/app/service/cronjob_helper.go
View file

@ -31,12 +31,12 @@ func (u *CronjobService) HandleJob(cronjob *model.Cronjob) {
global.LOG.Errorf("new task for exec shell failed, err: %v", err) global.LOG.Errorf("new task for exec shell failed, err: %v", err)
return return
} }
go func() { if cronjob.Type == "snapshot" {
err = u.loadTask(cronjob, &record, taskItem) go func() {
if cronjob.Type == "snapshot" { _ = cronjobRepo.UpdateRecords(record.ID, map[string]interface{}{"records": record.Records})
if err != nil { if err = u.handleSnapshot(*cronjob, record, taskItem); err != nil {
taskItem, _ := taskRepo.GetFirst(taskRepo.WithByID(record.TaskID)) item, _ := taskRepo.GetFirst(taskRepo.WithByID(record.TaskID))
if len(taskItem.ID) == 0 { if len(item.ID) == 0 {
record.TaskID = "" record.TaskID = ""
} }
cronjobRepo.EndRecords(record, constant.StatusFailed, err.Error(), record.Records) cronjobRepo.EndRecords(record, constant.StatusFailed, err.Error(), record.Records)
@ -44,15 +44,20 @@ func (u *CronjobService) HandleJob(cronjob *model.Cronjob) {
return return
} }
cronjobRepo.EndRecords(record, constant.StatusSuccess, "", record.Records) cronjobRepo.EndRecords(record, constant.StatusSuccess, "", record.Records)
return }()
} return
if err != nil { }
global.LOG.Debugf("preper to handle cron job [%s] %s failed, err: %v", cronjob.Type, cronjob.Name, err) if err = u.loadTask(cronjob, &record, taskItem); err != nil {
global.LOG.Debugf("preper to handle cron job [%s] %s failed, err: %v", cronjob.Type, cronjob.Name, err)
item, _ := taskRepo.GetFirst(taskRepo.WithByID(record.TaskID))
if len(item.ID) == 0 {
record.TaskID = "" record.TaskID = ""
cronjobRepo.EndRecords(record, constant.StatusFailed, err.Error(), record.Records)
handleCronJobAlert(cronjob)
return
} }
cronjobRepo.EndRecords(record, constant.StatusFailed, err.Error(), record.Records)
handleCronJobAlert(cronjob)
return
}
go func() {
if err := taskItem.Execute(); err != nil { if err := taskItem.Execute(); err != nil {
taskItem, _ := taskRepo.GetFirst(taskRepo.WithByID(record.TaskID)) taskItem, _ := taskRepo.GetFirst(taskRepo.WithByID(record.TaskID))
if len(taskItem.ID) == 0 { if len(taskItem.ID) == 0 {
@ -110,9 +115,6 @@ func (u *CronjobService) loadTask(cronjob *model.Cronjob, record *model.JobRecor
err = u.handleDirectory(*cronjob, record.StartTime, taskItem) err = u.handleDirectory(*cronjob, record.StartTime, taskItem)
case "log": case "log":
err = u.handleSystemLog(*cronjob, record.StartTime, taskItem) err = u.handleSystemLog(*cronjob, record.StartTime, taskItem)
case "snapshot":
_ = cronjobRepo.UpdateRecords(record.ID, map[string]interface{}{"records": record.Records})
err = u.handleSnapshot(*cronjob, *record, taskItem)
} }
return err return err
} }

2
agent/app/service/snapshot.go
View file

@ -31,7 +31,7 @@ type SnapshotService struct {
type ISnapshotService interface { type ISnapshotService interface {
SearchWithPage(req dto.PageSnapshot) (int64, interface{}, error) SearchWithPage(req dto.PageSnapshot) (int64, interface{}, error)
LoadSnapshotData() (dto.SnapshotData, error) LoadSnapshotData() (dto.SnapshotData, error)
SnapshotCreate(parentTask *task.Task, req dto.SnapshotCreate, jobID, retry, timeout uint) error SnapshotCreate(parentTask *task.Task, req dto.SnapshotCreate, jobID, retry uint) error
SnapshotReCreate(id uint) error SnapshotReCreate(id uint) error
SnapshotRecover(req dto.SnapshotRecover) error SnapshotRecover(req dto.SnapshotRecover) error
SnapshotRollback(req dto.SnapshotRecover) error SnapshotRollback(req dto.SnapshotRecover) error

4
agent/app/service/snapshot_create.go
View file

@ -27,7 +27,7 @@ import (
"gorm.io/gorm" "gorm.io/gorm"
) )
func (u *SnapshotService) SnapshotCreate(parentTask *task.Task, req dto.SnapshotCreate, jobID, retry, timeout uint) error { func (u *SnapshotService) SnapshotCreate(parentTask *task.Task, req dto.SnapshotCreate, jobID, retry uint) error {
versionItem, _ := settingRepo.Get(settingRepo.WithByKey("SystemVersion")) versionItem, _ := settingRepo.Get(settingRepo.WithByKey("SystemVersion"))
scope := "core" scope := "core"
@ -85,7 +85,7 @@ func (u *SnapshotService) SnapshotCreate(parentTask *task.Task, req dto.Snapshot
return nil return nil
} }
return handleSnapshot(req, taskItem, jobID, retry, timeout) return handleSnapshot(req, taskItem, jobID, retry, snap.Timeout)
} }
func (u *SnapshotService) SnapshotReCreate(id uint) error { func (u *SnapshotService) SnapshotReCreate(id uint) error {