1Panel-dev/1Panel
1
1
Fork 0
mirror of https://github.com/1Panel-dev/1Panel.git synced 2025-09-06 22:55:38 +08:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: Optimize backup logic for cronjob (#10163)

Browse source
This commit is contained in:
ssongliu 2025-08-27 14:35:28 +08:00 • committed by GitHub
parent abf3247c50
commit 5a34e2d380
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 52 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
agent/app/api/v2/snapshot.go
View file

@ -36,7 +36,7 @@ func (b *BaseApi) CreateSnapshot(c *gin.Context) {
return
}
if err := snapshotService.SnapshotCreate(nil, req, 0, 0, 0); err != nil {
if err := snapshotService.SnapshotCreate(nil, req, 0, 3); err != nil {
helper.InternalServer(c, err)
return
}

10
agent/app/service/backup.go
View file

@ -396,9 +396,9 @@ type backupClientHelper struct {
backupPath string
client cloud_storage.CloudStorageClient
isOk bool
hasBackuped bool
message string
isOk bool
hasBackup bool
message string
}
func NewBackupClientMap(ids []string) map[string]backupClientHelper {
@ -438,7 +438,7 @@ func uploadWithMap(taskItem task.Task, accountMap map[string]backupClientHelper,
if !ok {
continue
}
if itemBackup.hasBackuped {
if itemBackup.hasBackup {
continue
}
if !itemBackup.isOk {
@ -460,7 +460,7 @@ func uploadWithMap(taskItem task.Task, accountMap map[string]backupClientHelper,
break
}
}
itemBackup.hasBackuped = true
itemBackup.hasBackup = true
accountMap[account] = itemBackup
}
os.RemoveAll(src)

3
agent/app/service/backup_record.go
View file

@ -259,6 +259,9 @@ func (u *BackupRecordService) LoadRecordSize(req dto.SearchForSize) ([]dto.Recor
for i := 0; i < len(list); i++ {
datas = append(datas, dto.RecordFileSize{ID: list[i].ID})
if val, ok := clientMap[fmt.Sprintf("%v", list[i].DownloadID)]; ok {
if !val.isOk {
continue
}
wg.Add(1)
go func(index int) {
defer wg.Done()

26
agent/app/service/cronjob_backup.go
View file

@ -51,6 +51,7 @@ func (u *CronjobService) handleApp(cronjob model.Cronjob, startTime time.Time, t
return err
} else {
task.Log(i18n.GetMsgWithDetail("IgnoreBackupErr", err.Error()))
cleanAccountMap(accountMap)
return nil
}
}
@ -63,6 +64,7 @@ func (u *CronjobService) handleApp(cronjob model.Cronjob, startTime time.Time, t
return err
}
task.Log(i18n.GetMsgWithDetail("IgnoreUploadErr", err.Error()))
cleanAccountMap(accountMap)
return nil
}
record.FileDir = path.Dir(dst)
@ -71,6 +73,7 @@ func (u *CronjobService) handleApp(cronjob model.Cronjob, startTime time.Time, t
return err
}
u.removeExpiredBackup(cronjob, accountMap, record)
cleanAccountMap(accountMap)
return nil
}, nil, int(cronjob.RetryTimes), time.Duration(cronjob.Timeout)*time.Second)
}
@ -105,6 +108,7 @@ func (u *CronjobService) handleWebsite(cronjob model.Cronjob, startTime time.Tim
return err
} else {
task.Log(i18n.GetMsgWithDetail("IgnoreBackupErr", err.Error()))
cleanAccountMap(accountMap)
return nil
}
}
@ -117,6 +121,7 @@ func (u *CronjobService) handleWebsite(cronjob model.Cronjob, startTime time.Tim
return err
}
task.Log(i18n.GetMsgWithDetail("IgnoreUploadErr", err.Error()))
cleanAccountMap(accountMap)
return nil
}
record.FileDir = path.Dir(dst)
@ -125,6 +130,7 @@ func (u *CronjobService) handleWebsite(cronjob model.Cronjob, startTime time.Tim
return err
}
u.removeExpiredBackup(cronjob, accountMap, record)
cleanAccountMap(accountMap)
return nil
}, nil, int(cronjob.RetryTimes), time.Duration(cronjob.Timeout)*time.Second)
}
@ -161,6 +167,7 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Ti
return err
} else {
task.Log(i18n.GetMsgWithDetail("IgnoreBackupErr", err.Error()))
cleanAccountMap(accountMap)
return nil
}
}
@ -171,6 +178,7 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Ti
return err
} else {
task.Log(i18n.GetMsgWithDetail("IgnoreBackupErr", err.Error()))
cleanAccountMap(accountMap)
return nil
}
}
@ -184,6 +192,7 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Ti
return err
}
task.Log(i18n.GetMsgWithDetail("IgnoreUploadErr", err.Error()))
cleanAccountMap(accountMap)
return nil
}
record.FileDir = path.Dir(dst)
@ -192,6 +201,7 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Ti
return err
}
u.removeExpiredBackup(cronjob, accountMap, record)
cleanAccountMap(accountMap)
return nil
}, nil, int(cronjob.RetryTimes), time.Duration(cronjob.Timeout)*time.Second)
}
@ -306,9 +316,10 @@ func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, jobRecord model.J
return err
}
req := dto.SnapshotCreate{
Name: fmt.Sprintf("snapshot-1panel-%s-%s-linux-%s-%s", scope, versionItem.Value, loadOs(), jobRecord.StartTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5)),
Secret: cronjob.Secret,
TaskID: jobRecord.TaskID,
Name: fmt.Sprintf("snapshot-1panel-%s-%s-linux-%s-%s", scope, versionItem.Value, loadOs(), jobRecord.StartTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5)),
Secret: cronjob.Secret,
TaskID: jobRecord.TaskID,
Timeout: cronjob.Timeout,
SourceAccountIDs: record.SourceAccountIDs,
DownloadAccountID: cronjob.DownloadAccountID,
@ -324,7 +335,7 @@ func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, jobRecord model.J
IgnoreFiles: strings.Split(cronjob.ExclusionRules, ","),
}
if err := NewISnapshotService().SnapshotCreate(taskItem, req, jobRecord.ID, cronjob.RetryTimes, cronjob.Timeout); err != nil {
if err := NewISnapshotService().SnapshotCreate(taskItem, req, jobRecord.ID, cronjob.RetryTimes); err != nil {
return err
}
record.FileName = req.Name + ".tar.gz"
@ -579,3 +590,10 @@ func simplifiedFileName(name string) string {
name = strings.ReplaceAll(name, "|", "_")
return name
}
func cleanAccountMap(accountMap map[string]backupClientHelper) {
for key, val := range accountMap {
val.hasBackup = false
accountMap[key] = val
}
}

34
agent/app/service/cronjob_helper.go
View file

@ -31,12 +31,12 @@ func (u *CronjobService) HandleJob(cronjob *model.Cronjob) {
global.LOG.Errorf("new task for exec shell failed, err: %v", err)
return
}
go func() {
err = u.loadTask(cronjob, &record, taskItem)
if cronjob.Type == "snapshot" {
if err != nil {
taskItem, _ := taskRepo.GetFirst(taskRepo.WithByID(record.TaskID))
if len(taskItem.ID) == 0 {
if cronjob.Type == "snapshot" {
go func() {
_ = cronjobRepo.UpdateRecords(record.ID, map[string]interface{}{"records": record.Records})
if err = u.handleSnapshot(*cronjob, record, taskItem); err != nil {
item, _ := taskRepo.GetFirst(taskRepo.WithByID(record.TaskID))
if len(item.ID) == 0 {
record.TaskID = ""
}
cronjobRepo.EndRecords(record, constant.StatusFailed, err.Error(), record.Records)
@ -44,15 +44,20 @@ func (u *CronjobService) HandleJob(cronjob *model.Cronjob) {
return
}
cronjobRepo.EndRecords(record, constant.StatusSuccess, "", record.Records)
return
}
if err != nil {
global.LOG.Debugf("preper to handle cron job [%s] %s failed, err: %v", cronjob.Type, cronjob.Name, err)
}()
return
}
if err = u.loadTask(cronjob, &record, taskItem); err != nil {
global.LOG.Debugf("preper to handle cron job [%s] %s failed, err: %v", cronjob.Type, cronjob.Name, err)
item, _ := taskRepo.GetFirst(taskRepo.WithByID(record.TaskID))
if len(item.ID) == 0 {
record.TaskID = ""
cronjobRepo.EndRecords(record, constant.StatusFailed, err.Error(), record.Records)
handleCronJobAlert(cronjob)
return
}
cronjobRepo.EndRecords(record, constant.StatusFailed, err.Error(), record.Records)
handleCronJobAlert(cronjob)
return
}
go func() {
if err := taskItem.Execute(); err != nil {
taskItem, _ := taskRepo.GetFirst(taskRepo.WithByID(record.TaskID))
if len(taskItem.ID) == 0 {
@ -110,9 +115,6 @@ func (u *CronjobService) loadTask(cronjob *model.Cronjob, record *model.JobRecor
err = u.handleDirectory(*cronjob, record.StartTime, taskItem)
case "log":
err = u.handleSystemLog(*cronjob, record.StartTime, taskItem)
case "snapshot":
_ = cronjobRepo.UpdateRecords(record.ID, map[string]interface{}{"records": record.Records})
err = u.handleSnapshot(*cronjob, *record, taskItem)
}
return err
}

2
agent/app/service/snapshot.go
View file

@ -31,7 +31,7 @@ type SnapshotService struct {
type ISnapshotService interface {
SearchWithPage(req dto.PageSnapshot) (int64, interface{}, error)
LoadSnapshotData() (dto.SnapshotData, error)
SnapshotCreate(parentTask *task.Task, req dto.SnapshotCreate, jobID, retry, timeout uint) error
SnapshotCreate(parentTask *task.Task, req dto.SnapshotCreate, jobID, retry uint) error
SnapshotReCreate(id uint) error
SnapshotRecover(req dto.SnapshotRecover) error
SnapshotRollback(req dto.SnapshotRecover) error

4
agent/app/service/snapshot_create.go
View file

@ -27,7 +27,7 @@ import (
"gorm.io/gorm"
)
func (u *SnapshotService) SnapshotCreate(parentTask *task.Task, req dto.SnapshotCreate, jobID, retry, timeout uint) error {
func (u *SnapshotService) SnapshotCreate(parentTask *task.Task, req dto.SnapshotCreate, jobID, retry uint) error {
versionItem, _ := settingRepo.Get(settingRepo.WithByKey("SystemVersion"))
scope := "core"
@ -85,7 +85,7 @@ func (u *SnapshotService) SnapshotCreate(parentTask *task.Task, req dto.Snapshot
return nil
}
return handleSnapshot(req, taskItem, jobID, retry, timeout)
return handleSnapshot(req, taskItem, jobID, retry, snap.Timeout)
}
func (u *SnapshotService) SnapshotReCreate(id uint) error {