1Panel-dev/1Panel
1
1
Fork 0
mirror of https://github.com/1Panel-dev/1Panel.git synced 2025-12-09 11:55:52 +08:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: Modify the execution mode of the planned tasks (#8580)

Browse source
This commit is contained in:
ssongliu 2025-05-09 15:55:48 +08:00 • committed by GitHub
parent 6dd99dbb25
commit e1635aa39d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 484 additions and 492 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
agent/app/api/v2/snapshot.go
View file

@ -36,7 +36,7 @@ func (b *BaseApi) CreateSnapshot(c *gin.Context) {
return
}
if err := snapshotService.SnapshotCreate(req, 0); err != nil {
if err := snapshotService.SnapshotCreate(nil, req, 0, 0, 0); err != nil {
helper.InternalServer(c, err)
return
}

2
agent/app/repo/cronjob.go
View file

@ -147,9 +147,7 @@ func (u *CronjobRepo) StartRecords(cronjobID uint, targetPath, cronjobType strin
var record model.JobRecords
record.StartTime = time.Now()
record.CronjobID = cronjobID
if cronjobType != "directory" && cronjobType != "log" && cronjobType != "cutWebsiteLog" {
record.TaskID = uuid.New().String()
}
record.Status = constant.StatusWaiting
if err := global.DB.Create(&record).Error; err != nil {
global.LOG.Errorf("create record status failed, err: %v", err)

78
agent/app/service/backup_app.go
View file

@ -143,45 +143,10 @@ func handleAppBackup(install *model.AppInstall, parentTask *task.Task, backupDir
}
}
backupApp := func(t *task.Task) error {
fileOp := files.NewFileOp()
tmpDir := fmt.Sprintf("%s/%s", backupDir, strings.ReplaceAll(fileName, ".tar.gz", ""))
if !fileOp.Stat(tmpDir) {
if err := os.MkdirAll(tmpDir, os.ModePerm); err != nil {
return fmt.Errorf("mkdir %s failed, err: %v", backupDir, err)
}
}
defer func() {
_ = os.RemoveAll(tmpDir)
}()
remarkInfo, _ := json.Marshal(install)
remarkInfoPath := fmt.Sprintf("%s/app.json", tmpDir)
if err := fileOp.SaveFile(remarkInfoPath, string(remarkInfo), fs.ModePerm); err != nil {
return err
}
appPath := install.GetPath()
if err := fileOp.TarGzCompressPro(true, appPath, path.Join(tmpDir, "app.tar.gz"), "", excludes); err != nil {
return err
}
resources, _ := appInstallResourceRepo.GetBy(appInstallResourceRepo.WithAppInstallId(install.ID))
for _, resource := range resources {
if err = backupDatabaseWithTask(t, resource.Key, tmpDir, install.Name, resource.ResourceId); err != nil {
return err
}
}
t.LogStart(i18n.GetMsgByKey("CompressDir"))
if err := fileOp.TarGzCompressPro(true, tmpDir, path.Join(backupDir, fileName), secret, ""); err != nil {
return err
}
t.Log(i18n.GetWithName("CompressFileSuccess", fileName))
return nil
}
backupTask.AddSubTask(task.GetTaskName(install.Name, task.TaskBackup, task.TaskScopeApp), backupApp, nil)
itemHandler := doAppBackup(install, backupTask, backupDir, fileName, excludes, secret)
backupTask.AddSubTask(task.GetTaskName(install.Name, task.TaskBackup, task.TaskScopeApp), func(t *task.Task) error { return itemHandler }, nil)
if parentTask != nil {
return backupApp(parentTask)
return itemHandler
}
return backupTask.Execute()
}
@ -367,6 +332,43 @@ func handleAppRecover(install *model.AppInstall, parentTask *task.Task, recoverF
return recoverTask.Execute()
}
func doAppBackup(install *model.AppInstall, parentTask *task.Task, backupDir, fileName, excludes, secret string) error {
fileOp := files.NewFileOp()
tmpDir := fmt.Sprintf("%s/%s", backupDir, strings.ReplaceAll(fileName, ".tar.gz", ""))
if !fileOp.Stat(tmpDir) {
if err := os.MkdirAll(tmpDir, os.ModePerm); err != nil {
return fmt.Errorf("mkdir %s failed, err: %v", backupDir, err)
}
}
defer func() {
_ = os.RemoveAll(tmpDir)
}()
remarkInfo, _ := json.Marshal(install)
remarkInfoPath := fmt.Sprintf("%s/app.json", tmpDir)
if err := fileOp.SaveFile(remarkInfoPath, string(remarkInfo), fs.ModePerm); err != nil {
return err
}
appPath := install.GetPath()
if err := fileOp.TarGzCompressPro(true, appPath, path.Join(tmpDir, "app.tar.gz"), "", excludes); err != nil {
return err
}
resources, _ := appInstallResourceRepo.GetBy(appInstallResourceRepo.WithAppInstallId(install.ID))
for _, resource := range resources {
if err := backupDatabaseWithTask(parentTask, resource.Key, tmpDir, install.Name, resource.ResourceId); err != nil {
return err
}
}
parentTask.LogStart(i18n.GetMsgByKey("CompressDir"))
if err := fileOp.TarGzCompressPro(true, tmpDir, path.Join(backupDir, fileName), secret, ""); err != nil {
return err
}
parentTask.Log(i18n.GetWithName("CompressFileSuccess", fileName))
return nil
}
func reCreateDB(dbID uint, database model.Database, oldEnv string) (*model.DatabaseMysql, map[string]interface{}, error) {
mysqlService := NewIMysqlService()
ctx := context.Background()

54
agent/app/service/backup_mysql.go
View file

@ -107,45 +107,27 @@ func (u *BackupService) MysqlRecoverByUpload(req dto.CommonRecover) error {
func handleMysqlBackup(db DatabaseHelper, parentTask *task.Task, targetDir, fileName, taskID string) error {
var (
err error
itemTask *task.Task
backupTask *task.Task
)
itemTask = parentTask
backupTask = parentTask
dbInfo, err := mysqlRepo.Get(repo.WithByName(db.Name), mysqlRepo.WithByMysqlName(db.Database))
if err != nil {
return err
}
itemName := fmt.Sprintf("%s[%s] - %s", db.Database, db.DBType, db.Name)
if parentTask == nil {
itemTask, err = task.NewTaskWithOps(itemName, task.TaskBackup, task.TaskScopeDatabase, taskID, dbInfo.ID)
backupTask, err = task.NewTaskWithOps(itemName, task.TaskBackup, task.TaskScopeDatabase, taskID, dbInfo.ID)
if err != nil {
return err
}
}
backupDatabase := func(t *task.Task) error {
cli, version, err := LoadMysqlClientByFrom(db.Database)
if err != nil {
return err
}
backupInfo := client.BackupInfo{
Name: db.Name,
Type: db.DBType,
Version: version,
Format: dbInfo.Format,
TargetDir: targetDir,
FileName: fileName,
Timeout: 300,
}
return cli.Backup(backupInfo)
}
itemTask.AddSubTask(i18n.GetMsgByKey("TaskBackup"), backupDatabase, nil)
itemHandler := doMysqlBackup(db, targetDir, fileName)
backupTask.AddSubTask(task.GetTaskName(itemName, task.TaskBackup, task.TaskScopeApp), func(t *task.Task) error { return itemHandler }, nil)
if parentTask != nil {
return backupDatabase(parentTask)
return itemHandler
}
return itemTask.Execute()
return backupTask.Execute()
}
func handleMysqlRecover(req dto.CommonRecover, parentTask *task.Task, isRollback bool, taskID string) error {
@ -239,3 +221,25 @@ func handleMysqlRecover(req dto.CommonRecover, parentTask *task.Task, isRollback
return itemTask.Execute()
}
func doMysqlBackup(db DatabaseHelper, targetDir, fileName string) error {
dbInfo, err := mysqlRepo.Get(repo.WithByName(db.Name), mysqlRepo.WithByMysqlName(db.Database))
if err != nil {
return err
}
cli, version, err := LoadMysqlClientByFrom(db.Database)
if err != nil {
return err
}
backupInfo := client.BackupInfo{
Name: db.Name,
Type: db.DBType,
Version: version,
Format: dbInfo.Format,
TargetDir: targetDir,
FileName: fileName,
Timeout: 300,
}
return cli.Backup(backupInfo)
}

44
agent/app/service/backup_postgresql.go
View file

@ -107,37 +107,23 @@ func (u *BackupService) PostgresqlRecoverByUpload(req dto.CommonRecover) error {
func handlePostgresqlBackup(db DatabaseHelper, parentTask *task.Task, targetDir, fileName, taskID string) error {
var (
err error
itemTask *task.Task
backupTask *task.Task
)
itemTask = parentTask
backupTask = parentTask
itemName := fmt.Sprintf("%s - %s", db.Database, db.Name)
if parentTask == nil {
itemTask, err = task.NewTaskWithOps(itemName, task.TaskBackup, task.TaskScopeDatabase, taskID, db.ID)
backupTask, err = task.NewTaskWithOps(itemName, task.TaskBackup, task.TaskScopeDatabase, taskID, db.ID)
if err != nil {
return err
}
}
backupDatabase := func(t *task.Task) error {
cli, err := LoadPostgresqlClientByFrom(db.Database)
if err != nil {
return err
}
defer cli.Close()
backupInfo := pgclient.BackupInfo{
Name: db.Name,
TargetDir: targetDir,
FileName: fileName,
Timeout: 300,
}
return cli.Backup(backupInfo)
}
itemTask.AddSubTask(i18n.GetMsgByKey("TaskBackup"), backupDatabase, nil)
itemHandler := doPostgresqlgBackup(db, targetDir, fileName)
backupTask.AddSubTask(task.GetTaskName(itemName, task.TaskBackup, task.TaskScopeApp), func(task *task.Task) error { return itemHandler }, nil)
if parentTask != nil {
return backupDatabase(parentTask)
return itemHandler
}
return itemTask.Execute()
return backupTask.Execute()
}
func handlePostgresqlRecover(req dto.CommonRecover, parentTask *task.Task, isRollback bool) error {
@ -219,3 +205,19 @@ func handlePostgresqlRecover(req dto.CommonRecover, parentTask *task.Task, isRol
return itemTask.Execute()
}
func doPostgresqlgBackup(db DatabaseHelper, targetDir, fileName string) error {
cli, err := LoadPostgresqlClientByFrom(db.Database)
if err != nil {
return err
}
defer cli.Close()
backupInfo := pgclient.BackupInfo{
Name: db.Name,
TargetDir: targetDir,
FileName: fileName,
Timeout: 300,
}
return cli.Backup(backupInfo)
}

43
agent/app/service/backup_website.go
View file

@ -215,8 +215,15 @@ func handleWebsiteBackup(website *model.Website, parentTask *task.Task, backupDi
return err
}
}
itemHandler := doWebsiteBackup(website, backupTask, backupDir, fileName, excludes, secret)
backupTask.AddSubTask(task.GetTaskName(website.Alias, task.TaskBackup, task.TaskScopeApp), func(task *task.Task) error { return itemHandler }, nil)
if parentTask != nil {
return itemHandler
}
return backupTask.Execute()
}
backupWebsite := func(t *task.Task) error {
func doWebsiteBackup(website *model.Website, parentTask *task.Task, backupDir, fileName, excludes, secret string) error {
fileOp := files.NewFileOp()
tmpDir := fmt.Sprintf("%s/%s", backupDir, strings.ReplaceAll(fileName, ".tar.gz", ""))
if !fileOp.Stat(tmpDir) {
@ -229,14 +236,14 @@ func handleWebsiteBackup(website *model.Website, parentTask *task.Task, backupDi
}()
remarkInfo, _ := json.Marshal(website)
if err = fileOp.SaveFile(tmpDir+"/website.json", string(remarkInfo), fs.ModePerm); err != nil {
if err := fileOp.SaveFile(tmpDir+"/website.json", string(remarkInfo), fs.ModePerm); err != nil {
return err
}
nginxConfFile := GetSitePath(*website, SiteConf)
if err = fileOp.CopyFile(nginxConfFile, tmpDir); err != nil {
if err := fileOp.CopyFile(nginxConfFile, tmpDir); err != nil {
return err
}
t.Log(i18n.GetMsgByKey("BackupNginxConfig"))
parentTask.Log(i18n.GetMsgByKey("BackupNginxConfig"))
switch website.Type {
case constant.Deployment:
@ -244,51 +251,45 @@ func handleWebsiteBackup(website *model.Website, parentTask *task.Task, backupDi
if err != nil {
return err
}
t.LogStart(task.GetTaskName(app.Name, task.TaskBackup, task.TaskScopeApp))
if err = handleAppBackup(&app, backupTask, tmpDir, fmt.Sprintf("%s.app.tar.gz", website.Alias), excludes, "", ""); err != nil {
parentTask.LogStart(task.GetTaskName(app.Name, task.TaskBackup, task.TaskScopeApp))
if err = handleAppBackup(&app, parentTask, tmpDir, fmt.Sprintf("%s.app.tar.gz", website.Alias), excludes, "", ""); err != nil {
return err
}
t.LogSuccess(task.GetTaskName(app.Name, task.TaskBackup, task.TaskScopeApp))
parentTask.LogSuccess(task.GetTaskName(app.Name, task.TaskBackup, task.TaskScopeApp))
case constant.Runtime:
runtime, err := runtimeRepo.GetFirst(context.Background(), repo.WithByID(website.RuntimeID))
if err != nil {
return err
}
t.LogStart(task.GetTaskName(runtime.Name, task.TaskBackup, task.TaskScopeRuntime))
parentTask.LogStart(task.GetTaskName(runtime.Name, task.TaskBackup, task.TaskScopeRuntime))
if err = handleRuntimeBackup(runtime, tmpDir, fmt.Sprintf("%s.runtime.tar.gz", website.Alias), excludes, ""); err != nil {
return err
}
t.LogSuccess(task.GetTaskName(runtime.Name, task.TaskBackup, task.TaskScopeRuntime))
parentTask.LogSuccess(task.GetTaskName(runtime.Name, task.TaskBackup, task.TaskScopeRuntime))
if website.DbID > 0 {
if err = backupDatabaseWithTask(t, website.DbType, tmpDir, website.Alias, website.DbID); err != nil {
if err = backupDatabaseWithTask(parentTask, website.DbType, tmpDir, website.Alias, website.DbID); err != nil {
return err
}
}
case constant.Static:
if website.DbID > 0 {
if err = backupDatabaseWithTask(t, website.DbType, tmpDir, website.Alias, website.DbID); err != nil {
if err := backupDatabaseWithTask(parentTask, website.DbType, tmpDir, website.Alias, website.DbID); err != nil {
return err
}
}
}
websiteDir := GetSitePath(*website, SiteDir)
t.LogStart(i18n.GetMsgByKey("CompressDir"))
if err = fileOp.TarGzCompressPro(true, websiteDir, path.Join(tmpDir, fmt.Sprintf("%s.web.tar.gz", website.Alias)), "", excludes); err != nil {
parentTask.LogStart(i18n.GetMsgByKey("CompressDir"))
if err := fileOp.TarGzCompressPro(true, websiteDir, path.Join(tmpDir, fmt.Sprintf("%s.web.tar.gz", website.Alias)), "", excludes); err != nil {
return err
}
if err = fileOp.TarGzCompressPro(true, tmpDir, path.Join(backupDir, fileName), secret, ""); err != nil {
if err := fileOp.TarGzCompressPro(true, tmpDir, path.Join(backupDir, fileName), secret, ""); err != nil {
return err
}
t.Log(i18n.GetWithName("CompressFileSuccess", fileName))
parentTask.Log(i18n.GetWithName("CompressFileSuccess", fileName))
return nil
}
backupTask.AddSubTask(task.GetTaskName(website.Alias, task.TaskBackup, task.TaskScopeApp), backupWebsite, nil)
if parentTask != nil {
return backupWebsite(parentTask)
}
return backupTask.Execute()
}
func checkValidOfWebsite(oldWebsite, website *model.Website) error {
if oldWebsite.Alias != website.Alias || oldWebsite.Type != website.Type {

22
agent/app/service/cronjob.go
View file

@ -1,7 +1,6 @@
package service
import (
"bufio"
"encoding/json"
"fmt"
"os"
@ -461,24 +460,3 @@ func (u *CronjobService) AddCronJob(cronjob *model.Cronjob) (int, error) {
global.LOG.Infof("start cronjob entryID: %d", entryID)
return int(entryID), nil
}
func mkdirAndWriteFile(cronjob *model.Cronjob, startTime time.Time, msg []byte) (string, error) {
dir := fmt.Sprintf("%s/task/%s/%s", global.Dir.DataDir, cronjob.Type, cronjob.Name)
if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) {
if err = os.MkdirAll(dir, os.ModePerm); err != nil {
return "", err
}
}
path := fmt.Sprintf("%s/%s.log", dir, startTime.Format(constant.DateTimeSlimLayout))
global.LOG.Infof("cronjob %s has generated some logs %s", cronjob.Name, path)
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, constant.FilePerm)
if err != nil {
return "", err
}
defer file.Close()
write := bufio.NewWriter(file)
_, _ = write.WriteString(string(msg))
write.Flush()
return path, nil
}

64
agent/app/service/cronjob_backup.go
View file

@ -10,6 +10,8 @@ import (
"time"
"github.com/1Panel-dev/1Panel/agent/app/repo"
"github.com/1Panel-dev/1Panel/agent/app/task"
"github.com/1Panel-dev/1Panel/agent/i18n"
"github.com/1Panel-dev/1Panel/agent/app/dto"
"github.com/1Panel-dev/1Panel/agent/app/model"
@ -20,7 +22,7 @@ import (
"github.com/pkg/errors"
)
func (u *CronjobService) handleApp(cronjob model.Cronjob, startTime time.Time, taskID string) error {
func (u *CronjobService) handleApp(cronjob model.Cronjob, startTime time.Time, taskItem *task.Task) error {
var apps []model.AppInstall
if cronjob.AppID == "all" {
apps, _ = appInstallRepo.ListBy(context.Background())
@ -40,6 +42,7 @@ func (u *CronjobService) handleApp(cronjob model.Cronjob, startTime time.Time, t
return err
}
for _, app := range apps {
taskItem.AddSubTaskWithOps(task.GetTaskName(app.Name, task.TaskBackup, task.TaskScopeCronjob), func(task *task.Task) error {
var record model.BackupRecord
record.From = "cronjob"
record.Type = "app"
@ -49,7 +52,7 @@ func (u *CronjobService) handleApp(cronjob model.Cronjob, startTime time.Time, t
record.DownloadAccountID, record.SourceAccountIDs = cronjob.DownloadAccountID, cronjob.SourceAccountIDs
backupDir := path.Join(global.Dir.TmpDir, fmt.Sprintf("app/%s/%s", app.App.Key, app.Name))
record.FileName = fmt.Sprintf("app_%s_%s.tar.gz", app.Name, startTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5))
if err := handleAppBackup(&app, nil, backupDir, record.FileName, cronjob.ExclusionRules, cronjob.Secret, taskID); err != nil {
if err := doAppBackup(&app, task, backupDir, record.FileName, cronjob.ExclusionRules, cronjob.Secret); err != nil {
return err
}
downloadPath, err := u.uploadCronjobBackFile(cronjob, accountMap, path.Join(backupDir, record.FileName))
@ -62,11 +65,13 @@ func (u *CronjobService) handleApp(cronjob model.Cronjob, startTime time.Time, t
return err
}
u.removeExpiredBackup(cronjob, accountMap, record)
return nil
}, nil, int(cronjob.RetryTimes), time.Duration(cronjob.Timeout)*time.Second)
}
return nil
}
func (u *CronjobService) handleWebsite(cronjob model.Cronjob, startTime time.Time, taskID string) error {
func (u *CronjobService) handleWebsite(cronjob model.Cronjob, startTime time.Time, taskItem *task.Task) error {
webs := loadWebsForJob(cronjob)
if len(webs) == 0 {
return errors.New("no such website in database!")
@ -76,6 +81,7 @@ func (u *CronjobService) handleWebsite(cronjob model.Cronjob, startTime time.Tim
return err
}
for _, web := range webs {
taskItem.AddSubTaskWithOps(task.GetTaskName(web.Alias, task.TaskBackup, task.TaskScopeCronjob), func(task *task.Task) error {
var record model.BackupRecord
record.From = "cronjob"
record.Type = "website"
@ -85,9 +91,11 @@ func (u *CronjobService) handleWebsite(cronjob model.Cronjob, startTime time.Tim
record.DownloadAccountID, record.SourceAccountIDs = cronjob.DownloadAccountID, cronjob.SourceAccountIDs
backupDir := path.Join(global.Dir.TmpDir, fmt.Sprintf("website/%s", web.Alias))
record.FileName = fmt.Sprintf("website_%s_%s.tar.gz", web.Alias, startTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5))
if err := handleWebsiteBackup(&web, nil, backupDir, record.FileName, cronjob.ExclusionRules, cronjob.Secret, taskID); err != nil {
if err := doWebsiteBackup(&web, taskItem, backupDir, record.FileName, cronjob.ExclusionRules, cronjob.Secret); err != nil {
return err
}
downloadPath, err := u.uploadCronjobBackFile(cronjob, accountMap, path.Join(backupDir, record.FileName))
if err != nil {
return err
@ -98,11 +106,14 @@ func (u *CronjobService) handleWebsite(cronjob model.Cronjob, startTime time.Tim
return err
}
u.removeExpiredBackup(cronjob, accountMap, record)
return nil
}, nil, int(cronjob.RetryTimes), time.Duration(cronjob.Timeout)*time.Second)
return nil
}
return nil
}
func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Time, taskID string) error {
func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Time, taskItem *task.Task) error {
dbs := loadDbsForJob(cronjob)
if len(dbs) == 0 {
return errors.New("no such db in database!")
@ -112,6 +123,8 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Ti
return err
}
for _, dbInfo := range dbs {
itemName := fmt.Sprintf("%s[%s] - %s", dbInfo.Database, dbInfo.DBType, dbInfo.Name)
taskItem.AddSubTaskWithOps(task.GetTaskName(itemName, task.TaskBackup, task.TaskScopeCronjob), func(task *task.Task) error {
var record model.BackupRecord
record.From = "cronjob"
record.Type = dbInfo.DBType
@ -123,14 +136,15 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Ti
backupDir := path.Join(global.Dir.TmpDir, fmt.Sprintf("database/%s/%s/%s", dbInfo.DBType, record.Name, dbInfo.Name))
record.FileName = fmt.Sprintf("db_%s_%s.sql.gz", dbInfo.Name, startTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5))
if cronjob.DBType == "mysql" || cronjob.DBType == "mariadb" {
if err := handleMysqlBackup(dbInfo, nil, backupDir, record.FileName, taskID); err != nil {
if err := doMysqlBackup(dbInfo, backupDir, record.FileName); err != nil {
return err
}
} else {
if err := handlePostgresqlBackup(dbInfo, nil, backupDir, record.FileName, taskID); err != nil {
if err := doPostgresqlgBackup(dbInfo, backupDir, record.FileName); err != nil {
return err
}
}
downloadPath, err := u.uploadCronjobBackFile(cronjob, accountMap, path.Join(backupDir, record.FileName))
if err != nil {
return err
@ -141,11 +155,14 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Ti
return err
}
u.removeExpiredBackup(cronjob, accountMap, record)
return nil
}, nil, int(cronjob.RetryTimes), time.Duration(cronjob.Timeout)*time.Second)
}
return nil
}
func (u *CronjobService) handleDirectory(cronjob model.Cronjob, startTime time.Time) error {
func (u *CronjobService) handleDirectory(cronjob model.Cronjob, startTime time.Time, taskItem *task.Task) error {
taskItem.AddSubTaskWithOps(task.GetTaskName(i18n.GetMsgByKey("BackupFileOrDir"), task.TaskBackup, task.TaskScopeCronjob), func(task *task.Task) error {
accountMap, err := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ","))
if err != nil {
return err
@ -155,10 +172,12 @@ func (u *CronjobService) handleDirectory(cronjob model.Cronjob, startTime time.T
fileOp := files.NewFileOp()
if cronjob.IsDir {
taskItem.Logf("Dir: %s, Excludes: %s", cronjob.SourceDir, cronjob.ExclusionRules)
if err := fileOp.TarGzCompressPro(true, cronjob.SourceDir, path.Join(backupDir, fileName), cronjob.Secret, cronjob.ExclusionRules); err != nil {
return err
}
} else {
taskItem.Logf("Files: %s", cronjob.SourceDir)
fileLists := strings.Split(cronjob.SourceDir, ",")
if err := fileOp.TarGzFilesWithCompressPro(fileLists, path.Join(backupDir, fileName), cronjob.Secret); err != nil {
return err
@ -172,19 +191,23 @@ func (u *CronjobService) handleDirectory(cronjob model.Cronjob, startTime time.T
record.DownloadAccountID, record.SourceAccountIDs = cronjob.DownloadAccountID, cronjob.SourceAccountIDs
downloadPath, err := u.uploadCronjobBackFile(cronjob, accountMap, path.Join(backupDir, fileName))
if err != nil {
taskItem.LogFailedWithErr("Upload backup file", err)
return err
}
record.FileDir = path.Dir(downloadPath)
record.FileName = fileName
if err := backupRepo.CreateRecord(&record); err != nil {
global.LOG.Errorf("save backup record failed, err: %v", err)
taskItem.LogFailedWithErr("Save record", err)
return err
}
u.removeExpiredBackup(cronjob, accountMap, record)
return nil
}, nil, int(cronjob.RetryTimes), time.Duration(cronjob.Timeout)*time.Second)
return nil
}
func (u *CronjobService) handleSystemLog(cronjob model.Cronjob, startTime time.Time) error {
func (u *CronjobService) handleSystemLog(cronjob model.Cronjob, startTime time.Time, taskItem *task.Task) error {
taskItem.AddSubTaskWithOps(task.GetTaskName(i18n.GetMsgByKey("BackupSystemLog"), task.TaskBackup, task.TaskScopeCronjob), func(task *task.Task) error {
accountMap, err := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ","))
if err != nil {
return err
@ -192,7 +215,7 @@ func (u *CronjobService) handleSystemLog(cronjob model.Cronjob, startTime time.T
nameItem := startTime.Format(constant.DateTimeSlimLayout) + common.RandStrAndNum(5)
fileName := fmt.Sprintf("system_log_%s.tar.gz", nameItem)
backupDir := path.Join(global.Dir.TmpDir, "log", nameItem)
if err := handleBackupLogs(backupDir, fileName, cronjob.Secret); err != nil {
if err := handleBackupLogs(taskItem, backupDir, fileName, cronjob.Secret); err != nil {
return err
}
var record model.BackupRecord
@ -203,19 +226,22 @@ func (u *CronjobService) handleSystemLog(cronjob model.Cronjob, startTime time.T
record.DownloadAccountID, record.SourceAccountIDs = cronjob.DownloadAccountID, cronjob.SourceAccountIDs
downloadPath, err := u.uploadCronjobBackFile(cronjob, accountMap, path.Join(path.Dir(backupDir), fileName))
if err != nil {
taskItem.LogFailedWithErr("Upload backup file", err)
return err
}
record.FileDir = path.Dir(downloadPath)
record.FileName = fileName
if err := backupRepo.CreateRecord(&record); err != nil {
global.LOG.Errorf("save backup record failed, err: %v", err)
taskItem.LogFailedWithErr("Save record", err)
return err
}
u.removeExpiredBackup(cronjob, accountMap, record)
return nil
}, nil, int(cronjob.RetryTimes), time.Duration(cronjob.Timeout)*time.Second)
return nil
}
func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, jobRecord model.JobRecords) error {
func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, jobRecord model.JobRecords, taskItem *task.Task) error {
accountMap, err := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ","))
if err != nil {
return err
@ -255,7 +281,7 @@ func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, jobRecord model.J
WithTaskLog: true,
}
if err := NewISnapshotService().SnapshotCreate(req, jobRecord.ID); err != nil {
if err := NewISnapshotService().SnapshotCreate(taskItem, req, jobRecord.ID, cronjob.RetryTimes, cronjob.Timeout); err != nil {
return err
}
record.FileName = req.Name + ".tar.gz"
@ -336,7 +362,7 @@ func loadWebsForJob(cronjob model.Cronjob) []model.Website {
return weblist
}
func handleBackupLogs(targetDir, fileName string, secret string) error {
func handleBackupLogs(taskItem *task.Task, targetDir, fileName string, secret string) error {
fileOp := files.NewFileOp()
websites, err := websiteRepo.List()
if err != nil {
@ -345,6 +371,7 @@ func handleBackupLogs(targetDir, fileName string, secret string) error {
if len(websites) != 0 {
webItem := GetOpenrestyDir(SitesRootDir)
for _, website := range websites {
taskItem.Logf("%s Website logs %s...", i18n.GetMsgByKey("TaskBackup"), website.Alias)
dirItem := path.Join(targetDir, "website", website.Alias)
if _, err := os.Stat(dirItem); err != nil && os.IsNotExist(err) {
if err = os.MkdirAll(dirItem, os.ModePerm); err != nil {
@ -370,7 +397,6 @@ func handleBackupLogs(targetDir, fileName string, secret string) error {
}
}
}
global.LOG.Debug("backup website log successful!")
}
systemDir := path.Join(targetDir, "system")
@ -379,6 +405,8 @@ func handleBackupLogs(targetDir, fileName string, secret string) error {
return err
}
}
taskItem.Logf("%s System logs...", i18n.GetMsgByKey("TaskBackup"))
systemLogFiles, _ := os.ReadDir(global.Dir.LogDir)
if len(systemLogFiles) != 0 {
for i := 0; i < len(systemLogFiles); i++ {
@ -387,8 +415,8 @@ func handleBackupLogs(targetDir, fileName string, secret string) error {
}
}
}
global.LOG.Debug("backup system log successful!")
taskItem.Logf("%s SSH logs...", i18n.GetMsgByKey("TaskBackup"))
loginLogFiles, _ := os.ReadDir("/var/log")
loginDir := path.Join(targetDir, "login")
if _, err := os.Stat(loginDir); err != nil && os.IsNotExist(err) {
@ -403,7 +431,7 @@ func handleBackupLogs(targetDir, fileName string, secret string) error {
}
}
}
global.LOG.Debug("backup ssh log successful!")
taskItem.Log("backup ssh log successful!")
if err := fileOp.TarGzCompressPro(true, targetDir, path.Join(path.Dir(targetDir), fileName), secret, ""); err != nil {
return err

179
agent/app/service/cronjob_helper.go
View file

@ -12,7 +12,6 @@ import (
"github.com/1Panel-dev/1Panel/agent/app/model"
"github.com/1Panel-dev/1Panel/agent/app/repo"
"github.com/1Panel-dev/1Panel/agent/app/task"
"github.com/1Panel-dev/1Panel/agent/buserr"
"github.com/1Panel-dev/1Panel/agent/constant"
"github.com/1Panel-dev/1Panel/agent/global"
"github.com/1Panel-dev/1Panel/agent/i18n"
@ -25,35 +24,14 @@ import (
func (u *CronjobService) HandleJob(cronjob *model.Cronjob) {
record := cronjobRepo.StartRecords(cronjob.ID, "", cronjob.Type)
go func() {
var (
message []byte
err error
)
cronjob.RetryTimes = cronjob.RetryTimes + 1
for i := uint(0); i < cronjob.RetryTimes; i++ {
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(cronjob.Timeout)*time.Second)
done := make(chan error)
go func() {
message, err = u.handleJob(cronjob, record)
taskItem, err := task.NewTaskWithOps(fmt.Sprintf("cronjob-%s", cronjob.Name), task.TaskHandle, task.TaskScopeCronjob, record.TaskID, cronjob.ID)
if err != nil {
global.LOG.Debugf("try handle cron job [%s] %s failed %d/%d, err: %v", cronjob.Type, cronjob.Name, i+1, cronjob.RetryTimes, err)
}
close(done)
}()
select {
case <-done:
cancel()
case <-ctx.Done():
global.LOG.Debugf("try handle cron job [%s] %s failed %d/%d, err: timeout", cronjob.Type, cronjob.Name, i+1, cronjob.RetryTimes)
err = fmt.Errorf("handle timeout")
cancel()
continue
}
global.LOG.Errorf("new task for exec shell failed, err: %v", err)
return
}
err = u.loadTask(cronjob, &record, taskItem)
if cronjob.Type == "snapshot" {
if err != nil {
if len(message) != 0 {
record.Records, _ = mkdirAndWriteFile(cronjob, record.StartTime, message)
}
taskItem, _ := taskRepo.GetFirst(taskRepo.WithByID(record.TaskID))
if len(taskItem.ID) == 0 {
record.TaskID = ""
@ -62,77 +40,78 @@ func (u *CronjobService) HandleJob(cronjob *model.Cronjob) {
handleCronJobAlert(cronjob)
return
}
if len(message) != 0 {
record.Records, err = mkdirAndWriteFile(cronjob, record.StartTime, message)
if err != nil {
global.LOG.Errorf("save file %s failed, err: %v", record.Records, err)
}
}
cronjobRepo.EndRecords(record, constant.StatusSuccess, "", record.Records)
return
}
if err != nil {
global.LOG.Debugf("preper to handle cron job [%s] %s failed, err: %v", cronjob.Type, cronjob.Name, err)
record.TaskID = ""
cronjobRepo.EndRecords(record, constant.StatusFailed, err.Error(), record.Records)
return
}
if err := taskItem.Execute(); err != nil {
taskItem, _ := taskRepo.GetFirst(taskRepo.WithByID(record.TaskID))
if len(taskItem.ID) == 0 {
record.TaskID = ""
}
cronjobRepo.EndRecords(record, constant.StatusFailed, err.Error(), record.Records)
handleCronJobAlert(cronjob)
} else {
cronjobRepo.EndRecords(record, constant.StatusSuccess, "", record.Records)
}
}()
}
func (u *CronjobService) handleJob(cronjob *model.Cronjob, record model.JobRecords) ([]byte, error) {
var (
message []byte
err error
)
func (u *CronjobService) loadTask(cronjob *model.Cronjob, record *model.JobRecords, taskItem *task.Task) error {
var err error
switch cronjob.Type {
case "shell":
if cronjob.ScriptMode == "library" {
scriptItem, _ := scriptRepo.Get(repo.WithByID(cronjob.ScriptID))
if scriptItem.ID == 0 {
return nil, fmt.Errorf("load script from db failed, err: %v", err)
return fmt.Errorf("load script from db failed, err: %v", err)
}
cronjob.Script = scriptItem.Script
cronjob.ScriptMode = "input"
}
if len(cronjob.Script) == 0 {
return nil, fmt.Errorf("the script content is empty and is skipped")
return fmt.Errorf("the script content is empty and is skipped")
}
err = u.handleShell(*cronjob, record.TaskID)
u.handleShell(*cronjob, taskItem)
case "curl":
if len(cronjob.URL) == 0 {
return nil, fmt.Errorf("the url is empty and is skipped")
return fmt.Errorf("the url is empty and is skipped")
}
err = u.handleCurl(*cronjob, record.TaskID)
u.handleCurl(*cronjob, taskItem)
case "ntp":
err = u.handleNtpSync(*cronjob, record.TaskID)
u.handleNtpSync(*cronjob, taskItem)
case "cutWebsiteLog":
var messageItem []string
messageItem, record.File, err = u.handleCutWebsiteLog(cronjob, record.StartTime)
message = []byte(strings.Join(messageItem, "\n"))
err = u.handleCutWebsiteLog(cronjob, record, taskItem)
case "clean":
err = u.handleSystemClean(*cronjob, record.TaskID)
u.handleSystemClean(*cronjob, taskItem)
case "website":
err = u.handleWebsite(*cronjob, record.StartTime, record.TaskID)
err = u.handleWebsite(*cronjob, record.StartTime, taskItem)
case "app":
err = u.handleApp(*cronjob, record.StartTime, record.TaskID)
err = u.handleApp(*cronjob, record.StartTime, taskItem)
case "database":
err = u.handleDatabase(*cronjob, record.StartTime, record.TaskID)
err = u.handleDatabase(*cronjob, record.StartTime, taskItem)
case "directory":
if len(cronjob.SourceDir) == 0 {
return nil, fmt.Errorf("the source dir is empty and is skipped")
return fmt.Errorf("the source dir is empty and is skipped")
}
err = u.handleDirectory(*cronjob, record.StartTime)
err = u.handleDirectory(*cronjob, record.StartTime, taskItem)
case "log":
err = u.handleSystemLog(*cronjob, record.StartTime)
err = u.handleSystemLog(*cronjob, record.StartTime, taskItem)
case "snapshot":
_ = cronjobRepo.UpdateRecords(record.ID, map[string]interface{}{"records": record.Records})
err = u.handleSnapshot(*cronjob, record)
err = u.handleSnapshot(*cronjob, *record, taskItem)
}
return message, err
}
func (u *CronjobService) handleShell(cronjob model.Cronjob, taskID string) error {
taskItem, err := task.NewTaskWithOps(fmt.Sprintf("cronjob-%s", cronjob.Name), task.TaskHandle, task.TaskScopeCronjob, taskID, cronjob.ID)
if err != nil {
global.LOG.Errorf("new task for exec shell failed, err: %v", err)
return err
}
cmdMgr := cmd.NewCommandMgr(cmd.WithTask(*taskItem), cmd.WithTimeout(24*time.Hour))
taskItem.AddSubTask(i18n.GetWithName("HandleShell", cronjob.Name), func(t *task.Task) error {
func (u *CronjobService) handleShell(cronjob model.Cronjob, taskItem *task.Task) {
cmdMgr := cmd.NewCommandMgr(cmd.WithTask(*taskItem))
taskItem.AddSubTaskWithOps(i18n.GetWithName("HandleShell", cronjob.Name), func(t *task.Task) error {
if len(cronjob.ContainerName) != 0 {
command := "sh"
if len(cronjob.Command) != 0 {
@ -166,36 +145,18 @@ func (u *CronjobService) handleShell(cronjob model.Cronjob, taskID string) error
return err
}
return nil
},
nil,
)
return taskItem.Execute()
}, nil, int(cronjob.RetryTimes), time.Duration(cronjob.Timeout)*time.Second)
}
func (u *CronjobService) handleCurl(cronjob model.Cronjob, taskID string) error {
taskItem, err := task.NewTaskWithOps(fmt.Sprintf("cronjob-%s", cronjob.Name), task.TaskHandle, task.TaskScopeCronjob, taskID, cronjob.ID)
if err != nil {
global.LOG.Errorf("new task for exec shell failed, err: %v", err)
return err
}
taskItem.AddSubTask(i18n.GetWithName("HandleShell", cronjob.Name), func(t *task.Task) error {
cmdMgr := cmd.NewCommandMgr(cmd.WithTask(*taskItem), cmd.WithTimeout(time.Hour))
func (u *CronjobService) handleCurl(cronjob model.Cronjob, taskItem *task.Task) {
taskItem.AddSubTaskWithOps(i18n.GetWithName("HandleShell", cronjob.Name), func(t *task.Task) error {
cmdMgr := cmd.NewCommandMgr(cmd.WithTask(*taskItem))
return cmdMgr.Run("curl", cronjob.URL)
},
nil,
)
return taskItem.Execute()
}, nil, int(cronjob.RetryTimes), time.Duration(cronjob.Timeout)*time.Second)
}
func (u *CronjobService) handleNtpSync(cronjob model.Cronjob, taskID string) error {
taskItem, err := task.NewTaskWithOps(fmt.Sprintf("cronjob-%s", cronjob.Name), task.TaskHandle, task.TaskScopeCronjob, taskID, cronjob.ID)
if err != nil {
global.LOG.Errorf("new task for exec shell failed, err: %v", err)
return err
}
taskItem.AddSubTask(i18n.GetMsgByKey("HandleNtpSync"), func(t *task.Task) error {
func (u *CronjobService) handleNtpSync(cronjob model.Cronjob, taskItem *task.Task) {
taskItem.AddSubTaskWithOps(i18n.GetMsgByKey("HandleNtpSync"), func(t *task.Task) error {
ntpServer, err := settingRepo.Get(settingRepo.WithByKey("NtpSite"))
if err != nil {
return err
@ -209,20 +170,17 @@ func (u *CronjobService) handleNtpSync(cronjob model.Cronjob, taskID string) err
return err
}
return nil
}, nil)
return taskItem.Execute()
}, nil, int(cronjob.RetryTimes), time.Duration(cronjob.Timeout)*time.Second)
}
func (u *CronjobService) handleCutWebsiteLog(cronjob *model.Cronjob, startTime time.Time) ([]string, string, error) {
var (
err error
filePaths []string
msgs []string
)
func (u *CronjobService) handleCutWebsiteLog(cronjob *model.Cronjob, record *model.JobRecords, taskItem *task.Task) error {
taskItem.AddSubTaskWithOps(i18n.GetWithName("CutWebsiteLog", cronjob.Name), func(t *task.Task) error {
var filePaths []string
websites := loadWebsForJob(*cronjob)
fileOp := files.NewFileOp()
baseDir := GetOpenrestyDir(SitesRootDir)
for _, website := range websites {
taskItem.Log(website.Alias)
websiteLogDir := pathUtils.Join(baseDir, website.Alias, "log")
srcAccessLogPath := pathUtils.Join(websiteLogDir, "access.log")
srcErrorLogPath := pathUtils.Join(websiteLogDir, "error.log")
@ -231,25 +189,24 @@ func (u *CronjobService) handleCutWebsiteLog(cronjob *model.Cronjob, startTime t
_ = os.MkdirAll(dstLogDir, constant.DirPerm)
}
dstName := fmt.Sprintf("%s_log_%s.gz", website.PrimaryDomain, startTime.Format(constant.DateTimeSlimLayout))
dstName := fmt.Sprintf("%s_log_%s.gz", website.PrimaryDomain, record.StartTime.Format(constant.DateTimeSlimLayout))
dstFilePath := pathUtils.Join(dstLogDir, dstName)
filePaths = append(filePaths, dstFilePath)
if err = backupLogFile(dstFilePath, websiteLogDir, fileOp); err != nil {
websiteErr := buserr.WithNameAndErr("ErrCutWebsiteLog", website.PrimaryDomain, err)
err = websiteErr
msgs = append(msgs, websiteErr.Error())
global.LOG.Error(websiteErr.Error())
if err := backupLogFile(dstFilePath, websiteLogDir, fileOp); err != nil {
taskItem.LogFailedWithErr("CutWebsiteLog", err)
continue
} else {
_ = fileOp.WriteFile(srcAccessLogPath, strings.NewReader(""), constant.DirPerm)
_ = fileOp.WriteFile(srcErrorLogPath, strings.NewReader(""), constant.DirPerm)
}
msg := i18n.GetMsgWithMap("CutWebsiteLogSuccess", map[string]interface{}{"name": website.PrimaryDomain, "path": dstFilePath})
msgs = append(msgs, msg)
taskItem.Log(i18n.GetMsgWithMap("CutWebsiteLogSuccess", map[string]interface{}{"name": website.PrimaryDomain, "path": dstFilePath}))
}
u.removeExpiredLog(*cronjob)
return msgs, strings.Join(filePaths, ","), err
record.File = strings.Join(filePaths, ",")
return nil
}, nil, int(cronjob.RetryTimes), time.Duration(cronjob.Timeout)*time.Second)
return nil
}
func backupLogFile(dstFilePath, websiteLogDir string, fileOp files.FileOp) error {
@ -272,13 +229,9 @@ func backupLogFile(dstFilePath, websiteLogDir string, fileOp files.FileOp) error
return nil
}
func (u *CronjobService) handleSystemClean(cronjob model.Cronjob, taskID string) error {
taskItem, err := task.NewTaskWithOps(fmt.Sprintf("cronjob-%s", cronjob.Name), task.TaskHandle, task.TaskScopeCronjob, taskID, cronjob.ID)
if err != nil {
global.LOG.Errorf("new task for system clean failed, err: %v", err)
return err
}
return systemClean(taskItem)
func (u *CronjobService) handleSystemClean(cronjob model.Cronjob, taskItem *task.Task) {
cleanTask := doSystemClean(taskItem)
taskItem.AddSubTaskWithOps(i18n.GetMsgByKey("HandleSystemClean"), cleanTask, nil, int(cronjob.RetryTimes), time.Duration(cronjob.Timeout)*time.Second)
}
func (u *CronjobService) uploadCronjobBackFile(cronjob model.Cronjob, accountMap map[string]backupClientHelper, file string) (string, error) {

8
agent/app/service/device_clean.go
View file

@ -10,7 +10,6 @@ import (
"time"
"github.com/1Panel-dev/1Panel/agent/constant"
"github.com/1Panel-dev/1Panel/agent/i18n"
"github.com/1Panel-dev/1Panel/agent/utils/docker"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
@ -301,8 +300,8 @@ func (u *DeviceService) Clean(req []dto.Clean) {
}
}
func systemClean(taskItem *task.Task) error {
taskItem.AddSubTask(i18n.GetMsgByKey("HandleSystemClean"), func(t *task.Task) error {
func doSystemClean(taskItem *task.Task) func(t *task.Task) error {
return func(t *task.Task) error {
size := int64(0)
fileCount := 0
dropWithTask(path.Join(global.Dir.BaseDir, "1panel_original"), taskItem, &size, &fileCount)
@ -363,8 +362,7 @@ func systemClean(taskItem *task.Task) error {
_ = settingRepo.Update("LastCleanData", fmt.Sprintf("%v", fileCount))
return nil
}, nil)
return taskItem.Execute()
}
}
func loadSnapshotTree(fileOp fileUtils.FileOp) []dto.CleanTree {

3
agent/app/service/snapshot.go
View file

@ -8,6 +8,7 @@ import (
"strings"
"github.com/1Panel-dev/1Panel/agent/app/repo"
"github.com/1Panel-dev/1Panel/agent/app/task"
"github.com/1Panel-dev/1Panel/agent/buserr"
"github.com/1Panel-dev/1Panel/agent/app/dto"
@ -30,7 +31,7 @@ type SnapshotService struct {
type ISnapshotService interface {
SearchWithPage(req dto.PageSnapshot) (int64, interface{}, error)
LoadSnapshotData() (dto.SnapshotData, error)
SnapshotCreate(req dto.SnapshotCreate, jobID uint) error
SnapshotCreate(parentTask *task.Task, req dto.SnapshotCreate, jobID, retry, timeout uint) error
SnapshotReCreate(id uint) error
SnapshotRecover(req dto.SnapshotRecover) error
SnapshotRollback(req dto.SnapshotRecover) error

54
agent/app/service/snapshot_create.go
View file

@ -26,7 +26,7 @@ import (
"gorm.io/gorm"
)
func (u *SnapshotService) SnapshotCreate(req dto.SnapshotCreate, jobID uint) error {
func (u *SnapshotService) SnapshotCreate(parentTask *task.Task, req dto.SnapshotCreate, jobID, retry, timeout uint) error {
versionItem, _ := settingRepo.Get(settingRepo.WithByKey("SystemVersion"))
scope := "core"
@ -65,19 +65,23 @@ func (u *SnapshotService) SnapshotCreate(req dto.SnapshotCreate, jobID uint) err
}
req.ID = snap.ID
taskItem, err := task.NewTaskWithOps(req.Name, task.TaskCreate, task.TaskScopeSnapshot, req.TaskID, req.ID)
var err error
taskItem := parentTask
if parentTask == nil {
taskItem, err = task.NewTaskWithOps(req.Name, task.TaskCreate, task.TaskScopeSnapshot, req.TaskID, req.ID)
if err != nil {
global.LOG.Errorf("new task for create snapshot failed, err: %v", err)
return err
}
}
if jobID == 0 {
go func() {
_ = handleSnapshot(req, taskItem, jobID)
_ = handleSnapshot(req, taskItem, jobID, 0, 0)
}()
return nil
}
return handleSnapshot(req, taskItem, jobID)
return handleSnapshot(req, taskItem, jobID, retry, timeout)
}
func (u *SnapshotService) SnapshotReCreate(id uint) error {
@ -108,20 +112,23 @@ func (u *SnapshotService) SnapshotReCreate(id uint) error {
return err
}
go func() {
_ = handleSnapshot(req, taskItem, 0)
_ = handleSnapshot(req, taskItem, 0, 0, 0)
}()
return nil
}
func handleSnapshot(req dto.SnapshotCreate, taskItem *task.Task, jobID uint) error {
func handleSnapshot(req dto.SnapshotCreate, taskItem *task.Task, jobID, retry, timeout uint) error {
rootDir := path.Join(global.Dir.TmpDir, "system", req.Name)
openrestyDir, _ := settingRepo.GetValueByKey("WEBSITE_DIR")
itemHelper := snapHelper{SnapID: req.ID, Task: *taskItem, FileOp: files.NewFileOp(), Ctx: context.Background(), OpenrestyDir: openrestyDir}
baseDir := path.Join(rootDir, "base")
_ = os.MkdirAll(baseDir, os.ModePerm)
taskItem.AddSubTaskWithAlias(
if timeout == 0 {
timeout = 1800
}
taskItem.AddSubTaskWithAliasAndOps(
"SnapDBInfo",
func(t *task.Task) error {
if err := loadDbConn(&itemHelper, rootDir, req); err != nil {
@ -132,68 +139,65 @@ func handleSnapshot(req dto.SnapshotCreate, taskItem *task.Task, jobID uint) err
_ = itemHelper.snapAgentDB.Where("id = ?", jobID).Delete(&model.JobRecords{}).Error
}
return nil
},
nil,
}, nil, int(retry), time.Duration(timeout)*time.Second,
)
if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapBaseInfo" {
taskItem.AddSubTaskWithAlias(
taskItem.AddSubTaskWithAliasAndOps(
"SnapBaseInfo",
func(t *task.Task) error { return snapBaseData(itemHelper, baseDir) },
nil,
nil, int(retry), time.Duration(timeout)*time.Second,
)
req.InterruptStep = ""
}
if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapInstallApp" {
taskItem.AddSubTaskWithAlias(
taskItem.AddSubTaskWithAliasAndOps(
"SnapInstallApp",
func(t *task.Task) error { return snapAppImage(itemHelper, req, rootDir) },
nil,
nil, int(retry), time.Duration(timeout)*time.Second,
)
req.InterruptStep = ""
}
if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapLocalBackup" {
taskItem.AddSubTaskWithAlias(
taskItem.AddSubTaskWithAliasAndOps(
"SnapLocalBackup",
func(t *task.Task) error { return snapBackupData(itemHelper, req, rootDir) },
nil,
nil, int(retry), time.Duration(timeout)*time.Second,
)
req.InterruptStep = ""
}
if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapPanelData" {
taskItem.AddSubTaskWithAlias(
taskItem.AddSubTaskWithAliasAndOps(
"SnapPanelData",
func(t *task.Task) error { return snapPanelData(itemHelper, req, rootDir) },
nil,
nil, int(retry), time.Duration(timeout)*time.Second,
)
req.InterruptStep = ""
}
taskItem.AddSubTaskWithAlias(
taskItem.AddSubTaskWithAliasAndOps(
"SnapCloseDBConn",
func(t *task.Task) error {
taskItem.Log("---------------------- 6 / 8 ----------------------")
common.CloseDB(itemHelper.snapAgentDB)
common.CloseDB(itemHelper.snapCoreDB)
return nil
},
nil,
}, nil, int(retry), time.Duration(timeout)*time.Second,
)
if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapCompress" {
taskItem.AddSubTaskWithAlias(
taskItem.AddSubTaskWithAliasAndOps(
"SnapCompress",
func(t *task.Task) error { return snapCompress(itemHelper, rootDir, req.Secret) },
nil,
nil, int(retry), time.Duration(timeout)*time.Second,
)
req.InterruptStep = ""
}
if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapUpload" {
taskItem.AddSubTaskWithAlias(
taskItem.AddSubTaskWithAliasAndOps(
"SnapUpload",
func(t *task.Task) error {
return snapUpload(itemHelper, req.SourceAccountIDs, fmt.Sprintf("%s.tar.gz", rootDir))
},
nil,
}, nil, int(retry), time.Duration(timeout)*time.Second,
)
req.InterruptStep = ""
}

7
agent/app/task/task.go
View file

@ -3,6 +3,7 @@ package task
import (
"bufio"
"context"
"errors"
"fmt"
"log"
"os"
@ -156,6 +157,11 @@ func (t *Task) AddSubTaskWithOps(name string, action ActionFunc, rollback Rollba
t.SubTasks = append(t.SubTasks, subTask)
}
func (t *Task) AddSubTaskWithAliasAndOps(key string, action ActionFunc, rollback RollbackFunc, retry int, timeout time.Duration) {
subTask := &SubTask{RootTask: t, Name: i18n.GetMsgByKey(key), Retry: retry, Timeout: timeout, Action: action, Rollback: rollback}
t.SubTasks = append(t.SubTasks, subTask)
}
func (t *Task) AddSubTaskWithIgnoreErr(name string, action ActionFunc) {
subTask := &SubTask{RootTask: t, Name: name, Retry: 0, Timeout: 30 * time.Minute, Action: action, Rollback: nil, IgnoreErr: true}
t.SubTasks = append(t.SubTasks, subTask)
@ -182,6 +188,7 @@ func (s *SubTask) Execute() error {
select {
case <-ctx.Done():
s.RootTask.Log(i18n.GetWithName("TaskTimeout", subTaskName))
err = errors.New("timeout!")
case err = <-done:
if err != nil {
s.RootTask.Log(i18n.GetWithNameAndErr("SubTaskFailed", subTaskName, err))

4
agent/i18n/lang/en.yaml
View file

@ -192,11 +192,13 @@ ErrConfigAlreadyExist: 'A configuration file with the same name already exists'
ErrUserFindErr: 'User {{ .name }} search failed {{ .err }}'
#cronjob
ErrCutWebsiteLog: '{{ .name }} website log cutting failed, error {{ .err }}'
CutWebsiteLogSuccess: '{{ .name }} website log cut successfully, backup path {{ .path }}'
HandleShell: 'Execute script {{ .name }}'
HandleNtpSync: 'System time synchronization'
HandleSystemClean: 'System cache cleanup'
SystemLog: 'System Log'
CutWebsiteLog: 'Rotate Website Log'
FileOrDir: 'Directory / File'
#toolbox
ErrNotExistUser: 'The current user does not exist, please modify and try again!'

4
agent/i18n/lang/ja.yaml
View file

@ -192,11 +192,13 @@ ErrConfigAlreadyExist: '同じ名前の設定ファイルがすでに存在し
ErrUserFindErr: 'ユーザー {{ .name }} の検索に失敗しました {{ .err }}'
#cronjob
ErrCutWebsiteLog: '{{ .name }} ウェブサイトのログの切り取りに失敗しました。エラー {{ .err }}'
CutWebsiteLogSuccess: '{{ .name }} ウェブサイトのログが正常にカットされました。バックアップ パス {{ .path }}'
HandleShell: 'スクリプト {{ .name }} を実行します'
HandleNtpSync: 'システム時刻の同期'
HandleSystemClean: 'システム キャッシュのクリーンアップ'
SystemLog: 'システムログ'
CutWebsiteLog: 'ウェブサイトログのローテーション'
FileOrDir: 'ディレクトリ / ファイル'
#toolbox
ErrNotExistUser: '現在のユーザーは存在しません。変更してもう一度お試しください。'

4
agent/i18n/lang/ko.yaml
View file

@ -192,11 +192,13 @@ ErrConfigAlreadyExist: '같은 이름의 구성 파일이 이미 존재합니다
ErrUserFindErr: '사용자 {{ .name }} 검색에 실패했습니다 {{ .err }}'
#크론잡
ErrCutWebsiteLog: '{{ .name }} 웹사이트 로그 잘라내기에 실패했습니다. 오류 {{ .err }}'
CutWebsiteLogSuccess: '{{ .name }} 웹사이트 로그가 성공적으로 잘렸습니다. 백업 경로 {{ .path }}'
HandleShell: '스크립트 {{ .name }} 실행'
HandleNtpSync: '시스템 시간 동기화'
HandleSystemClean: '시스템 캐시 정리'
SystemLog: '시스템 로그'
CutWebsiteLog: '웹사이트 로그 회전'
FileOrDir: '디렉터리 / 파일'
#도구상자
ErrNotExistUser: '현재 사용자가 존재하지 않습니다. 수정한 후 다시 시도하세요!'

4
agent/i18n/lang/ms.yaml
View file

@ -192,11 +192,13 @@ ErrConfigAlreadyExist: 'Fail konfigurasi dengan nama yang sama sudah wujud'
ErrUserFindErr: 'Pengguna {{ .name }} carian gagal {{ .err }}'
#cronjob
ErrCutWebsiteLog: '{{ .name }} pemotongan log tapak web gagal, ralat {{ .err }}'
CutWebsiteLogSuccess: '{{ .name }} log tapak web berjaya dipotong, laluan sandaran {{ .path }}'
HandleShell: 'Laksanakan skrip {{ .name }}'
HandleNtpSync: 'Penyegerakan masa sistem'
HandleSystemClean: 'Pembersihan cache sistem'
SystemLog: 'Log Sistem'
CutWebsiteLog: 'Putar Log Laman Web'
FileOrDir: 'Direktori / Fail'
#kotak alat
ErrNotExistUser: 'Pengguna semasa tidak wujud, sila ubah suai dan cuba lagi!'

4
agent/i18n/lang/pt-BR.yaml
View file

@ -192,11 +192,13 @@ ErrConfigAlreadyExist: 'Um arquivo de configuração com o mesmo nome já existe
ErrUserFindErr: 'Falha na pesquisa do usuário {{ .name }} {{ .err }}'
#cronjob
ErrCutWebsiteLog: '{{ .name }} falha ao cortar o log do site, erro {{ .err }}'
CutWebsiteLogSuccess: '{{ .name }} registro do site cortado com sucesso, caminho de backup {{ .path }}'
HandleShell: 'Executar script {{ .name }}'
HandleNtpSync: 'Sincronização de hora do sistema'
HandleSystemClean: 'Limpeza de cache do sistema'
SystemLog: 'Log do Sistema'
CutWebsiteLog: 'Rotacionar Log do Website'
FileOrDir: 'Diretório / Arquivo'
#caixa de ferramentas
ErrNotExistUser: 'O usuário atual não existe, modifique e tente novamente!'

4
agent/i18n/lang/ru.yaml
View file

@ -192,11 +192,13 @@ ErrConfigAlreadyExist: 'Файл конфигурации с таким имен
ErrUserFindErr: 'Поиск пользователя {{ .name }} не удался {{ .err }}'
#cronjob
ErrCutWebsiteLog: '{{ .name }} не удалось вырезать журнал веб-сайта, ошибка {{ .err }}'
CutWebsiteLogSuccess: 'Журнал веб-сайта {{ .name }} успешно вырезан, путь к резервной копии {{ .path }}'
HandleShell: 'Выполнить скрипт {{ .name }}'
HandleNtpSync: 'Синхронизация системного времени'
HandleSystemClean: 'Очистка системного кэша'
SystemLog: 'Системный лог'
CutWebsiteLog: 'Ротация логов сайта'
FileOrDir: 'Каталог / Файл'
#ящик для инструментов
ErrNotExistUser: 'Текущий пользователь не существует, измените его и повторите попытку!'

4
agent/i18n/lang/zh-Hant.yaml
View file

@ -192,11 +192,13 @@ ErrConfigAlreadyExist: '已存在同名設定檔'
ErrUserFindErr: '使用者{{ .name }} 尋找失敗{{ .err }}'
#cronjob
ErrCutWebsiteLog: '{{ .name }} 網站日誌切割失敗，錯誤{{ .err }}'
CutWebsiteLogSuccess: '{{ .name }} 網站日誌切割成功，備份路徑{{ .path }}'
HandleShell: '執行腳本{{ .name }}'
HandleNtpSync: '系統時間同步'
HandleSystemClean: '系統快取清理'
SystemLog: '系統日誌'
CutWebsiteLog: '切割網站日誌'
FileOrDir: '目錄 / 檔案'
#toolbox
ErrNotExistUser: '目前使用者不存在，請修改後重試！'

4
agent/i18n/lang/zh.yaml
View file

@ -191,11 +191,13 @@ ErrConfigAlreadyExist: "已存在同名配置文件"
ErrUserFindErr: "用户 {{ .name }} 查找失败 {{ .err }}"
#cronjob
ErrCutWebsiteLog: "{{ .name }} 网站日志切割失败，错误 {{ .err }}"
CutWebsiteLogSuccess: "{{ .name }} 网站日志切割成功，备份路径 {{ .path }}"
HandleShell: "执行脚本 {{ .name }}"
HandleNtpSync: "系统时间同步"
HandleSystemClean: "系统缓存清理"
SystemLog: "系统日志"
CutWebsiteLog: "切割网站日志"
FileOrDir: "目录 / 文件"
#toolbox
ErrNotExistUser: "当前用户不存在，请修改后重试！"

2
agent/utils/files/file_op.go
View file

@ -742,7 +742,7 @@ func (f FileOp) TarGzCompressPro(withDir bool, src, dst, secret, exclusionRules
itemPrefix = ""
}
if len(secret) != 0 {
commands = fmt.Sprintf("tar --warning=no-file-changed --ignore-failed-read --exclude-from=<(find %s -type s -printf '%s' | sed 's|^|%s/|') -zcf - %s | openssl enc -aes-256-cbc -salt -k '%s' -out %s", src, "%P\n", itemPrefix, srcItem, secret, dst)
commands = fmt.Sprintf("tar --warning=no-file-changed --ignore-failed-read %s --exclude-from=<(find %s -type s -printf '%s' | sed 's|^|%s/|') -zcf - %s | openssl enc -aes-256-cbc -salt -k '%s' -out %s", exStr, src, "%P\n", itemPrefix, srcItem, secret, dst)
global.LOG.Debug(strings.ReplaceAll(commands, fmt.Sprintf(" %s ", secret), "******"))
} else {
commands = fmt.Sprintf("tar --warning=no-file-changed --ignore-failed-read --exclude-from=<(find %s -type s -printf '%s' | sed 's|^|%s/|') -zcf %s %s %s", src, "%P\n", itemPrefix, dst, exStr, srcItem)