diff --git a/agent/app/service/backup_app.go b/agent/app/service/backup_app.go index 91b5e34de..e2a35add8 100644 --- a/agent/app/service/backup_app.go +++ b/agent/app/service/backup_app.go @@ -4,14 +4,15 @@ import ( "context" "encoding/json" "fmt" - "github.com/1Panel-dev/1Panel/agent/app/task" - "github.com/1Panel-dev/1Panel/agent/i18n" "io/fs" "os" "path" "strings" "time" + "github.com/1Panel-dev/1Panel/agent/app/task" + "github.com/1Panel-dev/1Panel/agent/i18n" + "github.com/1Panel-dev/1Panel/agent/buserr" "github.com/1Panel-dev/1Panel/agent/app/dto" @@ -103,7 +104,8 @@ func backupDatabaseWithTask(parentTask *task.Task, resourceKey, tmpDir, name str return err } parentTask.LogStart(task.GetTaskName(db.Name, task.TaskBackup, task.TaskScopeDatabase)) - if err := handleMysqlBackup(db.MysqlName, resourceKey, db.Name, tmpDir, fmt.Sprintf("%s.sql.gz", name)); err != nil { + databaseHelper := DatabaseHelper{Database: db.MysqlName, DBType: resourceKey, Name: db.Name} + if err := handleMysqlBackup(databaseHelper, parentTask, tmpDir, fmt.Sprintf("%s.sql.gz", name), ""); err != nil { return err } parentTask.LogSuccess(task.GetTaskName(db.Name, task.TaskBackup, task.TaskScopeDatabase)) @@ -113,7 +115,8 @@ func backupDatabaseWithTask(parentTask *task.Task, resourceKey, tmpDir, name str return err } parentTask.LogStart(task.GetTaskName(db.Name, task.TaskBackup, task.TaskScopeDatabase)) - if err := handlePostgresqlBackup(db.PostgresqlName, db.Name, tmpDir, fmt.Sprintf("%s.sql.gz", name)); err != nil { + databaseHelper := DatabaseHelper{Database: db.PostgresqlName, DBType: resourceKey, Name: db.Name} + if err := handlePostgresqlBackup(databaseHelper, parentTask, tmpDir, fmt.Sprintf("%s.sql.gz", name), ""); err != nil { return err } parentTask.LogSuccess(task.GetTaskName(db.Name, task.TaskBackup, task.TaskScopeDatabase)) @@ -257,7 +260,7 @@ func handleAppRecover(install *model.AppInstall, parentTask *task.Task, recoverF Name: database.Name, DetailName: db.Name, File: fmt.Sprintf("%s/%s.sql.gz", tmpPath, install.Name), - }, true); err != nil { + }, parentTask, true); err != nil { t.LogFailedWithErr(taskName, err) return err } @@ -286,7 +289,7 @@ func handleAppRecover(install *model.AppInstall, parentTask *task.Task, recoverF Name: newDB.MysqlName, DetailName: newDB.Name, File: fmt.Sprintf("%s/%s.sql.gz", tmpPath, install.Name), - }, true); err != nil { + }, parentTask, true, ""); err != nil { t.LogFailedWithErr(taskName, err) return err } diff --git a/agent/app/service/backup_mysql.go b/agent/app/service/backup_mysql.go index ff94497ca..a61950b54 100644 --- a/agent/app/service/backup_mysql.go +++ b/agent/app/service/backup_mysql.go @@ -9,11 +9,13 @@ import ( "time" "github.com/1Panel-dev/1Panel/agent/constant" + "github.com/1Panel-dev/1Panel/agent/i18n" "github.com/1Panel-dev/1Panel/agent/buserr" "github.com/1Panel-dev/1Panel/agent/app/dto" "github.com/1Panel-dev/1Panel/agent/app/model" + "github.com/1Panel-dev/1Panel/agent/app/task" "github.com/1Panel-dev/1Panel/agent/global" "github.com/1Panel-dev/1Panel/agent/utils/common" "github.com/1Panel-dev/1Panel/agent/utils/files" @@ -26,7 +28,8 @@ func (u *BackupService) MysqlBackup(req dto.CommonBackup) error { targetDir := path.Join(global.CONF.System.Backup, itemDir) fileName := fmt.Sprintf("%s_%s.sql.gz", req.DetailName, timeNow+common.RandStrAndNum(5)) - if err := handleMysqlBackup(req.Name, req.Type, req.DetailName, targetDir, fileName); err != nil { + databaseHelper := DatabaseHelper{Database: req.Name, DBType: req.Type, Name: req.DetailName} + if err := handleMysqlBackup(databaseHelper, nil, targetDir, fileName, req.TaskID); err != nil { return err } @@ -46,7 +49,7 @@ func (u *BackupService) MysqlBackup(req dto.CommonBackup) error { } func (u *BackupService) MysqlRecover(req dto.CommonRecover) error { - if err := handleMysqlRecover(req, false); err != nil { + if err := handleMysqlRecover(req, nil, false, req.TaskID); err != nil { return err } return nil @@ -90,100 +93,143 @@ func (u *BackupService) MysqlRecoverByUpload(req dto.CommonRecover) error { } req.File = path.Dir(file) + "/" + fileName - if err := handleMysqlRecover(req, false); err != nil { + if err := handleMysqlRecover(req, nil, false, req.TaskID); err != nil { return err } global.LOG.Info("recover from uploads successful!") return nil } -func handleMysqlBackup(database, dbType, dbName, targetDir, fileName string) error { - dbInfo, err := mysqlRepo.Get(commonRepo.WithByName(dbName), mysqlRepo.WithByMysqlName(database)) +func handleMysqlBackup(db DatabaseHelper, parentTask *task.Task, targetDir, fileName, taskID string) error { + var ( + err error + itemTask *task.Task + ) + itemTask = parentTask + dbInfo, err := mysqlRepo.Get(commonRepo.WithByName(db.Name), mysqlRepo.WithByMysqlName(db.Database)) if err != nil { return err } - cli, version, err := LoadMysqlClientByFrom(database) - if err != nil { - return err + itemName := fmt.Sprintf("%s[%s] - %s", db.Database, db.DBType, db.Name) + if parentTask == nil { + itemTask, err = task.NewTaskWithOps(itemName, task.TaskBackup, task.TaskScopeDatabase, taskID, dbInfo.ID) + if err != nil { + return err + } } - backupInfo := client.BackupInfo{ - Name: dbName, - Type: dbType, - Version: version, - Format: dbInfo.Format, - TargetDir: targetDir, - FileName: fileName, + backupDatabase := func(t *task.Task) error { + cli, version, err := LoadMysqlClientByFrom(db.Database) + if err != nil { + return err + } + backupInfo := client.BackupInfo{ + Name: db.Name, + Type: db.DBType, + Version: version, + Format: dbInfo.Format, + TargetDir: targetDir, + FileName: fileName, - Timeout: 300, + Timeout: 300, + } + return cli.Backup(backupInfo) } - if err := cli.Backup(backupInfo); err != nil { - return err + + itemTask.AddSubTask(i18n.GetMsgByKey("TaskBackup"), backupDatabase, nil) + if parentTask != nil { + return backupDatabase(parentTask) } - return nil + + return itemTask.Execute() } -func handleMysqlRecover(req dto.CommonRecover, isRollback bool) error { - isOk := false - fileOp := files.NewFileOp() - if !fileOp.Stat(req.File) { - return buserr.WithName("ErrFileNotFound", req.File) - } +func handleMysqlRecover(req dto.CommonRecover, parentTask *task.Task, isRollback bool, taskID string) error { + var ( + err error + itemTask *task.Task + ) + itemTask = parentTask dbInfo, err := mysqlRepo.Get(commonRepo.WithByName(req.DetailName), mysqlRepo.WithByMysqlName(req.Name)) if err != nil { return err } - cli, version, err := LoadMysqlClientByFrom(req.Name) - if err != nil { - return err + itemName := fmt.Sprintf("%s[%s] - %s", req.Name, req.Type, req.DetailName) + if parentTask == nil { + itemTask, err = task.NewTaskWithOps(itemName, task.TaskRecover, task.TaskScopeDatabase, taskID, dbInfo.ID) + if err != nil { + return err + } } - if !isRollback { - rollbackFile := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("database/%s/%s_%s.sql.gz", req.Type, req.DetailName, time.Now().Format(constant.DateTimeSlimLayout))) - if err := cli.Backup(client.BackupInfo{ - Name: req.DetailName, - Type: req.Type, - Version: version, - Format: dbInfo.Format, - TargetDir: path.Dir(rollbackFile), - FileName: path.Base(rollbackFile), + recoverDatabase := func(t *task.Task) error { + isOk := false + fileOp := files.NewFileOp() + if !fileOp.Stat(req.File) { + return buserr.WithName("ErrFileNotFound", req.File) + } + dbInfo, err := mysqlRepo.Get(commonRepo.WithByName(req.DetailName), mysqlRepo.WithByMysqlName(req.Name)) + if err != nil { + return err + } + cli, version, err := LoadMysqlClientByFrom(req.Name) + if err != nil { + return err + } + + if !isRollback { + rollbackFile := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("database/%s/%s_%s.sql.gz", req.Type, req.DetailName, time.Now().Format(constant.DateTimeSlimLayout))) + if err := cli.Backup(client.BackupInfo{ + Name: req.DetailName, + Type: req.Type, + Version: version, + Format: dbInfo.Format, + TargetDir: path.Dir(rollbackFile), + FileName: path.Base(rollbackFile), + + Timeout: 300, + }); err != nil { + return fmt.Errorf("backup mysql db %s for rollback before recover failed, err: %v", req.DetailName, err) + } + defer func() { + if !isOk { + global.LOG.Info("recover failed, start to rollback now") + if err := cli.Recover(client.RecoverInfo{ + Name: req.DetailName, + Type: req.Type, + Version: version, + Format: dbInfo.Format, + SourceFile: rollbackFile, + + Timeout: 300, + }); err != nil { + global.LOG.Errorf("rollback mysql db %s from %s failed, err: %v", req.DetailName, rollbackFile, err) + } + global.LOG.Infof("rollback mysql db %s from %s successful", req.DetailName, rollbackFile) + _ = os.RemoveAll(rollbackFile) + } else { + _ = os.RemoveAll(rollbackFile) + } + }() + } + if err := cli.Recover(client.RecoverInfo{ + Name: req.DetailName, + Type: req.Type, + Version: version, + Format: dbInfo.Format, + SourceFile: req.File, Timeout: 300, }); err != nil { - return fmt.Errorf("backup mysql db %s for rollback before recover failed, err: %v", req.DetailName, err) + return err } - defer func() { - if !isOk { - global.LOG.Info("recover failed, start to rollback now") - if err := cli.Recover(client.RecoverInfo{ - Name: req.DetailName, - Type: req.Type, - Version: version, - Format: dbInfo.Format, - SourceFile: rollbackFile, - - Timeout: 300, - }); err != nil { - global.LOG.Errorf("rollback mysql db %s from %s failed, err: %v", req.DetailName, rollbackFile, err) - } - global.LOG.Infof("rollback mysql db %s from %s successful", req.DetailName, rollbackFile) - _ = os.RemoveAll(rollbackFile) - } else { - _ = os.RemoveAll(rollbackFile) - } - }() + isOk = true + return nil } - if err := cli.Recover(client.RecoverInfo{ - Name: req.DetailName, - Type: req.Type, - Version: version, - Format: dbInfo.Format, - SourceFile: req.File, - - Timeout: 300, - }); err != nil { - return err + itemTask.AddSubTask(i18n.GetMsgByKey("TaskRecover"), recoverDatabase, nil) + if parentTask != nil { + return recoverDatabase(parentTask) } - isOk = true - return nil + + return itemTask.Execute() } diff --git a/agent/app/service/backup_postgresql.go b/agent/app/service/backup_postgresql.go index a709289cf..1a54ad3c0 100644 --- a/agent/app/service/backup_postgresql.go +++ b/agent/app/service/backup_postgresql.go @@ -9,6 +9,7 @@ import ( "time" "github.com/1Panel-dev/1Panel/agent/constant" + "github.com/1Panel-dev/1Panel/agent/i18n" "github.com/1Panel-dev/1Panel/agent/buserr" "github.com/1Panel-dev/1Panel/agent/utils/common" @@ -16,6 +17,7 @@ import ( "github.com/1Panel-dev/1Panel/agent/app/dto" "github.com/1Panel-dev/1Panel/agent/app/model" + "github.com/1Panel-dev/1Panel/agent/app/task" "github.com/1Panel-dev/1Panel/agent/global" "github.com/1Panel-dev/1Panel/agent/utils/files" "github.com/1Panel-dev/1Panel/agent/utils/postgresql/client" @@ -27,7 +29,8 @@ func (u *BackupService) PostgresqlBackup(req dto.CommonBackup) error { targetDir := path.Join(global.CONF.System.Backup, itemDir) fileName := fmt.Sprintf("%s_%s.sql.gz", req.DetailName, timeNow+common.RandStrAndNum(5)) - if err := handlePostgresqlBackup(req.Name, req.DetailName, targetDir, fileName); err != nil { + databaseHelper := DatabaseHelper{Database: req.Name, DBType: req.Type, Name: req.DetailName} + if err := handlePostgresqlBackup(databaseHelper, nil, targetDir, fileName, req.TaskID); err != nil { return err } @@ -46,7 +49,7 @@ func (u *BackupService) PostgresqlBackup(req dto.CommonBackup) error { return nil } func (u *BackupService) PostgresqlRecover(req dto.CommonRecover) error { - if err := handlePostgresqlRecover(req, false); err != nil { + if err := handlePostgresqlRecover(req, nil, false); err != nil { return err } return nil @@ -90,85 +93,123 @@ func (u *BackupService) PostgresqlRecoverByUpload(req dto.CommonRecover) error { } req.File = path.Dir(file) + "/" + fileName - if err := handlePostgresqlRecover(req, false); err != nil { + if err := handlePostgresqlRecover(req, nil, false); err != nil { return err } global.LOG.Info("recover from uploads successful!") return nil } -func handlePostgresqlBackup(database, dbName, targetDir, fileName string) error { - cli, err := LoadPostgresqlClientByFrom(database) - if err != nil { - return err - } - defer cli.Close() - backupInfo := pgclient.BackupInfo{ - Name: dbName, - TargetDir: targetDir, - FileName: fileName, +func handlePostgresqlBackup(db DatabaseHelper, parentTask *task.Task, targetDir, fileName, taskID string) error { + var ( + err error + itemTask *task.Task + ) + itemTask = parentTask + itemName := fmt.Sprintf("%s - %s", db.Database, db.Name) + if parentTask == nil { + itemTask, err = task.NewTaskWithOps(itemName, task.TaskBackup, task.TaskScopeDatabase, taskID, db.ID) + if err != nil { + return err + } + } + backupDatabase := func(t *task.Task) error { + cli, err := LoadPostgresqlClientByFrom(db.Database) + if err != nil { + return err + } + defer cli.Close() + backupInfo := pgclient.BackupInfo{ + Name: db.Name, + TargetDir: targetDir, + FileName: fileName, - Timeout: 300, + Timeout: 300, + } + return cli.Backup(backupInfo) } - if err := cli.Backup(backupInfo); err != nil { - return err + itemTask.AddSubTask(i18n.GetMsgByKey("TaskBackup"), backupDatabase, nil) + if parentTask != nil { + return backupDatabase(parentTask) } - return nil + + return itemTask.Execute() } -func handlePostgresqlRecover(req dto.CommonRecover, isRollback bool) error { - isOk := false - fileOp := files.NewFileOp() - if !fileOp.Stat(req.File) { - return buserr.WithName("ErrFileNotFound", req.File) - } +func handlePostgresqlRecover(req dto.CommonRecover, parentTask *task.Task, isRollback bool) error { + var ( + err error + itemTask *task.Task + ) dbInfo, err := postgresqlRepo.Get(commonRepo.WithByName(req.DetailName), postgresqlRepo.WithByPostgresqlName(req.Name)) if err != nil { return err } - cli, err := LoadPostgresqlClientByFrom(req.Name) - if err != nil { - return err - } - defer cli.Close() - - if !isRollback { - rollbackFile := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("database/%s/%s_%s.sql.gz", req.Type, req.DetailName, time.Now().Format(constant.DateTimeSlimLayout))) - if err := cli.Backup(client.BackupInfo{ - Name: req.DetailName, - TargetDir: path.Dir(rollbackFile), - FileName: path.Base(rollbackFile), - - Timeout: 300, - }); err != nil { - return fmt.Errorf("backup postgresql db %s for rollback before recover failed, err: %v", req.DetailName, err) + itemTask = parentTask + if parentTask == nil { + itemTask, err = task.NewTaskWithOps("Redis", task.TaskRecover, task.TaskScopeDatabase, req.TaskID, dbInfo.ID) + if err != nil { + return err } - defer func() { - if !isOk { - global.LOG.Info("recover failed, start to rollback now") - if err := cli.Recover(client.RecoverInfo{ - Name: req.DetailName, - SourceFile: rollbackFile, + } - Timeout: 300, - }); err != nil { - global.LOG.Errorf("rollback postgresql db %s from %s failed, err: %v", req.DetailName, rollbackFile, err) - } - global.LOG.Infof("rollback postgresql db %s from %s successful", req.DetailName, rollbackFile) - _ = os.RemoveAll(rollbackFile) - } else { - _ = os.RemoveAll(rollbackFile) + recoverDatabase := func(t *task.Task) error { + isOk := false + fileOp := files.NewFileOp() + if !fileOp.Stat(req.File) { + return buserr.WithName("ErrFileNotFound", req.File) + } + + cli, err := LoadPostgresqlClientByFrom(req.Name) + if err != nil { + return err + } + defer cli.Close() + + if !isRollback { + rollbackFile := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("database/%s/%s_%s.sql.gz", req.Type, req.DetailName, time.Now().Format(constant.DateTimeSlimLayout))) + if err := cli.Backup(client.BackupInfo{ + Name: req.DetailName, + TargetDir: path.Dir(rollbackFile), + FileName: path.Base(rollbackFile), + + Timeout: 300, + }); err != nil { + return fmt.Errorf("backup postgresql db %s for rollback before recover failed, err: %v", req.DetailName, err) } - }() + defer func() { + if !isOk { + global.LOG.Info("recover failed, start to rollback now") + if err := cli.Recover(client.RecoverInfo{ + Name: req.DetailName, + SourceFile: rollbackFile, + + Timeout: 300, + }); err != nil { + global.LOG.Errorf("rollback postgresql db %s from %s failed, err: %v", req.DetailName, rollbackFile, err) + } + global.LOG.Infof("rollback postgresql db %s from %s successful", req.DetailName, rollbackFile) + _ = os.RemoveAll(rollbackFile) + } else { + _ = os.RemoveAll(rollbackFile) + } + }() + } + if err := cli.Recover(client.RecoverInfo{ + Name: req.DetailName, + SourceFile: req.File, + Username: dbInfo.Username, + Timeout: 300, + }); err != nil { + return err + } + isOk = true + return nil } - if err := cli.Recover(client.RecoverInfo{ - Name: req.DetailName, - SourceFile: req.File, - Username: dbInfo.Username, - Timeout: 300, - }); err != nil { - return err + itemTask.AddSubTask(i18n.GetMsgByKey("TaskRecover"), recoverDatabase, nil) + if parentTask != nil { + return recoverDatabase(parentTask) } - isOk = true - return nil + + return itemTask.Execute() } diff --git a/agent/app/service/backup_redis.go b/agent/app/service/backup_redis.go index 699d3c863..0ac23dc86 100644 --- a/agent/app/service/backup_redis.go +++ b/agent/app/service/backup_redis.go @@ -10,9 +10,11 @@ import ( "github.com/1Panel-dev/1Panel/agent/app/dto" "github.com/1Panel-dev/1Panel/agent/app/model" "github.com/1Panel-dev/1Panel/agent/app/repo" + "github.com/1Panel-dev/1Panel/agent/app/task" "github.com/1Panel-dev/1Panel/agent/buserr" "github.com/1Panel-dev/1Panel/agent/constant" "github.com/1Panel-dev/1Panel/agent/global" + "github.com/1Panel-dev/1Panel/agent/i18n" "github.com/1Panel-dev/1Panel/agent/utils/cmd" "github.com/1Panel-dev/1Panel/agent/utils/common" "github.com/1Panel-dev/1Panel/agent/utils/compose" @@ -20,8 +22,8 @@ import ( "github.com/pkg/errors" ) -func (u *BackupService) RedisBackup(db dto.CommonBackup) error { - redisInfo, err := appInstallRepo.LoadBaseInfo("redis", db.Name) +func (u *BackupService) RedisBackup(req dto.CommonBackup) error { + redisInfo, err := appInstallRepo.LoadBaseInfo("redis", req.Name) if err != nil { return err } @@ -42,12 +44,12 @@ func (u *BackupService) RedisBackup(db dto.CommonBackup) error { } itemDir := fmt.Sprintf("database/redis/%s", redisInfo.Name) backupDir := path.Join(global.CONF.System.Backup, itemDir) - if err := handleRedisBackup(redisInfo, backupDir, fileName, db.Secret); err != nil { + if err := handleRedisBackup(redisInfo, nil, backupDir, fileName, req.Secret, req.TaskID); err != nil { return err } record := &model.BackupRecord{ Type: "redis", - Name: db.Name, + Name: req.Name, SourceAccountIDs: "1", DownloadAccountID: 1, FileDir: itemDir, @@ -66,125 +68,165 @@ func (u *BackupService) RedisRecover(req dto.CommonRecover) error { return err } global.LOG.Infof("recover redis from backup file %s", req.File) - if err := handleRedisRecover(redisInfo, req.File, false, req.Secret); err != nil { + if err := handleRedisRecover(redisInfo, nil, req.File, false, req.Secret, req.TaskID); err != nil { return err } return nil } -func handleRedisBackup(redisInfo *repo.RootInfo, backupDir, fileName string, secret string) error { - fileOp := files.NewFileOp() - if !fileOp.Stat(backupDir) { - if err := os.MkdirAll(backupDir, os.ModePerm); err != nil { - return fmt.Errorf("mkdir %s failed, err: %v", backupDir, err) - } - } - - stdout, err := cmd.Execf("docker exec %s redis-cli -a %s --no-auth-warning save", redisInfo.ContainerName, redisInfo.Password) - if err != nil { - return errors.New(string(stdout)) - } - - if strings.HasSuffix(fileName, ".tar.gz") { - redisDataDir := fmt.Sprintf("%s/%s/%s/data/appendonlydir", constant.AppInstallDir, "redis", redisInfo.Name) - if err := handleTar(redisDataDir, backupDir, fileName, "", secret); err != nil { +func handleRedisBackup(redisInfo *repo.RootInfo, parentTask *task.Task, backupDir, fileName, secret, taskID string) error { + var ( + err error + itemTask *task.Task + ) + itemTask = parentTask + if parentTask == nil { + itemTask, err = task.NewTaskWithOps("Redis", task.TaskBackup, task.TaskScopeDatabase, taskID, redisInfo.ID) + if err != nil { return err } - return nil } - if strings.HasSuffix(fileName, ".aof") { - stdout1, err := cmd.Execf("docker cp %s:/data/appendonly.aof %s/%s", redisInfo.ContainerName, backupDir, fileName) + + backupDatabase := func(t *task.Task) error { + fileOp := files.NewFileOp() + if !fileOp.Stat(backupDir) { + if err := os.MkdirAll(backupDir, os.ModePerm); err != nil { + return fmt.Errorf("mkdir %s failed, err: %v", backupDir, err) + } + } + + stdout, err := cmd.Execf("docker exec %s redis-cli -a %s --no-auth-warning save", redisInfo.ContainerName, redisInfo.Password) if err != nil { + return errors.New(string(stdout)) + } + + if strings.HasSuffix(fileName, ".tar.gz") { + redisDataDir := fmt.Sprintf("%s/%s/%s/data/appendonlydir", constant.AppInstallDir, "redis", redisInfo.Name) + if err := handleTar(redisDataDir, backupDir, fileName, "", secret); err != nil { + return err + } + return nil + } + if strings.HasSuffix(fileName, ".aof") { + stdout1, err := cmd.Execf("docker cp %s:/data/appendonly.aof %s/%s", redisInfo.ContainerName, backupDir, fileName) + if err != nil { + return errors.New(string(stdout1)) + } + return nil + } + + stdout1, err1 := cmd.Execf("docker cp %s:/data/dump.rdb %s/%s", redisInfo.ContainerName, backupDir, fileName) + if err1 != nil { return errors.New(string(stdout1)) } return nil } - - stdout1, err1 := cmd.Execf("docker cp %s:/data/dump.rdb %s/%s", redisInfo.ContainerName, backupDir, fileName) - if err1 != nil { - return errors.New(string(stdout1)) + itemTask.AddSubTask(i18n.GetMsgByKey("TaskBackup"), backupDatabase, nil) + if parentTask != nil { + return backupDatabase(parentTask) } - return nil + + return itemTask.Execute() } -func handleRedisRecover(redisInfo *repo.RootInfo, recoverFile string, isRollback bool, secret string) error { - fileOp := files.NewFileOp() - if !fileOp.Stat(recoverFile) { - return buserr.WithName("ErrFileNotFound", recoverFile) - } - - appendonly, err := configGetStr(redisInfo.ContainerName, redisInfo.Password, "appendonly") - if err != nil { - return err - } - - if appendonly == "yes" { - if strings.HasPrefix(redisInfo.Version, "6.") && !strings.HasSuffix(recoverFile, ".aof") { - return buserr.New(constant.ErrTypeOfRedis) - } - if strings.HasPrefix(redisInfo.Version, "7.") && !strings.HasSuffix(recoverFile, ".tar.gz") { - return buserr.New(constant.ErrTypeOfRedis) - } - } else { - if !strings.HasSuffix(recoverFile, ".rdb") { - return buserr.New(constant.ErrTypeOfRedis) - } - } - - global.LOG.Infof("appendonly in redis conf is %s", appendonly) - isOk := false - if !isRollback { - suffix := "rdb" - if appendonly == "yes" { - if strings.HasPrefix(redisInfo.Version, "6.") { - suffix = "aof" - } else { - suffix = "tar.gz" - } - } - rollbackFile := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("database/redis/%s_%s.%s", redisInfo.Name, time.Now().Format(constant.DateTimeSlimLayout), suffix)) - if err := handleRedisBackup(redisInfo, path.Dir(rollbackFile), path.Base(rollbackFile), secret); err != nil { - return fmt.Errorf("backup database %s for rollback before recover failed, err: %v", redisInfo.Name, err) - } - defer func() { - if !isOk { - global.LOG.Info("recover failed, start to rollback now") - if err := handleRedisRecover(redisInfo, rollbackFile, true, secret); err != nil { - global.LOG.Errorf("rollback redis from %s failed, err: %v", rollbackFile, err) - return - } - global.LOG.Infof("rollback redis from %s successful", rollbackFile) - _ = os.RemoveAll(rollbackFile) - } else { - _ = os.RemoveAll(rollbackFile) - } - }() - } - composeDir := fmt.Sprintf("%s/redis/%s", constant.AppInstallDir, redisInfo.Name) - if _, err := compose.Down(composeDir + "/docker-compose.yml"); err != nil { - return err - } - if appendonly == "yes" && strings.HasPrefix(redisInfo.Version, "7.") { - redisDataDir := fmt.Sprintf("%s/%s/%s/data", constant.AppInstallDir, "redis", redisInfo.Name) - if err := handleUnTar(recoverFile, redisDataDir, secret); err != nil { - return err - } - } else { - itemName := "dump.rdb" - if appendonly == "yes" && strings.HasPrefix(redisInfo.Version, "6.") { - itemName = "appendonly.aof" - } - input, err := os.ReadFile(recoverFile) +func handleRedisRecover(redisInfo *repo.RootInfo, parentTask *task.Task, recoverFile string, isRollback bool, secret, taskID string) error { + var ( + err error + itemTask *task.Task + ) + itemTask = parentTask + if parentTask == nil { + itemTask, err = task.NewTaskWithOps("Redis", task.TaskRecover, task.TaskScopeDatabase, taskID, redisInfo.ID) if err != nil { return err } - if err = os.WriteFile(composeDir+"/data/"+itemName, input, 0640); err != nil { + } + + recoverDatabase := func(t *task.Task) error { + fileOp := files.NewFileOp() + if !fileOp.Stat(recoverFile) { + return buserr.WithName("ErrFileNotFound", recoverFile) + } + + appendonly, err := configGetStr(redisInfo.ContainerName, redisInfo.Password, "appendonly") + if err != nil { return err } + + if appendonly == "yes" { + if strings.HasPrefix(redisInfo.Version, "6.") && !strings.HasSuffix(recoverFile, ".aof") { + return buserr.New(constant.ErrTypeOfRedis) + } + if strings.HasPrefix(redisInfo.Version, "7.") && !strings.HasSuffix(recoverFile, ".tar.gz") { + return buserr.New(constant.ErrTypeOfRedis) + } + } else { + if !strings.HasSuffix(recoverFile, ".rdb") { + return buserr.New(constant.ErrTypeOfRedis) + } + } + + global.LOG.Infof("appendonly in redis conf is %s", appendonly) + isOk := false + if !isRollback { + suffix := "rdb" + if appendonly == "yes" { + if strings.HasPrefix(redisInfo.Version, "6.") { + suffix = "aof" + } else { + suffix = "tar.gz" + } + } + rollbackFile := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("database/redis/%s_%s.%s", redisInfo.Name, time.Now().Format(constant.DateTimeSlimLayout), suffix)) + if err := handleRedisBackup(redisInfo, nil, path.Dir(rollbackFile), path.Base(rollbackFile), secret, ""); err != nil { + return fmt.Errorf("backup database %s for rollback before recover failed, err: %v", redisInfo.Name, err) + } + defer func() { + if !isOk { + global.LOG.Info("recover failed, start to rollback now") + if err := handleRedisRecover(redisInfo, itemTask, rollbackFile, true, secret, ""); err != nil { + global.LOG.Errorf("rollback redis from %s failed, err: %v", rollbackFile, err) + return + } + global.LOG.Infof("rollback redis from %s successful", rollbackFile) + _ = os.RemoveAll(rollbackFile) + } else { + _ = os.RemoveAll(rollbackFile) + } + }() + } + composeDir := fmt.Sprintf("%s/redis/%s", constant.AppInstallDir, redisInfo.Name) + if _, err := compose.Down(composeDir + "/docker-compose.yml"); err != nil { + return err + } + if appendonly == "yes" && strings.HasPrefix(redisInfo.Version, "7.") { + redisDataDir := fmt.Sprintf("%s/%s/%s/data", constant.AppInstallDir, "redis", redisInfo.Name) + if err := handleUnTar(recoverFile, redisDataDir, secret); err != nil { + return err + } + } else { + itemName := "dump.rdb" + if appendonly == "yes" && strings.HasPrefix(redisInfo.Version, "6.") { + itemName = "appendonly.aof" + } + input, err := os.ReadFile(recoverFile) + if err != nil { + return err + } + if err = os.WriteFile(composeDir+"/data/"+itemName, input, 0640); err != nil { + return err + } + } + if _, err := compose.Up(composeDir + "/docker-compose.yml"); err != nil { + return err + } + isOk = true + return nil } - if _, err := compose.Up(composeDir + "/docker-compose.yml"); err != nil { - return err + itemTask.AddSubTask(i18n.GetMsgByKey("TaskRecover"), recoverDatabase, nil) + if parentTask != nil { + return recoverDatabase(parentTask) } - isOk = true - return nil + + return itemTask.Execute() } diff --git a/agent/app/service/backup_website.go b/agent/app/service/backup_website.go index bf5d42624..2e73e1e05 100644 --- a/agent/app/service/backup_website.go +++ b/agent/app/service/backup_website.go @@ -3,17 +3,18 @@ package service import ( "encoding/json" "fmt" - "github.com/1Panel-dev/1Panel/agent/app/task" - "github.com/1Panel-dev/1Panel/agent/i18n" - "github.com/1Panel-dev/1Panel/agent/utils/cmd" - "github.com/1Panel-dev/1Panel/agent/utils/compose" - "github.com/pkg/errors" "io/fs" "os" "path" "strings" "time" + "github.com/1Panel-dev/1Panel/agent/app/task" + "github.com/1Panel-dev/1Panel/agent/i18n" + "github.com/1Panel-dev/1Panel/agent/utils/cmd" + "github.com/1Panel-dev/1Panel/agent/utils/compose" + "github.com/pkg/errors" + "github.com/1Panel-dev/1Panel/agent/app/dto" "github.com/1Panel-dev/1Panel/agent/app/model" "github.com/1Panel-dev/1Panel/agent/buserr" @@ -310,7 +311,7 @@ func recoverWebsiteDatabase(t *task.Task, dbID uint, dbType, tmpPath, websiteKey Name: db.PostgresqlName, DetailName: db.Name, File: fmt.Sprintf("%s/%s.sql.gz", tmpPath, websiteKey), - }, true); err != nil { + }, t, true); err != nil { t.LogFailedWithErr(taskName, err) return err } @@ -326,7 +327,7 @@ func recoverWebsiteDatabase(t *task.Task, dbID uint, dbType, tmpPath, websiteKey Name: db.MysqlName, DetailName: db.Name, File: fmt.Sprintf("%s/%s.sql.gz", tmpPath, websiteKey), - }, true); err != nil { + }, t, true, ""); err != nil { t.LogFailedWithErr(taskName, err) return err } diff --git a/agent/app/service/cronjob_backup.go b/agent/app/service/cronjob_backup.go index d017159db..dea6e648b 100644 --- a/agent/app/service/cronjob_backup.go +++ b/agent/app/service/cronjob_backup.go @@ -109,11 +109,11 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Ti backupDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("database/%s/%s/%s", dbInfo.DBType, record.Name, dbInfo.Name)) record.FileName = fmt.Sprintf("db_%s_%s.sql.gz", dbInfo.Name, startTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5)) if cronjob.DBType == "mysql" || cronjob.DBType == "mariadb" { - if err := handleMysqlBackup(dbInfo.Database, dbInfo.DBType, dbInfo.Name, backupDir, record.FileName); err != nil { + if err := handleMysqlBackup(dbInfo, nil, backupDir, record.FileName, ""); err != nil { return err } } else { - if err := handlePostgresqlBackup(dbInfo.Database, dbInfo.Name, backupDir, record.FileName); err != nil { + if err := handlePostgresqlBackup(dbInfo, nil, backupDir, record.FileName, ""); err != nil { return err } } @@ -200,7 +200,7 @@ func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, startTime time.Ti var record model.BackupRecord record.From = "cronjob" - record.Type = "directory" + record.Type = "snapshot" record.CronjobID = cronjob.ID record.Name = cronjob.Name record.DownloadAccountID, record.SourceAccountIDs = cronjob.DownloadAccountID, cronjob.SourceAccountIDs @@ -227,19 +227,21 @@ func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, startTime time.Ti return nil } -type databaseHelper struct { +type DatabaseHelper struct { + ID uint DBType string Database string Name string } -func loadDbsForJob(cronjob model.Cronjob) []databaseHelper { - var dbs []databaseHelper +func loadDbsForJob(cronjob model.Cronjob) []DatabaseHelper { + var dbs []DatabaseHelper if cronjob.DBName == "all" { if cronjob.DBType == "mysql" || cronjob.DBType == "mariadb" { mysqlItems, _ := mysqlRepo.List() for _, mysql := range mysqlItems { - dbs = append(dbs, databaseHelper{ + dbs = append(dbs, DatabaseHelper{ + ID: mysql.ID, DBType: cronjob.DBType, Database: mysql.MysqlName, Name: mysql.Name, @@ -248,7 +250,8 @@ func loadDbsForJob(cronjob model.Cronjob) []databaseHelper { } else { pgItems, _ := postgresqlRepo.List() for _, pg := range pgItems { - dbs = append(dbs, databaseHelper{ + dbs = append(dbs, DatabaseHelper{ + ID: pg.ID, DBType: cronjob.DBType, Database: pg.PostgresqlName, Name: pg.Name, @@ -260,14 +263,16 @@ func loadDbsForJob(cronjob model.Cronjob) []databaseHelper { itemID, _ := strconv.Atoi(cronjob.DBName) if cronjob.DBType == "mysql" || cronjob.DBType == "mariadb" { mysqlItem, _ := mysqlRepo.Get(commonRepo.WithByID(uint(itemID))) - dbs = append(dbs, databaseHelper{ + dbs = append(dbs, DatabaseHelper{ + ID: mysqlItem.ID, DBType: cronjob.DBType, Database: mysqlItem.MysqlName, Name: mysqlItem.Name, }) } else { pgItem, _ := postgresqlRepo.Get(commonRepo.WithByID(uint(itemID))) - dbs = append(dbs, databaseHelper{ + dbs = append(dbs, DatabaseHelper{ + ID: pgItem.ID, DBType: cronjob.DBType, Database: pgItem.PostgresqlName, Name: pgItem.Name,