1Panel-dev/1Panel
1
1
Fork 0
mirror of https://github.com/1Panel-dev/1Panel.git synced 2025-02-19 12:23:54 +08:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: 计划任务备份数据库增加任务日志

Browse source
This commit is contained in:
ssongliu 2024-10-15 10:23:38 +08:00
parent 2565373db6
commit e33a2038ce
6 changed files with 396 additions and 258 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
agent/app/service/backup_app.go
View file

@ -4,14 +4,15 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/1Panel-dev/1Panel/agent/app/task"
"github.com/1Panel-dev/1Panel/agent/i18n"
"io/fs"
"os"
"path"
"strings"
"time"
"github.com/1Panel-dev/1Panel/agent/app/task"
"github.com/1Panel-dev/1Panel/agent/i18n"
"github.com/1Panel-dev/1Panel/agent/buserr"
"github.com/1Panel-dev/1Panel/agent/app/dto"
@ -103,7 +104,8 @@ func backupDatabaseWithTask(parentTask *task.Task, resourceKey, tmpDir, name str
return err
}
parentTask.LogStart(task.GetTaskName(db.Name, task.TaskBackup, task.TaskScopeDatabase))
if err := handleMysqlBackup(db.MysqlName, resourceKey, db.Name, tmpDir, fmt.Sprintf("%s.sql.gz", name)); err != nil {
databaseHelper := DatabaseHelper{Database: db.MysqlName, DBType: resourceKey, Name: db.Name}
if err := handleMysqlBackup(databaseHelper, parentTask, tmpDir, fmt.Sprintf("%s.sql.gz", name), ""); err != nil {
return err
}
parentTask.LogSuccess(task.GetTaskName(db.Name, task.TaskBackup, task.TaskScopeDatabase))
@ -113,7 +115,8 @@ func backupDatabaseWithTask(parentTask *task.Task, resourceKey, tmpDir, name str
return err
}
parentTask.LogStart(task.GetTaskName(db.Name, task.TaskBackup, task.TaskScopeDatabase))
if err := handlePostgresqlBackup(db.PostgresqlName, db.Name, tmpDir, fmt.Sprintf("%s.sql.gz", name)); err != nil {
databaseHelper := DatabaseHelper{Database: db.PostgresqlName, DBType: resourceKey, Name: db.Name}
if err := handlePostgresqlBackup(databaseHelper, parentTask, tmpDir, fmt.Sprintf("%s.sql.gz", name), ""); err != nil {
return err
}
parentTask.LogSuccess(task.GetTaskName(db.Name, task.TaskBackup, task.TaskScopeDatabase))
@ -257,7 +260,7 @@ func handleAppRecover(install *model.AppInstall, parentTask *task.Task, recoverF
Name: database.Name,
DetailName: db.Name,
File: fmt.Sprintf("%s/%s.sql.gz", tmpPath, install.Name),
}, true); err != nil {
}, parentTask, true); err != nil {
t.LogFailedWithErr(taskName, err)
return err
}
@ -286,7 +289,7 @@ func handleAppRecover(install *model.AppInstall, parentTask *task.Task, recoverF
Name: newDB.MysqlName,
DetailName: newDB.Name,
File: fmt.Sprintf("%s/%s.sql.gz", tmpPath, install.Name),
}, true); err != nil {
}, parentTask, true, ""); err != nil {
t.LogFailedWithErr(taskName, err)
return err
}

186
agent/app/service/backup_mysql.go
View file

@ -9,11 +9,13 @@ import (
"time"
"github.com/1Panel-dev/1Panel/agent/constant"
"github.com/1Panel-dev/1Panel/agent/i18n"
"github.com/1Panel-dev/1Panel/agent/buserr"
"github.com/1Panel-dev/1Panel/agent/app/dto"
"github.com/1Panel-dev/1Panel/agent/app/model"
"github.com/1Panel-dev/1Panel/agent/app/task"
"github.com/1Panel-dev/1Panel/agent/global"
"github.com/1Panel-dev/1Panel/agent/utils/common"
"github.com/1Panel-dev/1Panel/agent/utils/files"
@ -26,7 +28,8 @@ func (u *BackupService) MysqlBackup(req dto.CommonBackup) error {
targetDir := path.Join(global.CONF.System.Backup, itemDir)
fileName := fmt.Sprintf("%s_%s.sql.gz", req.DetailName, timeNow+common.RandStrAndNum(5))
if err := handleMysqlBackup(req.Name, req.Type, req.DetailName, targetDir, fileName); err != nil {
databaseHelper := DatabaseHelper{Database: req.Name, DBType: req.Type, Name: req.DetailName}
if err := handleMysqlBackup(databaseHelper, nil, targetDir, fileName, req.TaskID); err != nil {
return err
}
@ -46,7 +49,7 @@ func (u *BackupService) MysqlBackup(req dto.CommonBackup) error {
}
func (u *BackupService) MysqlRecover(req dto.CommonRecover) error {
if err := handleMysqlRecover(req, false); err != nil {
if err := handleMysqlRecover(req, nil, false, req.TaskID); err != nil {
return err
}
return nil
@ -90,100 +93,143 @@ func (u *BackupService) MysqlRecoverByUpload(req dto.CommonRecover) error {
}
req.File = path.Dir(file) + "/" + fileName
if err := handleMysqlRecover(req, false); err != nil {
if err := handleMysqlRecover(req, nil, false, req.TaskID); err != nil {
return err
}
global.LOG.Info("recover from uploads successful!")
return nil
}
func handleMysqlBackup(database, dbType, dbName, targetDir, fileName string) error {
dbInfo, err := mysqlRepo.Get(commonRepo.WithByName(dbName), mysqlRepo.WithByMysqlName(database))
func handleMysqlBackup(db DatabaseHelper, parentTask *task.Task, targetDir, fileName, taskID string) error {
var (
err error
itemTask *task.Task
)
itemTask = parentTask
dbInfo, err := mysqlRepo.Get(commonRepo.WithByName(db.Name), mysqlRepo.WithByMysqlName(db.Database))
if err != nil {
return err
}
cli, version, err := LoadMysqlClientByFrom(database)
if err != nil {
return err
itemName := fmt.Sprintf("%s[%s] - %s", db.Database, db.DBType, db.Name)
if parentTask == nil {
itemTask, err = task.NewTaskWithOps(itemName, task.TaskBackup, task.TaskScopeDatabase, taskID, dbInfo.ID)
if err != nil {
return err
}
}
backupInfo := client.BackupInfo{
Name: dbName,
Type: dbType,
Version: version,
Format: dbInfo.Format,
TargetDir: targetDir,
FileName: fileName,
backupDatabase := func(t *task.Task) error {
cli, version, err := LoadMysqlClientByFrom(db.Database)
if err != nil {
return err
}
backupInfo := client.BackupInfo{
Name: db.Name,
Type: db.DBType,
Version: version,
Format: dbInfo.Format,
TargetDir: targetDir,
FileName: fileName,
Timeout: 300,
Timeout: 300,
}
return cli.Backup(backupInfo)
}
if err := cli.Backup(backupInfo); err != nil {
return err
itemTask.AddSubTask(i18n.GetMsgByKey("TaskBackup"), backupDatabase, nil)
if parentTask != nil {
return backupDatabase(parentTask)
}
return nil
return itemTask.Execute()
}
func handleMysqlRecover(req dto.CommonRecover, isRollback bool) error {
isOk := false
fileOp := files.NewFileOp()
if !fileOp.Stat(req.File) {
return buserr.WithName("ErrFileNotFound", req.File)
}
func handleMysqlRecover(req dto.CommonRecover, parentTask *task.Task, isRollback bool, taskID string) error {
var (
err error
itemTask *task.Task
)
itemTask = parentTask
dbInfo, err := mysqlRepo.Get(commonRepo.WithByName(req.DetailName), mysqlRepo.WithByMysqlName(req.Name))
if err != nil {
return err
}
cli, version, err := LoadMysqlClientByFrom(req.Name)
if err != nil {
return err
itemName := fmt.Sprintf("%s[%s] - %s", req.Name, req.Type, req.DetailName)
if parentTask == nil {
itemTask, err = task.NewTaskWithOps(itemName, task.TaskRecover, task.TaskScopeDatabase, taskID, dbInfo.ID)
if err != nil {
return err
}
}
if !isRollback {
rollbackFile := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("database/%s/%s_%s.sql.gz", req.Type, req.DetailName, time.Now().Format(constant.DateTimeSlimLayout)))
if err := cli.Backup(client.BackupInfo{
Name: req.DetailName,
Type: req.Type,
Version: version,
Format: dbInfo.Format,
TargetDir: path.Dir(rollbackFile),
FileName: path.Base(rollbackFile),
recoverDatabase := func(t *task.Task) error {
isOk := false
fileOp := files.NewFileOp()
if !fileOp.Stat(req.File) {
return buserr.WithName("ErrFileNotFound", req.File)
}
dbInfo, err := mysqlRepo.Get(commonRepo.WithByName(req.DetailName), mysqlRepo.WithByMysqlName(req.Name))
if err != nil {
return err
}
cli, version, err := LoadMysqlClientByFrom(req.Name)
if err != nil {
return err
}
if !isRollback {
rollbackFile := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("database/%s/%s_%s.sql.gz", req.Type, req.DetailName, time.Now().Format(constant.DateTimeSlimLayout)))
if err := cli.Backup(client.BackupInfo{
Name: req.DetailName,
Type: req.Type,
Version: version,
Format: dbInfo.Format,
TargetDir: path.Dir(rollbackFile),
FileName: path.Base(rollbackFile),
Timeout: 300,
}); err != nil {
return fmt.Errorf("backup mysql db %s for rollback before recover failed, err: %v", req.DetailName, err)
}
defer func() {
if !isOk {
global.LOG.Info("recover failed, start to rollback now")
if err := cli.Recover(client.RecoverInfo{
Name: req.DetailName,
Type: req.Type,
Version: version,
Format: dbInfo.Format,
SourceFile: rollbackFile,
Timeout: 300,
}); err != nil {
global.LOG.Errorf("rollback mysql db %s from %s failed, err: %v", req.DetailName, rollbackFile, err)
}
global.LOG.Infof("rollback mysql db %s from %s successful", req.DetailName, rollbackFile)
_ = os.RemoveAll(rollbackFile)
} else {
_ = os.RemoveAll(rollbackFile)
}
}()
}
if err := cli.Recover(client.RecoverInfo{
Name: req.DetailName,
Type: req.Type,
Version: version,
Format: dbInfo.Format,
SourceFile: req.File,
Timeout: 300,
}); err != nil {
return fmt.Errorf("backup mysql db %s for rollback before recover failed, err: %v", req.DetailName, err)
return err
}
defer func() {
if !isOk {
global.LOG.Info("recover failed, start to rollback now")
if err := cli.Recover(client.RecoverInfo{
Name: req.DetailName,
Type: req.Type,
Version: version,
Format: dbInfo.Format,
SourceFile: rollbackFile,
Timeout: 300,
}); err != nil {
global.LOG.Errorf("rollback mysql db %s from %s failed, err: %v", req.DetailName, rollbackFile, err)
}
global.LOG.Infof("rollback mysql db %s from %s successful", req.DetailName, rollbackFile)
_ = os.RemoveAll(rollbackFile)
} else {
_ = os.RemoveAll(rollbackFile)
}
}()
isOk = true
return nil
}
if err := cli.Recover(client.RecoverInfo{
Name: req.DetailName,
Type: req.Type,
Version: version,
Format: dbInfo.Format,
SourceFile: req.File,
Timeout: 300,
}); err != nil {
return err
itemTask.AddSubTask(i18n.GetMsgByKey("TaskRecover"), recoverDatabase, nil)
if parentTask != nil {
return recoverDatabase(parentTask)
}
isOk = true
return nil
return itemTask.Execute()
}

167
agent/app/service/backup_postgresql.go
View file

@ -9,6 +9,7 @@ import (
"time"
"github.com/1Panel-dev/1Panel/agent/constant"
"github.com/1Panel-dev/1Panel/agent/i18n"
"github.com/1Panel-dev/1Panel/agent/buserr"
"github.com/1Panel-dev/1Panel/agent/utils/common"
@ -16,6 +17,7 @@ import (
"github.com/1Panel-dev/1Panel/agent/app/dto"
"github.com/1Panel-dev/1Panel/agent/app/model"
"github.com/1Panel-dev/1Panel/agent/app/task"
"github.com/1Panel-dev/1Panel/agent/global"
"github.com/1Panel-dev/1Panel/agent/utils/files"
"github.com/1Panel-dev/1Panel/agent/utils/postgresql/client"
@ -27,7 +29,8 @@ func (u *BackupService) PostgresqlBackup(req dto.CommonBackup) error {
targetDir := path.Join(global.CONF.System.Backup, itemDir)
fileName := fmt.Sprintf("%s_%s.sql.gz", req.DetailName, timeNow+common.RandStrAndNum(5))
if err := handlePostgresqlBackup(req.Name, req.DetailName, targetDir, fileName); err != nil {
databaseHelper := DatabaseHelper{Database: req.Name, DBType: req.Type, Name: req.DetailName}
if err := handlePostgresqlBackup(databaseHelper, nil, targetDir, fileName, req.TaskID); err != nil {
return err
}
@ -46,7 +49,7 @@ func (u *BackupService) PostgresqlBackup(req dto.CommonBackup) error {
return nil
}
func (u *BackupService) PostgresqlRecover(req dto.CommonRecover) error {
if err := handlePostgresqlRecover(req, false); err != nil {
if err := handlePostgresqlRecover(req, nil, false); err != nil {
return err
}
return nil
@ -90,85 +93,123 @@ func (u *BackupService) PostgresqlRecoverByUpload(req dto.CommonRecover) error {
}
req.File = path.Dir(file) + "/" + fileName
if err := handlePostgresqlRecover(req, false); err != nil {
if err := handlePostgresqlRecover(req, nil, false); err != nil {
return err
}
global.LOG.Info("recover from uploads successful!")
return nil
}
func handlePostgresqlBackup(database, dbName, targetDir, fileName string) error {
cli, err := LoadPostgresqlClientByFrom(database)
if err != nil {
return err
}
defer cli.Close()
backupInfo := pgclient.BackupInfo{
Name: dbName,
TargetDir: targetDir,
FileName: fileName,
func handlePostgresqlBackup(db DatabaseHelper, parentTask *task.Task, targetDir, fileName, taskID string) error {
var (
err error
itemTask *task.Task
)
itemTask = parentTask
itemName := fmt.Sprintf("%s - %s", db.Database, db.Name)
if parentTask == nil {
itemTask, err = task.NewTaskWithOps(itemName, task.TaskBackup, task.TaskScopeDatabase, taskID, db.ID)
if err != nil {
return err
}
}
backupDatabase := func(t *task.Task) error {
cli, err := LoadPostgresqlClientByFrom(db.Database)
if err != nil {
return err
}
defer cli.Close()
backupInfo := pgclient.BackupInfo{
Name: db.Name,
TargetDir: targetDir,
FileName: fileName,
Timeout: 300,
Timeout: 300,
}
return cli.Backup(backupInfo)
}
if err := cli.Backup(backupInfo); err != nil {
return err
itemTask.AddSubTask(i18n.GetMsgByKey("TaskBackup"), backupDatabase, nil)
if parentTask != nil {
return backupDatabase(parentTask)
}
return nil
return itemTask.Execute()
}
func handlePostgresqlRecover(req dto.CommonRecover, isRollback bool) error {
isOk := false
fileOp := files.NewFileOp()
if !fileOp.Stat(req.File) {
return buserr.WithName("ErrFileNotFound", req.File)
}
func handlePostgresqlRecover(req dto.CommonRecover, parentTask *task.Task, isRollback bool) error {
var (
err error
itemTask *task.Task
)
dbInfo, err := postgresqlRepo.Get(commonRepo.WithByName(req.DetailName), postgresqlRepo.WithByPostgresqlName(req.Name))
if err != nil {
return err
}
cli, err := LoadPostgresqlClientByFrom(req.Name)
if err != nil {
return err
}
defer cli.Close()
if !isRollback {
rollbackFile := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("database/%s/%s_%s.sql.gz", req.Type, req.DetailName, time.Now().Format(constant.DateTimeSlimLayout)))
if err := cli.Backup(client.BackupInfo{
Name: req.DetailName,
TargetDir: path.Dir(rollbackFile),
FileName: path.Base(rollbackFile),
Timeout: 300,
}); err != nil {
return fmt.Errorf("backup postgresql db %s for rollback before recover failed, err: %v", req.DetailName, err)
itemTask = parentTask
if parentTask == nil {
itemTask, err = task.NewTaskWithOps("Redis", task.TaskRecover, task.TaskScopeDatabase, req.TaskID, dbInfo.ID)
if err != nil {
return err
}
defer func() {
if !isOk {
global.LOG.Info("recover failed, start to rollback now")
if err := cli.Recover(client.RecoverInfo{
Name: req.DetailName,
SourceFile: rollbackFile,
}
Timeout: 300,
}); err != nil {
global.LOG.Errorf("rollback postgresql db %s from %s failed, err: %v", req.DetailName, rollbackFile, err)
}
global.LOG.Infof("rollback postgresql db %s from %s successful", req.DetailName, rollbackFile)
_ = os.RemoveAll(rollbackFile)
} else {
_ = os.RemoveAll(rollbackFile)
recoverDatabase := func(t *task.Task) error {
isOk := false
fileOp := files.NewFileOp()
if !fileOp.Stat(req.File) {
return buserr.WithName("ErrFileNotFound", req.File)
}
cli, err := LoadPostgresqlClientByFrom(req.Name)
if err != nil {
return err
}
defer cli.Close()
if !isRollback {
rollbackFile := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("database/%s/%s_%s.sql.gz", req.Type, req.DetailName, time.Now().Format(constant.DateTimeSlimLayout)))
if err := cli.Backup(client.BackupInfo{
Name: req.DetailName,
TargetDir: path.Dir(rollbackFile),
FileName: path.Base(rollbackFile),
Timeout: 300,
}); err != nil {
return fmt.Errorf("backup postgresql db %s for rollback before recover failed, err: %v", req.DetailName, err)
}
}()
defer func() {
if !isOk {
global.LOG.Info("recover failed, start to rollback now")
if err := cli.Recover(client.RecoverInfo{
Name: req.DetailName,
SourceFile: rollbackFile,
Timeout: 300,
}); err != nil {
global.LOG.Errorf("rollback postgresql db %s from %s failed, err: %v", req.DetailName, rollbackFile, err)
}
global.LOG.Infof("rollback postgresql db %s from %s successful", req.DetailName, rollbackFile)
_ = os.RemoveAll(rollbackFile)
} else {
_ = os.RemoveAll(rollbackFile)
}
}()
}
if err := cli.Recover(client.RecoverInfo{
Name: req.DetailName,
SourceFile: req.File,
Username: dbInfo.Username,
Timeout: 300,
}); err != nil {
return err
}
isOk = true
return nil
}
if err := cli.Recover(client.RecoverInfo{
Name: req.DetailName,
SourceFile: req.File,
Username: dbInfo.Username,
Timeout: 300,
}); err != nil {
return err
itemTask.AddSubTask(i18n.GetMsgByKey("TaskRecover"), recoverDatabase, nil)
if parentTask != nil {
return recoverDatabase(parentTask)
}
isOk = true
return nil
return itemTask.Execute()
}

246
agent/app/service/backup_redis.go
View file

@ -10,9 +10,11 @@ import (
"github.com/1Panel-dev/1Panel/agent/app/dto"
"github.com/1Panel-dev/1Panel/agent/app/model"
"github.com/1Panel-dev/1Panel/agent/app/repo"
"github.com/1Panel-dev/1Panel/agent/app/task"
"github.com/1Panel-dev/1Panel/agent/buserr"
"github.com/1Panel-dev/1Panel/agent/constant"
"github.com/1Panel-dev/1Panel/agent/global"
"github.com/1Panel-dev/1Panel/agent/i18n"
"github.com/1Panel-dev/1Panel/agent/utils/cmd"
"github.com/1Panel-dev/1Panel/agent/utils/common"
"github.com/1Panel-dev/1Panel/agent/utils/compose"
@ -20,8 +22,8 @@ import (
"github.com/pkg/errors"
)
func (u *BackupService) RedisBackup(db dto.CommonBackup) error {
redisInfo, err := appInstallRepo.LoadBaseInfo("redis", db.Name)
func (u *BackupService) RedisBackup(req dto.CommonBackup) error {
redisInfo, err := appInstallRepo.LoadBaseInfo("redis", req.Name)
if err != nil {
return err
}
@ -42,12 +44,12 @@ func (u *BackupService) RedisBackup(db dto.CommonBackup) error {
}
itemDir := fmt.Sprintf("database/redis/%s", redisInfo.Name)
backupDir := path.Join(global.CONF.System.Backup, itemDir)
if err := handleRedisBackup(redisInfo, backupDir, fileName, db.Secret); err != nil {
if err := handleRedisBackup(redisInfo, nil, backupDir, fileName, req.Secret, req.TaskID); err != nil {
return err
}
record := &model.BackupRecord{
Type: "redis",
Name: db.Name,
Name: req.Name,
SourceAccountIDs: "1",
DownloadAccountID: 1,
FileDir: itemDir,
@ -66,125 +68,165 @@ func (u *BackupService) RedisRecover(req dto.CommonRecover) error {
return err
}
global.LOG.Infof("recover redis from backup file %s", req.File)
if err := handleRedisRecover(redisInfo, req.File, false, req.Secret); err != nil {
if err := handleRedisRecover(redisInfo, nil, req.File, false, req.Secret, req.TaskID); err != nil {
return err
}
return nil
}
func handleRedisBackup(redisInfo *repo.RootInfo, backupDir, fileName string, secret string) error {
fileOp := files.NewFileOp()
if !fileOp.Stat(backupDir) {
if err := os.MkdirAll(backupDir, os.ModePerm); err != nil {
return fmt.Errorf("mkdir %s failed, err: %v", backupDir, err)
}
}
stdout, err := cmd.Execf("docker exec %s redis-cli -a %s --no-auth-warning save", redisInfo.ContainerName, redisInfo.Password)
if err != nil {
return errors.New(string(stdout))
}
if strings.HasSuffix(fileName, ".tar.gz") {
redisDataDir := fmt.Sprintf("%s/%s/%s/data/appendonlydir", constant.AppInstallDir, "redis", redisInfo.Name)
if err := handleTar(redisDataDir, backupDir, fileName, "", secret); err != nil {
func handleRedisBackup(redisInfo *repo.RootInfo, parentTask *task.Task, backupDir, fileName, secret, taskID string) error {
var (
err error
itemTask *task.Task
)
itemTask = parentTask
if parentTask == nil {
itemTask, err = task.NewTaskWithOps("Redis", task.TaskBackup, task.TaskScopeDatabase, taskID, redisInfo.ID)
if err != nil {
return err
}
return nil
}
if strings.HasSuffix(fileName, ".aof") {
stdout1, err := cmd.Execf("docker cp %s:/data/appendonly.aof %s/%s", redisInfo.ContainerName, backupDir, fileName)
backupDatabase := func(t *task.Task) error {
fileOp := files.NewFileOp()
if !fileOp.Stat(backupDir) {
if err := os.MkdirAll(backupDir, os.ModePerm); err != nil {
return fmt.Errorf("mkdir %s failed, err: %v", backupDir, err)
}
}
stdout, err := cmd.Execf("docker exec %s redis-cli -a %s --no-auth-warning save", redisInfo.ContainerName, redisInfo.Password)
if err != nil {
return errors.New(string(stdout))
}
if strings.HasSuffix(fileName, ".tar.gz") {
redisDataDir := fmt.Sprintf("%s/%s/%s/data/appendonlydir", constant.AppInstallDir, "redis", redisInfo.Name)
if err := handleTar(redisDataDir, backupDir, fileName, "", secret); err != nil {
return err
}
return nil
}
if strings.HasSuffix(fileName, ".aof") {
stdout1, err := cmd.Execf("docker cp %s:/data/appendonly.aof %s/%s", redisInfo.ContainerName, backupDir, fileName)
if err != nil {
return errors.New(string(stdout1))
}
return nil
}
stdout1, err1 := cmd.Execf("docker cp %s:/data/dump.rdb %s/%s", redisInfo.ContainerName, backupDir, fileName)
if err1 != nil {
return errors.New(string(stdout1))
}
return nil
}
stdout1, err1 := cmd.Execf("docker cp %s:/data/dump.rdb %s/%s", redisInfo.ContainerName, backupDir, fileName)
if err1 != nil {
return errors.New(string(stdout1))
itemTask.AddSubTask(i18n.GetMsgByKey("TaskBackup"), backupDatabase, nil)
if parentTask != nil {
return backupDatabase(parentTask)
}
return nil
return itemTask.Execute()
}
func handleRedisRecover(redisInfo *repo.RootInfo, recoverFile string, isRollback bool, secret string) error {
fileOp := files.NewFileOp()
if !fileOp.Stat(recoverFile) {
return buserr.WithName("ErrFileNotFound", recoverFile)
}
appendonly, err := configGetStr(redisInfo.ContainerName, redisInfo.Password, "appendonly")
if err != nil {
return err
}
if appendonly == "yes" {
if strings.HasPrefix(redisInfo.Version, "6.") && !strings.HasSuffix(recoverFile, ".aof") {
return buserr.New(constant.ErrTypeOfRedis)
}
if strings.HasPrefix(redisInfo.Version, "7.") && !strings.HasSuffix(recoverFile, ".tar.gz") {
return buserr.New(constant.ErrTypeOfRedis)
}
} else {
if !strings.HasSuffix(recoverFile, ".rdb") {
return buserr.New(constant.ErrTypeOfRedis)
}
}
global.LOG.Infof("appendonly in redis conf is %s", appendonly)
isOk := false
if !isRollback {
suffix := "rdb"
if appendonly == "yes" {
if strings.HasPrefix(redisInfo.Version, "6.") {
suffix = "aof"
} else {
suffix = "tar.gz"
}
}
rollbackFile := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("database/redis/%s_%s.%s", redisInfo.Name, time.Now().Format(constant.DateTimeSlimLayout), suffix))
if err := handleRedisBackup(redisInfo, path.Dir(rollbackFile), path.Base(rollbackFile), secret); err != nil {
return fmt.Errorf("backup database %s for rollback before recover failed, err: %v", redisInfo.Name, err)
}
defer func() {
if !isOk {
global.LOG.Info("recover failed, start to rollback now")
if err := handleRedisRecover(redisInfo, rollbackFile, true, secret); err != nil {
global.LOG.Errorf("rollback redis from %s failed, err: %v", rollbackFile, err)
return
}
global.LOG.Infof("rollback redis from %s successful", rollbackFile)
_ = os.RemoveAll(rollbackFile)
} else {
_ = os.RemoveAll(rollbackFile)
}
}()
}
composeDir := fmt.Sprintf("%s/redis/%s", constant.AppInstallDir, redisInfo.Name)
if _, err := compose.Down(composeDir + "/docker-compose.yml"); err != nil {
return err
}
if appendonly == "yes" && strings.HasPrefix(redisInfo.Version, "7.") {
redisDataDir := fmt.Sprintf("%s/%s/%s/data", constant.AppInstallDir, "redis", redisInfo.Name)
if err := handleUnTar(recoverFile, redisDataDir, secret); err != nil {
return err
}
} else {
itemName := "dump.rdb"
if appendonly == "yes" && strings.HasPrefix(redisInfo.Version, "6.") {
itemName = "appendonly.aof"
}
input, err := os.ReadFile(recoverFile)
func handleRedisRecover(redisInfo *repo.RootInfo, parentTask *task.Task, recoverFile string, isRollback bool, secret, taskID string) error {
var (
err error
itemTask *task.Task
)
itemTask = parentTask
if parentTask == nil {
itemTask, err = task.NewTaskWithOps("Redis", task.TaskRecover, task.TaskScopeDatabase, taskID, redisInfo.ID)
if err != nil {
return err
}
if err = os.WriteFile(composeDir+"/data/"+itemName, input, 0640); err != nil {
}
recoverDatabase := func(t *task.Task) error {
fileOp := files.NewFileOp()
if !fileOp.Stat(recoverFile) {
return buserr.WithName("ErrFileNotFound", recoverFile)
}
appendonly, err := configGetStr(redisInfo.ContainerName, redisInfo.Password, "appendonly")
if err != nil {
return err
}
if appendonly == "yes" {
if strings.HasPrefix(redisInfo.Version, "6.") && !strings.HasSuffix(recoverFile, ".aof") {
return buserr.New(constant.ErrTypeOfRedis)
}
if strings.HasPrefix(redisInfo.Version, "7.") && !strings.HasSuffix(recoverFile, ".tar.gz") {
return buserr.New(constant.ErrTypeOfRedis)
}
} else {
if !strings.HasSuffix(recoverFile, ".rdb") {
return buserr.New(constant.ErrTypeOfRedis)
}
}
global.LOG.Infof("appendonly in redis conf is %s", appendonly)
isOk := false
if !isRollback {
suffix := "rdb"
if appendonly == "yes" {
if strings.HasPrefix(redisInfo.Version, "6.") {
suffix = "aof"
} else {
suffix = "tar.gz"
}
}
rollbackFile := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("database/redis/%s_%s.%s", redisInfo.Name, time.Now().Format(constant.DateTimeSlimLayout), suffix))
if err := handleRedisBackup(redisInfo, nil, path.Dir(rollbackFile), path.Base(rollbackFile), secret, ""); err != nil {
return fmt.Errorf("backup database %s for rollback before recover failed, err: %v", redisInfo.Name, err)
}
defer func() {
if !isOk {
global.LOG.Info("recover failed, start to rollback now")
if err := handleRedisRecover(redisInfo, itemTask, rollbackFile, true, secret, ""); err != nil {
global.LOG.Errorf("rollback redis from %s failed, err: %v", rollbackFile, err)
return
}
global.LOG.Infof("rollback redis from %s successful", rollbackFile)
_ = os.RemoveAll(rollbackFile)
} else {
_ = os.RemoveAll(rollbackFile)
}
}()
}
composeDir := fmt.Sprintf("%s/redis/%s", constant.AppInstallDir, redisInfo.Name)
if _, err := compose.Down(composeDir + "/docker-compose.yml"); err != nil {
return err
}
if appendonly == "yes" && strings.HasPrefix(redisInfo.Version, "7.") {
redisDataDir := fmt.Sprintf("%s/%s/%s/data", constant.AppInstallDir, "redis", redisInfo.Name)
if err := handleUnTar(recoverFile, redisDataDir, secret); err != nil {
return err
}
} else {
itemName := "dump.rdb"
if appendonly == "yes" && strings.HasPrefix(redisInfo.Version, "6.") {
itemName = "appendonly.aof"
}
input, err := os.ReadFile(recoverFile)
if err != nil {
return err
}
if err = os.WriteFile(composeDir+"/data/"+itemName, input, 0640); err != nil {
return err
}
}
if _, err := compose.Up(composeDir + "/docker-compose.yml"); err != nil {
return err
}
isOk = true
return nil
}
if _, err := compose.Up(composeDir + "/docker-compose.yml"); err != nil {
return err
itemTask.AddSubTask(i18n.GetMsgByKey("TaskRecover"), recoverDatabase, nil)
if parentTask != nil {
return recoverDatabase(parentTask)
}
isOk = true
return nil
return itemTask.Execute()
}

15
agent/app/service/backup_website.go
View file

@ -3,17 +3,18 @@ package service
import (
"encoding/json"
"fmt"
"github.com/1Panel-dev/1Panel/agent/app/task"
"github.com/1Panel-dev/1Panel/agent/i18n"
"github.com/1Panel-dev/1Panel/agent/utils/cmd"
"github.com/1Panel-dev/1Panel/agent/utils/compose"
"github.com/pkg/errors"
"io/fs"
"os"
"path"
"strings"
"time"
"github.com/1Panel-dev/1Panel/agent/app/task"
"github.com/1Panel-dev/1Panel/agent/i18n"
"github.com/1Panel-dev/1Panel/agent/utils/cmd"
"github.com/1Panel-dev/1Panel/agent/utils/compose"
"github.com/pkg/errors"
"github.com/1Panel-dev/1Panel/agent/app/dto"
"github.com/1Panel-dev/1Panel/agent/app/model"
"github.com/1Panel-dev/1Panel/agent/buserr"
@ -310,7 +311,7 @@ func recoverWebsiteDatabase(t *task.Task, dbID uint, dbType, tmpPath, websiteKey
Name: db.PostgresqlName,
DetailName: db.Name,
File: fmt.Sprintf("%s/%s.sql.gz", tmpPath, websiteKey),
}, true); err != nil {
}, t, true); err != nil {
t.LogFailedWithErr(taskName, err)
return err
}
@ -326,7 +327,7 @@ func recoverWebsiteDatabase(t *task.Task, dbID uint, dbType, tmpPath, websiteKey
Name: db.MysqlName,
DetailName: db.Name,
File: fmt.Sprintf("%s/%s.sql.gz", tmpPath, websiteKey),
}, true); err != nil {
}, t, true, ""); err != nil {
t.LogFailedWithErr(taskName, err)
return err
}

25
agent/app/service/cronjob_backup.go
View file

@ -109,11 +109,11 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Ti
backupDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("database/%s/%s/%s", dbInfo.DBType, record.Name, dbInfo.Name))
record.FileName = fmt.Sprintf("db_%s_%s.sql.gz", dbInfo.Name, startTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5))
if cronjob.DBType == "mysql" || cronjob.DBType == "mariadb" {
if err := handleMysqlBackup(dbInfo.Database, dbInfo.DBType, dbInfo.Name, backupDir, record.FileName); err != nil {
if err := handleMysqlBackup(dbInfo, nil, backupDir, record.FileName, ""); err != nil {
return err
}
} else {
if err := handlePostgresqlBackup(dbInfo.Database, dbInfo.Name, backupDir, record.FileName); err != nil {
if err := handlePostgresqlBackup(dbInfo, nil, backupDir, record.FileName, ""); err != nil {
return err
}
}
@ -200,7 +200,7 @@ func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, startTime time.Ti
var record model.BackupRecord
record.From = "cronjob"
record.Type = "directory"
record.Type = "snapshot"
record.CronjobID = cronjob.ID
record.Name = cronjob.Name
record.DownloadAccountID, record.SourceAccountIDs = cronjob.DownloadAccountID, cronjob.SourceAccountIDs
@ -227,19 +227,21 @@ func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, startTime time.Ti
return nil
}
type databaseHelper struct {
type DatabaseHelper struct {
ID uint
DBType string
Database string
Name string
}
func loadDbsForJob(cronjob model.Cronjob) []databaseHelper {
var dbs []databaseHelper
func loadDbsForJob(cronjob model.Cronjob) []DatabaseHelper {
var dbs []DatabaseHelper
if cronjob.DBName == "all" {
if cronjob.DBType == "mysql" || cronjob.DBType == "mariadb" {
mysqlItems, _ := mysqlRepo.List()
for _, mysql := range mysqlItems {
dbs = append(dbs, databaseHelper{
dbs = append(dbs, DatabaseHelper{
ID: mysql.ID,
DBType: cronjob.DBType,
Database: mysql.MysqlName,
Name: mysql.Name,
@ -248,7 +250,8 @@ func loadDbsForJob(cronjob model.Cronjob) []databaseHelper {
} else {
pgItems, _ := postgresqlRepo.List()
for _, pg := range pgItems {
dbs = append(dbs, databaseHelper{
dbs = append(dbs, DatabaseHelper{
ID: pg.ID,
DBType: cronjob.DBType,
Database: pg.PostgresqlName,
Name: pg.Name,
@ -260,14 +263,16 @@ func loadDbsForJob(cronjob model.Cronjob) []databaseHelper {
itemID, _ := strconv.Atoi(cronjob.DBName)
if cronjob.DBType == "mysql" || cronjob.DBType == "mariadb" {
mysqlItem, _ := mysqlRepo.Get(commonRepo.WithByID(uint(itemID)))
dbs = append(dbs, databaseHelper{
dbs = append(dbs, DatabaseHelper{
ID: mysqlItem.ID,
DBType: cronjob.DBType,
Database: mysqlItem.MysqlName,
Name: mysqlItem.Name,
})
} else {
pgItem, _ := postgresqlRepo.Get(commonRepo.WithByID(uint(itemID)))
dbs = append(dbs, databaseHelper{
dbs = append(dbs, DatabaseHelper{
ID: pgItem.ID,
DBType: cronjob.DBType,
Database: pgItem.PostgresqlName,
Name: pgItem.Name,