1Panel-dev/1Panel
1
1
Fork 0
mirror of https://github.com/1Panel-dev/1Panel.git synced 2025-10-06 05:24:33 +08:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: 计划任务备份数据库增加任务日志 (#6717)

Browse source
This commit is contained in:
ssongliu 2024-10-15 16:03:21 +08:00 • committed by GitHub
parent 07bdcdc91b
commit f3fd02fcb3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 396 additions and 258 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
agent/app/service/backup_app.go
View file

@ -4,14 +4,15 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/1Panel-dev/1Panel/agent/app/task"
"github.com/1Panel-dev/1Panel/agent/i18n"
"io/fs"
"os"
"path"
"strings"
"time"
"github.com/1Panel-dev/1Panel/agent/app/task"
"github.com/1Panel-dev/1Panel/agent/i18n"
"github.com/1Panel-dev/1Panel/agent/buserr"
"github.com/1Panel-dev/1Panel/agent/app/dto"
@ -103,7 +104,8 @@ func backupDatabaseWithTask(parentTask *task.Task, resourceKey, tmpDir, name str
return err
}
parentTask.LogStart(task.GetTaskName(db.Name, task.TaskBackup, task.TaskScopeDatabase))
if err := handleMysqlBackup(db.MysqlName, resourceKey, db.Name, tmpDir, fmt.Sprintf("%s.sql.gz", name)); err != nil {
databaseHelper := DatabaseHelper{Database: db.MysqlName, DBType: resourceKey, Name: db.Name}
if err := handleMysqlBackup(databaseHelper, parentTask, tmpDir, fmt.Sprintf("%s.sql.gz", name), ""); err != nil {
return err
}
parentTask.LogSuccess(task.GetTaskName(db.Name, task.TaskBackup, task.TaskScopeDatabase))
@ -113,7 +115,8 @@ func backupDatabaseWithTask(parentTask *task.Task, resourceKey, tmpDir, name str
return err
}
parentTask.LogStart(task.GetTaskName(db.Name, task.TaskBackup, task.TaskScopeDatabase))
if err := handlePostgresqlBackup(db.PostgresqlName, db.Name, tmpDir, fmt.Sprintf("%s.sql.gz", name)); err != nil {
databaseHelper := DatabaseHelper{Database: db.PostgresqlName, DBType: resourceKey, Name: db.Name}
if err := handlePostgresqlBackup(databaseHelper, parentTask, tmpDir, fmt.Sprintf("%s.sql.gz", name), ""); err != nil {
return err
}
parentTask.LogSuccess(task.GetTaskName(db.Name, task.TaskBackup, task.TaskScopeDatabase))
@ -257,7 +260,7 @@ func handleAppRecover(install *model.AppInstall, parentTask *task.Task, recoverF
Name: database.Name,
DetailName: db.Name,
File: fmt.Sprintf("%s/%s.sql.gz", tmpPath, install.Name),
}, true); err != nil {
}, parentTask, true); err != nil {
t.LogFailedWithErr(taskName, err)
return err
}
@ -286,7 +289,7 @@ func handleAppRecover(install *model.AppInstall, parentTask *task.Task, recoverF
Name: newDB.MysqlName,
DetailName: newDB.Name,
File: fmt.Sprintf("%s/%s.sql.gz", tmpPath, install.Name),
}, true); err != nil {
}, parentTask, true, ""); err != nil {
t.LogFailedWithErr(taskName, err)
return err
}

70
agent/app/service/backup_mysql.go
View file

@ -9,11 +9,13 @@ import (
"time"
"github.com/1Panel-dev/1Panel/agent/constant"
"github.com/1Panel-dev/1Panel/agent/i18n"
"github.com/1Panel-dev/1Panel/agent/buserr"
"github.com/1Panel-dev/1Panel/agent/app/dto"
"github.com/1Panel-dev/1Panel/agent/app/model"
"github.com/1Panel-dev/1Panel/agent/app/task"
"github.com/1Panel-dev/1Panel/agent/global"
"github.com/1Panel-dev/1Panel/agent/utils/common"
"github.com/1Panel-dev/1Panel/agent/utils/files"
@ -26,7 +28,8 @@ func (u *BackupService) MysqlBackup(req dto.CommonBackup) error {
targetDir := path.Join(global.CONF.System.Backup, itemDir)
fileName := fmt.Sprintf("%s_%s.sql.gz", req.DetailName, timeNow+common.RandStrAndNum(5))
if err := handleMysqlBackup(req.Name, req.Type, req.DetailName, targetDir, fileName); err != nil {
databaseHelper := DatabaseHelper{Database: req.Name, DBType: req.Type, Name: req.DetailName}
if err := handleMysqlBackup(databaseHelper, nil, targetDir, fileName, req.TaskID); err != nil {
return err
}
@ -46,7 +49,7 @@ func (u *BackupService) MysqlBackup(req dto.CommonBackup) error {
}
func (u *BackupService) MysqlRecover(req dto.CommonRecover) error {
if err := handleMysqlRecover(req, false); err != nil {
if err := handleMysqlRecover(req, nil, false, req.TaskID); err != nil {
return err
}
return nil
@ -90,26 +93,39 @@ func (u *BackupService) MysqlRecoverByUpload(req dto.CommonRecover) error {
}
req.File = path.Dir(file) + "/" + fileName
if err := handleMysqlRecover(req, false); err != nil {
if err := handleMysqlRecover(req, nil, false, req.TaskID); err != nil {
return err
}
global.LOG.Info("recover from uploads successful!")
return nil
}
func handleMysqlBackup(database, dbType, dbName, targetDir, fileName string) error {
dbInfo, err := mysqlRepo.Get(commonRepo.WithByName(dbName), mysqlRepo.WithByMysqlName(database))
func handleMysqlBackup(db DatabaseHelper, parentTask *task.Task, targetDir, fileName, taskID string) error {
var (
err error
itemTask *task.Task
)
itemTask = parentTask
dbInfo, err := mysqlRepo.Get(commonRepo.WithByName(db.Name), mysqlRepo.WithByMysqlName(db.Database))
if err != nil {
return err
}
cli, version, err := LoadMysqlClientByFrom(database)
itemName := fmt.Sprintf("%s[%s] - %s", db.Database, db.DBType, db.Name)
if parentTask == nil {
itemTask, err = task.NewTaskWithOps(itemName, task.TaskBackup, task.TaskScopeDatabase, taskID, dbInfo.ID)
if err != nil {
return err
}
}
backupDatabase := func(t *task.Task) error {
cli, version, err := LoadMysqlClientByFrom(db.Database)
if err != nil {
return err
}
backupInfo := client.BackupInfo{
Name: dbName,
Type: dbType,
Name: db.Name,
Type: db.DBType,
Version: version,
Format: dbInfo.Format,
TargetDir: targetDir,
@ -117,13 +133,36 @@ func handleMysqlBackup(database, dbType, dbName, targetDir, fileName string) err
Timeout: 300,
}
if err := cli.Backup(backupInfo); err != nil {
return err
return cli.Backup(backupInfo)
}
return nil
itemTask.AddSubTask(i18n.GetMsgByKey("TaskBackup"), backupDatabase, nil)
if parentTask != nil {
return backupDatabase(parentTask)
}
return itemTask.Execute()
}
func handleMysqlRecover(req dto.CommonRecover, isRollback bool) error {
func handleMysqlRecover(req dto.CommonRecover, parentTask *task.Task, isRollback bool, taskID string) error {
var (
err error
itemTask *task.Task
)
itemTask = parentTask
dbInfo, err := mysqlRepo.Get(commonRepo.WithByName(req.DetailName), mysqlRepo.WithByMysqlName(req.Name))
if err != nil {
return err
}
itemName := fmt.Sprintf("%s[%s] - %s", req.Name, req.Type, req.DetailName)
if parentTask == nil {
itemTask, err = task.NewTaskWithOps(itemName, task.TaskRecover, task.TaskScopeDatabase, taskID, dbInfo.ID)
if err != nil {
return err
}
}
recoverDatabase := func(t *task.Task) error {
isOk := false
fileOp := files.NewFileOp()
if !fileOp.Stat(req.File) {
@ -186,4 +225,11 @@ func handleMysqlRecover(req dto.CommonRecover, isRollback bool) error {
}
isOk = true
return nil
}
itemTask.AddSubTask(i18n.GetMsgByKey("TaskRecover"), recoverDatabase, nil)
if parentTask != nil {
return recoverDatabase(parentTask)
}
return itemTask.Execute()
}

71
agent/app/service/backup_postgresql.go
View file

@ -9,6 +9,7 @@ import (
"time"
"github.com/1Panel-dev/1Panel/agent/constant"
"github.com/1Panel-dev/1Panel/agent/i18n"
"github.com/1Panel-dev/1Panel/agent/buserr"
"github.com/1Panel-dev/1Panel/agent/utils/common"
@ -16,6 +17,7 @@ import (
"github.com/1Panel-dev/1Panel/agent/app/dto"
"github.com/1Panel-dev/1Panel/agent/app/model"
"github.com/1Panel-dev/1Panel/agent/app/task"
"github.com/1Panel-dev/1Panel/agent/global"
"github.com/1Panel-dev/1Panel/agent/utils/files"
"github.com/1Panel-dev/1Panel/agent/utils/postgresql/client"
@ -27,7 +29,8 @@ func (u *BackupService) PostgresqlBackup(req dto.CommonBackup) error {
targetDir := path.Join(global.CONF.System.Backup, itemDir)
fileName := fmt.Sprintf("%s_%s.sql.gz", req.DetailName, timeNow+common.RandStrAndNum(5))
if err := handlePostgresqlBackup(req.Name, req.DetailName, targetDir, fileName); err != nil {
databaseHelper := DatabaseHelper{Database: req.Name, DBType: req.Type, Name: req.DetailName}
if err := handlePostgresqlBackup(databaseHelper, nil, targetDir, fileName, req.TaskID); err != nil {
return err
}
@ -46,7 +49,7 @@ func (u *BackupService) PostgresqlBackup(req dto.CommonBackup) error {
return nil
}
func (u *BackupService) PostgresqlRecover(req dto.CommonRecover) error {
if err := handlePostgresqlRecover(req, false); err != nil {
if err := handlePostgresqlRecover(req, nil, false); err != nil {
return err
}
return nil
@ -90,42 +93,73 @@ func (u *BackupService) PostgresqlRecoverByUpload(req dto.CommonRecover) error {
}
req.File = path.Dir(file) + "/" + fileName
if err := handlePostgresqlRecover(req, false); err != nil {
if err := handlePostgresqlRecover(req, nil, false); err != nil {
return err
}
global.LOG.Info("recover from uploads successful!")
return nil
}
func handlePostgresqlBackup(database, dbName, targetDir, fileName string) error {
cli, err := LoadPostgresqlClientByFrom(database)
func handlePostgresqlBackup(db DatabaseHelper, parentTask *task.Task, targetDir, fileName, taskID string) error {
var (
err error
itemTask *task.Task
)
itemTask = parentTask
itemName := fmt.Sprintf("%s - %s", db.Database, db.Name)
if parentTask == nil {
itemTask, err = task.NewTaskWithOps(itemName, task.TaskBackup, task.TaskScopeDatabase, taskID, db.ID)
if err != nil {
return err
}
}
backupDatabase := func(t *task.Task) error {
cli, err := LoadPostgresqlClientByFrom(db.Database)
if err != nil {
return err
}
defer cli.Close()
backupInfo := pgclient.BackupInfo{
Name: dbName,
Name: db.Name,
TargetDir: targetDir,
FileName: fileName,
Timeout: 300,
}
if err := cli.Backup(backupInfo); err != nil {
return err
return cli.Backup(backupInfo)
}
return nil
itemTask.AddSubTask(i18n.GetMsgByKey("TaskBackup"), backupDatabase, nil)
if parentTask != nil {
return backupDatabase(parentTask)
}
return itemTask.Execute()
}
func handlePostgresqlRecover(req dto.CommonRecover, isRollback bool) error {
func handlePostgresqlRecover(req dto.CommonRecover, parentTask *task.Task, isRollback bool) error {
var (
err error
itemTask *task.Task
)
dbInfo, err := postgresqlRepo.Get(commonRepo.WithByName(req.DetailName), postgresqlRepo.WithByPostgresqlName(req.Name))
if err != nil {
return err
}
itemTask = parentTask
if parentTask == nil {
itemTask, err = task.NewTaskWithOps("Redis", task.TaskRecover, task.TaskScopeDatabase, req.TaskID, dbInfo.ID)
if err != nil {
return err
}
}
recoverDatabase := func(t *task.Task) error {
isOk := false
fileOp := files.NewFileOp()
if !fileOp.Stat(req.File) {
return buserr.WithName("ErrFileNotFound", req.File)
}
dbInfo, err := postgresqlRepo.Get(commonRepo.WithByName(req.DetailName), postgresqlRepo.WithByPostgresqlName(req.Name))
if err != nil {
return err
}
cli, err := LoadPostgresqlClientByFrom(req.Name)
if err != nil {
return err
@ -171,4 +205,11 @@ func handlePostgresqlRecover(req dto.CommonRecover, isRollback bool) error {
}
isOk = true
return nil
}
itemTask.AddSubTask(i18n.GetMsgByKey("TaskRecover"), recoverDatabase, nil)
if parentTask != nil {
return recoverDatabase(parentTask)
}
return itemTask.Execute()
}

60
agent/app/service/backup_redis.go
View file

@ -10,9 +10,11 @@ import (
"github.com/1Panel-dev/1Panel/agent/app/dto"
"github.com/1Panel-dev/1Panel/agent/app/model"
"github.com/1Panel-dev/1Panel/agent/app/repo"
"github.com/1Panel-dev/1Panel/agent/app/task"
"github.com/1Panel-dev/1Panel/agent/buserr"
"github.com/1Panel-dev/1Panel/agent/constant"
"github.com/1Panel-dev/1Panel/agent/global"
"github.com/1Panel-dev/1Panel/agent/i18n"
"github.com/1Panel-dev/1Panel/agent/utils/cmd"
"github.com/1Panel-dev/1Panel/agent/utils/common"
"github.com/1Panel-dev/1Panel/agent/utils/compose"
@ -20,8 +22,8 @@ import (
"github.com/pkg/errors"
)
func (u *BackupService) RedisBackup(db dto.CommonBackup) error {
redisInfo, err := appInstallRepo.LoadBaseInfo("redis", db.Name)
func (u *BackupService) RedisBackup(req dto.CommonBackup) error {
redisInfo, err := appInstallRepo.LoadBaseInfo("redis", req.Name)
if err != nil {
return err
}
@ -42,12 +44,12 @@ func (u *BackupService) RedisBackup(db dto.CommonBackup) error {
}
itemDir := fmt.Sprintf("database/redis/%s", redisInfo.Name)
backupDir := path.Join(global.CONF.System.Backup, itemDir)
if err := handleRedisBackup(redisInfo, backupDir, fileName, db.Secret); err != nil {
if err := handleRedisBackup(redisInfo, nil, backupDir, fileName, req.Secret, req.TaskID); err != nil {
return err
}
record := &model.BackupRecord{
Type: "redis",
Name: db.Name,
Name: req.Name,
SourceAccountIDs: "1",
DownloadAccountID: 1,
FileDir: itemDir,
@ -66,13 +68,26 @@ func (u *BackupService) RedisRecover(req dto.CommonRecover) error {
return err
}
global.LOG.Infof("recover redis from backup file %s", req.File)
if err := handleRedisRecover(redisInfo, req.File, false, req.Secret); err != nil {
if err := handleRedisRecover(redisInfo, nil, req.File, false, req.Secret, req.TaskID); err != nil {
return err
}
return nil
}
func handleRedisBackup(redisInfo *repo.RootInfo, backupDir, fileName string, secret string) error {
func handleRedisBackup(redisInfo *repo.RootInfo, parentTask *task.Task, backupDir, fileName, secret, taskID string) error {
var (
err error
itemTask *task.Task
)
itemTask = parentTask
if parentTask == nil {
itemTask, err = task.NewTaskWithOps("Redis", task.TaskBackup, task.TaskScopeDatabase, taskID, redisInfo.ID)
if err != nil {
return err
}
}
backupDatabase := func(t *task.Task) error {
fileOp := files.NewFileOp()
if !fileOp.Stat(backupDir) {
if err := os.MkdirAll(backupDir, os.ModePerm); err != nil {
@ -105,9 +120,29 @@ func handleRedisBackup(redisInfo *repo.RootInfo, backupDir, fileName string, sec
return errors.New(string(stdout1))
}
return nil
}
itemTask.AddSubTask(i18n.GetMsgByKey("TaskBackup"), backupDatabase, nil)
if parentTask != nil {
return backupDatabase(parentTask)
}
return itemTask.Execute()
}
func handleRedisRecover(redisInfo *repo.RootInfo, recoverFile string, isRollback bool, secret string) error {
func handleRedisRecover(redisInfo *repo.RootInfo, parentTask *task.Task, recoverFile string, isRollback bool, secret, taskID string) error {
var (
err error
itemTask *task.Task
)
itemTask = parentTask
if parentTask == nil {
itemTask, err = task.NewTaskWithOps("Redis", task.TaskRecover, task.TaskScopeDatabase, taskID, redisInfo.ID)
if err != nil {
return err
}
}
recoverDatabase := func(t *task.Task) error {
fileOp := files.NewFileOp()
if !fileOp.Stat(recoverFile) {
return buserr.WithName("ErrFileNotFound", recoverFile)
@ -143,13 +178,13 @@ func handleRedisRecover(redisInfo *repo.RootInfo, recoverFile string, isRollback
}
}
rollbackFile := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("database/redis/%s_%s.%s", redisInfo.Name, time.Now().Format(constant.DateTimeSlimLayout), suffix))
if err := handleRedisBackup(redisInfo, path.Dir(rollbackFile), path.Base(rollbackFile), secret); err != nil {
if err := handleRedisBackup(redisInfo, nil, path.Dir(rollbackFile), path.Base(rollbackFile), secret, ""); err != nil {
return fmt.Errorf("backup database %s for rollback before recover failed, err: %v", redisInfo.Name, err)
}
defer func() {
if !isOk {
global.LOG.Info("recover failed, start to rollback now")
if err := handleRedisRecover(redisInfo, rollbackFile, true, secret); err != nil {
if err := handleRedisRecover(redisInfo, itemTask, rollbackFile, true, secret, ""); err != nil {
global.LOG.Errorf("rollback redis from %s failed, err: %v", rollbackFile, err)
return
}
@ -187,4 +222,11 @@ func handleRedisRecover(redisInfo *repo.RootInfo, recoverFile string, isRollback
}
isOk = true
return nil
}
itemTask.AddSubTask(i18n.GetMsgByKey("TaskRecover"), recoverDatabase, nil)
if parentTask != nil {
return recoverDatabase(parentTask)
}
return itemTask.Execute()
}

15
agent/app/service/backup_website.go
View file

@ -3,17 +3,18 @@ package service
import (
"encoding/json"
"fmt"
"github.com/1Panel-dev/1Panel/agent/app/task"
"github.com/1Panel-dev/1Panel/agent/i18n"
"github.com/1Panel-dev/1Panel/agent/utils/cmd"
"github.com/1Panel-dev/1Panel/agent/utils/compose"
"github.com/pkg/errors"
"io/fs"
"os"
"path"
"strings"
"time"
"github.com/1Panel-dev/1Panel/agent/app/task"
"github.com/1Panel-dev/1Panel/agent/i18n"
"github.com/1Panel-dev/1Panel/agent/utils/cmd"
"github.com/1Panel-dev/1Panel/agent/utils/compose"
"github.com/pkg/errors"
"github.com/1Panel-dev/1Panel/agent/app/dto"
"github.com/1Panel-dev/1Panel/agent/app/model"
"github.com/1Panel-dev/1Panel/agent/buserr"
@ -310,7 +311,7 @@ func recoverWebsiteDatabase(t *task.Task, dbID uint, dbType, tmpPath, websiteKey
Name: db.PostgresqlName,
DetailName: db.Name,
File: fmt.Sprintf("%s/%s.sql.gz", tmpPath, websiteKey),
}, true); err != nil {
}, t, true); err != nil {
t.LogFailedWithErr(taskName, err)
return err
}
@ -326,7 +327,7 @@ func recoverWebsiteDatabase(t *task.Task, dbID uint, dbType, tmpPath, websiteKey
Name: db.MysqlName,
DetailName: db.Name,
File: fmt.Sprintf("%s/%s.sql.gz", tmpPath, websiteKey),
}, true); err != nil {
}, t, true, ""); err != nil {
t.LogFailedWithErr(taskName, err)
return err
}

25
agent/app/service/cronjob_backup.go
View file

@ -109,11 +109,11 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Ti
backupDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("database/%s/%s/%s", dbInfo.DBType, record.Name, dbInfo.Name))
record.FileName = fmt.Sprintf("db_%s_%s.sql.gz", dbInfo.Name, startTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5))
if cronjob.DBType == "mysql" || cronjob.DBType == "mariadb" {
if err := handleMysqlBackup(dbInfo.Database, dbInfo.DBType, dbInfo.Name, backupDir, record.FileName); err != nil {
if err := handleMysqlBackup(dbInfo, nil, backupDir, record.FileName, ""); err != nil {
return err
}
} else {
if err := handlePostgresqlBackup(dbInfo.Database, dbInfo.Name, backupDir, record.FileName); err != nil {
if err := handlePostgresqlBackup(dbInfo, nil, backupDir, record.FileName, ""); err != nil {
return err
}
}
@ -200,7 +200,7 @@ func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, startTime time.Ti
var record model.BackupRecord
record.From = "cronjob"
record.Type = "directory"
record.Type = "snapshot"
record.CronjobID = cronjob.ID
record.Name = cronjob.Name
record.DownloadAccountID, record.SourceAccountIDs = cronjob.DownloadAccountID, cronjob.SourceAccountIDs
@ -227,19 +227,21 @@ func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, startTime time.Ti
return nil
}
type databaseHelper struct {
type DatabaseHelper struct {
ID uint
DBType string
Database string
Name string
}
func loadDbsForJob(cronjob model.Cronjob) []databaseHelper {
var dbs []databaseHelper
func loadDbsForJob(cronjob model.Cronjob) []DatabaseHelper {
var dbs []DatabaseHelper
if cronjob.DBName == "all" {
if cronjob.DBType == "mysql" || cronjob.DBType == "mariadb" {
mysqlItems, _ := mysqlRepo.List()
for _, mysql := range mysqlItems {
dbs = append(dbs, databaseHelper{
dbs = append(dbs, DatabaseHelper{
ID: mysql.ID,
DBType: cronjob.DBType,
Database: mysql.MysqlName,
Name: mysql.Name,
@ -248,7 +250,8 @@ func loadDbsForJob(cronjob model.Cronjob) []databaseHelper {
} else {
pgItems, _ := postgresqlRepo.List()
for _, pg := range pgItems {
dbs = append(dbs, databaseHelper{
dbs = append(dbs, DatabaseHelper{
ID: pg.ID,
DBType: cronjob.DBType,
Database: pg.PostgresqlName,
Name: pg.Name,
@ -260,14 +263,16 @@ func loadDbsForJob(cronjob model.Cronjob) []databaseHelper {
itemID, _ := strconv.Atoi(cronjob.DBName)
if cronjob.DBType == "mysql" || cronjob.DBType == "mariadb" {
mysqlItem, _ := mysqlRepo.Get(commonRepo.WithByID(uint(itemID)))
dbs = append(dbs, databaseHelper{
dbs = append(dbs, DatabaseHelper{
ID: mysqlItem.ID,
DBType: cronjob.DBType,
Database: mysqlItem.MysqlName,
Name: mysqlItem.Name,
})
} else {
pgItem, _ := postgresqlRepo.Get(commonRepo.WithByID(uint(itemID)))
dbs = append(dbs, databaseHelper{
dbs = append(dbs, DatabaseHelper{
ID: pgItem.ID,
DBType: cronjob.DBType,
Database: pgItem.PostgresqlName,
Name: pgItem.Name,