1Panel-dev/1Panel
1
1
Fork 0
mirror of https://github.com/1Panel-dev/1Panel.git synced 2024-09-20 15:45:57 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: 解决备份所有时，保留份数错误的问题 (#578)

Browse source
This commit is contained in:
ssongliu 2023-04-11 17:00:29 +08:00 committed by GitHub
parent ef16934952
commit e935fa128f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 111 additions and 121 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

186
backend/app/service/cronjob_helper.go
View file

@ -1,8 +1,10 @@
package service
import (
"context"
"fmt"
"os"
"path"
"strings"
"time"
@ -22,7 +24,6 @@ func (u *CronjobService) HandleJob(cronjob *model.Cronjob) {
err error
)
record := cronjobRepo.StartRecords(cronjob.ID, cronjob.KeepLocal, "")
logDir := fmt.Sprintf("%s/1panel/task/%s/%s", global.CONF.System.BaseDir, cronjob.Type, cronjob.Name)
go func() {
switch cronjob.Type {
case "shell":
@ -34,7 +35,7 @@ func (u *CronjobService) HandleJob(cronjob *model.Cronjob) {
err = errExec
}
message = []byte(stdout)
u.HandleRmExpired("LOCAL", logDir, cronjob, nil)
u.HandleRmExpired("LOCAL", "", cronjob, nil)
case "website":
record.File, err = u.HandleBackup(cronjob, record.StartTime)
case "database":
@ -53,7 +54,7 @@ func (u *CronjobService) HandleJob(cronjob *model.Cronjob) {
err = errCurl
}
message = []byte(stdout)
u.HandleRmExpired("LOCAL", logDir, cronjob, nil)
u.HandleRmExpired("LOCAL", "", cronjob, nil)
}
if err != nil {
cronjobRepo.EndRecords(record, constant.StatusFailed, err.Error(), string(message))
@ -86,27 +87,11 @@ func (u *CronjobService) HandleBackup(cronjob *model.Cronjob, startTime time.Tim
if err != nil {
return "", err
}
if err := u.handleDatabase(*cronjob, app, backup, startTime); err != nil {
return "", err
}
if cronjob.DBName != "all" {
if !cronjob.KeepLocal && backup.Type != "LOCAL" {
return fmt.Sprintf("database/mysql/%s/%s/db_%s_%s.sql.gz", app.Name, cronjob.DBName, cronjob.DBName, startTime.Format("20060102150405")), nil
}
return fmt.Sprintf("%s/database/mysql/%s/%s/db_%s_%s.sql.gz", localDir, app.Name, cronjob.DBName, cronjob.DBName, startTime.Format("20060102150405")), nil
}
return "", nil
paths, err := u.handleDatabase(*cronjob, app, backup, startTime)
return strings.Join(paths, ","), err
case "website":
if err := u.handleWebsite(*cronjob, backup, startTime); err != nil {
return "", err
}
if cronjob.DBName != "all" {
if !cronjob.KeepLocal && backup.Type != "LOCAL" {
return fmt.Sprintf("website/%s/website_%s_%s.tar.gz", cronjob.Website, cronjob.Website, startTime.Format("20060102150405")), nil
}
return fmt.Sprintf("%s/website/%s/website_%s_%s.tar.gz", localDir, cronjob.Website, cronjob.Website, startTime.Format("20060102150405")), nil
}
return "", nil
paths, err := u.handleWebsite(*cronjob, backup, startTime)
return strings.Join(paths, ","), err
default:
fileName := fmt.Sprintf("directory%s_%s.tar.gz", strings.ReplaceAll(cronjob.SourceDir, "/", "_"), startTime.Format("20060102150405"))
backupDir := fmt.Sprintf("%s/%s/%s", localDir, cronjob.Type, cronjob.Name)
@ -115,25 +100,24 @@ func (u *CronjobService) HandleBackup(cronjob *model.Cronjob, startTime time.Tim
if err := handleTar(cronjob.SourceDir, backupDir, fileName, cronjob.ExclusionRules); err != nil {
return "", err
}
if backup.Type == "LOCAL" {
u.HandleRmExpired(backup.Type, backupDir, cronjob, nil)
return fmt.Sprintf("%s/%s", backupDir, fileName), nil
}
var client cloud_storage.CloudStorageClient
if backup.Type != "LOCAL" {
if !cronjob.KeepLocal {
defer func() {
_ = os.RemoveAll(fmt.Sprintf("%s/%s", backupDir, fileName))
}()
}
client, err := NewIBackupService().NewClient(&backup)
client, err = NewIBackupService().NewClient(&backup)
if err != nil {
return "", err
}
if _, err = client.Upload(backupDir+"/"+fileName, itemFileDir+"/"+fileName); err != nil {
return "", err
}
u.HandleRmExpired(backup.Type, itemFileDir, cronjob, client)
if cronjob.KeepLocal {
u.HandleRmExpired("LOCAL", backupDir, cronjob, client)
}
u.HandleRmExpired(backup.Type, localDir, cronjob, client)
if backup.Type == "LOCAL" || cronjob.KeepLocal {
return fmt.Sprintf("%s/%s/%s/%s", localDir, cronjob.Type, cronjob.Name, fileName), nil
}
return fmt.Sprintf("%s/%s/%s", cronjob.Type, cronjob.Name, fileName), nil
}
@ -158,25 +142,23 @@ func (u *CronjobService) HandleDelete(id uint) error {
return nil
}
func (u *CronjobService) HandleRmExpired(backType, backupDir string, cronjob *model.Cronjob, backClient cloud_storage.CloudStorageClient) {
func (u *CronjobService) HandleRmExpired(backType, localDir string, cronjob *model.Cronjob, backClient cloud_storage.CloudStorageClient) {
global.LOG.Infof("start to handle remove expired, retain copies: %d", cronjob.RetainCopies)
if backType != "LOCAL" {
currentObjs, err := backClient.ListObjects(backupDir + "/")
if err != nil {
global.LOG.Errorf("list bucket object %s failed, err: %v", backupDir, err)
return
}
for i := 0; i < len(currentObjs)-int(cronjob.RetainCopies); i++ {
_, _ = backClient.Delete(currentObjs[i].(string))
}
return
}
records, _ := cronjobRepo.ListRecord(cronjobRepo.WithByJobID(int(cronjob.ID)), commonRepo.WithOrderBy("created_at desc"))
if len(records) > int(cronjob.RetainCopies) {
for i := int(cronjob.RetainCopies); i < len(records); i++ {
files := strings.Split(records[i].File, ",")
for _, file := range files {
if backType != "LOCAL" {
_, _ = backClient.Delete(strings.ReplaceAll(file, localDir+"/", ""))
_ = os.Remove(file)
} else {
_ = os.Remove(file)
}
_ = backupRepo.DeleteRecord(context.TODO(), backupRepo.WithByFileName(path.Base(file)))
}
_ = cronjobRepo.DeleteRecord(commonRepo.WithByID(uint(records[i].ID)))
_ = os.Remove(records[i].File)
_ = os.Remove(records[i].Records)
}
}
@ -233,11 +215,33 @@ func handleUnTar(sourceFile, targetDir string) error {
return nil
}
func (u *CronjobService) handleDatabase(cronjob model.Cronjob, app *repo.RootInfo, backup model.BackupAccount, startTime time.Time) error {
func (u *CronjobService) handleDatabase(cronjob model.Cronjob, app *repo.RootInfo, backup model.BackupAccount, startTime time.Time) ([]string, error) {
var paths []string
localDir, err := loadLocalDir()
if err != nil {
return err
return paths, err
}
var dblist []string
if cronjob.DBName == "all" {
mysqlService := NewIMysqlService()
dblist, err = mysqlService.ListDBName()
if err != nil {
return paths, err
}
} else {
dblist = append(dblist, cronjob.DBName)
}
var client cloud_storage.CloudStorageClient
if backup.Type != "LOCAL" {
client, err = NewIBackupService().NewClient(&backup)
if err != nil {
return paths, err
}
}
for _, dbName := range dblist {
var record model.BackupRecord
record.Type = "mysql"
@ -245,21 +249,10 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, app *repo.RootInf
record.Source = "LOCAL"
record.BackupType = backup.Type
var dblist []string
if cronjob.DBName == "all" {
mysqlService := NewIMysqlService()
dblist, err = mysqlService.ListDBName()
if err != nil {
return err
}
} else {
dblist = append(dblist, cronjob.DBName)
}
for _, dbName := range dblist {
backupDir := fmt.Sprintf("%s/database/mysql/%s/%s", localDir, app.Name, dbName)
record.FileName = fmt.Sprintf("db_%s_%s.sql.gz", dbName, startTime.Format("20060102150405"))
if err = handleMysqlBackup(app, backupDir, dbName, record.FileName); err != nil {
return err
return paths, err
}
record.DetailName = dbName
record.FileDir = backupDir
@ -268,58 +261,61 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, app *repo.RootInf
record.Source = backup.Type
record.FileDir = itemFileDir
}
paths = append(paths, fmt.Sprintf("%s/%s", record.FileDir, record.FileName))
if err := backupRepo.CreateRecord(&record); err != nil {
global.LOG.Errorf("save backup record failed, err: %v", err)
return err
}
if backup.Type == "LOCAL" {
u.HandleRmExpired(backup.Type, backupDir, &cronjob, nil)
return nil
return paths, err
}
if backup.Type != "LOCAL" {
if !cronjob.KeepLocal {
defer func() {
_ = os.RemoveAll(fmt.Sprintf("%s/%s", backupDir, record.FileName))
}()
}
client, err := NewIBackupService().NewClient(&backup)
if err != nil {
return err
}
if _, err = client.Upload(backupDir+"/"+record.FileName, itemFileDir+"/"+record.FileName); err != nil {
return err
}
u.HandleRmExpired(backup.Type, itemFileDir, &cronjob, client)
if cronjob.KeepLocal {
u.HandleRmExpired("LOCAL", backupDir, &cronjob, client)
return paths, err
}
}
return nil
}
u.HandleRmExpired(backup.Type, localDir, &cronjob, client)
return paths, nil
}
func (u *CronjobService) handleWebsite(cronjob model.Cronjob, backup model.BackupAccount, startTime time.Time) error {
func (u *CronjobService) handleWebsite(cronjob model.Cronjob, backup model.BackupAccount, startTime time.Time) ([]string, error) {
var paths []string
localDir, err := loadLocalDir()
if err != nil {
return err
return paths, err
}
var record model.BackupRecord
record.Type = "website"
record.Name = cronjob.Website
record.Source = "LOCAL"
record.BackupType = backup.Type
var weblist []string
if cronjob.Website == "all" {
weblist, err = NewIWebsiteService().GetWebsiteOptions()
if err != nil {
return err
return paths, err
}
} else {
weblist = append(weblist, cronjob.Website)
}
var client cloud_storage.CloudStorageClient
if backup.Type != "LOCAL" {
client, err = NewIBackupService().NewClient(&backup)
if err != nil {
return paths, err
}
}
for _, websiteItem := range weblist {
var record model.BackupRecord
record.Type = "website"
record.Name = cronjob.Website
record.Source = "LOCAL"
record.BackupType = backup.Type
website, err := websiteRepo.GetFirst(websiteRepo.WithDomain(websiteItem))
if err != nil {
return err
return paths, err
}
backupDir := fmt.Sprintf("%s/website/%s", localDir, website.PrimaryDomain)
record.FileDir = backupDir
@ -329,34 +325,26 @@ func (u *CronjobService) handleWebsite(cronjob model.Cronjob, backup model.Backu
record.FileDir = strings.ReplaceAll(backupDir, localDir+"/", "")
}
record.FileName = fmt.Sprintf("website_%s_%s.tar.gz", website.PrimaryDomain, startTime.Format("20060102150405"))
paths = append(paths, fmt.Sprintf("%s/%s", record.FileDir, record.FileName))
if err := handleWebsiteBackup(&website, backupDir, record.FileName); err != nil {
return err
return paths, err
}
record.Name = website.PrimaryDomain
if err := backupRepo.CreateRecord(&record); err != nil {
global.LOG.Errorf("save backup record failed, err: %v", err)
return err
}
if backup.Type == "LOCAL" {
u.HandleRmExpired(backup.Type, backupDir, &cronjob, nil)
return nil
return paths, err
}
if backup.Type != "LOCAL" {
if !cronjob.KeepLocal {
defer func() {
_ = os.RemoveAll(fmt.Sprintf("%s/%s", backupDir, record.FileName))
}()
}
client, err := NewIBackupService().NewClient(&backup)
if err != nil {
return err
}
if _, err = client.Upload(backupDir+"/"+record.FileName, itemFileDir+"/"+record.FileName); err != nil {
return err
}
u.HandleRmExpired(backup.Type, itemFileDir, &cronjob, client)
if cronjob.KeepLocal {
u.HandleRmExpired("LOCAL", backupDir, &cronjob, client)
return paths, err
}
}
return nil
}
u.HandleRmExpired(backup.Type, localDir, &cronjob, client)
return paths, nil
}

3
frontend/src/lang/modules/en.ts
View file

@ -284,6 +284,7 @@ const message = {
logout: 'Logout',
},
database: {
database: 'database',
delete: 'Delete operation cannot be rolled back, please input "',
deleteHelper: '" to delete this database',
create: 'Create database',
@ -594,7 +595,7 @@ const message = {
directory: 'Backup directory',
sourceDir: 'Backup directory',
allOptionHelper:
'The current task plan is to back up all {0}. Direct download is not supported at the moment. You can check the backup list of {0} menu.',
'The current task plan is to back up all [{0}]. Direct download is not supported at the moment. You can check the backup list of [{0}] menu.',
exclusionRules: 'Exclusive rule',
saveLocal: 'Retain local backups (the same as the number of cloud storage copies)',
url: 'URL Address',

3
frontend/src/lang/modules/zh.ts
View file

@ -287,6 +287,7 @@ const message = {
logout: '退出登录',
},
database: {
database: '数据库',
delete: '删除操作无法回滚，请输入 "',
deleteHelper: '" 删除此数据库',
create: '创建数据库',
@ -595,7 +596,7 @@ const message = {
cleanHelper: '该操作将所有任务执行记录、备份文件和日志文件，是否继续？',
directory: '备份目录',
sourceDir: '备份目录',
allOptionHelper: '当前计划任务为备份所有 {0}，暂不支持直接下载，可在 {0} 备份列表中查看',
allOptionHelper: '当前计划任务为备份所有【{0}】，暂不支持直接下载，可在【{0}】备份列表中查看',
exclusionRules: '排除规则',
saveLocal: '同时保留本地备份（和云存储保留份数一致）',
url: 'URL 地址',

2
frontend/src/views/cronjob/record/index.vue
View file

@ -163,7 +163,7 @@
<template #label>
<span class="status-label">{{ $t('cronjob.database') }}</span>
</template>
<span v-if="dialogData.rowData!.website !== 'all'" class="status-count">
<span v-if="dialogData.rowData!.dbName !== 'all'" class="status-count">
{{ dialogData.rowData!.dbName }}
</span>
<span v-else class="status-count">

2
frontend/src/views/setting/backup-account/index.vue
View file

@ -126,7 +126,7 @@
>
{{ $t('commons.button.edit') }}
</el-button>
<el-button round :disabled="s3Data.id === 0" @click="onBatchDelete(cosData)">
<el-button round :disabled="cosData.id === 0" @click="onBatchDelete(cosData)">
{{ $t('commons.button.delete') }}
</el-button>
</div>