From be6d3ea337e918c1a2a0441b1bb59e656c4ca544 Mon Sep 17 00:00:00 2001 From: ssongliu <73214554+ssongliu@users.noreply.github.com> Date: Fri, 27 Jun 2025 17:25:23 +0800 Subject: [PATCH] fix: Update compression backup exclusion rules (#9323) --- agent/app/dto/setting.go | 2 - agent/app/dto/snapshot.go | 13 ++-- agent/app/model/snapshot.go | 1 + agent/app/service/cronjob_backup.go | 1 + agent/app/service/snapshot_create.go | 29 ++++---- agent/init/migration/migrate.go | 1 + agent/init/migration/migrations/init.go | 12 ++-- agent/utils/files/file_op.go | 7 +- frontend/src/api/interface/cronjob.ts | 1 + .../file-batch}/index.vue | 71 +++++++------------ .../views/cronjob/cronjob/operate/index.vue | 17 +++-- .../views/setting/snapshot/create/index.vue | 8 ++- frontend/src/views/setting/snapshot/index.vue | 12 +--- .../views/setting/snapshot/status/index.vue | 4 +- 14 files changed, 85 insertions(+), 94 deletions(-) rename frontend/src/{views/setting/snapshot/ignore-rule => components/file-batch}/index.vue (65%) diff --git a/agent/app/dto/setting.go b/agent/app/dto/setting.go index 7c09099ce..50e90c48b 100644 --- a/agent/app/dto/setting.go +++ b/agent/app/dto/setting.go @@ -23,8 +23,6 @@ type SettingInfo struct { AppStoreSyncStatus string `json:"appStoreSyncStatus"` FileRecycleBin string `json:"fileRecycleBin"` - - SnapshotIgnore string `json:"snapshotIgnore"` } type SettingUpdate struct { diff --git a/agent/app/dto/snapshot.go b/agent/app/dto/snapshot.go index 7e428eb9d..29d8642e3 100644 --- a/agent/app/dto/snapshot.go +++ b/agent/app/dto/snapshot.go @@ -28,6 +28,8 @@ type SnapshotCreate struct { WithOperationLog bool `json:"withOperationLog"` WithSystemLog bool `json:"withSystemLog"` WithTaskLog bool `json:"withTaskLog"` + + IgnoreFiles []string `json:"ignoreFiles"` } type SnapshotData struct { @@ -35,11 +37,12 @@ type SnapshotData struct { BackupData []DataTree `json:"backupData"` PanelData []DataTree `json:"panelData"` - WithMonitorData bool `json:"withMonitorData"` - WithLoginLog bool `json:"withLoginLog"` - WithOperationLog bool `json:"withOperationLog"` - WithSystemLog bool `json:"withSystemLog"` - WithTaskLog bool `json:"withTaskLog"` + WithMonitorData bool `json:"withMonitorData"` + WithLoginLog bool `json:"withLoginLog"` + WithOperationLog bool `json:"withOperationLog"` + WithSystemLog bool `json:"withSystemLog"` + WithTaskLog bool `json:"withTaskLog"` + IgnoreFiles []string `json:"ignoreFiles"` } type DataTree struct { ID string `json:"id"` diff --git a/agent/app/model/snapshot.go b/agent/app/model/snapshot.go index d0c15cd44..afb4610e7 100644 --- a/agent/app/model/snapshot.go +++ b/agent/app/model/snapshot.go @@ -23,6 +23,7 @@ type Snapshot struct { WithOperationLog bool `json:"withOperationLog"` WithSystemLog bool `json:"withSystemLog"` WithTaskLog bool `json:"withTaskLog"` + IgnoreFiles string `json:"ignoreFiles"` InterruptStep string `json:"interruptStep"` RecoverStatus string `json:"recoverStatus"` diff --git a/agent/app/service/cronjob_backup.go b/agent/app/service/cronjob_backup.go index 8f4bfeada..f1a3f6959 100644 --- a/agent/app/service/cronjob_backup.go +++ b/agent/app/service/cronjob_backup.go @@ -285,6 +285,7 @@ func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, jobRecord model.J WithOperationLog: true, WithSystemLog: true, WithTaskLog: true, + IgnoreFiles: strings.Split(cronjob.ExclusionRules, ","), } if err := NewISnapshotService().SnapshotCreate(taskItem, req, jobRecord.ID, cronjob.RetryTimes, cronjob.Timeout); err != nil { diff --git a/agent/app/service/snapshot_create.go b/agent/app/service/snapshot_create.go index 2a597e2e1..bb47e3712 100644 --- a/agent/app/service/snapshot_create.go +++ b/agent/app/service/snapshot_create.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path" + "path/filepath" "strings" "sync" "time" @@ -55,6 +56,7 @@ func (u *SnapshotService) SnapshotCreate(parentTask *task.Task, req dto.Snapshot WithOperationLog: req.WithOperationLog, WithTaskLog: req.WithTaskLog, WithSystemLog: req.WithSystemLog, + IgnoreFiles: strings.Join(req.IgnoreFiles, ","), Version: versionItem.Value, Status: constant.StatusWaiting, @@ -398,6 +400,7 @@ func snapBackupData(snap snapHelper, req dto.SnapshotCreate, targetDir string) e snap.Task.LogStart(i18n.GetMsgByKey("SnapLocalBackup")) excludes := loadBackupExcludes(snap, req.BackupData) + excludes = append(excludes, req.IgnoreFiles...) excludes = append(excludes, "./system_snapshot") for _, item := range req.AppData { for _, itemApp := range item.Children { @@ -421,7 +424,8 @@ func loadBackupExcludes(snap snapHelper, req []dto.DataTree) []string { if err := snap.snapAgentDB.Where("file_dir = ? AND file_name = ?", strings.TrimPrefix(path.Dir(item.Path), global.Dir.LocalBackupDir+"/"), path.Base(item.Path)).Delete(&model.BackupRecord{}).Error; err != nil { snap.Task.LogWithStatus("delete backup file from database", err) } - excludes = append(excludes, "."+strings.TrimPrefix(item.Path, global.Dir.LocalBackupDir)) + itemDir, _ := filepath.Rel(item.Path, global.Dir.LocalBackupDir) + excludes = append(excludes, itemDir) } else { excludes = append(excludes, loadBackupExcludes(snap, item.Children)...) } @@ -433,7 +437,8 @@ func loadAppBackupExcludes(req []dto.DataTree) []string { for _, item := range req { if len(item.Children) == 0 { if !item.IsCheck { - excludes = append(excludes, "."+strings.TrimPrefix(item.Path, path.Join(global.Dir.LocalBackupDir))) + itemDir, _ := filepath.Rel(item.Path, global.Dir.LocalBackupDir) + excludes = append(excludes, itemDir) } } else { excludes = append(excludes, loadAppBackupExcludes(item.Children)...) @@ -466,26 +471,21 @@ func snapPanelData(snap snapHelper, req dto.SnapshotCreate, targetDir string) er rootDir := global.Dir.DataDir if strings.Contains(global.Dir.LocalBackupDir, rootDir) { - excludes = append(excludes, "."+strings.ReplaceAll(global.Dir.LocalBackupDir, rootDir, "")) + itemDir, _ := filepath.Rel(rootDir, global.Dir.LocalBackupDir) + excludes = append(excludes, itemDir) } if len(snap.OpenrestyDir) != 0 && strings.Contains(snap.OpenrestyDir, rootDir) { - excludes = append(excludes, "."+strings.ReplaceAll(snap.OpenrestyDir, rootDir, "")) - } - ignoreVal, _ := settingRepo.Get(settingRepo.WithByKey("SnapshotIgnore")) - rules := strings.Split(ignoreVal.Value, ",") - for _, ignore := range rules { - if len(ignore) == 0 || cmd.CheckIllegal(ignore) { - continue - } - excludes = append(excludes, "."+strings.ReplaceAll(ignore, rootDir, "")) + itemDir, _ := filepath.Rel(rootDir, snap.OpenrestyDir) + excludes = append(excludes, itemDir) } + excludes = append(excludes, req.IgnoreFiles...) err := snap.FileOp.TarGzCompressPro(false, rootDir, path.Join(targetDir, "1panel_data.tar.gz"), "", strings.Join(excludes, ",")) snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapCompressPanel"), err) if err != nil { return err } if len(snap.OpenrestyDir) != 0 { - err := snap.FileOp.TarGzCompressPro(false, snap.OpenrestyDir, path.Join(targetDir, "website.tar.gz"), "", "") + err := snap.FileOp.TarGzCompressPro(false, snap.OpenrestyDir, path.Join(targetDir, "website.tar.gz"), "", strings.Join(req.IgnoreFiles, ",")) snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapWebsite"), err) if err != nil { return err @@ -499,7 +499,8 @@ func loadPanelExcludes(req []dto.DataTree) []string { for _, item := range req { if len(item.Children) == 0 { if !item.IsCheck { - excludes = append(excludes, "."+strings.TrimPrefix(item.Path, path.Join(global.Dir.BaseDir, "1panel"))) + itemDir, _ := filepath.Rel(item.Path, path.Join(global.Dir.BaseDir, "1panel")) + excludes = append(excludes, itemDir) } } else { excludes = append(excludes, loadPanelExcludes(item.Children)...) diff --git a/agent/init/migration/migrate.go b/agent/init/migration/migrate.go index feddce0ba..707998e1c 100644 --- a/agent/init/migration/migrate.go +++ b/agent/init/migration/migrate.go @@ -27,6 +27,7 @@ func InitAgentDB() { migrations.UpdateRuntime, migrations.AddSnapshotRule, migrations.UpdatePHPRuntime, + migrations.AddSnapshotIgnore, }) if err := m.Migrate(); err != nil { global.LOG.Error(err) diff --git a/agent/init/migration/migrations/init.go b/agent/init/migration/migrations/init.go index 6bc84aeec..d38ce61c7 100644 --- a/agent/init/migration/migrations/init.go +++ b/agent/init/migration/migrations/init.go @@ -185,9 +185,6 @@ var InitSetting = &gormigrate.Migration{ if err := tx.Create(&model.Setting{Key: "FileRecycleBin", Value: constant.StatusEnable}).Error; err != nil { return err } - if err := tx.Create(&model.Setting{Key: "SnapshotIgnore", Value: "*.sock"}).Error; err != nil { - return err - } if err := tx.Create(&model.Setting{Key: "LocalSSHConn", Value: ""}).Error; err != nil { return err @@ -332,7 +329,6 @@ var AddSnapshotRule = &gormigrate.Migration{ ) }, } - var UpdatePHPRuntime = &gormigrate.Migration{ ID: "20250624-update-php-runtime", Migrate: func(tx *gorm.DB) error { @@ -340,3 +336,11 @@ var UpdatePHPRuntime = &gormigrate.Migration{ return nil }, } +var AddSnapshotIgnore = &gormigrate.Migration{ + ID: "20250627-add-snapshot-ignore", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate( + &model.Snapshot{}, + ) + }, +} diff --git a/agent/utils/files/file_op.go b/agent/utils/files/file_op.go index c6d3df772..a3fab7ac7 100644 --- a/agent/utils/files/file_op.go +++ b/agent/utils/files/file_op.go @@ -815,6 +815,9 @@ func (f FileOp) TarGzCompressPro(withDir bool, src, dst, secret, exclusionRules if len(exclude) == 0 { continue } + if strings.HasPrefix(exclude, "/") { + exclude, _ = filepath.Rel(src, exclude) + } if _, ok := exMap[exclude]; ok { continue } @@ -822,10 +825,6 @@ func (f FileOp) TarGzCompressPro(withDir bool, src, dst, secret, exclusionRules exMap[exclude] = struct{}{} } - itemPrefix := filepath.Base(src) - if itemPrefix == "/" { - itemPrefix = "" - } if len(secret) != 0 { commands = fmt.Sprintf("tar %s -zcf - %s | openssl enc -aes-256-cbc -salt -k '%s' -out %s", exStr, srcItem, secret, dst) global.LOG.Debug(strings.ReplaceAll(commands, fmt.Sprintf(" %s ", secret), "******")) diff --git a/frontend/src/api/interface/cronjob.ts b/frontend/src/api/interface/cronjob.ts index a92808a8a..50ecc6ac2 100644 --- a/frontend/src/api/interface/cronjob.ts +++ b/frontend/src/api/interface/cronjob.ts @@ -23,6 +23,7 @@ export namespace Cronjob { appID: string; website: string; exclusionRules: string; + ignoreFiles: Array; dbType: string; dbName: string; url: string; diff --git a/frontend/src/views/setting/snapshot/ignore-rule/index.vue b/frontend/src/components/file-batch/index.vue similarity index 65% rename from frontend/src/views/setting/snapshot/ignore-rule/index.vue rename to frontend/src/components/file-batch/index.vue index 304b53e92..fb093913a 100644 --- a/frontend/src/views/setting/snapshot/ignore-rule/index.vue +++ b/frontend/src/components/file-batch/index.vue @@ -1,6 +1,5 @@ - - + diff --git a/frontend/src/views/cronjob/cronjob/operate/index.vue b/frontend/src/views/cronjob/cronjob/operate/index.vue index cd0163a5b..12559851c 100644 --- a/frontend/src/views/cronjob/cronjob/operate/index.vue +++ b/frontend/src/views/cronjob/cronjob/operate/index.vue @@ -601,11 +601,7 @@ - + {{ $t('cronjob.exclusionRulesHelper') }} @@ -703,6 +699,7 @@ import { ElForm } from 'element-plus'; import { Cronjob } from '@/api/interface/cronjob'; import { addCronjob, editCronjob, loadCronjobInfo, loadNextHandle, loadScriptOptions } from '@/api/modules/cronjob'; import CodemirrorPro from '@/components/codemirror-pro/index.vue'; +import IgnoreFile from '@/components/file-batch/index.vue'; import LayoutCol from '@/components/layout-col/form.vue'; import { listDbItems } from '@/api/modules/database'; import { getWebsiteOptions } from '@/api/modules/website'; @@ -756,6 +753,7 @@ const form = reactive({ scriptID: null, appID: '', website: '', + ignoreFiles: [], exclusionRules: '', dbType: 'mysql', dbName: '', @@ -835,6 +833,7 @@ const search = async () => { form.website = res.data.website; form.websiteList = res.data.website.split(',') || []; form.exclusionRules = res.data.exclusionRules; + form.ignoreFiles = res.data.exclusionRules.split(','); form.dbType = res.data.dbType; form.dbName = res.data.dbName; form.dbNameList = res.data.dbName.split(',') || []; @@ -1280,7 +1279,12 @@ function isBackup() { } function hasExclusionRules() { - return form.type === 'app' || form.type === 'website' || (form.type === 'directory' && form.isDir); + return ( + form.type === 'app' || + form.type === 'website' || + form.type === 'snapshot' || + (form.type === 'directory' && form.isDir) + ); } function hasScript() { @@ -1327,6 +1331,7 @@ const onSubmit = async (formEl: FormInstance | undefined) => { form.dbName = form.dbNameList.join(','); } + form.exclusionRules = form.ignoreFiles.join(','); form.snapshotRule = { withImage: form.withImage, ignoreAppIDs: form.ignoreAppIDs }; form.alertCount = form.hasAlert && isProductPro.value ? form.alertCount : 0; form.alertTitle = diff --git a/frontend/src/views/setting/snapshot/create/index.vue b/frontend/src/views/setting/snapshot/create/index.vue index 7e15b4e83..ccad55b5b 100644 --- a/frontend/src/views/setting/snapshot/create/index.vue +++ b/frontend/src/views/setting/snapshot/create/index.vue @@ -143,11 +143,14 @@ + + + @@ -235,7 +231,6 @@ import { import { onMounted, reactive, ref } from 'vue'; import { computeSize, dateFormat, newUUID } from '@/utils/util'; import { ElForm } from 'element-plus'; -import IgnoreRule from '@/views/setting/snapshot/ignore-rule/index.vue'; import i18n from '@/lang'; import { Setting } from '@/api/interface/setting'; import TaskLog from '@/components/log/task/index.vue'; @@ -263,7 +258,6 @@ const searchName = ref(); const opRef = ref(); const createRef = ref(); -const ignoreRef = ref(); const recoverStatusRef = ref(); const importRef = ref(); const isRecordShow = ref(); @@ -335,10 +329,6 @@ const reRollback = (row: any) => { }); }; -const onIgnore = () => { - ignoreRef.value.acceptParams(); -}; - const onChange = async (info: any) => { await updateSnapshotDescription({ id: info.id, description: info.description }); MsgSuccess(i18n.global.t('commons.msg.operationSuccess')); diff --git a/frontend/src/views/setting/snapshot/status/index.vue b/frontend/src/views/setting/snapshot/status/index.vue index 5b7aa5542..d3986fd97 100644 --- a/frontend/src/views/setting/snapshot/status/index.vue +++ b/frontend/src/views/setting/snapshot/status/index.vue @@ -131,6 +131,7 @@ import { snapshotRollback } from '@/api/modules/setting'; import { MsgSuccess } from '@/utils/message'; import { loadOsInfo } from '@/api/modules/dashboard'; import SnapRecover from '@/views/setting/snapshot/recover/index.vue'; +import { newUUID } from '@/utils/util'; const drawerVisible = ref(false); const snapInfo = ref(); @@ -181,7 +182,8 @@ const rollbackSnapshot = async () => { type: 'info', }).then(async () => { loading.value = true; - await snapshotRollback({ id: snapInfo.value.id, isNew: false, reDownload: false, secret: '' }) + let taskID = newUUID(); + await snapshotRollback({ id: snapInfo.value.id, isNew: false, reDownload: false, secret: '', taskID: taskID }) .then(() => { emit('search'); loading.value = false;