From a0dfbde0d35d49e6b0349f6c1b579f33c7ffcd9d Mon Sep 17 00:00:00 2001 From: ssongliu <73214554+ssongliu@users.noreply.github.com> Date: Mon, 12 May 2025 13:57:59 +0800 Subject: [PATCH] feat: Support downloading scheduled task website log rotation files (#8597) --- agent/app/dto/cronjob.go | 12 ++-- agent/app/service/cronjob.go | 18 +++++- agent/app/service/cronjob_helper.go | 58 +++++++++---------- frontend/src/api/interface/cronjob.ts | 1 + frontend/src/api/modules/cronjob.ts | 4 +- frontend/src/lang/modules/en.ts | 1 + frontend/src/lang/modules/ja.ts | 1 + frontend/src/lang/modules/ko.ts | 1 + frontend/src/lang/modules/ms.ts | 1 + frontend/src/lang/modules/pt-br.ts | 1 + frontend/src/lang/modules/ru.ts | 1 + frontend/src/lang/modules/zh-Hant.ts | 1 + frontend/src/lang/modules/zh.ts | 1 + frontend/src/views/cronjob/cronjob/helper.ts | 12 ++++ frontend/src/views/cronjob/cronjob/index.vue | 21 +++---- .../views/cronjob/cronjob/record/index.vue | 39 ++++++++----- frontend/src/views/setting/license/index.vue | 2 +- 17 files changed, 104 insertions(+), 71 deletions(-) diff --git a/agent/app/dto/cronjob.go b/agent/app/dto/cronjob.go index 94b832382..13d953cef 100644 --- a/agent/app/dto/cronjob.go +++ b/agent/app/dto/cronjob.go @@ -61,14 +61,16 @@ type CronjobDownload struct { } type CronjobClean struct { - IsDelete bool `json:"isDelete"` - CleanData bool `json:"cleanData"` - CronjobID uint `json:"cronjobID" validate:"required"` + IsDelete bool `json:"isDelete"` + CleanData bool `json:"cleanData"` + CronjobID uint `json:"cronjobID" validate:"required"` + CleanRemoteData bool `json:"cleanRemoteData"` } type CronjobBatchDelete struct { - CleanData bool `json:"cleanData"` - IDs []uint `json:"ids" validate:"required"` + CleanData bool `json:"cleanData"` + CleanRemoteData bool `json:"cleanRemoteData"` + IDs []uint `json:"ids" validate:"required"` } type CronjobInfo struct { diff --git a/agent/app/service/cronjob.go b/agent/app/service/cronjob.go index bb407554e..67e0dccc6 100644 --- a/agent/app/service/cronjob.go +++ b/agent/app/service/cronjob.go @@ -202,10 +202,15 @@ func (u *CronjobService) CleanRecord(req dto.CronjobClean) error { if err != nil { return err } + if !req.CleanRemoteData { + for key := range accountMap { + if key != constant.Local { + delete(accountMap, key) + } + } + } cronjob.RetainCopies = 0 u.removeExpiredBackup(cronjob, accountMap, model.BackupRecord{}) - } else { - u.removeExpiredLog(cronjob) } } if req.IsDelete { @@ -272,6 +277,13 @@ func (u *CronjobService) Create(req dto.CronjobOperate) error { if err := copier.Copy(&cronjob, &req); err != nil { return buserr.WithDetail("ErrStructTransform", err.Error(), nil) } + if cronjob.Type == "cutWebsiteLog" { + backupAccount, err := backupRepo.Get(repo.WithByType(constant.Local)) + if backupAccount.ID == 0 { + return fmt.Errorf("load local backup dir failed, err: %v", err) + } + cronjob.DownloadAccountID, cronjob.SourceAccountIDs = backupAccount.ID, fmt.Sprintf("%v", backupAccount.ID) + } cronjob.Status = constant.StatusEnable global.LOG.Infof("create cronjob %s successful, spec: %s", cronjob.Name, cronjob.Spec) @@ -334,7 +346,7 @@ func (u *CronjobService) Delete(req dto.CronjobBatchDelete) error { global.Cron.Remove(cron.EntryID(idItem)) } global.LOG.Infof("stop cronjob entryID: %s", cronjob.EntryIDs) - if err := u.CleanRecord(dto.CronjobClean{CronjobID: id, CleanData: req.CleanData, IsDelete: true}); err != nil { + if err := u.CleanRecord(dto.CronjobClean{CronjobID: id, CleanData: req.CleanData, CleanRemoteData: req.CleanRemoteData, IsDelete: true}); err != nil { return err } if err := cronjobRepo.Delete(repo.WithByID(id)); err != nil { diff --git a/agent/app/service/cronjob_helper.go b/agent/app/service/cronjob_helper.go index 7b0aafe31..9ec7dc6b1 100644 --- a/agent/app/service/cronjob_helper.go +++ b/agent/app/service/cronjob_helper.go @@ -86,7 +86,7 @@ func (u *CronjobService) loadTask(cronjob *model.Cronjob, record *model.JobRecor case "ntp": u.handleNtpSync(*cronjob, taskItem) case "cutWebsiteLog": - err = u.handleCutWebsiteLog(cronjob, record, taskItem) + err = u.handleCutWebsiteLog(cronjob, record.StartTime, taskItem) case "clean": u.handleSystemClean(*cronjob, taskItem) case "website": @@ -174,26 +174,40 @@ func (u *CronjobService) handleNtpSync(cronjob model.Cronjob, taskItem *task.Tas }, nil, int(cronjob.RetryTimes), time.Duration(cronjob.Timeout)*time.Second) } -func (u *CronjobService) handleCutWebsiteLog(cronjob *model.Cronjob, record *model.JobRecords, taskItem *task.Task) error { +func (u *CronjobService) handleCutWebsiteLog(cronjob *model.Cronjob, startTime time.Time, taskItem *task.Task) error { taskItem.AddSubTaskWithOps(i18n.GetWithName("CutWebsiteLog", cronjob.Name), func(t *task.Task) error { - var filePaths []string websites := loadWebsForJob(*cronjob) fileOp := files.NewFileOp() baseDir := GetOpenrestyDir(SitesRootDir) + clientMap, err := NewBackupClientMap([]string{fmt.Sprintf("%v", cronjob.DownloadAccountID)}) + if err != nil { + return fmt.Errorf("load local backup client failed, err: %v", err) + } for _, website := range websites { taskItem.Log(website.Alias) + var record model.BackupRecord + record.From = "cronjob" + record.Type = "cut-website-log" + record.CronjobID = cronjob.ID + record.Name = website.Alias + record.DetailName = website.Alias + record.DownloadAccountID, record.SourceAccountIDs = cronjob.DownloadAccountID, cronjob.SourceAccountIDs + backupDir := pathUtils.Join(global.Dir.LocalBackupDir, "log", "website", website.Alias) + if !fileOp.Stat(backupDir) { + _ = os.MkdirAll(backupDir, constant.DirPerm) + } + record.FileDir = strings.TrimPrefix(backupDir, global.Dir.LocalBackupDir+"/") + record.FileName = fmt.Sprintf("%s_log_%s.gz", website.PrimaryDomain, startTime.Format(constant.DateTimeSlimLayout)) + if err := backupRepo.CreateRecord(&record); err != nil { + global.LOG.Errorf("save backup record failed, err: %v", err) + return err + } + websiteLogDir := pathUtils.Join(baseDir, website.Alias, "log") srcAccessLogPath := pathUtils.Join(websiteLogDir, "access.log") srcErrorLogPath := pathUtils.Join(websiteLogDir, "error.log") - dstLogDir := pathUtils.Join(global.Dir.LocalBackupDir, "log", "website", website.Alias) - if !fileOp.Stat(dstLogDir) { - _ = os.MkdirAll(dstLogDir, constant.DirPerm) - } - - dstName := fmt.Sprintf("%s_log_%s.gz", website.PrimaryDomain, record.StartTime.Format(constant.DateTimeSlimLayout)) - dstFilePath := pathUtils.Join(dstLogDir, dstName) - filePaths = append(filePaths, dstFilePath) + dstFilePath := pathUtils.Join(backupDir, record.FileName) if err := backupLogFile(dstFilePath, websiteLogDir, fileOp); err != nil { taskItem.LogFailedWithErr("CutWebsiteLog", err) continue @@ -202,9 +216,8 @@ func (u *CronjobService) handleCutWebsiteLog(cronjob *model.Cronjob, record *mod _ = fileOp.WriteFile(srcErrorLogPath, strings.NewReader(""), constant.DirPerm) } taskItem.Log(i18n.GetMsgWithMap("CutWebsiteLogSuccess", map[string]interface{}{"name": website.PrimaryDomain, "path": dstFilePath})) + u.removeExpiredBackup(*cronjob, clientMap, record) } - u.removeExpiredLog(*cronjob) - record.File = strings.Join(filePaths, ",") return nil }, nil, int(cronjob.RetryTimes), time.Duration(cronjob.Timeout)*time.Second) return nil @@ -287,25 +300,8 @@ func (u *CronjobService) removeExpiredBackup(cronjob model.Cronjob, accountMap m } } -func (u *CronjobService) removeExpiredLog(cronjob model.Cronjob) { - records, _ := cronjobRepo.ListRecord(cronjobRepo.WithByJobID(int(cronjob.ID)), repo.WithOrderBy("created_at desc")) - if len(records) <= int(cronjob.RetainCopies) { - return - } - for i := int(cronjob.RetainCopies); i < len(records); i++ { - if len(records[i].File) != 0 { - files := strings.Split(records[i].File, ",") - for _, file := range files { - _ = os.Remove(file) - } - } - _ = cronjobRepo.DeleteRecord(repo.WithByID(records[i].ID)) - _ = os.Remove(records[i].Records) - } -} - func hasBackup(cronjobType string) bool { - return cronjobType == "app" || cronjobType == "database" || cronjobType == "website" || cronjobType == "directory" || cronjobType == "snapshot" || cronjobType == "log" + return cronjobType == "app" || cronjobType == "database" || cronjobType == "website" || cronjobType == "directory" || cronjobType == "snapshot" || cronjobType == "log" || cronjobType == "cutWebsiteLog" } func handleCronJobAlert(cronjob *model.Cronjob) { diff --git a/frontend/src/api/interface/cronjob.ts b/frontend/src/api/interface/cronjob.ts index 8059967b3..d98f857fb 100644 --- a/frontend/src/api/interface/cronjob.ts +++ b/frontend/src/api/interface/cronjob.ts @@ -98,6 +98,7 @@ export namespace Cronjob { export interface CronjobDelete { ids: Array; cleanData: boolean; + cleanRemoteData: boolean; } export interface UpdateStatus { id: number; diff --git a/frontend/src/api/modules/cronjob.ts b/frontend/src/api/modules/cronjob.ts index 0bfdcb5cd..4ae4796a1 100644 --- a/frontend/src/api/modules/cronjob.ts +++ b/frontend/src/api/modules/cronjob.ts @@ -39,8 +39,8 @@ export const searchRecords = (params: Cronjob.SearchRecord) => { return http.post>(`cronjobs/search/records`, params); }; -export const cleanRecords = (id: number, cleanData: boolean) => { - return http.post(`cronjobs/records/clean`, { cronjobID: id, cleanData: cleanData }); +export const cleanRecords = (id: number, cleanData: boolean, cleanRemoteData: boolean) => { + return http.post(`cronjobs/records/clean`, { cronjobID: id, cleanData: cleanData, cleanRemoteData }); }; export const getRecordDetail = (params: string) => { diff --git a/frontend/src/lang/modules/en.ts b/frontend/src/lang/modules/en.ts index 32e05d0db..a1ebe819d 100644 --- a/frontend/src/lang/modules/en.ts +++ b/frontend/src/lang/modules/en.ts @@ -1060,6 +1060,7 @@ const message = { errHandle: 'Cronjob execution failure', noRecord: 'The execution did not generate any logs', cleanData: 'Clean data', + cleanRemoteData: 'Delete remote data', cleanDataHelper: 'Delete the backup file generated during this task.', noLogs: 'No task output yet...', errPath: 'Backup path [{0}] error, cannot download!', diff --git a/frontend/src/lang/modules/ja.ts b/frontend/src/lang/modules/ja.ts index 5761eb5ac..7b09584c2 100644 --- a/frontend/src/lang/modules/ja.ts +++ b/frontend/src/lang/modules/ja.ts @@ -1019,6 +1019,7 @@ const message = { errHandle: 'cronjob実行障害', noRecord: 'Cronジョブをトリガーすると、ここにレコードが表示されます。', cleanData: 'クリーンデータ', + cleanRemoteData: 'リモートデータを削除', cleanDataHelper: 'このタスク中に生成されたバックアップファイルを削除します。', noLogs: 'タスク出力はまだありません...', errPath: 'バックアップパス[{0}]エラー、ダウンロードできません！', diff --git a/frontend/src/lang/modules/ko.ts b/frontend/src/lang/modules/ko.ts index 3514fb1ff..ebf873c8a 100644 --- a/frontend/src/lang/modules/ko.ts +++ b/frontend/src/lang/modules/ko.ts @@ -1014,6 +1014,7 @@ const message = { errHandle: '크론 작업 실행 실패', noRecord: '크론 작업을 트리거하고 나면 여기에 레코드가 표시됩니다.', cleanData: '데이터 정리', + cleanRemoteData: '원격 데이터 삭제', cleanDataHelper: '이 작업에서 생성된 백업 파일을 삭제합니다.', noLogs: '작업 출력이 아직 없습니다...', errPath: '백업 경로 [{0}] 오류, 다운로드할 수 없습니다!', diff --git a/frontend/src/lang/modules/ms.ts b/frontend/src/lang/modules/ms.ts index 77761117a..8cad9ccb1 100644 --- a/frontend/src/lang/modules/ms.ts +++ b/frontend/src/lang/modules/ms.ts @@ -1049,6 +1049,7 @@ const message = { errHandle: 'Kegagalan pelaksanaan tugas cron', noRecord: 'Picu Tugas Cron, dan anda akan melihat rekod di sini.', cleanData: 'Bersihkan data', + cleanRemoteData: 'Padam data jarak jauh', cleanDataHelper: 'Padam fail sandaran yang dijana semasa tugas ini.', noLogs: 'Tiada keluaran tugas lagi...', errPath: 'Laluan sandaran [{0}] salah, tidak boleh dimuat turun!', diff --git a/frontend/src/lang/modules/pt-br.ts b/frontend/src/lang/modules/pt-br.ts index 82962c527..bc07007a3 100644 --- a/frontend/src/lang/modules/pt-br.ts +++ b/frontend/src/lang/modules/pt-br.ts @@ -1038,6 +1038,7 @@ const message = { errHandle: 'Falha na execução do Cronjob', noRecord: 'Acione a tarefa Cron e você verá os registros aqui.', cleanData: 'Limpar dados', + cleanRemoteData: 'Excluir dados remotos', cleanDataHelper: 'Excluir o arquivo de backup gerado durante esta tarefa.', noLogs: 'Ainda não há saída de tarefa...', errPath: 'Caminho de backup [{0}] com erro, não é possível fazer o download!', diff --git a/frontend/src/lang/modules/ru.ts b/frontend/src/lang/modules/ru.ts index 535748ad1..5049a8215 100644 --- a/frontend/src/lang/modules/ru.ts +++ b/frontend/src/lang/modules/ru.ts @@ -1043,6 +1043,7 @@ const message = { errHandle: 'Сбой выполнения задачи Cron', noRecord: 'Запустите задачу Cron, и вы увидите записи здесь.', cleanData: 'Очистить данные', + cleanRemoteData: 'Удалить удалённые данные', cleanDataHelper: 'Удалить файл резервной копии, созданный во время этой задачи.', noLogs: 'Пока нет вывода задачи...', errPath: 'Ошибка пути резервной копии [{0}], невозможно скачать!', diff --git a/frontend/src/lang/modules/zh-Hant.ts b/frontend/src/lang/modules/zh-Hant.ts index bc753d01f..65f5ade2a 100644 --- a/frontend/src/lang/modules/zh-Hant.ts +++ b/frontend/src/lang/modules/zh-Hant.ts @@ -1007,6 +1007,7 @@ const message = { errHandle: '任務執行失敗', noRecord: '當前計劃任務暫未產生記錄', cleanData: '刪除備份文件', + cleanRemoteData: '刪除遠端備份檔案', cleanDataHelper: '刪除該任務執行過程中產生的備份文件', noLogs: '暫無任務輸出...', errPath: '備份路徑 [{0}] 錯誤，無法下載！', diff --git a/frontend/src/lang/modules/zh.ts b/frontend/src/lang/modules/zh.ts index f83750025..8ab42a3b1 100644 --- a/frontend/src/lang/modules/zh.ts +++ b/frontend/src/lang/modules/zh.ts @@ -1005,6 +1005,7 @@ const message = { errHandle: '任务执行失败', noRecord: '当前计划任务暂未产生记录', cleanData: '删除备份文件', + cleanRemoteData: '删除远程备份文件', cleanDataHelper: '删除该任务执行过程中产生的备份文件', noLogs: '暂无任务输出...', errPath: '备份路径 [{0}] 错误，无法下载！', diff --git a/frontend/src/views/cronjob/cronjob/helper.ts b/frontend/src/views/cronjob/cronjob/helper.ts index 6251458e4..12a0aad98 100644 --- a/frontend/src/views/cronjob/cronjob/helper.ts +++ b/frontend/src/views/cronjob/cronjob/helper.ts @@ -189,3 +189,15 @@ export function transSpecToStr(spec: string): string { return i18n.global.t('cronjob.perNSecondHelper', [loadZero(specObj.second)]); } } + +export function hasBackup(type: string) { + return ( + type === 'app' || + type === 'website' || + type === 'database' || + type === 'directory' || + type === 'snapshot' || + type === 'log' || + type === 'cutWebsiteLog' + ); +} diff --git a/frontend/src/views/cronjob/cronjob/index.vue b/frontend/src/views/cronjob/cronjob/index.vue index 558caf52e..0e64e7af4 100644 --- a/frontend/src/views/cronjob/cronjob/index.vue +++ b/frontend/src/views/cronjob/cronjob/index.vue @@ -143,6 +143,11 @@ + {{ $t('cronjob.cleanDataHelper') }} @@ -164,7 +169,7 @@ import i18n from '@/lang'; import { Cronjob } from '@/api/interface/cronjob'; import { ElMessageBox } from 'element-plus'; import { MsgSuccess } from '@/utils/message'; -import { transSpecToStr } from './helper'; +import { hasBackup, transSpecToStr } from './helper'; import { GlobalStore } from '@/store'; import router from '@/routers'; @@ -181,6 +186,7 @@ const operateIDs = ref(); const opRef = ref(); const showClean = ref(); const cleanData = ref(); +const cleanRemoteData = ref(); const data = ref(); const paginationConfig = reactive({ @@ -257,7 +263,7 @@ const onDelete = async (row: Cronjob.CronjobInfo | null) => { const onSubmitDelete = async () => { loading.value = true; - await deleteCronjob({ ids: operateIDs.value, cleanData: cleanData.value }) + await deleteCronjob({ ids: operateIDs.value, cleanData: cleanData.value, cleanRemoteData: cleanRemoteData.value }) .then(() => { loading.value = false; MsgSuccess(i18n.global.t('commons.msg.deleteSuccess')); @@ -311,17 +317,6 @@ const onHandle = async (row: Cronjob.CronjobInfo) => { }); }; -const hasBackup = (type: string) => { - return ( - type === 'app' || - type === 'website' || - type === 'database' || - type === 'directory' || - type === 'snapshot' || - type === 'log' - ); -}; - const loadDetail = (row: any) => { isRecordShow.value = true; let params = { diff --git a/frontend/src/views/cronjob/cronjob/record/index.vue b/frontend/src/views/cronjob/cronjob/record/index.vue index de0aec2dc..67684245e 100644 --- a/frontend/src/views/cronjob/cronjob/record/index.vue +++ b/frontend/src/views/cronjob/cronjob/record/index.vue @@ -200,6 +200,7 @@ + {{ $t('cronjob.cleanDataHelper') }} @@ -231,6 +232,7 @@ import { MsgSuccess } from '@/utils/message'; import { listDbItems } from '@/api/modules/database'; import { listAppInstalled } from '@/api/modules/app'; import { shortcuts } from '@/utils/shortcuts'; +import { hasBackup } from '../helper'; const loading = ref(); const refresh = ref(false); @@ -253,6 +255,7 @@ const currentRecordDetail = ref(''); const open = ref(); const delLoading = ref(); const cleanData = ref(); +const cleanRemoteData = ref(); const acceptParams = async (params: DialogProps): Promise => { let itemSize = Number(localStorage.getItem(searchInfo.cacheSizeKey)); @@ -402,26 +405,30 @@ const loadRecord = async (row: Cronjob.Record) => { }; const onClean = async () => { - ElMessageBox.confirm(i18n.global.t('commons.msg.clean'), i18n.global.t('commons.button.delete'), { - confirmButtonText: i18n.global.t('commons.button.confirm'), - cancelButtonText: i18n.global.t('commons.button.cancel'), - type: 'warning', - }).then(async () => { - await cleanRecords(dialogData.value.rowData.id, cleanData.value) - .then(() => { - delLoading.value = false; - MsgSuccess(i18n.global.t('commons.msg.operationSuccess')); - search(); - }) - .catch(() => { - delLoading.value = false; - }); - }); + if (!hasBackup(dialogData.value.rowData.type)) { + ElMessageBox.confirm(i18n.global.t('commons.msg.clean'), i18n.global.t('commons.msg.deleteTitle'), { + confirmButtonText: i18n.global.t('commons.button.confirm'), + cancelButtonText: i18n.global.t('commons.button.cancel'), + type: 'warning', + }).then(async () => { + await cleanRecords(dialogData.value.rowData.id, cleanData.value, cleanRemoteData.value) + .then(() => { + delLoading.value = false; + MsgSuccess(i18n.global.t('commons.msg.operationSuccess')); + search(); + }) + .catch(() => { + delLoading.value = false; + }); + }); + } else { + open.value = true; + } }; const cleanRecord = async () => { delLoading.value = true; - await cleanRecords(dialogData.value.rowData.id, cleanData.value) + await cleanRecords(dialogData.value.rowData.id, cleanData.value, false) .then(() => { delLoading.value = false; open.value = false; diff --git a/frontend/src/views/setting/license/index.vue b/frontend/src/views/setting/license/index.vue index 786c5e9dd..589c7543f 100644 --- a/frontend/src/views/setting/license/index.vue +++ b/frontend/src/views/setting/license/index.vue @@ -41,7 +41,7 @@