diff --git a/agent/app/repo/task.go b/agent/app/repo/task.go index 8667e8a45..b5dea011f 100644 --- a/agent/app/repo/task.go +++ b/agent/app/repo/task.go @@ -24,6 +24,7 @@ type ITaskRepo interface { DeleteAll() error WithByID(id string) DBOption + WithByIDNotIn(ids []string) DBOption WithResourceID(id uint) DBOption WithOperate(taskOperate string) DBOption WithByStatus(status string) DBOption @@ -58,6 +59,12 @@ func (t TaskRepo) WithByID(id string) DBOption { } } +func (t TaskRepo) WithByIDNotIn(ids []string) DBOption { + return func(g *gorm.DB) *gorm.DB { + return g.Where("id not in (?)", ids) + } +} + func (t TaskRepo) WithOperate(taskOperate string) DBOption { return func(g *gorm.DB) *gorm.DB { return g.Where("operate = ?", taskOperate) diff --git a/agent/app/service/backup_app.go b/agent/app/service/backup_app.go index c67836d22..01920c38d 100644 --- a/agent/app/service/backup_app.go +++ b/agent/app/service/backup_app.go @@ -128,7 +128,7 @@ func handleAppBackup(install *model.AppInstall, parentTask *task.Task, recordID ) backupTask = parentTask if parentTask == nil { - backupTask, err = task.NewTaskWithOps(install.Name, task.TaskBackup, task.TaskScopeApp, taskID, install.ID) + backupTask, err = task.NewTaskWithOps(install.Name, task.TaskBackup, task.TaskScopeBackup, taskID, install.ID) if err != nil { return err } @@ -139,7 +139,7 @@ func handleAppBackup(install *model.AppInstall, parentTask *task.Task, recordID return itemHandler() } - backupTask.AddSubTaskWithOps(task.GetTaskName(install.Name, task.TaskBackup, task.TaskScopeApp), func(t *task.Task) error { return itemHandler() }, nil, 3, time.Hour) + backupTask.AddSubTaskWithOps(task.GetTaskName(install.Name, task.TaskBackup, task.TaskScopeBackup), func(t *task.Task) error { return itemHandler() }, nil, 3, time.Hour) go func() { if err := backupTask.Execute(); err != nil { backupRepo.UpdateRecordByMap(recordID, map[string]interface{}{"status": constant.StatusFailed, "message": err.Error()}) @@ -160,7 +160,7 @@ func handleAppRecover(install *model.AppInstall, parentTask *task.Task, recoverF ) recoverTask = parentTask if parentTask == nil { - recoverTask, err = task.NewTaskWithOps(install.Name, task.TaskRecover, task.TaskScopeApp, taskID, install.ID) + recoverTask, err = task.NewTaskWithOps(install.Name, task.TaskRecover, task.TaskScopeBackup, taskID, install.ID) if err != nil { return err } @@ -328,7 +328,7 @@ func handleAppRecover(install *model.AppInstall, parentTask *task.Task, recoverF return recoverApp(parentTask) } - recoverTask.AddSubTask(task.GetTaskName(install.Name, task.TaskRecover, task.TaskScopeApp), recoverApp, rollBackApp) + recoverTask.AddSubTask(task.GetTaskName(install.Name, task.TaskRecover, task.TaskScopeBackup), recoverApp, rollBackApp) go func() { _ = recoverTask.Execute() }() diff --git a/agent/app/service/backup_mysql.go b/agent/app/service/backup_mysql.go index ba75ac21d..9b075ff89 100644 --- a/agent/app/service/backup_mysql.go +++ b/agent/app/service/backup_mysql.go @@ -84,7 +84,7 @@ func handleMysqlBackup(db DatabaseHelper, parentTask *task.Task, recordID uint, } itemName := fmt.Sprintf("%s[%s] - %s", db.Database, db.DBType, db.Name) if parentTask == nil { - backupTask, err = task.NewTaskWithOps(itemName, task.TaskBackup, task.TaskScopeDatabase, taskID, dbInfo.ID) + backupTask, err = task.NewTaskWithOps(itemName, task.TaskBackup, task.TaskScopeBackup, taskID, dbInfo.ID) if err != nil { return err } @@ -94,7 +94,7 @@ func handleMysqlBackup(db DatabaseHelper, parentTask *task.Task, recordID uint, if parentTask != nil { return itemHandler() } - backupTask.AddSubTaskWithOps(task.GetTaskName(itemName, task.TaskBackup, task.TaskScopeDatabase), func(t *task.Task) error { return itemHandler() }, nil, 3, time.Hour) + backupTask.AddSubTaskWithOps(task.GetTaskName(itemName, task.TaskBackup, task.TaskScopeBackup), func(t *task.Task) error { return itemHandler() }, nil, 3, time.Hour) go func() { if err := backupTask.Execute(); err != nil { backupRepo.UpdateRecordByMap(recordID, map[string]interface{}{"status": constant.StatusFailed, "message": err.Error()}) @@ -117,7 +117,7 @@ func handleMysqlRecover(req dto.CommonRecover, parentTask *task.Task, isRollback } itemName := fmt.Sprintf("%s[%s] - %s", req.Name, req.Type, req.DetailName) if parentTask == nil { - itemTask, err = task.NewTaskWithOps(itemName, task.TaskRecover, task.TaskScopeDatabase, taskID, dbInfo.ID) + itemTask, err = task.NewTaskWithOps(itemName, task.TaskRecover, task.TaskScopeBackup, taskID, dbInfo.ID) if err != nil { return err } diff --git a/agent/app/service/backup_postgresql.go b/agent/app/service/backup_postgresql.go index 8a21fb690..d95c6caee 100644 --- a/agent/app/service/backup_postgresql.go +++ b/agent/app/service/backup_postgresql.go @@ -79,7 +79,7 @@ func handlePostgresqlBackup(db DatabaseHelper, parentTask *task.Task, recordID u backupTask = parentTask itemName := fmt.Sprintf("%s - %s", db.Database, db.Name) if parentTask == nil { - backupTask, err = task.NewTaskWithOps(itemName, task.TaskBackup, task.TaskScopeDatabase, taskID, db.ID) + backupTask, err = task.NewTaskWithOps(itemName, task.TaskBackup, task.TaskScopeBackup, taskID, db.ID) if err != nil { return err } @@ -89,7 +89,7 @@ func handlePostgresqlBackup(db DatabaseHelper, parentTask *task.Task, recordID u if parentTask != nil { return itemHandler() } - backupTask.AddSubTaskWithOps(task.GetTaskName(itemName, task.TaskBackup, task.TaskScopeDatabase), func(t *task.Task) error { return itemHandler() }, nil, 3, time.Hour) + backupTask.AddSubTaskWithOps(task.GetTaskName(itemName, task.TaskBackup, task.TaskScopeBackup), func(t *task.Task) error { return itemHandler() }, nil, 3, time.Hour) go func() { if err := backupTask.Execute(); err != nil { backupRepo.UpdateRecordByMap(recordID, map[string]interface{}{"status": constant.StatusFailed, "message": err.Error()}) @@ -111,7 +111,7 @@ func handlePostgresqlRecover(req dto.CommonRecover, parentTask *task.Task, isRol } itemTask = parentTask if parentTask == nil { - itemTask, err = task.NewTaskWithOps(req.Name, task.TaskRecover, task.TaskScopeDatabase, req.TaskID, dbInfo.ID) + itemTask, err = task.NewTaskWithOps(req.Name, task.TaskRecover, task.TaskScopeBackup, req.TaskID, dbInfo.ID) if err != nil { return err } diff --git a/agent/app/service/backup_redis.go b/agent/app/service/backup_redis.go index 4deabac41..6e8b30938 100644 --- a/agent/app/service/backup_redis.go +++ b/agent/app/service/backup_redis.go @@ -84,7 +84,7 @@ func handleRedisBackup(redisInfo *repo.RootInfo, parentTask *task.Task, recordID ) itemTask = parentTask if parentTask == nil { - itemTask, err = task.NewTaskWithOps("Redis", task.TaskBackup, task.TaskScopeDatabase, taskID, redisInfo.ID) + itemTask, err = task.NewTaskWithOps("Redis", task.TaskBackup, task.TaskScopeBackup, taskID, redisInfo.ID) if err != nil { return err } @@ -147,7 +147,7 @@ func handleRedisRecover(redisInfo *repo.RootInfo, parentTask *task.Task, recover ) itemTask = parentTask if parentTask == nil { - itemTask, err = task.NewTaskWithOps("Redis", task.TaskRecover, task.TaskScopeDatabase, taskID, redisInfo.ID) + itemTask, err = task.NewTaskWithOps("Redis", task.TaskRecover, task.TaskScopeBackup, taskID, redisInfo.ID) if err != nil { return err } diff --git a/agent/app/service/backup_website.go b/agent/app/service/backup_website.go index 35f74bf69..0c4896344 100644 --- a/agent/app/service/backup_website.go +++ b/agent/app/service/backup_website.go @@ -73,11 +73,11 @@ func (u *BackupService) WebsiteRecover(req dto.CommonRecover) error { } func handleWebsiteRecover(website *model.Website, recoverFile string, isRollback bool, secret, taskID string) error { - recoverTask, err := task.NewTaskWithOps(website.PrimaryDomain, task.TaskRecover, task.TaskScopeWebsite, taskID, website.ID) + recoverTask, err := task.NewTaskWithOps(website.PrimaryDomain, task.TaskRecover, task.TaskScopeBackup, taskID, website.ID) if err != nil { return err } - recoverTask.AddSubTask(task.GetTaskName(website.PrimaryDomain, task.TaskRecover, task.TaskScopeWebsite), func(t *task.Task) error { + recoverTask.AddSubTask(task.GetTaskName(website.PrimaryDomain, task.TaskRecover, task.TaskScopeBackup), func(t *task.Task) error { isOk := false if !isRollback { rollbackFile := path.Join(global.Dir.TmpDir, fmt.Sprintf("website/%s_%s.tar.gz", website.Alias, time.Now().Format(constant.DateTimeSlimLayout))) @@ -212,7 +212,7 @@ func handleWebsiteBackup(website *model.Website, parentTask *task.Task, recordID ) backupTask = parentTask if parentTask == nil { - backupTask, err = task.NewTaskWithOps(website.Alias, task.TaskBackup, task.TaskScopeWebsite, taskID, website.ID) + backupTask, err = task.NewTaskWithOps(website.Alias, task.TaskBackup, task.TaskScopeBackup, taskID, website.ID) if err != nil { return err } @@ -221,7 +221,7 @@ func handleWebsiteBackup(website *model.Website, parentTask *task.Task, recordID if parentTask != nil { return itemHandler() } - backupTask.AddSubTaskWithOps(task.GetTaskName(website.Alias, task.TaskBackup, task.TaskScopeWebsite), func(t *task.Task) error { return itemHandler() }, nil, 3, time.Hour) + backupTask.AddSubTaskWithOps(task.GetTaskName(website.Alias, task.TaskBackup, task.TaskScopeBackup), func(t *task.Task) error { return itemHandler() }, nil, 3, time.Hour) go func() { if err := backupTask.Execute(); err != nil { backupRepo.UpdateRecordByMap(recordID, map[string]interface{}{"status": constant.StatusFailed, "message": err.Error()}) diff --git a/agent/app/service/device_clean.go b/agent/app/service/device_clean.go index 4236ffa04..67e41ea92 100644 --- a/agent/app/service/device_clean.go +++ b/agent/app/service/device_clean.go @@ -7,6 +7,7 @@ import ( "path" "path/filepath" "sort" + "strconv" "strings" "time" @@ -18,6 +19,7 @@ import ( "github.com/docker/docker/api/types/filters" "github.com/1Panel-dev/1Panel/agent/app/dto" + "github.com/1Panel-dev/1Panel/agent/app/model" "github.com/1Panel-dev/1Panel/agent/app/repo" "github.com/1Panel-dev/1Panel/agent/app/task" "github.com/1Panel-dev/1Panel/agent/global" @@ -74,8 +76,21 @@ func (u *DeviceService) Scan() dto.CleanData { sort.Slice(upgradeTree.Children, func(i, j int) bool { return common.CompareVersion(upgradeTree.Children[i].Label, upgradeTree.Children[j].Label) }) - upgradeTree.Children[0].IsCheck = false - upgradeTree.Children[0].IsRecommend = false + if global.IsMaster { + var copiesSeeting model.Setting + _ = global.CoreDB.Where("key = ?", "UpgradeBackupCopies").First(&copiesSeeting).Error + copies, _ := strconv.Atoi(copiesSeeting.Value) + if copies == 0 || copies > len(upgradeTree.Children) { + copies = len(upgradeTree.Children) + } + for i := 0; i < copies; i++ { + upgradeTree.Children[i].IsCheck = false + upgradeTree.Children[i].IsRecommend = false + } + } else { + upgradeTree.Children[0].IsCheck = false + upgradeTree.Children[0].IsRecommend = false + } } treeData = append(treeData, upgradeTree) @@ -214,7 +229,7 @@ func (u *DeviceService) Clean(req []dto.Clean) { } case "task_log": if len(item.Name) == 0 { - files, _ := os.ReadDir(path.Join(global.Dir.TaskDir)) + files, _ := os.ReadDir(global.Dir.TaskDir) if len(files) == 0 { continue } @@ -222,15 +237,10 @@ func (u *DeviceService) Clean(req []dto.Clean) { if file.Name() == "ssl" || !file.IsDir() { continue } - dropFileOrDir(path.Join(global.Dir.TaskDir, file.Name())) + dropTaskLog(path.Join(global.Dir.TaskDir, file.Name())) } - _ = taskRepo.DeleteAll() } else { - pathItem := path.Join(global.Dir.TaskDir, item.Name) - dropFileOrDir(pathItem) - if len(item.Name) != 0 { - _ = taskRepo.Delete(repo.WithByType(item.Name)) - } + dropTaskLog(path.Join(global.Dir.TaskDir, item.Name)) } case "script": dropFileOrDir(path.Join(global.Dir.TmpDir, "script", item.Name)) @@ -530,7 +540,6 @@ func loadTreeWithAllFile(isCheck bool, originalPath, treeType, pathItem string, } func dropFileOrDir(itemPath string) { - global.LOG.Debugf("drop file %s", itemPath) if err := os.RemoveAll(itemPath); err != nil { global.LOG.Errorf("drop file %s failed, err %v", itemPath, err) } @@ -606,6 +615,62 @@ func dropVolumes() (int, int) { return len(res.VolumesDeleted), int(res.SpaceReclaimed) } +func dropTaskLog(logDir string) { + files, err := os.ReadDir(logDir) + if err != nil { + return + } + taskType := path.Base(logDir) + var usedTasks []string + switch taskType { + case "Cronjob": + _ = global.DB.Model(&model.JobRecords{}).Where("task_id != ?", "").Select("task_id").Find(&usedTasks).Error + case "Snapshot": + var ( + snapIDs []string + recoverIDs []string + rollbackIDs []string + ) + _ = global.DB.Model(&model.Snapshot{}).Where("task_id != ?", "").Select("task_id").Find(&snapIDs).Error + _ = global.DB.Model(&model.Snapshot{}).Where("task_recover_id != ", "").Select("task_id").Find(&recoverIDs).Error + _ = global.DB.Model(&model.Snapshot{}).Where("task_rollback_id != ?", "").Select("task_id").Find(&rollbackIDs).Error + usedTasks = append(usedTasks, snapIDs...) + usedTasks = append(usedTasks, recoverIDs...) + usedTasks = append(usedTasks, rollbackIDs...) + case "Backup": + _ = global.DB.Model(&model.BackupRecord{}).Where("task_id != ?", "").Select("task_id").Find(&usedTasks).Error + case "Clam": + _ = global.DB.Model(&model.ClamRecord{}).Where("task_id != ?", "").Select("task_id").Find(&usedTasks).Error + case "Tamper": + xpackDB, err := common.LoadDBConnByPathWithErr(path.Join(global.CONF.Base.InstallDir, "1panel/db/xpack.db"), "xpack.db") + if err == nil { + _ = xpackDB.Table("tampers").Where("task_id != ?", "").Select("task_id").Find(&usedTasks).Error + } + case "System": + xpackDB, err := common.LoadDBConnByPathWithErr(path.Join(global.CONF.Base.InstallDir, "1panel/db/xpack.db"), "xpack.db") + if err == nil { + _ = xpackDB.Model("nodes").Where("task_id != ?", "").Select("task_id").Find(&usedTasks).Error + } + default: + dropFileOrDir(logDir) + _ = taskRepo.Delete(repo.WithByType(taskType)) + return + } + usedMap := make(map[string]struct{}) + for _, item := range usedTasks { + if _, ok := usedMap[item]; !ok { + usedMap[item] = struct{}{} + } + } + for _, item := range files { + if _, ok := usedMap[strings.TrimSuffix(item.Name(), ".log")]; ok { + continue + } + _ = os.Remove(logDir + "/" + item.Name()) + } + _ = taskRepo.Delete(repo.WithByType(taskType), taskRepo.WithByIDNotIn(usedTasks)) +} + func dropWithExclude(pathToDelete string, excludeSubDirs []string, taskItem *task.Task, size *int64, count *int) { entries, err := os.ReadDir(pathToDelete) if err != nil { diff --git a/agent/app/task/task.go b/agent/app/task/task.go index d2c72f4af..18b8b99f5 100644 --- a/agent/app/task/task.go +++ b/agent/app/task/task.go @@ -87,6 +87,7 @@ const ( TaskScopeContainer = "Container" TaskScopeCompose = "Compose" TaskScopeImage = "Image" + TaskScopeBackup = "Backup " TaskScopeRuntimeExtension = "RuntimeExtension" TaskScopeCustomAppstore = "CustomAppstore" TaskScopeTamper = "Tamper"