fix: Fix the issue where cache cleanup fails to exclude logs of curre… (#10497)

…ntly used

Refs #10484

agent/app/repo/task.go

@@ -24,6 +24,7 @@ type ITaskRepo interface {
 	DeleteAll() error
 
 	WithByID(id string) DBOption
+	WithByIDNotIn(ids []string) DBOption
 	WithResourceID(id uint) DBOption
 	WithOperate(taskOperate string) DBOption
 	WithByStatus(status string) DBOption
@@ -58,6 +59,12 @@ func (t TaskRepo) WithByID(id string) DBOption {
 	}
 }
 
+func (t TaskRepo) WithByIDNotIn(ids []string) DBOption {
+	return func(g *gorm.DB) *gorm.DB {
+		return g.Where("id not in (?)", ids)
+	}
+}
+
 func (t TaskRepo) WithOperate(taskOperate string) DBOption {
 	return func(g *gorm.DB) *gorm.DB {
 		return g.Where("operate = ?", taskOperate)

agent/app/service/backup_app.go

@@ -128,7 +128,7 @@ func handleAppBackup(install *model.AppInstall, parentTask *task.Task, recordID
 	)
 	backupTask = parentTask
 	if parentTask == nil {
-		backupTask, err = task.NewTaskWithOps(install.Name, task.TaskBackup, task.TaskScopeApp, taskID, install.ID)
+		backupTask, err = task.NewTaskWithOps(install.Name, task.TaskBackup, task.TaskScopeBackup, taskID, install.ID)
 		if err != nil {
 			return err
 		}
@@ -139,7 +139,7 @@ func handleAppBackup(install *model.AppInstall, parentTask *task.Task, recordID
 		return itemHandler()
 	}
 
-	backupTask.AddSubTaskWithOps(task.GetTaskName(install.Name, task.TaskBackup, task.TaskScopeApp), func(t *task.Task) error { return itemHandler() }, nil, 3, time.Hour)
+	backupTask.AddSubTaskWithOps(task.GetTaskName(install.Name, task.TaskBackup, task.TaskScopeBackup), func(t *task.Task) error { return itemHandler() }, nil, 3, time.Hour)
 	go func() {
 		if err := backupTask.Execute(); err != nil {
 			backupRepo.UpdateRecordByMap(recordID, map[string]interface{}{"status": constant.StatusFailed, "message": err.Error()})
@@ -160,7 +160,7 @@ func handleAppRecover(install *model.AppInstall, parentTask *task.Task, recoverF
 	)
 	recoverTask = parentTask
 	if parentTask == nil {
-		recoverTask, err = task.NewTaskWithOps(install.Name, task.TaskRecover, task.TaskScopeApp, taskID, install.ID)
+		recoverTask, err = task.NewTaskWithOps(install.Name, task.TaskRecover, task.TaskScopeBackup, taskID, install.ID)
 		if err != nil {
 			return err
 		}
@@ -328,7 +328,7 @@ func handleAppRecover(install *model.AppInstall, parentTask *task.Task, recoverF
 		return recoverApp(parentTask)
 	}
 
-	recoverTask.AddSubTask(task.GetTaskName(install.Name, task.TaskRecover, task.TaskScopeApp), recoverApp, rollBackApp)
+	recoverTask.AddSubTask(task.GetTaskName(install.Name, task.TaskRecover, task.TaskScopeBackup), recoverApp, rollBackApp)
 	go func() {
 		_ = recoverTask.Execute()
 	}()

agent/app/service/backup_mysql.go

@@ -84,7 +84,7 @@ func handleMysqlBackup(db DatabaseHelper, parentTask *task.Task, recordID uint,
 	}
 	itemName := fmt.Sprintf("%s[%s] - %s", db.Database, db.DBType, db.Name)
 	if parentTask == nil {
-		backupTask, err = task.NewTaskWithOps(itemName, task.TaskBackup, task.TaskScopeDatabase, taskID, dbInfo.ID)
+		backupTask, err = task.NewTaskWithOps(itemName, task.TaskBackup, task.TaskScopeBackup, taskID, dbInfo.ID)
 		if err != nil {
 			return err
 		}
@@ -94,7 +94,7 @@ func handleMysqlBackup(db DatabaseHelper, parentTask *task.Task, recordID uint,
 	if parentTask != nil {
 		return itemHandler()
 	}
-	backupTask.AddSubTaskWithOps(task.GetTaskName(itemName, task.TaskBackup, task.TaskScopeDatabase), func(t *task.Task) error { return itemHandler() }, nil, 3, time.Hour)
+	backupTask.AddSubTaskWithOps(task.GetTaskName(itemName, task.TaskBackup, task.TaskScopeBackup), func(t *task.Task) error { return itemHandler() }, nil, 3, time.Hour)
 	go func() {
 		if err := backupTask.Execute(); err != nil {
 			backupRepo.UpdateRecordByMap(recordID, map[string]interface{}{"status": constant.StatusFailed, "message": err.Error()})
@@ -117,7 +117,7 @@ func handleMysqlRecover(req dto.CommonRecover, parentTask *task.Task, isRollback
 	}
 	itemName := fmt.Sprintf("%s[%s] - %s", req.Name, req.Type, req.DetailName)
 	if parentTask == nil {
-		itemTask, err = task.NewTaskWithOps(itemName, task.TaskRecover, task.TaskScopeDatabase, taskID, dbInfo.ID)
+		itemTask, err = task.NewTaskWithOps(itemName, task.TaskRecover, task.TaskScopeBackup, taskID, dbInfo.ID)
 		if err != nil {
 			return err
 		}

agent/app/service/backup_postgresql.go

@@ -79,7 +79,7 @@ func handlePostgresqlBackup(db DatabaseHelper, parentTask *task.Task, recordID u
 	backupTask = parentTask
 	itemName := fmt.Sprintf("%s - %s", db.Database, db.Name)
 	if parentTask == nil {
-		backupTask, err = task.NewTaskWithOps(itemName, task.TaskBackup, task.TaskScopeDatabase, taskID, db.ID)
+		backupTask, err = task.NewTaskWithOps(itemName, task.TaskBackup, task.TaskScopeBackup, taskID, db.ID)
 		if err != nil {
 			return err
 		}
@@ -89,7 +89,7 @@ func handlePostgresqlBackup(db DatabaseHelper, parentTask *task.Task, recordID u
 	if parentTask != nil {
 		return itemHandler()
 	}
-	backupTask.AddSubTaskWithOps(task.GetTaskName(itemName, task.TaskBackup, task.TaskScopeDatabase), func(t *task.Task) error { return itemHandler() }, nil, 3, time.Hour)
+	backupTask.AddSubTaskWithOps(task.GetTaskName(itemName, task.TaskBackup, task.TaskScopeBackup), func(t *task.Task) error { return itemHandler() }, nil, 3, time.Hour)
 	go func() {
 		if err := backupTask.Execute(); err != nil {
 			backupRepo.UpdateRecordByMap(recordID, map[string]interface{}{"status": constant.StatusFailed, "message": err.Error()})
@@ -111,7 +111,7 @@ func handlePostgresqlRecover(req dto.CommonRecover, parentTask *task.Task, isRol
 	}
 	itemTask = parentTask
 	if parentTask == nil {
-		itemTask, err = task.NewTaskWithOps(req.Name, task.TaskRecover, task.TaskScopeDatabase, req.TaskID, dbInfo.ID)
+		itemTask, err = task.NewTaskWithOps(req.Name, task.TaskRecover, task.TaskScopeBackup, req.TaskID, dbInfo.ID)
 		if err != nil {
 			return err
 		}

agent/app/service/backup_redis.go

@@ -84,7 +84,7 @@ func handleRedisBackup(redisInfo *repo.RootInfo, parentTask *task.Task, recordID
 	)
 	itemTask = parentTask
 	if parentTask == nil {
-		itemTask, err = task.NewTaskWithOps("Redis", task.TaskBackup, task.TaskScopeDatabase, taskID, redisInfo.ID)
+		itemTask, err = task.NewTaskWithOps("Redis", task.TaskBackup, task.TaskScopeBackup, taskID, redisInfo.ID)
 		if err != nil {
 			return err
 		}
@@ -147,7 +147,7 @@ func handleRedisRecover(redisInfo *repo.RootInfo, parentTask *task.Task, recover
 	)
 	itemTask = parentTask
 	if parentTask == nil {
-		itemTask, err = task.NewTaskWithOps("Redis", task.TaskRecover, task.TaskScopeDatabase, taskID, redisInfo.ID)
+		itemTask, err = task.NewTaskWithOps("Redis", task.TaskRecover, task.TaskScopeBackup, taskID, redisInfo.ID)
 		if err != nil {
 			return err
 		}

agent/app/service/backup_website.go

@@ -73,11 +73,11 @@ func (u *BackupService) WebsiteRecover(req dto.CommonRecover) error {
 }
 
 func handleWebsiteRecover(website *model.Website, recoverFile string, isRollback bool, secret, taskID string) error {
-	recoverTask, err := task.NewTaskWithOps(website.PrimaryDomain, task.TaskRecover, task.TaskScopeWebsite, taskID, website.ID)
+	recoverTask, err := task.NewTaskWithOps(website.PrimaryDomain, task.TaskRecover, task.TaskScopeBackup, taskID, website.ID)
 	if err != nil {
 		return err
 	}
-	recoverTask.AddSubTask(task.GetTaskName(website.PrimaryDomain, task.TaskRecover, task.TaskScopeWebsite), func(t *task.Task) error {
+	recoverTask.AddSubTask(task.GetTaskName(website.PrimaryDomain, task.TaskRecover, task.TaskScopeBackup), func(t *task.Task) error {
 		isOk := false
 		if !isRollback {
 			rollbackFile := path.Join(global.Dir.TmpDir, fmt.Sprintf("website/%s_%s.tar.gz", website.Alias, time.Now().Format(constant.DateTimeSlimLayout)))
@@ -212,7 +212,7 @@ func handleWebsiteBackup(website *model.Website, parentTask *task.Task, recordID
 	)
 	backupTask = parentTask
 	if parentTask == nil {
-		backupTask, err = task.NewTaskWithOps(website.Alias, task.TaskBackup, task.TaskScopeWebsite, taskID, website.ID)
+		backupTask, err = task.NewTaskWithOps(website.Alias, task.TaskBackup, task.TaskScopeBackup, taskID, website.ID)
 		if err != nil {
 			return err
 		}
@@ -221,7 +221,7 @@ func handleWebsiteBackup(website *model.Website, parentTask *task.Task, recordID
 	if parentTask != nil {
 		return itemHandler()
 	}
-	backupTask.AddSubTaskWithOps(task.GetTaskName(website.Alias, task.TaskBackup, task.TaskScopeWebsite), func(t *task.Task) error { return itemHandler() }, nil, 3, time.Hour)
+	backupTask.AddSubTaskWithOps(task.GetTaskName(website.Alias, task.TaskBackup, task.TaskScopeBackup), func(t *task.Task) error { return itemHandler() }, nil, 3, time.Hour)
 	go func() {
 		if err := backupTask.Execute(); err != nil {
 			backupRepo.UpdateRecordByMap(recordID, map[string]interface{}{"status": constant.StatusFailed, "message": err.Error()})

agent/app/service/device_clean.go

@@ -7,6 +7,7 @@ import (
 	"path"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -18,6 +19,7 @@ import (
 	"github.com/docker/docker/api/types/filters"
 
 	"github.com/1Panel-dev/1Panel/agent/app/dto"
+	"github.com/1Panel-dev/1Panel/agent/app/model"
 	"github.com/1Panel-dev/1Panel/agent/app/repo"
 	"github.com/1Panel-dev/1Panel/agent/app/task"
 	"github.com/1Panel-dev/1Panel/agent/global"
@@ -74,9 +76,22 @@ func (u *DeviceService) Scan() dto.CleanData {
 		sort.Slice(upgradeTree.Children, func(i, j int) bool {
 			return common.CompareVersion(upgradeTree.Children[i].Label, upgradeTree.Children[j].Label)
 		})
+		if global.IsMaster {
+			var copiesSeeting model.Setting
+			_ = global.CoreDB.Where("key = ?", "UpgradeBackupCopies").First(&copiesSeeting).Error
+			copies, _ := strconv.Atoi(copiesSeeting.Value)
+			if copies == 0 || copies > len(upgradeTree.Children) {
+				copies = len(upgradeTree.Children)
+			}
+			for i := 0; i < copies; i++ {
+				upgradeTree.Children[i].IsCheck = false
+				upgradeTree.Children[i].IsRecommend = false
+			}
+		} else {
 			upgradeTree.Children[0].IsCheck = false
 			upgradeTree.Children[0].IsRecommend = false
 		}
+	}
 	treeData = append(treeData, upgradeTree)
 
 	tmpBackupTree := loadTmpBackupTree(fileOp)
@@ -214,7 +229,7 @@ func (u *DeviceService) Clean(req []dto.Clean) {
 			}
 		case "task_log":
 			if len(item.Name) == 0 {
-				files, _ := os.ReadDir(path.Join(global.Dir.TaskDir))
+				files, _ := os.ReadDir(global.Dir.TaskDir)
 				if len(files) == 0 {
 					continue
 				}
@@ -222,15 +237,10 @@ func (u *DeviceService) Clean(req []dto.Clean) {
 					if file.Name() == "ssl" || !file.IsDir() {
 						continue
 					}
-					dropFileOrDir(path.Join(global.Dir.TaskDir, file.Name()))
+					dropTaskLog(path.Join(global.Dir.TaskDir, file.Name()))
 				}
-				_ = taskRepo.DeleteAll()
 			} else {
-				pathItem := path.Join(global.Dir.TaskDir, item.Name)
+				dropTaskLog(path.Join(global.Dir.TaskDir, item.Name))
-				dropFileOrDir(pathItem)
-				if len(item.Name) != 0 {
-					_ = taskRepo.Delete(repo.WithByType(item.Name))
-				}
 			}
 		case "script":
 			dropFileOrDir(path.Join(global.Dir.TmpDir, "script", item.Name))
@@ -530,7 +540,6 @@ func loadTreeWithAllFile(isCheck bool, originalPath, treeType, pathItem string,
 }
 
 func dropFileOrDir(itemPath string) {
-	global.LOG.Debugf("drop file %s", itemPath)
 	if err := os.RemoveAll(itemPath); err != nil {
 		global.LOG.Errorf("drop file %s failed, err %v", itemPath, err)
 	}
@@ -606,6 +615,62 @@ func dropVolumes() (int, int) {
 	return len(res.VolumesDeleted), int(res.SpaceReclaimed)
 }
 
+func dropTaskLog(logDir string) {
+	files, err := os.ReadDir(logDir)
+	if err != nil {
+		return
+	}
+	taskType := path.Base(logDir)
+	var usedTasks []string
+	switch taskType {
+	case "Cronjob":
+		_ = global.DB.Model(&model.JobRecords{}).Where("task_id != ?", "").Select("task_id").Find(&usedTasks).Error
+	case "Snapshot":
+		var (
+			snapIDs     []string
+			recoverIDs  []string
+			rollbackIDs []string
+		)
+		_ = global.DB.Model(&model.Snapshot{}).Where("task_id != ?", "").Select("task_id").Find(&snapIDs).Error
+		_ = global.DB.Model(&model.Snapshot{}).Where("task_recover_id != ", "").Select("task_id").Find(&recoverIDs).Error
+		_ = global.DB.Model(&model.Snapshot{}).Where("task_rollback_id != ?", "").Select("task_id").Find(&rollbackIDs).Error
+		usedTasks = append(usedTasks, snapIDs...)
+		usedTasks = append(usedTasks, recoverIDs...)
+		usedTasks = append(usedTasks, rollbackIDs...)
+	case "Backup":
+		_ = global.DB.Model(&model.BackupRecord{}).Where("task_id != ?", "").Select("task_id").Find(&usedTasks).Error
+	case "Clam":
+		_ = global.DB.Model(&model.ClamRecord{}).Where("task_id != ?", "").Select("task_id").Find(&usedTasks).Error
+	case "Tamper":
+		xpackDB, err := common.LoadDBConnByPathWithErr(path.Join(global.CONF.Base.InstallDir, "1panel/db/xpack.db"), "xpack.db")
+		if err == nil {
+			_ = xpackDB.Table("tampers").Where("task_id != ?", "").Select("task_id").Find(&usedTasks).Error
+		}
+	case "System":
+		xpackDB, err := common.LoadDBConnByPathWithErr(path.Join(global.CONF.Base.InstallDir, "1panel/db/xpack.db"), "xpack.db")
+		if err == nil {
+			_ = xpackDB.Model("nodes").Where("task_id != ?", "").Select("task_id").Find(&usedTasks).Error
+		}
+	default:
+		dropFileOrDir(logDir)
+		_ = taskRepo.Delete(repo.WithByType(taskType))
+		return
+	}
+	usedMap := make(map[string]struct{})
+	for _, item := range usedTasks {
+		if _, ok := usedMap[item]; !ok {
+			usedMap[item] = struct{}{}
+		}
+	}
+	for _, item := range files {
+		if _, ok := usedMap[strings.TrimSuffix(item.Name(), ".log")]; ok {
+			continue
+		}
+		_ = os.Remove(logDir + "/" + item.Name())
+	}
+	_ = taskRepo.Delete(repo.WithByType(taskType), taskRepo.WithByIDNotIn(usedTasks))
+}
+
 func dropWithExclude(pathToDelete string, excludeSubDirs []string, taskItem *task.Task, size *int64, count *int) {
 	entries, err := os.ReadDir(pathToDelete)
 	if err != nil {

agent/app/task/task.go

@@ -87,6 +87,7 @@ const (
 	TaskScopeContainer        = "Container"
 	TaskScopeCompose          = "Compose"
 	TaskScopeImage            = "Image"
+	TaskScopeBackup           = "Backup "
 	TaskScopeRuntimeExtension = "RuntimeExtension"
 	TaskScopeCustomAppstore   = "CustomAppstore"
 	TaskScopeTamper           = "Tamper"