diff --git a/agent/app/api/v2/snapshot.go b/agent/app/api/v2/snapshot.go index fffff3e13..88f8d4543 100644 --- a/agent/app/api/v2/snapshot.go +++ b/agent/app/api/v2/snapshot.go @@ -7,6 +7,21 @@ import ( "github.com/gin-gonic/gin" ) +// @Tags System Setting +// @Summary Load system snapshot data +// @Description 获取系统快照数据 +// @Success 200 +// @Security ApiKeyAuth +// @Router /settings/snapshot/load [get] +func (b *BaseApi) LoadSnapshotData(c *gin.Context) { + data, err := snapshotService.LoadSnapshotData() + if err != nil { + helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err) + return + } + helper.SuccessWithData(c, data) +} + // @Tags System Setting // @Summary Create system snapshot // @Description 创建系统快照 diff --git a/agent/app/dto/setting.go b/agent/app/dto/setting.go index a7ac6d679..d9d015f3d 100644 --- a/agent/app/dto/setting.go +++ b/agent/app/dto/setting.go @@ -1,7 +1,5 @@ package dto -import "time" - type SettingInfo struct { SystemIP string `json:"systemIP"` DockerSockPath string `json:"dockerSockPath"` @@ -34,64 +32,6 @@ type SettingUpdate struct { Value string `json:"value"` } -type SnapshotStatus struct { - Panel string `json:"panel"` - PanelInfo string `json:"panelInfo"` - DaemonJson string `json:"daemonJson"` - AppData string `json:"appData"` - PanelData string `json:"panelData"` - BackupData string `json:"backupData"` - - Compress string `json:"compress"` - Size string `json:"size"` - Upload string `json:"upload"` -} - -type SnapshotCreate struct { - ID uint `json:"id"` - SourceAccountIDs string `json:"sourceAccountIDs" validate:"required"` - DownloadAccountID uint `json:"downloadAccountID" validate:"required"` - Description string `json:"description" validate:"max=256"` - Secret string `json:"secret"` -} -type SnapshotRecover struct { - IsNew bool `json:"isNew"` - ReDownload bool `json:"reDownload"` - ID uint `json:"id" validate:"required"` - Secret string `json:"secret"` -} -type SnapshotBatchDelete struct { - DeleteWithFile bool `json:"deleteWithFile"` - Ids []uint `json:"ids" validate:"required"` -} - -type SnapshotImport struct { - BackupAccountID uint `json:"backupAccountID"` - Names []string `json:"names"` - Description string `json:"description" validate:"max=256"` -} - -type SnapshotInfo struct { - ID uint `json:"id"` - Name string `json:"name"` - Description string `json:"description" validate:"max=256"` - From string `json:"from"` - DefaultDownload string `json:"defaultDownload"` - Status string `json:"status"` - Message string `json:"message"` - CreatedAt time.Time `json:"createdAt"` - Version string `json:"version"` - Size int64 `json:"size"` - - InterruptStep string `json:"interruptStep"` - RecoverStatus string `json:"recoverStatus"` - RecoverMessage string `json:"recoverMessage"` - LastRecoveredAt string `json:"lastRecoveredAt"` - RollbackStatus string `json:"rollbackStatus"` - RollbackMessage string `json:"rollbackMessage"` - LastRollbackedAt string `json:"lastRollbackedAt"` -} - type SyncTime struct { NtpSite string `json:"ntpSite" validate:"required"` } diff --git a/agent/app/dto/snapshot.go b/agent/app/dto/snapshot.go new file mode 100644 index 000000000..9504cfc96 --- /dev/null +++ b/agent/app/dto/snapshot.go @@ -0,0 +1,90 @@ +package dto + +import "time" + +type SnapshotStatus struct { + BaseData string `json:"baseData"` + AppImage string `json:"appImage"` + PanelData string `json:"panelData"` + BackupData string `json:"backupData"` + + Compress string `json:"compress"` + Size string `json:"size"` + Upload string `json:"upload"` +} + +type SnapshotCreate struct { + ID uint `json:"id"` + SourceAccountIDs string `json:"sourceAccountIDs" validate:"required"` + DownloadAccountID uint `json:"downloadAccountID" validate:"required"` + Description string `json:"description" validate:"max=256"` + Secret string `json:"secret"` + + AppData []DataTree `json:"appData"` + BackupData []DataTree `json:"backupData"` + PanelData []DataTree `json:"panelData"` + + WithMonitorData bool `json:"withMonitorData"` + WithLoginLog bool `json:"withLoginLog"` + WithOperationLog bool `json:"withOperationLog"` +} + +type SnapshotData struct { + AppData []DataTree `json:"appData"` + BackupData []DataTree `json:"backupData"` + PanelData []DataTree `json:"panelData"` + + WithMonitorData bool `json:"withMonitorData"` + WithLoginLog bool `json:"withLoginLog"` + WithOperationLog bool `json:"withOperationLog"` +} +type DataTree struct { + ID string `json:"id"` + Label string `json:"label"` + Key string `json:"key"` + Name string `json:"name"` + Size uint64 `json:"size"` + IsCheck bool `json:"isCheck"` + IsDisable bool `json:"isDisable"` + + Path string `json:"path"` + + Children []DataTree `json:"children"` +} +type SnapshotRecover struct { + IsNew bool `json:"isNew"` + ReDownload bool `json:"reDownload"` + ID uint `json:"id" validate:"required"` + Secret string `json:"secret"` +} +type SnapshotBatchDelete struct { + DeleteWithFile bool `json:"deleteWithFile"` + Ids []uint `json:"ids" validate:"required"` +} + +type SnapshotImport struct { + BackupAccountID uint `json:"backupAccountID"` + Names []string `json:"names"` + Description string `json:"description" validate:"max=256"` +} + +type SnapshotInfo struct { + ID uint `json:"id"` + Name string `json:"name"` + Description string `json:"description" validate:"max=256"` + From string `json:"from"` + DefaultDownload string `json:"defaultDownload"` + Status string `json:"status"` + Message string `json:"message"` + CreatedAt time.Time `json:"createdAt"` + Version string `json:"version"` + Size int64 `json:"size"` + + InterruptStep string `json:"interruptStep"` + RecoverStatus string `json:"recoverStatus"` + RecoverMessage string `json:"recoverMessage"` + LastRecoveredAt string `json:"lastRecoveredAt"` + RollbackStatus string `json:"rollbackStatus"` + RollbackMessage string `json:"rollbackMessage"` + LastRollbackedAt string `json:"lastRollbackedAt"` +} diff --git a/agent/app/model/snapshot.go b/agent/app/model/snapshot.go index 177105112..bf8cc3b43 100644 --- a/agent/app/model/snapshot.go +++ b/agent/app/model/snapshot.go @@ -10,6 +10,13 @@ type Snapshot struct { Message string `json:"message"` Version string `json:"version"` + AppData string `json:"appData"` + PanelData string `json:"panelData"` + BackupData string `json:"backupData"` + WithMonitorData bool `json:"withMonitorData"` + WithLoginLog bool `json:"withLoginLog"` + WithOperationLog bool `json:"withOperationLog"` + InterruptStep string `json:"interruptStep"` RecoverStatus string `json:"recoverStatus"` RecoverMessage string `json:"recoverMessage"` @@ -21,11 +28,10 @@ type Snapshot struct { type SnapshotStatus struct { BaseModel - SnapID uint `json:"snapID"` - Panel string `json:"panel" gorm:"default:Running"` - PanelInfo string `json:"panelInfo" gorm:"default:Running"` - DaemonJson string `json:"daemonJson" gorm:"default:Running"` - AppData string `json:"appData" gorm:"default:Running"` + SnapID uint `json:"snapID"` + + BaseData string `json:"baseData" gorm:"default:Running"` + AppImage string `json:"appImage" gorm:"default:Running"` PanelData string `json:"panelData" gorm:"default:Running"` BackupData string `json:"backupData" gorm:"default:Running"` diff --git a/agent/app/service/snapshot.go b/agent/app/service/snapshot.go index b398efe1c..2b4a5efec 100644 --- a/agent/app/service/snapshot.go +++ b/agent/app/service/snapshot.go @@ -16,7 +16,11 @@ import ( "github.com/1Panel-dev/1Panel/agent/global" "github.com/1Panel-dev/1Panel/agent/utils/cmd" "github.com/1Panel-dev/1Panel/agent/utils/compose" + "github.com/1Panel-dev/1Panel/agent/utils/docker" "github.com/1Panel-dev/1Panel/agent/utils/files" + fileUtils "github.com/1Panel-dev/1Panel/agent/utils/files" + "github.com/docker/docker/api/types/image" + "github.com/google/uuid" "github.com/jinzhu/copier" "github.com/pkg/errors" "github.com/shirou/gopsutil/v3/host" @@ -28,6 +32,7 @@ type SnapshotService struct { type ISnapshotService interface { SearchWithPage(req dto.SearchWithPage) (int64, interface{}, error) + LoadSnapshotData() (dto.SnapshotData, error) SnapshotCreate(req dto.SnapshotCreate) error SnapshotRecover(req dto.SnapshotRecover) error SnapshotRollback(req dto.SnapshotRecover) error @@ -92,6 +97,37 @@ func (u *SnapshotService) SnapshotImport(req dto.SnapshotImport) error { return nil } +func (u *SnapshotService) LoadSnapshotData() (dto.SnapshotData, error) { + var ( + data dto.SnapshotData + err error + ) + fileOp := fileUtils.NewFileOp() + data.AppData, err = loadApps(fileOp) + if err != nil { + return data, err + } + data.PanelData, err = loadPanelFile(fileOp) + if err != nil { + return data, err + } + itemBackups, err := loadFile(global.CONF.System.Backup, 8, fileOp) + if err != nil { + return data, err + } + for i, item := range itemBackups { + if item.Label == "app" { + data.BackupData = append(itemBackups[:i], itemBackups[i+1:]...) + break + } + } + data.WithLoginLog = true + data.WithOperationLog = true + data.WithMonitorData = false + + return data, nil +} + func (u *SnapshotService) UpdateDescription(req dto.UpdateDescription) error { return snapshotRepo.Update(req.ID, map[string]interface{}{"description": req.Description}) } @@ -109,16 +145,9 @@ func (u *SnapshotService) LoadSnapShotStatus(id uint) (*dto.SnapshotStatus, erro } type SnapshotJson struct { - OldBaseDir string `json:"oldBaseDir"` - OldDockerDataDir string `json:"oldDockerDataDir"` - OldBackupDataDir string `json:"oldBackupDataDir"` - OldPanelDataDir string `json:"oldPanelDataDir"` - - BaseDir string `json:"baseDir"` - DockerDataDir string `json:"dockerDataDir"` - BackupDataDir string `json:"backupDataDir"` - PanelDataDir string `json:"panelDataDir"` - LiveRestoreEnabled bool `json:"liveRestoreEnabled"` + BaseDir string `json:"baseDir"` + BackupDataDir string `json:"backupDataDir"` + Size uint64 `json:"size"` } func (u *SnapshotService) SnapshotCreate(req dto.SnapshotCreate) error { @@ -148,7 +177,7 @@ func (u *SnapshotService) SnapshotRecover(req dto.SnapshotRecover) error { _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"recover_status": constant.StatusWaiting}) _ = settingRepo.Update("SystemStatus", "Recovering") - go u.HandleSnapshotRecover(snap, true, req) + go u.HandleSnapshotRecover(snap, req) return nil } @@ -158,9 +187,11 @@ func (u *SnapshotService) SnapshotRollback(req dto.SnapshotRecover) error { if err != nil { return err } - req.IsNew = false - snap.InterruptStep = "Readjson" - go u.HandleSnapshotRecover(snap, false, req) + go func() { + if err := handleRollback(snap.Name); err != nil { + global.LOG.Errorf("handle roll back snapshot failed, err: %v", err) + } + }() return nil } @@ -194,15 +225,26 @@ func (u *SnapshotService) HandleSnapshot(isCronjob bool, logPath string, req dto if isCronjob { name = fmt.Sprintf("snapshot_1panel_%s_%s_%s", versionItem.Value, loadOs(), timeNow) } - rootDir = path.Join(global.CONF.System.Backup, "system", name) + rootDir = path.Join(global.CONF.System.BaseDir, "1panel/tmp/system", name) + appItem, _ := json.Marshal(req.AppData) + panelItem, _ := json.Marshal(req.PanelData) + backupItem, _ := json.Marshal(req.BackupData) snap = model.Snapshot{ Name: name, Description: req.Description, SourceAccountIDs: req.SourceAccountIDs, DownloadAccountID: req.DownloadAccountID, - Version: versionItem.Value, - Status: constant.StatusWaiting, + + AppData: string(appItem), + PanelData: string(panelItem), + BackupData: string(backupItem), + WithMonitorData: req.WithMonitorData, + WithLoginLog: req.WithLoginLog, + WithOperationLog: req.WithOperationLog, + + Version: versionItem.Value, + Status: constant.StatusWaiting, } _ = snapshotRepo.Create(&snap) snapStatus.SnapID = snap.ID @@ -218,57 +260,43 @@ func (u *SnapshotService) HandleSnapshot(isCronjob bool, logPath string, req dto snapStatus.SnapID = snap.ID _ = snapshotRepo.CreateStatus(&snapStatus) } - rootDir = path.Join(global.CONF.System.Backup, fmt.Sprintf("system/%s", snap.Name)) + rootDir = path.Join(global.CONF.System.BaseDir, "1panel/tmp/system", snap.Name) } var wg sync.WaitGroup itemHelper := snapHelper{SnapID: snap.ID, Status: &snapStatus, Wg: &wg, FileOp: files.NewFileOp(), Ctx: context.Background()} - backupPanelDir := path.Join(rootDir, "1panel") - _ = os.MkdirAll(backupPanelDir, os.ModePerm) - backupDockerDir := path.Join(rootDir, "docker") - _ = os.MkdirAll(backupDockerDir, os.ModePerm) + baseDir := path.Join(rootDir, "base") + _ = os.MkdirAll(baseDir, os.ModePerm) + if err := loadDbConn(&itemHelper, rootDir, req); err != nil { + return "", fmt.Errorf("load snapshot db conn failed, err: %v", err) + } - jsonItem := SnapshotJson{ - BaseDir: global.CONF.System.BaseDir, - BackupDataDir: global.CONF.System.Backup, - PanelDataDir: path.Join(global.CONF.System.BaseDir, "1panel"), - } loadLogByStatus(snapStatus, logPath) - if snapStatus.PanelInfo != constant.StatusDone { + if snapStatus.BaseData != constant.StatusDone { wg.Add(1) - go snapJson(itemHelper, jsonItem, rootDir) + go snapBaseData(itemHelper, baseDir) } - if snapStatus.Panel != constant.StatusDone { + if snapStatus.AppImage != constant.StatusDone { wg.Add(1) - go snapPanel(itemHelper, backupPanelDir) - } - if snapStatus.DaemonJson != constant.StatusDone { - wg.Add(1) - go snapDaemonJson(itemHelper, backupDockerDir) - } - if snapStatus.AppData != constant.StatusDone { - wg.Add(1) - go snapAppData(itemHelper, backupDockerDir) + go snapAppImage(itemHelper, req, rootDir) } if snapStatus.BackupData != constant.StatusDone { wg.Add(1) - go snapBackup(itemHelper, backupPanelDir) + go snapBackupData(itemHelper, req, rootDir) + } + if snapStatus.PanelData != constant.StatusDone { + wg.Add(1) + go snapPanelData(itemHelper, req, rootDir) } - if !isCronjob { go func() { wg.Wait() + closeDatabase(itemHelper.snapAgentDB) + closeDatabase(itemHelper.snapCoreDB) if !checkIsAllDone(snap.ID) { _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) return } - if snapStatus.PanelData != constant.StatusDone { - snapPanelData(itemHelper, backupPanelDir) - } - if snapStatus.PanelData != constant.StatusDone { - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) - return - } if snapStatus.Compress != constant.StatusDone { snapCompress(itemHelper, rootDir, secret) } @@ -284,23 +312,20 @@ func (u *SnapshotService) HandleSnapshot(isCronjob bool, logPath string, req dto return } _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusSuccess}) + // _ = snapshotRepo.DeleteStatus(itemHelper.SnapID) + _ = os.RemoveAll(rootDir) }() return "", nil } wg.Wait() + closeDatabase(itemHelper.snapAgentDB) + closeDatabase(itemHelper.snapCoreDB) if !checkIsAllDone(snap.ID) { _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) loadLogByStatus(snapStatus, logPath) return snap.Name, fmt.Errorf("snapshot %s backup failed", snap.Name) } loadLogByStatus(snapStatus, logPath) - snapPanelData(itemHelper, backupPanelDir) - if snapStatus.PanelData != constant.StatusDone { - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) - loadLogByStatus(snapStatus, logPath) - return snap.Name, fmt.Errorf("snapshot %s 1panel data failed", snap.Name) - } - loadLogByStatus(snapStatus, logPath) snapCompress(itemHelper, rootDir, secret) if snapStatus.Compress != constant.StatusDone { _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) @@ -316,6 +341,7 @@ func (u *SnapshotService) HandleSnapshot(isCronjob bool, logPath string, req dto } _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusSuccess}) loadLogByStatus(snapStatus, logPath) + _ = os.RemoveAll(rootDir) return snap.Name, nil } @@ -341,71 +367,6 @@ func (u *SnapshotService) Delete(req dto.SnapshotBatchDelete) error { return nil } -func updateRecoverStatus(id uint, isRecover bool, interruptStep, status, message string) { - if isRecover { - if status != constant.StatusSuccess { - global.LOG.Errorf("recover failed, err: %s", message) - } - if err := snapshotRepo.Update(id, map[string]interface{}{ - "interrupt_step": interruptStep, - "recover_status": status, - "recover_message": message, - "last_recovered_at": time.Now().Format(constant.DateTimeLayout), - }); err != nil { - global.LOG.Errorf("update snap recover status failed, err: %v", err) - } - _ = settingRepo.Update("SystemStatus", "Free") - return - } - _ = settingRepo.Update("SystemStatus", "Free") - if status == constant.StatusSuccess { - if err := snapshotRepo.Update(id, map[string]interface{}{ - "recover_status": "", - "recover_message": "", - "interrupt_step": "", - "rollback_status": "", - "rollback_message": "", - "last_rollbacked_at": time.Now().Format(constant.DateTimeLayout), - }); err != nil { - global.LOG.Errorf("update snap recover status failed, err: %v", err) - } - return - } - global.LOG.Errorf("rollback failed, err: %s", message) - if err := snapshotRepo.Update(id, map[string]interface{}{ - "rollback_status": status, - "rollback_message": message, - "last_rollbacked_at": time.Now().Format(constant.DateTimeLayout), - }); err != nil { - global.LOG.Errorf("update snap recover status failed, err: %v", err) - } -} - -func (u *SnapshotService) handleUnTar(sourceDir, targetDir string, secret string) error { - if _, err := os.Stat(targetDir); err != nil && os.IsNotExist(err) { - if err = os.MkdirAll(targetDir, os.ModePerm); err != nil { - return err - } - } - commands := "" - if len(secret) != 0 { - extraCmd := "openssl enc -d -aes-256-cbc -k '" + secret + "' -in " + sourceDir + " | " - commands = fmt.Sprintf("%s tar -zxvf - -C %s", extraCmd, targetDir+" > /dev/null 2>&1") - global.LOG.Debug(strings.ReplaceAll(commands, fmt.Sprintf(" %s ", secret), "******")) - } else { - commands = fmt.Sprintf("tar zxvfC %s %s", sourceDir, targetDir) - global.LOG.Debug(commands) - } - stdout, err := cmd.ExecWithTimeOut(commands, 30*time.Minute) - if err != nil { - if len(stdout) != 0 { - global.LOG.Errorf("do handle untar failed, stdout: %s, err: %v", stdout, err) - return fmt.Errorf("do handle untar failed, stdout: %s, err: %v", stdout, err) - } - } - return nil -} - func rebuildAllAppInstall() error { global.LOG.Debug("start to rebuild all app") appInstalls, err := appInstallRepo.ListBy() @@ -449,17 +410,14 @@ func checkIsAllDone(snapID uint) bool { } func checkAllDone(status model.SnapshotStatus) (bool, string) { - if status.Panel != constant.StatusDone { - return false, status.Panel + if status.BaseData != constant.StatusDone { + return false, status.BaseData } - if status.PanelInfo != constant.StatusDone { - return false, status.PanelInfo + if status.PanelData != constant.StatusDone { + return false, status.PanelData } - if status.DaemonJson != constant.StatusDone { - return false, status.DaemonJson - } - if status.AppData != constant.StatusDone { - return false, status.AppData + if status.AppImage != constant.StatusDone { + return false, status.AppImage } if status.BackupData != constant.StatusDone { return false, status.BackupData @@ -469,10 +427,8 @@ func checkAllDone(status model.SnapshotStatus) (bool, string) { func loadLogByStatus(status model.SnapshotStatus, logPath string) { logs := "" - logs += fmt.Sprintf("Write 1Panel basic information: %s \n", status.PanelInfo) - logs += fmt.Sprintf("Backup 1Panel system files: %s \n", status.Panel) - logs += fmt.Sprintf("Backup Docker configuration file: %s \n", status.DaemonJson) - logs += fmt.Sprintf("Backup installed apps from 1Panel: %s \n", status.AppData) + logs += fmt.Sprintf("Backup 1Panel base files: %s \n", status.BaseData) + logs += fmt.Sprintf("Backup installed apps from 1Panel: %s \n", status.AppImage) logs += fmt.Sprintf("Backup 1Panel data directory: %s \n", status.PanelData) logs += fmt.Sprintf("Backup local backup directory for 1Panel: %s \n", status.BackupData) logs += fmt.Sprintf("Create snapshot file: %s \n", status.Compress) @@ -544,3 +500,154 @@ func loadSnapSize(records []model.Snapshot) ([]dto.SnapshotInfo, error) { wg.Wait() return datas, nil } + +func loadApps(fileOp fileUtils.FileOp) ([]dto.DataTree, error) { + var data []dto.DataTree + apps, err := appInstallRepo.ListBy() + if err != nil { + return data, err + } + client, err := docker.NewDockerClient() + hasDockerClient := true + if err != nil { + hasDockerClient = false + global.LOG.Errorf("new docker client failed, err: %v", err) + } else { + defer client.Close() + } + imageList, err := client.ImageList(context.Background(), image.ListOptions{}) + if err != nil { + hasDockerClient = false + global.LOG.Errorf("load image list failed, err: %v", err) + } + + for _, app := range apps { + itemApp := dto.DataTree{ID: uuid.NewString(), Label: fmt.Sprintf("%s - %s", app.App.Name, app.Name), Key: app.App.Key, Name: app.Name} + appPath := path.Join(global.CONF.System.BaseDir, "1panel/apps", app.App.Key, app.Name) + itemAppData := dto.DataTree{ID: uuid.NewString(), Label: "appData", Key: app.App.Key, Name: app.Name, IsCheck: true, Path: appPath} + sizeItem, err := fileOp.GetDirSize(appPath) + if err == nil { + itemAppData.Size = uint64(sizeItem) + } + itemApp.Size += itemAppData.Size + itemApp.Children = append(itemApp.Children, itemAppData) + + appBackupPath := path.Join(global.CONF.System.BaseDir, "1panel/backup/app", app.App.Key, app.Name) + itemAppBackupTree, err := loadFile(appBackupPath, 8, fileOp) + itemAppBackup := dto.DataTree{ID: uuid.NewString(), Label: "appBackup", IsCheck: true, Children: itemAppBackupTree, Path: appBackupPath} + if err == nil { + backupSizeItem, err := fileOp.GetDirSize(appBackupPath) + if err == nil { + itemAppBackup.Size = uint64(backupSizeItem) + itemApp.Size += itemAppBackup.Size + } + itemApp.Children = append(itemApp.Children, itemAppBackup) + } + + itemAppImage := dto.DataTree{ID: uuid.NewString(), Label: "appImage"} + stdout, err := cmd.Execf("cat %s | grep image: ", path.Join(global.CONF.System.BaseDir, "1panel/apps", app.App.Key, app.Name, "docker-compose.yml")) + if err != nil { + itemApp.Children = append(itemApp.Children, itemAppImage) + data = append(data, itemApp) + continue + } + itemAppImage.Name = strings.ReplaceAll(strings.ReplaceAll(strings.TrimSpace(stdout), "\n", ""), "image: ", "") + if !hasDockerClient { + itemApp.Children = append(itemApp.Children, itemAppImage) + data = append(data, itemApp) + continue + } + for _, imageItem := range imageList { + for _, tag := range imageItem.RepoTags { + if tag == itemAppImage.Name { + itemAppImage.Size = uint64(imageItem.Size) + break + } + } + } + itemApp.Children = append(itemApp.Children, itemAppImage) + data = append(data, itemApp) + } + return data, nil +} + +func loadPanelFile(fileOp fileUtils.FileOp) ([]dto.DataTree, error) { + var data []dto.DataTree + snapFiles, err := os.ReadDir(path.Join(global.CONF.System.BaseDir, "1panel")) + if err != nil { + return data, err + } + for _, fileItem := range snapFiles { + itemData := dto.DataTree{ + ID: uuid.NewString(), + Label: fileItem.Name(), + IsCheck: true, + Path: path.Join(global.CONF.System.BaseDir, "1panel", fileItem.Name()), + } + switch itemData.Label { + case "agent", "conf", "db", "runtime", "secret": + itemData.IsDisable = true + case "log", "docker", "task", "clamav": + panelPath := path.Join(global.CONF.System.BaseDir, "1panel", itemData.Label) + itemData.Children, _ = loadFile(panelPath, 5, fileOp) + default: + continue + } + if fileItem.IsDir() { + sizeItem, err := fileOp.GetDirSize(path.Join(global.CONF.System.BaseDir, "1panel", itemData.Label)) + if err != nil { + continue + } + itemData.Size = uint64(sizeItem) + } else { + fileInfo, err := fileItem.Info() + if err != nil { + continue + } + itemData.Size = uint64(fileInfo.Size()) + } + if itemData.IsCheck && itemData.Size == 0 { + itemData.IsCheck = false + itemData.IsDisable = true + } + + data = append(data, itemData) + } + + return data, nil +} + +func loadFile(pathItem string, index int, fileOp fileUtils.FileOp) ([]dto.DataTree, error) { + var data []dto.DataTree + snapFiles, err := os.ReadDir(pathItem) + if err != nil { + return data, err + } + i := 0 + for _, fileItem := range snapFiles { + itemData := dto.DataTree{ + ID: uuid.NewString(), + Label: fileItem.Name(), + Name: fileItem.Name(), + Path: path.Join(pathItem, fileItem.Name()), + IsCheck: true, + } + if fileItem.IsDir() { + sizeItem, err := fileOp.GetDirSize(path.Join(pathItem, itemData.Label)) + if err != nil { + continue + } + itemData.Size = uint64(sizeItem) + itemData.Children, _ = loadFile(path.Join(pathItem, itemData.Label), index-1, fileOp) + } else { + fileInfo, err := fileItem.Info() + if err != nil { + continue + } + itemData.Size = uint64(fileInfo.Size()) + } + data = append(data, itemData) + i++ + } + return data, nil +} diff --git a/agent/app/service/snapshot_create.go b/agent/app/service/snapshot_create.go index e4a42b6c3..a93fcfee1 100644 --- a/agent/app/service/snapshot_create.go +++ b/agent/app/service/snapshot_create.go @@ -6,145 +6,219 @@ import ( "fmt" "os" "path" - "regexp" "strings" "sync" "time" + "github.com/1Panel-dev/1Panel/agent/app/dto" "github.com/1Panel-dev/1Panel/agent/app/model" "github.com/1Panel-dev/1Panel/agent/constant" "github.com/1Panel-dev/1Panel/agent/global" "github.com/1Panel-dev/1Panel/agent/utils/cmd" "github.com/1Panel-dev/1Panel/agent/utils/common" "github.com/1Panel-dev/1Panel/agent/utils/files" + "github.com/glebarez/sqlite" + "gorm.io/gorm" ) type snapHelper struct { - SnapID uint - Status *model.SnapshotStatus - Ctx context.Context - FileOp files.FileOp - Wg *sync.WaitGroup + SnapID uint + snapAgentDB *gorm.DB + snapCoreDB *gorm.DB + Status *model.SnapshotStatus + Ctx context.Context + FileOp files.FileOp + Wg *sync.WaitGroup } -func snapJson(snap snapHelper, snapJson SnapshotJson, targetDir string) { - defer snap.Wg.Done() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"panel_info": constant.Running}) - status := constant.StatusDone - remarkInfo, _ := json.MarshalIndent(snapJson, "", "\t") - if err := os.WriteFile(fmt.Sprintf("%s/snapshot.json", targetDir), remarkInfo, 0640); err != nil { - status = err.Error() +func loadDbConn(snap *snapHelper, targetDir string, req dto.SnapshotCreate) error { + global.LOG.Debug("start load snapshot db conn") + + if err := snap.FileOp.CopyDir(path.Join(global.CONF.System.BaseDir, "1panel/db"), targetDir); err != nil { + return err } - snap.Status.PanelInfo = status - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"panel_info": status}) + agentDb, err := newSnapDB(path.Join(targetDir, "db"), "agent.db") + if err != nil { + return err + } + snap.snapAgentDB = agentDb + coreDb, err := newSnapDB(path.Join(targetDir, "db"), "core.db") + if err != nil { + return err + } + snap.snapCoreDB = coreDb + + if !req.WithMonitorData { + _ = os.Remove(path.Join(targetDir, "db/monitor.db")) + } + if !req.WithOperationLog { + _ = snap.snapCoreDB.Exec("DELETE FROM operation_logs") + } + if !req.WithLoginLog { + _ = snap.snapCoreDB.Exec("DELETE FROM login_logs") + } + if err := snap.snapAgentDB.Where("id = ?", snap.SnapID).Delete(&model.Snapshot{}).Error; err != nil { + global.LOG.Errorf("delete current snapshot record failed, err: %v", err) + } + if err := snap.snapAgentDB.Where("snap_id = ?", snap.SnapID).Delete(&model.SnapshotStatus{}).Error; err != nil { + global.LOG.Errorf("delete current snapshot status record failed, err: %v", err) + } + + return nil } -func snapPanel(snap snapHelper, targetDir string) { +func snapBaseData(snap snapHelper, targetDir string) { defer snap.Wg.Done() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"panel": constant.Running}) + _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"base_data": constant.Running}) status := constant.StatusDone - if err := common.CopyFile("/usr/local/bin/1panel", path.Join(targetDir, "1panel")); err != nil { + if err := common.CopyFile("/usr/local/bin/1panel", targetDir); err != nil { + status = err.Error() + } + if err := common.CopyFile("/usr/local/bin/1panel_agent", targetDir); err != nil { status = err.Error() } - if err := common.CopyFile("/usr/local/bin/1pctl", targetDir); err != nil { status = err.Error() } - if err := common.CopyFile("/etc/systemd/system/1panel.service", targetDir); err != nil { status = err.Error() } - snap.Status.Panel = status - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"panel": status}) -} - -func snapDaemonJson(snap snapHelper, targetDir string) { - defer snap.Wg.Done() - status := constant.StatusDone - if !snap.FileOp.Stat("/etc/docker/daemon.json") { - snap.Status.DaemonJson = status - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"daemon_json": status}) - return - } - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"daemon_json": constant.Running}) - if err := common.CopyFile("/etc/docker/daemon.json", targetDir); err != nil { + if err := common.CopyFile("/etc/systemd/system/1panel_agent.service", targetDir); err != nil { status = err.Error() } - snap.Status.DaemonJson = status - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"daemon_json": status}) + + if snap.FileOp.Stat("/etc/docker/daemon.json") { + if err := common.CopyFile("/etc/docker/daemon.json", targetDir); err != nil { + status = err.Error() + } + } + + remarkInfo, _ := json.MarshalIndent(SnapshotJson{ + BaseDir: global.CONF.System.BaseDir, + BackupDataDir: global.CONF.System.Backup, + }, "", "\t") + if err := os.WriteFile(fmt.Sprintf("%s/snapshot.json", targetDir), remarkInfo, 0640); err != nil { + status = err.Error() + } + snap.Status.BaseData = status + _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"base_data": status}) } -func snapAppData(snap snapHelper, targetDir string) { +func snapAppImage(snap snapHelper, req dto.SnapshotCreate, targetDir string) { defer snap.Wg.Done() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_data": constant.Running}) - appInstalls, err := appInstallRepo.ListBy() - if err != nil { - snap.Status.AppData = err.Error() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_data": err.Error()}) - return - } - runtimes, err := runtimeRepo.List() - if err != nil { - snap.Status.AppData = err.Error() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_data": err.Error()}) - return - } - imageRegex := regexp.MustCompile(`image:\s*(.*)`) - var imageSaveList []string + _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_image": constant.Running}) + + var imageList []string existStr, _ := cmd.Exec("docker images | awk '{print $1\":\"$2}' | grep -v REPOSITORY:TAG") existImages := strings.Split(existStr, "\n") - duplicateMap := make(map[string]bool) - for _, app := range appInstalls { - matches := imageRegex.FindAllStringSubmatch(app.DockerCompose, -1) - for _, match := range matches { - for _, existImage := range existImages { - if match[1] == existImage && !duplicateMap[match[1]] { - imageSaveList = append(imageSaveList, match[1]) - duplicateMap[match[1]] = true + for _, app := range req.AppData { + for _, item := range app.Children { + if item.Label == "appImage" && item.IsCheck { + for _, existImage := range existImages { + if existImage == item.Name { + imageList = append(imageList, item.Name) + } } } } } - for _, runtime := range runtimes { - for _, existImage := range existImages { - if runtime.Image == existImage && !duplicateMap[runtime.Image] { - imageSaveList = append(imageSaveList, runtime.Image) - duplicateMap[runtime.Image] = true - } - } - } - if len(imageSaveList) != 0 { - global.LOG.Debugf("docker save %s | gzip -c > %s", strings.Join(imageSaveList, " "), path.Join(targetDir, "docker_image.tar")) - std, err := cmd.Execf("docker save %s | gzip -c > %s", strings.Join(imageSaveList, " "), path.Join(targetDir, "docker_image.tar")) + if len(imageList) != 0 { + global.LOG.Debugf("docker save %s | gzip -c > %s", strings.Join(imageList, " "), path.Join(targetDir, "images.tar.gz")) + std, err := cmd.Execf("docker save %s | gzip -c > %s", strings.Join(imageList, " "), path.Join(targetDir, "images.tar.gz")) if err != nil { - snap.Status.AppData = err.Error() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_data": std}) + snap.Status.AppImage = err.Error() + _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_image": std}) return } } - snap.Status.AppData = constant.StatusDone - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_data": constant.StatusDone}) + snap.Status.AppImage = constant.StatusDone + _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_image": constant.StatusDone}) } -func snapBackup(snap snapHelper, targetDir string) { +func snapBackupData(snap snapHelper, req dto.SnapshotCreate, targetDir string) { defer snap.Wg.Done() _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"backup_data": constant.Running}) status := constant.StatusDone - if err := handleSnapTar(global.CONF.System.Backup, targetDir, "1panel_backup.tar.gz", "./system;./system_snapshot;", ""); err != nil { + + excludes := loadBackupExcludes(snap, req.BackupData) + for _, item := range req.AppData { + for _, itemApp := range item.Children { + if itemApp.Label == "appBackup" { + excludes = append(excludes, loadAppBackupExcludes([]dto.DataTree{itemApp})...) + } + } + } + global.LOG.Debugf("excludes backup file: %v\n", strings.Join(excludes, "\n")) + + if err := snap.FileOp.TarGzCompressPro(false, global.CONF.System.Backup, path.Join(targetDir, "1panel_backup.tar.gz"), "", strings.Join(excludes, ";")); err != nil { status = err.Error() } snap.Status.BackupData = status _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"backup_data": status}) } +func loadBackupExcludes(snap snapHelper, req []dto.DataTree) []string { + var excludes []string + for _, item := range req { + if len(item.Children) == 0 { + if item.IsCheck { + continue + } + if strings.HasPrefix(item.Path, path.Join(global.CONF.System.Backup, "system_snapshot")) { + fmt.Println(strings.TrimSuffix(item.Name, ".tar.gz")) + if err := snap.snapAgentDB.Debug().Where("name = ? AND download_account_id = ?", strings.TrimSuffix(item.Name, ".tar.gz"), "1").Delete(&model.Snapshot{}).Error; err != nil { + global.LOG.Errorf("delete snapshot from database failed, err: %v", err) + } + } else { + fmt.Println(strings.TrimPrefix(path.Dir(item.Path), global.CONF.System.Backup+"/"), path.Base(item.Path)) + if err := snap.snapAgentDB.Debug().Where("file_dir = ? AND file_name = ?", strings.TrimPrefix(path.Dir(item.Path), global.CONF.System.Backup+"/"), path.Base(item.Path)).Delete(&model.BackupRecord{}).Error; err != nil { + global.LOG.Errorf("delete backup file from database failed, err: %v", err) + } + } + fmt.Println(strings.TrimPrefix(item.Path, global.CONF.System.Backup)) + excludes = append(excludes, "."+strings.TrimPrefix(item.Path, global.CONF.System.Backup)) + } else { + excludes = append(excludes, loadBackupExcludes(snap, item.Children)...) + } + } + return excludes +} +func loadAppBackupExcludes(req []dto.DataTree) []string { + var excludes []string + for _, item := range req { + if len(item.Children) == 0 { + if !item.IsCheck { + excludes = append(excludes, "."+strings.TrimPrefix(item.Path, path.Join(global.CONF.System.Backup))) + } + } else { + excludes = append(excludes, loadAppBackupExcludes(item.Children)...) + } + } + return excludes +} -func snapPanelData(snap snapHelper, targetDir string) { +func snapPanelData(snap snapHelper, req dto.SnapshotCreate, targetDir string) { + defer snap.Wg.Done() _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"panel_data": constant.Running}) status := constant.StatusDone - dataDir := path.Join(global.CONF.System.BaseDir, "1panel") - exclusionRules := "./tmp;./log;./cache;./db/1Panel.db-*;" - if strings.Contains(global.CONF.System.Backup, dataDir) { - exclusionRules += ("." + strings.ReplaceAll(global.CONF.System.Backup, dataDir, "") + ";") + + excludes := loadPanelExcludes(req.PanelData) + for _, item := range req.AppData { + for _, itemApp := range item.Children { + if itemApp.Label == "appData" { + excludes = append(excludes, loadPanelExcludes([]dto.DataTree{itemApp})...) + } + } + } + global.LOG.Debugf("excludes panel file: %v\n", strings.Join(excludes, "\n")) + excludes = append(excludes, "./tmp") + excludes = append(excludes, "./cache") + excludes = append(excludes, "./uploads") + excludes = append(excludes, "./db") + excludes = append(excludes, "./resource") + rootDir := path.Join(global.CONF.System.BaseDir, "1panel") + if strings.Contains(global.CONF.System.Backup, rootDir) { + excludes = append(excludes, "."+strings.ReplaceAll(global.CONF.System.Backup, rootDir, "")) } ignoreVal, _ := settingRepo.Get(settingRepo.WithByKey("SnapshotIgnore")) rules := strings.Split(ignoreVal.Value, ",") @@ -152,27 +226,37 @@ func snapPanelData(snap snapHelper, targetDir string) { if len(ignore) == 0 || cmd.CheckIllegal(ignore) { continue } - exclusionRules += ("." + strings.ReplaceAll(ignore, dataDir, "") + ";") + excludes = append(excludes, "."+strings.ReplaceAll(ignore, rootDir, "")) } - _ = snapshotRepo.Update(snap.SnapID, map[string]interface{}{"status": "OnSaveData"}) - sysIP, _ := settingRepo.Get(settingRepo.WithByKey("SystemIP")) - _ = settingRepo.Update("SystemIP", "") - checkPointOfWal() - if err := handleSnapTar(dataDir, targetDir, "1panel_data.tar.gz", exclusionRules, ""); err != nil { + + _ = snap.snapAgentDB.Model(&model.Setting{}).Where("key = ?", "SystemIP").Updates(map[string]interface{}{"SystemIP": ""}) + + if err := snap.FileOp.TarGzCompressPro(false, rootDir, path.Join(targetDir, "1panel_data.tar.gz"), "", strings.Join(excludes, ";")); err != nil { status = err.Error() } - _ = snapshotRepo.Update(snap.SnapID, map[string]interface{}{"status": constant.StatusWaiting}) snap.Status.PanelData = status _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"panel_data": status}) - _ = settingRepo.Update("SystemIP", sysIP.Value) +} +func loadPanelExcludes(req []dto.DataTree) []string { + var excludes []string + for _, item := range req { + if len(item.Children) == 0 { + if !item.IsCheck { + excludes = append(excludes, "."+strings.TrimPrefix(item.Path, path.Join(global.CONF.System.BaseDir, "1panel"))) + } + } else { + excludes = append(excludes, loadPanelExcludes(item.Children)...) + } + } + return excludes } func snapCompress(snap snapHelper, rootDir string, secret string) { _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"compress": constant.StatusRunning}) tmpDir := path.Join(global.CONF.System.TmpDir, "system") fileName := fmt.Sprintf("%s.tar.gz", path.Base(rootDir)) - if err := handleSnapTar(rootDir, tmpDir, fileName, "", secret); err != nil { + if err := snap.FileOp.TarGzCompressPro(true, rootDir, path.Join(tmpDir, fileName), secret, ""); err != nil { snap.Status.Compress = err.Error() _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"compress": err.Error()}) return @@ -207,12 +291,12 @@ func snapUpload(snap snapHelper, accounts string, file string) { for _, item := range targetAccounts { global.LOG.Debugf("start upload snapshot to %s, path: %s", item, path.Join(accountMap[item].backupPath, "system_snapshot", path.Base(file))) if _, err := accountMap[item].client.Upload(source, path.Join(accountMap[item].backupPath, "system_snapshot", path.Base(file))); err != nil { - global.LOG.Debugf("upload to %s failed, err: %v", item, err) + global.LOG.Debugf("upload to %s failed, err: %v", accountMap[item].name, err) snap.Status.Upload = err.Error() _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"upload": err.Error()}) return } - global.LOG.Debugf("upload to %s successful", item) + global.LOG.Debugf("upload to %s successful", accountMap[item].name) } snap.Status.Upload = constant.StatusDone _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"upload": constant.StatusDone}) @@ -221,60 +305,25 @@ func snapUpload(snap snapHelper, accounts string, file string) { _ = os.Remove(source) } -func handleSnapTar(sourceDir, targetDir, name, exclusionRules string, secret string) error { - if _, err := os.Stat(targetDir); err != nil && os.IsNotExist(err) { - if err = os.MkdirAll(targetDir, os.ModePerm); err != nil { - return err - } - } - - exMap := make(map[string]struct{}) - exStr := "" - excludes := strings.Split(exclusionRules, ";") - excludes = append(excludes, "*.sock") - for _, exclude := range excludes { - if len(exclude) == 0 { - continue - } - if _, ok := exMap[exclude]; ok { - continue - } - exStr += " --exclude " - exStr += exclude - exMap[exclude] = struct{}{} - } - path := "" - if strings.Contains(sourceDir, "/") { - itemDir := strings.ReplaceAll(sourceDir[strings.LastIndex(sourceDir, "/"):], "/", "") - aheadDir := sourceDir[:strings.LastIndex(sourceDir, "/")] - if len(aheadDir) == 0 { - aheadDir = "/" - } - path += fmt.Sprintf("-C %s %s", aheadDir, itemDir) - } else { - path = sourceDir - } - commands := "" - if len(secret) != 0 { - extraCmd := "| openssl enc -aes-256-cbc -salt -k '" + secret + "' -out" - commands = fmt.Sprintf("tar --warning=no-file-changed --ignore-failed-read -zcf %s %s %s %s", " -"+exStr, path, extraCmd, targetDir+"/"+name) - global.LOG.Debug(strings.ReplaceAll(commands, fmt.Sprintf(" %s ", secret), "******")) - } else { - commands = fmt.Sprintf("tar --warning=no-file-changed --ignore-failed-read -zcf %s %s -C %s .", targetDir+"/"+name, exStr, sourceDir) - global.LOG.Debug(commands) - } - stdout, err := cmd.ExecWithTimeOut(commands, 30*time.Minute) +func newSnapDB(dir, file string) (*gorm.DB, error) { + db, _ := gorm.Open(sqlite.Open(path.Join(dir, file)), &gorm.Config{ + DisableForeignKeyConstraintWhenMigrating: true, + }) + sqlDB, err := db.DB() if err != nil { - if len(stdout) != 0 { - global.LOG.Errorf("do handle tar failed, stdout: %s, err: %v", stdout, err) - return fmt.Errorf("do handle tar failed, stdout: %s, err: %v", stdout, err) - } + return nil, err } - return nil + sqlDB.SetConnMaxIdleTime(10) + sqlDB.SetMaxOpenConns(100) + sqlDB.SetConnMaxLifetime(time.Hour) + // global.LOG.Debug("load snapshot db conn successful!") + return db, nil } -func checkPointOfWal() { - if err := global.DB.Exec("PRAGMA wal_checkpoint(TRUNCATE);").Error; err != nil { - global.LOG.Errorf("handle check point failed, err: %v", err) +func closeDatabase(db *gorm.DB) { + sqlDB, err := db.DB() + if err != nil { + return } + _ = sqlDB.Close() } diff --git a/agent/app/service/snapshot_recover.go b/agent/app/service/snapshot_recover.go index f125c6e75..9c283835f 100644 --- a/agent/app/service/snapshot_recover.go +++ b/agent/app/service/snapshot_recover.go @@ -1,124 +1,98 @@ package service import ( - "context" "fmt" "os" "path" "strings" - "sync" + "time" "github.com/1Panel-dev/1Panel/agent/app/dto" "github.com/1Panel-dev/1Panel/agent/app/model" "github.com/1Panel-dev/1Panel/agent/constant" "github.com/1Panel-dev/1Panel/agent/global" "github.com/1Panel-dev/1Panel/agent/utils/cmd" - "github.com/1Panel-dev/1Panel/agent/utils/common" "github.com/1Panel-dev/1Panel/agent/utils/files" "github.com/pkg/errors" ) -func (u *SnapshotService) HandleSnapshotRecover(snap model.Snapshot, isRecover bool, req dto.SnapshotRecover) { +func (u *SnapshotService) HandleSnapshotRecover(snap model.Snapshot, req dto.SnapshotRecover) { _ = global.Cron.Stop() defer func() { global.Cron.Start() }() - snapFileDir := "" - if isRecover { - baseDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("system/%s", snap.Name)) - if _, err := os.Stat(baseDir); err != nil && os.IsNotExist(err) { - _ = os.MkdirAll(baseDir, os.ModePerm) - } - if req.IsNew || snap.InterruptStep == "Download" || req.ReDownload { - if err := handleDownloadSnapshot(snap, baseDir); err != nil { - updateRecoverStatus(snap.ID, isRecover, "Backup", constant.StatusFailed, err.Error()) - return - } - global.LOG.Debugf("download snapshot file to %s successful!", baseDir) - req.IsNew = true - } - if req.IsNew || snap.InterruptStep == "Decompress" { - if err := handleUnTar(fmt.Sprintf("%s/%s.tar.gz", baseDir, snap.Name), baseDir, req.Secret); err != nil { - updateRecoverStatus(snap.ID, isRecover, "Decompress", constant.StatusFailed, fmt.Sprintf("decompress file failed, err: %v", err)) - return - } - global.LOG.Debug("decompress snapshot file successful!", baseDir) - req.IsNew = true - } - if req.IsNew || snap.InterruptStep == "Backup" { - if err := backupBeforeRecover(snap); err != nil { - updateRecoverStatus(snap.ID, isRecover, "Backup", constant.StatusFailed, fmt.Sprintf("handle backup before recover failed, err: %v", err)) - return - } - global.LOG.Debug("handle backup before recover successful!") - req.IsNew = true - } - snapFileDir = fmt.Sprintf("%s/%s", baseDir, snap.Name) - if _, err := os.Stat(snapFileDir); err != nil { - snapFileDir = baseDir - } - } else { - snapFileDir = fmt.Sprintf("%s/1panel_original/original_%s", global.CONF.System.BaseDir, snap.Name) - if _, err := os.Stat(snapFileDir); err != nil { - updateRecoverStatus(snap.ID, isRecover, "", constant.StatusFailed, fmt.Sprintf("cannot find the backup file %s, please try to recover again.", snapFileDir)) + fileOp := files.NewFileOp() + baseDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("system/%s", snap.Name)) + if _, err := os.Stat(baseDir); err != nil && os.IsNotExist(err) { + _ = os.MkdirAll(baseDir, os.ModePerm) + } + if req.IsNew || snap.InterruptStep == "Download" || req.ReDownload { + if err := handleDownloadSnapshot(snap, baseDir); err != nil { + updateRecoverStatus(snap.ID, "Download", constant.StatusFailed, err.Error()) return } + global.LOG.Debugf("download snapshot file to %s successful!", baseDir) + req.IsNew = true } - snapJson, err := u.readFromJson(fmt.Sprintf("%s/snapshot.json", snapFileDir)) + if req.IsNew || snap.InterruptStep == "Decompress" { + if err := fileOp.TarGzExtractPro(fmt.Sprintf("%s/%s.tar.gz", baseDir, snap.Name), baseDir, req.Secret); err != nil { + updateRecoverStatus(snap.ID, "Decompress", constant.StatusFailed, fmt.Sprintf("decompress file failed, err: %v", err)) + return + } + global.LOG.Debug("decompress snapshot file successful!", baseDir) + req.IsNew = true + } + if req.IsNew || snap.InterruptStep == "Backup" { + if err := backupBeforeRecover(snap.Name); err != nil { + updateRecoverStatus(snap.ID, "Backup", constant.StatusFailed, fmt.Sprintf("handle backup before recover failed, err: %v", err)) + return + } + global.LOG.Debug("handle backup before recover successful!") + req.IsNew = true + } + snapFileDir := fmt.Sprintf("%s/%s", baseDir, snap.Name) + if _, err := os.Stat(snapFileDir); err != nil { + snapFileDir = baseDir + } + snapJson, err := u.readFromJson(path.Join(snapFileDir, "base/snapshot.json")) if err != nil { - updateRecoverStatus(snap.ID, isRecover, "Readjson", constant.StatusFailed, fmt.Sprintf("decompress file failed, err: %v", err)) + updateRecoverStatus(snap.ID, "Readjson", constant.StatusFailed, fmt.Sprintf("decompress file failed, err: %v", err)) return } if snap.InterruptStep == "Readjson" { req.IsNew = true } - if isRecover && (req.IsNew || snap.InterruptStep == "AppData") { + if req.IsNew || snap.InterruptStep == "AppImage" { if err := recoverAppData(snapFileDir); err != nil { - updateRecoverStatus(snap.ID, isRecover, "DockerDir", constant.StatusFailed, fmt.Sprintf("handle recover app data failed, err: %v", err)) + updateRecoverStatus(snap.ID, "AppImage", constant.StatusFailed, fmt.Sprintf("handle recover app data failed, err: %v", err)) return } - global.LOG.Debug("recover app data from snapshot file successful!") - req.IsNew = true - } - if req.IsNew || snap.InterruptStep == "DaemonJson" { - fileOp := files.NewFileOp() - if err := recoverDaemonJson(snapFileDir, fileOp); err != nil { - updateRecoverStatus(snap.ID, isRecover, "DaemonJson", constant.StatusFailed, err.Error()) - return - } - global.LOG.Debug("recover daemon.json from snapshot file successful!") + global.LOG.Debug("recover app images from snapshot file successful!") req.IsNew = true } - if req.IsNew || snap.InterruptStep == "1PanelBinary" { - if err := recoverPanel(path.Join(snapFileDir, "1panel/1panel"), "/usr/local/bin"); err != nil { - updateRecoverStatus(snap.ID, isRecover, "1PanelBinary", constant.StatusFailed, err.Error()) + if req.IsNew || snap.InterruptStep == "BaseData" { + if err := recoverBaseData(path.Join(snapFileDir, "base"), fileOp); err != nil { + updateRecoverStatus(snap.ID, "BaseData", constant.StatusFailed, err.Error()) return } - global.LOG.Debug("recover 1panel binary from snapshot file successful!") + global.LOG.Debug("recover base data from snapshot file successful!") req.IsNew = true } - if req.IsNew || snap.InterruptStep == "1PctlBinary" { - if err := recoverPanel(path.Join(snapFileDir, "1panel/1pctl"), "/usr/local/bin"); err != nil { - updateRecoverStatus(snap.ID, isRecover, "1PctlBinary", constant.StatusFailed, err.Error()) + + if req.IsNew || snap.InterruptStep == "DBData" { + if err := recoverDBData(path.Join(snapFileDir, "db"), fileOp); err != nil { + updateRecoverStatus(snap.ID, "DBData", constant.StatusFailed, err.Error()) return } - global.LOG.Debug("recover 1pctl from snapshot file successful!") - req.IsNew = true - } - if req.IsNew || snap.InterruptStep == "1PanelService" { - if err := recoverPanel(path.Join(snapFileDir, "1panel/1panel.service"), "/etc/systemd/system"); err != nil { - updateRecoverStatus(snap.ID, isRecover, "1PanelService", constant.StatusFailed, err.Error()) - return - } - global.LOG.Debug("recover 1panel service from snapshot file successful!") + global.LOG.Debug("recover db data from snapshot file successful!") req.IsNew = true } if req.IsNew || snap.InterruptStep == "1PanelBackups" { - if err := u.handleUnTar(path.Join(snapFileDir, "/1panel/1panel_backup.tar.gz"), snapJson.BackupDataDir, ""); err != nil { - updateRecoverStatus(snap.ID, isRecover, "1PanelBackups", constant.StatusFailed, err.Error()) + if err := fileOp.TarGzExtractPro(path.Join(snapFileDir, "/1panel_backup.tar.gz"), snapJson.BackupDataDir, ""); err != nil { + updateRecoverStatus(snap.ID, "1PanelBackups", constant.StatusFailed, err.Error()) return } global.LOG.Debug("recover 1panel backups from snapshot file successful!") @@ -126,9 +100,8 @@ func (u *SnapshotService) HandleSnapshotRecover(snap model.Snapshot, isRecover b } if req.IsNew || snap.InterruptStep == "1PanelData" { - checkPointOfWal() - if err := u.handleUnTar(path.Join(snapFileDir, "/1panel/1panel_data.tar.gz"), path.Join(snapJson.BaseDir, "1panel"), ""); err != nil { - updateRecoverStatus(snap.ID, isRecover, "1PanelData", constant.StatusFailed, err.Error()) + if err := fileOp.TarGzExtractPro(path.Join(snapFileDir, "/1panel_data.tar.gz"), path.Join(snapJson.BaseDir, "1panel"), ""); err != nil { + updateRecoverStatus(snap.ID, "1PanelData", constant.StatusFailed, err.Error()) return } global.LOG.Debug("recover 1panel data from snapshot file successful!") @@ -138,51 +111,11 @@ func (u *SnapshotService) HandleSnapshotRecover(snap model.Snapshot, isRecover b restartCompose(path.Join(snapJson.BaseDir, "1panel/docker/compose")) global.LOG.Info("recover successful") - if !isRecover { - oriPath := fmt.Sprintf("%s/1panel_original/original_%s", global.CONF.System.BaseDir, snap.Name) - global.LOG.Debugf("remove the file %s after the operation is successful", oriPath) - _ = os.RemoveAll(oriPath) - } else { - global.LOG.Debugf("remove the file %s after the operation is successful", path.Dir(snapFileDir)) - _ = os.RemoveAll(path.Dir(snapFileDir)) - } + global.LOG.Debugf("remove the file %s after the operation is successful", path.Dir(snapFileDir)) + _ = os.RemoveAll(path.Dir(snapFileDir)) _, _ = cmd.Exec("systemctl daemon-reload && systemctl restart 1panel.service") } -func backupBeforeRecover(snap model.Snapshot) error { - baseDir := fmt.Sprintf("%s/1panel_original/original_%s", global.CONF.System.BaseDir, snap.Name) - var wg sync.WaitGroup - var status model.SnapshotStatus - itemHelper := snapHelper{SnapID: 0, Status: &status, Wg: &wg, FileOp: files.NewFileOp(), Ctx: context.Background()} - - jsonItem := SnapshotJson{ - BaseDir: global.CONF.System.BaseDir, - BackupDataDir: global.CONF.System.Backup, - PanelDataDir: path.Join(global.CONF.System.BaseDir, "1panel"), - } - _ = os.MkdirAll(path.Join(baseDir, "1panel"), os.ModePerm) - _ = os.MkdirAll(path.Join(baseDir, "docker"), os.ModePerm) - - wg.Add(4) - itemHelper.Wg = &wg - go snapJson(itemHelper, jsonItem, baseDir) - go snapPanel(itemHelper, path.Join(baseDir, "1panel")) - go snapDaemonJson(itemHelper, path.Join(baseDir, "docker")) - go snapBackup(itemHelper, path.Join(baseDir, "1panel")) - wg.Wait() - itemHelper.Status.AppData = constant.StatusDone - - allDone, msg := checkAllDone(status) - if !allDone { - return errors.New(msg) - } - snapPanelData(itemHelper, path.Join(baseDir, "1panel")) - if status.PanelData != constant.StatusDone { - return errors.New(status.PanelData) - } - return nil -} - func handleDownloadSnapshot(snap model.Snapshot, targetDir string) error { account, client, err := NewBackupClientWithID(snap.DownloadAccountID) if err != nil { @@ -202,18 +135,34 @@ func handleDownloadSnapshot(snap model.Snapshot, targetDir string) error { } func recoverAppData(src string) error { - if _, err := os.Stat(path.Join(src, "docker/docker_image.tar")); err != nil { + if _, err := os.Stat(path.Join(src, "images.tar.gz")); err != nil { global.LOG.Debug("no such docker images in snapshot") return nil } - std, err := cmd.Execf("docker load < %s", path.Join(src, "docker/docker_image.tar")) + std, err := cmd.Execf("docker load < %s", path.Join(src, "images.tar.gz")) if err != nil { return errors.New(std) } return err } -func recoverDaemonJson(src string, fileOp files.FileOp) error { +func recoverBaseData(src string, fileOp files.FileOp) error { + if err := fileOp.CopyFile(path.Join(src, "1pctl"), "/usr/local/bin"); err != nil { + return err + } + if err := fileOp.CopyFile(path.Join(src, "1panel"), "/usr/local/bin"); err != nil { + return err + } + if err := fileOp.CopyFile(path.Join(src, "1panel_agent"), "/usr/local/bin"); err != nil { + return err + } + if err := fileOp.CopyFile(path.Join(src, "1panel.service"), "/etc/systemd/system"); err != nil { + return err + } + if err := fileOp.CopyFile(path.Join(src, "1panel_agent.service"), "/etc/systemd/system"); err != nil { + return err + } + daemonJsonPath := "/etc/docker/daemon.json" _, errSrc := os.Stat(path.Join(src, "docker/daemon.json")) _, errPath := os.Stat(daemonJsonPath) @@ -231,14 +180,8 @@ func recoverDaemonJson(src string, fileOp files.FileOp) error { return nil } -func recoverPanel(src string, dst string) error { - if _, err := os.Stat(src); err != nil { - return fmt.Errorf("file is not found in %s, err: %v", src, err) - } - if err := common.CopyFile(src, dst); err != nil { - return fmt.Errorf("cp file failed, err: %v", err) - } - return nil +func recoverDBData(src string, fileOp files.FileOp) error { + return fileOp.CopyDir(src, path.Join(global.CONF.System.BaseDir, "1panel", "db")) } func restartCompose(composePath string) { @@ -259,3 +202,86 @@ func restartCompose(composePath string) { } global.LOG.Debug("restart all compose successful!") } + +func backupBeforeRecover(name string) error { + rootDir := fmt.Sprintf("%s/1panel_original/original_%s", global.CONF.System.BaseDir, name) + baseDir := path.Join(rootDir, "base") + if _, err := os.Stat(baseDir); err != nil { + _ = os.MkdirAll(baseDir, os.ModePerm) + } + + FileOp := files.NewFileOp() + if err := FileOp.CopyDirWithExclude(path.Join(global.CONF.System.BaseDir, "1panel"), rootDir, []string{"cache", "tmp"}); err != nil { + return err + } + if err := FileOp.CopyDir(global.CONF.System.Backup, rootDir); err != nil { + return err + } + if err := FileOp.CopyFile("/usr/local/bin/1pctl", baseDir); err != nil { + return err + } + if err := FileOp.CopyFile("/usr/local/bin/1panel", baseDir); err != nil { + return err + } + if err := FileOp.CopyFile("/usr/local/bin/1panel_agent", baseDir); err != nil { + return err + } + if err := FileOp.CopyFile("/etc/systemd/system/1panel.service", baseDir); err != nil { + return err + } + if err := FileOp.CopyFile("/etc/systemd/system/1panel_agent.service", baseDir); err != nil { + return err + } + if err := FileOp.CopyFile("/etc/docker/daemon.json", baseDir); err != nil { + return err + } + return nil +} + +func handleRollback(name string) error { + rootDir := fmt.Sprintf("%s/1panel_original/original_%s", global.CONF.System.BaseDir, name) + baseDir := path.Join(rootDir, "base") + + FileOp := files.NewFileOp() + if err := FileOp.CopyDir(path.Join(rootDir, "1panel"), global.CONF.System.BaseDir); err != nil { + return err + } + if err := FileOp.CopyDir(path.Join(rootDir, "backup"), path.Dir(global.CONF.System.Backup)); err != nil { + return err + } + if err := FileOp.CopyFile(path.Join(baseDir, "1pctl"), "/usr/local/bin/1pctl"); err != nil { + return err + } + if err := FileOp.CopyFile(path.Join(baseDir, "1panel"), "/usr/local/bin/1panel"); err != nil { + return err + } + if err := FileOp.CopyFile(path.Join(baseDir, "1panel_agent"), "/usr/local/bin/1panel_agent"); err != nil { + return err + } + if err := FileOp.CopyFile(path.Join(baseDir, "1panel.service"), "/etc/systemd/system/1panel.service"); err != nil { + return err + } + if err := FileOp.CopyFile(path.Join(baseDir, "1panel_agent.service"), "/etc/systemd/system/1panel_agent.service"); err != nil { + return err + } + if err := FileOp.CopyFile(path.Join(baseDir, "daemon.json"), "/etc/docker/daemon.json"); err != nil { + return err + } + _ = os.RemoveAll(rootDir) + return nil +} + +func updateRecoverStatus(id uint, interruptStep, status, message string) { + if status != constant.StatusSuccess { + global.LOG.Errorf("recover failed, err: %s", message) + } + if err := snapshotRepo.Update(id, map[string]interface{}{ + "interrupt_step": interruptStep, + "recover_status": status, + "recover_message": message, + "last_recovered_at": time.Now().Format(constant.DateTimeLayout), + }); err != nil { + global.LOG.Errorf("update snap recover status failed, err: %v", err) + } + _ = settingRepo.Update("SystemStatus", "Free") +} diff --git a/agent/init/hook/hook.go b/agent/init/hook/hook.go index f2d7cc45b..7cc33863a 100644 --- a/agent/init/hook/hook.go +++ b/agent/init/hook/hook.go @@ -72,18 +72,18 @@ func handleSnapStatus() { status, _ := snapRepo.GetStatusList() for _, item := range status { updates := make(map[string]interface{}) - if item.Panel == constant.StatusRunning { - updates["panel"] = constant.StatusFailed - } - if item.PanelInfo == constant.StatusRunning { - updates["panel_info"] = constant.StatusFailed - } - if item.DaemonJson == constant.StatusRunning { - updates["daemon_json"] = constant.StatusFailed - } - if item.AppData == constant.StatusRunning { - updates["app_data"] = constant.StatusFailed - } + // if item.Panel == constant.StatusRunning { + // updates["panel"] = constant.StatusFailed + // } + // if item.PanelInfo == constant.StatusRunning { + // updates["panel_info"] = constant.StatusFailed + // } + // if item.DaemonJson == constant.StatusRunning { + // updates["daemon_json"] = constant.StatusFailed + // } + // if item.AppData == constant.StatusRunning { + // updates["app_data"] = constant.StatusFailed + // } if item.PanelData == constant.StatusRunning { updates["panel_data"] = constant.StatusFailed } diff --git a/agent/init/migration/migrate.go b/agent/init/migration/migrate.go index 54f693f19..cf5b14619 100644 --- a/agent/init/migration/migrate.go +++ b/agent/init/migration/migrate.go @@ -21,6 +21,7 @@ func Init() { migrations.UpdateApp, migrations.AddTaskDB, migrations.UpdateAppInstall, + migrations.UpdateSnapshot, }) if err := m.Migrate(); err != nil { global.LOG.Error(err) diff --git a/agent/init/migration/migrations/init.go b/agent/init/migration/migrations/init.go index 8e09ccc82..8ec8c8a83 100644 --- a/agent/init/migration/migrations/init.go +++ b/agent/init/migration/migrations/init.go @@ -259,3 +259,10 @@ var UpdateAppInstall = &gormigrate.Migration{ &model.AppInstall{}) }, } + +var UpdateSnapshot = &gormigrate.Migration{ + ID: "20240913-update-snapshot", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&model.Snapshot{}, &model.SnapshotStatus{}) + }, +} diff --git a/agent/router/ro_setting.go b/agent/router/ro_setting.go index 67ecd12ea..ec956ccac 100644 --- a/agent/router/ro_setting.go +++ b/agent/router/ro_setting.go @@ -15,6 +15,7 @@ func (s *SettingRouter) InitRouter(Router *gin.RouterGroup) { settingRouter.GET("/search/available", baseApi.GetSystemAvailable) settingRouter.POST("/update", baseApi.UpdateSetting) + settingRouter.GET("/snapshot/load", baseApi.LoadSnapshotData) settingRouter.POST("/snapshot", baseApi.CreateSnapshot) settingRouter.POST("/snapshot/status", baseApi.LoadSnapShotStatus) settingRouter.POST("/snapshot/search", baseApi.SearchSnapshot) diff --git a/agent/utils/files/file_op.go b/agent/utils/files/file_op.go index 8fe90bc6e..415c6c34d 100644 --- a/agent/utils/files/file_op.go +++ b/agent/utils/files/file_op.go @@ -450,6 +450,48 @@ func (f FileOp) CopyDir(src, dst string) error { return cmd.ExecCmd(fmt.Sprintf(`cp -rf '%s' '%s'`, src, dst+"/")) } +func (f FileOp) CopyDirWithExclude(src, dst string, excludeNames []string) error { + srcInfo, err := f.Fs.Stat(src) + if err != nil { + return err + } + dstDir := filepath.Join(dst, srcInfo.Name()) + if err = f.Fs.MkdirAll(dstDir, srcInfo.Mode()); err != nil { + return err + } + if len(excludeNames) == 0 { + return cmd.ExecCmd(fmt.Sprintf(`cp -rf '%s' '%s'`, src, dst+"/")) + } + tmpFiles, err := os.ReadDir(src) + if err != nil { + return err + } + for _, item := range tmpFiles { + isExclude := false + for _, name := range excludeNames { + if item.Name() == name { + isExclude = true + break + } + } + if isExclude { + continue + } + if item.IsDir() { + fmt.Println(path.Join(src, item.Name()), dstDir) + if err := f.CopyDir(path.Join(src, item.Name()), dstDir); err != nil { + return err + } + continue + } + if err := f.CopyFile(path.Join(src, item.Name()), dstDir); err != nil { + return err + } + } + + return nil +} + func (f FileOp) CopyFile(src, dst string) error { dst = filepath.Clean(dst) + string(filepath.Separator) return cmd.ExecCmd(fmt.Sprintf(`cp -f '%s' '%s'`, src, dst+"/")) @@ -664,3 +706,56 @@ func ZipFile(files []archiver.File, dst afero.File) error { } return nil } + +func (f FileOp) TarGzCompressPro(withDir bool, src, dst, secret, exclusionRules string) error { + workdir := src + srcItem := "." + if withDir { + workdir = path.Dir(src) + srcItem = path.Base(src) + } + commands := "" + + exMap := make(map[string]struct{}) + exStr := "" + excludes := strings.Split(exclusionRules, ";") + excludes = append(excludes, "*.sock") + for _, exclude := range excludes { + if len(exclude) == 0 { + continue + } + if _, ok := exMap[exclude]; ok { + continue + } + exStr += " --exclude " + exStr += exclude + exMap[exclude] = struct{}{} + } + + if len(secret) != 0 { + commands = fmt.Sprintf("tar -zcf - %s | openssl enc -aes-256-cbc -salt -pbkdf2 -k '%s' -out %s", srcItem, secret, dst) + global.LOG.Debug(strings.ReplaceAll(commands, fmt.Sprintf(" %s ", secret), "******")) + } else { + commands = fmt.Sprintf("tar zcf %s %s %s", dst, exStr, srcItem) + global.LOG.Debug(commands) + } + return cmd.ExecCmdWithDir(commands, workdir) +} + +func (f FileOp) TarGzExtractPro(src, dst string, secret string) error { + if _, err := os.Stat(path.Dir(dst)); err != nil && os.IsNotExist(err) { + if err = os.MkdirAll(path.Dir(dst), os.ModePerm); err != nil { + return err + } + } + + commands := "" + if len(secret) != 0 { + commands = fmt.Sprintf("openssl enc -d -aes-256-cbc -salt -pbkdf2 -k '%s' -in %s | tar -zxf - > /root/log", secret, src) + global.LOG.Debug(strings.ReplaceAll(commands, fmt.Sprintf(" %s ", secret), "******")) + } else { + commands = fmt.Sprintf("tar zxvf %s", src) + global.LOG.Debug(commands) + } + return cmd.ExecCmdWithDir(commands, dst) +} diff --git a/agent/utils/files/tar_gz.go b/agent/utils/files/tar_gz.go index 93a82c54b..f59a113fb 100644 --- a/agent/utils/files/tar_gz.go +++ b/agent/utils/files/tar_gz.go @@ -2,6 +2,8 @@ package files import ( "fmt" + "os" + "path" "path/filepath" "strings" @@ -59,3 +61,56 @@ func (t TarGzArchiver) Compress(sourcePaths []string, dstFile string, secret str } return nil } + +func (t TarGzArchiver) CompressPro(withDir bool, src, dst, secret, exclusionRules string) error { + workdir := src + srcItem := "." + if withDir { + workdir = path.Dir(src) + srcItem = path.Base(src) + } + commands := "" + + exMap := make(map[string]struct{}) + exStr := "" + excludes := strings.Split(exclusionRules, ";") + excludes = append(excludes, "*.sock") + for _, exclude := range excludes { + if len(exclude) == 0 { + continue + } + if _, ok := exMap[exclude]; ok { + continue + } + exStr += " --exclude " + exStr += exclude + exMap[exclude] = struct{}{} + } + + if len(secret) != 0 { + commands = fmt.Sprintf("tar -zcf - %s | openssl enc -aes-256-cbc -salt -pbkdf2 -k '%s' -out %s", srcItem, secret, dst) + global.LOG.Debug(strings.ReplaceAll(commands, fmt.Sprintf(" %s ", secret), "******")) + } else { + commands = fmt.Sprintf("tar zcf %s %s %s", dst, exStr, srcItem) + global.LOG.Debug(commands) + } + return cmd.ExecCmdWithDir(commands, workdir) +} + +func (t TarGzArchiver) ExtractPro(src, dst string, secret string) error { + if _, err := os.Stat(path.Dir(dst)); err != nil && os.IsNotExist(err) { + if err = os.MkdirAll(path.Dir(dst), os.ModePerm); err != nil { + return err + } + } + + commands := "" + if len(secret) != 0 { + commands = fmt.Sprintf("openssl enc -d -aes-256-cbc -salt -pbkdf2 -k '%s' -in %s | tar -zxf - > /root/log", secret, src) + global.LOG.Debug(strings.ReplaceAll(commands, fmt.Sprintf(" %s ", secret), "******")) + } else { + commands = fmt.Sprintf("tar zxvf %s", src) + global.LOG.Debug(commands) + } + return cmd.ExecCmdWithDir(commands, dst) +} diff --git a/frontend/src/api/interface/setting.ts b/frontend/src/api/interface/setting.ts index e358879a7..6c3ceb578 100644 --- a/frontend/src/api/interface/setting.ts +++ b/frontend/src/api/interface/setting.ts @@ -119,11 +119,18 @@ export namespace Setting { export interface SnapshotCreate { id: number; - from: string; - fromAccounts: Array; - defaultDownload: string; + sourceAccountIDs: string; + downloadAccountID: string; description: string; secret: string; + + appData: Array; + panelData: Array; + backupData: Array; + + withMonitorData: boolean; + withLoginLog: boolean; + withOperationLog: boolean; } export interface SnapshotImport { from: string; @@ -155,11 +162,31 @@ export namespace Setting { lastRollbackedAt: string; secret: string; } + export interface SnapshotData { + appData: Array; + panelData: Array; + backupData: Array; + + withMonitorData: boolean; + withLoginLog: boolean; + withOperationLog: boolean; + } + export interface DataTree { + id: string; + label: string; + key: string; + name: string; + size: number; + isCheck: boolean; + isDisable: boolean; + + path: string; + + Children: Array; + } export interface SnapshotStatus { - panel: string; - panelInfo: string; - daemonJson: string; - appData: string; + baseData: string; + appImage: string; panelData: string; backupData: string; diff --git a/frontend/src/api/modules/setting.ts b/frontend/src/api/modules/setting.ts index abd3b0e7e..922b1ecbe 100644 --- a/frontend/src/api/modules/setting.ts +++ b/frontend/src/api/modules/setting.ts @@ -88,6 +88,9 @@ export const bindMFA = (param: Setting.MFABind) => { }; // snapshot +export const loadSnapshotSetting = () => { + return http.get(`/settings/snapshot/load`); +}; export const snapshotCreate = (param: Setting.SnapshotCreate) => { return http.post(`/settings/snapshot`, param); }; diff --git a/frontend/src/lang/modules/en.ts b/frontend/src/lang/modules/en.ts index 4d64924c0..d34f3d1ea 100644 --- a/frontend/src/lang/modules/en.ts +++ b/frontend/src/lang/modules/en.ts @@ -1585,6 +1585,36 @@ const message = { 'Backup files not in the current backup list, please try downloading from the file directory and importing for backup.', snapshot: 'Snapshot', + stepBaseData: 'Base Data', + stepAppData: 'System Application', + stepPanelData: 'System Data', + stepBackupData: 'Backup Data', + stepOtherData: 'Other Data', + loginLog: 'Backup System Login Records', + OperationLog: 'Backup System Operation Log Login Records', + monitorData: 'Backup System Monitoring Data', + selectAllImage: 'Backup All Application Images', + agentLabel: 'Node Configuration', + appDataLabel: 'Application Data', + appImage: 'Application Image', + appBackup: 'Application Backup', + backupLabel: 'Backup Directory', + cacheLabel: 'Cache Directory', + confLabel: 'Configuration File', + dbLabel: 'Database Directory', + dockerLabel: 'Container Related Logs', + logLabel: 'Log Directory', + taskLabel: 'Scheduled Task Report', + resourceLabel: 'Application Resource Directory', + runtimeLabel: 'Runtime Directory', + appLabel: 'Application', + databaseLabel: 'Database', + websiteLabel: 'Website', + directoryLabel: 'Directory', + appStoreLabel: 'Application Store', + shellLabel: 'Script', + tmpLabel: 'Temporary Directory', + sslLabel: 'Certificate Directory', deleteHelper: 'All backup files for the snapshot, including those in the third-party backup account, will be deleted.', status: 'Snapshot status', diff --git a/frontend/src/lang/modules/tw.ts b/frontend/src/lang/modules/tw.ts index 93718d475..723bf87d7 100644 --- a/frontend/src/lang/modules/tw.ts +++ b/frontend/src/lang/modules/tw.ts @@ -1403,6 +1403,36 @@ const message = { backupJump: '未在當前備份列表中的備份檔案，請嘗試從檔案目錄中下載後導入備份。', snapshot: '快照', + stepBaseData: '基礎數據', + stepAppData: '系統應用', + stepPanelData: '系統數據', + stepBackupData: '備份數據', + stepOtherData: '其他數據', + loginLog: '備份系統登錄記錄', + OperationLog: '備份系統操作日誌登錄記錄', + monitorData: '備份系統監控數據', + selectAllImage: '備份所有應用鏡像', + agentLabel: '節點配置', + appDataLabel: '應用數據', + appImage: '應用鏡像', + appBackup: '應用備份', + backupLabel: '備份目錄', + cacheLabel: '緩存目錄', + confLabel: '配置文件', + dbLabel: '數據庫目錄', + dockerLabel: '容器相關日誌', + logLabel: '日誌目錄', + taskLabel: '計劃任務報告', + resourceLabel: '應用資源目錄', + runtimeLabel: '運行時目錄', + appLabel: '應用', + databaseLabel: '數據庫', + websiteLabel: '網站', + directoryLabel: '目錄', + appStoreLabel: '應用商店', + shellLabel: '腳本', + tmpLabel: '臨時目錄', + sslLabel: '證書目錄', deleteHelper: '將刪除該快照的所有備份文件，包括第三方備份賬號中的文件。', status: '快照狀態', ignoreRule: '排除規則', diff --git a/frontend/src/lang/modules/zh.ts b/frontend/src/lang/modules/zh.ts index b1a79ebc8..8c61d2719 100644 --- a/frontend/src/lang/modules/zh.ts +++ b/frontend/src/lang/modules/zh.ts @@ -1405,6 +1405,36 @@ const message = { backupJump: '未在当前备份列表中的备份文件，请尝试从文件目录中下载后导入备份。', snapshot: '快照', + stepBaseData: '基础数据', + stepAppData: '系统应用', + stepPanelData: '系统数据', + stepBackupData: '备份数据', + stepOtherData: '其他数据', + loginLog: '备份系统登陆记录', + OperationLog: '备份系统备份系统操作日志登陆记录', + monitorData: '备份系统监控数据', + selectAllImage: '备份所有应用镜像', + agentLabel: '节点配置', + appDataLabel: '应用数据', + appImage: '应用镜像', + appBackup: '应用备份', + backupLabel: '备份目录', + cacheLabel: '缓存目录', + confLabel: '配置文件', + dbLabel: '数据库目录', + dockerLabel: '容器相关日志', + logLabel: '日志目录', + taskLabel: '计划任务报告', + resourceLabel: '应用资源目录', + runtimeLabel: '运行时目录', + appLabel: '应用', + databaseLabel: '数据库', + websiteLabel: '网站', + directoryLabel: '目录', + appStoreLabel: '应用商店', + shellLabel: '脚本', + tmpLabel: '临时目录', + sslLabel: '证书目录', deleteHelper: '将删除该快照的所有备份文件，包括第三方备份账号中的文件。', ignoreRule: '排除规则', ignoreHelper: '快照时将使用该规则对 1Panel 数据目录进行压缩备份，请谨慎修改。', diff --git a/frontend/src/views/setting/snapshot/create/index.vue b/frontend/src/views/setting/snapshot/create/index.vue new file mode 100644 index 000000000..001ffc4de --- /dev/null +++ b/frontend/src/views/setting/snapshot/create/index.vue @@ -0,0 +1,485 @@ + + + + diff --git a/frontend/src/views/setting/snapshot/index.vue b/frontend/src/views/setting/snapshot/index.vue index 42799fd97..1f5458d3c 100644 --- a/frontend/src/views/setting/snapshot/index.vue +++ b/frontend/src/views/setting/snapshot/index.vue @@ -88,7 +88,7 @@ {{ $t('commons.status.error') }} - + {{ $t('commons.status.success') }} @@ -115,52 +115,8 @@ + - - - - - - - - - - - - - - - - - - - - - diff --git a/frontend/src/views/setting/snapshot/status/index.vue b/frontend/src/views/setting/snapshot/status/index.vue index 4816fe4aa..b49ba8f57 100644 --- a/frontend/src/views/setting/snapshot/status/index.vue +++ b/frontend/src/views/setting/snapshot/status/index.vue @@ -1,220 +1,295 @@ - - -