fix: Improve the node synchronization logic (#8467)

agent/app/api/v2/backup.go

@@ -25,20 +25,6 @@ func (b *BaseApi) CheckBackupUsed(c *gin.Context) {
 	helper.Success(c)
 }
 
-func (b *BaseApi) SyncBackupAccount(c *gin.Context) {
-	var req dto.SyncFromMaster
-	if err := helper.CheckBindAndValidate(&req, c); err != nil {
-		return
-	}
-
-	if err := backupService.Sync(req); err != nil {
-		helper.BadRequest(c, err)
-		return
-	}
-
-	helper.Success(c)
-}
-
 // @Tags Backup Account
 // @Summary Create backup account
 // @Accept json

agent/app/service/backup.go

@@ -11,10 +11,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/1Panel-dev/1Panel/agent/app/repo"
-
 	"github.com/1Panel-dev/1Panel/agent/app/dto"
 	"github.com/1Panel-dev/1Panel/agent/app/model"
+	"github.com/1Panel-dev/1Panel/agent/app/repo"
 	"github.com/1Panel-dev/1Panel/agent/buserr"
 	"github.com/1Panel-dev/1Panel/agent/constant"
 	"github.com/1Panel-dev/1Panel/agent/global"
@@ -29,7 +28,6 @@ type BackupService struct{}
 
 type IBackupService interface {
 	CheckUsed(name string, isPublic bool) error
-	Sync(req dto.SyncFromMaster) error
 
 	LoadBackupOptions() ([]dto.BackupOption, error)
 	SearchWithPage(search dto.SearchPageWithType) (int64, interface{}, error)
@@ -346,39 +344,6 @@ func (u *BackupService) checkBackupConn(backup *model.BackupAccount) (bool, erro
 	return true, nil
 }
 
-func (u *BackupService) Sync(req dto.SyncFromMaster) error {
-	var accountItem model.BackupAccount
-	if err := json.Unmarshal([]byte(req.Data), &accountItem); err != nil {
-		return err
-	}
-	accountItem.AccessKey, _ = encrypt.StringEncryptWithBase64(accountItem.AccessKey)
-	accountItem.Credential, _ = encrypt.StringEncryptWithBase64(accountItem.Credential)
-	account, _ := backupRepo.Get(repo.WithByName(req.Name))
-	switch req.Operation {
-	case "create":
-		if account.ID != 0 {
-			accountItem.ID = account.ID
-			return backupRepo.Save(&accountItem)
-		}
-		return backupRepo.Create(&accountItem)
-	case "delete":
-		if account.ID == 0 {
-			return buserr.New("ErrRecordNotFound")
-		}
-		return backupRepo.Delete(repo.WithByID(account.ID))
-	case "update":
-		if account.ID == 0 {
-			return buserr.New("ErrRecordNotFound")
-		}
-		accountItem.ID = account.ID
-		accountItem.CreatedAt = account.CreatedAt
-		accountItem.UpdatedAt = account.UpdatedAt
-		return backupRepo.Save(&accountItem)
-	default:
-		return fmt.Errorf("not support such operation %s", req.Operation)
-	}
-}
-
 func (u *BackupService) LoadBackupOptions() ([]dto.BackupOption, error) {
 	accounts, err := backupRepo.List(repo.WithOrderBy("created_at desc"))
 	if err != nil {

agent/router/backup.go

@@ -12,7 +12,6 @@ func (s *BackupRouter) InitRouter(Router *gin.RouterGroup) {
 	baseApi := v2.ApiGroupApp.BaseApi
 	{
 		backupRouter.GET("/check/:name", baseApi.CheckBackupUsed)
-		backupRouter.POST("/sync", baseApi.SyncBackupAccount)
 		backupRouter.GET("/options", baseApi.LoadBackupOptions)
 		backupRouter.POST("/search", baseApi.SearchBackup)
 

core/app/service/backup.go

@@ -2,7 +2,6 @@ package service
 
 import (
 	"bufio"
-	"bytes"
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
@@ -121,7 +120,11 @@ func (u *BackupService) Create(req dto.BackupOperate) error {
 	if err := backupRepo.Create(&backup); err != nil {
 		return err
 	}
-	go syncAccountToAgent(backup, "create")
+	go func() {
+		if err := xpack.Sync(constant.SyncBackupAccounts); err != nil {
+			global.LOG.Errorf("sync backup account to node failed, err: %v", err)
+		}
+	}()
 	return nil
 }
 
@@ -176,7 +179,11 @@ func (u *BackupService) Delete(name string) error {
 		return buserr.New("ErrBackupInUsed")
 	}
 
-	go syncAccountToAgent(backup, "delete")
+	go func() {
+		if err := xpack.Sync(constant.SyncBackupAccounts); err != nil {
+			global.LOG.Errorf("sync backup account to node failed, err: %v", err)
+		}
+	}()
 	return backupRepo.Delete(repo.WithByName(name))
 }
 
@@ -234,7 +241,11 @@ func (u *BackupService) Update(req dto.BackupOperate) error {
 	if err := backupRepo.Save(&newBackup); err != nil {
 		return err
 	}
-	go syncAccountToAgent(newBackup, "update")
+	go func() {
+		if err := xpack.Sync(constant.SyncBackupAccounts); err != nil {
+			global.LOG.Errorf("sync backup account to node failed, err: %v", err)
+		}
+	}()
 	return nil
 }
 
@@ -365,16 +376,3 @@ func (u *BackupService) checkBackupConn(backup *model.BackupAccount) (bool, erro
 	_, _ = client.Delete(path.Join(backup.BackupPath, "test/1panel"))
 	return true, nil
 }
-
-func syncAccountToAgent(backup model.BackupAccount, operation string) {
-	if !backup.IsPublic {
-		return
-	}
-	backup.AccessKey, _ = encrypt.StringDecryptWithBase64(backup.AccessKey)
-	backup.Credential, _ = encrypt.StringDecryptWithBase64(backup.Credential)
-	itemData, _ := json.Marshal(backup)
-	itemJson := dto.SyncToAgent{Name: backup.Name, Operation: operation, Data: string(itemData)}
-	bodyItem, _ := json.Marshal(itemJson)
-	_ = xpack.RequestToAllAgent("/api/v2/backups/sync", http.MethodPost, bytes.NewReader((bodyItem)))
-	_, _ = proxy_local.NewLocalClient("/api/v2/backups/sync", http.MethodPost, bytes.NewReader((bodyItem)))
-}

core/constant/common.go

@@ -42,6 +42,11 @@ const (
 	FilePerm = 0644
 )
 
+const (
+	SyncSystemProxy    = "SyncSystemProxy"
+	SyncBackupAccounts = "SyncBackupAccounts"
+)
+
 var WebUrlMap = map[string]struct{}{
 	"/apps":           {},
 	"/apps/all":       {},

core/i18n/lang/en.yaml

@@ -93,6 +93,13 @@ SubTask: "Subtask"
 #script 
 ScriptLibrary: "Script Library"
 
+Node: "Node"
+SyncNode: "Sync node data {{ .name }}"
+SyncPackageBackups: "Package backup account data"
+SyncPackageProxy: "Package system proxy data"
+SyncPackagePanelJson: "Package node basic data"
+SyncPackageEncrypt: "Data package encryption"
+
 #upgrade node
 NodeUpgrade: "Upgrade node {{ .name }}"
 UpgradeCheck: "Check for node updates"

core/i18n/lang/ja.yaml

@@ -94,6 +94,13 @@ SubTask: "サブタスク"
 #script 
 ScriptLibrary: "スクリプトライブラリ"
 
+Node: "ノード"
+SyncNode: "ノードデータ同期 {{ .name }}"
+SyncPackageBackups: "バックアップアカウントデータのパッケージ化"
+SyncPackageProxy: "システムプロキシデータのパッケージ化"
+SyncPackagePanelJson: "ノード基本データのパッケージ化"
+SyncPackageEncrypt: "データパッケージの暗号化"
+
 #upgrade node
 NodeUpgrade: "{{ .name }} ノードのアップグレード"
 UpgradeCheck: "ノードの更新を確認"

core/i18n/lang/ko.yaml

@@ -93,6 +93,13 @@ SubTask: "서브 작업"
 #script 
 ScriptLibrary: "스크립트 라이브러리"
 
+Node: "노드"
+SyncNode: "노드 데이터 동기화 {{ .name }}"
+SyncPackageBackups: "백업 계정 데이터 패키징"
+SyncPackageProxy: "시스템 프록시 데이터 패키징"
+SyncPackagePanelJson: "노드 기본 데이터 패키징"
+SyncPackageEncrypt: "데이터 패키지 암호화"
+
 #upgrade node
 NodeUpgrade: "{{ .name }} 노드 업그레이드"
 UpgradeCheck: "노드 업데이트 확인"

core/i18n/lang/ms.yml

@@ -93,6 +93,13 @@ SubTask: "Tugas Sub"
 #script 
 ScriptLibrary: "Pustaka Skrip"
 
+Node: "Nod"
+SyncNode: "Segerakkan data nod {{ .name }}"
+SyncPackageBackups: "Pakej data akaun sandaran"
+SyncPackageProxy: "Pakej data proksi sistem"
+SyncPackagePanelJson: "Pakej data asas nod"
+SyncPackageEncrypt: "Enkripsi pakej data"
+
 #upgrade node
 NodeUpgrade: "Naik taraf node {{ .name }}"
 UpgradeCheck: "Periksa kemas kini nod"

core/i18n/lang/pt-BR.yaml

@@ -93,6 +93,13 @@ SubTask: "Subtarefa"
 #script 
 ScriptLibrary: "Biblioteca de Scripts"
 
+Node: "Nó"
+SyncNode: "Sincronizar dados do nó {{ .name }}"
+SyncPackageBackups: "Empacotar dados da conta de backup"
+SyncPackageProxy: "Empacotar dados de proxy do sistema"
+SyncPackagePanelJson: "Empacotar dados básicos do nó"
+SyncPackageEncrypt: "Criptografia de pacote de dados"
+
 #upgrade node
 NodeUpgrade: "Atualizar nó {{ .name }}"
 UpgradeCheck: "Verificar atualizações do nó"

core/i18n/lang/ru.yaml

@@ -93,6 +93,13 @@ SubTask: "Подзадача"
 #script 
 ScriptLibrary: "Библиотека скриптов"
 
+Node: "Узел"
+SyncNode: "Синхронизация данных узла {{ .name }}"
+SyncPackageBackups: "Упаковка данных резервных копий аккаунтов"
+SyncPackageProxy: "Упаковка данных системного прокси"
+SyncPackagePanelJson: "Упаковка базовых данных узла"
+SyncPackageEncrypt: "Шифрование пакета данных"
+
 #upgrade node
 NodeUpgrade: "Обновление узла {{ .name }}"
 UpgradeCheck: "Проверить обновления узла"

core/i18n/lang/zh-Hant.yaml

@@ -93,6 +93,13 @@ SubTask: "子任務"
 #script 
 ScriptLibrary: "腳本庫"
 
+Node: "節點"
+SyncNode: "同步節點數據 {{ .name }}"
+SyncPackageBackups: "打包備份帳號數據"
+SyncPackageProxy: "打包系統代理數據"
+SyncPackagePanelJson: "打包節點基礎數據"
+SyncPackageEncrypt: "數據包加密"
+
 #upgrade node
 NodeUpgrade: "升級節點 {{ .name }}"
 UpgradeCheck: "檢查節點更新"

core/i18n/lang/zh.yaml

@@ -93,6 +93,14 @@ SubTask: "子任务"
 #script 
 ScriptLibrary: "脚本库"
 
+#sync node
+Node: "节点"
+SyncNode: "同步节点数据 {{ .name }}"
+SyncPackageBackups: "打包备份账号数据"
+SyncPackageProxy: "打包系统代理数据"
+SyncPackagePanelJson: "打包节点基础数据"
+SyncPackageEncrypt: "数据包加密"
+
 #upgrade node
 NodeUpgrade: "升级节点 {{ .name }}"
 UpgradeCheck: "检查节点更新"

core/init/cron/job/backup.go

@@ -8,6 +8,7 @@ import (
 	"github.com/1Panel-dev/1Panel/core/constant"
 	"github.com/1Panel-dev/1Panel/core/global"
 	"github.com/1Panel-dev/1Panel/core/utils/cloud_storage/client"
+	"github.com/1Panel-dev/1Panel/core/utils/xpack"
 )
 
 type backup struct{}
@@ -57,5 +58,10 @@ func (b *backup) Run() {
 		varsItem, _ := json.Marshal(varMap)
 		_ = global.DB.Model(&model.BackupAccount{}).Where("id = ?", backupItem.ID).Updates(map[string]interface{}{"vars": string(varsItem)}).Error
 		global.LOG.Infof("Refresh %s-%s access_token successful!", backupItem.Type, backupItem.Name)
+		go func() {
+			if err := xpack.Sync(constant.SyncBackupAccounts); err != nil {
+				global.LOG.Errorf("sync backup account to node failed, err: %v", err)
+			}
+		}()
 	}
 }

core/utils/xpack/xpack.go

@@ -4,7 +4,6 @@ package xpack
 
 import (
 	"crypto/tls"
-	"io"
 	"net"
 	"net/http"
 	"time"
@@ -19,8 +18,6 @@ func UpdateGroup(name string, group, newGroup uint) error { return nil }
 
 func CheckBackupUsed(name string) error { return nil }
 
-func RequestToAllAgent(reqUrl, reqMethod string, reqBody io.Reader) error { return nil }
-
 func LoadRequestTransport() *http.Transport {
 	return &http.Transport{
 		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
@@ -37,3 +34,5 @@ func LoadRequestTransport() *http.Transport {
 func LoadNodeInfo(currentNode string) (*ssh.ConnInfo, string, error) {
 	return nil, "", nil
 }
+
+func Sync(dataType string) error { return nil }

frontend/src/lang/modules/en.ts

@@ -3259,6 +3259,10 @@ const message = {
             addNode: 'Add Node',
             connInfo: 'Connection Information',
             nodeInfo: 'Node Information',
+            syncInfo: 'Data synchronization',
+            syncHelper: 'When master node data changes, it synchronizes to this child node in real-time',
+            syncBackupAccount: 'Backup account data',
+            syncProxy: 'System proxy data',
             nodeSyncHelper: 'Node information synchronization will sync the following information:',
             nodeSyncHelper1: '1. Public backup account information',
             nodeSyncHelper2: '2. Connection information between the main node and sub-nodes',

frontend/src/lang/modules/ja.ts

@@ -3118,6 +3118,10 @@ const message = {
             addNode: 'ノードを追加',
             connInfo: '接続情報',
             nodeInfo: 'ノード情報',
+            syncInfo: 'データ同期,',
+            syncHelper: 'マスターノードのデータが変更されると、この子ノードにリアルタイムで同期されます,',
+            syncBackupAccount: 'アカウントデータのバックアップ,',
+            syncProxy: 'システムプロキシデータ,',
             nodeSyncHelper: 'ノード情報の同期は、以下の情報を同期します：',
             nodeSyncHelper1: '1. 公共のバックアップアカウント情報',
             nodeSyncHelper2: '2. 主ノードとサブノードの接続情報',

frontend/src/lang/modules/ko.ts

@@ -3066,6 +3066,10 @@ const message = {
             addNode: '노드 추가',
             connInfo: '연결 정보',
             nodeInfo: '노드 정보',
+            syncInfo: '데이터 동기화,',
+            syncHelper: '마스터 노드 데이터가 변경되면, 이 자식 노드에 실시간으로 동기화됩니다,',
+            syncBackupAccount: '계정 데이터 백업,',
+            syncProxy: '시스템 프록시 데이터,',
             nodeSyncHelper: '노드 정보 동기화는 다음 정보를 동기화합니다:',
             nodeSyncHelper1: '1. 공용 백업 계정 정보',
             nodeSyncHelper2: '2. 주 노드와 하위 노드 간의 연결 정보',

frontend/src/lang/modules/ms.ts

@@ -3187,6 +3187,10 @@ const message = {
             addNode: 'Tambah Nod',
             connInfo: 'Maklumat Sambungan',
             nodeInfo: 'Maklumat Nod',
+            syncInfo: 'Penyegerakan data,',
+            syncHelper: 'Apabila data nod induk berubah, ia akan disegerakkan ke nod anak ini secara masa nyata,',
+            syncBackupAccount: 'Data akaun sandaran,',
+            syncProxy: 'Data proksi sistem,',
             nodeSyncHelper: 'Penyelarasan maklumat nod akan menyelaraskan maklumat berikut:',
             nodeSyncHelper1: '1. Maklumat akaun sandaran awam',
             nodeSyncHelper2: '2. Maklumat sambungan antara nod utama dan nod sub',

frontend/src/lang/modules/pt-br.ts

@@ -3193,6 +3193,10 @@ const message = {
             addNode: 'Adicionar Nó',
             connInfo: 'Informações de Conexão',
             nodeInfo: 'Informações do Nó',
+            syncInfo: 'Sincronização de dados,',
+            syncHelper: 'Quando os dados do nó mestre mudam, são sincronizados em tempo real para este nó filho,',
+            syncBackupAccount: 'Dados de conta de backup,',
+            syncProxy: 'Dados de proxy do sistema,',
             nodeSyncHelper: 'A sincronização das informações do nó irá sincronizar as seguintes informações:',
             nodeSyncHelper1: '1. Informações da conta de backup pública',
             nodeSyncHelper2: '2. Informações de conexão entre o nó principal e os sub-nós',

frontend/src/lang/modules/ru.ts

@@ -3181,6 +3181,11 @@ const message = {
             addNode: 'Добавить узел',
             connInfo: 'Информация о подключении',
             nodeInfo: 'Информация об узле',
+            syncInfo: 'Синхронизация данных,',
+            syncHelper:
+                'При изменении данных главного узла, происходит синхронизация с этим дочерним узлом в реальном времени,',
+            syncBackupAccount: 'Резервные данные аккаунта,',
+            syncProxy: 'Данные системного прокси,',
             nodeSyncHelper: 'Синхронизация информации о узле будет синхронизировать следующую информацию:',
             nodeSyncHelper1: '1. Информация о публичной резервной учетной записи',
             nodeSyncHelper2: '2. Информация о соединении между основным узлом и подузлами',

frontend/src/lang/modules/zh-Hant.ts

@@ -3022,6 +3022,10 @@ const message = {
             addNode: '新增節點',
             connInfo: '連接資訊',
             nodeInfo: '節點資訊',
+            syncInfo: '數據同步,',
+            syncHelper: '當主節點數據發生變化時，實時同步到該子節點,',
+            syncBackupAccount: '備份帳號數據,',
+            syncProxy: '系統代理數據,',
             nodeSyncHelper: '節點信息同步將同步以下信息：',
             nodeSyncHelper1: '1. 公用的備份帳號信息',
             nodeSyncHelper2: '2. 主節點與子節點的連接信息',

frontend/src/lang/modules/zh.ts

@@ -3004,6 +3004,10 @@ const message = {
             addNode: '添加节点',
             connInfo: '连接信息',
             nodeInfo: '节点信息',
+            syncInfo: '数据同步',
+            syncHelper: '主节点数据发生变化时，实时同步到该子节点',
+            syncBackupAccount: '备份账号数据',
+            syncProxy: '系统代理数据',
             nodeSyncHelper: '节点信息同步将同步以下信息：',
             nodeSyncHelper1: '1. 公用的备份账号信息',
             nodeSyncHelper2: '2. 主节点与子节点的连接信息',