From aeabed70db779beaa0f40af5e1793c63e3578eb7 Mon Sep 17 00:00:00 2001 From: zhengkunwang223 Date: Mon, 15 May 2023 22:40:05 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=BA=94=E7=94=A8=E5=95=86=E5=BA=97?= =?UTF-8?q?=E5=AF=B9=E6=8E=A5=E8=BF=9C=E7=A8=8B=E5=BA=94=E7=94=A8=E6=9C=8D?= =?UTF-8?q?=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/dto/app.go | 47 +- backend/app/dto/response/app.go | 9 +- backend/app/dto/setting.go | 3 +- backend/app/model/app.go | 39 +- backend/app/model/app_detail.go | 17 +- backend/app/repo/app.go | 5 + backend/app/service/app.go | 478 +++++++++--------- backend/app/service/app_install.go | 4 + backend/app/service/app_utils.go | 84 ++- backend/configs/system.go | 37 +- backend/constant/app.go | 14 +- backend/constant/errs.go | 11 +- backend/i18n/lang/en.yaml | 2 + backend/i18n/lang/zh.yaml | 2 + backend/init/migration/migrate.go | 2 + backend/init/migration/migrations/init.go | 23 + cmd/server/conf/app.yaml | 1 + frontend/src/api/interface/app.ts | 1 + frontend/src/views/app-store/apps/index.vue | 3 + frontend/src/views/app-store/detail/index.vue | 8 +- 20 files changed, 457 insertions(+), 333 deletions(-) diff --git a/backend/app/dto/app.go b/backend/app/dto/app.go index 0f77a03f3..cb1d7870c 100644 --- a/backend/app/dto/app.go +++ b/backend/app/dto/app.go @@ -31,20 +31,42 @@ type AppVersion struct { DetailId uint `json:"detailId"` } +//type AppList struct { +// Version string `json:"version"` +// Tags []Tag `json:"tags"` +// Items []AppDefine `json:"items"` +//} + type AppList struct { - Version string `json:"version"` - Tags []Tag `json:"tags"` - Items []AppDefine `json:"items"` + Valid bool `json:"valid"` + Violations []string `json:"violations"` + LastModified int `json:"lastModified"` + + Apps []AppDefine `json:"apps"` + Extra ExtraProperties `json:"additionalProperties"` } type AppDefine struct { - Key string `json:"key"` + Icon string `json:"icon"` + Name string `json:"name"` + ReadMe string `json:"readMe"` + LastModified int `json:"lastModified"` + + AppProperty AppProperty `json:"additionalProperties"` + Versions []AppConfigVersion `json:"versions"` +} + +type ExtraProperties struct { + Tags []Tag `json:"tags"` +} + +type AppProperty struct { Name string `json:"name"` + Type string `json:"type"` Tags []string `json:"tags"` - Versions []string `json:"versions"` ShortDescZh string `json:"shortDescZh"` ShortDescEn string `json:"shortDescEn"` - Type string `json:"type"` + Key string `json:"key"` Required []string `json:"Required"` CrossVersionUpdate bool `json:"crossVersionUpdate"` Limit int `json:"limit"` @@ -54,8 +76,16 @@ type AppDefine struct { Document string `json:"document"` } -func (define AppDefine) GetRequired() string { - by, _ := json.Marshal(define.Required) +type AppConfigVersion struct { + Name string `json:"name"` + LastModified int `json:"lastModified"` + DownloadUrl string `json:"downloadUrl"` + DownloadCallBackUrl string `json:"downloadCallBackUrl"` + AppForm interface{} `json:"additionalProperties"` +} + +func (config AppProperty) GetRequired() string { + by, _ := json.Marshal(config.Required) return string(by) } @@ -79,6 +109,7 @@ type AppFormFields struct { Edit bool `json:"edit"` Rule string `json:"rule"` Multiple bool `json:"multiple"` + Child interface{} `json:"child"` Values []AppFormValue `json:"values"` } diff --git a/backend/app/dto/response/app.go b/backend/app/dto/response/app.go index f9ac0b3de..ca3aa5a24 100644 --- a/backend/app/dto/response/app.go +++ b/backend/app/dto/response/app.go @@ -1,6 +1,7 @@ package response import ( + "github.com/1Panel-dev/1Panel/backend/app/dto" "time" "github.com/1Panel-dev/1Panel/backend/app/model" @@ -12,9 +13,11 @@ type AppRes struct { } type AppUpdateRes struct { - Version string `json:"version"` - CanUpdate bool `json:"canUpdate"` - DownloadPath string `json:"downloadPath"` + //Version string `json:"version"` + //DownloadPath string `json:"downloadPath"` + CanUpdate bool `json:"canUpdate"` + AppStoreLastModified int `json:"appStoreLastModified"` + List dto.AppList `json:"list"` } type AppDTO struct { diff --git a/backend/app/dto/setting.go b/backend/app/dto/setting.go index 9a3aee038..3cbc8ac87 100644 --- a/backend/app/dto/setting.go +++ b/backend/app/dto/setting.go @@ -33,7 +33,8 @@ type SettingInfo struct { WeChatVars string `json:"weChatVars"` DingVars string `json:"dingVars"` - AppStoreVersion string `json:"appStoreVersion"` + AppStoreVersion string `json:"appStoreVersion"` + AppStoreLastModified string `json:"appStoreLastModified"` } type SettingUpdate struct { diff --git a/backend/app/model/app.go b/backend/app/model/app.go index 1772d4042..7fe76bbb5 100644 --- a/backend/app/model/app.go +++ b/backend/app/model/app.go @@ -2,22 +2,25 @@ package model type App struct { BaseModel - Name string `json:"name" gorm:"type:varchar(64);not null"` - Key string `json:"key" gorm:"type:varchar(64);not null;uniqueIndex"` - ShortDescZh string `json:"shortDescZh" gorm:"type:longtext;"` - ShortDescEn string `json:"shortDescEn" gorm:"type:longtext;"` - Icon string `json:"icon" gorm:"type:longtext;"` - Type string `json:"type" gorm:"type:varchar(64);not null"` - Status string `json:"status" gorm:"type:varchar(64);not null"` - Required string `json:"required" gorm:"type:varchar(64);not null"` - CrossVersionUpdate bool `json:"crossVersionUpdate"` - Limit int `json:"limit" gorm:"type:Integer;not null"` - Website string `json:"website" gorm:"type:varchar(64);not null"` - Github string `json:"github" gorm:"type:varchar(64);not null"` - Document string `json:"document" gorm:"type:varchar(64);not null"` - Recommend int `json:"recommend" gorm:"type:Integer;not null"` - Resource string `json:"resource" gorm:"type:varchar;not null;default:remote"` - Details []AppDetail `json:"-" gorm:"-:migration"` - TagsKey []string `json:"-" gorm:"-"` - AppTags []AppTag `json:"-" gorm:"-:migration"` + Name string `json:"name" gorm:"type:varchar(64);not null"` + Key string `json:"key" gorm:"type:varchar(64);not null;uniqueIndex"` + ShortDescZh string `json:"shortDescZh" gorm:"type:longtext;"` + ShortDescEn string `json:"shortDescEn" gorm:"type:longtext;"` + Icon string `json:"icon" gorm:"type:longtext;"` + Type string `json:"type" gorm:"type:varchar(64);not null"` + Status string `json:"status" gorm:"type:varchar(64);not null"` + Required string `json:"required" gorm:"type:varchar(64);not null"` + CrossVersionUpdate bool `json:"crossVersionUpdate"` + Limit int `json:"limit" gorm:"type:Integer;not null"` + Website string `json:"website" gorm:"type:varchar(64);not null"` + Github string `json:"github" gorm:"type:varchar(64);not null"` + Document string `json:"document" gorm:"type:varchar(64);not null"` + Recommend int `json:"recommend" gorm:"type:Integer;not null"` + Resource string `json:"resource" gorm:"type:varchar;not null;default:remote"` + ReadMe string `json:"readMe" gorm:"type:varchar;"` + LastModified int `json:"lastModified" gorm:"type:Integer;"` + + Details []AppDetail `json:"-" gorm:"-:migration"` + TagsKey []string `json:"-" gorm:"-"` + AppTags []AppTag `json:"-" gorm:"-:migration"` } diff --git a/backend/app/model/app_detail.go b/backend/app/model/app_detail.go index 26b3fe877..e2a8269e4 100644 --- a/backend/app/model/app_detail.go +++ b/backend/app/model/app_detail.go @@ -2,11 +2,14 @@ package model type AppDetail struct { BaseModel - AppId uint `json:"appId" gorm:"type:integer;not null"` - Version string `json:"version" gorm:"type:varchar(64);not null"` - Params string `json:"-" gorm:"type:longtext;"` - DockerCompose string `json:"-" gorm:"type:longtext;not null"` - Readme string `json:"readme" gorm:"type:longtext;"` - Status string `json:"status" gorm:"type:varchar(64);not null"` - LastVersion string `json:"lastVersion" gorm:"type:varchar(64);"` + AppId uint `json:"appId" gorm:"type:integer;not null"` + Version string `json:"version" gorm:"type:varchar(64);not null"` + Params string `json:"-" gorm:"type:longtext;"` + DockerCompose string `json:"-" gorm:"type:longtext;"` + Status string `json:"status" gorm:"type:varchar(64);not null"` + LastVersion string `json:"lastVersion" gorm:"type:varchar(64);"` + LastModified int `json:"lastModified" gorm:"type:integer;"` + DownloadUrl string `json:"downloadUrl" gorm:"type:varchar;"` + DownloadCallBackUrl string `json:"downloadCallBackUrl" gorm:"type:longtext;"` + Update bool `json:"update"` } diff --git a/backend/app/repo/app.go b/backend/app/repo/app.go index 0e9ca1af9..c6bb64133 100644 --- a/backend/app/repo/app.go +++ b/backend/app/repo/app.go @@ -24,6 +24,7 @@ type IAppRepo interface { GetByKey(ctx context.Context, key string) (model.App, error) Create(ctx context.Context, app *model.App) error Save(ctx context.Context, app *model.App) error + BatchDelete(ctx context.Context, apps []model.App) error } func NewIAppRepo() IAppRepo { @@ -106,3 +107,7 @@ func (a AppRepo) Create(ctx context.Context, app *model.App) error { func (a AppRepo) Save(ctx context.Context, app *model.App) error { return getTx(ctx).Omit(clause.Associations).Save(app).Error } + +func (a AppRepo) BatchDelete(ctx context.Context, apps []model.App) error { + return getTx(ctx).Omit(clause.Associations).Delete(&apps).Error +} diff --git a/backend/app/service/app.go b/backend/app/service/app.go index 24d6b155b..8a316b797 100644 --- a/backend/app/service/app.go +++ b/backend/app/service/app.go @@ -5,14 +5,12 @@ import ( "encoding/base64" "encoding/json" "fmt" - "io" - "net/http" - "os" - "path" - "strings" - "github.com/1Panel-dev/1Panel/backend/buserr" "github.com/1Panel-dev/1Panel/backend/utils/docker" + "io" + "net/http" + "path" + "strconv" "github.com/1Panel-dev/1Panel/backend/app/dto" "github.com/1Panel-dev/1Panel/backend/app/dto/request" @@ -253,7 +251,7 @@ func (a AppService) Install(ctx context.Context, req request.AppInstallCreate) ( } app, err = appRepo.GetFirst(commonRepo.WithByID(appDetail.AppId)) if err != nil { - return nil, err + return } if err = checkRequiredAndLimit(app); err != nil { return @@ -276,7 +274,7 @@ func (a AppService) Install(ctx context.Context, req request.AppInstallCreate) ( value, ok := composeMap["services"] if !ok { - err = buserr.New("") + err = buserr.New(constant.ErrFileParse) return } servicesMap := value.(map[string]interface{}) @@ -318,20 +316,11 @@ func (a AppService) Install(ctx context.Context, req request.AppInstallCreate) ( } } }() - - if err = copyAppData(app.Key, appDetail.Version, req.Name, req.Params, app.Resource == constant.AppResourceLocal); err != nil { - return - } - fileOp := files.NewFileOp() - if err = fileOp.WriteFile(appInstall.GetComposePath(), strings.NewReader(string(composeByte)), 0775); err != nil { - return - } paramByte, err = json.Marshal(req.Params) if err != nil { return } appInstall.Env = string(paramByte) - if err = appInstallRepo.Create(ctx, appInstall); err != nil { return } @@ -341,7 +330,13 @@ func (a AppService) Install(ctx context.Context, req request.AppInstallCreate) ( if err = upAppPre(app, appInstall); err != nil { return } - go upApp(appInstall) + go func() { + if err = downloadApp(app, appDetail, appInstall, req); err != nil { + _ = appInstallRepo.Save(ctx, appInstall) + return + } + upApp(appInstall) + }() go updateToolApp(appInstall) return } @@ -354,7 +349,7 @@ func (a AppService) GetAppUpdate() (*response.AppUpdateRes, error) { if err != nil { return nil, err } - versionUrl := fmt.Sprintf("%s/%s/%s/appstore/apps.json", global.CONF.System.RepoUrl, global.CONF.System.Mode, setting.SystemVersion) + versionUrl := fmt.Sprintf("%s/%s/1panel.json", global.CONF.System.AppRepo, global.CONF.System.Mode) versionRes, err := http.Get(versionUrl) if err != nil { return nil, err @@ -368,162 +363,165 @@ func (a AppService) GetAppUpdate() (*response.AppUpdateRes, error) { if err = json.Unmarshal(body, list); err != nil { return nil, err } - res.Version = list.Version - if setting.AppStoreVersion == "" || common.CompareVersion(list.Version, setting.AppStoreVersion) { + res.AppStoreLastModified = list.LastModified + res.List = *list + + appStoreLastModified, _ := strconv.Atoi(setting.AppStoreLastModified) + if setting.AppStoreLastModified == "" || list.LastModified > appStoreLastModified { res.CanUpdate = true - res.DownloadPath = fmt.Sprintf("%s/%s/%s/appstore/apps-%s.tar.gz", global.CONF.System.RepoUrl, global.CONF.System.Mode, setting.SystemVersion, list.Version) return res, err } + res.CanUpdate = true return res, nil } func (a AppService) SyncAppListFromLocal() { - fileOp := files.NewFileOp() - appDir := constant.LocalAppResourceDir - listFile := path.Join(appDir, "list.json") - if !fileOp.Stat(listFile) { - return - } - global.LOG.Infof("start sync local apps...") - content, err := fileOp.GetContent(listFile) - if err != nil { - global.LOG.Errorf("get list.json content failed %s", err.Error()) - return - } - list := &dto.AppList{} - if err := json.Unmarshal(content, list); err != nil { - global.LOG.Errorf("unmarshal list.json failed %s", err.Error()) - return - } - oldApps, _ := appRepo.GetBy(appRepo.WithResource(constant.AppResourceLocal)) - appsMap := getApps(oldApps, list.Items, true) - for _, l := range list.Items { - localKey := "local" + l.Key - app := appsMap[localKey] - icon, err := os.ReadFile(path.Join(appDir, l.Key, "metadata", "logo.png")) - if err != nil { - global.LOG.Errorf("get [%s] icon error: %s", l.Name, err.Error()) - continue - } - iconStr := base64.StdEncoding.EncodeToString(icon) - app.Icon = iconStr - app.TagsKey = append(l.Tags, "Local") - app.Recommend = 9999 - versions := l.Versions - detailsMap := getAppDetails(app.Details, versions) - - for _, v := range versions { - detail := detailsMap[v] - detailPath := path.Join(appDir, l.Key, "versions", v) - if _, err := os.Stat(detailPath); err != nil { - global.LOG.Errorf("get [%s] folder error: %s", detailPath, err.Error()) - continue - } - readmeStr, err := os.ReadFile(path.Join(detailPath, "README.md")) - if err != nil { - global.LOG.Errorf("get [%s] README error: %s", detailPath, err.Error()) - } - detail.Readme = string(readmeStr) - dockerComposeStr, err := os.ReadFile(path.Join(detailPath, "docker-compose.yml")) - if err != nil { - global.LOG.Errorf("get [%s] docker-compose.yml error: %s", detailPath, err.Error()) - continue - } - detail.DockerCompose = string(dockerComposeStr) - paramStr, err := os.ReadFile(path.Join(detailPath, "config.json")) - if err != nil { - global.LOG.Errorf("get [%s] form.json error: %s", detailPath, err.Error()) - } - detail.Params = string(paramStr) - detailsMap[v] = detail - } - var newDetails []model.AppDetail - for _, v := range detailsMap { - newDetails = append(newDetails, v) - } - app.Details = newDetails - appsMap[localKey] = app - } - var ( - addAppArray []model.App - updateArray []model.App - appIds []uint - ) - for _, v := range appsMap { - if v.ID == 0 { - addAppArray = append(addAppArray, v) - } else { - updateArray = append(updateArray, v) - appIds = append(appIds, v.ID) - } - } - tx, ctx := getTxAndContext() - if len(addAppArray) > 0 { - if err := appRepo.BatchCreate(ctx, addAppArray); err != nil { - tx.Rollback() - return - } - } - for _, update := range updateArray { - if err := appRepo.Save(ctx, &update); err != nil { - tx.Rollback() - return - } - } - if err := appTagRepo.DeleteByAppIds(ctx, appIds); err != nil { - tx.Rollback() - return - } - apps := append(addAppArray, updateArray...) - var ( - addDetails []model.AppDetail - updateDetails []model.AppDetail - appTags []*model.AppTag - ) - tags, _ := tagRepo.All() - tagMap := make(map[string]uint, len(tags)) - for _, app := range tags { - tagMap[app.Key] = app.ID - } - for _, a := range apps { - for _, t := range a.TagsKey { - tagId, ok := tagMap[t] - if ok { - appTags = append(appTags, &model.AppTag{ - AppId: a.ID, - TagId: tagId, - }) - } - } - for _, d := range a.Details { - d.AppId = a.ID - if d.ID == 0 { - addDetails = append(addDetails, d) - } else { - updateDetails = append(updateDetails, d) - } - } - } - if len(addDetails) > 0 { - if err := appDetailRepo.BatchCreate(ctx, addDetails); err != nil { - tx.Rollback() - return - } - } - for _, u := range updateDetails { - if err := appDetailRepo.Update(ctx, u); err != nil { - tx.Rollback() - return - } - } - if len(appTags) > 0 { - if err := appTagRepo.BatchCreate(ctx, appTags); err != nil { - tx.Rollback() - return - } - } - tx.Commit() - global.LOG.Infof("sync local apps success") + //fileOp := files.NewFileOp() + //appDir := constant.LocalAppResourceDir + //listFile := path.Join(appDir, "list.json") + //if !fileOp.Stat(listFile) { + // return + //} + //global.LOG.Infof("start sync local apps...") + //content, err := fileOp.GetContent(listFile) + //if err != nil { + // global.LOG.Errorf("get list.json content failed %s", err.Error()) + // return + //} + //list := &dto.AppList{} + //if err := json.Unmarshal(content, list); err != nil { + // global.LOG.Errorf("unmarshal list.json failed %s", err.Error()) + // return + //} + //oldApps, _ := appRepo.GetBy(appRepo.WithResource(constant.AppResourceLocal)) + //appsMap := getApps(oldApps, list.Apps, true) + //for _, l := range list.Apps { + // localKey := "local" + l.Config.Key + // app := appsMap[localKey] + // icon, err := os.ReadFile(path.Join(appDir, l.Config.Key, "metadata", "logo.png")) + // if err != nil { + // global.LOG.Errorf("get [%s] icon error: %s", l.Name, err.Error()) + // continue + // } + // iconStr := base64.StdEncoding.EncodeToString(icon) + // app.Icon = iconStr + // app.TagsKey = append(l.Tags, "Local") + // app.Recommend = 9999 + // versions := l.Versions + // detailsMap := getAppDetails(app.Details, versions) + // + // for _, v := range versions { + // detail := detailsMap[v] + // detailPath := path.Join(appDir, l.Key, "versions", v) + // if _, err := os.Stat(detailPath); err != nil { + // global.LOG.Errorf("get [%s] folder error: %s", detailPath, err.Error()) + // continue + // } + // readmeStr, err := os.ReadFile(path.Join(detailPath, "README.md")) + // if err != nil { + // global.LOG.Errorf("get [%s] README error: %s", detailPath, err.Error()) + // } + // detail.Readme = string(readmeStr) + // dockerComposeStr, err := os.ReadFile(path.Join(detailPath, "docker-compose.yml")) + // if err != nil { + // global.LOG.Errorf("get [%s] docker-compose.yml error: %s", detailPath, err.Error()) + // continue + // } + // detail.DockerCompose = string(dockerComposeStr) + // paramStr, err := os.ReadFile(path.Join(detailPath, "config.json")) + // if err != nil { + // global.LOG.Errorf("get [%s] form.json error: %s", detailPath, err.Error()) + // } + // detail.Params = string(paramStr) + // detailsMap[v] = detail + // } + // var newDetails []model.AppDetail + // for _, v := range detailsMap { + // newDetails = append(newDetails, v) + // } + // app.Details = newDetails + // appsMap[localKey] = app + //} + //var ( + // addAppArray []model.App + // updateArray []model.App + // appIds []uint + //) + //for _, v := range appsMap { + // if v.ID == 0 { + // addAppArray = append(addAppArray, v) + // } else { + // updateArray = append(updateArray, v) + // appIds = append(appIds, v.ID) + // } + //} + //tx, ctx := getTxAndContext() + //if len(addAppArray) > 0 { + // if err := appRepo.BatchCreate(ctx, addAppArray); err != nil { + // tx.Rollback() + // return + // } + //} + //for _, update := range updateArray { + // if err := appRepo.Save(ctx, &update); err != nil { + // tx.Rollback() + // return + // } + //} + //if err := appTagRepo.DeleteByAppIds(ctx, appIds); err != nil { + // tx.Rollback() + // return + //} + //apps := append(addAppArray, updateArray...) + //var ( + // addDetails []model.AppDetail + // updateDetails []model.AppDetail + // appTags []*model.AppTag + //) + //tags, _ := tagRepo.All() + //tagMap := make(map[string]uint, len(tags)) + //for _, app := range tags { + // tagMap[app.Key] = app.ID + //} + //for _, a := range apps { + // for _, t := range a.TagsKey { + // tagId, ok := tagMap[t] + // if ok { + // appTags = append(appTags, &model.AppTag{ + // AppId: a.ID, + // TagId: tagId, + // }) + // } + // } + // for _, d := range a.Details { + // d.AppId = a.ID + // if d.ID == 0 { + // addDetails = append(addDetails, d) + // } else { + // updateDetails = append(updateDetails, d) + // } + // } + //} + //if len(addDetails) > 0 { + // if err := appDetailRepo.BatchCreate(ctx, addDetails); err != nil { + // tx.Rollback() + // return + // } + //} + //for _, u := range updateDetails { + // if err := appDetailRepo.Update(ctx, u); err != nil { + // tx.Rollback() + // return + // } + //} + //if len(appTags) > 0 { + // if err := appTagRepo.BatchCreate(ctx, appTags); err != nil { + // tx.Rollback() + // return + // } + //} + //tx.Commit() + //global.LOG.Infof("sync local apps success") } func (a AppService) SyncAppListFromRemote() error { updateRes, err := a.GetAppUpdate() @@ -531,28 +529,15 @@ func (a AppService) SyncAppListFromRemote() error { return err } if !updateRes.CanUpdate { - global.LOG.Infof("The latest version is [%s] The app store is already up to date", updateRes.Version) + //global.LOG.Infof("The latest version is [%s] The app store is already up to date", updateRes.Version) return nil } - if err := getAppFromRepo(updateRes.DownloadPath, updateRes.Version); err != nil { - global.LOG.Errorf("get app from oss error: %s", err.Error()) - return err - } - appDir := constant.AppResourceDir - listFile := path.Join(appDir, "list.json") - content, err := os.ReadFile(listFile) - if err != nil { - return err - } - list := &dto.AppList{} - if err := json.Unmarshal(content, list); err != nil { - return err - } var ( tags []*model.Tag appTags []*model.AppTag + list = updateRes.List ) - for _, t := range list.Tags { + for _, t := range list.Extra.Tags { tags = append(tags, &model.Tag{ Key: t.Key, Name: t.Name, @@ -562,70 +547,86 @@ func (a AppService) SyncAppListFromRemote() error { if err != nil { return err } - appsMap := getApps(oldApps, list.Items, false) - for _, l := range list.Items { - app := appsMap[l.Key] - icon, err := os.ReadFile(path.Join(appDir, l.Key, "metadata", "logo.png")) + baseRemoteUrl := fmt.Sprintf("%s/%s/1panel", global.CONF.System.AppRepo, global.CONF.System.Mode) + appsMap := getApps(oldApps, list.Apps, false) + for _, l := range list.Apps { + app := appsMap[l.AppProperty.Key] + iconRes, err := http.Get(l.Icon) if err != nil { - global.LOG.Errorf("get [%s] icon error: %s", l.Name, err.Error()) - continue + return err } - iconStr := base64.StdEncoding.EncodeToString(icon) + body, err := io.ReadAll(iconRes.Body) + if err != nil { + return err + } + iconStr := base64.StdEncoding.EncodeToString(body) app.Icon = iconStr - app.TagsKey = l.Tags - if l.Recommend > 0 { - app.Recommend = l.Recommend + app.TagsKey = l.AppProperty.Tags + if l.AppProperty.Recommend > 0 { + app.Recommend = l.AppProperty.Recommend } else { app.Recommend = 9999 } - + app.ReadMe = l.ReadMe + app.LastModified = l.LastModified versions := l.Versions detailsMap := getAppDetails(app.Details, versions) - for _, v := range versions { - detail := detailsMap[v] - detailPath := path.Join(appDir, l.Key, "versions", v) - if _, err := os.Stat(detailPath); err != nil { - global.LOG.Errorf("get [%s] folder error: %s", detailPath, err.Error()) - continue - } - readmeStr, err := os.ReadFile(path.Join(detailPath, "README.md")) + version := v.Name + detail := detailsMap[version] + + dockerComposeUrl := fmt.Sprintf("%s/%s/%s/%s", baseRemoteUrl, app.Key, version, "docker-compose.yml") + composeRes, err := http.Get(dockerComposeUrl) if err != nil { - global.LOG.Errorf("get [%s] README error: %s", detailPath, err.Error()) + return err } - detail.Readme = string(readmeStr) - dockerComposeStr, err := os.ReadFile(path.Join(detailPath, "docker-compose.yml")) + bodyContent, err := io.ReadAll(composeRes.Body) if err != nil { - global.LOG.Errorf("get [%s] docker-compose.yml error: %s", detailPath, err.Error()) - continue + return err } - detail.DockerCompose = string(dockerComposeStr) - paramStr, err := os.ReadFile(path.Join(detailPath, "config.json")) - if err != nil { - global.LOG.Errorf("get [%s] form.json error: %s", detailPath, err.Error()) + detail.DockerCompose = string(bodyContent) + + paramByte, _ := json.Marshal(v.AppForm) + detail.Params = string(paramByte) + detail.DownloadUrl = v.DownloadUrl + detail.DownloadCallBackUrl = v.DownloadCallBackUrl + if v.LastModified > detail.LastModified { + detail.Update = true + detail.LastModified = v.LastModified } - detail.Params = string(paramStr) - detailsMap[v] = detail + detailsMap[version] = detail } var newDetails []model.AppDetail - for _, v := range detailsMap { - newDetails = append(newDetails, v) + for _, detail := range detailsMap { + newDetails = append(newDetails, detail) } app.Details = newDetails - appsMap[l.Key] = app + appsMap[l.AppProperty.Key] = app } var ( - addAppArray []model.App - updateArray []model.App - tagMap = make(map[string]uint, len(tags)) + addAppArray []model.App + updateAppArray []model.App + deleteAppArray []model.App + deleteIds []uint + tagMap = make(map[string]uint, len(tags)) ) for _, v := range appsMap { if v.ID == 0 { addAppArray = append(addAppArray, v) } else { - updateArray = append(updateArray, v) + if v.Status == constant.AppTakeDown { + installs, _ := appInstallRepo.ListBy(appInstallRepo.WithAppId(v.ID)) + if len(installs) > 0 { + updateAppArray = append(updateAppArray, v) + continue + } + deleteAppArray = append(deleteAppArray, v) + deleteIds = append(deleteIds, v.ID) + } else { + updateAppArray = append(updateAppArray, v) + } } } tx, ctx := getTxAndContext() @@ -635,6 +636,16 @@ func (a AppService) SyncAppListFromRemote() error { return err } } + if len(deleteAppArray) > 0 { + if err := appRepo.BatchDelete(ctx, deleteAppArray); err != nil { + tx.Rollback() + return err + } + if err := appDetailRepo.DeleteByAppIds(ctx, deleteIds); err != nil { + tx.Rollback() + return err + } + } if err := tagRepo.DeleteAll(ctx); err != nil { tx.Rollback() return err @@ -648,34 +659,39 @@ func (a AppService) SyncAppListFromRemote() error { tagMap[t.Key] = t.ID } } - for _, update := range updateArray { + for _, update := range updateAppArray { if err := appRepo.Save(ctx, &update); err != nil { tx.Rollback() return err } } - apps := append(addAppArray, updateArray...) + apps := append(addAppArray, updateAppArray...) var ( addDetails []model.AppDetail updateDetails []model.AppDetail + deleteDetails []model.AppDetail ) - for _, a := range apps { - for _, t := range a.TagsKey { + for _, app := range apps { + for _, t := range app.TagsKey { tagId, ok := tagMap[t] if ok { appTags = append(appTags, &model.AppTag{ - AppId: a.ID, + AppId: app.ID, TagId: tagId, }) } } - for _, d := range a.Details { - d.AppId = a.ID + for _, d := range app.Details { + d.AppId = app.ID if d.ID == 0 { addDetails = append(addDetails, d) } else { - updateDetails = append(updateDetails, d) + if d.Status == constant.AppTakeDown { + deleteDetails = append(deleteDetails, d) + } else { + updateDetails = append(updateDetails, d) + } } } } @@ -691,6 +707,7 @@ func (a AppService) SyncAppListFromRemote() error { return err } } + if err := appTagRepo.DeleteAll(ctx); err != nil { tx.Rollback() return err @@ -702,5 +719,8 @@ func (a AppService) SyncAppListFromRemote() error { } } tx.Commit() + if err := NewISettingService().Update("AppStoreLastModified", strconv.Itoa(list.LastModified)); err != nil { + return err + } return nil } diff --git a/backend/app/service/app_install.go b/backend/app/service/app_install.go index 95cf4b43d..935641503 100644 --- a/backend/app/service/app_install.go +++ b/backend/app/service/app_install.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/1Panel-dev/1Panel/backend/utils/files" "math" "os" "path" @@ -183,6 +184,9 @@ func (a *AppInstallService) Operate(req request.AppInstalledOperate) error { if err != nil { return err } + if !req.ForceDelete && !files.NewFileOp().Stat(install.GetPath()) { + return buserr.New(constant.ErrInstallDirNotFound) + } dockerComposePath := install.GetComposePath() switch req.Operate { case constant.Rebuild: diff --git a/backend/app/service/app_utils.go b/backend/app/service/app_utils.go index 9a80f2850..94c463a70 100644 --- a/backend/app/service/app_utils.go +++ b/backend/app/service/app_utils.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "github.com/1Panel-dev/1Panel/backend/app/api/v1/helper" + "github.com/1Panel-dev/1Panel/backend/app/dto/request" "github.com/compose-spec/compose-go/types" "github.com/subosito/gotenv" "math" @@ -225,7 +226,7 @@ func upgradeInstall(installId uint, detailId uint) error { return err } - detailDir := path.Join(constant.ResourceDir, "apps", install.App.Key, "versions", detail.Version) + detailDir := path.Join(constant.ResourceDir, "apps", install.App.Resource, install.App.Key, detail.Version) if install.App.Resource == constant.AppResourceLocal { detailDir = path.Join(constant.ResourceDir, "localApps", strings.TrimPrefix(install.App.Key, "local"), "versions", detail.Version) } @@ -359,24 +360,48 @@ func handleMap(params map[string]interface{}, envParams map[string]string) { } } -func copyAppData(key, version, installName string, params map[string]interface{}, isLocal bool) (err error) { +func downloadApp(app model.App, appDetail model.AppDetail, appInstall *model.AppInstall, req request.AppInstallCreate) (err error) { fileOp := files.NewFileOp() - appResourceDir := constant.AppResourceDir - installAppDir := path.Join(constant.AppInstallDir, key) - appKey := key - if isLocal { + appResourceDir := path.Join(constant.AppResourceDir, app.Resource) + + if app.Resource == constant.AppResourceRemote && appDetail.Update { + appDownloadDir := path.Join(appResourceDir, app.Key) + if !fileOp.Stat(appDownloadDir) { + _ = fileOp.CreateDir(appDownloadDir, 0755) + } + appVersionDir := path.Join(appDownloadDir, appDetail.Version) + if !fileOp.Stat(appVersionDir) { + _ = fileOp.CreateDir(appVersionDir, 0755) + } + global.LOG.Infof("download app[%s] from %s", app.Name, appDetail.DownloadUrl) + filePath := path.Join(appVersionDir, appDetail.Version+".tar.gz") + if err = fileOp.DownloadFile(appDetail.DownloadUrl, filePath); err != nil { + appInstall.Status = constant.DownloadErr + global.LOG.Errorf("download app[%s] error %v", app.Name, err) + return + } + if err = fileOp.Decompress(filePath, appVersionDir, files.TarGz); err != nil { + global.LOG.Errorf("decompress app[%s] error %v", app.Name, err) + appInstall.Status = constant.DownloadErr + return + } + _ = fileOp.DeleteFile(filePath) + } + appKey := app.Key + installAppDir := path.Join(constant.AppInstallDir, app.Key) + if app.Resource == constant.AppResourceLocal { appResourceDir = constant.LocalAppResourceDir - appKey = strings.TrimPrefix(key, "local") + appKey = strings.TrimPrefix(app.Resource, "local") installAppDir = path.Join(constant.LocalAppInstallDir, appKey) } - resourceDir := path.Join(appResourceDir, appKey, "versions", version) + resourceDir := path.Join(appResourceDir, appKey, appDetail.Version) if !fileOp.Stat(installAppDir) { if err = fileOp.CreateDir(installAppDir, 0755); err != nil { return } } - appDir := path.Join(installAppDir, installName) + appDir := path.Join(installAppDir, req.Name) if fileOp.Stat(appDir) { if err = fileOp.DeleteDir(appDir); err != nil { return @@ -385,14 +410,14 @@ func copyAppData(key, version, installName string, params map[string]interface{} if err = fileOp.Copy(resourceDir, installAppDir); err != nil { return } - versionDir := path.Join(installAppDir, version) + versionDir := path.Join(installAppDir, appDetail.Version) if err = fileOp.Rename(versionDir, appDir); err != nil { return } envPath := path.Join(appDir, ".env") - envParams := make(map[string]string, len(params)) - handleMap(params, envParams) + envParams := make(map[string]string, len(req.Params)) + handleMap(req.Params, envParams) if err = env.Write(envParams, envPath); err != nil { return } @@ -473,21 +498,21 @@ func rebuildApp(appInstall model.AppInstall) error { return syncById(appInstall.ID) } -func getAppDetails(details []model.AppDetail, versions []string) map[string]model.AppDetail { +func getAppDetails(details []model.AppDetail, versions []dto.AppConfigVersion) map[string]model.AppDetail { appDetails := make(map[string]model.AppDetail, len(details)) for _, old := range details { old.Status = constant.AppTakeDown appDetails[old.Version] = old } - for _, v := range versions { - detail, ok := appDetails[v] + version := v.Name + detail, ok := appDetails[version] if ok { detail.Status = constant.AppNormal - appDetails[v] = detail + appDetails[version] = detail } else { - appDetails[v] = model.AppDetail{ - Version: v, + appDetails[version] = model.AppDetail{ + Version: version, Status: constant.AppNormal, } } @@ -502,7 +527,8 @@ func getApps(oldApps []model.App, items []dto.AppDefine, isLocal bool) map[strin apps[old.Key] = old } for _, item := range items { - key := item.Key + config := item.AppProperty + key := config.Key if isLocal { key = "local" + key } @@ -516,17 +542,19 @@ func getApps(oldApps []model.App, items []dto.AppDefine, isLocal bool) map[strin app.Resource = constant.AppResourceRemote } app.Name = item.Name - app.Limit = item.Limit + app.Limit = config.Limit app.Key = key - app.ShortDescZh = item.ShortDescZh - app.ShortDescEn = item.ShortDescEn - app.Website = item.Website - app.Document = item.Document - app.Github = item.Github - app.Type = item.Type - app.CrossVersionUpdate = item.CrossVersionUpdate - app.Required = item.GetRequired() + app.ShortDescZh = config.ShortDescZh + app.ShortDescEn = config.ShortDescEn + app.Website = config.Website + app.Document = config.Document + app.Github = config.Github + app.Type = config.Type + app.CrossVersionUpdate = config.CrossVersionUpdate + app.Required = config.GetRequired() app.Status = constant.AppNormal + app.LastModified = item.LastModified + app.ReadMe = item.ReadMe apps[key] = app } return apps diff --git a/backend/configs/system.go b/backend/configs/system.go index e144c4675..51a540593 100644 --- a/backend/configs/system.go +++ b/backend/configs/system.go @@ -1,23 +1,24 @@ package configs type System struct { - Port string `mapstructure:"port"` - SSL string `mapstructure:"ssl"` - DbFile string `mapstructure:"db_file"` - DbPath string `mapstructure:"db_path"` - LogPath string `mapstructure:"log_path"` - DataDir string `mapstructure:"data_dir"` - TmpDir string `mapstructure:"tmp_dir"` - Cache string `mapstructure:"cache"` - Backup string `mapstructure:"backup"` - EncryptKey string `mapstructure:"encrypt_key"` - BaseDir string `mapstructure:"base_dir"` - Mode string `mapstructure:"mode"` - RepoUrl string `mapstructure:"repo_url"` - Version string `mapstructure:"version"` - Username string `mapstructure:"username"` - Password string `mapstructure:"password"` - Entrance string `mapstructure:"entrance"` - IsDemo bool `mapstructure:"is_demo"` + Port string `mapstructure:"port"` + SSL string `mapstructure:"ssl"` + DbFile string `mapstructure:"db_file"` + DbPath string `mapstructure:"db_path"` + LogPath string `mapstructure:"log_path"` + DataDir string `mapstructure:"data_dir"` + TmpDir string `mapstructure:"tmp_dir"` + Cache string `mapstructure:"cache"` + Backup string `mapstructure:"backup"` + EncryptKey string `mapstructure:"encrypt_key"` + BaseDir string `mapstructure:"base_dir"` + Mode string `mapstructure:"mode"` + RepoUrl string `mapstructure:"repo_url"` + Version string `mapstructure:"version"` + Username string `mapstructure:"username"` + Password string `mapstructure:"password"` + Entrance string `mapstructure:"entrance"` + IsDemo bool `mapstructure:"is_demo"` + AppRepo string `mapstructure:"app_repo"` ChangeUserInfo bool `mapstructure:"change_user_info"` } diff --git a/backend/constant/app.go b/backend/constant/app.go index 9ef2e59d9..5011bec85 100644 --- a/backend/constant/app.go +++ b/backend/constant/app.go @@ -1,12 +1,14 @@ package constant const ( - Running = "Running" - UnHealthy = "UnHealthy" - Error = "Error" - Stopped = "Stopped" - Installing = "Installing" - Syncing = "Syncing" + Running = "Running" + UnHealthy = "UnHealthy" + Error = "Error" + Stopped = "Stopped" + Installing = "Installing" + Syncing = "Syncing" + DownloadErr = "DownloadErr" + DirNotFound = "DirNotFound" ContainerPrefix = "1Panel-" diff --git a/backend/constant/errs.go b/backend/constant/errs.go index 142cc78a5..f9f5fe23a 100644 --- a/backend/constant/errs.go +++ b/backend/constant/errs.go @@ -30,23 +30,16 @@ var ( ErrInvalidParams = errors.New("ErrInvalidParams") ErrTokenParse = errors.New("ErrTokenParse") - - ErrPageGenerate = errors.New("generate page info failed") - ErrRepoNotValid = "ErrRepoNotValid" ) // api var ( ErrTypeInternalServer = "ErrInternalServer" ErrTypeInvalidParams = "ErrInvalidParams" - ErrTypeToken = "ErrToken" - ErrTypeTokenTimeOut = "ErrTokenTimeOut" ErrTypeNotLogin = "ErrNotLogin" ErrTypePasswordExpired = "ErrPasswordExpired" - ErrTypeNotSafety = "ErrNotSafety" ErrNameIsExist = "ErrNameIsExist" ErrDemoEnvironment = "ErrDemoEnvironment" - ErrInitUser = "ErrInitUser" ) // app @@ -55,20 +48,20 @@ var ( ErrAppLimit = "ErrAppLimit" ErrAppRequired = "ErrAppRequired" ErrFileCanNotRead = "ErrFileCanNotRead" - ErrFileToLarge = "ErrFileToLarge" ErrNotInstall = "ErrNotInstall" ErrPortInOtherApp = "ErrPortInOtherApp" ErrDbUserNotValid = "ErrDbUserNotValid" ErrUpdateBuWebsite = "ErrUpdateBuWebsite" Err1PanelNetworkFailed = "Err1PanelNetworkFailed" ErrCmdTimeout = "ErrCmdTimeout" + ErrFileParse = "ErrFileParse" + ErrInstallDirNotFound = "ErrInstallDirNotFound" ) // website var ( ErrDomainIsExist = "ErrDomainIsExist" ErrAliasIsExist = "ErrAliasIsExist" - ErrAppDelete = "ErrAppDelete" ErrGroupIsUsed = "ErrGroupIsUsed" ErrUsernameIsExist = "ErrUsernameIsExist" ErrUsernameIsNotExist = "ErrUsernameIsNotExist" diff --git a/backend/i18n/lang/en.yaml b/backend/i18n/lang/en.yaml index 145756022..1218304d6 100644 --- a/backend/i18n/lang/en.yaml +++ b/backend/i18n/lang/en.yaml @@ -29,6 +29,8 @@ ErrDbUserNotValid: "Stock database, username and password do not match！" ErrDockerComposeNotValid: "docker-compose file format error!" ErrUpdateBuWebsite: 'The application was updated successfully, but the modification of the website configuration file failed, please check the configuration!' Err1PanelNetworkFailed: 'Default container network creation failed! {{ .detail }}' +ErrFileParse: 'Application docker-compose file parsing failed!' +ErrInstallDirNotFound: 'installation directory does not exist' #file ErrFileCanNotRead: "File can not read" diff --git a/backend/i18n/lang/zh.yaml b/backend/i18n/lang/zh.yaml index 9c8956f87..734b817f9 100644 --- a/backend/i18n/lang/zh.yaml +++ b/backend/i18n/lang/zh.yaml @@ -29,6 +29,8 @@ ErrDbUserNotValid: "存量数据库，用户名密码不匹配！" ErrDockerComposeNotValid: "docker-compose 文件格式错误" ErrUpdateBuWebsite: '应用更新成功，但是网站配置文件修改失败，请检查配置！' Err1PanelNetworkFailed: '默认容器网络创建失败！{{ .detail }}' +ErrFileParse: '应用 docker-compose 文件解析失败!' +ErrInstallDirNotFound: '安装目录不存在' #file ErrFileCanNotRead: "此文件不支持预览" diff --git a/backend/init/migration/migrate.go b/backend/init/migration/migrate.go index 992795602..f61dfd91a 100644 --- a/backend/init/migration/migrate.go +++ b/backend/init/migration/migrate.go @@ -26,6 +26,8 @@ func Init() { migrations.UpdateTableHost, migrations.UpdateTableWebsite, migrations.AddEntranceAndSSL, + migrations.UpdateTableSetting, + migrations.UpdateTableAppDetail, }) if err := m.Migrate(); err != nil { global.LOG.Error(err) diff --git a/backend/init/migration/migrations/init.go b/backend/init/migration/migrations/init.go index e6f5702ba..0780aa585 100644 --- a/backend/init/migration/migrations/init.go +++ b/backend/init/migration/migrations/init.go @@ -312,3 +312,26 @@ var AddEntranceAndSSL = &gormigrate.Migration{ return tx.AutoMigrate(&model.Website{}) }, } + +var UpdateTableSetting = &gormigrate.Migration{ + ID: "20200511-update-table-setting", + Migrate: func(tx *gorm.DB) error { + if err := tx.AutoMigrate(&model.App{}); err != nil { + return err + } + if err := tx.Create(&model.Setting{Key: "AppStoreLastModified", Value: ""}).Error; err != nil { + return err + } + return nil + }, +} + +var UpdateTableAppDetail = &gormigrate.Migration{ + ID: "20200513-update-table-app-detail", + Migrate: func(tx *gorm.DB) error { + if err := tx.AutoMigrate(&model.AppDetail{}); err != nil { + return err + } + return nil + }, +} diff --git a/cmd/server/conf/app.yaml b/cmd/server/conf/app.yaml index 20865f3ae..0da0e5163 100644 --- a/cmd/server/conf/app.yaml +++ b/cmd/server/conf/app.yaml @@ -3,6 +3,7 @@ system: base_dir: /opt mode: dev repo_url: https://resource.fit2cloud.com/1panel/package + app_repo: https://apps-assets.fit2cloud.com is_demo: false port: 9999 username: admin diff --git a/frontend/src/api/interface/app.ts b/frontend/src/api/interface/app.ts index 1fb035872..6c01cd2a0 100644 --- a/frontend/src/api/interface/app.ts +++ b/frontend/src/api/interface/app.ts @@ -11,6 +11,7 @@ export namespace App { author: string; source: string; type: string; + status: string; } export interface AppDTO extends App { diff --git a/frontend/src/views/app-store/apps/index.vue b/frontend/src/views/app-store/apps/index.vue index c8a362a4a..8a975e031 100644 --- a/frontend/src/views/app-store/apps/index.vue +++ b/frontend/src/views/app-store/apps/index.vue @@ -84,6 +84,9 @@ {{ language == 'zh' ? tag.name : tag.key }} + + 已废弃 + diff --git a/frontend/src/views/app-store/detail/index.vue b/frontend/src/views/app-store/detail/index.vue index 437d68796..4d0743778 100644 --- a/frontend/src/views/app-store/detail/index.vue +++ b/frontend/src/views/app-store/detail/index.vue @@ -82,12 +82,8 @@ -
- +
+