From 17224b89203d1d62f601faa15fe1d34f29408b95 Mon Sep 17 00:00:00 2001 From: CityFun <31820853+zhengkunwang223@users.noreply.github.com> Date: Sat, 18 Oct 2025 22:49:48 +0800 Subject: [PATCH] feat: ai model support tensorRT LLM (#10688) --- agent/app/api/v2/entry.go | 5 +- agent/app/api/v2/tensorrt_llm.go | 68 +++ agent/app/dto/request/tensorrt_llm.go | 32 ++ agent/app/dto/response/tensorrt_llm.go | 16 + agent/app/model/tensorrt_llm.go | 13 + agent/app/repo/tensorrt_llm.go | 56 +++ agent/app/service/entry.go | 5 +- agent/app/service/mcp_server.go | 4 +- agent/app/service/tensorrt_llm.go | 305 ++++++++++++ agent/cmd/server/{mcp/mcp.go => ai/ai.go} | 5 +- agent/cmd/server/{mcp => ai}/compose.yml | 0 agent/cmd/server/ai/llm-compose.yml | 32 ++ agent/global/config.go | 1 + agent/init/dir/dir.go | 1 + agent/init/migration/migrate.go | 1 + agent/init/migration/migrations/init.go | 7 + agent/router/ro_ai.go | 6 + frontend/src/api/interface/ai.ts | 31 ++ frontend/src/api/modules/ai.ts | 20 + frontend/src/lang/modules/en.ts | 4 + frontend/src/lang/modules/es-es.ts | 4 + frontend/src/lang/modules/ja.ts | 4 + frontend/src/lang/modules/ko.ts | 4 + frontend/src/lang/modules/ms.ts | 4 + frontend/src/lang/modules/pt-br.ts | 4 + frontend/src/lang/modules/ru.ts | 4 + frontend/src/lang/modules/tr.ts | 4 + frontend/src/lang/modules/zh-Hant.ts | 4 + frontend/src/lang/modules/zh.ts | 4 + frontend/src/routers/modules/ai.ts | 15 +- frontend/src/views/ai/model/index.vue | 462 +----------------- .../views/ai/model/{ => ollama}/add/index.vue | 0 .../ai/model/{ => ollama}/conn/index.vue | 0 .../views/ai/model/{ => ollama}/del/index.vue | 0 .../ai/model/{ => ollama}/domain/index.vue | 0 frontend/src/views/ai/model/ollama/index.vue | 461 +++++++++++++++++ .../ai/model/{ => ollama}/terminal/index.vue | 0 .../src/views/ai/model/tensorrt/index.vue | 224 +++++++++ .../views/ai/model/tensorrt/operate/index.vue | 178 +++++++ 39 files changed, 1525 insertions(+), 463 deletions(-) create mode 100644 agent/app/api/v2/tensorrt_llm.go create mode 100644 agent/app/dto/request/tensorrt_llm.go create mode 100644 agent/app/dto/response/tensorrt_llm.go create mode 100644 agent/app/model/tensorrt_llm.go create mode 100644 agent/app/repo/tensorrt_llm.go create mode 100644 agent/app/service/tensorrt_llm.go rename agent/cmd/server/{mcp/mcp.go => ai/ai.go} (50%) rename agent/cmd/server/{mcp => ai}/compose.yml (100%) create mode 100644 agent/cmd/server/ai/llm-compose.yml rename frontend/src/views/ai/model/{ => ollama}/add/index.vue (100%) rename frontend/src/views/ai/model/{ => ollama}/conn/index.vue (100%) rename frontend/src/views/ai/model/{ => ollama}/del/index.vue (100%) rename frontend/src/views/ai/model/{ => ollama}/domain/index.vue (100%) create mode 100644 frontend/src/views/ai/model/ollama/index.vue rename frontend/src/views/ai/model/{ => ollama}/terminal/index.vue (100%) create mode 100644 frontend/src/views/ai/model/tensorrt/index.vue create mode 100644 frontend/src/views/ai/model/tensorrt/operate/index.vue diff --git a/agent/app/api/v2/entry.go b/agent/app/api/v2/entry.go index 0f0258799..aa403c95c 100644 --- a/agent/app/api/v2/entry.go +++ b/agent/app/api/v2/entry.go @@ -17,8 +17,9 @@ var ( appInstallService = service.NewIAppInstalledService() appIgnoreUpgradeService = service.NewIAppIgnoreUpgradeService() - aiToolService = service.NewIAIToolService() - mcpServerService = service.NewIMcpServerService() + aiToolService = service.NewIAIToolService() + mcpServerService = service.NewIMcpServerService() + tensorrtLLMService = service.NewITensorRTLLMService() containerService = service.NewIContainerService() composeTemplateService = service.NewIComposeTemplateService() diff --git a/agent/app/api/v2/tensorrt_llm.go b/agent/app/api/v2/tensorrt_llm.go new file mode 100644 index 000000000..1af51004b --- /dev/null +++ b/agent/app/api/v2/tensorrt_llm.go @@ -0,0 +1,68 @@ +package v2 + +import ( + "github.com/1Panel-dev/1Panel/agent/app/api/v2/helper" + "github.com/1Panel-dev/1Panel/agent/app/dto/request" + "github.com/gin-gonic/gin" +) + +func (b *BaseApi) PageTensorRTLLMs(c *gin.Context) { + var req request.TensorRTLLMSearch + if err := helper.CheckBindAndValidate(&req, c); err != nil { + return + } + list := tensorrtLLMService.Page(req) + helper.SuccessWithData(c, list) +} + +func (b *BaseApi) CreateTensorRTLLM(c *gin.Context) { + var req request.TensorRTLLMCreate + if err := helper.CheckBindAndValidate(&req, c); err != nil { + return + } + err := tensorrtLLMService.Create(req) + if err != nil { + helper.InternalServer(c, err) + return + } + helper.Success(c) +} + +func (b *BaseApi) UpdateTensorRTLLM(c *gin.Context) { + var req request.TensorRTLLMUpdate + if err := helper.CheckBindAndValidate(&req, c); err != nil { + return + } + err := tensorrtLLMService.Update(req) + if err != nil { + helper.InternalServer(c, err) + return + } + helper.Success(c) +} + +func (b *BaseApi) DeleteTensorRTLLM(c *gin.Context) { + var req request.TensorRTLLMDelete + if err := helper.CheckBindAndValidate(&req, c); err != nil { + return + } + err := tensorrtLLMService.Delete(req.ID) + if err != nil { + helper.InternalServer(c, err) + return + } + helper.Success(c) +} + +func (b *BaseApi) OperateTensorRTLLM(c *gin.Context) { + var req request.TensorRTLLMOperate + if err := helper.CheckBindAndValidate(&req, c); err != nil { + return + } + err := tensorrtLLMService.Operate(req) + if err != nil { + helper.InternalServer(c, err) + return + } + helper.Success(c) +} diff --git a/agent/app/dto/request/tensorrt_llm.go b/agent/app/dto/request/tensorrt_llm.go new file mode 100644 index 000000000..d69f075d9 --- /dev/null +++ b/agent/app/dto/request/tensorrt_llm.go @@ -0,0 +1,32 @@ +package request + +import "github.com/1Panel-dev/1Panel/agent/app/dto" + +type TensorRTLLMSearch struct { + dto.PageInfo + Name string `json:"name"` +} + +type TensorRTLLMCreate struct { + Name string `json:"name" validate:"required"` + ContainerName string `json:"containerName"` + Port int `json:"port" validate:"required"` + Version string `json:"version" validate:"required"` + ModelDir string `json:"modelDir" validate:"required"` + Model string `json:"model" validate:"required"` + HostIP string `json:"hostIP"` +} + +type TensorRTLLMUpdate struct { + ID uint `json:"id" validate:"required"` + TensorRTLLMCreate +} + +type TensorRTLLMDelete struct { + ID uint `json:"id" validate:"required"` +} + +type TensorRTLLMOperate struct { + ID uint `json:"id" validate:"required"` + Operate string `json:"operate" validate:"required"` +} diff --git a/agent/app/dto/response/tensorrt_llm.go b/agent/app/dto/response/tensorrt_llm.go new file mode 100644 index 000000000..18a37f606 --- /dev/null +++ b/agent/app/dto/response/tensorrt_llm.go @@ -0,0 +1,16 @@ +package response + +import "github.com/1Panel-dev/1Panel/agent/app/model" + +type TensorRTLLMsRes struct { + Items []TensorRTLLMDTO `json:"items"` + Total int64 `json:"total"` +} + +type TensorRTLLMDTO struct { + model.TensorRTLLM + Version string `json:"version"` + Model string `json:"model"` + Dir string `json:"dir"` + ModelDir string `json:"modelDir"` +} diff --git a/agent/app/model/tensorrt_llm.go b/agent/app/model/tensorrt_llm.go new file mode 100644 index 000000000..8ffc3f8cc --- /dev/null +++ b/agent/app/model/tensorrt_llm.go @@ -0,0 +1,13 @@ +package model + +type TensorRTLLM struct { + BaseModel + Name string `json:"name"` + DockerCompose string `json:"dockerCompose"` + ContainerName string `json:"containerName"` + Message string `json:"message"` + Port int `json:"port"` + Status string `json:"status"` + Env string `json:"env"` + TaskID string `json:"taskID"` +} diff --git a/agent/app/repo/tensorrt_llm.go b/agent/app/repo/tensorrt_llm.go new file mode 100644 index 000000000..6df2b43d2 --- /dev/null +++ b/agent/app/repo/tensorrt_llm.go @@ -0,0 +1,56 @@ +package repo + +import "github.com/1Panel-dev/1Panel/agent/app/model" + +type TensorRTLLMRepo struct { +} + +type ITensorRTLLMRepo interface { + Page(page, size int, opts ...DBOption) (int64, []model.TensorRTLLM, error) + GetFirst(opts ...DBOption) (*model.TensorRTLLM, error) + Create(tensorrtLLM *model.TensorRTLLM) error + Save(tensorrtLLM *model.TensorRTLLM) error + DeleteBy(opts ...DBOption) error + List(opts ...DBOption) ([]model.TensorRTLLM, error) +} + +func NewITensorRTLLMRepo() ITensorRTLLMRepo { + return &TensorRTLLMRepo{} +} + +func (t TensorRTLLMRepo) Page(page, size int, opts ...DBOption) (int64, []model.TensorRTLLM, error) { + var servers []model.TensorRTLLM + db := getDb(opts...).Model(&model.TensorRTLLM{}) + count := int64(0) + db = db.Count(&count) + err := db.Limit(size).Offset(size * (page - 1)).Find(&servers).Error + return count, servers, err +} + +func (t TensorRTLLMRepo) GetFirst(opts ...DBOption) (*model.TensorRTLLM, error) { + var tensorrtLLM model.TensorRTLLM + if err := getDb(opts...).First(&tensorrtLLM).Error; err != nil { + return nil, err + } + return &tensorrtLLM, nil +} + +func (t TensorRTLLMRepo) List(opts ...DBOption) ([]model.TensorRTLLM, error) { + var tensorrtLLMs []model.TensorRTLLM + if err := getDb(opts...).Find(&tensorrtLLMs).Error; err != nil { + return nil, err + } + return tensorrtLLMs, nil +} + +func (t TensorRTLLMRepo) Create(tensorrtLLM *model.TensorRTLLM) error { + return getDb().Create(tensorrtLLM).Error +} + +func (t TensorRTLLMRepo) Save(tensorrtLLM *model.TensorRTLLM) error { + return getDb().Save(tensorrtLLM).Error +} + +func (t TensorRTLLMRepo) DeleteBy(opts ...DBOption) error { + return getDb(opts...).Delete(&model.TensorRTLLM{}).Error +} diff --git a/agent/app/service/entry.go b/agent/app/service/entry.go index b9deff6a5..90213e707 100644 --- a/agent/app/service/entry.go +++ b/agent/app/service/entry.go @@ -12,8 +12,9 @@ var ( appInstallResourceRepo = repo.NewIAppInstallResourceRpo() appIgnoreUpgradeRepo = repo.NewIAppIgnoreUpgradeRepo() - aiRepo = repo.NewIAiRepo() - mcpServerRepo = repo.NewIMcpServerRepo() + aiRepo = repo.NewIAiRepo() + mcpServerRepo = repo.NewIMcpServerRepo() + tensorrtLLMRepo = repo.NewITensorRTLLMRepo() mysqlRepo = repo.NewIMysqlRepo() postgresqlRepo = repo.NewIPostgresqlRepo() diff --git a/agent/app/service/mcp_server.go b/agent/app/service/mcp_server.go index 3c51e7a95..efb6cf1a4 100644 --- a/agent/app/service/mcp_server.go +++ b/agent/app/service/mcp_server.go @@ -10,7 +10,7 @@ import ( "github.com/1Panel-dev/1Panel/agent/app/model" "github.com/1Panel-dev/1Panel/agent/app/repo" "github.com/1Panel-dev/1Panel/agent/buserr" - "github.com/1Panel-dev/1Panel/agent/cmd/server/mcp" + "github.com/1Panel-dev/1Panel/agent/cmd/server/ai" "github.com/1Panel-dev/1Panel/agent/cmd/server/nginx_conf" "github.com/1Panel-dev/1Panel/agent/constant" "github.com/1Panel-dev/1Panel/agent/global" @@ -565,7 +565,7 @@ func handleEnv(mcpServer *model.McpServer) gotenv.Env { func handleCreateParams(mcpServer *model.McpServer, environments []request.Environment, volumes []request.Volume) error { var composeContent []byte if mcpServer.ID == 0 { - composeContent = mcp.DefaultMcpCompose + composeContent = ai.DefaultMcpCompose } else { composeContent = []byte(mcpServer.DockerCompose) } diff --git a/agent/app/service/tensorrt_llm.go b/agent/app/service/tensorrt_llm.go new file mode 100644 index 000000000..a73483f23 --- /dev/null +++ b/agent/app/service/tensorrt_llm.go @@ -0,0 +1,305 @@ +package service + +import ( + "github.com/1Panel-dev/1Panel/agent/app/dto/request" + "github.com/1Panel-dev/1Panel/agent/app/dto/response" + "github.com/1Panel-dev/1Panel/agent/app/model" + "github.com/1Panel-dev/1Panel/agent/app/repo" + "github.com/1Panel-dev/1Panel/agent/buserr" + "github.com/1Panel-dev/1Panel/agent/cmd/server/ai" + "github.com/1Panel-dev/1Panel/agent/constant" + "github.com/1Panel-dev/1Panel/agent/global" + "github.com/1Panel-dev/1Panel/agent/utils/compose" + "github.com/1Panel-dev/1Panel/agent/utils/docker" + "github.com/1Panel-dev/1Panel/agent/utils/files" + "github.com/subosito/gotenv" + "gopkg.in/yaml.v3" + "path" + "strconv" +) + +type TensorRTLLMService struct{} + +type ITensorRTLLMService interface { + Page(req request.TensorRTLLMSearch) response.TensorRTLLMsRes + Create(create request.TensorRTLLMCreate) error + Update(req request.TensorRTLLMUpdate) error + Delete(id uint) error + Operate(req request.TensorRTLLMOperate) error +} + +func NewITensorRTLLMService() ITensorRTLLMService { + return &TensorRTLLMService{} +} + +func (t TensorRTLLMService) Page(req request.TensorRTLLMSearch) response.TensorRTLLMsRes { + var ( + res response.TensorRTLLMsRes + items []response.TensorRTLLMDTO + ) + + total, data, _ := tensorrtLLMRepo.Page(req.PageInfo.Page, req.PageInfo.PageSize) + for _, item := range data { + _ = syncTensorRTLLMContainerStatus(&item) + serverDTO := response.TensorRTLLMDTO{ + TensorRTLLM: item, + } + env, _ := gotenv.Unmarshal(item.Env) + serverDTO.Version = env["VERSION"] + serverDTO.Model = env["MODEL_NAME"] + serverDTO.ModelDir = env["MODEL_PATH"] + serverDTO.Dir = path.Join(global.Dir.TensorRTLLMDir, item.Name) + items = append(items, serverDTO) + } + res.Total = total + res.Items = items + return res +} + +func handleLLMParams(llm *model.TensorRTLLM) error { + var composeContent []byte + if llm.ID == 0 { + composeContent = ai.DefaultTensorrtLLMCompose + } else { + composeContent = []byte(llm.DockerCompose) + } + composeMap := make(map[string]interface{}) + if err := yaml.Unmarshal(composeContent, &composeMap); err != nil { + return err + } + services, serviceValid := composeMap["services"].(map[string]interface{}) + if !serviceValid { + return buserr.New("ErrFileParse") + } + serviceName := "" + serviceValue := make(map[string]interface{}) + + if llm.ID > 0 { + serviceName = llm.Name + serviceValue = services[serviceName].(map[string]interface{}) + } else { + for name, service := range services { + serviceName = name + serviceValue = service.(map[string]interface{}) + break + } + delete(services, serviceName) + } + + services[llm.Name] = serviceValue + composeByte, err := yaml.Marshal(composeMap) + if err != nil { + return err + } + llm.DockerCompose = string(composeByte) + return nil +} + +func handleLLMEnv(llm *model.TensorRTLLM, create request.TensorRTLLMCreate) gotenv.Env { + env := make(gotenv.Env) + env["CONTAINER_NAME"] = create.ContainerName + env["PANEL_APP_PORT_HTTP"] = strconv.Itoa(llm.Port) + env["MODEL_PATH"] = create.ModelDir + env["MODEL_NAME"] = create.Model + env["VERSION"] = create.Version + if create.HostIP != "" { + env["HOST_IP"] = create.HostIP + } else { + env["HOST_IP"] = "" + } + envStr, _ := gotenv.Marshal(env) + llm.Env = envStr + return env +} + +func (t TensorRTLLMService) Create(create request.TensorRTLLMCreate) error { + servers, _ := tensorrtLLMRepo.List() + for _, server := range servers { + if server.Port == create.Port { + return buserr.New("ErrPortInUsed") + } + if server.ContainerName == create.ContainerName { + return buserr.New("ErrContainerName") + } + if server.Name == create.Name { + return buserr.New("ErrNameIsExist") + } + } + if err := checkPortExist(create.Port); err != nil { + return err + } + if err := checkContainerName(create.ContainerName); err != nil { + return err + } + + tensorrtLLMDir := path.Join(global.Dir.TensorRTLLMDir, create.Name) + filesOP := files.NewFileOp() + if !filesOP.Stat(tensorrtLLMDir) { + _ = filesOP.CreateDir(tensorrtLLMDir, 0644) + } + tensorrtLLM := &model.TensorRTLLM{ + Name: create.Name, + ContainerName: create.ContainerName, + Port: create.Port, + Status: constant.StatusStarting, + } + + if err := handleLLMParams(tensorrtLLM); err != nil { + return err + } + env := handleLLMEnv(tensorrtLLM, create) + llmDir := path.Join(global.Dir.TensorRTLLMDir, create.Name) + envPath := path.Join(llmDir, ".env") + if err := gotenv.Write(env, envPath); err != nil { + return err + } + dockerComposePath := path.Join(llmDir, "docker-compose.yml") + if err := filesOP.SaveFile(dockerComposePath, tensorrtLLM.DockerCompose, 0644); err != nil { + return err + } + tensorrtLLM.Status = constant.StatusStarting + + if err := tensorrtLLMRepo.Create(tensorrtLLM); err != nil { + return err + } + go startTensorRTLLM(tensorrtLLM) + return nil +} + +func (t TensorRTLLMService) Update(req request.TensorRTLLMUpdate) error { + tensorrtLLM, err := tensorrtLLMRepo.GetFirst(repo.WithByID(req.ID)) + if err != nil { + return err + } + if tensorrtLLM.Port != req.Port { + if err := checkPortExist(req.Port); err != nil { + return err + } + } + if tensorrtLLM.ContainerName != req.ContainerName { + if err := checkContainerName(req.ContainerName); err != nil { + return err + } + } + + tensorrtLLM.ContainerName = req.ContainerName + tensorrtLLM.Port = req.Port + if err := handleLLMParams(tensorrtLLM); err != nil { + return err + } + + newEnv, err := gotenv.Unmarshal(tensorrtLLM.Env) + if err != nil { + return err + } + newEnv["CONTAINER_NAME"] = req.ContainerName + newEnv["PANEL_APP_PORT_HTTP"] = strconv.Itoa(tensorrtLLM.Port) + newEnv["MODEL_PATH"] = req.ModelDir + newEnv["MODEL_NAME"] = req.Model + newEnv["VERSION"] = req.Version + if req.HostIP != "" { + newEnv["HOST_IP"] = req.HostIP + } else { + newEnv["HOST_IP"] = "" + } + envStr, _ := gotenv.Marshal(newEnv) + tensorrtLLM.Env = envStr + llmDir := path.Join(global.Dir.TensorRTLLMDir, tensorrtLLM.Name) + envPath := path.Join(llmDir, ".env") + if err := gotenv.Write(newEnv, envPath); err != nil { + return err + } + tensorrtLLM.Status = constant.StatusStarting + if err := tensorrtLLMRepo.Save(tensorrtLLM); err != nil { + return err + } + go startTensorRTLLM(tensorrtLLM) + return nil +} + +func (t TensorRTLLMService) Delete(id uint) error { + tensorrtLLM, err := tensorrtLLMRepo.GetFirst(repo.WithByID(id)) + if err != nil { + return err + } + composePath := path.Join(global.Dir.TensorRTLLMDir, tensorrtLLM.Name, "docker-compose.yml") + _, _ = compose.Down(composePath) + _ = files.NewFileOp().DeleteDir(path.Join(global.Dir.TensorRTLLMDir, tensorrtLLM.Name)) + return tensorrtLLMRepo.DeleteBy(repo.WithByID(id)) +} + +func (t TensorRTLLMService) Operate(req request.TensorRTLLMOperate) error { + tensorrtLLM, err := tensorrtLLMRepo.GetFirst(repo.WithByID(req.ID)) + if err != nil { + return err + } + composePath := path.Join(global.Dir.TensorRTLLMDir, tensorrtLLM.Name, "docker-compose.yml") + var out string + switch req.Operate { + case "start": + out, err = compose.Up(composePath) + tensorrtLLM.Status = constant.StatusRunning + case "stop": + out, err = compose.Down(composePath) + tensorrtLLM.Status = constant.StatusStopped + case "restart": + out, err = compose.Restart(composePath) + tensorrtLLM.Status = constant.StatusRunning + } + if err != nil { + tensorrtLLM.Status = constant.StatusError + tensorrtLLM.Message = out + } + return tensorrtLLMRepo.Save(tensorrtLLM) +} + +func startTensorRTLLM(tensorrtLLM *model.TensorRTLLM) { + composePath := path.Join(global.Dir.TensorRTLLMDir, tensorrtLLM.Name, "docker-compose.yml") + if tensorrtLLM.Status != constant.StatusNormal { + _, _ = compose.Down(composePath) + } + if out, err := compose.Up(composePath); err != nil { + tensorrtLLM.Status = constant.StatusError + tensorrtLLM.Message = out + } else { + tensorrtLLM.Status = constant.StatusRunning + tensorrtLLM.Message = "" + } + _ = syncTensorRTLLMContainerStatus(tensorrtLLM) +} + +func syncTensorRTLLMContainerStatus(tensorrtLLM *model.TensorRTLLM) error { + containerNames := []string{tensorrtLLM.ContainerName} + cli, err := docker.NewClient() + if err != nil { + return err + } + defer cli.Close() + containers, err := cli.ListContainersByName(containerNames) + if err != nil { + return err + } + if len(containers) == 0 { + if tensorrtLLM.Status == constant.StatusStarting { + return nil + } + tensorrtLLM.Status = constant.StatusStopped + return tensorrtLLMRepo.Save(tensorrtLLM) + } + container := containers[0] + switch container.State { + case "exited": + tensorrtLLM.Status = constant.StatusError + case "running": + tensorrtLLM.Status = constant.StatusRunning + case "paused": + tensorrtLLM.Status = constant.StatusStopped + case "restarting": + tensorrtLLM.Status = constant.StatusRestarting + default: + if tensorrtLLM.Status != constant.StatusStarting { + tensorrtLLM.Status = constant.StatusStopped + } + } + return tensorrtLLMRepo.Save(tensorrtLLM) +} diff --git a/agent/cmd/server/mcp/mcp.go b/agent/cmd/server/ai/ai.go similarity index 50% rename from agent/cmd/server/mcp/mcp.go rename to agent/cmd/server/ai/ai.go index 0e87d1335..8888eee8c 100644 --- a/agent/cmd/server/mcp/mcp.go +++ b/agent/cmd/server/ai/ai.go @@ -1,4 +1,4 @@ -package mcp +package ai import ( _ "embed" @@ -6,3 +6,6 @@ import ( //go:embed compose.yml var DefaultMcpCompose []byte + +//go:embed llm-compose.yml +var DefaultTensorrtLLMCompose []byte diff --git a/agent/cmd/server/mcp/compose.yml b/agent/cmd/server/ai/compose.yml similarity index 100% rename from agent/cmd/server/mcp/compose.yml rename to agent/cmd/server/ai/compose.yml diff --git a/agent/cmd/server/ai/llm-compose.yml b/agent/cmd/server/ai/llm-compose.yml new file mode 100644 index 000000000..d4423ccdb --- /dev/null +++ b/agent/cmd/server/ai/llm-compose.yml @@ -0,0 +1,32 @@ +services: + tensorrt-llm: + image: nvcr.io/nvidia/tensorrt-llm/release:${VERSION} + container_name: ${CONTAINER_NAME} + restart: always + runtime: nvidia + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + networks: + - 1panel-network + volumes: + - ${MODEL_PATH}:/models + ports: + - ${PANEL_APP_PORT_HTTP}:8000 + ipc: host + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65535 + hard: 65535 + stack: 67108864 + command: bach -c "trtllm-serve /models/${MODEL_NAME} --host 0.0.0.0 --port 8000" +networks: + 1panel-network: + external: true \ No newline at end of file diff --git a/agent/global/config.go b/agent/global/config.go index 90f2253d6..20d1dad62 100644 --- a/agent/global/config.go +++ b/agent/global/config.go @@ -46,6 +46,7 @@ type SystemDir struct { SSLLogDir string McpDir string ConvertLogDir string + TensorRTLLMDir string } type LogConfig struct { diff --git a/agent/init/dir/dir.go b/agent/init/dir/dir.go index 1d26cb47b..fe2dc51ba 100644 --- a/agent/init/dir/dir.go +++ b/agent/init/dir/dir.go @@ -33,4 +33,5 @@ func Init() { global.Dir.SSLLogDir, _ = fileOp.CreateDirWithPath(true, path.Join(baseDir, "1panel/log/ssl")) global.Dir.McpDir, _ = fileOp.CreateDirWithPath(true, path.Join(baseDir, "1panel/mcp")) global.Dir.ConvertLogDir, _ = fileOp.CreateDirWithPath(true, path.Join(baseDir, "1panel/log/convert")) + global.Dir.TensorRTLLMDir, _ = fileOp.CreateDirWithPath(true, path.Join(baseDir, "1panel/ai/tensorrt_llm")) } diff --git a/agent/init/migration/migrate.go b/agent/init/migration/migrate.go index df06a4cd8..74bdd743a 100644 --- a/agent/init/migration/migrate.go +++ b/agent/init/migration/migrate.go @@ -46,6 +46,7 @@ func InitAgentDB() { migrations.AddTimeoutForClam, migrations.UpdateCronjobSpec, migrations.UpdateWebsiteSSLAddColumn, + migrations.AddTensorRTLLMModel, }) if err := m.Migrate(); err != nil { global.LOG.Error(err) diff --git a/agent/init/migration/migrations/init.go b/agent/init/migration/migrations/init.go index 46ca9a22d..7c8461790 100644 --- a/agent/init/migration/migrations/init.go +++ b/agent/init/migration/migrations/init.go @@ -627,3 +627,10 @@ var UpdateWebsiteSSLAddColumn = &gormigrate.Migration{ return nil }, } + +var AddTensorRTLLMModel = &gormigrate.Migration{ + ID: "20251018-add-tensorrt-llm-model", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&model.TensorRTLLM{}) + }, +} diff --git a/agent/router/ro_ai.go b/agent/router/ro_ai.go index 1c590c556..47246c6aa 100644 --- a/agent/router/ro_ai.go +++ b/agent/router/ro_ai.go @@ -33,5 +33,11 @@ func (a *AIToolsRouter) InitRouter(Router *gin.RouterGroup) { aiToolsRouter.POST("/mcp/domain/bind", baseApi.BindMcpDomain) aiToolsRouter.GET("/mcp/domain/get", baseApi.GetMcpBindDomain) aiToolsRouter.POST("/mcp/domain/update", baseApi.UpdateMcpBindDomain) + + aiToolsRouter.POST("/tensorrt/search", baseApi.PageTensorRTLLMs) + aiToolsRouter.POST("/tensorrt/create", baseApi.CreateTensorRTLLM) + aiToolsRouter.POST("/tensorrt/update", baseApi.UpdateTensorRTLLM) + aiToolsRouter.POST("/tensorrt/delete", baseApi.DeleteTensorRTLLM) + aiToolsRouter.POST("/tensorrt/operate", baseApi.OperateTensorRTLLM) } } diff --git a/frontend/src/api/interface/ai.ts b/frontend/src/api/interface/ai.ts index 4ff02dae9..dec849479 100644 --- a/frontend/src/api/interface/ai.ts +++ b/frontend/src/api/interface/ai.ts @@ -183,4 +183,35 @@ export namespace AI { containerName: string; environments: Environment[]; } + + export interface TensorRTLLM { + id?: number; + name: string; + containerName: string; + port: number; + version: string; + modelDir: string; + model: string; + hostIP: string; + status?: string; + message?: string; + createdAt?: string; + } + + export interface TensorRTLLMDTO extends TensorRTLLM { + dir?: string; + } + + export interface TensorRTLLMSearch extends ReqPage { + name: string; + } + + export interface TensorRTLLMDelete { + id: number; + } + + export interface TensorRTLLMOperate { + id: number; + operate: string; + } } diff --git a/frontend/src/api/modules/ai.ts b/frontend/src/api/modules/ai.ts index 11c6eca2f..fd2bfa429 100644 --- a/frontend/src/api/modules/ai.ts +++ b/frontend/src/api/modules/ai.ts @@ -71,3 +71,23 @@ export const getMcpDomain = () => { export const updateMcpDomain = (req: AI.McpBindDomainUpdate) => { return http.post(`/ai/mcp/domain/update`, req); }; + +export const pageTensorRTLLM = (req: AI.TensorRTLLMSearch) => { + return http.post>(`/ai/tensorrt/search`, req); +}; + +export const createTensorRTLLM = (req: AI.TensorRTLLM) => { + return http.post(`/ai/tensorrt/create`, req); +}; + +export const updateTensorRTLLM = (req: AI.TensorRTLLM) => { + return http.post(`/ai/tensorrt/update`, req); +}; + +export const deleteTensorRTLLM = (req: AI.TensorRTLLMDelete) => { + return http.post(`/ai/tensorrt/delete`, req); +}; + +export const operateTensorRTLLM = (req: AI.TensorRTLLMOperate) => { + return http.post(`/ai/tensorrt/operate`, req); +}; diff --git a/frontend/src/lang/modules/en.ts b/frontend/src/lang/modules/en.ts index 73afdc227..0c34238a1 100644 --- a/frontend/src/lang/modules/en.ts +++ b/frontend/src/lang/modules/en.ts @@ -724,6 +724,10 @@ const message = { npxHelper: 'Suitable for mcp started with npx or binary', uvxHelper: 'Suitable for mcp started with uvx', }, + tensorRT: { + llm: 'TensorRT LLM', + modelDir: 'Model Directory', + }, }, container: { create: 'Create', diff --git a/frontend/src/lang/modules/es-es.ts b/frontend/src/lang/modules/es-es.ts index aa2f30a41..6debc618c 100644 --- a/frontend/src/lang/modules/es-es.ts +++ b/frontend/src/lang/modules/es-es.ts @@ -725,6 +725,10 @@ const message = { npxHelper: 'Adecuado para mcp iniciado con npx o binario', uvxHelper: 'Adecuado para mcp iniciado con uvx', }, + tensorRT: { + llm: 'TensorRT LLM', + modelDir: 'Directorio del Modelo', + }, }, container: { create: 'Crear', diff --git a/frontend/src/lang/modules/ja.ts b/frontend/src/lang/modules/ja.ts index db9205fa8..ce60da888 100644 --- a/frontend/src/lang/modules/ja.ts +++ b/frontend/src/lang/modules/ja.ts @@ -712,6 +712,10 @@ const message = { npxHelper: 'npx またはバイナリで起動する mcp に適しています', uvxHelper: 'uvx で起動する mcp に適しています', }, + tensorRT: { + llm: 'TensorRT LLM', + modelDir: 'モデルディレクトリ', + }, }, container: { create: 'コンテナを作成します', diff --git a/frontend/src/lang/modules/ko.ts b/frontend/src/lang/modules/ko.ts index 541a39d62..697e4bb93 100644 --- a/frontend/src/lang/modules/ko.ts +++ b/frontend/src/lang/modules/ko.ts @@ -708,6 +708,10 @@ const message = { npxHelper: 'npx 또는 바이너리로 시작하는 mcp에 적합', uvxHelper: 'uvx로 시작하는 mcp에 적합', }, + tensorRT: { + llm: 'TensorRT LLM', + modelDir: '모델 디렉토리', + }, }, container: { create: '컨테이너 만들기', diff --git a/frontend/src/lang/modules/ms.ts b/frontend/src/lang/modules/ms.ts index 466ccf7ac..4159733d1 100644 --- a/frontend/src/lang/modules/ms.ts +++ b/frontend/src/lang/modules/ms.ts @@ -725,6 +725,10 @@ const message = { npxHelper: 'Sesuai untuk mcp yang dimulakan dengan npx atau binari', uvxHelper: 'Sesuai untuk mcp yang dimulakan dengan uvx', }, + tensorRT: { + llm: 'TensorRT LLM', + modelDir: 'Direktori Model', + }, }, container: { create: 'Cipta kontena', diff --git a/frontend/src/lang/modules/pt-br.ts b/frontend/src/lang/modules/pt-br.ts index 3c3cbcb2c..fa7726b54 100644 --- a/frontend/src/lang/modules/pt-br.ts +++ b/frontend/src/lang/modules/pt-br.ts @@ -721,6 +721,10 @@ const message = { npxHelper: 'Adequado para mcp iniciado com npx ou binário', uvxHelper: 'Adequado para mcp iniciado com uvx', }, + tensorRT: { + llm: 'TensorRT LLM', + modelDir: 'Diretório do Modelo', + }, }, container: { create: 'Criar contêiner', diff --git a/frontend/src/lang/modules/ru.ts b/frontend/src/lang/modules/ru.ts index 987bfc997..f3a90902d 100644 --- a/frontend/src/lang/modules/ru.ts +++ b/frontend/src/lang/modules/ru.ts @@ -719,6 +719,10 @@ const message = { npxHelper: 'Подходит для mcp, запущенного с помощью npx или бинарного файла', uvxHelper: 'Подходит для mcp, запущенного с помощью uvx', }, + tensorRT: { + llm: 'TensorRT LLM', + modelDir: 'Каталог модели', + }, }, container: { create: 'Создать контейнер', diff --git a/frontend/src/lang/modules/tr.ts b/frontend/src/lang/modules/tr.ts index e6924af23..b889a5f50 100644 --- a/frontend/src/lang/modules/tr.ts +++ b/frontend/src/lang/modules/tr.ts @@ -733,6 +733,10 @@ const message = { npxHelper: 'npx veya ikili dosya ile başlatılan mcp için uygundur', uvxHelper: 'uvx ile başlatılan mcp için uygundur', }, + tensorRT: { + llm: 'TensorRT LLM', + modelDir: 'Model Dizini', + }, }, container: { create: 'Oluştur', diff --git a/frontend/src/lang/modules/zh-Hant.ts b/frontend/src/lang/modules/zh-Hant.ts index 242ba9cb4..bd35fd3df 100644 --- a/frontend/src/lang/modules/zh-Hant.ts +++ b/frontend/src/lang/modules/zh-Hant.ts @@ -698,6 +698,10 @@ const message = { npxHelper: '適合 npx 或者 二進制啟動的 mcp', uvxHelper: '適合 uvx 啟動的 mcp', }, + tensorRT: { + llm: 'TensorRT LLM', + modelDir: '模型目錄', + }, }, container: { create: '建立容器', diff --git a/frontend/src/lang/modules/zh.ts b/frontend/src/lang/modules/zh.ts index 1e9274cd5..ec1cde61d 100644 --- a/frontend/src/lang/modules/zh.ts +++ b/frontend/src/lang/modules/zh.ts @@ -697,6 +697,10 @@ const message = { npxHelper: '适合 npx 或者 二进制启动的 mcp', uvxHelper: '适合 uvx 启动的 mcp', }, + tensorRT: { + llm: 'TensorRT LLM', + modelDir: '模型目录', + }, }, container: { create: '创建容器', diff --git a/frontend/src/routers/modules/ai.ts b/frontend/src/routers/modules/ai.ts index c6dfd58ff..a84789340 100644 --- a/frontend/src/routers/modules/ai.ts +++ b/frontend/src/routers/modules/ai.ts @@ -12,9 +12,9 @@ const databaseRouter = { }, children: [ { - path: '/ai/model', + path: '/ai/model/ollama', name: 'OllamaModel', - component: () => import('@/views/ai/model/index.vue'), + component: () => import('@/views/ai/model/ollama/index.vue'), meta: { icon: 'p-moxing-menu', title: 'aiTools.model.model', @@ -41,6 +41,17 @@ const databaseRouter = { requiresAuth: true, }, }, + { + path: '/ai/model/tensorrt', + hidden: true, + name: 'TensorRTLLm', + component: () => import('@/views/ai/model/tensorrt/index.vue'), + meta: { + title: 'aiTools.tensorRT.llm', + activeMenu: '/ai/model/ollama', + requiresAuth: true, + }, + }, ], }; diff --git a/frontend/src/views/ai/model/index.vue b/frontend/src/views/ai/model/index.vue index b568d2f38..53b3a737f 100644 --- a/frontend/src/views/ai/model/index.vue +++ b/frontend/src/views/ai/model/index.vue @@ -1,467 +1,21 @@ - - diff --git a/frontend/src/views/ai/model/add/index.vue b/frontend/src/views/ai/model/ollama/add/index.vue similarity index 100% rename from frontend/src/views/ai/model/add/index.vue rename to frontend/src/views/ai/model/ollama/add/index.vue diff --git a/frontend/src/views/ai/model/conn/index.vue b/frontend/src/views/ai/model/ollama/conn/index.vue similarity index 100% rename from frontend/src/views/ai/model/conn/index.vue rename to frontend/src/views/ai/model/ollama/conn/index.vue diff --git a/frontend/src/views/ai/model/del/index.vue b/frontend/src/views/ai/model/ollama/del/index.vue similarity index 100% rename from frontend/src/views/ai/model/del/index.vue rename to frontend/src/views/ai/model/ollama/del/index.vue diff --git a/frontend/src/views/ai/model/domain/index.vue b/frontend/src/views/ai/model/ollama/domain/index.vue similarity index 100% rename from frontend/src/views/ai/model/domain/index.vue rename to frontend/src/views/ai/model/ollama/domain/index.vue diff --git a/frontend/src/views/ai/model/ollama/index.vue b/frontend/src/views/ai/model/ollama/index.vue new file mode 100644 index 000000000..94eb24f78 --- /dev/null +++ b/frontend/src/views/ai/model/ollama/index.vue @@ -0,0 +1,461 @@ + + + + + diff --git a/frontend/src/views/ai/model/terminal/index.vue b/frontend/src/views/ai/model/ollama/terminal/index.vue similarity index 100% rename from frontend/src/views/ai/model/terminal/index.vue rename to frontend/src/views/ai/model/ollama/terminal/index.vue diff --git a/frontend/src/views/ai/model/tensorrt/index.vue b/frontend/src/views/ai/model/tensorrt/index.vue new file mode 100644 index 000000000..d56ae9cc4 --- /dev/null +++ b/frontend/src/views/ai/model/tensorrt/index.vue @@ -0,0 +1,224 @@ + + + diff --git a/frontend/src/views/ai/model/tensorrt/operate/index.vue b/frontend/src/views/ai/model/tensorrt/operate/index.vue new file mode 100644 index 000000000..4ef68bb0f --- /dev/null +++ b/frontend/src/views/ai/model/tensorrt/operate/index.vue @@ -0,0 +1,178 @@ + + +