diff --git a/backend/app/api/v1/file.go b/backend/app/api/v1/file.go index 207277920..ba03a19a8 100644 --- a/backend/app/api/v1/file.go +++ b/backend/app/api/v1/file.go @@ -6,7 +6,10 @@ import ( "github.com/1Panel-dev/1Panel/app/dto" "github.com/1Panel-dev/1Panel/constant" "github.com/1Panel-dev/1Panel/global" + websocket2 "github.com/1Panel-dev/1Panel/utils/websocket" "github.com/gin-gonic/gin" + "github.com/gorilla/websocket" + "net/http" "path" ) @@ -174,11 +177,14 @@ func (b *BaseApi) WgetFile(c *gin.Context) { helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err) return } - if err := fileService.Wget(req); err != nil { + key, err := fileService.Wget(req) + if err != nil { helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err) return } - helper.SuccessWithData(c, nil) + helper.SuccessWithData(c, dto.FileWgetRes{ + Key: key, + }) } func (b *BaseApi) MoveFile(c *gin.Context) { @@ -221,3 +227,37 @@ func (b *BaseApi) Size(c *gin.Context) { } helper.SuccessWithData(c, res) } + +var wsUpgrade = websocket.Upgrader{ + CheckOrigin: func(r *http.Request) bool { + return true + }, +} + +var WsManager = websocket2.Manager{ + Group: make(map[string]*websocket2.Client), + Register: make(chan *websocket2.Client, 128), + UnRegister: make(chan *websocket2.Client, 128), + ClientCount: 0, +} + +func (b *BaseApi) Ws(c *gin.Context) { + ws, err := wsUpgrade.Upgrade(c.Writer, c.Request, nil) + if err != nil { + return + } + wsClient := websocket2.NewWsClient("13232", ws) + go wsClient.Read() + go wsClient.Write() +} + +func (b *BaseApi) Keys(c *gin.Context) { + res := &dto.FileProcessKeys{} + keys, err := global.CACHE.PrefixScanKey("file-wget-") + if err != nil { + helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err) + return + } + res.Keys = keys + helper.SuccessWithData(c, res) +} diff --git a/backend/app/dto/file.go b/backend/app/dto/file.go index 9d3afeb78..e25fb7cb5 100644 --- a/backend/app/dto/file.go +++ b/backend/app/dto/file.go @@ -83,3 +83,22 @@ type DirSizeReq struct { type DirSizeRes struct { Size float64 `json:"size" validate:"required"` } + +type FileProcess struct { + Total uint64 `json:"total"` + Written uint64 `json:"written"` + Percent float64 `json:"percent"` + Name string `json:"name"` +} + +type FileProcessReq struct { + Key string +} + +type FileProcessKeys struct { + Keys []string `json:"keys"` +} + +type FileWgetRes struct { + Key string +} diff --git a/backend/app/service/file.go b/backend/app/service/file.go index a5a540e45..0d08779b3 100644 --- a/backend/app/service/file.go +++ b/backend/app/service/file.go @@ -7,6 +7,7 @@ import ( "github.com/1Panel-dev/1Panel/global" "github.com/1Panel-dev/1Panel/utils/files" "github.com/pkg/errors" + uuid "github.com/satori/go.uuid" "io" "io/fs" "os" @@ -123,9 +124,10 @@ func (f FileService) ChangeName(re dto.FileRename) error { return fo.Rename(re.OldName, re.NewName) } -func (f FileService) Wget(w dto.FileWget) error { +func (f FileService) Wget(w dto.FileWget) (string, error) { fo := files.NewFileOp() - return fo.DownloadFile(w.Url, filepath.Join(w.Path, w.Name)) + key := "file-wget-" + uuid.NewV4().String() + return key, fo.DownloadFile(w.Url, filepath.Join(w.Path, w.Name), key) } func (f FileService) MvFile(m dto.FileMove) error { diff --git a/backend/init/cache/badger_db/badger_db.go b/backend/init/cache/badger_db/badger_db.go index eeed465f1..46588e336 100644 --- a/backend/init/cache/badger_db/badger_db.go +++ b/backend/init/cache/badger_db/badger_db.go @@ -68,3 +68,20 @@ func (c *Cache) SetWithTTL(key string, value interface{}, duration time.Duration }) return err } + +func (c *Cache) PrefixScanKey(prefixStr string) ([]string, error) { + var res []string + err := c.db.View(func(txn *badger.Txn) error { + it := txn.NewIterator(badger.DefaultIteratorOptions) + defer it.Close() + prefix := []byte(prefixStr) + for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { + item := it.Item() + k := item.Key() + res = append(res, string(k)) + return nil + } + return nil + }) + return res, err +} diff --git a/backend/router/ro_file.go b/backend/router/ro_file.go index 60d94c168..6ccb0666c 100644 --- a/backend/router/ro_file.go +++ b/backend/router/ro_file.go @@ -30,6 +30,8 @@ func (f *FileRouter) InitFileRouter(Router *gin.RouterGroup) { fileRouter.POST("/move", baseApi.MoveFile) fileRouter.POST("/download", baseApi.Download) fileRouter.POST("/size", baseApi.Size) + fileRouter.GET("/ws", baseApi.Ws) + fileRouter.GET("/keys", baseApi.Keys) } } diff --git a/backend/utils/files/file_op.go b/backend/utils/files/file_op.go index 90decb1c8..5256293a5 100644 --- a/backend/utils/files/file_op.go +++ b/backend/utils/files/file_op.go @@ -2,6 +2,8 @@ package files import ( "context" + "encoding/json" + "fmt" "github.com/1Panel-dev/1Panel/global" "github.com/mholt/archiver/v4" "github.com/pkg/errors" @@ -12,7 +14,9 @@ import ( "os" "path" "path/filepath" + "strconv" "sync" + "time" ) type FileOp struct { @@ -83,23 +87,77 @@ func (f FileOp) Rename(oldName string, newName string) error { return f.Fs.Rename(oldName, newName) } -func (f FileOp) DownloadFile(url, dst string) error { +type WriteCounter struct { + Total uint64 + Written uint64 + Key string + Name string +} + +type Process struct { + Total uint64 `json:"total"` + Written uint64 `json:"written"` + Percent float64 `json:"percent"` + Name string `json:"name"` +} + +func (w *WriteCounter) Write(p []byte) (n int, err error) { + n = len(p) + w.Written += uint64(n) + w.SaveProcess() + return n, nil +} + +func (w *WriteCounter) SaveProcess() { + percent := float64(w.Written) / float64(w.Total) * 100 + percentValue, _ := strconv.ParseFloat(fmt.Sprintf("%.2f", percent), 64) + + process := Process{ + Total: w.Total, + Written: w.Written, + Percent: percentValue, + Name: w.Name, + } + + by, _ := json.Marshal(process) + + if percentValue < 100 { + if err := global.CACHE.Set(w.Key, string(by)); err != nil { + global.LOG.Errorf("save cache error, err %s", err.Error()) + } + } else { + if err := global.CACHE.SetWithTTL(w.Key, string(by), time.Second*time.Duration(10)); err != nil { + global.LOG.Errorf("save cache error, err %s", err.Error()) + } + } + +} + +func (f FileOp) DownloadFile(url, dst, key string) error { resp, err := http.Get(url) if err != nil { - panic(err) + global.LOG.Errorf("get download file [%s] error, err %s", dst, err.Error()) + } + header, err := http.Head(url) + if err != nil { + global.LOG.Errorf("get download file [%s] error, err %s", dst, err.Error()) } - defer resp.Body.Close() out, err := os.Create(dst) if err != nil { - panic(err) + global.LOG.Errorf("create download file [%s] error, err %s", dst, err.Error()) } - defer out.Close() go func() { - if _, err = io.Copy(out, resp.Body); err != nil { + counter := &WriteCounter{} + counter.Key = key + counter.Total = uint64(header.ContentLength) + counter.Name = filepath.Base(dst) + if _, err = io.Copy(out, io.TeeReader(resp.Body, counter)); err != nil { global.LOG.Errorf("save download file [%s] error, err %s", dst, err.Error()) } + out.Close() + resp.Body.Close() }() return nil diff --git a/backend/utils/websocket/client.go b/backend/utils/websocket/client.go new file mode 100644 index 000000000..8a51ca032 --- /dev/null +++ b/backend/utils/websocket/client.go @@ -0,0 +1,79 @@ +package websocket + +import ( + "encoding/json" + "github.com/1Panel-dev/1Panel/global" + "github.com/1Panel-dev/1Panel/utils/files" + "github.com/gorilla/websocket" +) + +type WsMsg struct { + Type string + Keys []string +} + +type Client struct { + ID string + Socket *websocket.Conn + Msg chan []byte +} + +func NewWsClient(ID string, socket *websocket.Conn) *Client { + return &Client{ + ID: ID, + Socket: socket, + Msg: make(chan []byte, 100), + } +} + +func (c *Client) Read() { + defer func() { + close(c.Msg) + }() + + for { + _, message, err := c.Socket.ReadMessage() + if err != nil { + return + } + msg := &WsMsg{} + _ = json.Unmarshal(message, msg) + ProcessData(c, msg) + } +} + +func (c *Client) Write() { + defer func() { + c.Socket.Close() + }() + + for { + select { + case message, ok := <-c.Msg: + if !ok { + return + } + _ = c.Socket.WriteMessage(websocket.TextMessage, message) + } + } +} + +func ProcessData(c *Client, msg *WsMsg) { + + if msg.Type == "wget" { + var res []files.Process + for _, k := range msg.Keys { + value, err := global.CACHE.Get(k) + if err != nil { + global.LOG.Errorf("get cache error,err %s", err.Error()) + return + } + + process := &files.Process{} + json.Unmarshal(value, process) + res = append(res, *process) + } + reByte, _ := json.Marshal(res) + c.Msg <- reByte + } +} diff --git a/backend/utils/websocket/client_manager.go b/backend/utils/websocket/client_manager.go new file mode 100644 index 000000000..73548ecd6 --- /dev/null +++ b/backend/utils/websocket/client_manager.go @@ -0,0 +1,38 @@ +package websocket + +import "sync" + +type Manager struct { + Group map[string]*Client + Lock sync.Mutex + Register, UnRegister chan *Client + ClientCount uint +} + +func (m *Manager) Start() { + for { + select { + case client := <-m.Register: + m.Lock.Lock() + m.Group[client.ID] = client + m.ClientCount++ + m.Lock.Unlock() + case client := <-m.UnRegister: + m.Lock.Lock() + if _, ok := m.Group[client.ID]; ok { + close(client.Msg) + delete(m.Group, client.ID) + m.ClientCount-- + } + m.Lock.Unlock() + } + } +} + +func (m *Manager) RegisterClient(client *Client) { + m.Register <- client +} + +func (m *Manager) UnRegisterClient(client *Client) { + m.UnRegister <- client +} diff --git a/frontend/src/api/interface/file.ts b/frontend/src/api/interface/file.ts index d993d5689..9a727d475 100644 --- a/frontend/src/api/interface/file.ts +++ b/frontend/src/api/interface/file.ts @@ -80,6 +80,14 @@ export namespace File { url: string; } + export interface FileWgetRes { + key: string; + } + + export interface FileKeys { + keys: string[]; + } + export interface FileMove { oldPaths: string[]; newPath: string; diff --git a/frontend/src/api/modules/files.ts b/frontend/src/api/modules/files.ts index 699c33104..e9408b5f2 100644 --- a/frontend/src/api/modules/files.ts +++ b/frontend/src/api/modules/files.ts @@ -47,7 +47,7 @@ export const RenameRile = (params: File.FileRename) => { }; export const WgetFile = (params: File.FileWget) => { - return http.post('files/wget', params); + return http.post('files/wget', params); }; export const MoveFile = (params: File.FileMove) => { @@ -61,3 +61,7 @@ export const DownloadFile = (params: File.FileDownload) => { export const ComputeDirSize = (params: File.DirSizeReq) => { return http.post('files/size', params); }; + +export const FileKeys = () => { + return http.get('files/keys'); +}; diff --git a/frontend/src/lang/modules/en.ts b/frontend/src/lang/modules/en.ts index 14ac17395..0ed93e50a 100644 --- a/frontend/src/lang/modules/en.ts +++ b/frontend/src/lang/modules/en.ts @@ -245,5 +245,6 @@ export default { calculate: 'Calculate', canNotDeCompress: 'Can not DeCompress this File', uploadSuccess: 'Upload Success!', + downloadProcess: 'Download Process', }, }; diff --git a/frontend/src/lang/modules/zh.ts b/frontend/src/lang/modules/zh.ts index 170e6b2b5..b055c1fd9 100644 --- a/frontend/src/lang/modules/zh.ts +++ b/frontend/src/lang/modules/zh.ts @@ -245,5 +245,6 @@ export default { calculate: '计算', canNotDeCompress: '无法解压此文件', uploadSuccess: '上传成功!', + downloadProcess: '下载进度', }, }; diff --git a/frontend/src/views/host/file-management/index.vue b/frontend/src/views/host/file-management/index.vue index de56a32d8..fd6c0cd49 100644 --- a/frontend/src/views/host/file-management/index.vue +++ b/frontend/src/views/host/file-management/index.vue @@ -172,6 +172,7 @@ :name="downloadPage.name" @close="closeDownload" > + @@ -206,6 +207,7 @@ import Wget from './wget/index.vue'; import Move from './move/index.vue'; import Download from './download/index.vue'; import { Mimetypes } from '@/global/mimetype'; +import Process from './process/index.vue'; const data = ref(); let selects = ref([]); @@ -227,6 +229,7 @@ const renamePage = reactive({ open: false, path: '', oldName: '' }); const wgetPage = reactive({ open: false, path: '' }); const movePage = reactive({ open: false, oldPaths: [''], type: '' }); const downloadPage = reactive({ open: false, paths: [''], name: '' }); +const processPage = reactive({ open: false }); const defaultProps = { children: 'children', @@ -445,9 +448,21 @@ const openWget = () => { wgetPage.path = req.path; }; -const closeWget = () => { +const closeWget = (submit: any) => { + console.log(submit); wgetPage.open = false; search(req); + if (submit) { + openProcess(); + } +}; + +const openProcess = () => { + processPage.open = true; +}; + +const closeProcess = () => { + processPage.open = false; }; const openRename = (item: File.File) => { @@ -490,7 +505,6 @@ const closeDownload = () => { downloadPage.open = false; search(req); }; - const saveContent = (content: string) => { editorPage.loading = true; SaveFileContent({ path: codeReq.path, content: content }).finally(() => { diff --git a/frontend/src/views/host/file-management/process/index.vue b/frontend/src/views/host/file-management/process/index.vue new file mode 100644 index 000000000..956b67d55 --- /dev/null +++ b/frontend/src/views/host/file-management/process/index.vue @@ -0,0 +1,94 @@ + + + diff --git a/frontend/src/views/host/file-management/wget/index.vue b/frontend/src/views/host/file-management/wget/index.vue index 477a7f193..7cc152c2d 100644 --- a/frontend/src/views/host/file-management/wget/index.vue +++ b/frontend/src/views/host/file-management/wget/index.vue @@ -22,7 +22,7 @@