1Panel-dev/1Panel
1
1
Fork 0
mirror of https://github.com/1Panel-dev/1Panel.git synced 2024-09-20 07:35:58 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: 增加文件分片上传

Browse source
This commit is contained in:
zhengkunwang223 2023-03-11 19:55:37 +08:00 committed by zhengkunwang223
parent f2ca4a88dd
commit 3d79c06f71
10 changed files with 205 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
backend/app/api/v1/file.go
View file

@ -3,23 +3,21 @@ package v1
import (
"errors"
"fmt"
"io/ioutil"
"net/http"
"os"
"path"
"strings"
"github.com/1Panel-dev/1Panel/backend/app/api/v1/helper"
"github.com/1Panel-dev/1Panel/backend/app/dto"
"github.com/1Panel-dev/1Panel/backend/app/dto/request"
"github.com/1Panel-dev/1Panel/backend/app/dto/response"
"github.com/1Panel-dev/1Panel/backend/buserr"
"github.com/1Panel-dev/1Panel/backend/app/api/v1/helper"
"github.com/1Panel-dev/1Panel/backend/app/dto"
"github.com/1Panel-dev/1Panel/backend/constant"
"github.com/1Panel-dev/1Panel/backend/global"
websocket2 "github.com/1Panel-dev/1Panel/backend/utils/websocket"
"github.com/gin-gonic/gin"
"github.com/gorilla/websocket"
"io/ioutil"
"net/http"
"os"
"path"
"strings"
)
// @Tags File

110
backend/app/api/v1/file_upload.go Normal file
View file

@ -0,0 +1,110 @@
package v1
import (
"fmt"
"github.com/1Panel-dev/1Panel/backend/app/api/v1/helper"
"github.com/1Panel-dev/1Panel/backend/constant"
"github.com/1Panel-dev/1Panel/backend/global"
"github.com/1Panel-dev/1Panel/backend/utils/files"
"github.com/gin-gonic/gin"
"io/ioutil"
"os"
"path/filepath"
"strconv"
)
func mergeChunks(fileName string, fileDir string, dstDir string, chunkCount int) error {
//fileInfoList, err := ioutil.ReadDir(fileDir)
//if err != nil {
// return err
//}
targetFile, err := os.Create(filepath.Join(dstDir, fileName))
if err != nil {
return err
}
defer targetFile.Close()
for i := 0; i < chunkCount; i++ {
chunkPath := filepath.Join(fileDir, fmt.Sprintf("%s.%d", fileName, i))
chunkData, err := ioutil.ReadFile(chunkPath)
if err != nil {
return err
}
_, err = targetFile.Write(chunkData)
if err != nil {
return err
}
}
return files.NewFileOp().DeleteDir(fileDir)
}
func (b *BaseApi) UploadChunkFiles(c *gin.Context) {
fileForm, err := c.FormFile("chunk")
if err != nil {
helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err)
return
}
uploadFile, err := fileForm.Open()
if err != nil {
helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err)
return
}
chunkIndex, err := strconv.Atoi(c.PostForm("chunkIndex"))
if err != nil {
helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err)
return
}
chunkCount, err := strconv.Atoi(c.PostForm("chunkCount"))
if err != nil {
helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err)
return
}
fileOp := files.NewFileOp()
if err := fileOp.CreateDir("uploads", 0755); err != nil {
helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err)
return
}
//fileID := uuid.New().String()
filename := c.PostForm("filename")
fileDir := filepath.Join(global.CONF.System.DataDir, "upload", filename)
os.MkdirAll(fileDir, 0755)
filePath := filepath.Join(fileDir, filename)
emptyFile, err := os.Create(filePath)
if err != nil {
helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err)
return
}
emptyFile.Close()
chunkData, err := ioutil.ReadAll(uploadFile)
if err != nil {
helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err)
return
}
chunkPath := filepath.Join(fileDir, fmt.Sprintf("%s.%d", filename, chunkIndex))
err = ioutil.WriteFile(chunkPath, chunkData, 0644)
if err != nil {
helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err)
return
}
if chunkIndex+1 == chunkCount {
err = mergeChunks(filename, fileDir, c.PostForm("path"), chunkCount)
if err != nil {
fmt.Println(err.Error())
helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrAppDelete, err)
return
}
helper.SuccessWithData(c, true)
} else {
return
}
}

4
backend/app/service/auth.go
View file

@ -12,8 +12,8 @@ import (
"github.com/1Panel-dev/1Panel/backend/utils/jwt"
"github.com/1Panel-dev/1Panel/backend/utils/mfa"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"github.com/pkg/errors"
uuid "github.com/satori/go.uuid"
)
type AuthService struct{}
@ -128,7 +128,7 @@ func (u *AuthService) generateSession(c *gin.Context, name, authMethod string) (
sID, _ := c.Cookie(constant.SessionName)
sessionUser, err := global.SESSION.Get(sID)
if err != nil {
sID = uuid.NewV4().String()
sID = uuid.New().String()
c.SetCookie(constant.SessionName, sID, 604800, "", "", false, false)
err := global.SESSION.Set(sID, sessionUser, lifeTime)
if err != nil {

3
backend/app/service/file.go
View file

@ -17,7 +17,6 @@ import (
"github.com/1Panel-dev/1Panel/backend/utils/common"
"github.com/1Panel-dev/1Panel/backend/utils/files"
"github.com/pkg/errors"
uuid "github.com/satori/go.uuid"
)
type FileService struct {
@ -181,7 +180,7 @@ func (f FileService) ChangeName(req request.FileRename) error {
func (f FileService) Wget(w request.FileWget) (string, error) {
fo := files.NewFileOp()
key := "file-wget-" + uuid.NewV4().String()
key := "file-wget-" + common.GetUuid()
return key, fo.DownloadFileWithProcess(w.Url, filepath.Join(w.Path, w.Name), key)
}

2
backend/i18n/i18n.go
View file

@ -8,7 +8,7 @@ import (
"github.com/gin-gonic/gin"
"github.com/nicksnyder/go-i18n/v2/i18n"
"golang.org/x/text/language"
"gopkg.in/yaml.v2"
"gopkg.in/yaml.v3"
)
func GetMsgWithMap(msg string, maps map[string]interface{}) string {

2
backend/router/ro_file.go
View file

@ -26,7 +26,7 @@ func (f *FileRouter) InitFileRouter(Router *gin.RouterGroup) {
fileRouter.POST("/content", baseApi.GetContent)
fileRouter.POST("/save", baseApi.SaveContent)
fileRouter.POST("/check", baseApi.CheckFile)
fileRouter.POST("/upload", baseApi.UploadFiles)
fileRouter.POST("/upload", baseApi.UploadChunkFiles)
fileRouter.POST("/rename", baseApi.ChangeFileName)
fileRouter.POST("/wget", baseApi.WgetFile)
fileRouter.POST("/move", baseApi.MoveFile)

1
cmd/server/conf/app.yaml
View file

@ -4,6 +4,7 @@ system:
mode: dev
repo_url: https://resource.fit2cloud.com/1panel/package
is_demo: false
port: 9999
log:
level: debug

133
frontend/src/views/host/file-management/upload/index.vue
View file

@ -1,40 +1,39 @@
<template>
<div>
<el-drawer v-model="open" :before-close="handleClose" size="40%">
<template #header>
<DrawerHeader :header="$t('file.upload')" :back="handleClose" />
</template>
<el-upload
action="#"
drag
:auto-upload="false"
ref="uploadRef"
:multiple="true"
:on-change="fileOnChange"
v-loading="loading"
>
<el-icon class="el-icon--upload"><upload-filled /></el-icon>
<div class="el-upload__text">
{{ $t('database.dropHelper') }}
<em>{{ $t('database.clickHelper') }}</em>
</div>
</el-upload>
<el-progress
v-if="loading"
:text-inside="true"
:stroke-width="26"
:percentage="uploadPrecent"
></el-progress>
<template #footer>
<span class="dialog-footer">
<el-button @click="handleClose" :disabled="loading">{{ $t('commons.button.cancel') }}</el-button>
<el-button type="primary" @click="submit()" :disabled="loading">
{{ $t('commons.button.confirm') }}
</el-button>
</span>
</template>
</el-drawer>
</div>
<el-drawer
v-model="open"
:before-close="handleClose"
size="40%"
:destroy-on-close="true"
:close-on-click-modal="false"
>
<template #header>
<DrawerHeader :header="$t('file.upload')" :back="handleClose" />
</template>
<el-upload
action="#"
drag
:auto-upload="false"
ref="uploadRef"
:multiple="true"
:on-change="fileOnChange"
v-loading="loading"
>
<el-icon class="el-icon--upload"><upload-filled /></el-icon>
<div class="el-upload__text">
{{ $t('database.dropHelper') }}
<em>{{ $t('database.clickHelper') }}</em>
</div>
</el-upload>
<el-progress v-if="loading" :text-inside="true" :stroke-width="26" :percentage="uploadPrecent"></el-progress>
<template #footer>
<span class="dialog-footer">
<el-button @click="handleClose" :disabled="loading">{{ $t('commons.button.cancel') }}</el-button>
<el-button type="primary" @click="submit()" :disabled="loading">
{{ $t('commons.button.confirm') }}
</el-button>
</span>
</template>
</el-drawer>
</template>
<script setup lang="ts">
@ -68,33 +67,57 @@ const fileOnChange = (_uploadFile: UploadFile, uploadFiles: UploadFiles) => {
uploaderFiles.value = uploadFiles;
};
const onProcess = (e: any) => {
const { loaded, total } = e;
uploadPrecent.value = ((loaded / total) * 100) | 0;
};
// const onProcess = (e: any) => {
// const { loaded, total } = e;
// uploadPrecent.value = ((loaded / total) * 100) | 0;
// };
const submit = () => {
const formData = new FormData();
for (const file of uploaderFiles.value) {
if (file.raw != undefined) {
formData.append('file', file.raw);
const submit = async () => {
loading.value = true;
const file = uploaderFiles.value[0];
const CHUNK_SIZE = 1024 * 1024; // 1MB
const fileSize = file.size;
const chunkCount = Math.ceil(fileSize / CHUNK_SIZE);
let uploadedChunkCount = 0;
for (let i = 0; i < chunkCount; i++) {
const start = i * CHUNK_SIZE;
const end = Math.min(start + CHUNK_SIZE, fileSize);
const chunk = file.raw.slice(start, end);
const formData = new FormData();
formData.append('filename', file.name);
formData.append('path', path.value);
formData.append('chunk', chunk);
formData.append('chunkIndex', i.toString());
formData.append('chunkCount', chunkCount.toString());
try {
await UploadFileData(formData, {
onUploadProgress: (progressEvent) => {
const progress = Math.round(
((uploadedChunkCount + progressEvent.loaded / progressEvent.total) * 100) / chunkCount,
);
uploadPrecent.value = progress;
},
});
uploadedChunkCount++;
} catch (error) {
loading.value = false;
}
if (uploadedChunkCount == chunkCount) {
loading.value = false;
MsgSuccess(i18n.global.t('file.uploadSuccess'));
}
}
formData.append('path', path.value);
loading.value = true;
UploadFileData(formData, { onUploadProgress: onProcess })
.then(() => {
MsgSuccess(i18n.global.t('file.uploadSuccess'));
handleClose();
})
.finally(() => {
loading.value = false;
});
};
const acceptParams = (props: UploadProps) => {
path.value = props.path;
open.value = true;
uploadPrecent.value = 0;
};
defineExpose({ acceptParams });

9
go.mod
View file

@ -21,6 +21,7 @@ require (
github.com/go-sql-driver/mysql v1.6.0
github.com/gogf/gf v1.16.9
github.com/golang-jwt/jwt/v4 v4.4.2
github.com/google/uuid v1.3.0
github.com/gorilla/websocket v1.5.0
github.com/jinzhu/copier v0.3.5
github.com/joho/godotenv v1.5.1
@ -32,7 +33,6 @@ require (
github.com/pkg/errors v0.9.1
github.com/pkg/sftp v1.13.1
github.com/robfig/cron/v3 v3.0.1
github.com/satori/go.uuid v1.2.0
github.com/shirou/gopsutil/v3 v3.23.1
github.com/sirupsen/logrus v1.9.0
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
@ -47,7 +47,6 @@ require (
golang.org/x/net v0.7.0
golang.org/x/text v0.7.0
gopkg.in/square/go-jose.v2 v2.6.0
gopkg.in/yaml.v2 v2.4.0
gopkg.in/yaml.v3 v3.0.1
gorm.io/driver/sqlite v1.4.4
gorm.io/gorm v1.24.5
@ -91,7 +90,6 @@ require (
github.com/golang/snappy v0.0.4 // indirect
github.com/google/flatbuffers v1.12.1 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-retryablehttp v0.7.1 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
@ -156,8 +154,7 @@ require (
golang.org/x/tools v0.1.12 // indirect
google.golang.org/protobuf v1.28.0 // indirect
gopkg.in/ini.v1 v1.66.6 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)
replace (
golang.org/x/net => golang.org/x/net v0.7.0
)
replace golang.org/x/net => golang.org/x/net v0.7.0

2
go.sum
View file

@ -857,7 +857,6 @@ github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U=
github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
@ -1053,6 +1052,7 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190501045829-6d32002ffd75/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.5.0 h1:5JMiNunQeQw++mMOz48/ISeNu3Iweh/JaZU8ZLqHRrI=
golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=