fix: 解决部分 OneDrive 上传大文件失败的问题 (#4117)

Refs #4125
This commit is contained in:
ssongliu 2024-03-08 16:12:06 +08:00 committed by GitHub
parent a6784772b9
commit 21b1242a94
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 22 additions and 15 deletions

View file

@ -154,10 +154,6 @@ func snapPanelData(snap snapHelper, localDir, targetDir string) {
} }
func snapCompress(snap snapHelper, rootDir string) { func snapCompress(snap snapHelper, rootDir string) {
defer func() {
global.LOG.Debugf("remove snapshot file %s", rootDir)
_ = os.RemoveAll(rootDir)
}()
_ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"compress": constant.StatusRunning}) _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"compress": constant.StatusRunning})
tmpDir := path.Join(global.CONF.System.TmpDir, "system") tmpDir := path.Join(global.CONF.System.TmpDir, "system")
fileName := fmt.Sprintf("%s.tar.gz", path.Base(rootDir)) fileName := fmt.Sprintf("%s.tar.gz", path.Base(rootDir))
@ -178,15 +174,13 @@ func snapCompress(snap snapHelper, rootDir string) {
snap.Status.Compress = constant.StatusDone snap.Status.Compress = constant.StatusDone
snap.Status.Size = size snap.Status.Size = size
_ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"compress": constant.StatusDone, "size": size}) _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"compress": constant.StatusDone, "size": size})
global.LOG.Debugf("remove snapshot file %s", rootDir)
_ = os.RemoveAll(rootDir)
} }
func snapUpload(snap snapHelper, accounts string, file string) { func snapUpload(snap snapHelper, accounts string, file string) {
source := path.Join(global.CONF.System.TmpDir, "system", path.Base(file)) source := path.Join(global.CONF.System.TmpDir, "system", path.Base(file))
defer func() {
global.LOG.Debugf("remove snapshot file %s", source)
_ = os.Remove(source)
}()
_ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"upload": constant.StatusUploading}) _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"upload": constant.StatusUploading})
accountMap, err := loadClientMap(accounts) accountMap, err := loadClientMap(accounts)
if err != nil { if err != nil {
@ -207,6 +201,9 @@ func snapUpload(snap snapHelper, accounts string, file string) {
} }
snap.Status.Upload = constant.StatusDone snap.Status.Upload = constant.StatusDone
_ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"upload": constant.StatusDone}) _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"upload": constant.StatusDone})
global.LOG.Debugf("remove snapshot file %s", source)
_ = os.Remove(source)
} }
func handleSnapTar(sourceDir, targetDir, name, exclusionRules string) error { func handleSnapTar(sourceDir, targetDir, name, exclusionRules string) error {

View file

@ -4,6 +4,7 @@ import (
"bufio" "bufio"
"bytes" "bytes"
"context" "context"
"crypto/tls"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
@ -14,6 +15,7 @@ import (
"path" "path"
"strconv" "strconv"
"strings" "strings"
"time"
"github.com/1Panel-dev/1Panel/backend/utils/files" "github.com/1Panel-dev/1Panel/backend/utils/files"
odsdk "github.com/goh-chunlin/go-onedrive/onedrive" odsdk "github.com/goh-chunlin/go-onedrive/onedrive"
@ -321,7 +323,6 @@ func (o *oneDriveClient) upSmall(srcPath, folderID string, fileSize int64) (bool
if err := o.client.Do(context.Background(), req, false, &response); err != nil { if err := o.client.Do(context.Background(), req, false, &response); err != nil {
return false, fmt.Errorf("do request for list failed, err: %v", err) return false, fmt.Errorf("do request for list failed, err: %v", err)
} }
fmt.Println(response)
return true, nil return true, nil
} }
@ -362,7 +363,13 @@ func (o *oneDriveClient) upBig(ctx context.Context, srcPath, folderID string, fi
splitCount += 1 splitCount += 1
} }
bfReader := bufio.NewReader(file) bfReader := bufio.NewReader(file)
var fileUploadResp *UploadSessionUploadResponse httpClient := http.Client{
Timeout: time.Minute * 10,
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
},
}
for splitNow := int64(0); splitNow < splitCount; splitNow++ { for splitNow := int64(0); splitNow < splitCount; splitNow++ {
length, err := bfReader.Read(buffer) length, err := bfReader.Read(buffer)
if err != nil { if err != nil {
@ -376,12 +383,15 @@ func (o *oneDriveClient) upBig(ctx context.Context, srcPath, folderID string, fi
if err != nil { if err != nil {
return false, err return false, err
} }
if err := o.client.Do(ctx, sessionFileUploadReq, false, &fileUploadResp); err != nil { res, err := httpClient.Do(sessionFileUploadReq)
if err != nil {
return false, err return false, err
} }
} if res.StatusCode != 201 && res.StatusCode != 202 && res.StatusCode != 200 {
if fileUploadResp.Id == "" { data, _ := io.ReadAll(res.Body)
return false, errors.New("something went wrong. file upload incomplete. consider upload the file in a step-by-step manner") res.Body.Close()
return false, errors.New(string(data))
}
} }
return true, nil return true, nil
} }