1Panel-dev/1Panel
1
1
Fork 0
mirror of https://github.com/1Panel-dev/1Panel.git synced 2025-09-05 22:25:49 +08:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: Optimize file log reading (#10218)

Browse source
This commit is contained in:
CityFun 2025-09-01 17:39:40 +08:00 • committed by GitHub
parent 4c287f4a2d
commit f2335d313c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 86 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
agent/app/dto/file.go Normal file
View file

@ -0,0 +1,8 @@
package dto
type LogFileRes struct {
Lines []string `json:"lines"`
IsEndOfFile bool `json:"isEndOfFile"`
TotalPages int `json:"totalPages"`
TotalLines int `json:"totalLines"`
}

1
agent/app/dto/response/file.go
View file

@ -44,6 +44,7 @@ type FileLineContent struct {
TaskStatus string `json:"taskStatus"`
Lines []string `json:"lines"`
Scope string `json:"scope"`
TotalLines int `json:"totalLines"`
}
type FileExist struct {

18
agent/app/service/file.go
View file

@ -568,37 +568,41 @@ func (f *FileService) ReadLogByLine(req request.FileReadByLineReq) (*response.Fi
var (
lines []string
isEndOfFile bool
total int
scope string
logFileRes *dto.LogFileRes
)
if stat.Size() > 500*1024*1024 {
lines, err = files.TailFromEnd(logFilePath, req.PageSize)
isEndOfFile = true
scope = "tail"
} else {
lines, isEndOfFile, total, err = files.ReadFileByLine(logFilePath, req.Page, req.PageSize, req.Latest)
logFileRes, err = files.ReadFileByLine(logFilePath, req.Page, req.PageSize, req.Latest)
if err != nil {
return nil, err
}
if req.Latest && req.Page == 1 && len(lines) < 1000 && total > 1 {
preLines, _, _, err := files.ReadFileByLine(logFilePath, total-1, req.PageSize, false)
if req.Latest && req.Page == 1 && len(logFileRes.Lines) < 1000 && logFileRes.TotalPages > 1 {
res, err := files.ReadFileByLine(logFilePath, logFileRes.TotalPages-1, req.PageSize, false)
if err != nil {
return nil, err
}
lines = append(preLines, lines...)
logFileRes.Lines = append(res.Lines, logFileRes.Lines...)
}
scope = "page"
lines = logFileRes.Lines
}
res := &response.FileLineContent{
End: isEndOfFile,
Path: logFilePath,
Total: total,
TaskStatus: taskStatus,
Lines: lines,
Scope: scope,
}
if logFileRes != nil {
res.TotalLines = logFileRes.TotalLines
res.Total = logFileRes.TotalPages
res.End = logFileRes.IsEndOfFile
}
return res, nil
}

6
agent/app/service/website.go
View file

@ -1233,13 +1233,13 @@ func (w WebsiteService) OpWebsiteLog(req request.WebsiteLogReq) (*response.Websi
}
}
filePath := path.Join(sitePath, "log", req.LogType)
lines, end, _, err := files.ReadFileByLine(filePath, req.Page, req.PageSize, false)
logFileRes, err := files.ReadFileByLine(filePath, req.Page, req.PageSize, false)
if err != nil {
return nil, err
}
res.End = end
res.End = logFileRes.IsEndOfFile
res.Path = filePath
res.Content = strings.Join(lines, "\n")
res.Content = strings.Join(logFileRes.Lines, "\n")
return res, nil
case constant.DisableLog:
params := dto.NginxParam{}

2
agent/go.sum
View file

@ -919,8 +919,6 @@ github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2
github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY=
github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/upyun/go-sdk v2.1.0+incompatible h1:OdjXghQ/TVetWV16Pz3C1/SUpjhGBVPr+cLiqZLLyq0=

14
agent/utils/files/utils.go
View file

@ -4,6 +4,7 @@ import (
"bufio"
"bytes"
"fmt"
"github.com/1Panel-dev/1Panel/agent/app/dto"
"github.com/1Panel-dev/1Panel/agent/buserr"
"github.com/1Panel-dev/1Panel/agent/constant"
"github.com/1Panel-dev/1Panel/agent/utils/req_helper"
@ -161,7 +162,7 @@ func TailFromEnd(filename string, lines int) ([]string, error) {
return result, nil
}
func ReadFileByLine(filename string, page, pageSize int, latest bool) (lines []string, isEndOfFile bool, total int, err error) {
func ReadFileByLine(filename string, page, pageSize int, latest bool) (res *dto.LogFileRes, err error) {
if !NewFileOp().Stat(filename) {
return
}
@ -185,7 +186,10 @@ func ReadFileByLine(filename string, page, pageSize int, latest bool) (lines []s
if err != nil {
return
}
total = (totalLines + pageSize - 1) / pageSize
res = &dto.LogFileRes{}
total := (totalLines + pageSize - 1) / pageSize
res.TotalPages = total
res.TotalLines = totalLines
reader := bufio.NewReaderSize(file, 8192)
if latest {
@ -194,7 +198,7 @@ func ReadFileByLine(filename string, page, pageSize int, latest bool) (lines []s
currentLine := 0
startLine := (page - 1) * pageSize
endLine := startLine + pageSize
lines := make([]string, 0, pageSize)
for {
line, _, err := reader.ReadLine()
if err == io.EOF {
@ -208,8 +212,8 @@ func ReadFileByLine(filename string, page, pageSize int, latest bool) (lines []s
break
}
}
isEndOfFile = currentLine < endLine
res.Lines = lines
res.IsEndOfFile = currentLine < endLine
return
}

66
frontend/src/components/log/file/index.vue
View file

@ -37,7 +37,7 @@
>
<hightlight :log="log" :type="config.colorMode ?? 'nginx'"></hightlight>
</div>
<hightlight v-if="visibleLogs.length === 0" :log="$t('commons.log.noLog')" type="system"></hightlight>
<hightlight v-if="logs.length === 0" :log="$t('commons.log.noLog')" type="system"></hightlight>
</div>
</div>
</template>
@ -151,13 +151,20 @@ const containerHeight = ref(500);
const visibleCount = computed(() => Math.ceil(containerHeight.value / logHeight));
const startIndex = ref(0);
const lastScrollTop = ref(0);
const totalLines = ref(0);
const stopReading = ref(false);
const visibleLogs = computed(() => {
return logs.value.slice(startIndex.value, startIndex.value + visibleCount.value);
const safeStartIndex = Math.max(0, Math.min(startIndex.value, Math.max(0, logs.value.length - visibleCount.value)));
if (safeStartIndex !== startIndex.value) {
nextTick(() => {
startIndex.value = safeStartIndex;
});
}
return logs.value.slice(safeStartIndex, safeStartIndex + visibleCount.value);
});
const onScroll = async () => {
if (!logContainer.value || isLoading.value) return;
if (!logContainer.value) return;
const scrollTop = logContainer.value.scrollTop;
const scrollHeight = logContainer.value.scrollHeight;
@ -165,8 +172,15 @@ const onScroll = async () => {
lastScrollTop.value = scrollTop;
const newStartIndex = Math.max(0, Math.floor(scrollTop / logHeight));
startIndex.value = newStartIndex;
if (!isLoading.value) {
const newStartIndex = Math.max(
0,
Math.min(Math.floor(scrollTop / logHeight), Math.max(0, logs.value.length - visibleCount.value)),
);
startIndex.value = newStartIndex;
}
if (isLoading.value) return;
if (scrollTop <= 50 && readReq.page > 1) {
if (minPage.value <= 1) {
@ -178,7 +192,7 @@ const onScroll = async () => {
return;
}
if (scrollHeight - scrollTop - clientHeight <= 50 && !end.value && maxPage.value > 1) {
if (scrollHeight - scrollTop - clientHeight <= 50 && !end.value && readReq.page < maxPage.value) {
readReq.page = maxPage.value;
await getContent(false);
}
@ -210,6 +224,7 @@ const changeTail = (fromOutSide: boolean) => {
const clearLog = (): void => {
logs.value = [];
startIndex.value = 0;
readReq.page = 1;
lastLogs.value = [];
};
@ -231,6 +246,12 @@ const getContent = async (pre: boolean) => {
isLoading.value = false;
firstLoading.value = false;
}
if (res.data.scope != 'tail' && !pre && res.data.end && res.data.totalLines == totalLines.value) {
isLoading.value = false;
return;
}
totalLines.value = res.data.totalLines;
if (res.data.scope == 'tail') {
showTail.value = false;
@ -238,6 +259,7 @@ const getContent = async (pre: boolean) => {
if (res.data.taskStatus && res.data.taskStatus !== 'Executing') {
isTailDisabled.value = true;
tailLog.value = false;
}
logPath.value = res.data.path;
@ -279,18 +301,29 @@ const getContent = async (pre: boolean) => {
if (end.value) {
logs.value = [...lastLogs.value, ...newLogs];
} else {
logs.value = [...logs.value, ...newLogs];
if (newLogs.length > logs.value.length) {
logs.value = newLogs;
} else {
logs.value = [...logs.value, ...newLogs];
}
}
}
}
nextTick(() => {
if (logContainer.value) {
if (pre) {
if (readReq.page > 1) {
const addedLines = newLogs.length;
const newScrollPosition = lastScrollTop.value + (addedLines * logHeight) / 3;
const newScrollPosition = lastScrollTop.value + addedLines * logHeight;
logContainer.value.scrollTop = newScrollPosition;
startIndex.value = Math.max(0, Math.floor(newScrollPosition / logHeight));
startIndex.value = Math.max(
0,
Math.min(
Math.floor(newScrollPosition / logHeight),
Math.max(0, logs.value.length - visibleCount.value),
),
);
}
} else {
logContainer.value.scrollTop = totalHeight.value;
@ -313,24 +346,33 @@ const getContent = async (pre: boolean) => {
} else {
minPage.value = res.data.total;
}
} else {
maxPage.value = Math.max(maxPage.value, readReq.page);
}
if (logs.value && logs.value.length > 3000) {
const removedCount = readReq.pageSize;
if (pre) {
logs.value.splice(logs.value.length - readReq.pageSize, readReq.pageSize);
logs.value.splice(logs.value.length - removedCount, removedCount);
if (maxPage.value > 1) {
maxPage.value--;
}
} else {
logs.value.splice(0, readReq.pageSize);
logs.value.splice(0, removedCount);
startIndex.value = Math.max(0, startIndex.value - removedCount);
if (minPage.value > 1) {
minPage.value++;
}
}
logCount.value = logs.value.length;
}
isLoading.value = false;
};
const onCloseLog = async () => {
if (stopReading.value) {
return;
}
stopReading.value = true;
tailLog.value = false;
if (timer) {
clearInterval(Number(timer));