feat: Optimize log file reading (#10249)

agent/app/service/file.go

@@ -580,13 +580,6 @@ func (f *FileService) ReadLogByLine(req request.FileReadByLineReq) (*response.Fi
 		if err != nil {
 			return nil, err
 		}
-		if req.Latest && req.Page == 1 && len(logFileRes.Lines) < 1000 && logFileRes.TotalPages > 1 {
-			res, err := files.ReadFileByLine(logFilePath, logFileRes.TotalPages-1, req.PageSize, false)
-			if err != nil {
-				return nil, err
-			}
-			logFileRes.Lines = append(res.Lines, logFileRes.Lines...)
-		}
 		scope = "page"
 		lines = logFileRes.Lines
 	}

frontend/src/components/log/file/index.vue

@@ -29,20 +29,22 @@
         </div>
         <div class="log-container" ref="logContainer" @scroll="onScroll" :style="containerStyle">
             <div class="log-spacer" :style="{ height: `${totalHeight}px` }"></div>
-            <div
+            <div class="log-viewport" :style="{ transform: `translateY(${offsetY}px)` }">
-                v-for="(log, index) in visibleLogs"
+                <div
-                :key="startIndex + index"
+                    v-for="(log, index) in visibleLogs"
-                class="log-item"
+                    :key="`${startIndex + index}-${log}`"
-                :style="{ top: `${(startIndex + index) * logHeight}px` }"
+                    class="log-item"
-            >
+                    :style="{ height: `${logHeight}px` }"
-                <hightlight :log="log" :type="config.colorMode ?? 'nginx'"></hightlight>
+                >
+                    <hightlight :log="log" :type="config.colorMode ?? 'nginx'"></hightlight>
+                </div>
             </div>
             <hightlight v-if="logs.length === 0" :log="$t('commons.log.noLog')" type="system"></hightlight>
         </div>
     </div>
 </template>
 <script lang="ts" setup>
-import { nextTick, onMounted, onUnmounted, reactive, ref } from 'vue';
+import { nextTick, onMounted, onUnmounted, reactive, ref, computed } from 'vue';
 import { downloadFile } from '@/utils/util';
 import { readByLine } from '@/api/modules/files';
 import { GlobalStore } from '@/store';
@@ -144,46 +146,60 @@ const isTailDisabled = ref();
 const firstLoading = ref(false);
 const logs = ref<string[]>([]);
 const logContainer = ref<HTMLElement | null>(null);
-const logHeight = 20;
+const logHeight = 23;
-const logCount = ref(0);
-const totalHeight = computed(() => logHeight * logCount.value);
 const containerHeight = ref(500);
-const visibleCount = computed(() => Math.ceil(containerHeight.value / logHeight));
+const scrollTop = ref(0);
-const startIndex = ref(0);
 const lastScrollTop = ref(0);
 const totalLines = ref(0);
 const stopReading = ref(false);
 const totalPages = ref(0);
-const visibleLogs = computed(() => {
+let resizeObserver: ResizeObserver | null = null;
-    const safeStartIndex = Math.max(0, Math.min(startIndex.value, Math.max(0, logs.value.length - visibleCount.value)));
+const isEndOfFile = ref(false);
-    if (safeStartIndex !== startIndex.value) {
+
-        nextTick(() => {
+const totalHeight = computed(() => logs.value.length * logHeight);
-            startIndex.value = safeStartIndex;
+
-        });
+const visibleCount = computed(() => {
-    }
+    const buffer = 5;
-    return logs.value.slice(safeStartIndex, safeStartIndex + visibleCount.value);
+    return Math.ceil(containerHeight.value / logHeight) + buffer * 2;
 });
 
+const startIndex = computed(() => {
+    const buffer = 5;
+    const index = Math.floor(scrollTop.value / logHeight) - buffer;
+    return Math.max(0, index);
+});
+
+const endIndex = computed(() => {
+    return Math.min(logs.value.length, startIndex.value + visibleCount.value);
+});
+
+const visibleLogs = computed(() => {
+    return logs.value.slice(startIndex.value, endIndex.value);
+});
+
+const offsetY = computed(() => {
+    return startIndex.value * logHeight;
+});
+
+const updateContainerHeight = () => {
+    if (logContainer.value) {
+        const rect = logContainer.value.getBoundingClientRect();
+        containerHeight.value = rect.height;
+    }
+};
+
 const onScroll = async () => {
     if (!logContainer.value) return;
 
-    const scrollTop = logContainer.value.scrollTop;
+    scrollTop.value = logContainer.value.scrollTop;
     const scrollHeight = logContainer.value.scrollHeight;
     const clientHeight = logContainer.value.clientHeight;
 
-    lastScrollTop.value = scrollTop;
+    lastScrollTop.value = scrollTop.value;
-
-    if (!isLoading.value) {
-        const newStartIndex = Math.max(
-            0,
-            Math.min(Math.floor(scrollTop / logHeight), Math.max(0, logs.value.length - visibleCount.value)),
-        );
-        startIndex.value = newStartIndex;
-    }
 
     if (isLoading.value) return;
 
-    if (scrollTop <= 50 && readReq.page > 1) {
+    if (scrollTop.value <= 50 && readReq.page > 1) {
         if (minPage.value <= 1) {
             return;
         }
@@ -192,7 +208,7 @@ const onScroll = async () => {
         await getContent(true);
         return;
     }
-    if (scrollHeight - scrollTop - clientHeight <= 50 && !end.value) {
+    if (scrollHeight - scrollTop.value - clientHeight <= 50 && !end.value && !isEndOfFile.value) {
         if (readReq.page < maxPage.value) {
             readReq.page = maxPage.value;
             await getContent(false);
@@ -230,7 +246,7 @@ const changeTail = (fromOutSide: boolean) => {
 
 const clearLog = (): void => {
     logs.value = [];
-    startIndex.value = 0;
+    scrollTop.value = 0;
     readReq.page = 1;
     lastLogs.value = [];
 };
@@ -251,6 +267,7 @@ const getContent = async (pre: boolean) => {
     } catch (error) {
         isLoading.value = false;
         firstLoading.value = false;
+        return;
     }
 
     totalLines.value = res.data.totalLines;
@@ -319,24 +336,14 @@ const getContent = async (pre: boolean) => {
                         const addedLines = newLogs.length;
                         const newScrollPosition = lastScrollTop.value + addedLines * logHeight;
                         logContainer.value.scrollTop = newScrollPosition;
-                        startIndex.value = Math.max(
-                            0,
-                            Math.min(
-                                Math.floor(newScrollPosition / logHeight),
-                                Math.max(0, logs.value.length - visibleCount.value),
-                            ),
-                        );
                     }
                 } else {
-                    logContainer.value.scrollTop = totalHeight.value;
+                    logContainer.value.scrollTop = logContainer.value.scrollHeight;
-                    containerHeight.value = logContainer.value.getBoundingClientRect().height;
-                    startIndex.value = Math.max(0, logs.value.length - visibleCount.value);
                 }
             }
         });
     }
 
-    logCount.value = logs.value.length;
     end.value = res.data.end;
     totalPages.value = res.data.total;
     emit('update:hasContent', logs.value.length > 0);
@@ -344,6 +351,7 @@ const getContent = async (pre: boolean) => {
         readReq.page = res.data.total;
         readReq.latest = false;
         maxPage.value = res.data.total;
+        isEndOfFile.value = true;
         if (res.data.lines && res.data.lines.length > 500) {
             minPage.value = res.data.total - 1;
         } else {
@@ -354,19 +362,26 @@ const getContent = async (pre: boolean) => {
     }
     if (logs.value && logs.value.length > 3000) {
         const removedCount = readReq.pageSize;
+        const currentScrollRatio = scrollTop.value / (logs.value.length * logHeight);
+
         if (pre) {
             logs.value.splice(logs.value.length - removedCount, removedCount);
             if (maxPage.value > 1) {
                 maxPage.value--;
             }
         } else {
+            isEndOfFile.value = false;
             logs.value.splice(0, removedCount);
-            startIndex.value = Math.max(0, startIndex.value - removedCount);
+            nextTick(() => {
+                if (logContainer.value) {
+                    const newScrollTop = currentScrollRatio * (logs.value.length * logHeight);
+                    logContainer.value.scrollTop = Math.max(0, newScrollTop - removedCount * logHeight);
+                }
+            });
             if (minPage.value > 1) {
                 minPage.value++;
             }
         }
-        logCount.value = logs.value.length;
     }
     isLoading.value = false;
 };
@@ -396,16 +411,19 @@ watch(
 );
 
 const init = async () => {
+    tailLog.value = false;
     if (props.config.tail) {
         tailLog.value = props.config.tail;
-    } else {
-        tailLog.value = false;
     }
     if (tailLog.value) {
         changeTail(false);
     }
     readReq.latest = true;
     await getContent(false);
+    if (readReq.page > 1 && totalPages.value == maxPage.value) {
+        readReq.page--;
+        await getContent(true);
+    }
 };
 
 const containerStyle = computed(() => ({
@@ -425,16 +443,29 @@ onMounted(async () => {
     readReq.taskOperate = props.config.taskOperate;
     readReq.resourceID = props.config.resourceID;
     await init();
+
+    updateContainerHeight();
+
+    if (logContainer.value) {
+        resizeObserver = new ResizeObserver(() => {
+            updateContainerHeight();
+        });
+        resizeObserver.observe(logContainer.value);
+    }
+
     nextTick(() => {
         if (logContainer.value) {
-            logContainer.value.scrollTop = totalHeight.value;
+            logContainer.value.scrollTop = logContainer.value.scrollHeight;
-            containerHeight.value = logContainer.value.getBoundingClientRect().height;
         }
     });
 });
 
 onUnmounted(() => {
     onCloseLog();
+    if (resizeObserver && logContainer.value) {
+        resizeObserver.unobserve(logContainer.value);
+        resizeObserver.disconnect();
+    }
 });
 
 defineExpose({ changeTail, onDownload, clearLog });
@@ -453,13 +484,21 @@ defineExpose({ changeTail, onDownload, clearLog });
     width: 100%;
 }
 
-.log-item {
+.log-viewport {
     position: absolute;
+    top: 0;
+    left: 0;
+    right: 0;
+    will-change: transform;
+}
+
+.log-item {
     width: 100%;
     padding: 5px;
     color: #f5f5f5;
     box-sizing: border-box;
     white-space: nowrap;
+    overflow: hidden;
 }
 
 .log-content {