Limit standard output to last 1000 lines (#1063)

assets/js/hooks/scroll_on_update.js

@@ -1,3 +1,5 @@
+import { scrollToEnd } from "../lib/utils";
+
 /**
  * A hook used to scroll to the bottom of an element whenever it
  * receives LV update.
@@ -12,7 +14,7 @@ const ScrollOnUpdate = {
   },
 
   scroll() {
-    this.el.scrollTop = this.el.scrollHeight;
+    scrollToEnd(this.el);
   },
 };
 

assets/js/hooks/virtualized_lines.js

@@ -1,10 +1,16 @@
 import HyperList from "hyperlist";
 import {
+  getAttributeOrDefault,
   getAttributeOrThrow,
   parseBoolean,
   parseInteger,
 } from "../lib/attribute";
-import { findChildOrThrow, getLineHeight, isScrolledToEnd } from "../lib/utils";
+import {
+  findChildOrThrow,
+  getLineHeight,
+  isScrolledToEnd,
+  scrollToEnd,
+} from "../lib/utils";
 
 /**
  * A hook used to render text lines as a virtual list, so that only
@@ -16,7 +22,13 @@ import { findChildOrThrow, getLineHeight, isScrolledToEnd } from "../lib/utils";
  *     this height enables scrolling
  *
  *   * `data-follow` - whether to automatically scroll to the bottom as
- *     new lines appear
+ *     new lines appear. Defaults to false
+ *
+ *   * `data-max-lines` - the maximum number of lines to keep in the DOM.
+ *     By default all lines are kept
+ *
+ *   * `data-ignore-trailing-empty-line` - whether to ignore the last
+ *     line if it is empty. Defaults to false
  *
  * The element should have two children:
  *
@@ -34,32 +46,58 @@ const VirtualizedLines = {
     this.templateEl = findChildOrThrow(this.el, "[data-template]");
     this.contentEl = findChildOrThrow(this.el, "[data-content]");
 
+    this.capLines();
+
     const config = this.hyperListConfig();
     this.virtualizedList = new HyperList(this.contentEl, config);
+
+    if (this.props.follow) {
+      scrollToEnd(this.contentEl);
+    }
   },
 
   updated() {
     this.props = this.getProps();
 
-    const scrollToEnd = this.props.follow && isScrolledToEnd(this.contentEl);
+    this.capLines();
+
+    const shouldScrollToEnd =
+      this.props.follow && isScrolledToEnd(this.contentEl);
 
     const config = this.hyperListConfig();
     this.virtualizedList.refresh(this.contentEl, config);
 
-    if (scrollToEnd) {
+    if (shouldScrollToEnd) {
-      this.contentEl.scrollTop = this.contentEl.scrollHeight;
+      scrollToEnd(this.contentEl);
     }
   },
 
   getProps() {
     return {
       maxHeight: getAttributeOrThrow(this.el, "data-max-height", parseInteger),
-      follow: getAttributeOrThrow(this.el, "data-follow", parseBoolean),
+      follow: getAttributeOrDefault(
+        this.el,
+        "data-follow",
+        false,
+        parseBoolean
+      ),
+      maxLines: getAttributeOrDefault(
+        this.el,
+        "data-max-lines",
+        null,
+        parseInteger
+      ),
+      ignoreTrailingEmptyLine: getAttributeOrDefault(
+        this.el,
+        "data-ignore-trailing-empty-line",
+        false,
+        parseBoolean
+      ),
     };
   },
 
   hyperListConfig() {
-    const lineEls = this.templateEl.querySelectorAll("[data-line]");
+    const lineEls = this.getLineElements();
     const numberOfLines = lineEls.length;
 
     const height = Math.min(
@@ -89,6 +127,38 @@ const VirtualizedLines = {
       },
     };
   },
+
+  getLineElements() {
+    const lineEls = Array.from(this.templateEl.querySelectorAll("[data-line]"));
+
+    if (lineEls.length === 0) {
+      return [];
+    }
+
+    const lastLineEl = lineEls[lineEls.length - 1];
+
+    if (this.props.ignoreTrailingEmptyLine && lastLineEl.innerText === "") {
+      return lineEls.slice(0, -1);
+    } else {
+      return lineEls;
+    }
+  },
+
+  capLines() {
+    if (this.props.maxLines) {
+      const lineEls = Array.from(
+        this.templateEl.querySelectorAll("[data-line]")
+      );
+      const ignoredLineEls = lineEls.slice(0, -this.props.maxLines);
+
+      const [first, ...rest] = ignoredLineEls;
+      rest.forEach((lineEl) => lineEl.remove());
+
+      if (first) {
+        first.innerHTML = "...";
+      }
+    }
+  },
 };
 
 export default VirtualizedLines;

assets/js/lib/attribute.js

@@ -13,14 +13,15 @@ export function getAttributeOrThrow(element, attr, transform = null) {
 export function getAttributeOrDefault(
   element,
   attr,
-  defaultAttrVal,
+  defaultValue,
   transform = null
 ) {
-  const value = element.hasAttribute(attr)
+  if (element.hasAttribute(attr)) {
-    ? element.getAttribute(attr)
+    const value = element.getAttribute(attr);
-    : defaultAttrVal;
-
     return transform ? transform(value) : value;
+  } else {
+    return defaultValue;
+  }
 }
 
 export function parseBoolean(value) {

assets/js/lib/utils.js

@@ -66,6 +66,10 @@ export function isScrolledToEnd(element) {
   );
 }
 
+export function scrollToEnd(element) {
+  element.scrollTop = element.scrollHeight;
+}
+
 /**
  * Transforms a UTF8 string into base64 encoding.
  */

lib/livebook/notebook.ex

@@ -612,7 +612,7 @@ defmodule Livebook.Notebook do
   defp apply_frame_update(outputs, new_outputs, :append), do: new_outputs ++ outputs
 
   defp add_output([], {idx, {:stdout, text}}),
-    do: [{idx, {:stdout, Livebook.Utils.apply_rewind(text)}}]
+    do: [{idx, {:stdout, normalize_stdout(text)}}]
 
   defp add_output([], output), do: [output]
 
@@ -626,11 +626,28 @@ defmodule Livebook.Notebook do
 
   # Session server keeps all outputs, so we merge consecutive stdouts
   defp add_output([{idx, {:stdout, text}} | tail], {_idx, {:stdout, cont}}) do
-    [{idx, {:stdout, Livebook.Utils.apply_rewind(text <> cont)}} | tail]
+    [{idx, {:stdout, normalize_stdout(text <> cont)}} | tail]
   end
 
   defp add_output(outputs, output), do: [output | outputs]
 
+  @doc """
+  Normalizes a text chunk coming form the standard output.
+
+  Handles CR rawinds and caps output lines.
+  """
+  @spec normalize_stdout(String.t()) :: String.t()
+  def normalize_stdout(text) do
+    text
+    |> Livebook.Utils.apply_rewind()
+    |> Livebook.Utils.cap_lines(max_stdout_lines())
+  end
+
+  @doc """
+  The maximum desired number of lines of the standard output.
+  """
+  def max_stdout_lines(), do: 1_000
+
   @doc """
   Recursively adds index to all outputs, including frames.
   """

lib/livebook/utils.ex

@@ -360,13 +360,50 @@ defmodule Livebook.Utils do
       "Hola\r\nHey\r"
   """
   @spec apply_rewind(String.t()) :: String.t()
-  def apply_rewind(string) when is_binary(string) do
+  def apply_rewind(text) when is_binary(text) do
-    string
+    apply_rewind(text, "", "")
-    |> String.split("\n")
+  end
-    |> Enum.map(fn line ->
+
-      String.replace(line, ~r/^.*\r([^\r].*)$/, "\\1")
+  defp apply_rewind(<<?\n, rest::binary>>, acc, line),
-    end)
+    do: apply_rewind(rest, <<acc::binary, line::binary, ?\n>>, "")
-    |> Enum.join("\n")
+
+  defp apply_rewind(<<?\r, byte, rest::binary>>, acc, _line) when byte != ?\n,
+    do: apply_rewind(rest, acc, <<byte>>)
+
+  defp apply_rewind(<<byte, rest::binary>>, acc, line),
+    do: apply_rewind(rest, acc, <<line::binary, byte>>)
+
+  defp apply_rewind("", acc, line), do: acc <> line
+
+  @doc ~S"""
+  Limits `text` to last `max_lines`.
+
+  Replaces the removed lines with `"..."`.
+
+  ## Examples
+
+      iex> Livebook.Utils.cap_lines("Line 1\nLine 2\nLine 3\nLine 4", 2)
+      "...\nLine 3\nLine 4"
+
+      iex> Livebook.Utils.cap_lines("Line 1\nLine 2", 2)
+      "Line 1\nLine 2"
+
+      iex> Livebook.Utils.cap_lines("Line 1\nLine 2", 3)
+      "Line 1\nLine 2"
+  """
+  @spec cap_lines(String.t(), non_neg_integer()) :: String.t()
+  def cap_lines(text, max_lines) do
+    text
+    |> :binary.matches("\n")
+    |> Enum.at(-max_lines)
+    |> case do
+      nil ->
+        text
+
+      {pos, _len} ->
+        <<_ignore::binary-size(pos), rest::binary>> = text
+        "..." <> rest
+    end
   end
 
   @doc """

lib/livebook_web/live/output.ex

@@ -38,14 +38,14 @@ defmodule LivebookWeb.Output do
 
   defp render_output({:stdout, text}, %{id: id}) do
     text = if(text == :__pruned__, do: nil, else: text)
-    live_component(Output.StdoutComponent, id: id, text: text, follow: true)
+    live_component(Output.StdoutComponent, id: id, text: text)
   end
 
   defp render_output({:text, text}, %{id: id}) do
     assigns = %{id: id, text: text}
 
     ~H"""
-    <Output.TextComponent.render id={@id} content={@text} follow={false} />
+    <Output.TextComponent.render id={@id} content={@text} />
     """
   end
 

lib/livebook_web/live/output/stdout_component.ex

@@ -15,12 +15,7 @@ defmodule LivebookWeb.Output.StdoutComponent do
     if text do
       text = (socket.assigns.last_line || "") <> text
 
-      # Captured output usually has a trailing newline that we ignore
+      text = Livebook.Notebook.normalize_stdout(text)
-      # for HTML conversion, since each line is an HTML block anyway
-      trailing_newline? = String.ends_with?(text, "\n")
-      text = String.replace_suffix(text, "\n", "")
-
-      text = Livebook.Utils.apply_rewind(text)
 
       last_line =
         case Livebook.Utils.split_at_last_occurrence(text, "\n") do
@@ -28,8 +23,6 @@ defmodule LivebookWeb.Output.StdoutComponent do
           {:ok, _, last_line} -> last_line
         end
 
-      last_line = last_line <> if(trailing_newline?, do: "\n", else: "")
-
       {html_lines, modifiers} =
         LivebookWeb.Helpers.ANSI.ansi_string_to_html_lines_step(text, socket.assigns.modifiers)
 
@@ -54,7 +47,9 @@ defmodule LivebookWeb.Output.StdoutComponent do
       class="relative"
       phx-hook="VirtualizedLines"
       data-max-height="300"
-      data-follow={to_string(@follow)}>
+      data-follow="true"
+      data-max-lines={Livebook.Notebook.max_stdout_lines()}
+      data-ignore-trailing-empty-line="true">
       <%# Note 1: We add a newline to each element, so that multiple lines can be copied properly as element.textContent %>
       <%# Note 2: We use comments to avoid inserting unintended whitespace %>
       <div data-template class="hidden" id={"virtualized-text-#{@id}-template"}><%#

lib/livebook_web/live/output/text_component.ex

@@ -7,8 +7,7 @@ defmodule LivebookWeb.Output.TextComponent do
     <div id={"virtualized-text-#{@id}"}
       class="relative"
       phx-hook="VirtualizedLines"
-      data-max-height="300"
+      data-max-height="300">
-      data-follow={to_string(@follow)}>
       <%# Add a newline to each element, so that multiple lines can be copied properly %>
       <div data-template class="hidden"
         id={"virtualized-text-#{@id}-template"}