Update export flow for JS outputs (#2186)

lib/livebook/live_markdown/export.ex

@@ -7,9 +7,9 @@ defmodule Livebook.LiveMarkdown.Export do
     include_outputs? = Keyword.get(opts, :include_outputs, notebook.persist_outputs)
     include_stamp? = Keyword.get(opts, :include_stamp, true)
 
-    js_ref_with_data = if include_outputs?, do: collect_js_output_data(notebook), else: %{}
+    js_ref_with_export = if include_outputs?, do: collect_js_output_export(notebook), else: %{}
 
-    ctx = %{include_outputs?: include_outputs?, js_ref_with_data: js_ref_with_data}
+    ctx = %{include_outputs?: include_outputs?, js_ref_with_export: js_ref_with_export}
 
     iodata = render_notebook(notebook, ctx)
 
@@ -24,20 +24,42 @@ defmodule Livebook.LiveMarkdown.Export do
     {source, footer_warnings}
   end
 
-  defp collect_js_output_data(notebook) do
+  defp collect_js_output_export(notebook) do
-    for section <- notebook.sections,
+    for(
-        %{outputs: outputs} <- section.cells,
+      section <- notebook.sections,
-        {_idx, %{type: :js, js_view: %{ref: ref, pid: pid}, export: %{}}} <- outputs do
+      %{outputs: outputs} <- section.cells,
-      Task.async(fn ->
+      {_idx, %{type: :js, js_view: js_view, export: export}} <- outputs,
-        {ref, get_js_output_data(pid, ref)}
+      export == true or is_map(export),
-      end)
+      do: {js_view.ref, js_view.pid, export},
-    end
+      uniq: true
+    )
+    |> Enum.map(fn {ref, pid, export} ->
+      Task.async(fn -> {ref, get_js_output_export(pid, ref, export)} end)
+    end)
     |> Task.await_many(:infinity)
     |> Map.new()
   end
 
-  defp get_js_output_data(pid, ref) do
+  defp get_js_output_export(pid, ref, true) do
-    send(pid, {:connect, self(), %{origin: self(), ref: ref}})
+    send(pid, {:export, self(), %{ref: ref}})
+
+    monitor_ref = Process.monitor(pid)
+
+    data =
+      receive do
+        {:export_reply, export_result, %{ref: ^ref}} -> export_result
+        {:DOWN, ^monitor_ref, :process, _pid, _reason} -> nil
+      end
+
+    Process.demonitor(monitor_ref, [:flush])
+
+    data
+  end
+
+  # TODO: remove on Livebook v0.13
+  # Handle old flow for backward compatibility with Kino <= 0.10.0
+  defp get_js_output_export(pid, ref, %{info_string: info_string, key: key}) do
+    send(pid, {:connect, self(), %{origin: inspect(self()), ref: ref}})
 
     monitor_ref = Process.monitor(pid)
 
@@ -49,7 +71,10 @@ defmodule Livebook.LiveMarkdown.Export do
 
     Process.demonitor(monitor_ref, [:flush])
 
-    data
+    if data do
+      payload = if key && is_map(data), do: data[key], else: data
+      {info_string, payload}
+    end
   end
 
   defp render_notebook(notebook, ctx) do
@@ -221,23 +246,15 @@ defmodule Livebook.LiveMarkdown.Export do
     |> prepend_metadata(%{output: true})
   end
 
-  defp render_output(
+  defp render_output(%{type: :js, js_view: %{ref: ref}}, ctx) do
-         %{type: :js, export: %{info_string: info_string, key: key}, js_view: %{ref: ref}},
+    with {info_string, payload} <- ctx.js_ref_with_export[ref],
-         ctx
+         {:ok, binary} <- encode_js_data(payload) do
-       )
+      delimiter = MarkdownHelpers.code_block_delimiter(binary)
-       when is_binary(info_string) do
-    data = ctx.js_ref_with_data[ref]
-    payload = if key && is_map(data), do: data[key], else: data
 
-    case encode_js_data(payload) do
+      [delimiter, info_string, "\n", binary, "\n", delimiter]
-      {:ok, binary} ->
+      |> prepend_metadata(%{output: true})
-        delimiter = MarkdownHelpers.code_block_delimiter(binary)
+    else
-
+      _ -> :ignored
-        [delimiter, info_string, "\n", binary, "\n", delimiter]
-        |> prepend_metadata(%{output: true})
-
-      _ ->
-        :ignored
     end
   end
 

lib/livebook/runtime.ex

@@ -142,25 +142,24 @@ defprotocol Livebook.Runtime do
 
   ## Export
 
-  The `:export` map describes how the output should be persisted.
+  The `:export` specifies whether the given output supports persistence.
-  The output data is put in a Markdown fenced code block.
+  When enabled, the JS view server should handle the following message:
 
-    * `:info_string` - used as the info string for the Markdown
+      {:export, pid(), info :: %{ref: ref()}}
-      code block
 
-    * `:key` - in case the data is a map and only a specific part
+  And reply with:
-      should be exported
 
+      {:export_reply, export_result, info :: %{ref: ref()}}
+
+  Where `export_result` is a tuple `{info_string, payload}`.
+  `info_string` is used as the info string for the Markdown code block,
+  while `payload` is its content. Payload can be either a string,
+  otherwise it is serialized into JSON.
   """
   @type js_output() :: %{
           type: :js,
           js_view: js_view(),
-          export:
+          export: boolean()
-            nil
-            | %{
-                info_string: String.t(),
-                key: nil | term()
-              }
         }
 
   @typedoc """

lib/livebook/session.ex

@@ -2880,6 +2880,7 @@ defmodule Livebook.Session do
     %{type: :ignored}
   end
 
+  # Rewrite tuples to maps for backward compatibility with Kino <= 0.10.0
   defp normalize_runtime_output({:text, text}) do
     %{type: :terminal_text, text: text, chunk: false}
   end
@@ -2896,6 +2897,13 @@ defmodule Livebook.Session do
     %{type: :image, content: content, mime_type: mime_type}
   end
 
+  # Rewrite older output format for backward compatibility with Kino <= 0.5.2
+  defp normalize_runtime_output({:js, %{ref: ref, pid: pid, assets: assets, export: export}}) do
+    normalize_runtime_output(
+      {:js, %{js_view: %{ref: ref, pid: pid, assets: assets}, export: export}}
+    )
+  end
+
   defp normalize_runtime_output({:js, info}) do
     %{type: :js, js_view: info.js_view, export: info.export}
   end

lib/livebook/session/data.ex

@@ -1226,19 +1226,6 @@ defmodule Livebook.Session.Data do
     |> update_cell_eval_info!(cell.id, &%{&1 | status: :ready})
   end
 
-  # Rewrite older output format for backward compatibility with Kino <= 0.5.2
-  defp add_cell_output(
-         data_actions,
-         cell,
-         {:js, %{ref: ref, pid: pid, assets: assets, export: export}}
-       ) do
-    add_cell_output(
-      data_actions,
-      cell,
-      {:js, %{js_view: %{ref: ref, pid: pid, assets: assets}, export: export}}
-    )
-  end
-
   defp add_cell_output({data, _} = data_actions, cell, output) do
     {[indexed_output], _counter} = Notebook.index_outputs([output], 0)
 

test/livebook/live_markdown/export_test.exs

@@ -758,7 +758,7 @@ defmodule Livebook.LiveMarkdown.ExportTest do
       assert expected_document == document
     end
 
-    test "does not include js output with no export info" do
+    test "does not include js output with export disabled" do
       notebook = %{
         Notebook.new()
         | name: "My Notebook",
@@ -772,15 +772,15 @@ defmodule Livebook.LiveMarkdown.ExportTest do
                     | source: ":ok",
                       outputs: [
                         {0,
-                         {:js,
+                         %{
-                          %{
+                           type: :js,
-                            js_view: %{
+                           js_view: %{
-                              ref: "1",
+                             ref: "1",
-                              pid: spawn_widget_with_data("1", "data"),
+                             pid: spawn_widget_with_data("1", "data"),
-                              assets: %{archive_path: "", hash: "abcd", js_path: "main.js"}
+                             assets: %{archive_path: "", hash: "abcd", js_path: "main.js"}
-                            },
+                           },
-                            export: nil
+                           export: false
-                          }}}
+                         }}
                       ]
                   }
                 ]
@@ -803,7 +803,59 @@ defmodule Livebook.LiveMarkdown.ExportTest do
       assert expected_document == document
     end
 
-    test "includes js output if export info is set" do
+    test "includes js output with export enabled" do
+      notebook = %{
+        Notebook.new()
+        | name: "My Notebook",
+          sections: [
+            %{
+              Notebook.Section.new()
+              | name: "Section 1",
+                cells: [
+                  %{
+                    Notebook.Cell.new(:code)
+                    | source: ":ok",
+                      outputs: [
+                        {0,
+                         %{
+                           type: :js,
+                           js_view: %{
+                             ref: "1",
+                             pid: spawn_widget_with_export("1", {"mermaid", "graph TD;\nA-->B;"}),
+                             assets: %{archive_path: "", hash: "abcd", js_path: "main.js"}
+                           },
+                           export: true
+                         }}
+                      ]
+                  }
+                ]
+            }
+          ]
+      }
+
+      expected_document = """
+      # My Notebook
+
+      ## Section 1
+
+      ```elixir
+      :ok
+      ```
+
+      <!-- livebook:{"output":true} -->
+
+      ```mermaid
+      graph TD;
+      A-->B;
+      ```
+      """
+
+      {document, []} = Export.notebook_to_livemd(notebook, include_outputs: true)
+
+      assert expected_document == document
+    end
+
+    test "includes js output with legacy export info" do
       notebook = %{
         Notebook.new()
         | name: "My Notebook",
@@ -1439,6 +1491,15 @@ defmodule Livebook.LiveMarkdown.ExportTest do
     metadata
   end
 
+  defp spawn_widget_with_export(ref, export_result) do
+    spawn(fn ->
+      receive do
+        {:export, pid, %{ref: ^ref}} ->
+          send(pid, {:export_reply, export_result, %{ref: ref}})
+      end
+    end)
+  end
+
   defp spawn_widget_with_data(ref, data) do
     spawn(fn ->
       receive do