Markdown snippets (#56)

* Extend LiveMarkdown format to support Elixir snippets in Markdown cells

* Highlight Markdown code blocks using Monaco editor API

* Use livebook metadata for forcing markdown as well

assets/css/components.css

@@ -71,7 +71,7 @@
   @apply hidden;
 }
 
-.radio-button .radio-button__input[checked] +  .radio-button__label {
+.radio-button .radio-button__input[checked] + .radio-button__label {
   @apply bg-gray-700 text-white;
 }
 

assets/js/cell/live_editor/elixir/monarch_language.js

@@ -198,8 +198,8 @@ const ElixirMonarchLanguage = {
             "@declarationKeywords": "keyword.declaration",
             "@namespaceKeywords": "keyword",
             "@otherKeywords": "keyword",
-            "@default": "function.call"
+            "@default": "function.call",
-          }
+          },
         },
       ],
       [

assets/js/cell/markdown.js

@@ -1,6 +1,23 @@
 import marked from "marked";
 import morphdom from "morphdom";
 import DOMPurify from "dompurify";
+import monaco from "./live_editor/monaco";
+
+// Reuse Monaco highlighter for Markdown code blocks
+marked.setOptions({
+  highlight: (code, lang, callback) => {
+    monaco.editor
+      .colorize(code, lang)
+      .then((result) => {
+        // `colorize` always adds additional newline, so we remove it
+        result = result.replace(/<br\/>$/, "");
+        callback(null, result);
+      })
+      .catch((error) => {
+        callback(error, null);
+      });
+  },
+});
 
 /**
  * Renders markdown content in the given container.
@@ -19,27 +36,31 @@ class Markdown {
   }
 
   __render() {
-    const html = this.__getHtml();
+    this.__getHtml().then((html) => {
-    // Wrap the HTML in another element, so that we
+      // Wrap the HTML in another element, so that we
-    // can use morphdom's childrenOnly option.
+      // can use morphdom's childrenOnly option.
-    const wrappedHtml = `<div>${html}</div>`;
+      const wrappedHtml = `<div>${html}</div>`;
 
-    morphdom(this.container, wrappedHtml, { childrenOnly: true });
+      morphdom(this.container, wrappedHtml, { childrenOnly: true });
+    });
   }
 
   __getHtml() {
-    const html = marked(this.content);
+    return new Promise((resolve, reject) => {
-    const sanitizedHtml = DOMPurify.sanitize(html);
+      marked(this.content, (error, html) => {
+        const sanitizedHtml = DOMPurify.sanitize(html);
 
-    if (sanitizedHtml) {
+        if (sanitizedHtml) {
-      return sanitizedHtml;
+          resolve(sanitizedHtml);
-    } else {
+        } else {
-      return `
+          resolve(`
-        <div class="text-gray-300">
+            <div class="text-gray-300">
-          Empty markdown cell
+              Empty markdown cell
-        </div>
+            </div>
-      `;
+          `);
-    }
+        }
+      });
+    });
   }
 }
 

lib/live_book/live_markdown.ex

@@ -15,8 +15,10 @@ defmodule LiveBook.LiveMarkdown do
   # 2. Every *Heading 2* starts a new section.
   # 3. Every Elixir code block maps to an Elixir cell.
   # 4. Adjacent regular Markdown text maps to a Markdown cell.
-  # 5. Comments of the form `<!--live_book:json_object-->` hold metadata
+  # 5. Comments of the form `<!-- livebook:json_object -->` hold metadata
   #    any apply to the element they directly precede (e.g. an Elixir cell).
+  #    Such comments may appear anywhere, for instance `<!-- livebook:{"force_markdown":true} -->`
+  #    forces the next Markdown block to be treated as prat of Markdown cell (even if it's Elixir code block).
   #
   # ## Example
   #
@@ -32,7 +34,8 @@ defmodule LiveBook.LiveMarkdown do
   #     * Elixir
   #     * PostgreSQL
   #
-  #     <!--live_book:{"readonly":true}-->
+  #     <!-- livebook:{"readonly":true} -->
+  #
   #     ```elixir
   #     Enum.to_list(1..10)
   #     ```

lib/live_book/live_markdown/export.ex

@@ -47,14 +47,14 @@ defmodule LiveBook.LiveMarkdown.Export do
 
   defp render_metadata(metadata) do
     metadata_json = Jason.encode!(metadata)
-    "<!--live_book:#{metadata_json}-->"
+    "<!-- livebook:#{metadata_json} -->"
   end
 
   defp prepend_metadata(iodata, metadata) when metadata == %{}, do: iodata
 
   defp prepend_metadata(iodata, metadata) do
     content = render_metadata(metadata)
-    [content, "\n", iodata]
+    [content, "\n\n", iodata]
   end
 
   defp format_markdown_source(markdown) do
@@ -69,6 +69,7 @@ defmodule LiveBook.LiveMarkdown.Export do
   defp rewrite_ast(ast) do
     ast
     |> remove_reserved_headings()
+    |> add_markdown_annotation_before_elixir_block()
   end
 
   defp remove_reserved_headings(ast) do
@@ -78,4 +79,14 @@ defmodule LiveBook.LiveMarkdown.Export do
       _ast_node -> true
     end)
   end
+
+  defp add_markdown_annotation_before_elixir_block(ast) do
+    Enum.flat_map(ast, fn
+      {"pre", _, [{"code", [{"class", "elixir"}], [_source], %{}}], %{}} = ast_node ->
+        [{:comment, [], [~s/livebook:{"force_markdown":true}/], %{comment: true}}, ast_node]
+
+      ast_node ->
+        [ast_node]
+    end)
+  end
 end

lib/live_book/live_markdown/import.ex

@@ -32,6 +32,7 @@ defmodule LiveBook.LiveMarkdown.Import do
   defp rewrite_ast(ast) do
     {ast, messages1} = rewrite_multiple_primary_headings(ast)
     {ast, messages2} = move_primary_heading_top(ast)
+    ast = trim_comments(ast)
 
     {ast, messages1 ++ messages2}
   end
@@ -92,6 +93,18 @@ defmodule LiveBook.LiveMarkdown.Import do
     {Enum.reverse(left_rev), Enum.reverse(right_rev)}
   end
 
+  # Trims one-line comments to allow nice pattern matching
+  # on LiveBook-specific annotations with no regard to surrounding whitespace.
+  defp trim_comments(ast) do
+    Enum.map(ast, fn
+      {:comment, attrs, [line], %{comment: true}} ->
+        {:comment, attrs, [String.trim(line)], %{comment: true}}
+
+      ast_node ->
+        ast_node
+    end)
+  end
+
   # Builds a list of classified elements from the AST.
   defp group_elements(ast, elems \\ [])
 
@@ -105,8 +118,30 @@ defmodule LiveBook.LiveMarkdown.Import do
     group_elements(ast, [{:section_name, content} | elems])
   end
 
+  # The <!-- livebook:{"force_markdown":true} --> annotation forces the next node
+  # to be interpreted as Markdown cell content.
   defp group_elements(
-         [{:comment, _, ["live_book:" <> metadata_json], %{comment: true}} | ast],
+         [
+           {:comment, _, [~s/livebook:{"force_markdown":true}/], %{comment: true}},
+           ast_node | ast
+         ],
+         [{:cell, :markdown, md_ast} | rest]
+       ) do
+    group_elements(ast, [{:cell, :markdown, [ast_node | md_ast]} | rest])
+  end
+
+  defp group_elements(
+         [
+           {:comment, _, [~s/livebook:{"force_markdown":true}/], %{comment: true}},
+           ast_node | ast
+         ],
+         elems
+       ) do
+    group_elements(ast, [{:cell, :markdown, [ast_node]} | elems])
+  end
+
+  defp group_elements(
+         [{:comment, _, ["livebook:" <> metadata_json], %{comment: true}} | ast],
          elems
        ) do
     group_elements(ast, [{:metadata, metadata_json} | elems])

lib/live_book/live_markdown/markdown_helpers.ex

@@ -89,7 +89,7 @@ defmodule LiveBook.LiveMarkdown.MarkdownHelpers do
 
   defp build_md(iodata, [{:comment, _, lines, %{comment: true}} | ast]) do
     render_comment(lines)
-    |> append_inline(iodata)
+    |> append_block(iodata)
     |> build_md(ast)
   end
 

test/live_book/live_markdown/export_test.exs

@@ -60,20 +60,24 @@ defmodule LiveBook.LiveMarkdown.ExportTest do
     }
 
     expected_document = """
-    <!--live_book:{"author":"Sherlock Holmes"}-->
+    <!-- livebook:{"author":"Sherlock Holmes"} -->
+
     # My Notebook
 
-    <!--live_book:{"created_at":"2021-02-15"}-->
+    <!-- livebook:{"created_at":"2021-02-15"} -->
+
     ## Section 1
 
-    <!--live_book:{"updated_at":"2021-02-15"}-->
+    <!-- livebook:{"updated_at":"2021-02-15"} -->
+
     Make sure to install:
 
     * Erlang
     * Elixir
     * PostgreSQL
 
-    <!--live_book:{"readonly":true}-->
+    <!-- livebook:{"readonly":true} -->
+
     ```elixir
     Enum.to_list(1..10)
     ```
@@ -221,4 +225,74 @@ defmodule LiveBook.LiveMarkdown.ExportTest do
 
     assert expected_document == document
   end
+
+  test "marks elixir snippets in markdown cells as such" do
+    notebook = %{
+      Notebook.new()
+      | name: "My Notebook",
+        metadata: %{},
+        sections: [
+          %{
+            Notebook.Section.new()
+            | name: "Section 1",
+              metadata: %{},
+              cells: [
+                %{
+                  Notebook.Cell.new(:markdown)
+                  | metadata: %{},
+                    source: """
+                    ```elixir
+                    [1, 2, 3]
+                    ```\
+                    """
+                }
+              ]
+          },
+          %{
+            Notebook.Section.new()
+            | name: "Section 2",
+              metadata: %{},
+              cells: [
+                %{
+                  Notebook.Cell.new(:markdown)
+                  | metadata: %{},
+                    source: """
+                    Some markdown.
+
+                    ```elixir
+                    [1, 2, 3]
+                    ```\
+                    """
+                }
+              ]
+          }
+        ]
+    }
+
+    expected_document = """
+    # My Notebook
+
+    ## Section 1
+
+    <!-- livebook:{"force_markdown":true} -->
+
+    ```elixir
+    [1, 2, 3]
+    ```
+
+    ## Section 2
+
+    Some markdown.
+
+    <!-- livebook:{"force_markdown":true} -->
+
+    ```elixir
+    [1, 2, 3]
+    ```
+    """
+
+    document = Export.notebook_to_markdown(notebook)
+
+    assert expected_document == document
+  end
 end

test/live_book/live_markdown/import_test.exs

@@ -6,20 +6,24 @@ defmodule LiveBook.LiveMarkdown.ImportTest do
 
   test "acceptance" do
     markdown = """
-    <!--live_book:{"author":"Sherlock Holmes"}-->
+    <!-- livebook:{"author":"Sherlock Holmes"} -->
+
     # My Notebook
 
-    <!--live_book:{"created_at":"2021-02-15"}-->
+    <!-- livebook:{"created_at":"2021-02-15"} -->
+
     ## Section 1
 
-    <!--live_book:{"updated_at":"2021-02-15"}-->
+    <!-- livebook:{"updated_at":"2021-02-15"} -->
+
     Make sure to install:
 
     * Erlang
     * Elixir
     * PostgreSQL
 
-    <!--live_book:{"readonly":true}-->
+    <!-- livebook:{"readonly":true} -->
+
     ```elixir
     Enum.to_list(1..10)
     ```
@@ -276,7 +280,8 @@ defmodule LiveBook.LiveMarkdown.ImportTest do
     markdown = """
     Cool notebook.
 
-    <!--live_book:{"author":"Sherlock Holmes"}-->
+    <!-- livebook:{"author":"Sherlock Holmes"} -->
+
     # My Notebook
 
     Some markdown.
@@ -375,4 +380,64 @@ defmodule LiveBook.LiveMarkdown.ImportTest do
              ]
            } = notebook
   end
+
+  test "imports elixir snippets as part of markdown cells if marked as such" do
+    markdown = """
+    # My Notebook
+
+    ## Section 1
+
+    <!-- livebook:{"force_markdown":true} -->
+
+    ```elixir
+    [1, 2, 3]
+    ```
+
+    ## Section 2
+
+    Some markdown.
+
+    <!-- livebook:{"force_markdown":true} -->
+
+    ```elixir
+    [1, 2, 3]
+    ```
+    """
+
+    {notebook, []} = Import.notebook_from_markdown(markdown)
+
+    assert %Notebook{
+             name: "My Notebook",
+             sections: [
+               %Notebook.Section{
+                 name: "Section 1",
+                 cells: [
+                   %Notebook.Cell{
+                     type: :markdown,
+                     source: """
+                     ```elixir
+                     [1, 2, 3]
+                     ```\
+                     """
+                   }
+                 ]
+               },
+               %Notebook.Section{
+                 name: "Section 2",
+                 cells: [
+                   %Notebook.Cell{
+                     type: :markdown,
+                     source: """
+                     Some markdown.
+
+                     ```elixir
+                     [1, 2, 3]
+                     ```\
+                     """
+                   }
+                 ]
+               }
+             ]
+           } = notebook
+  end
 end