Keep inline Markdown unchanged when importing/exporting (#487)

* Merge: [WIP] Make Markdown formatter math-safe #447

* Keep inline Markdown unchanged when importing/exporting

lib/livebook/live_markdown/export.ex

@@ -79,7 +79,7 @@ defmodule Livebook.LiveMarkdown.Export do
   end
 
   defp render_cell(%Cell.Elixir{} = cell, ctx) do
-    delimiter = code_block_delimiter(cell.source)
+    delimiter = MarkdownHelpers.code_block_delimiter(cell.source)
     code = get_elixir_cell_code(cell)
     outputs = if ctx.include_outputs?, do: render_outputs(cell), else: []
 
@@ -131,13 +131,13 @@ defmodule Livebook.LiveMarkdown.Export do
 
   defp render_output(text) when is_binary(text) do
     text = String.replace_suffix(text, "\n", "")
-    delimiter = code_block_delimiter(text)
+    delimiter = MarkdownHelpers.code_block_delimiter(text)
     text = strip_ansi(text)
     [delimiter, "output\n", text, "\n", delimiter]
   end
 
   defp render_output({:text, text}) do
-    delimiter = code_block_delimiter(text)
+    delimiter = MarkdownHelpers.code_block_delimiter(text)
     text = strip_ansi(text)
     [delimiter, "output\n", text, "\n", delimiter]
   end
@@ -163,7 +163,7 @@ defmodule Livebook.LiveMarkdown.Export do
 
   defp format_markdown_source(markdown) do
     markdown
-    |> EarmarkParser.as_ast()
+    |> MarkdownHelpers.markdown_to_block_ast()
     |> elem(1)
     |> rewrite_ast()
     |> MarkdownHelpers.markdown_from_ast()
@@ -202,15 +202,6 @@ defmodule Livebook.LiveMarkdown.Export do
     end
   end
 
-  defp code_block_delimiter(code) do
-    max_streak =
-      Regex.scan(~r/`{3,}/, code)
-      |> Enum.map(fn [string] -> byte_size(string) end)
-      |> Enum.max(&>=/2, fn -> 2 end)
-
-    String.duplicate("`", max_streak + 1)
-  end
-
   defp put_unless_implicit(map, entries) do
     Enum.reduce(entries, map, fn {key, value}, map ->
       if value in [false, %{}] do

lib/livebook/live_markdown/import.ex

@@ -10,7 +10,7 @@ defmodule Livebook.LiveMarkdown.Import do
   """
   @spec notebook_from_markdown(String.t()) :: {Notebook.t(), list(String.t())}
   def notebook_from_markdown(markdown) do
-    {_, ast, earmark_messages} = EarmarkParser.as_ast(markdown)
+    {_, ast, earmark_messages} = MarkdownHelpers.markdown_to_block_ast(markdown)
     earmark_messages = Enum.map(earmark_messages, &earmark_message_to_string/1)
 
     {ast, rewrite_messages} = rewrite_ast(ast)
@@ -221,7 +221,7 @@ defmodule Livebook.LiveMarkdown.Import do
   end
 
   defp build_notebook([{:section_name, content} | elems], cells, sections) do
-    name = MarkdownHelpers.text_from_ast(content)
+    name = text_from_markdown(content)
     {metadata, elems} = grab_metadata(elems)
     attrs = section_metadata_to_attrs(metadata)
     section = %{Notebook.Section.new() | name: name, cells: cells} |> Map.merge(attrs)
@@ -242,7 +242,7 @@ defmodule Livebook.LiveMarkdown.Import do
   end
 
   defp build_notebook([{:notebook_name, content} | elems], [], sections) do
-    name = MarkdownHelpers.text_from_ast(content)
+    name = text_from_markdown(content)
     {metadata, []} = grab_metadata(elems)
     attrs = notebook_metadata_to_attrs(metadata)
     %{Notebook.new() | name: name, sections: sections} |> Map.merge(attrs)
@@ -253,6 +253,13 @@ defmodule Livebook.LiveMarkdown.Import do
     %{Notebook.new() | sections: sections}
   end
 
+  defp text_from_markdown(markdown) do
+    markdown
+    |> MarkdownHelpers.markdown_to_ast()
+    |> elem(1)
+    |> MarkdownHelpers.text_from_ast()
+  end
+
   # Takes optional leading metadata JSON object and returns {metadata, rest}.
   defp grab_metadata([{:metadata, metadata} | elems]) do
     {metadata, elems}

lib/livebook/live_markdown/markdown_helpers.ex

@@ -1,13 +1,32 @@
 defmodule Livebook.LiveMarkdown.MarkdownHelpers do
   @moduledoc false
 
+  @doc """
+  Wraps `EarmarkParser.as_ast/2`.
+  """
+  @spec markdown_to_ast(String.t()) :: {:ok | :error, EarmarkParser.ast(), list()}
+  def markdown_to_ast(markdown) do
+    EarmarkParser.as_ast(markdown)
+  end
+
+  @doc """
+  Wraps `EarmarkParser.as_ast/2`.
+
+  Markdown blocks are parsed into the AST, while inline
+  content is kept as is.
+  """
+  @spec markdown_to_block_ast(String.t()) :: {:ok | :error, EarmarkParser.ast(), list()}
+  def markdown_to_block_ast(markdown) do
+    EarmarkParser.as_ast(markdown, parse_inline: false)
+  end
+
   @doc """
   Reformats the given markdown document.
   """
   @spec reformat(String.t()) :: String.t()
   def reformat(markdown) do
     markdown
-    |> EarmarkParser.as_ast()
+    |> markdown_to_block_ast()
     |> elem(1)
     |> markdown_from_ast()
   end
@@ -27,6 +46,20 @@ defmodule Livebook.LiveMarkdown.MarkdownHelpers do
   def text_from_ast(ast) when is_binary(ast), do: ast
   def text_from_ast({_, _, ast, _}), do: text_from_ast(ast)
 
+  @doc """
+  Determines suitable Markdown fence delimiter for the
+  given code.
+  """
+  @spec code_block_delimiter(String.t()) :: String.t()
+  def code_block_delimiter(code) do
+    max_streak =
+      Regex.scan(~r/`{3,}/, code)
+      |> Enum.map(fn [string] -> byte_size(string) end)
+      |> Enum.max(&>=/2, fn -> 2 end)
+
+    String.duplicate("`", max_streak + 1)
+  end
+
   @doc """
   Renders Markdown string from the given `EarmarkParser` AST.
   """
@@ -161,12 +194,12 @@ defmodule Livebook.LiveMarkdown.MarkdownHelpers do
   @void_elements ~W(area base br col command embed hr img input keygen link meta param source track wbr)
 
   defp render_html(tag, attrs, []) when tag in @void_elements do
-    ["<", tag, " ", attrs_to_string(attrs), " />"]
+    ["<", tag, attrs_to_string(attrs), " />"]
   end
 
   defp render_html(tag, attrs, lines) do
     inner = Enum.intersperse(lines, "\n")
-    ["<", tag, " ", attrs_to_string(attrs), ">\n", inner, "\n</", tag, ">"]
+    ["<", tag, attrs_to_string(attrs), ">\n", inner, "\n</", tag, ">"]
   end
 
   defp render_emphasis(content) do
@@ -243,8 +276,9 @@ defmodule Livebook.LiveMarkdown.MarkdownHelpers do
   end
 
   defp render_code_block(content, attrs) do
+    delimiter = code_block_delimiter(content)
     language = get_attr(attrs, "class", "")
-    ["```", language, "\n", content, "\n```"]
+    [delimiter, language, "\n", content, "\n", delimiter]
   end
 
   defp render_blockquote(content) do
@@ -376,9 +410,7 @@ defmodule Livebook.LiveMarkdown.MarkdownHelpers do
   end
 
   defp attrs_to_string(attrs) do
-    attrs
-    |> Enum.map(fn {key, value} -> ~s/#{key}="#{value}"/ end)
-    |> Enum.join(" ")
+    Enum.map(attrs, fn {key, value} -> ~s/ #{key}="#{value}"/ end)
   end
 
   defp blank?(string), do: String.trim(string) == ""

mix.exs

@@ -47,7 +47,9 @@ defmodule Livebook.MixProject do
       {:telemetry_poller, "~> 0.4"},
       {:jason, "~> 1.0"},
       {:plug_cowboy, "~> 2.0"},
-      {:earmark_parser, "~> 1.4"},
+      # {:earmark_parser, "~> 1.4"},
+      {:earmark_parser, "~> 1.4",
+       github: "jonatanklosko/earmark_parser", branch: "jk-optional-inline"},
       {:bypass, "~> 2.1", only: :test},
       {:castore, "~> 0.1.0"}
     ]

mix.lock

@@ -4,7 +4,7 @@
   "cowboy": {:hex, :cowboy, "2.9.0", "865dd8b6607e14cf03282e10e934023a1bd8be6f6bacf921a7e2a96d800cd452", [:make, :rebar3], [{:cowlib, "2.11.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "2c729f934b4e1aa149aff882f57c6372c15399a20d54f65c8d67bef583021bde"},
   "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.3.1", "ebd1a1d7aff97f27c66654e78ece187abdc646992714164380d8a041eda16754", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3a6efd3366130eab84ca372cbd4a7d3c3a97bdfcfb4911233b035d117063f0af"},
   "cowlib": {:hex, :cowlib, "2.11.0", "0b9ff9c346629256c42ebe1eeb769a83c6cb771a6ee5960bd110ab0b9b872063", [:make, :rebar3], [], "hexpm", "2b3e9da0b21c4565751a6d4901c20d1b4cc25cbb7fd50d91d2ab6dd287bc86a9"},
-  "earmark_parser": {:hex, :earmark_parser, "1.4.13", "0c98163e7d04a15feb62000e1a891489feb29f3d10cb57d4f845c405852bbef8", [:mix], [], "hexpm", "d602c26af3a0af43d2f2645613f65841657ad6efc9f0e361c3b6c06b578214ba"},
+  "earmark_parser": {:git, "https://github.com/jonatanklosko/earmark_parser.git", "288ed586d45825a28b824d1cfd42fcd0959cd628", [branch: "jk-optional-inline"]},
   "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"},
   "floki": {:hex, :floki, "0.31.0", "f05ee8a8e6a3ced4e62beeb2c79a63bc8e12ab98fbaaf6e6a3d9b76b1278e23f", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "b05afa372f5c345a5bf240ac25ea1f0f3d5fcfd7490ac0beeb4a203f9444891e"},
   "html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"},

test/livebook/live_markdown/export_test.exs

@@ -20,7 +20,9 @@ defmodule Livebook.LiveMarkdown.ExportTest do
 
                     * Erlang
                     * Elixir
-                    * PostgreSQL\
+                    * PostgreSQL
+
+                    $x_{i} + y_{i}$\
                     """
                 },
                 %{
@@ -92,6 +94,8 @@ defmodule Livebook.LiveMarkdown.ExportTest do
     * Elixir
     * PostgreSQL
 
+    $x_{i} + y_{i}$
+
     <!-- livebook:{"disable_formatting":true} -->
 
     ```elixir

test/livebook/live_markdown/import_test.exs

@@ -17,6 +17,8 @@ defmodule Livebook.LiveMarkdown.ImportTest do
     * Elixir
     * PostgreSQL
 
+    $x_{i} + y_{i}$
+
     <!-- livebook:{"disable_formatting": true} -->
 
     ```elixir
@@ -60,7 +62,9 @@ defmodule Livebook.LiveMarkdown.ImportTest do
 
                      * Erlang
                      * Elixir
-                     * PostgreSQL\
+                     * PostgreSQL
+
+                     $x_{i} + y_{i}$\
                      """
                    },
                    %Cell.Elixir{
@@ -120,9 +124,6 @@ defmodule Livebook.LiveMarkdown.ImportTest do
 
     ## Section 1
 
-    Line 1.\s\s
-    Line 2.
-
     |State|Abbrev|Capital|
     | --: | :-: | --- |
     | Texas | TX | Austin |
@@ -139,9 +140,6 @@ defmodule Livebook.LiveMarkdown.ImportTest do
                  cells: [
                    %Cell.Markdown{
                      source: """
-                     Line 1.\\
-                     Line 2.
-
                      | State | Abbrev | Capital |
                      | ----: | :----: | ------- |
                      | Texas | TX     | Austin  |

test/livebook/live_markdown/markdown_helpers_test.exs

@@ -9,6 +9,16 @@ defmodule Livebook.LiveMarkdown.MarkdownHelpersTest do
       assert markdown == reformat(markdown)
     end
 
+    test "other emphasis" do
+      markdown = "The Game, _Mrs Hudson_, is on!"
+      assert markdown == reformat(markdown)
+    end
+
+    test "nested emphasis" do
+      markdown = "The *Game, _Mrs Hudson_, is* on!"
+      assert markdown == reformat(markdown)
+    end
+
     test "bold" do
       markdown = "The Game, **Mrs Hudson**, is on!"
       assert markdown == reformat(markdown)
@@ -24,16 +34,50 @@ defmodule Livebook.LiveMarkdown.MarkdownHelpersTest do
       assert markdown == reformat(markdown)
     end
 
+    test "inline code with a line break" do
+      markdown = "The Game, `Mrs\nHudson`, is on!"
+      assert markdown == reformat(markdown)
+    end
+
+    test "inline code with extra spaces" do
+      markdown = "The Game, `Mrs Huds  on`, is on!"
+      assert markdown == reformat(markdown)
+    end
+
     test "combined" do
       markdown = "The Game, ~~***`Mrs Hudson`***~~, is on!"
       assert markdown == reformat(markdown)
     end
 
+    test "inline math" do
+      markdown = "The Game, $x_{i} + y_{i}$, is on!"
+
+      assert markdown == reformat(markdown)
+    end
+
     test "link" do
       markdown = "The Game, [Mrs Hudson](https://youtu.be/M-KqaO1oH2E), is on!"
       assert markdown == reformat(markdown)
     end
 
+    test "link with IAL" do
+      markdown = "[link](url){: .classy}"
+
+      assert markdown == reformat(markdown)
+    end
+
+    test "link followed by escaped IAL" do
+      markdown = "[link](url)\\{: .classy}"
+
+      assert markdown == reformat(markdown)
+    end
+
+    test "autolink" do
+      markdown = "<https://elixir-lang.com>"
+
+      assert markdown == reformat(markdown)
+    end
+
     test "basic image" do
       markdown = "The Game, ![Mrs Hudson](https://example.com), is on!"
       assert markdown == reformat(markdown)
@@ -49,6 +93,11 @@ defmodule Livebook.LiveMarkdown.MarkdownHelpersTest do
       assert markdown == reformat(markdown)
     end
 
+    test "line break" do
+      markdown = "Line 1.\\\nLine 2."
+      assert markdown == reformat(markdown)
+    end
+
     test "oneline comment" do
       markdown = "<!-- The Game, Mrs Hudson, is on! -->"
       assert markdown == reformat(markdown)
@@ -108,6 +157,32 @@ defmodule Livebook.LiveMarkdown.MarkdownHelpersTest do
       assert markdown == reformat(markdown)
     end
 
+    test "code block with fences inside it" do
+      markdown = """
+      ````elixir
+      before
+
+      ```
+      _inside_
+      ```
+
+      after
+      ````\
+      """
+
+      assert markdown == reformat(markdown)
+    end
+
+    test "display math" do
+      markdown = """
+      $$
+      R_{ij}^{kl} = R_{ij} - \Gamma^k_{kl}
+      $$\
+      """
+
+      assert markdown == reformat(markdown)
+    end
+
     test "blockquote" do
       markdown = """
       > The Game, Mrs Hudson,
@@ -243,6 +318,18 @@ defmodule Livebook.LiveMarkdown.MarkdownHelpersTest do
       assert markdown == reformat(markdown)
     end
 
+    test "surprise ordered list" do
+      markdown = "1986\\. What a great season."
+
+      assert markdown == reformat(markdown)
+    end
+
+    test "escaped Markdown" do
+      markdown = "not a \\[link\\]()"
+
+      assert markdown == reformat(markdown)
+    end
+
     test "raw html" do
       markdown = """
       <div class="box" aria-label="box">
@@ -253,6 +340,18 @@ defmodule Livebook.LiveMarkdown.MarkdownHelpersTest do
       assert markdown == reformat(markdown)
     end
 
+    test "raw html at the beginning of a line" do
+      markdown = "line\n\n<hr />"
+
+      assert markdown == reformat(markdown)
+    end
+
+    test "raw html not at the beginning of a line" do
+      markdown = "line\n  <hr />"
+
+      assert markdown == reformat(markdown)
+    end
+
     test "separates blocks with a single line" do
       markdown = """
       The first paragraph,
@@ -274,7 +373,9 @@ defmodule Livebook.LiveMarkdown.MarkdownHelpersTest do
     # By reformatting we can assert correct rendering
     # by comparing against the original content.
     defp reformat(markdown) do
-      {:ok, ast, []} = EarmarkParser.as_ast(markdown)
+      # Note: we don't parse inline content, so some of the tests
+      # above are not stricly necessary, but we keep them for completeness.
+      {:ok, ast, []} = MarkdownHelpers.markdown_to_block_ast(markdown)
       MarkdownHelpers.markdown_from_ast(ast)
     end
   end