diff --git a/lib/livebook/live_markdown/export.ex b/lib/livebook/live_markdown/export.ex index 56d28da4e..06d2c2f65 100644 --- a/lib/livebook/live_markdown/export.ex +++ b/lib/livebook/live_markdown/export.ex @@ -79,7 +79,7 @@ defmodule Livebook.LiveMarkdown.Export do end defp render_cell(%Cell.Elixir{} = cell, ctx) do - delimiter = code_block_delimiter(cell.source) + delimiter = MarkdownHelpers.code_block_delimiter(cell.source) code = get_elixir_cell_code(cell) outputs = if ctx.include_outputs?, do: render_outputs(cell), else: [] @@ -131,13 +131,13 @@ defmodule Livebook.LiveMarkdown.Export do defp render_output(text) when is_binary(text) do text = String.replace_suffix(text, "\n", "") - delimiter = code_block_delimiter(text) + delimiter = MarkdownHelpers.code_block_delimiter(text) text = strip_ansi(text) [delimiter, "output\n", text, "\n", delimiter] end defp render_output({:text, text}) do - delimiter = code_block_delimiter(text) + delimiter = MarkdownHelpers.code_block_delimiter(text) text = strip_ansi(text) [delimiter, "output\n", text, "\n", delimiter] end @@ -163,7 +163,7 @@ defmodule Livebook.LiveMarkdown.Export do defp format_markdown_source(markdown) do markdown - |> EarmarkParser.as_ast() + |> MarkdownHelpers.markdown_to_block_ast() |> elem(1) |> rewrite_ast() |> MarkdownHelpers.markdown_from_ast() @@ -202,15 +202,6 @@ defmodule Livebook.LiveMarkdown.Export do end end - defp code_block_delimiter(code) do - max_streak = - Regex.scan(~r/`{3,}/, code) - |> Enum.map(fn [string] -> byte_size(string) end) - |> Enum.max(&>=/2, fn -> 2 end) - - String.duplicate("`", max_streak + 1) - end - defp put_unless_implicit(map, entries) do Enum.reduce(entries, map, fn {key, value}, map -> if value in [false, %{}] do diff --git a/lib/livebook/live_markdown/import.ex b/lib/livebook/live_markdown/import.ex index d3462a3b1..2fe4a4b13 100644 --- a/lib/livebook/live_markdown/import.ex +++ b/lib/livebook/live_markdown/import.ex @@ -10,7 +10,7 @@ defmodule Livebook.LiveMarkdown.Import do """ @spec notebook_from_markdown(String.t()) :: {Notebook.t(), list(String.t())} def notebook_from_markdown(markdown) do - {_, ast, earmark_messages} = EarmarkParser.as_ast(markdown) + {_, ast, earmark_messages} = MarkdownHelpers.markdown_to_block_ast(markdown) earmark_messages = Enum.map(earmark_messages, &earmark_message_to_string/1) {ast, rewrite_messages} = rewrite_ast(ast) @@ -221,7 +221,7 @@ defmodule Livebook.LiveMarkdown.Import do end defp build_notebook([{:section_name, content} | elems], cells, sections) do - name = MarkdownHelpers.text_from_ast(content) + name = text_from_markdown(content) {metadata, elems} = grab_metadata(elems) attrs = section_metadata_to_attrs(metadata) section = %{Notebook.Section.new() | name: name, cells: cells} |> Map.merge(attrs) @@ -242,7 +242,7 @@ defmodule Livebook.LiveMarkdown.Import do end defp build_notebook([{:notebook_name, content} | elems], [], sections) do - name = MarkdownHelpers.text_from_ast(content) + name = text_from_markdown(content) {metadata, []} = grab_metadata(elems) attrs = notebook_metadata_to_attrs(metadata) %{Notebook.new() | name: name, sections: sections} |> Map.merge(attrs) @@ -253,6 +253,13 @@ defmodule Livebook.LiveMarkdown.Import do %{Notebook.new() | sections: sections} end + defp text_from_markdown(markdown) do + markdown + |> MarkdownHelpers.markdown_to_ast() + |> elem(1) + |> MarkdownHelpers.text_from_ast() + end + # Takes optional leading metadata JSON object and returns {metadata, rest}. defp grab_metadata([{:metadata, metadata} | elems]) do {metadata, elems} diff --git a/lib/livebook/live_markdown/markdown_helpers.ex b/lib/livebook/live_markdown/markdown_helpers.ex index 20b1f5750..f019b5d54 100644 --- a/lib/livebook/live_markdown/markdown_helpers.ex +++ b/lib/livebook/live_markdown/markdown_helpers.ex @@ -1,13 +1,32 @@ defmodule Livebook.LiveMarkdown.MarkdownHelpers do @moduledoc false + @doc """ + Wraps `EarmarkParser.as_ast/2`. + """ + @spec markdown_to_ast(String.t()) :: {:ok | :error, EarmarkParser.ast(), list()} + def markdown_to_ast(markdown) do + EarmarkParser.as_ast(markdown) + end + + @doc """ + Wraps `EarmarkParser.as_ast/2`. + + Markdown blocks are parsed into the AST, while inline + content is kept as is. + """ + @spec markdown_to_block_ast(String.t()) :: {:ok | :error, EarmarkParser.ast(), list()} + def markdown_to_block_ast(markdown) do + EarmarkParser.as_ast(markdown, parse_inline: false) + end + @doc """ Reformats the given markdown document. """ @spec reformat(String.t()) :: String.t() def reformat(markdown) do markdown - |> EarmarkParser.as_ast() + |> markdown_to_block_ast() |> elem(1) |> markdown_from_ast() end @@ -27,6 +46,20 @@ defmodule Livebook.LiveMarkdown.MarkdownHelpers do def text_from_ast(ast) when is_binary(ast), do: ast def text_from_ast({_, _, ast, _}), do: text_from_ast(ast) + @doc """ + Determines suitable Markdown fence delimiter for the + given code. + """ + @spec code_block_delimiter(String.t()) :: String.t() + def code_block_delimiter(code) do + max_streak = + Regex.scan(~r/`{3,}/, code) + |> Enum.map(fn [string] -> byte_size(string) end) + |> Enum.max(&>=/2, fn -> 2 end) + + String.duplicate("`", max_streak + 1) + end + @doc """ Renders Markdown string from the given `EarmarkParser` AST. """ @@ -161,12 +194,12 @@ defmodule Livebook.LiveMarkdown.MarkdownHelpers do @void_elements ~W(area base br col command embed hr img input keygen link meta param source track wbr) defp render_html(tag, attrs, []) when tag in @void_elements do - ["<", tag, " ", attrs_to_string(attrs), " />"] + ["<", tag, attrs_to_string(attrs), " />"] end defp render_html(tag, attrs, lines) do inner = Enum.intersperse(lines, "\n") - ["<", tag, " ", attrs_to_string(attrs), ">\n", inner, "\n"] + ["<", tag, attrs_to_string(attrs), ">\n", inner, "\n"] end defp render_emphasis(content) do @@ -243,8 +276,9 @@ defmodule Livebook.LiveMarkdown.MarkdownHelpers do end defp render_code_block(content, attrs) do + delimiter = code_block_delimiter(content) language = get_attr(attrs, "class", "") - ["```", language, "\n", content, "\n```"] + [delimiter, language, "\n", content, "\n", delimiter] end defp render_blockquote(content) do @@ -376,9 +410,7 @@ defmodule Livebook.LiveMarkdown.MarkdownHelpers do end defp attrs_to_string(attrs) do - attrs - |> Enum.map(fn {key, value} -> ~s/#{key}="#{value}"/ end) - |> Enum.join(" ") + Enum.map(attrs, fn {key, value} -> ~s/ #{key}="#{value}"/ end) end defp blank?(string), do: String.trim(string) == "" diff --git a/mix.exs b/mix.exs index 0e57a3f88..16164ec84 100644 --- a/mix.exs +++ b/mix.exs @@ -47,7 +47,9 @@ defmodule Livebook.MixProject do {:telemetry_poller, "~> 0.4"}, {:jason, "~> 1.0"}, {:plug_cowboy, "~> 2.0"}, - {:earmark_parser, "~> 1.4"}, + # {:earmark_parser, "~> 1.4"}, + {:earmark_parser, "~> 1.4", + github: "jonatanklosko/earmark_parser", branch: "jk-optional-inline"}, {:bypass, "~> 2.1", only: :test}, {:castore, "~> 0.1.0"} ] diff --git a/mix.lock b/mix.lock index e30fd7bd8..aca46d42a 100644 --- a/mix.lock +++ b/mix.lock @@ -4,7 +4,7 @@ "cowboy": {:hex, :cowboy, "2.9.0", "865dd8b6607e14cf03282e10e934023a1bd8be6f6bacf921a7e2a96d800cd452", [:make, :rebar3], [{:cowlib, "2.11.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "2c729f934b4e1aa149aff882f57c6372c15399a20d54f65c8d67bef583021bde"}, "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.3.1", "ebd1a1d7aff97f27c66654e78ece187abdc646992714164380d8a041eda16754", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3a6efd3366130eab84ca372cbd4a7d3c3a97bdfcfb4911233b035d117063f0af"}, "cowlib": {:hex, :cowlib, "2.11.0", "0b9ff9c346629256c42ebe1eeb769a83c6cb771a6ee5960bd110ab0b9b872063", [:make, :rebar3], [], "hexpm", "2b3e9da0b21c4565751a6d4901c20d1b4cc25cbb7fd50d91d2ab6dd287bc86a9"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.13", "0c98163e7d04a15feb62000e1a891489feb29f3d10cb57d4f845c405852bbef8", [:mix], [], "hexpm", "d602c26af3a0af43d2f2645613f65841657ad6efc9f0e361c3b6c06b578214ba"}, + "earmark_parser": {:git, "https://github.com/jonatanklosko/earmark_parser.git", "288ed586d45825a28b824d1cfd42fcd0959cd628", [branch: "jk-optional-inline"]}, "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, "floki": {:hex, :floki, "0.31.0", "f05ee8a8e6a3ced4e62beeb2c79a63bc8e12ab98fbaaf6e6a3d9b76b1278e23f", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "b05afa372f5c345a5bf240ac25ea1f0f3d5fcfd7490ac0beeb4a203f9444891e"}, "html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"}, diff --git a/test/livebook/live_markdown/export_test.exs b/test/livebook/live_markdown/export_test.exs index 6c6f1ec7d..642a11c4a 100644 --- a/test/livebook/live_markdown/export_test.exs +++ b/test/livebook/live_markdown/export_test.exs @@ -20,7 +20,9 @@ defmodule Livebook.LiveMarkdown.ExportTest do * Erlang * Elixir - * PostgreSQL\ + * PostgreSQL + + $x_{i} + y_{i}$\ """ }, %{ @@ -92,6 +94,8 @@ defmodule Livebook.LiveMarkdown.ExportTest do * Elixir * PostgreSQL + $x_{i} + y_{i}$ + ```elixir diff --git a/test/livebook/live_markdown/import_test.exs b/test/livebook/live_markdown/import_test.exs index bf56ef4c9..d5bbfad73 100644 --- a/test/livebook/live_markdown/import_test.exs +++ b/test/livebook/live_markdown/import_test.exs @@ -17,6 +17,8 @@ defmodule Livebook.LiveMarkdown.ImportTest do * Elixir * PostgreSQL + $x_{i} + y_{i}$ + ```elixir @@ -60,7 +62,9 @@ defmodule Livebook.LiveMarkdown.ImportTest do * Erlang * Elixir - * PostgreSQL\ + * PostgreSQL + + $x_{i} + y_{i}$\ """ }, %Cell.Elixir{ @@ -120,9 +124,6 @@ defmodule Livebook.LiveMarkdown.ImportTest do ## Section 1 - Line 1.\s\s - Line 2. - |State|Abbrev|Capital| | --: | :-: | --- | | Texas | TX | Austin | @@ -139,9 +140,6 @@ defmodule Livebook.LiveMarkdown.ImportTest do cells: [ %Cell.Markdown{ source: """ - Line 1.\\ - Line 2. - | State | Abbrev | Capital | | ----: | :----: | ------- | | Texas | TX | Austin | diff --git a/test/livebook/live_markdown/markdown_helpers_test.exs b/test/livebook/live_markdown/markdown_helpers_test.exs index 1bc24ebc7..05b34c37c 100644 --- a/test/livebook/live_markdown/markdown_helpers_test.exs +++ b/test/livebook/live_markdown/markdown_helpers_test.exs @@ -9,6 +9,16 @@ defmodule Livebook.LiveMarkdown.MarkdownHelpersTest do assert markdown == reformat(markdown) end + test "other emphasis" do + markdown = "The Game, _Mrs Hudson_, is on!" + assert markdown == reformat(markdown) + end + + test "nested emphasis" do + markdown = "The *Game, _Mrs Hudson_, is* on!" + assert markdown == reformat(markdown) + end + test "bold" do markdown = "The Game, **Mrs Hudson**, is on!" assert markdown == reformat(markdown) @@ -24,16 +34,50 @@ defmodule Livebook.LiveMarkdown.MarkdownHelpersTest do assert markdown == reformat(markdown) end + test "inline code with a line break" do + markdown = "The Game, `Mrs\nHudson`, is on!" + assert markdown == reformat(markdown) + end + + test "inline code with extra spaces" do + markdown = "The Game, `Mrs Huds on`, is on!" + assert markdown == reformat(markdown) + end + test "combined" do markdown = "The Game, ~~***`Mrs Hudson`***~~, is on!" assert markdown == reformat(markdown) end + test "inline math" do + markdown = "The Game, $x_{i} + y_{i}$, is on!" + + assert markdown == reformat(markdown) + end + test "link" do markdown = "The Game, [Mrs Hudson](https://youtu.be/M-KqaO1oH2E), is on!" assert markdown == reformat(markdown) end + test "link with IAL" do + markdown = "[link](url){: .classy}" + + assert markdown == reformat(markdown) + end + + test "link followed by escaped IAL" do + markdown = "[link](url)\\{: .classy}" + + assert markdown == reformat(markdown) + end + + test "autolink" do + markdown = "" + + assert markdown == reformat(markdown) + end + test "basic image" do markdown = "The Game, ![Mrs Hudson](https://example.com), is on!" assert markdown == reformat(markdown) @@ -49,6 +93,11 @@ defmodule Livebook.LiveMarkdown.MarkdownHelpersTest do assert markdown == reformat(markdown) end + test "line break" do + markdown = "Line 1.\\\nLine 2." + assert markdown == reformat(markdown) + end + test "oneline comment" do markdown = "" assert markdown == reformat(markdown) @@ -108,6 +157,32 @@ defmodule Livebook.LiveMarkdown.MarkdownHelpersTest do assert markdown == reformat(markdown) end + test "code block with fences inside it" do + markdown = """ + ````elixir + before + + ``` + _inside_ + ``` + + after + ````\ + """ + + assert markdown == reformat(markdown) + end + + test "display math" do + markdown = """ + $$ + R_{ij}^{kl} = R_{ij} - \Gamma^k_{kl} + $$\ + """ + + assert markdown == reformat(markdown) + end + test "blockquote" do markdown = """ > The Game, Mrs Hudson, @@ -243,6 +318,18 @@ defmodule Livebook.LiveMarkdown.MarkdownHelpersTest do assert markdown == reformat(markdown) end + test "surprise ordered list" do + markdown = "1986\\. What a great season." + + assert markdown == reformat(markdown) + end + + test "escaped Markdown" do + markdown = "not a \\[link\\]()" + + assert markdown == reformat(markdown) + end + test "raw html" do markdown = """
@@ -253,6 +340,18 @@ defmodule Livebook.LiveMarkdown.MarkdownHelpersTest do assert markdown == reformat(markdown) end + test "raw html at the beginning of a line" do + markdown = "line\n\n
" + + assert markdown == reformat(markdown) + end + + test "raw html not at the beginning of a line" do + markdown = "line\n
" + + assert markdown == reformat(markdown) + end + test "separates blocks with a single line" do markdown = """ The first paragraph, @@ -274,7 +373,9 @@ defmodule Livebook.LiveMarkdown.MarkdownHelpersTest do # By reformatting we can assert correct rendering # by comparing against the original content. defp reformat(markdown) do - {:ok, ast, []} = EarmarkParser.as_ast(markdown) + # Note: we don't parse inline content, so some of the tests + # above are not stricly necessary, but we keep them for completeness. + {:ok, ast, []} = MarkdownHelpers.markdown_to_block_ast(markdown) MarkdownHelpers.markdown_from_ast(ast) end end