diff --git a/assets/css/markdown.css b/assets/css/markdown.css index bbaba8507..b9511d418 100644 --- a/assets/css/markdown.css +++ b/assets/css/markdown.css @@ -5,17 +5,29 @@ } .markdown h1 { - @apply text-gray-900 font-semibold text-3xl my-4; + @apply text-gray-900 font-semibold text-4xl my-4; } .markdown h2 { - @apply text-gray-900 font-semibold text-2xl my-4; + @apply text-gray-900 font-semibold text-3xl my-4; } .markdown h3 { + @apply text-gray-900 font-semibold text-2xl my-4; +} + +.markdown h4 { @apply text-gray-900 font-semibold text-xl my-4; } +.markdown h5 { + @apply text-gray-900 font-semibold text-lg my-4; +} + +.markdown h6 { + @apply text-gray-900 font-semibold text-base my-4; +} + .markdown p { @apply my-4; } @@ -97,3 +109,16 @@ .markdown :last-child { @apply mb-0; } + +/* Overrides for user-entered markdown */ + +.cell .markdown h1, +.cell .markdown h2 { + font-size: 0; +} + +.cell .markdown h1:after, +.cell .markdown h2:after { + @apply text-red-400 text-base font-medium; + content: "warning: heading levels 1 and 2 are reserved for notebook and section names, please use heading 3 and above."; +} diff --git a/lib/live_book/live_markdown.ex b/lib/live_book/live_markdown.ex new file mode 100644 index 000000000..6b703322f --- /dev/null +++ b/lib/live_book/live_markdown.ex @@ -0,0 +1,50 @@ +defmodule LiveBook.LiveMarkdown do + @moduledoc false + + # Notebook file format used by LiveBook. + # + # The format is based off of Markdown and preserves compatibility, + # in the sense that every LiveMarkdown file is a valid Markdown file. + # LiveMarkdown uses HTML comments for storing metadata, so a Markdown standard + # supporting that is assumed. Not every Markdown file is a valid LiveMarkdown file, + # but may be converted to such by applying tiny changes, which the import function does. + # + # Currently the format is straightforward and specifies the following: + # + # 1. The file should have a leading *Heading 1* holding the notebook name. + # 2. Every *Heading 2* starts a new section. + # 3. Every Elixir code block maps to an Elixir cell. + # 4. Adjacent regular Markdown text maps to a Markdown cell. + # 5. Comments of the form `` hold metadata + # any apply to the element they directly precede (e.g. an Elixir cell). + # + # ## Example + # + # Here's an example LiveMarkdown file: + # + # # My Notebook + # + # ## Section 1 + # + # Make sure to install: + # + # * Erlang + # * Elixir + # * PostgreSQL + # + # + # ```elixir + # Enum.to_list(1..10) + # ``` + # + # This is it for this section. + # + # ## Section 2 + # + # ```elixir + # # More Elixir code + # ``` + # + # This file defines a notebook named *My Notebook* with two sections. + # The first section includes 3 cells and the second section includes 1 Elixir cell. +end diff --git a/lib/live_book/live_markdown/export.ex b/lib/live_book/live_markdown/export.ex new file mode 100644 index 000000000..4748ca17a --- /dev/null +++ b/lib/live_book/live_markdown/export.ex @@ -0,0 +1,81 @@ +defmodule LiveBook.LiveMarkdown.Export do + alias LiveBook.Notebook + alias LiveBook.LiveMarkdown.MarkdownHelpers + + @doc """ + Converts the given notebook into a Markdown document. + """ + @spec notebook_to_markdown(Notebook.t()) :: String.t() + def notebook_to_markdown(notebook) do + iodata = render_notebook(notebook) + # Add trailing newline + IO.iodata_to_binary([iodata, "\n"]) + end + + defp render_notebook(notebook) do + name = "# #{notebook.name}" + sections = Enum.map(notebook.sections, &render_section/1) + + [name | sections] + |> Enum.intersperse("\n\n") + |> prepend_metadata(notebook.metadata) + end + + defp render_section(section) do + name = "## #{section.name}" + cells = Enum.map(section.cells, &render_cell/1) + + [name | cells] + |> Enum.intersperse("\n\n") + |> prepend_metadata(section.metadata) + end + + defp render_cell(%{type: :markdown} = cell) do + cell.source + |> format_markdown_source() + |> prepend_metadata(cell.metadata) + end + + defp render_cell(%{type: :elixir} = cell) do + """ + ```elixir + #{cell.source} + ```\ + """ + |> prepend_metadata(cell.metadata) + end + + defp render_metadata(metadata) do + metadata_json = Jason.encode!(metadata) + "" + end + + defp prepend_metadata(iodata, metadata) when metadata == %{}, do: iodata + + defp prepend_metadata(iodata, metadata) do + content = render_metadata(metadata) + [content, "\n", iodata] + end + + defp format_markdown_source(markdown) do + markdown + |> EarmarkParser.as_ast() + |> elem(1) + |> rewrite_ast() + |> MarkdownHelpers.markdown_from_ast() + end + + # Alters AST of the user-entered markdown. + defp rewrite_ast(ast) do + ast + |> remove_reserved_headings() + end + + defp remove_reserved_headings(ast) do + Enum.filter(ast, fn + {"h1", _, _, _} -> false + {"h2", _, _, _} -> false + _ast_node -> true + end) + end +end diff --git a/lib/live_book/live_markdown/import.ex b/lib/live_book/live_markdown/import.ex new file mode 100644 index 000000000..894e47668 --- /dev/null +++ b/lib/live_book/live_markdown/import.ex @@ -0,0 +1,187 @@ +defmodule LiveBook.LiveMarkdown.Import do + alias LiveBook.Notebook + alias LiveBook.LiveMarkdown.MarkdownHelpers + + @doc """ + Converts the given Markdown document into a notebook data structure. + + Returns the notebook structure and list if informative messages/warnings + related to the imported input. + """ + @spec notebook_from_markdown(String.t()) :: {Notebook.t(), list(String.t())} + def notebook_from_markdown(markdown) do + {_, ast, earmark_messages} = EarmarkParser.as_ast(markdown) + earmark_messages = Enum.map(earmark_messages, &earmark_message_to_string/1) + + {ast, rewrite_messages} = rewrite_ast(ast) + + notebook = + ast + |> group_elements() + |> build_notebook() + + {notebook, earmark_messages ++ rewrite_messages} + end + + defp earmark_message_to_string({_severity, line_number, message}) do + "Line #{line_number}: #{message}" + end + + # Does initial pre-processing of the AST, so that it conforms to the expected form. + # Returns {altered_ast, messages}. + defp rewrite_ast(ast) do + {ast, messages1} = rewrite_multiple_primary_headings(ast) + {ast, messages2} = move_primary_heading_top(ast) + + {ast, messages1 ++ messages2} + end + + # There should be only one h1 tag indicating notebook name, + # if there are many we downgrade all headings. + # This doesn't apply to documents exported from LiveBook, + # but may be the case for an arbitrary markdown file, + # so we do our best to preserve the intent. + defp rewrite_multiple_primary_headings(ast) do + primary_headings = Enum.count(ast, &(tag(&1) == "h1")) + + if primary_headings > 1 do + ast = Enum.map(ast, &downgrade_heading/1) + + message = + "Downgrading all headings, because #{primary_headings} instances of heading 1 were found" + + {ast, [message]} + else + {ast, []} + end + end + + defp downgrade_heading({"h1", attrs, content, meta}), do: {"h2", attrs, content, meta} + defp downgrade_heading({"h2", attrs, content, meta}), do: {"h3", attrs, content, meta} + defp downgrade_heading({"h3", attrs, content, meta}), do: {"h4", attrs, content, meta} + defp downgrade_heading({"h4", attrs, content, meta}), do: {"h5", attrs, content, meta} + defp downgrade_heading({"h5", attrs, content, meta}), do: {"h6", attrs, content, meta} + defp downgrade_heading({"h6", attrs, content, meta}), do: {"strong", attrs, content, meta} + defp downgrade_heading(ast_node), do: ast_node + + # This moves h1 together with any preceding comments to the top. + defp move_primary_heading_top(ast) do + case Enum.split_while(ast, &(tag(&1) != "h1")) do + {_ast, []} -> + {ast, []} + + {leading, [heading | rest]} -> + {leading, comments} = split_while_right(leading, &(tag(&1) == :comment)) + + if leading == [] do + {ast, []} + else + ast = comments ++ [heading] ++ leading ++ rest + message = "Moving heading 1 to the top of the notebook" + {ast, [message]} + end + end + end + + defp tag(ast_node) + defp tag({tag, _, _, _}), do: tag + defp tag(_), do: nil + + defp split_while_right(list, fun) do + {right_rev, left_rev} = list |> Enum.reverse() |> Enum.split_while(fun) + {Enum.reverse(left_rev), Enum.reverse(right_rev)} + end + + # Builds a list of classified elements from the AST. + defp group_elements(ast, elems \\ []) + + defp group_elements([], elems), do: elems + + defp group_elements([{"h1", _, content, %{}} | ast], elems) do + group_elements(ast, [{:notebook_name, content} | elems]) + end + + defp group_elements([{"h2", _, content, %{}} | ast], elems) do + group_elements(ast, [{:section_name, content} | elems]) + end + + defp group_elements( + [{:comment, _, ["live_book:" <> metadata_json], %{comment: true}} | ast], + elems + ) do + group_elements(ast, [{:metadata, metadata_json} | elems]) + end + + defp group_elements( + [{"pre", _, [{"code", [{"class", "elixir"}], [source], %{}}], %{}} | ast], + elems + ) do + group_elements(ast, [{:cell, :elixir, source} | elems]) + end + + defp group_elements([ast_node | ast], [{:cell, :markdown, md_ast} | rest]) do + group_elements(ast, [{:cell, :markdown, [ast_node | md_ast]} | rest]) + end + + defp group_elements([ast_node | ast], elems) do + group_elements(ast, [{:cell, :markdown, [ast_node]} | elems]) + end + + # Builds a notebook from the list of elements obtained in the previous step. + # Note that the list of elements is reversed: + # first we group elements by traversing Earmark AST top-down + # and then aggregate elements into data strictures going bottom-up. + defp build_notebook(elems, cells \\ [], sections \\ []) + + defp build_notebook([{:cell, :elixir, source} | elems], cells, sections) do + {metadata, elems} = grab_metadata(elems) + cell = %{Notebook.Cell.new(:elixir) | source: source, metadata: metadata} + build_notebook(elems, [cell | cells], sections) + end + + defp build_notebook([{:cell, :markdown, md_ast} | elems], cells, sections) do + {metadata, elems} = grab_metadata(elems) + source = md_ast |> Enum.reverse() |> MarkdownHelpers.markdown_from_ast() + cell = %{Notebook.Cell.new(:markdown) | source: source, metadata: metadata} + build_notebook(elems, [cell | cells], sections) + end + + defp build_notebook([{:section_name, content} | elems], cells, sections) do + name = MarkdownHelpers.text_from_ast(content) + {metadata, elems} = grab_metadata(elems) + section = %{Notebook.Section.new() | name: name, cells: cells, metadata: metadata} + build_notebook(elems, [], [section | sections]) + end + + # If there are section-less cells, put them in a default one. + defp build_notebook([{:notebook_name, _content} | _] = elems, cells, sections) + when cells != [] do + section = %{Notebook.Section.new() | cells: cells} + build_notebook(elems, [], [section | sections]) + end + + # If there are section-less cells, put them in a default one. + defp build_notebook([] = elems, cells, sections) when cells != [] do + section = %{Notebook.Section.new() | cells: cells} + build_notebook(elems, [], [section | sections]) + end + + defp build_notebook([{:notebook_name, content} | elems], [], sections) do + name = MarkdownHelpers.text_from_ast(content) + {metadata, []} = grab_metadata(elems) + %{Notebook.new() | name: name, sections: sections, metadata: metadata} + end + + # If there's no explicit notebook heading, use the defaults. + defp build_notebook([], [], sections) do + %{Notebook.new() | sections: sections} + end + + # Takes optional leading metadata JSON object and returns {metadata, rest}. + defp grab_metadata([{:metadata, metadata_json} | elems]) do + metadata = Jason.decode!(metadata_json) + {metadata, elems} + end + + defp grab_metadata(elems), do: {%{}, elems} +end diff --git a/lib/live_book/live_markdown/markdown_helpers.ex b/lib/live_book/live_markdown/markdown_helpers.ex new file mode 100644 index 000000000..9852b175c --- /dev/null +++ b/lib/live_book/live_markdown/markdown_helpers.ex @@ -0,0 +1,374 @@ +defmodule LiveBook.LiveMarkdown.MarkdownHelpers do + @doc """ + Reformats the given markdown document. + """ + @spec reformat(String.t()) :: String.t() + def reformat(markdown) do + markdown + |> EarmarkParser.as_ast() + |> elem(1) + |> markdown_from_ast() + end + + @doc """ + Extracts plain text from the given AST ignoring all the tags. + """ + @spec text_from_ast(EarmarkParser.ast()) :: String.t() + def text_from_ast(ast) + + def text_from_ast(ast) when is_list(ast) do + ast + |> Enum.map(&text_from_ast/1) + |> Enum.join("") + end + + def text_from_ast(ast) when is_binary(ast), do: ast + def text_from_ast({_, _, ast, _}), do: text_from_ast(ast) + + @doc """ + Renders Markdown string from the given `EarmarkParser` AST. + """ + @spec markdown_from_ast(EarmarkParser.ast()) :: String.t() + def markdown_from_ast(ast) do + build_md([], ast) + |> IO.iodata_to_binary() + |> String.trim() + end + + defp build_md(iodata, ast) + + defp build_md(iodata, []), do: iodata + + defp build_md(iodata, [string | ast]) when is_binary(string) do + string + |> append_inline(iodata) + |> build_md(ast) + end + + defp build_md(iodata, [{tag, attrs, lines, %{verbatim: true}} | ast]) do + render_html(tag, attrs, lines) + |> append_block(iodata) + |> build_md(ast) + end + + defp build_md(iodata, [{"em", _, content, %{}} | ast]) do + render_emphasis(content) + |> append_inline(iodata) + |> build_md(ast) + end + + defp build_md(iodata, [{"strong", _, content, %{}} | ast]) do + render_strong(content) + |> append_inline(iodata) + |> build_md(ast) + end + + defp build_md(iodata, [{"del", _, content, %{}} | ast]) do + render_strikethrough(content) + |> append_inline(iodata) + |> build_md(ast) + end + + defp build_md(iodata, [{"code", _, content, %{}} | ast]) do + render_code_inline(content) + |> append_inline(iodata) + |> build_md(ast) + end + + defp build_md(iodata, [{"a", attrs, content, %{}} | ast]) do + render_link(content, attrs) + |> append_inline(iodata) + |> build_md(ast) + end + + defp build_md(iodata, [{"img", attrs, [], %{}} | ast]) do + render_image(attrs) + |> append_inline(iodata) + |> build_md(ast) + end + + defp build_md(iodata, [{:comment, _, lines, %{comment: true}} | ast]) do + render_comment(lines) + |> append_inline(iodata) + |> build_md(ast) + end + + defp build_md(iodata, [{"hr", attrs, [], %{}} | ast]) do + render_ruler(attrs) + |> append_block(iodata) + |> build_md(ast) + end + + defp build_md(iodata, [{"p", _, content, %{}} | ast]) do + render_paragraph(content) + |> append_block(iodata) + |> build_md(ast) + end + + defp build_md(iodata, [{"h" <> n, _, content, %{}} | ast]) + when n in ["1", "2", "3", "4", "5", "6"] do + n = String.to_integer(n) + + render_heading(n, content) + |> append_block(iodata) + |> build_md(ast) + end + + defp build_md(iodata, [{"pre", _, [{"code", attrs, [content], %{}}], %{}} | ast]) do + render_code_block(content, attrs) + |> append_block(iodata) + |> build_md(ast) + end + + defp build_md(iodata, [{"blockquote", [], content, %{}} | ast]) do + render_blockquote(content) + |> append_block(iodata) + |> build_md(ast) + end + + defp build_md(iodata, [{"table", _, content, %{}} | ast]) do + render_table(content) + |> append_block(iodata) + |> build_md(ast) + end + + defp build_md(iodata, [{"ul", _, content, %{}} | ast]) do + render_unordered_list(content) + |> append_block(iodata) + |> build_md(ast) + end + + defp build_md(iodata, [{"ol", _, content, %{}} | ast]) do + render_ordered_list(content) + |> append_block(iodata) + |> build_md(ast) + end + + defp append_inline(md, iodata), do: [iodata, md] + defp append_block(md, iodata), do: [iodata, "\n", md, "\n"] + + # Renderers + + # https://www.w3.org/TR/2011/WD-html-markup-20110113/syntax.html#void-element + @void_elements ~W(area base br col command embed hr img input keygen link meta param source track wbr) + + defp render_html(tag, attrs, []) when tag in @void_elements do + "<#{tag} #{attrs_to_string(attrs)} />" + end + + defp render_html(tag, attrs, lines) do + inner = Enum.join(lines, "\n") + "<#{tag} #{attrs_to_string(attrs)}>\n#{inner}\n#{tag}>" + end + + defp render_emphasis(content) do + inner = markdown_from_ast(content) + "*#{inner}*" + end + + defp render_strong(content) do + inner = markdown_from_ast(content) + "**#{inner}**" + end + + defp render_strikethrough(content) do + inner = markdown_from_ast(content) + "~~#{inner}~~" + end + + defp render_code_inline(content) do + inner = markdown_from_ast(content) + "`#{inner}`" + end + + defp render_link(content, attrs) do + caption = markdown_from_ast(content) + href = get_attr(attrs, "href", "") + "[#{caption}](#{href})" + end + + defp render_image(attrs) do + alt = get_attr(attrs, "alt", "") + src = get_attr(attrs, "src", "") + title = get_attr(attrs, "title", "") + + if title == "" do + "" + else + ~s// + end + end + + defp render_comment([line]) do + line = String.trim(line) + "" + end + + defp render_comment(lines) do + lines = + lines + |> Enum.drop_while(&blank?/1) + |> Enum.reverse() + |> Enum.drop_while(&blank?/1) + |> Enum.reverse() + + Enum.join([""], "\n") + end + + defp render_ruler(attrs) do + class = get_attr(attrs, "class", "thin") + + case class do + "thin" -> "---" + "medium" -> "___" + "thick" -> "***" + end + end + + defp render_paragraph(content), do: markdown_from_ast(content) + + defp render_heading(n, content) do + title = markdown_from_ast(content) + String.duplicate("#", n) <> " " <> title + end + + defp render_code_block(content, attrs) do + language = get_attr(attrs, "class", "") + "```#{language}\n#{content}\n```" + end + + defp render_blockquote(content) do + inner = markdown_from_ast(content) + + inner + |> String.split("\n") + |> Enum.map(&("> " <> &1)) + |> Enum.join("\n") + end + + defp render_table([{"thead", _, [head_row], %{}}, {"tbody", _, body_rows, %{}}]) do + alignments = alignments_from_row(head_row) + cell_grid = cell_grid_from_rows([head_row | body_rows]) + column_widths = max_length_per_column(cell_grid) + [head_cells | body_cell_grid] = Enum.map(cell_grid, &pad_whitespace(&1, column_widths)) + separator_cells = build_separator_cells(alignments, column_widths) + cell_grid_to_md_table([head_cells, separator_cells | body_cell_grid]) + end + + defp render_table([{"tbody", _, body_rows, %{}}]) do + cell_grid = cell_grid_from_rows(body_rows) + column_widths = max_length_per_column(cell_grid) + cell_grid = Enum.map(cell_grid, &pad_whitespace(&1, column_widths)) + cell_grid_to_md_table(cell_grid) + end + + defp cell_grid_from_rows(rows) do + Enum.map(rows, fn {"tr", _, columns, %{}} -> + Enum.map(columns, fn {tag, _, content, %{}} when tag in ["th", "td"] -> + markdown_from_ast(content) + end) + end) + end + + defp alignments_from_row({"tr", _, columns, %{}}) do + Enum.map(columns, fn {tag, attrs, _, %{}} when tag in ["th", "td"] -> + style = get_attr(attrs, "style", nil) + + case style do + "text-align: left;" -> :left + "text-align: center;" -> :center + "text-align: right;" -> :right + end + end) + end + + defp build_separator_cells(alignments, widths) do + alignments + |> Enum.zip(widths) + |> Enum.map(fn + {:left, length} -> String.duplicate("-", length) + {:center, length} -> ":" <> String.duplicate("-", length - 2) <> ":" + {:right, length} -> String.duplicate("-", length - 1) <> ":" + end) + end + + defp max_length_per_column(cell_grid) do + cell_grid + |> List.zip() + |> Enum.map(&Tuple.to_list/1) + |> Enum.map(fn cells -> + cells + |> Enum.map(&String.length/1) + |> Enum.max() + end) + end + + defp pad_whitespace(cells, widths) do + cells + |> Enum.zip(widths) + |> Enum.map(fn {cell, width} -> + String.pad_trailing(cell, width, " ") + end) + end + + defp cell_grid_to_md_table(cell_grid) do + cell_grid + |> Enum.map(fn cells -> + "| " <> Enum.join(cells, " | ") <> " |" + end) + |> Enum.join("\n") + end + + defp render_unordered_list(content) do + marker_fun = fn _index -> "* " end + render_list(content, marker_fun, " ") + end + + defp render_ordered_list(content) do + marker_fun = fn index -> "#{index + 1}. " end + render_list(content, marker_fun, " ") + end + + defp render_list(items, marker_fun, indent) do + spaced? = spaced_list_items?(items) + item_separator = if(spaced?, do: "\n\n", else: "\n") + + items + |> Enum.map(fn {"li", _, content, %{}} -> markdown_from_ast(content) end) + |> Enum.with_index() + |> Enum.map(fn {inner, index} -> + [first_line | lines] = String.split(inner, "\n") + + first_line = marker_fun.(index) <> first_line + + lines = + Enum.map(lines, fn + "" -> "" + line -> indent <> line + end) + + Enum.join([first_line | lines], "\n") + end) + |> Enum.join(item_separator) + end + + defp spaced_list_items?([{"li", _, [{"p", _, _content, %{}} | _], %{}} | _items]), do: true + defp spaced_list_items?([_ | items]), do: spaced_list_items?(items) + defp spaced_list_items?([]), do: false + + # Helpers + + defp get_attr(attrs, key, default) do + Enum.find_value(attrs, default, fn {attr_key, attr_value} -> + attr_key == key && attr_value + end) + end + + defp attrs_to_string(attrs) do + attrs + |> Enum.map(fn {key, value} -> ~s/#{key}="#{value}"/ end) + |> Enum.join(" ") + end + + defp blank?(string), do: String.trim(string) == "" +end diff --git a/lib/live_book/notebook.ex b/lib/live_book/notebook.ex index ef95f9ef2..ea76b76ba 100644 --- a/lib/live_book/notebook.ex +++ b/lib/live_book/notebook.ex @@ -19,7 +19,7 @@ defmodule LiveBook.Notebook do name: String.t(), version: String.t(), sections: list(Section.t()), - metadata: %{atom() => term()} + metadata: %{String.t() => term()} } @version "1.0" diff --git a/lib/live_book/notebook/cell.ex b/lib/live_book/notebook/cell.ex index bc540333c..50bfdbc80 100644 --- a/lib/live_book/notebook/cell.ex +++ b/lib/live_book/notebook/cell.ex @@ -19,7 +19,7 @@ defmodule LiveBook.Notebook.Cell do type: type(), source: String.t(), outputs: list(), - metadata: %{atom() => term()} + metadata: %{String.t() => term()} } @doc """ diff --git a/lib/live_book/notebook/section.ex b/lib/live_book/notebook/section.ex index f80b59d73..84afe8de7 100644 --- a/lib/live_book/notebook/section.ex +++ b/lib/live_book/notebook/section.ex @@ -17,7 +17,7 @@ defmodule LiveBook.Notebook.Section do id: id(), name: String.t(), cells: list(Cell.t()), - metadata: %{atom() => term()} + metadata: %{String.t() => term()} } @doc """ diff --git a/lib/live_book_web/live/cell_component.ex b/lib/live_book_web/live/cell_component.ex index 04785d517..5f3e50e1b 100644 --- a/lib/live_book_web/live/cell_component.ex +++ b/lib/live_book_web/live/cell_component.ex @@ -3,7 +3,7 @@ defmodule LiveBookWeb.CellComponent do def render(assigns) do ~L""" -