Document how to render notebooks on GitHub (#677)

* Document how to render notebooks on GitHub

* Keep any comments before the notebook title heading

README.md

@@ -181,6 +181,24 @@ variables used by Elixir releases are also available](
 https://hexdocs.pm/mix/Mix.Tasks.Release.html#module-environment-variables).
 The notables ones are `RELEASE_NODE` and `RELEASE_DISTRIBUTION`.
 
+## Rendering notebooks as Markdown on GitHub
+
+By default GitHub renders the `.livemd` notebooks as regular text files. Depending
+on your use case and the target audience, you may find it useful to render notebooks
+content as Markdown files instead. There is an option to override how a particular
+file gets rendered on GitHub, so all you need to do is add a magic comment in every
+such notebook:
+
+```
+<!-- vim: syntax=markdown -->
+
+# My notebook
+
+...
+```
+
+For more details see [the documentation](https://github.com/github/linguist/blob/master/docs/overrides.md#using-emacs-or-vim-modelines).
+
 ## Development
 
 Livebook is primarily a Phoenix web application and can be setup as such:

lib/livebook/live_markdown.ex

@@ -40,6 +40,8 @@ defmodule Livebook.LiveMarkdown do
   #
   #        - cell metadata, recognised keys: `disable_formatting`
   #
+  #   6. Any comments before the leading heading are kept.
+  #
   # ## Example
   #
   # Here's an example LiveMarkdown file:

lib/livebook/live_markdown/export.ex

@@ -24,14 +24,23 @@ defmodule Livebook.LiveMarkdown.Export do
   end
 
   defp render_notebook(notebook, ctx) do
+    comments =
+      Enum.map(notebook.leading_comments, fn
+        [line] -> ["<!-- ", line, " -->"]
+        lines -> ["<!--\n", Enum.intersperse(lines, "\n"), "\n-->"]
+      end)
+
     name = ["# ", notebook.name]
     sections = Enum.map(notebook.sections, &render_section(&1, notebook, ctx))
 
     metadata = notebook_metadata(notebook)
 
+    notebook_with_metadata =
       [name | sections]
       |> Enum.intersperse("\n\n")
       |> prepend_metadata(metadata)
+
+    Enum.intersperse(comments ++ [notebook_with_metadata], "\n\n")
   end
 
   defp notebook_metadata(notebook) do

lib/livebook/live_markdown/import.ex

@@ -30,7 +30,7 @@ defmodule Livebook.LiveMarkdown.Import do
   defp rewrite_ast(ast) do
     {ast, messages1} = rewrite_multiple_primary_headings(ast)
     {ast, messages2} = move_primary_heading_top(ast)
-    ast = trim_comments(ast)
+    ast = normalize_comments(ast)
 
     {ast, messages1 ++ messages2}
   end
@@ -91,12 +91,12 @@ defmodule Livebook.LiveMarkdown.Import do
     {Enum.reverse(left_rev), Enum.reverse(right_rev)}
   end
 
-  # Trims one-line comments to allow nice pattern matching
+  # Normalizes comments to allow nice pattern matching on Livebook-specific
-  # on Livebook-specific annotations with no regard to surrounding whitespace.
+  # annotations with no regard to surrounding whitespace.
-  defp trim_comments(ast) do
+  defp normalize_comments(ast) do
     Enum.map(ast, fn
-      {:comment, attrs, [line], %{comment: true}} ->
+      {:comment, attrs, lines, %{comment: true}} ->
-        {:comment, attrs, [String.trim(line)], %{comment: true}}
+        {:comment, attrs, MarkdownHelpers.normalize_comment_lines(lines), %{comment: true}}
 
       ast_node ->
         ast_node
@@ -254,9 +254,15 @@ defmodule Livebook.LiveMarkdown.Import do
 
   defp build_notebook([{:notebook_name, content} | elems], [], sections, messages) do
     name = text_from_markdown(content)
-    {metadata, []} = grab_metadata(elems)
+    {metadata, elems} = grab_metadata(elems)
+    # If there are any non-metadata comments we keep them
+    {comments, []} = grab_leading_comments(elems)
     attrs = notebook_metadata_to_attrs(metadata)
-    notebook = %{Notebook.new() | name: name, sections: sections} |> Map.merge(attrs)
+
+    notebook =
+      %{Notebook.new() | name: name, sections: sections, leading_comments: comments}
+      |> Map.merge(attrs)
+
     {notebook, messages}
   end
 
@@ -280,6 +286,15 @@ defmodule Livebook.LiveMarkdown.Import do
 
   defp grab_metadata(elems), do: {%{}, elems}
 
+  defp grab_leading_comments([]), do: {[], []}
+
+  # Since these are not metadata comments they get wrapped in a markdown cell,
+  # so we unpack it
+  defp grab_leading_comments([{:cell, :markdown, md_ast}]) do
+    comments = for {:comment, _attrs, lines, %{comment: true}} <- Enum.reverse(md_ast), do: lines
+    {comments, []}
+  end
+
   defp parse_input_attrs(data) do
     with {:ok, type} <- parse_input_type(data["type"]) do
       warnings =

lib/livebook/live_markdown/markdown_helpers.ex

@@ -60,6 +60,27 @@ defmodule Livebook.LiveMarkdown.MarkdownHelpers do
     String.duplicate("`", max_streak + 1)
   end
 
+  @doc """
+  Normalizes comments.
+
+  In single-line comments trims all the whitespace and in multi-line
+  comments removes trailing/leading blank newlines.
+  """
+  @spec normalize_comment_lines(list(String.t())) :: list(String.t())
+  def normalize_comment_lines(lines)
+
+  def normalize_comment_lines([line]) do
+    [String.trim(line)]
+  end
+
+  def normalize_comment_lines(lines) do
+    lines
+    |> Enum.drop_while(&blank?/1)
+    |> Enum.reverse()
+    |> Enum.drop_while(&blank?/1)
+    |> Enum.reverse()
+  end
+
   @doc """
   Renders Markdown string from the given `EarmarkParser` AST.
   """
@@ -240,20 +261,11 @@ defmodule Livebook.LiveMarkdown.MarkdownHelpers do
     end
   end
 
-  defp render_comment([line]) do
-    line = String.trim(line)
-    ["<!-- ", line, " -->"]
-  end
-
   defp render_comment(lines) do
-    lines =
+    case normalize_comment_lines(lines) do
-      lines
+      [line] -> ["<!-- ", line, " -->"]
-      |> Enum.drop_while(&blank?/1)
+      lines -> ["<!--\n", Enum.intersperse(lines, "\n"), "\n-->"]
-      |> Enum.reverse()
+    end
-      |> Enum.drop_while(&blank?/1)
-      |> Enum.reverse()
-
-    ["<!--\n", Enum.intersperse(lines, "\n"), "\n-->"]
   end
 
   defp render_ruler(attrs) do

lib/livebook/notebook.ex

@@ -13,7 +13,14 @@ defmodule Livebook.Notebook do
   # A notebook is divided into a number of *sections*, each
   # containing a number of *cells*.
 
-  defstruct [:name, :version, :sections, :persist_outputs, :autosave_interval_s]
+  defstruct [
+    :name,
+    :version,
+    :sections,
+    :leading_comments,
+    :persist_outputs,
+    :autosave_interval_s
+  ]
 
   alias Livebook.Notebook.{Section, Cell}
   alias Livebook.Utils.Graph
@@ -23,6 +30,7 @@ defmodule Livebook.Notebook do
           name: String.t(),
           version: String.t(),
           sections: list(Section.t()),
+          leading_comments: list(list(line :: String.t())),
           persist_outputs: boolean(),
           autosave_interval_s: non_neg_integer() | nil
         }
@@ -38,6 +46,7 @@ defmodule Livebook.Notebook do
       name: "Untitled notebook",
       version: @version,
       sections: [],
+      leading_comments: [],
       persist_outputs: default_persist_outputs(),
       autosave_interval_s: default_autosave_interval_s()
     }

test/livebook/live_markdown/export_test.exs

@@ -515,6 +515,56 @@ defmodule Livebook.LiveMarkdown.ExportTest do
     assert expected_document == document
   end
 
+  test "exports leading notebook comments" do
+    notebook = %{
+      Notebook.new()
+      | name: "My Notebook",
+        persist_outputs: true,
+        leading_comments: [
+          ["vim: syntax=markdown"],
+          ["nowhitespace"],
+          ["  Multi", "  line"]
+        ],
+        sections: [
+          %{
+            Notebook.Section.new()
+            | name: "Section 1",
+              cells: [
+                %{
+                  Notebook.Cell.new(:markdown)
+                  | source: """
+                    Cell 1\
+                    """
+                }
+              ]
+          }
+        ]
+    }
+
+    expected_document = """
+    <!-- vim: syntax=markdown -->
+
+    <!-- nowhitespace -->
+
+    <!--
+      Multi
+      line
+    -->
+
+    <!-- livebook:{"persist_outputs":true} -->
+
+    # My Notebook
+
+    ## Section 1
+
+    Cell 1
+    """
+
+    document = Export.notebook_to_markdown(notebook)
+
+    assert expected_document == document
+  end
+
   describe "outputs" do
     test "does not include outputs by default" do
       notebook = %{

test/livebook/live_markdown/import_test.exs

@@ -517,6 +517,51 @@ defmodule Livebook.LiveMarkdown.ImportTest do
            } = notebook
   end
 
+  test "imports comments preceding the notebook title" do
+    markdown = """
+    <!-- vim: syntax=markdown -->
+
+    <!--nowhitespace-->
+
+    <!--
+      Multi
+      line
+    -->
+
+    <!-- livebook:{"persist_outputs":true} -->
+
+    # My Notebook
+
+    ## Section 1
+
+    Cell 1
+    """
+
+    {notebook, []} = Import.notebook_from_markdown(markdown)
+
+    assert %Notebook{
+             name: "My Notebook",
+             persist_outputs: true,
+             leading_comments: [
+               ["vim: syntax=markdown"],
+               ["nowhitespace"],
+               ["  Multi", "  line"]
+             ],
+             sections: [
+               %Notebook.Section{
+                 name: "Section 1",
+                 cells: [
+                   %Cell.Markdown{
+                     source: """
+                     Cell 1\
+                     """
+                   }
+                 ]
+               }
+             ]
+           } = notebook
+  end
+
   describe "outputs" do
     test "imports output snippets as cell textual outputs" do
       markdown = """
@@ -697,6 +742,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
   end
 
   describe "backward compatibility" do
+    test "warns if the imported notebook includes a reactive input" do
       markdown = """
       # My Notebook
 
@@ -711,4 +757,5 @@ defmodule Livebook.LiveMarkdown.ImportTest do
                "found a reactive input, but those are no longer supported, you can use automatically reevaluating cell instead"
              ] == messages
     end
+  end
 end