Set up importing notebook from URL or clipboard (#237)

* Set up importing notebook from URL or clipboard

* Extract loading functions into separate module

* Replace import icon with text

* Add TLS peer verification

* Remove unnecessary tests setup step

lib/livebook/content_loader.ex

@@ -0,0 +1,115 @@
+defmodule Livebook.ContentLoader do
+  @moduledoc false
+
+  @doc """
+  Rewrite known URLs, so that they point to plain text file rather than HTML.
+
+  Currently the rewerites handle:
+
+    * GitHub files
+    * Gist files
+  """
+  @spec rewrite_url(String.t()) :: String.t()
+  def rewrite_url(url) do
+    url
+    |> URI.parse()
+    |> do_rewrite_url()
+    |> URI.to_string()
+  end
+
+  defp do_rewrite_url(%URI{host: "github.com"} = uri) do
+    case String.split(uri.path, "/") do
+      ["", owner, repo, "blob", hash | file_path] ->
+        path = Enum.join(["", owner, repo, hash | file_path], "/")
+
+        %{
+          uri
+          | path: path,
+            host: "raw.githubusercontent.com",
+            authority: "raw.githubusercontent.com"
+        }
+
+      _ ->
+        uri
+    end
+  end
+
+  defp do_rewrite_url(%URI{host: "gist.github.com"} = uri) do
+    case String.split(uri.path, "/") do
+      ["", owner, hash] ->
+        path = Enum.join(["", owner, hash, "raw"], "/")
+
+        %{
+          uri
+          | path: path,
+            host: "gist.githubusercontent.com",
+            authority: "gist.githubusercontent.com"
+        }
+
+      _ ->
+        uri
+    end
+  end
+
+  defp do_rewrite_url(uri), do: uri
+
+  @doc """
+  Loads binary content from the given URl and validates if its plain text.
+  """
+  @spec fetch_content(String.t()) :: {:ok, String.t()} | {:error, String.t()}
+  def fetch_content(url) do
+    case :httpc.request(:get, {url, []}, http_opts(), body_format: :binary) do
+      {:ok, {{_, 200, _}, headers, body}} ->
+        valid_content? =
+          case fetch_content_type(headers) do
+            {:ok, content_type} -> content_type in ["text/plain", "text/markdown"]
+            :error -> false
+          end
+
+        if valid_content? do
+          {:ok, body}
+        else
+          {:error, "invalid content type, make sure the URL points to live markdown"}
+        end
+
+      _ ->
+        {:error, "failed to download notebook from the given URL"}
+    end
+  end
+
+  defp fetch_content_type(headers) do
+    case Enum.find(headers, fn {key, _} -> key == 'content-type' end) do
+      {_, value} ->
+        {:ok,
+         value
+         |> List.to_string()
+         |> String.split(";")
+         |> hd()}
+
+      _ ->
+        :error
+    end
+  end
+
+  crt_file = CAStore.file_path()
+  crt = File.read!(crt_file)
+  pems = :public_key.pem_decode(crt)
+  ders = Enum.map(pems, fn {:Certificate, der, _} -> der end)
+
+  # Note: we need to load the certificates at compilation time,
+  # as we don't have access to package files in Escript.
+  @cacerts ders
+
+  defp http_opts() do
+    [
+      # Use secure options, see https://gist.github.com/jonatanklosko/5e20ca84127f6b31bbe3906498e1a1d7
+      ssl: [
+        verify: :verify_peer,
+        cacerts: @cacerts,
+        customize_hostname_check: [
+          match_fun: :public_key.pkix_verify_hostname_match_fun(:https)
+        ]
+      ]
+    ]
+  end
+end

lib/livebook_web/live/home_live.ex

@@ -31,6 +31,10 @@ defmodule LivebookWeb.HomeLive do
                   <%= remix_icon("compass-line") %>
                 </button>
               </span>
+              <%= live_patch to: Routes.home_path(@socket, :import, "url"),
+                    class: "button button-outlined-gray whitespace-nowrap" do %>
+                Import
+              <% end %>
               <button class="button button-blue"
                 phx-click="new">
                 New notebook
@@ -54,7 +58,7 @@ defmodule LivebookWeb.HomeLive do
                   <span>Fork</span>
                 <% end %>
                 <%= if path_running?(@path, @session_summaries) do %>
-                  <%= live_patch "Join session", to: Routes.session_path(@socket, :page, session_id_by_path(@path, @session_summaries)),
+                  <%= live_redirect "Join session", to: Routes.session_path(@socket, :page, session_id_by_path(@path, @session_summaries)),
                     class: "button button-blue" %>
                 <% else %>
                   <%= tag :span, if(File.regular?(@path) and not file_writable?(@path),
@@ -86,7 +90,7 @@ defmodule LivebookWeb.HomeLive do
                 </div>
               </div>
             <% else %>
-              <%= live_component @socket, LivebookWeb.SessionLive.SessionsComponent,
+              <%= live_component @socket, LivebookWeb.HomeLive.SessionsComponent,
                 id: "sessions_list",
                 session_summaries: @session_summaries %>
             <% end %>
@@ -96,11 +100,18 @@ defmodule LivebookWeb.HomeLive do
     </div>
 
     <%= if @live_action == :close_session do %>
-      <%= live_modal @socket, LivebookWeb.SessionLive.CloseSessionComponent,
+      <%= live_modal @socket, LivebookWeb.HomeLive.CloseSessionComponent,
             id: :close_session_modal,
             return_to: Routes.home_path(@socket, :page),
             session_summary: @session_summary %>
     <% end %>
+
+    <%= if @live_action == :import do %>
+      <%= live_modal @socket, LivebookWeb.HomeLive.ImportComponent,
+            id: :import_modal,
+            return_to: Routes.home_path(@socket, :page),
+            tab: @tab %>
+    <% end %>
     """
   end
 
@@ -110,6 +121,10 @@ defmodule LivebookWeb.HomeLive do
     {:noreply, assign(socket, session_summary: session_summary)}
   end
 
+  def handle_params(%{"tab" => tab}, _url, socket) do
+    {:noreply, assign(socket, tab: tab)}
+  end
+
   def handle_params(_params, _url, socket), do: {:noreply, socket}
 
   @impl true
@@ -158,6 +173,12 @@ defmodule LivebookWeb.HomeLive do
     {:noreply, assign(socket, session_summaries: session_summaries)}
   end
 
+  def handle_info({:import_content, content}, socket) do
+    {notebook, messages} = Livebook.LiveMarkdown.Import.notebook_from_markdown(content)
+    socket = put_import_flash_messages(socket, messages)
+    create_session(socket, notebook: notebook)
+  end
+
   def handle_info(_message, socket), do: {:noreply, socket}
 
   defp default_path(), do: Livebook.Config.root_path() <> "/"

lib/livebook_web/live/home_live/close_session_component.ex

@@ -1,4 +1,4 @@
-defmodule LivebookWeb.SessionLive.CloseSessionComponent do
+defmodule LivebookWeb.HomeLive.CloseSessionComponent do
   use LivebookWeb, :live_component
 
   alias Livebook.SessionSupervisor

lib/livebook_web/live/home_live/import_component.ex

@@ -0,0 +1,43 @@
+defmodule LivebookWeb.HomeLive.ImportComponent do
+  use LivebookWeb, :live_component
+
+  @impl true
+  def render(assigns) do
+    ~L"""
+    <div class="p-6 pb-4 max-w-xl w-screen flex flex-col space-y-8">
+      <h3 class="text-2xl font-semibold text-gray-800">
+        Import notebook
+      </h3>
+      <div class="tabs">
+        <%= live_patch to: Routes.home_path(@socket, :import, "url"),
+          class: "tab #{if(@tab == "url", do: "active")}" do %>
+          <%= remix_icon("download-cloud-2-line", class: "align-middle") %>
+          <span class="font-medium">
+            From URL
+          </span>
+        <% end %>
+        <%= live_patch to: Routes.home_path(@socket, :import, "content"),
+          class: "tab #{if(@tab == "content", do: "active")}" do %>
+          <%= remix_icon("clipboard-line", class: "align-middle") %>
+          <span class="font-medium">
+            From clipboard
+          </span>
+        <% end %>
+        <div class="flex-grow tab">
+        </div>
+      </div>
+      <div>
+        <%= case @tab do %>
+          <% "url" -> %>
+            <%= live_component @socket, LivebookWeb.HomeLive.ImportUrlComponent,
+                  id: "import_url" %>
+
+          <% "content" -> %>
+            <%= live_component @socket, LivebookWeb.HomeLive.ImportContentComponent,
+                  id: "import_content" %>
+        <% end %>
+      </div>
+    </div>
+    """
+  end
+end

lib/livebook_web/live/home_live/import_content_component.ex

@@ -0,0 +1,44 @@
+defmodule LivebookWeb.HomeLive.ImportContentComponent do
+  use LivebookWeb, :live_component
+
+  @impl true
+  def mount(socket) do
+    {:ok, assign(socket, content: "")}
+  end
+
+  @impl true
+  def render(assigns) do
+    ~L"""
+    <div class="flex-col space-y-5">
+      <p class="text-gray-700">
+        Import notebook by directly pasting the <span class="font-semibold">live markdown</span> content.
+      </p>
+      <%= f = form_for :data, "#",
+                phx_submit: "import",
+                phx_change: "validate",
+                phx_target: @myself,
+                autocomplete: "off" %>
+        <%= textarea f, :content, value: @content, class: "input resize-none",
+              placeholder: "Notebook content",
+              spellcheck: "false",
+              rows: 5 %>
+
+        <%= submit "Import",
+              class: "mt-5 button button-blue",
+              disabled: @content == "" %>
+      </form>
+    </div>
+    """
+  end
+
+  @impl true
+  def handle_event("validate", %{"data" => %{"content" => content}}, socket) do
+    {:noreply, assign(socket, content: content)}
+  end
+
+  def handle_event("import", %{"data" => %{"content" => content}}, socket) do
+    send(self(), {:import_content, content})
+
+    {:noreply, socket}
+  end
+end

lib/livebook_web/live/home_live/import_url_component.ex

@@ -0,0 +1,63 @@
+defmodule LivebookWeb.HomeLive.ImportUrlComponent do
+  use LivebookWeb, :live_component
+
+  alias Livebook.ContentLoader
+
+  @impl true
+  def mount(socket) do
+    {:ok, assign(socket, url: "", error_message: nil)}
+  end
+
+  @impl true
+  def render(assigns) do
+    ~L"""
+    <div class="flex-col space-y-5">
+      <%= if @error_message do %>
+        <div class="mb-3 rounded-lg px-4 py-2 bg-red-100 text-red-400 font-medium">
+          <%= @error_message %>
+        </div>
+      <% end %>
+      <p class="text-gray-700">
+        Load notebook from an online source (e.g. a GitHub repository)
+        simply by pasting its URL.
+      </p>
+      <%= f = form_for :data, "#",
+                phx_submit: "import",
+                phx_change: "validate",
+                phx_target: @myself,
+                autocomplete: "off" %>
+        <%= text_input f, :url, value: @url, class: "input",
+              placeholder: "Notebook URL",
+              spellcheck: "false" %>
+        <%= submit "Import",
+              class: "mt-5 button button-blue",
+              disabled: not valid_url?(@url) %>
+      </form>
+    </div>
+    """
+  end
+
+  @impl true
+  def handle_event("validate", %{"data" => %{"url" => url}}, socket) do
+    {:noreply, assign(socket, url: url)}
+  end
+
+  def handle_event("import", %{"data" => %{"url" => url}}, socket) do
+    url
+    |> ContentLoader.rewrite_url()
+    |> ContentLoader.fetch_content()
+    |> case do
+      {:ok, content} ->
+        send(self(), {:import_content, content})
+        {:noreply, socket}
+
+      {:error, message} ->
+        {:noreply, assign(socket, error_message: String.capitalize(message))}
+    end
+  end
+
+  defp valid_url?(url) do
+    uri = URI.parse(url)
+    uri.scheme != nil and uri.host != nil and uri.host =~ "."
+  end
+end

lib/livebook_web/live/home_live/sessions_component.ex

@@ -1,4 +1,4 @@
-defmodule LivebookWeb.SessionLive.SessionsComponent do
+defmodule LivebookWeb.HomeLive.SessionsComponent do
   use LivebookWeb, :live_component
 
   @impl true

lib/livebook_web/router.ex

@@ -19,6 +19,7 @@ defmodule LivebookWeb.Router do
     pipe_through [:browser, :auth]
 
     live "/", HomeLive, :page
+    live "/home/import/:tab", HomeLive, :import
     live "/home/sessions/:session_id/close", HomeLive, :close_session
     live "/sessions/:id", SessionLive, :page
     live "/sessions/:id/shortcuts", SessionLive, :shortcuts

mix.exs

@@ -18,7 +18,7 @@ defmodule Livebook.MixProject do
   def application do
     [
       mod: {Livebook.Application, []},
-      extra_applications: [:logger, :runtime_tools, :os_mon]
+      extra_applications: [:logger, :runtime_tools, :os_mon, :inets, :ssl]
     ]
   end
 
@@ -37,7 +37,9 @@ defmodule Livebook.MixProject do
       {:telemetry_poller, "~> 0.4"},
       {:jason, "~> 1.0"},
       {:plug_cowboy, "~> 2.0"},
-      {:earmark_parser, "~> 1.4"}
+      {:earmark_parser, "~> 1.4"},
+      {:bypass, "~> 2.1", only: :test},
+      {:castore, "~> 0.1.0"}
     ]
   end
 

mix.lock

@@ -1,4 +1,6 @@
 %{
+  "bypass": {:hex, :bypass, "2.1.0", "909782781bf8e20ee86a9cabde36b259d44af8b9f38756173e8f5e2e1fabb9b1", [:mix], [{:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:ranch, "~> 1.3", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "d9b5df8fa5b7a6efa08384e9bbecfe4ce61c77d28a4282f79e02f1ef78d96b80"},
+  "castore": {:hex, :castore, "0.1.10", "b01a007416a0ae4188e70b3b306236021b16c11474038ead7aff79dd75538c23", [:mix], [], "hexpm", "a48314e0cb45682db2ea27b8ebfa11bd6fa0a6e21a65e5772ad83ca136ff2665"},
   "cowboy": {:hex, :cowboy, "2.8.0", "f3dc62e35797ecd9ac1b50db74611193c29815401e53bac9a5c0577bd7bc667d", [:rebar3], [{:cowlib, "~> 2.9.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "4643e4fba74ac96d4d152c75803de6fad0b3fa5df354c71afdd6cbeeb15fac8a"},
   "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.3.1", "ebd1a1d7aff97f27c66654e78ece187abdc646992714164380d8a041eda16754", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3a6efd3366130eab84ca372cbd4a7d3c3a97bdfcfb4911233b035d117063f0af"},
   "cowlib": {:hex, :cowlib, "2.9.1", "61a6c7c50cf07fdd24b2f45b89500bb93b6686579b069a89f88cb211e1125c78", [:rebar3], [], "hexpm", "e4175dc240a70d996156160891e1c62238ede1729e45740bdd38064dad476170"},

test/livebook/content_loader_test.exs

@@ -0,0 +1,67 @@
+defmodule Livebook.ContentLoaderTest do
+  use ExUnit.Case, async: true
+
+  alias Livebook.ContentLoader
+
+  describe "rewrite_url/1" do
+    test "rewrites GitHub URLs to their raw counterpart" do
+      url = "https://github.com/org/user/blob/main/notebooks/example.livemd"
+      expected_url = "https://raw.githubusercontent.com/org/user/main/notebooks/example.livemd"
+
+      assert ContentLoader.rewrite_url(url) == expected_url
+    end
+
+    test "rewrites Gist URLs to their raw counterpart" do
+      url = "https://gist.github.com/user/hash"
+      expected_url = "https://gist.githubusercontent.com/user/hash/raw"
+
+      assert ContentLoader.rewrite_url(url) == expected_url
+    end
+
+    test "leaves arbitrary URLs unchanged" do
+      url = "https://example.com/notebooks/example.livemd"
+
+      assert ContentLoader.rewrite_url(url) == url
+    end
+  end
+
+  describe "fetch_content/1" do
+    setup do
+      bypass = Bypass.open()
+      {:ok, bypass: bypass}
+    end
+
+    test "returns an error when the request fails", %{bypass: bypass} do
+      Bypass.expect_once(bypass, "GET", "/invalid", fn conn ->
+        Plug.Conn.resp(conn, 500, "Error")
+      end)
+
+      assert ContentLoader.fetch_content(url(bypass.port) <> "/invalid") ==
+               {:error, "failed to download notebook from the given URL"}
+    end
+
+    test "returns an error when the respone is HTML", %{bypass: bypass} do
+      Bypass.expect_once(bypass, "GET", "/html", fn conn ->
+        conn
+        |> Plug.Conn.put_resp_content_type("text/html")
+        |> Plug.Conn.resp(200, "<html></html>")
+      end)
+
+      assert ContentLoader.fetch_content(url(bypass.port) <> "/html") ==
+               {:error, "invalid content type, make sure the URL points to live markdown"}
+    end
+
+    test "returns response body when the response is plain text", %{bypass: bypass} do
+      Bypass.expect_once(bypass, "GET", "/notebook", fn conn ->
+        conn
+        |> Plug.Conn.put_resp_content_type("text/plain")
+        |> Plug.Conn.resp(200, "# My notebook")
+      end)
+
+      assert ContentLoader.fetch_content(url(bypass.port) <> "/notebook") ==
+               {:ok, "# My notebook"}
+    end
+  end
+
+  defp url(port), do: "http://localhost:#{port}"
+end

test/livebook_web/live/home_live_test.exs

@@ -172,6 +172,27 @@ defmodule LivebookWeb.HomeLiveTest do
     assert render(view) =~ "Welcome to Livebook"
   end
 
+  describe "notebook import" do
+    test "allows importing notebook directly from content", %{conn: conn} do
+      Phoenix.PubSub.subscribe(Livebook.PubSub, "sessions")
+
+      {:ok, view, _} = live(conn, "/home/import/content")
+
+      notebook_content = """
+      # My notebook
+      """
+
+      view
+      |> element("form", "Import")
+      |> render_submit(%{data: %{content: notebook_content}})
+
+      assert_receive {:session_created, id}
+
+      {:ok, view, _} = live(conn, "/sessions/#{id}")
+      assert render(view) =~ "My notebook"
+    end
+  end
+
   # Helpers
 
   defp test_notebook_path(name) do