Persist app settings and support deployment from directory (#1741)

Co-authored-by: José Valim <jose.valim@dashbit.co>

README.md

@@ -164,6 +164,14 @@ The following environment variables configure Livebook:
   * LIVEBOOK_APP_SERVICE_URL - sets the application url to manage this
     Livebook instance within the cloud provider platform.
 
+  * LIVEBOOK_APPS_PATH - the directory with app notebooks. When set, the apps
+    are deployed on Livebook startup with the persisted settings.
+    Password-protected notebooks will receive a random password,
+    unless LIVEBOOK_APPS_PATH_PASSWORD is set.
+
+  * LIVEBOOK_APPS_PATH_PASSWORD - the password to use for all protected apps
+    deployed from LIVEBOOK_APPS_PATH.
+
   * LIVEBOOK_BASE_URL_PATH - sets the base url path the web application is served on.
     Useful when deploying behind a reverse proxy.
 
@@ -194,7 +202,7 @@ The following environment variables configure Livebook:
 
   * LIVEBOOK_IFRAME_URL - sets the URL that Livebook loads iframes from.
     By default iframes are loaded from local LIVEBOOK_IFRAME_PORT when accessing
-    Livebook over http:// and from https://livebook.space when accessing over `https://`.
+    Livebook over http:// and from https://livebook.space when accessing over https://.
 
   * LIVEBOOK_IP - sets the ip address to start the web application on.
     Must be a valid IPv4 or IPv6 address.
@@ -219,7 +227,7 @@ The following environment variables configure Livebook:
     in the homepage. Set it to "true" to enable it.
 
   * LIVEBOOK_TOKEN_ENABLED - controls whether token authentication is enabled.
-    Enabled by default unless `LIVEBOOK_PASSWORD` is set. Set it to "false" to
+    Enabled by default unless LIVEBOOK_PASSWORD is set. Set it to "false" to
     disable it.
 
   * LIVEBOOK_UPDATE_INSTRUCTIONS_URL - sets the URL to direct the user to for

lib/livebook.ex

@@ -152,6 +152,14 @@ defmodule Livebook do
       config :livebook, :data_path, data_path
     end
 
+    if apps_path = Livebook.Config.readable_dir!("LIVEBOOK_APPS_PATH") do
+      config :livebook, :apps_path, apps_path
+    end
+
+    if apps_path_password = Livebook.Config.password!("LIVEBOOK_APPS_PATH_PASSWORD") do
+      config :livebook, :apps_path_password, apps_path_password
+    end
+
     if force_ssl_host = Livebook.Config.force_ssl_host!("LIVEBOOK_FORCE_SSL_HOST") do
       config :livebook, :force_ssl_host, force_ssl_host
     end

lib/livebook/application.ex

@@ -56,6 +56,7 @@ defmodule Livebook.Application do
         insert_personal_hub()
         Livebook.Hubs.connect_hubs()
         update_app_secrets_origin()
+        deploy_apps()
         result
 
       {:error, error} ->
@@ -222,6 +223,12 @@ defmodule Livebook.Application do
     end
   end
 
+  defp deploy_apps() do
+    if apps_path = Livebook.Config.apps_path() do
+      Livebook.Apps.deploy_apps_in_dir(apps_path, password: Livebook.Config.apps_path_password())
+    end
+  end
+
   defp iframe_server_specs() do
     server? = Phoenix.Endpoint.server?(:livebook, LivebookWeb.Endpoint)
     port = Livebook.Config.iframe_port()

lib/livebook/apps.ex

@@ -1,6 +1,8 @@
 defmodule Livebook.Apps do
   @moduledoc false
 
+  require Logger
+
   alias Livebook.Session
 
   @doc """
@@ -89,4 +91,49 @@ defmodule Livebook.Apps do
   end
 
   defp name(slug), do: {:app, slug}
+
+  @doc """
+  Deploys an app for each notebook in the given directory.
+
+  ## Options
+
+    * `:password` - a password to set for every loaded app
+
+  """
+  @spec deploy_apps_in_dir(String.t(), keyword()) :: :ok
+  def deploy_apps_in_dir(path, opts \\ []) do
+    opts = Keyword.validate!(opts, [:password])
+
+    pattern = Path.join([path, "**", "*.livemd"])
+    paths = Path.wildcard(pattern)
+
+    for path <- paths do
+      markdown = File.read!(path)
+      {notebook, warnings} = Livebook.LiveMarkdown.notebook_from_livemd(markdown)
+
+      if warnings != [] do
+        items = Enum.map(warnings, &("- " <> &1))
+
+        Logger.warning(
+          "Found warnings while importing app notebook at #{path}:\n\n" <> Enum.join(items, "\n")
+        )
+      end
+
+      notebook =
+        if password = opts[:password] do
+          put_in(notebook.app_settings.password, password)
+        else
+          notebook
+        end
+
+      if Livebook.Notebook.AppSettings.valid?(notebook.app_settings) do
+        {:ok, session} = Livebook.Sessions.create_session(notebook: notebook, mode: :app)
+        Livebook.Session.app_build(session.pid)
+      else
+        Logger.warning("Skipping app deployment at #{path} due to invalid settings")
+      end
+    end
+
+    :ok
+  end
 end

lib/livebook/config.ex

@@ -84,6 +84,22 @@ defmodule Livebook.Config do
     Application.get_env(:livebook, :data_path) || :filename.basedir(:user_data, "livebook")
   end
 
+  @doc """
+  Returns the apps path.
+  """
+  @spec apps_path() :: String.t() | nil
+  def apps_path() do
+    Application.get_env(:livebook, :apps_path)
+  end
+
+  @doc """
+  Returns the password configured for all apps deployed rom `app_path`.
+  """
+  @spec apps_path_password() :: String.t() | nil
+  def apps_path_password() do
+    Application.get_env(:livebook, :apps_path_password)
+  end
+
   @doc """
   Returns the configured port for the Livebook endpoint.
 
@@ -246,6 +262,33 @@ defmodule Livebook.Config do
     end
   end
 
+  @doc """
+  Parses and validates dir from env.
+  """
+  def readable_dir!(env) do
+    if dir = System.get_env(env) do
+      readable_dir!(env, dir)
+    end
+  end
+
+  @doc """
+  Validates `dir` within context.
+  """
+  def readable_dir!(context, dir) do
+    if readable_dir?(dir) do
+      Path.expand(dir)
+    else
+      abort!("expected #{context} to be a readable directory: #{dir}")
+    end
+  end
+
+  defp readable_dir?(path) do
+    case File.stat(path) do
+      {:ok, %{type: :directory, access: access}} when access in [:read_write, :read] -> true
+      _ -> false
+    end
+  end
+
   @doc """
   Parses and validates the secret from env.
   """

lib/livebook/live_markdown/export.ex

@@ -69,7 +69,25 @@ defmodule Livebook.LiveMarkdown.Export do
 
   defp notebook_metadata(notebook) do
     keys = [:persist_outputs, :autosave_interval_s]
-    put_unless_default(%{}, Map.take(notebook, keys), Map.take(Notebook.new(), keys))
+    metadata = put_unless_default(%{}, Map.take(notebook, keys), Map.take(Notebook.new(), keys))
+
+    app_settings_metadata = app_settings_metadata(notebook.app_settings)
+
+    if app_settings_metadata == %{} do
+      metadata
+    else
+      Map.put(metadata, :app_settings, app_settings_metadata)
+    end
+  end
+
+  defp app_settings_metadata(app_settings) do
+    keys = [:slug, :access_type]
+
+    put_unless_default(
+      %{},
+      Map.take(app_settings, keys),
+      Map.take(Notebook.AppSettings.new(), keys)
+    )
   end
 
   defp render_section(section, notebook, ctx) do

lib/livebook/live_markdown/import.ex

@@ -368,6 +368,28 @@ defmodule Livebook.LiveMarkdown.Import do
       {"autosave_interval_s", autosave_interval_s}, attrs ->
         Map.put(attrs, :autosave_interval_s, autosave_interval_s)
 
+      {"app_settings", app_settings_metadata}, attrs ->
+        app_settings =
+          Map.merge(
+            Notebook.AppSettings.new(),
+            app_settings_metadata_to_attrs(app_settings_metadata)
+          )
+
+        Map.put(attrs, :app_settings, app_settings)
+
+      _entry, attrs ->
+        attrs
+    end)
+  end
+
+  defp app_settings_metadata_to_attrs(metadata) do
+    Enum.reduce(metadata, %{}, fn
+      {"slug", slug}, attrs ->
+        Map.put(attrs, :slug, slug)
+
+      {"access_type", access_type}, attrs when access_type in ["public", "protected"] ->
+        Map.put(attrs, :access_type, String.to_atom(access_type))
+
       _entry, attrs ->
         attrs
     end)

lib/livebook/session/data.ex

@@ -839,6 +839,7 @@ defmodule Livebook.Session.Data do
     data
     |> with_actions()
     |> set_app_settings(settings)
+    |> set_dirty()
     |> wrap_ok()
   end
 

test/livebook/apps_test.exs

@@ -0,0 +1,76 @@
+defmodule Livebook.AppsTest do
+  use ExUnit.Case, async: true
+
+  import ExUnit.CaptureLog
+
+  describe "deploy_apps_in_dir/1" do
+    @tag :tmp_dir
+    test "deploys apps", %{tmp_dir: tmp_dir} do
+      app1_path = Path.join(tmp_dir, "app1.livemd")
+      app2_path = Path.join(tmp_dir, "app2.livemd")
+
+      File.write!(app1_path, """
+      <!-- livebook:{"app_settings":{"slug":"app1"}} -->
+
+      # App 1
+      """)
+
+      File.write!(app2_path, """
+      <!-- livebook:{"app_settings":{"slug":"app2"}} -->
+
+      # App 2
+      """)
+
+      Livebook.Sessions.subscribe()
+
+      Livebook.Apps.deploy_apps_in_dir(tmp_dir)
+
+      assert_receive {:session_created, %{app_info: %{slug: "app1"}}}
+
+      assert_receive {:session_updated,
+                      %{app_info: %{slug: "app1", status: :running, registered: true}} =
+                        app1_session}
+
+      assert_receive {:session_created, %{app_info: %{slug: "app2"}}}
+
+      assert_receive {:session_updated,
+                      %{app_info: %{slug: "app2", status: :running, registered: true}} =
+                        app2_session}
+
+      Livebook.Session.close([app1_session.pid, app2_session.pid])
+    end
+
+    @tag :tmp_dir
+    test "skips apps with incomplete config and warns", %{tmp_dir: tmp_dir} do
+      app_path = Path.join(tmp_dir, "app.livemd")
+
+      File.write!(app_path, """
+      # App
+      """)
+
+      assert capture_log(fn ->
+               Livebook.Apps.deploy_apps_in_dir(tmp_dir)
+             end) =~ "Skipping app deployment at #{app_path} due to invalid settings"
+    end
+
+    @tag :tmp_dir
+    test "overrides apps password when :password is set", %{tmp_dir: tmp_dir} do
+      app_path = Path.join(tmp_dir, "app.livemd")
+
+      File.write!(app_path, """
+      <!-- livebook:{"app_settings":{"slug":"app"}} -->
+
+      # App
+      """)
+
+      Livebook.Sessions.subscribe()
+
+      Livebook.Apps.deploy_apps_in_dir(tmp_dir, password: "verylongpass")
+
+      assert_receive {:session_created, %{app_info: %{slug: "app"}} = session}
+
+      %{access_type: :protected, password: "verylongpass"} =
+        Livebook.Session.get_app_settings(session.pid)
+    end
+  end
+end

test/livebook/live_markdown/export_test.exs

@@ -1121,6 +1121,49 @@ defmodule Livebook.LiveMarkdown.ExportTest do
     assert expected_document == document
   end
 
+  describe "app settings" do
+    test "persists non-default app settings" do
+      notebook = %{
+        Notebook.new()
+        | name: "My Notebook",
+          app_settings: %{Notebook.AppSettings.new() | slug: "app", access_type: :public}
+      }
+
+      expected_document = """
+      <!-- livebook:{"app_settings":{"access_type":"public","slug":"app"}} -->
+
+      # My Notebook
+      """
+
+      document = Export.notebook_to_livemd(notebook)
+
+      assert expected_document == document
+    end
+
+    test "does not persist password" do
+      notebook = %{
+        Notebook.new()
+        | name: "My Notebook",
+          app_settings: %{
+            Notebook.AppSettings.new()
+            | slug: "app",
+              access_type: :protected,
+              password: "verylongpass"
+          }
+      }
+
+      expected_document = """
+      <!-- livebook:{"app_settings":{"slug":"app"}} -->
+
+      # My Notebook
+      """
+
+      document = Export.notebook_to_livemd(notebook)
+
+      assert expected_document == document
+    end
+  end
+
   describe "setup cell" do
     test "includes the leading setup cell when it has content" do
       notebook =

test/livebook/live_markdown/import_test.exs

@@ -720,6 +720,38 @@ defmodule Livebook.LiveMarkdown.ImportTest do
     assert %Notebook{name: "My Notebook", autosave_interval_s: 10} = notebook
   end
 
+  describe "app settings" do
+    test "imports settings" do
+      markdown = """
+      <!-- livebook:{"app_settings":{"access_type":"public","slug":"app"}} -->
+
+      # My Notebook
+      """
+
+      {notebook, []} = Import.notebook_from_livemd(markdown)
+
+      assert %Notebook{
+               name: "My Notebook",
+               app_settings: %{slug: "app", access_type: :public}
+             } = notebook
+    end
+
+    test "correctly imports protected access" do
+      markdown = """
+      <!-- livebook:{"app_settings":{"slug":"app"}} -->
+
+      # My Notebook
+      """
+
+      {notebook, []} = Import.notebook_from_livemd(markdown)
+
+      assert %Notebook{
+               name: "My Notebook",
+               app_settings: %{slug: "app", access_type: :protected}
+             } = notebook
+    end
+  end
+
   describe "backward compatibility" do
     test "warns if the imported notebook includes an input" do
       markdown = """