Introduce abstraction for app deployment and permanent apps (#2524)

Co-authored-by: José Valim <jose.valim@dashbit.co>

config/config.exs

@@ -39,6 +39,8 @@ config :livebook,
   allowed_uri_schemes: [],
   aws_credentials: false
 
+config :livebook, Livebook.Apps.Manager, retry_backoff_base_ms: 5_000
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

config/test.exs

@@ -27,6 +27,8 @@ config :livebook,
   feature_flags: [deployment_groups: true],
   agent_name: "chonky-cat"
 
+config :livebook, Livebook.Apps.Manager, retry_backoff_base_ms: 0
+
 # Use longnames when running tests in CI, so that no host resolution is required,
 # see https://github.com/livebook-dev/livebook/pull/173#issuecomment-819468549
 if System.get_env("CI") == "true" do

lib/livebook.ex

@@ -251,7 +251,7 @@ defmodule Livebook do
   """
   @spec live_markdown_to_elixir(String.t()) :: String.t()
   def live_markdown_to_elixir(markdown) do
-    {notebook, _messages} = Livebook.LiveMarkdown.notebook_from_livemd(markdown)
+    {notebook, _info} = Livebook.LiveMarkdown.notebook_from_livemd(markdown)
     Livebook.Notebook.Export.Elixir.notebook_to_elixir(notebook)
   end
 end

lib/livebook/app.ex

@@ -19,9 +19,15 @@ defmodule Livebook.App do
     :notebook_name,
     :public?,
     :multi_session,
-    :sessions
+    :sessions,
+    :app_spec,
+    :permanent
   ]
 
+  use GenServer, restart: :temporary
+
+  require Logger
+
   @type t :: %{
           slug: slug(),
           pid: pid(),
@@ -30,7 +36,9 @@ defmodule Livebook.App do
           notebook_name: String.t(),
           public?: boolean(),
           multi_session: boolean(),
-          sessions: list(app_session())
+          sessions: list(app_session()),
+          app_spec: Livebook.Apps.AppSpec.t(),
+          permanent: boolean()
         }
 
   @type slug :: String.t()
@@ -45,28 +53,56 @@ defmodule Livebook.App do
           started_by: Livebook.Users.User.t() | nil
         }
 
-  use GenServer, restart: :temporary
+  @typedoc """
+  Notebook and related information for deploying an app version.
+
+  This information is used to start app sessions.
+
+    * `:notebook` - the notebook to use for the deployment
+
+    * `:files_tmp_path` - a path to local directory with notebook files.
+      This should be a temporary copy of the files that the app process
+      takes ownership over. The app is going to remove the directory
+      once no longer needed.
+
+      The app uses this local copy of the files, so that changes to
+      the original files can be made safely. Also, if the original
+      files are stored on a remote file system, we want to download
+      them once, rather than in each starting session. Finally, some
+      app specs (such as Teams) may need to unpack their files into
+      a temporary directory and only the app process knows when to
+      remove this directory
+
+    * `:app_spec` - the app spec that was used to load the notebook
+
+    * `:permanent` - whether the app is being deployed as a permanent,
+      typically by `Livebook.Apps.Manager`
+
+    * `:warnings` - a list of warnings to show for the deployment
+
+  """
+  @type deployment_bundle :: %{
+          notebook: Livebook.Notebook.t(),
+          files_tmp_path: String.t(),
+          app_spec: Livebook.Apps.AppSpec.t(),
+          permanent: boolean(),
+          warnings: list(String.t())
+        }
 
   @doc """
   Starts an apps process.
 
   ## Options
 
-    * `:notebook` (required) - the notebook for initial deployment
-
-    * `:warnings` - a list of warnings to show for the initial deployment
-
-    * `:files_source` - a location to fetch notebook files from, see
-      `Livebook.Session.start_link/1` for more details
+    * `:deployment_bundle` (required) - see `t:deployment_bundle/0`
 
   """
   @spec start_link(keyword()) :: {:ok, pid} | {:error, any()}
   def start_link(opts) do
-    notebook = Keyword.fetch!(opts, :notebook)
-    warnings = Keyword.get(opts, :warnings, [])
-    files_source = Keyword.get(opts, :files_source)
+    opts = Keyword.validate!(opts, [:deployment_bundle])
+    deployment_bundle = Keyword.fetch!(opts, :deployment_bundle)
 
-    GenServer.start_link(__MODULE__, {notebook, warnings, files_source})
+    GenServer.start_link(__MODULE__, deployment_bundle)
   end
 
   @doc """
@@ -115,11 +151,9 @@ defmodule Livebook.App do
   @doc """
   Deploys a new notebook into the app.
   """
-  @spec deploy(pid(), Livebook.Notebook.t(), keyword()) :: :ok
-  def deploy(pid, notebook, opts \\ []) do
-    warnings = Keyword.get(opts, :warnings, [])
-    files_source = Keyword.get(opts, :files_source)
-    GenServer.cast(pid, {:deploy, notebook, warnings, files_source})
+  @spec deploy(pid(), deployment_bundle()) :: :ok
+  def deploy(pid, deployment_bundle) do
+    GenServer.cast(pid, {:deploy, deployment_bundle})
   end
 
   @doc """
@@ -127,10 +161,19 @@ defmodule Livebook.App do
 
   This operation results in all app sessions being closed as well.
   """
+  @spec close(pid()) :: :ok
   def close(pid) do
     GenServer.call(pid, :close)
   end
 
+  @doc """
+  Sends an asynchronous request to close the session.
+  """
+  @spec close_async(pid()) :: :ok
+  def close_async(pid) do
+    GenServer.cast(pid, :close)
+  end
+
   @doc """
   Subscribes to app messages.
 
@@ -153,17 +196,33 @@ defmodule Livebook.App do
   end
 
   @impl true
-  def init({notebook, warnings, files_source}) do
-    {:ok,
-     %{
-       version: 1,
-       notebook: notebook,
-       files_source: files_source,
-       warnings: warnings,
-       sessions: [],
-       users: %{}
-     }
-     |> start_eagerly()}
+  def init(deployment_bundle) do
+    state = %{
+      version: 1,
+      deployment_bundle: deployment_bundle,
+      sessions: [],
+      users: %{}
+    }
+
+    app = self_from_state(state)
+
+    slug = deployment_bundle.app_spec.slug
+    name = Livebook.Apps.global_name(slug)
+
+    case :global.register_name(name, self(), &resolve_app_conflict/3) do
+      :yes ->
+        with :ok <- Livebook.Tracker.track_app(app) do
+          {:ok, state, {:continue, :after_init}}
+        end
+
+      :no ->
+        {:error, :already_started}
+    end
+  end
+
+  @impl true
+  def handle_continue(:after_init, state) do
+    {:noreply, state |> start_eagerly() |> notify_update()}
   end
 
   @impl true
@@ -173,7 +232,8 @@ defmodule Livebook.App do
 
   def handle_call({:get_session_id, user}, _from, state) do
     {session_id, state} =
-      case {state.notebook.app_settings.multi_session, single_session_app_session(state)} do
+      case {state.deployment_bundle.notebook.app_settings.multi_session,
+            single_session_app_session(state)} do
         {false, %{} = app_session} ->
           {app_session.id, state}
 
@@ -187,7 +247,7 @@ defmodule Livebook.App do
   end
 
   def handle_call(:get_settings, _from, state) do
-    {:reply, state.notebook.app_settings, state}
+    {:reply, state.deployment_bundle.notebook.app_settings, state}
   end
 
   def handle_call(:close, _from, state) do
@@ -195,22 +255,22 @@ defmodule Livebook.App do
   end
 
   @impl true
-  def handle_cast({:deploy, notebook, warnings, files_source}, state) do
-    true = notebook.app_settings.slug == state.notebook.app_settings.slug
+  def handle_cast({:deploy, deployment_bundle}, state) do
+    assert_valid_redeploy!(state.deployment_bundle, deployment_bundle)
+
+    cleanup_notebook_files_dir(state)
 
     {:noreply,
-     %{
-       state
-       | notebook: notebook,
-         version: state.version + 1,
-         warnings: warnings,
-         files_source: files_source
-     }
+     %{state | version: state.version + 1, deployment_bundle: deployment_bundle}
      |> start_eagerly()
      |> shutdown_old_versions()
      |> notify_update()}
   end
 
+  def handle_cast(:close, state) do
+    {:stop, :shutdown, state}
+  end
+
   @impl true
   def handle_info({:app_status_changed, session_id, status}, state) do
     state = update_app_session(state, session_id, &%{&1 | app_status: status})
@@ -233,24 +293,40 @@ defmodule Livebook.App do
     {:noreply, notify_update(state)}
   end
 
+  @impl true
+  def terminate(_reason, state) do
+    cleanup_notebook_files_dir(state)
+    :ok
+  end
+
   defp self_from_state(state) do
     %{
-      slug: state.notebook.app_settings.slug,
+      slug: state.deployment_bundle.notebook.app_settings.slug,
       pid: self(),
       version: state.version,
-      warnings: state.warnings,
-      notebook_name: state.notebook.name,
-      public?: state.notebook.app_settings.access_type == :public,
-      multi_session: state.notebook.app_settings.multi_session,
-      sessions: state.sessions
+      warnings: state.deployment_bundle.warnings,
+      notebook_name: state.deployment_bundle.notebook.name,
+      public?: state.deployment_bundle.notebook.app_settings.access_type == :public,
+      multi_session: state.deployment_bundle.notebook.app_settings.multi_session,
+      sessions: state.sessions,
+      app_spec: state.deployment_bundle.app_spec,
+      permanent: state.deployment_bundle.permanent
     }
   end
 
+  defp resolve_app_conflict({:app, slug}, pid1, pid2) do
+    Logger.info("[app=#{slug}] Closing duplicate app in the cluster")
+    [keep_pid, close_pid] = Enum.shuffle([pid1, pid2])
+    close_async(close_pid)
+    keep_pid
+  end
+
   defp single_session_app_session(state) do
     app_session = Enum.find(state.sessions, &(&1.version == state.version))
 
     if app_session do
-      if state.notebook.app_settings.zero_downtime and not status_ready?(app_session.app_status) do
+      if state.deployment_bundle.notebook.app_settings.zero_downtime and
+           not status_ready?(app_session.app_status) do
         Enum.find(state.sessions, &status_ready?(&1.app_status))
       end || app_session
     end
@@ -259,10 +335,11 @@ defmodule Livebook.App do
   defp status_ready?(%{execution: :executed, lifecycle: :active}), do: true
   defp status_ready?(_status), do: false
 
-  defp start_eagerly(state) when state.notebook.app_settings.multi_session, do: state
+  defp start_eagerly(state) when state.deployment_bundle.notebook.app_settings.multi_session,
+    do: state
 
   defp start_eagerly(state) do
-    if temporary_sessions?(state.notebook.app_settings) do
+    if temporary_sessions?(state.deployment_bundle.notebook.app_settings) do
       state
     else
       {:ok, state, _app_session} = start_app_session(state)
@@ -271,14 +348,19 @@ defmodule Livebook.App do
   end
 
   defp start_app_session(state, user \\ nil) do
-    user = if(state.notebook.teams_enabled, do: user)
+    user = if(state.deployment_bundle.notebook.teams_enabled, do: user)
+
+    files_source =
+      state.deployment_bundle.files_tmp_path
+      |> Livebook.FileSystem.Utils.ensure_dir_path()
+      |> Livebook.FileSystem.File.local()
 
     opts = [
-      notebook: state.notebook,
-      files_source: state.files_source,
+      notebook: state.deployment_bundle.notebook,
+      files_source: {:dir, files_source},
       mode: :app,
       app_pid: self(),
-      auto_shutdown_ms: state.notebook.app_settings.auto_shutdown_ms,
+      auto_shutdown_ms: state.deployment_bundle.notebook.app_settings.auto_shutdown_ms,
       started_by: user
     ]
 
@@ -316,7 +398,8 @@ defmodule Livebook.App do
 
   defp temporary_sessions?(app_settings), do: app_settings.auto_shutdown_ms != nil
 
-  defp shutdown_old_versions(state) when not state.notebook.app_settings.multi_session do
+  defp shutdown_old_versions(state)
+       when not state.deployment_bundle.notebook.app_settings.multi_session do
     single_session_app_session = single_session_app_session(state)
 
     for app_session <- state.sessions,
@@ -339,11 +422,23 @@ defmodule Livebook.App do
   defp notify_update(state) do
     app = self_from_state(state)
     Livebook.Apps.update_app(app)
-    broadcast_message(state.notebook.app_settings.slug, {:app_updated, app})
+    broadcast_message(state.deployment_bundle.notebook.app_settings.slug, {:app_updated, app})
     state
   end
 
   defp broadcast_message(slug, message) do
     Phoenix.PubSub.broadcast(Livebook.PubSub, "apps:#{slug}", message)
   end
+
+  defp cleanup_notebook_files_dir(state) do
+    if path = state.deployment_bundle.files_tmp_path do
+      File.rm_rf(path)
+    end
+  end
+
+  defp assert_valid_redeploy!(
+         %{app_spec: %module{slug: slug}, permanent: permanent},
+         %{app_spec: %module{slug: slug}, permanent: permanent}
+       ),
+       do: :ok
 end

lib/livebook/application.ex

@@ -10,40 +10,49 @@ defmodule Livebook.Application do
     set_cookie()
 
     children =
-      [
-        # Start the Telemetry supervisor
-        LivebookWeb.Telemetry,
-        # Start the PubSub system
-        {Phoenix.PubSub, name: Livebook.PubSub},
-        # Start a supervisor for Livebook tasks
-        {Task.Supervisor, name: Livebook.TaskSupervisor},
-        # Start the storage module
-        Livebook.Storage,
-        # Run migrations as soon as the storage is running
-        Livebook.Migration,
-        # Start the periodic version check
-        Livebook.UpdateCheck,
-        # Periodic measurement of system resources
-        Livebook.SystemResources,
-        # Start the notebook manager server
-        Livebook.NotebookManager,
-        # Start the tracker server on this node
-        {Livebook.Tracker, pubsub_server: Livebook.PubSub},
-        # Start the supervisor dynamically managing apps
-        {DynamicSupervisor, name: Livebook.AppSupervisor, strategy: :one_for_one},
-        # Start the supervisor dynamically managing sessions
-        {DynamicSupervisor, name: Livebook.SessionSupervisor, strategy: :one_for_one},
-        # Start the server responsible for associating files with sessions
-        Livebook.Session.FileGuard,
-        # Start the node pool for managing node names
-        Livebook.Runtime.NodePool,
-        # Start the unique task dependencies
-        Livebook.Utils.UniqueTask,
-        # Start the registry for managing unique connections
-        {Registry, keys: :unique, name: Livebook.HubsRegistry},
-        # Start the supervisor dynamically managing connections
-        {DynamicSupervisor, name: Livebook.HubsSupervisor, strategy: :one_for_one}
-      ] ++
+      if serverless?() do
+        []
+      else
+        [{DNSCluster, query: Application.get_env(:livebook, :dns_cluster_query) || :ignore}]
+      end ++
+        [
+          # Start the Telemetry supervisor
+          LivebookWeb.Telemetry,
+          # Start the PubSub system
+          {Phoenix.PubSub, name: Livebook.PubSub},
+          # Start a supervisor for Livebook tasks
+          {Task.Supervisor, name: Livebook.TaskSupervisor},
+          # Start the unique task dependencies
+          Livebook.Utils.UniqueTask,
+          # Start the storage module
+          Livebook.Storage,
+          # Run migrations as soon as the storage is running
+          {Livebook.Utils.SupervisionStep, {:migration, &Livebook.Migration.run/0}},
+          # Start the periodic version check
+          Livebook.UpdateCheck,
+          # Periodic measurement of system resources
+          Livebook.SystemResources,
+          # Start the notebook manager server
+          Livebook.NotebookManager,
+          # Start the tracker server for sessions and apps on this node
+          {Livebook.Tracker, pubsub_server: Livebook.PubSub},
+          # Start the node pool for managing node names
+          Livebook.Runtime.NodePool,
+          # Start the server responsible for associating files with sessions
+          Livebook.Session.FileGuard,
+          # Start the supervisor dynamically managing sessions
+          {DynamicSupervisor, name: Livebook.SessionSupervisor, strategy: :one_for_one},
+          # Start the registry for managing unique connections
+          {Registry, keys: :unique, name: Livebook.HubsRegistry},
+          # Start the supervisor dynamically managing connections
+          {DynamicSupervisor, name: Livebook.HubsSupervisor, strategy: :one_for_one},
+          # Run startup logic relying on the supervision tree
+          {Livebook.Utils.SupervisionStep, {:boot, &boot/0}},
+          # App manager supervision tree. We do it after boot, because
+          # permanent apps are going to be started right away and this
+          # depends on hubs being started
+          Livebook.Apps.DeploymentSupervisor
+        ] ++
         if serverless?() do
           []
         else
@@ -52,7 +61,6 @@ defmodule Livebook.Application do
           iframe_server_specs() ++
             [
               {module, name: LivebookWeb.ZTA, identity_key: key},
-              {DNSCluster, query: Application.get_env(:livebook, :dns_cluster_query) || :ignore},
               # We skip the access url as we do our own logging below
               {LivebookWeb.Endpoint, log_access_url: false}
             ] ++ app_specs()
@@ -62,16 +70,7 @@ defmodule Livebook.Application do
 
     case Supervisor.start_link(children, opts) do
       {:ok, _} = result ->
-        load_lb_env_vars()
-        create_teams_hub()
-        clear_env_vars()
         display_startup_info()
-        Livebook.Hubs.connect_hubs()
-
-        unless serverless?() do
-          deploy_apps()
-        end
-
         result
 
       {:error, error} ->
@@ -79,6 +78,17 @@ defmodule Livebook.Application do
     end
   end
 
+  def boot() do
+    load_lb_env_vars()
+    create_teams_hub()
+    clear_env_vars()
+    Livebook.Hubs.connect_hubs()
+
+    unless serverless?() do
+      load_apps_dir()
+    end
+  end
+
   # Tell Phoenix to update the endpoint configuration
   # whenever the application is updated.
   def config_change(changed, _new, removed) do
@@ -233,9 +243,30 @@ defmodule Livebook.Application do
     end
   end
 
-  defp clear_env_vars() do
-    for {var, _} <- System.get_env(), config_env_var?(var) do
-      System.delete_env(var)
+  if Mix.target() == :app do
+    defp app_specs, do: [LivebookApp]
+  else
+    defp app_specs, do: []
+  end
+
+  defp iframe_server_specs() do
+    server? = Phoenix.Endpoint.server?(:livebook, LivebookWeb.Endpoint)
+    port = Livebook.Config.iframe_port()
+
+    if server? do
+      http = Application.fetch_env!(:livebook, LivebookWeb.Endpoint)[:http]
+
+      iframe_opts =
+        [
+          scheme: :http,
+          plug: LivebookWeb.IframeEndpoint,
+          port: port,
+          thousand_island_options: [supervisor_options: [name: LivebookWeb.IframeEndpoint]]
+        ] ++ Keyword.take(http, [:ip])
+
+      [{Bandit, iframe_opts}]
+    else
+      []
     end
   end
 
@@ -254,6 +285,12 @@ defmodule Livebook.Application do
     Livebook.Secrets.set_startup_secrets(secrets)
   end
 
+  defp clear_env_vars() do
+    for {var, _} <- System.get_env(), config_env_var?(var) do
+      System.delete_env(var)
+    end
+  end
+
   defp create_teams_hub() do
     teams_key = System.get_env("LIVEBOOK_TEAMS_KEY")
     auth = System.get_env("LIVEBOOK_TEAMS_AUTH")
@@ -360,42 +397,18 @@ defmodule Livebook.Application do
   defp config_env_var?("MIX_ENV"), do: true
   defp config_env_var?(_), do: false
 
-  if Mix.target() == :app do
-    defp app_specs, do: [LivebookApp]
-  else
-    defp app_specs, do: []
-  end
-
-  defp deploy_apps() do
+  defp load_apps_dir() do
     if apps_path = Livebook.Config.apps_path() do
-      warmup = Livebook.Config.apps_path_warmup() == :auto
+      should_warmup = Livebook.Config.apps_path_warmup() == :auto
 
-      Livebook.Apps.deploy_apps_in_dir(apps_path,
-        password: Livebook.Config.apps_path_password(),
-        warmup: warmup,
-        start_only: true
-      )
-    end
-  end
+      specs =
+        Livebook.Apps.build_app_specs_in_dir(apps_path,
+          password: Livebook.Config.apps_path_password(),
+          hub_id: Livebook.Config.apps_path_hub_id(),
+          should_warmup: should_warmup
+        )
 
-  defp iframe_server_specs() do
-    server? = Phoenix.Endpoint.server?(:livebook, LivebookWeb.Endpoint)
-    port = Livebook.Config.iframe_port()
-
-    if server? do
-      http = Application.fetch_env!(:livebook, LivebookWeb.Endpoint)[:http]
-
-      iframe_opts =
-        [
-          scheme: :http,
-          plug: LivebookWeb.IframeEndpoint,
-          port: port,
-          thousand_island_options: [supervisor_options: [name: LivebookWeb.IframeEndpoint]]
-        ] ++ Keyword.take(http, [:ip])
-
-      [{Bandit, iframe_opts}]
-    else
-      []
+      Livebook.Apps.set_startup_app_specs(specs)
     end
   end
 

lib/livebook/apps.ex

@@ -1,5 +1,5 @@
 defmodule Livebook.Apps do
-  # This module is responsible for starting and discovering apps.
+  # Top-level module for keeping track of apps.
   #
   # App processes are tracked using `Livebook.Tracker` in the same way
   # that sessions are.
@@ -8,88 +8,12 @@ defmodule Livebook.Apps do
 
   alias Livebook.App
 
-  @doc """
-  Deploys the given notebook as an app.
-
-  If there is no app process under the corresponding slug, it is started.
-  Otherwise the notebook is deployed as a new version into the existing
-  app.
-
-  ## Options
-
-    * `:warnings` - a list of warnings to show for the new deployment
-
-    * `:files_source` - a location to fetch notebook files from, see
-      `Livebook.Session.start_link/1` for more details
-
-    * `:start_only` - when `true`, deploys only if the app does not
-      exist already. Defaults to `false`
-
-  """
-  @spec deploy(Livebook.Notebook.t(), keyword()) ::
-          {:ok, pid()} | {:error, :already_started} | {:error, term()}
-  def deploy(notebook, opts \\ []) do
-    opts = Keyword.validate!(opts, warnings: [], files_source: nil, start_only: false)
-
-    slug = notebook.app_settings.slug
-    name = name(slug)
-
-    case :global.whereis_name(name) do
-      :undefined ->
-        :global.trans({{:app_registration, name}, node()}, fn ->
-          case :global.whereis_name(name) do
-            :undefined ->
-              with {:ok, pid} <- start_app(notebook, opts[:warnings], opts[:files_source]) do
-                :yes = :global.register_name(name, pid)
-                {:ok, pid}
-              end
-
-            pid ->
-              redeploy_app(pid, notebook, opts)
-          end
-        end)
-
-      pid ->
-        redeploy_app(pid, notebook, opts)
-    end
-  end
-
-  defp start_app(notebook, warnings, files_source) do
-    opts = [notebook: notebook, warnings: warnings, files_source: files_source]
-
-    case DynamicSupervisor.start_child(Livebook.AppSupervisor, {App, opts}) do
-      {:ok, pid} ->
-        app = App.get_by_pid(pid)
-
-        case Livebook.Tracker.track_app(app) do
-          :ok ->
-            {:ok, pid}
-
-          {:error, reason} ->
-            App.close(pid)
-            {:error, reason}
-        end
-
-      {:error, reason} ->
-        {:error, reason}
-    end
-  end
-
-  defp redeploy_app(pid, notebook, opts) do
-    if opts[:start_only] do
-      {:error, :already_started}
-    else
-      App.deploy(pid, notebook, warnings: opts[:warnings], files_source: opts[:files_source])
-      {:ok, pid}
-    end
-  end
-
   @doc """
   Returns app process pid for the given slug.
   """
   @spec fetch_pid(App.slug()) :: {:ok, pid()} | :error
   def fetch_pid(slug) do
-    case :global.whereis_name(name(slug)) do
+    case :global.whereis_name(global_name(slug)) do
       :undefined -> :error
       pid -> {:ok, pid}
     end
@@ -100,9 +24,8 @@ defmodule Livebook.Apps do
   """
   @spec fetch_app(App.slug()) :: {:ok, App.t()} | :error
   def fetch_app(slug) do
-    case :global.whereis_name(name(slug)) do
-      :undefined -> :error
-      pid -> {:ok, App.get_by_pid(pid)}
+    with {:ok, pid} <- fetch_pid(slug) do
+      {:ok, App.get_by_pid(pid)}
     end
   end
 
@@ -111,7 +34,7 @@ defmodule Livebook.Apps do
   """
   @spec exists?(App.slug()) :: boolean()
   def exists?(slug) do
-    :global.whereis_name(name(slug)) != :undefined
+    :global.whereis_name(global_name(slug)) != :undefined
   end
 
   @doc """
@@ -125,7 +48,11 @@ defmodule Livebook.Apps do
     end
   end
 
-  defp name(slug), do: {:app, slug}
+  @doc """
+  Returns a global registration name for the given app slug.
+  """
+  @spec global_name(App.slug()) :: term()
+  def global_name(slug), do: {:app, slug}
 
   @doc """
   Returns all the running apps.
@@ -159,30 +86,71 @@ defmodule Livebook.Apps do
   end
 
   @doc """
-  Deploys an app for each notebook in the given directory.
+  Checks if the apps directory is configured and contains no notebooks.
+  """
+  @spec empty_apps_path?() :: boolean()
+  def empty_apps_path?() do
+    if path = Livebook.Config.apps_path() do
+      pattern = Path.join([path, "**", "*.livemd"])
+      Path.wildcard(pattern) == []
+    else
+      false
+    end
+  end
+
+  @doc """
+  Gets all permanent app specs.
+
+  Permanent apps are apps that should be kept running in the cluster.
+  This includes apps from local directory and apps from hubs.
+  """
+  @spec get_permanent_app_specs() :: list(Livebook.Apps.AppSpec.t())
+  def get_permanent_app_specs() do
+    app_specs = get_startup_app_specs() ++ Livebook.Hubs.get_app_specs()
+
+    # Just in case there is a slug conflict between startup specs and
+    # hub specs, we ensure slug uniqueness
+    Enum.uniq_by(app_specs, & &1.slug)
+  end
+
+  @startup_app_specs_key :livebook_startup_app_specs
+
+  @doc """
+  Sets permanent app specs that are kept only in memory.
+  """
+  @spec set_startup_app_specs(list(Livebook.Apps.AppSpec.t())) :: :ok
+  def set_startup_app_specs(app_specs) do
+    :persistent_term.put(@startup_app_specs_key, app_specs)
+  end
+
+  @doc """
+  Gets the startup permanent app specs.
+  """
+  @spec get_startup_app_specs() :: list(Livebook.Apps.AppSpec.t())
+  def get_startup_app_specs() do
+    :persistent_term.get(@startup_app_specs_key, [])
+  end
+
+  @doc """
+  Builds app specs for notebooks in the given directory.
 
   ## Options
 
     * `:password` - a password to set for every loaded app
 
-    * `:warmup` - when `true`, run setup cell for each of the
-      notebooks before the actual deployment. The setup cells are
-      run one by one to avoid race conditions. Defaults to `true`
+    * `:should_warmup` - whether the app should be warmed up before
+      deployment. Disabling warmup makes sense if the app setup has
+      already been cached. Defaults to `true`
 
-    * `:skip_deploy` - when `true`, the apps are not deployed.
-      This can be used to warmup apps without deployment. Defaults
-      to `false`
-
-    * `:start_only` - when `true`, deploys only if the app does not
-      exist already. Defaults to `false`
+    * `:hub_id` - when set, only imports notebooks from the given hub,
+      other notebooks are ignored
 
   """
-  @spec deploy_apps_in_dir(String.t(), keyword()) :: :ok
-  def deploy_apps_in_dir(path, opts \\ []) do
-    opts =
-      Keyword.validate!(opts, [:password, warmup: true, skip_deploy: false, start_only: false])
+  @spec build_app_specs_in_dir(String.t(), keyword()) :: list(Livebook.Apps.AppSpec.t())
+  def build_app_specs_in_dir(path, opts \\ []) do
+    opts = Keyword.validate!(opts, [:password, :hub_id, should_warmup: true])
 
-    infos = import_app_notebooks(path)
+    infos = import_app_notebooks(path, opts[:hub_id])
 
     if infos == [] do
       Logger.warning("No .livemd files were found for deployment at #{path}")
@@ -190,73 +158,66 @@ defmodule Livebook.Apps do
 
     for %{status: {:error, message}} = info <- infos do
       Logger.warning(
-        "Skipping deployment for app at #{info.relative_path}. #{Livebook.Utils.upcase_first(message)}."
+        "Ignoring app at #{info.relative_path}. #{Livebook.Utils.upcase_first(message)}."
       )
     end
 
     infos = Enum.filter(infos, &(&1.status == :ok))
 
+    infos =
+      infos
+      |> Enum.reduce({[], %{}}, fn info, {infos, slugs} ->
+        slug = info.notebook.app_settings.slug
+
+        case slugs do
+          %{^slug => other_path} ->
+            Logger.warning(
+              "Ignoring app at #{info.relative_path}. App with the same slug (#{slug}) is already present at #{other_path}"
+            )
+
+            {infos, slugs}
+
+          %{} ->
+            {[info | infos], Map.put(slugs, slug, info.relative_path)}
+        end
+      end)
+      |> elem(0)
+      |> Enum.reverse()
+
     for info <- infos, info.import_warnings != [] do
       items = Enum.map(info.import_warnings, &("- " <> &1))
 
       Logger.warning(
-        "Found warnings while importing app notebook at #{info.relative_path}:\n\n" <>
-          Enum.join(items, "\n")
+        "Found warnings while importing app notebook at #{info.relative_path}:\n  " <>
+          Enum.join(items, "\n  ")
       )
     end
 
-    if infos != [] and opts[:warmup] do
-      Logger.info("Running app warmups")
-
-      for info <- infos do
-        with {:error, message} <- run_app_setup_sync(info.notebook, info.files_source) do
-          Logger.warning(
-            "Failed to run setup for app at #{info.relative_path}. #{Livebook.Utils.upcase_first(message)}."
-          )
-        end
-      end
-    end
-
-    if infos != [] and not opts[:skip_deploy] do
-      Logger.info("Deploying apps")
-
-      for %{notebook: notebook} = info <- infos do
-        notebook =
-          if password = opts[:password] do
-            put_in(notebook.app_settings.password, password)
-          else
-            if notebook.app_settings.access_type == :protected do
-              Logger.warning(
-                "The app at #{info.relative_path} will use a random password." <>
-                  " You may want to set LIVEBOOK_APPS_PATH_PASSWORD or make the app public."
-              )
-            end
-
-            notebook
-          end
-
-        warnings = Enum.map(info.import_warnings, &("Import: " <> &1))
-
-        deploy(notebook,
-          warnings: warnings,
-          files_source: info.files_source,
-          start_only: opts[:start_only]
+    for %{notebook: notebook} = info <- infos do
+      if opts[:password] == nil and notebook.app_settings.access_type == :protected do
+        Logger.warning(
+          "The app at #{info.relative_path} will use a random password." <>
+            " You may want to set LIVEBOOK_APPS_PATH_PASSWORD or make the app public."
         )
       end
-    end
 
-    :ok
+      %Livebook.Apps.PathAppSpec{
+        slug: notebook.app_settings.slug,
+        path: info.path,
+        password: opts[:password],
+        should_warmup: opts[:should_warmup]
+      }
+    end
   end
 
-  defp import_app_notebooks(dir) do
+  defp import_app_notebooks(dir, hub_id) do
     pattern = Path.join([dir, "**", "*.livemd"])
 
     for path <- Path.wildcard(pattern) do
       markdown = File.read!(path)
 
-      {notebook, warnings} = Livebook.LiveMarkdown.notebook_from_livemd(markdown)
-
-      apps_path_hub_id = Livebook.Config.apps_path_hub_id()
+      {notebook, %{warnings: warnings, stamp_verified?: stamp_verified?}} =
+        Livebook.LiveMarkdown.notebook_from_livemd(markdown)
 
       status =
         cond do
@@ -264,8 +225,17 @@ defmodule Livebook.Apps do
             {:error,
              "the deployment settings are missing or invalid. Please configure them under the notebook deploy panel"}
 
-          apps_path_hub_id && apps_path_hub_id != notebook.hub_id ->
-            {:error, "the notebook is not verified to come from hub #{apps_path_hub_id}"}
+          # We only import notebooks from the given hub, but if that
+          # option is set then there should really be no other ones,
+          # so it makes sense to warn if there are
+          hub_id && notebook.hub_id != hub_id ->
+            {:error, "the notebook does not come from hub #{hub_id}"}
+
+          # We only deploy apps with valid stamp. We make an exception
+          # for personal hub, because the deployment instance has a
+          # different personal secret key anyway
+          notebook.hub_id != Livebook.Hubs.Personal.id() and not stamp_verified? ->
+            {:error, "the notebook does not have a valid stamp"}
 
           true ->
             :ok
@@ -275,6 +245,7 @@ defmodule Livebook.Apps do
       files_dir = Livebook.Session.files_dir_for_notebook(notebook_file)
 
       %{
+        path: path,
         relative_path: Path.relative_to(path, dir),
         status: status,
         notebook: notebook,
@@ -284,7 +255,54 @@ defmodule Livebook.Apps do
     end
   end
 
-  defp run_app_setup_sync(notebook, files_source) do
+  @doc """
+  Returns a temporary dir for app files.
+  """
+  @spec generate_files_tmp_path(String.t()) :: String.t()
+  def generate_files_tmp_path(slug) do
+    Path.join([
+      Livebook.Config.tmp_path(),
+      "app_files",
+      slug <> Livebook.Utils.random_short_id()
+    ])
+  end
+
+  @doc """
+  Runs app warmup.
+
+  This evaluates the setup cell in the given notebook to populate the
+  relevant caches, such as dependency installation.
+  """
+  @spec warmup_app(Livebook.Notebook.t(), String.t()) :: :ok | {:error, String.t()}
+  def warmup_app(notebook, files_tmp_path) do
+    run_app_setup_sync(notebook, files_tmp_path)
+  end
+
+  @doc """
+  Same as `warmup_app/2`, but loads app spec and immediately cleans up
+  afterwards.
+  """
+  @spec warmup_app(Livebook.Apps.AppSpec.t()) :: :ok | {:error, String.t()}
+  def warmup_app(app_spec) do
+    files_tmp_path = generate_files_tmp_path(app_spec.slug)
+
+    result =
+      with {:ok, %{notebook: notebook}} <- Livebook.Apps.AppSpec.load(app_spec, files_tmp_path) do
+        warmup_app(notebook, files_tmp_path)
+      end
+
+    File.rm_rf(files_tmp_path)
+
+    result
+  end
+
+  defp run_app_setup_sync(notebook, files_tmp_path) do
+    files_source =
+      {:dir,
+       files_tmp_path
+       |> Livebook.FileSystem.Utils.ensure_dir_path()
+       |> Livebook.FileSystem.File.local()}
+
     notebook = %{notebook | sections: []}
 
     opts = [
@@ -317,17 +335,4 @@ defmodule Livebook.Apps do
         {:error, "failed to start session, reason: #{inspect(reason)}"}
     end
   end
-
-  @doc """
-  Checks if the apps directory is configured and contains no notebooks.
-  """
-  @spec empty_apps_path?() :: boolean()
-  def empty_apps_path?() do
-    if path = Livebook.Config.apps_path() do
-      pattern = Path.join([path, "**", "*.livemd"])
-      Path.wildcard(pattern) == []
-    else
-      false
-    end
-  end
 end

lib/livebook/apps/app_spec.ex

@@ -0,0 +1,35 @@
+defprotocol Livebook.Apps.AppSpec do
+  # This protocol defines an interface for an app blueprint.
+  #
+  # An app spec is used to encapsulate information about how to load
+  # the source for an app. The spec is used by `Livebook.Apps.Manager`
+  # to deploy permanent apps across the cluster.
+  #
+  # Every struct implementing this protocol is also expected to have
+  # the `:slug` and `:version` (string) attributes.
+
+  @doc """
+  Loads the app notebook and other metadata relevant for deployment.
+
+  This function may load the notebook from an external source, so the
+  caller should avoid calling it multiple times.
+
+  The function receives a directory that the notebook files should be
+  copied into. It is the responsibility of the caller to remove this
+  directory, regardless of whether loading succeeds or fails.
+  """
+  @spec load(t(), String.t()) ::
+          {:ok, %{notebook: Livebook.Notebook.t(), warnings: list(String.t())}}
+          | {:error, String.t()}
+  def load(app_spec, files_tmp_path)
+
+  @doc """
+  Returns whether warmup procedure should run, before deploying the
+  app.
+
+  You may want to skip warmup, if the app has already been warmed up
+  beforehand.
+  """
+  @spec should_warmup?(t()) :: boolean()
+  def should_warmup?(app_spec)
+end

lib/livebook/apps/deployer.ex

@@ -0,0 +1,160 @@
+defmodule Livebook.Apps.Deployer do
+  # Deploys apps on the given node.
+  #
+  # Each node runs a single deployer, which deploys apps sequentially.
+  # This design makes sure that only one app warmup is going to run
+  # at a time, rather than concurrently, which reduces memory usage
+  # and prevents race conditions with `Mix.install/2`.
+
+  use GenServer
+
+  require Logger
+
+  alias Livebook.App
+  alias Livebook.Apps
+
+  @doc """
+  Starts a new deployer process.
+  """
+  @spec start_link(keyword()) :: GenServer.on_start()
+  def start_link(_opts) do
+    GenServer.start_link(__MODULE__, {})
+  end
+
+  @doc """
+  Asynchronously requests an app deployment.
+
+  If there is no app process under the requested slug, it is started.
+  Otherwise the notebook is deployed as a new version into the existing
+  app.
+
+  This function automatically starts monitoring the deployer process
+  and returns the monitor reference.
+
+  When the deployment finished the caller receives a message of the
+  form `{:deploy_result, ref, result}`, with the same monitor reference
+  and the result of `Apps.deploy/2`. The caller should then
+  demonitor the reference.
+
+  ## Options
+
+    * `:start_only` - when `true`, deploys only if the app does not
+      exist already. If the app does exist, the deployment finishes
+      with an error. Defaults to `false`
+
+    * `:permanent` - whether the app is deployed as permanent. Defaults
+      to `false`
+
+  """
+  @spec deploy_monitor(pid(), Apps.AppSpec.t(), keyword()) :: reference()
+  def deploy_monitor(pid, app_spec, opts \\ []) do
+    opts = Keyword.validate!(opts, start_only: false, permanent: false)
+
+    ref = Process.monitor(pid)
+    GenServer.cast(pid, {:deploy, app_spec, opts[:start_only], opts[:permanent], self(), ref})
+    ref
+  end
+
+  @doc """
+  Returns a list of all deployers present in the cluster.
+  """
+  @spec list_deployers() :: list(pid())
+  def list_deployers() do
+    :pg.get_members(Apps.Deployer.PG, Apps.Deployer)
+  end
+
+  @doc """
+  Returns a node-local deployer.
+  """
+  @spec local_deployer(node()) :: pid()
+  def local_deployer(node \\ node()) do
+    list_deployers() |> Enum.find(&(node(&1) == node)) || raise "no local deployer running"
+  end
+
+  @impl true
+  def init({}) do
+    :pg.join(Apps.Deployer.PG, __MODULE__, self())
+
+    {:ok, %{}}
+  end
+
+  @impl true
+  def handle_cast({:deploy, app_spec, start_only, permanent, from, ref}, state) do
+    Logger.info("[app=#{app_spec.slug}] Deploying app")
+
+    files_tmp_path = Apps.generate_files_tmp_path(app_spec.slug)
+
+    result =
+      with {:ok, %{notebook: notebook, warnings: warnings}} <-
+             Apps.AppSpec.load(app_spec, files_tmp_path) do
+        if Apps.AppSpec.should_warmup?(app_spec) do
+          with {:error, message} <- Apps.warmup_app(notebook, files_tmp_path) do
+            Logger.warning("[app=#{app_spec.slug}] App warmup failed, #{message}")
+          end
+        end
+
+        deployment_bundle = %{
+          notebook: notebook,
+          files_tmp_path: files_tmp_path,
+          app_spec: app_spec,
+          permanent: permanent,
+          warnings: warnings
+        }
+
+        name = Apps.global_name(app_spec.slug)
+        opts = [start_only: start_only]
+
+        with {:error, error} <- start_or_redeploy(name, deployment_bundle, opts) do
+          File.rm_rf(files_tmp_path)
+          {:error, error}
+        end
+      end
+
+    send(from, {:deploy_result, ref, result})
+
+    {:noreply, state}
+  end
+
+  defp start_or_redeploy(name, deployment_bundle, opts) do
+    case :global.whereis_name(name) do
+      :undefined ->
+        opts = [deployment_bundle: deployment_bundle]
+
+        case DynamicSupervisor.start_child(Livebook.AppSupervisor, {App, opts}) do
+          {:ok, pid} ->
+            {:ok, pid}
+
+          {:error, :already_started} ->
+            # We could use a global transaction to prevent the unlikely
+            # case of multiple nodes starting the app simultaneously.
+            # However, the global registration can still fail if the
+            # node joins the cluster while the app process is starting.
+            # So we handle both cases the same way in this branch.
+            start_or_redeploy(name, deployment_bundle, opts)
+
+          {:error, reason} ->
+            {:error, "app process failed to start, reason: #{inspect(reason)}"}
+        end
+
+      pid ->
+        redeploy_app(pid, deployment_bundle, opts)
+    end
+  end
+
+  defp redeploy_app(pid, deployment_bundle, opts) do
+    cond do
+      opts[:start_only] ->
+        {:error, :already_started}
+
+      node(pid) != node() ->
+        # This is relevant specifically when :files_source points to
+        # a local directory, but we generally expect redeployments to
+        # happen within the same node, so it's good to enforce this
+        {:error, "cannot deploy a new notebook to an app running on a different node"}
+
+      true ->
+        App.deploy(pid, deployment_bundle)
+        {:ok, pid}
+    end
+  end
+end

lib/livebook/apps/deployment_supervisor.ex

@@ -0,0 +1,40 @@
+defmodule Livebook.Apps.DeploymentSupervisor do
+  # Supervision tree related to orchestrating app deployments in the
+  # cluster.
+
+  use Supervisor
+
+  @name __MODULE__
+
+  @doc """
+  Starts the supervisor.
+  """
+  @spec start_link(keyword()) :: GenServer.on_start()
+  def start_link(_opts) do
+    Supervisor.start_link(__MODULE__, {}, name: @name)
+  end
+
+  @doc """
+  Starts a node-local instance of `Livebook.Apps.Manager`.
+  """
+  @spec start_manager() :: Supervisor.on_start_child()
+  def start_manager() do
+    Supervisor.start_child(@name, Livebook.Apps.Manager)
+  end
+
+  @impl true
+  def init({}) do
+    children = [
+      # Start the supervisor dynamically managing apps
+      {DynamicSupervisor, name: Livebook.AppSupervisor, strategy: :one_for_one},
+      # Process group for app deployers
+      %{id: Livebook.Apps.Deployer.PG, start: {:pg, :start_link, [Livebook.Apps.Deployer.PG]}},
+      # Node-local app deployer
+      Livebook.Apps.Deployer,
+      # Node-local app manager watcher
+      Livebook.Apps.ManagerWatcher
+    ]
+
+    Supervisor.init(children, strategy: :one_for_one)
+  end
+end

lib/livebook/apps/manager.ex

@@ -0,0 +1,249 @@
+defmodule Livebook.Apps.Manager do
+  # Orchestrates permanent app deployments across the cluster.
+  #
+  # Only a single instance of the manager runs in the cluster. This is
+  # ensured by registering it in :global. Each node also runs a single
+  # instance of `Livebook.Apps.ManagerWatcher`, which starts a new
+  # manager whenever needed.
+
+  use GenServer, restart: :temporary
+
+  require Logger
+
+  alias Livebook.Apps
+
+  @name __MODULE__
+
+  config = Application.compile_env(:livebook, __MODULE__)
+
+  @handle_app_close_debounce_ms 100
+  @retry_backoff_base_ms Keyword.fetch!(config, :retry_backoff_base_ms)
+
+  @doc """
+  Starts a new manager process.
+  """
+  @spec start_link(keyword()) :: GenServer.on_start()
+  def start_link(_opts) do
+    GenServer.start_link(__MODULE__, {}, name: {:global, @name})
+  end
+
+  @doc """
+  Checks if the manager is running on the current node.
+  """
+  @spec local?() :: boolean()
+  def local?() do
+    case :global.whereis_name(@name) do
+      :undefined -> false
+      pid -> node(pid) == node()
+    end
+  end
+
+  @doc """
+  Asynchronously asks the manager to reflect the currently configured
+  permanent apps.
+
+  Any permanent apps that are not running will be deployed. Apps that
+  should no longer be running will be closed.
+  """
+  @spec sync_permanent_apps() :: :ok
+  def sync_permanent_apps() do
+    GenServer.cast({:global, @name}, :sync_permanent_apps)
+  end
+
+  @impl true
+  def init({}) do
+    Apps.subscribe()
+
+    state = %{deployments: %{}, handle_app_close_timer_ref: nil}
+
+    {:ok, state, {:continue, :after_init}}
+  end
+
+  @impl true
+  def handle_continue(:after_init, state) do
+    {:noreply, sync_apps(state)}
+  end
+
+  @impl true
+  def handle_cast(:sync_permanent_apps, state) do
+    {:noreply, sync_apps(state)}
+  end
+
+  @impl true
+  def handle_info({:app_closed, _app}, state) do
+    # When a new node joins the cluster it has a node-local instance
+    # of the manager and it may have deployed apps. :global may resolve
+    # app conflicts before the manager conflicts (so both managers keep
+    # running). The app conflict resolution involves closing one of the
+    # running apps, which would cause the app node-local tracker emit
+    # :app_closed to that node-local manager immediately. At that point
+    # the given app may not be registered in :global temporarily, which
+    # would make us redeploy it unnecessarily.
+    #
+    # To avoid race conditions like that, we debounce the redeployment
+    # to give more time for conflicts to be resolved and the system to
+    # settle down.
+
+    if ref = state.handle_app_close_timer_ref do
+      Process.cancel_timer(ref)
+    end
+
+    handle_app_close_timer_ref =
+      Process.send_after(self(), :handle_app_close, @handle_app_close_debounce_ms)
+
+    {:noreply, %{state | handle_app_close_timer_ref: handle_app_close_timer_ref}}
+  end
+
+  def handle_info(:handle_app_close, state) do
+    state = %{state | handle_app_close_timer_ref: nil}
+    {:noreply, sync_apps(state)}
+  end
+
+  def handle_info({:DOWN, ref, :process, _pid, reason}, state) do
+    deployment = deployment_by_ref(state, ref)
+    message = "deployer terminated unexpectedly, reason: #{Exception.format_exit(reason)}"
+    {:noreply, handle_deployment_failure(state, deployment, message)}
+  end
+
+  def handle_info({:deploy_result, ref, result}, state) do
+    Process.demonitor(ref, [:flush])
+
+    deployment = deployment_by_ref(state, ref)
+
+    case result do
+      {:ok, _pid} ->
+        {_, state} = pop_in(state.deployments[deployment.slug])
+        Logger.info("[app=#{deployment.slug}] Deployment successful, app running")
+        # Run sync in case there is already a new version for this app
+        {:noreply, sync_apps(state)}
+
+      {:error, error} ->
+        {:noreply, handle_deployment_failure(state, deployment, error)}
+    end
+  end
+
+  def handle_info({:retry, slug}, state) do
+    {:noreply, retry(state, slug)}
+  end
+
+  def handle_info(_message, state) do
+    {:noreply, state}
+  end
+
+  defp sync_apps(state) do
+    permanent_app_specs = Apps.get_permanent_app_specs()
+    state = deploy_missing_apps(state, permanent_app_specs)
+    close_leftover_apps(permanent_app_specs)
+    state
+  end
+
+  defp deploy_missing_apps(state, permanent_app_specs) do
+    for app_spec <- permanent_app_specs,
+        not Map.has_key?(state.deployments, app_spec.slug),
+        reduce: state do
+      state ->
+        case fetch_app(app_spec.slug) do
+          {:ok, _state, app} when app.app_spec.version == app_spec.version ->
+            state
+
+          {:ok, :reachable, app} ->
+            ref = redeploy(app, app_spec)
+            track_deployment(state, app_spec, ref)
+
+          {:ok, :unreachable, _app} ->
+            state
+
+          :error ->
+            ref = deploy(app_spec)
+            track_deployment(state, app_spec, ref)
+        end
+    end
+  end
+
+  defp close_leftover_apps(permanent_app_specs) do
+    permanent_slugs = MapSet.new(permanent_app_specs, & &1.slug)
+
+    for app <- Apps.list_apps(),
+        app.permanent,
+        app.slug not in permanent_slugs do
+      Livebook.App.close_async(app.pid)
+    end
+  end
+
+  defp fetch_app(slug) do
+    # We check both global and the tracker. The app may be present in
+    # the tracker for longer, but if it is actually down we will get
+    # the :app_closed event eventually. On the other hand, if the app
+    # is already somewhere, global will tell us sooner than tracker.
+    case Apps.fetch_app(slug) do
+      {:ok, app} ->
+        {:ok, :reachable, app}
+
+      :error ->
+        case Livebook.Tracker.fetch_app(slug) do
+          {:ok, app} -> {:ok, :unreachable, app}
+          :error -> :error
+        end
+    end
+  end
+
+  defp app_definitely_down?(slug) do
+    not Apps.exists?(slug) and Livebook.Tracker.fetch_app(slug) == :error
+  end
+
+  defp deploy(app_spec) do
+    deployer_pid = Livebook.Apps.Deployer.list_deployers() |> Enum.random()
+    Logger.info("[app=#{app_spec.slug}] Scheduling app deployment on node #{node(deployer_pid)}")
+
+    Livebook.Apps.Deployer.deploy_monitor(deployer_pid, app_spec,
+      permanent: true,
+      start_only: true
+    )
+  end
+
+  defp redeploy(app, app_spec) do
+    # Redeploying pushes the new notebook to an existing app process,
+    # so we need to run deployment on the same node
+    node = node(app.pid)
+    deployer_pid = Livebook.Apps.Deployer.local_deployer(node)
+    Logger.info("[app=#{app_spec.slug}] Scheduling app deployment on node #{node(deployer_pid)}")
+    Livebook.Apps.Deployer.deploy_monitor(deployer_pid, app_spec, permanent: true)
+  end
+
+  defp track_deployment(state, app_spec, ref) do
+    put_in(state.deployments[app_spec.slug], %{
+      ref: ref,
+      retries: 0,
+      app_spec: app_spec,
+      slug: app_spec.slug
+    })
+  end
+
+  defp handle_deployment_failure(state, deployment, message) do
+    Logger.error("[app=#{deployment.slug}] Deployment failed, #{message}")
+
+    # Schedule retry
+    %{app_spec: app_spec, retries: retries} = deployment
+    retries = retries + 1
+    time = @retry_backoff_base_ms * min(retries, 6)
+    Process.send_after(self(), {:retry, app_spec.slug}, time)
+    put_in(state.deployments[app_spec.slug].retries, retries)
+  end
+
+  defp retry(state, slug) do
+    if app_definitely_down?(slug) do
+      %{app_spec: app_spec} = state.deployments[slug]
+      ref = deploy(app_spec)
+      put_in(state.deployments[slug].ref, ref)
+    else
+      {_, state} = pop_in(state.deployments[slug])
+      state
+    end
+  end
+
+  defp deployment_by_ref(state, ref) do
+    Enum.find_value(state.deployments, fn {_slug, deployment} ->
+      deployment.ref == ref && deployment
+    end)
+  end
+end

lib/livebook/apps/manager_watcher.ex

@@ -0,0 +1,55 @@
+defmodule Livebook.Apps.ManagerWatcher do
+  # Monitors the global `Livebook.Apps.Manager` and starts a new one
+  # whenever needed.
+
+  use GenServer
+
+  @name __MODULE__
+
+  @doc """
+  Starts a new watcher process.
+  """
+  @spec start_link(keyword()) :: GenServer.on_start()
+  def start_link(_opts) do
+    GenServer.start_link(__MODULE__, {}, name: @name)
+  end
+
+  @impl true
+  def init({}) do
+    # At this point the DeploymentSupervisor is still starting, so we
+    # start the Manager in handle_continue to avoid a dead lock
+    {:ok, {}, {:continue, :after_init}}
+  end
+
+  @impl true
+  def handle_continue(:after_init, {}) do
+    monitor_ref = maybe_start_and_monitor()
+    {:noreply, %{monitor_ref: monitor_ref}}
+  end
+
+  @impl true
+  def handle_info({:DOWN, ref, :process, _pid, _reason}, state) when ref == state.monitor_ref do
+    monitor_ref = maybe_start_and_monitor()
+    {:noreply, %{monitor_ref: monitor_ref}}
+  end
+
+  defp maybe_start_and_monitor() do
+    pid =
+      case Livebook.Apps.DeploymentSupervisor.start_manager() do
+        {:ok, pid} ->
+          report_manager_started(pid)
+          pid
+
+        {:error, {:already_started, pid}} ->
+          pid
+      end
+
+    Process.monitor(pid)
+  end
+
+  defp report_manager_started(pid) do
+    # We use this specifically for tests
+    message = {:manager_started, pid}
+    Phoenix.PubSub.direct_broadcast!(node(), Livebook.PubSub, "manager_watcher", message)
+  end
+end

lib/livebook/apps/path_app_spec.ex

@@ -0,0 +1,43 @@
+defmodule Livebook.Apps.PathAppSpec do
+  # App spec pointing to an app notebook in the local file system.
+  #
+  # Note that the struct holds the path, rather than the notebook and
+  # files source. The app spec may be sent across nodes and processes,
+  # so this way we do less copying.
+
+  @enforce_keys [:slug, :path]
+
+  defstruct [:slug, :path, :password, should_warmup: true, version: "1"]
+end
+
+defimpl Livebook.Apps.AppSpec, for: Livebook.Apps.PathAppSpec do
+  def load(app_spec, files_tmp_path) do
+    case File.read(app_spec.path) do
+      {:ok, source} ->
+        {notebook, %{warnings: warnings}} = Livebook.LiveMarkdown.notebook_from_livemd(source)
+        notebook = put_in(notebook.app_settings.password, app_spec.password)
+
+        notebook_file = Livebook.FileSystem.File.local(app_spec.path)
+        files_dir = Livebook.Session.files_dir_for_notebook(notebook_file)
+
+        warnings = Enum.map(warnings, &("Import: " <> &1))
+
+        files_tmp_dir =
+          files_tmp_path
+          |> Livebook.FileSystem.Utils.ensure_dir_path()
+          |> Livebook.FileSystem.File.local()
+
+        with :ok <- Livebook.Notebook.copy_files(notebook, files_dir, files_tmp_dir) do
+          {:ok, %{notebook: notebook, warnings: warnings}}
+        end
+
+      {:error, reason} ->
+        {:error,
+         "failed to read app notebook file at #{app_spec.path}, reason: #{:file.format_error(reason)}"}
+    end
+  end
+
+  def should_warmup?(app_spec) do
+    app_spec.should_warmup
+  end
+end

lib/livebook/apps/preview_app_spec.ex

@@ -0,0 +1,39 @@
+defmodule Livebook.Apps.PreviewAppSpec do
+  # App spec for deploying notebook directly from a session.
+  #
+  # This spec is used when deploying apps directly from the session,
+  # which we call previews.
+  #
+  # The notebook files may be stored on the local file system, so this
+  # app spec should always be deployed on the same node where it is
+  # built.
+
+  @enforce_keys [:slug, :session_id]
+
+  defstruct [:slug, :session_id, version: "1"]
+end
+
+defimpl Livebook.Apps.AppSpec, for: Livebook.Apps.PreviewAppSpec do
+  def load(app_spec, files_tmp_path) do
+    case Livebook.Sessions.fetch_session(app_spec.session_id) do
+      {:ok, %{pid: pid, files_dir: files_dir}} ->
+        notebook = Livebook.Session.get_notebook(pid)
+
+        files_tmp_dir =
+          files_tmp_path
+          |> Livebook.FileSystem.Utils.ensure_dir_path()
+          |> Livebook.FileSystem.File.local()
+
+        with :ok <- Livebook.Notebook.copy_files(notebook, files_dir, files_tmp_dir) do
+          {:ok, %{notebook: notebook, warnings: []}}
+        end
+
+      {:error, _reason} ->
+        {:error, "the session has already been closed"}
+    end
+  end
+
+  def should_warmup?(_app_spec) do
+    false
+  end
+end

lib/livebook/apps/teams_app_spec.ex

@@ -0,0 +1,95 @@
+defmodule Livebook.Apps.TeamsAppSpec do
+  # App spec for organization apps fetched from Livebook Teams.
+
+  @enforce_keys [:slug, :version, :hub_id, :app_deployment_id]
+
+  defstruct [:slug, :version, :hub_id, :app_deployment_id]
+end
+
+defimpl Livebook.Apps.AppSpec, for: Livebook.Apps.TeamsAppSpec do
+  alias Livebook.Hubs
+
+  def load(app_spec, files_tmp_path) do
+    with {:ok, hub} <- fetch_hub(app_spec.hub_id),
+         {:ok, app_deployment, derived_key} <-
+           fetch_download_info(hub.id, app_spec.app_deployment_id),
+         {:ok, encrypted_binary} <- download(hub, app_deployment),
+         {:ok, archive_binary} <- decrypt(encrypted_binary, derived_key),
+         {:ok, notebook_source} <- unzip(archive_binary, files_tmp_path) do
+      {notebook, %{warnings: warnings}} =
+        Livebook.LiveMarkdown.notebook_from_livemd(notebook_source)
+
+      {:ok, %{notebook: notebook, warnings: warnings}}
+    end
+  end
+
+  defp fetch_hub(hub_id) do
+    with :error <- Hubs.fetch_hub(hub_id) do
+      {:error, "the hub no longer exists"}
+    end
+  end
+
+  defp fetch_download_info(hub_id, app_deployment_id) do
+    with :error <- Hubs.TeamClient.get_app_deployment_download_info(hub_id, app_deployment_id) do
+      {:error, "the app deployment no longer exists"}
+    end
+  end
+
+  defp download(hub, app_deployment) do
+    case Livebook.Teams.Requests.download_revision(hub, app_deployment) do
+      {:ok, body} ->
+        {:ok, body}
+
+      {:error, %{} = error} ->
+        {:error, "downloading app archive failed, reason: #{inspect(error)}"}
+
+      {:error, message} ->
+        {:error, "downloading app archive failed, reason: #{message}"}
+
+      {:transport_error, message} ->
+        {:error, "downloading app archive failed, reason: #{message}"}
+    end
+  end
+
+  defp decrypt(binary, derived_key) do
+    with :error <- Livebook.Teams.decrypt(binary, derived_key) do
+      {:error, "failed to decrypt app archive"}
+    end
+  end
+
+  defp unzip(archive_binary, files_tmp_path) do
+    case :zip.extract(archive_binary, [:memory]) do
+      {:ok, entries} ->
+        Enum.reduce_while(entries, {:error, "notebook file missing in the app archive"}, fn
+          {~c"notebook.livemd", binary}, _acc ->
+            {:cont, {:ok, binary}}
+
+          {~c"files/" ++ name, binary}, acc ->
+            name = List.to_string(name)
+            destination = Path.join(files_tmp_path, name)
+
+            case File.write(destination, binary) do
+              :ok ->
+                {:cont, acc}
+
+              {:error, reason} ->
+                message =
+                  "failed to write a notebook file #{destination}, reason: #{:file.format_error(reason)}"
+
+                {:halt, {:error, message}}
+            end
+        end)
+
+      {:error, {name, reason}} ->
+        {:error,
+         "failed to unzip app archive, entry #{name} failed with reason: #{inspect(reason)}"}
+
+      {:error, reason} ->
+        {:error, "failed to unzip app archive, reason: #{inspect(reason)}"}
+    end
+  end
+
+  def should_warmup?(_app_spec) do
+    true
+  end
+end

lib/livebook/hubs.ex

@@ -303,4 +303,14 @@ defmodule Livebook.Hubs do
   def delete_file_system(hub, file_system) do
     Provider.delete_file_system(hub, file_system)
   end
+
+  @doc """
+  Gets a list of hub app specs.
+  """
+  @spec get_app_specs() :: list(Livebook.AppSpec.t())
+  def get_app_specs() do
+    for hub <- get_hubs(),
+        app_spec <- Provider.get_app_specs(hub),
+        do: app_spec
+  end
 end

lib/livebook/hubs/personal.ex

@@ -279,4 +279,6 @@ defimpl Livebook.Hubs.Provider, for: Livebook.Hubs.Personal do
   end
 
   def deployment_groups(_personal), do: nil
+
+  def get_app_specs(_personal), do: []
 end

lib/livebook/hubs/provider.ex

@@ -147,4 +147,10 @@ defprotocol Livebook.Hubs.Provider do
   @spec deployment_groups(t()) ::
           list(%{id: String.t(), name: String.t(), secrets: list(Secret.t())}) | nil
   def deployment_groups(hub)
+
+  @doc """
+  Gets app specs for permanent apps sourced from the given hub.
+  """
+  @spec get_app_specs(t()) :: list(Livebook.Apps.AppSpec.t())
+  def get_app_specs(hub)
 end

lib/livebook/hubs/team.ex

@@ -209,10 +209,6 @@ defimpl Livebook.Hubs.Provider, for: Livebook.Hubs.Team do
 
   def get_secrets(team), do: TeamClient.get_secrets(team.id)
 
-  @spec create_secret(Team.t(), Secret.t()) ::
-          :ok
-          | {:error, Ecto.Changeset.t()}
-          | {:transport_error, String.t()}
   def create_secret(%Team{} = team, %Secret{} = secret) do
     case Requests.create_secret(team, secret) do
       {:ok, %{"id" => _}} -> :ok
@@ -221,10 +217,6 @@ defimpl Livebook.Hubs.Provider, for: Livebook.Hubs.Team do
     end
   end
 
-  @spec update_secret(Team.t(), Secret.t()) ::
-          :ok
-          | {:error, Ecto.Changeset.t()}
-          | {:transport_error, String.t()}
   def update_secret(%Team{} = team, %Secret{} = secret) do
     case Requests.update_secret(team, secret) do
       {:ok, %{"id" => _}} -> :ok
@@ -233,10 +225,6 @@ defimpl Livebook.Hubs.Provider, for: Livebook.Hubs.Team do
     end
   end
 
-  @spec delete_secret(Team.t(), Secret.t()) ::
-          :ok
-          | {:error, Ecto.Changeset.t()}
-          | {:transport_error, String.t()}
   def delete_secret(%Team{} = team, %Secret{} = secret) do
     case Requests.delete_secret(team, secret) do
       {:ok, _} -> :ok
@@ -247,10 +235,6 @@ defimpl Livebook.Hubs.Provider, for: Livebook.Hubs.Team do
 
   def get_file_systems(team), do: TeamClient.get_file_systems(team.id)
 
-  @spec create_file_system(Team.t(), FileSystem.t()) ::
-          :ok
-          | {:error, Ecto.Changeset.t()}
-          | {:transport_error, String.t()}
   def create_file_system(%Team{} = team, file_system) do
     case Requests.create_file_system(team, file_system) do
       {:ok, %{"id" => _}} -> :ok
@@ -259,10 +243,6 @@ defimpl Livebook.Hubs.Provider, for: Livebook.Hubs.Team do
     end
   end
 
-  @spec update_file_system(Team.t(), FileSystem.t()) ::
-          :ok
-          | {:error, Ecto.Changeset.t()}
-          | {:transport_error, String.t()}
   def update_file_system(%Team{} = team, file_system) do
     case Requests.update_file_system(team, file_system) do
       {:ok, %{"id" => _}} -> :ok
@@ -271,10 +251,6 @@ defimpl Livebook.Hubs.Provider, for: Livebook.Hubs.Team do
     end
   end
 
-  @spec delete_file_system(Team.t(), FileSystem.t()) ::
-          :ok
-          | {:error, Ecto.Changeset.t()}
-          | {:transport_error, String.t()}
   def delete_file_system(%Team{} = team, file_system) do
     case Requests.delete_file_system(team, file_system) do
       {:ok, _} -> :ok
@@ -285,6 +261,19 @@ defimpl Livebook.Hubs.Provider, for: Livebook.Hubs.Team do
 
   def deployment_groups(team), do: TeamClient.get_deployment_groups(team.id)
 
+  def get_app_specs(team) do
+    app_deployments = TeamClient.get_agent_app_deployments(team.id)
+
+    for app_deployment <- app_deployments do
+      %Livebook.Apps.TeamsAppSpec{
+        slug: app_deployment.slug,
+        version: app_deployment.id,
+        hub_id: team.id,
+        app_deployment_id: app_deployment.id
+      }
+    end
+  end
+
   defp add_secret_errors(%Secret{} = secret, errors_map) do
     Requests.add_errors(secret, errors_map)
   end

lib/livebook/hubs/team_client.ex

@@ -33,6 +33,14 @@ defmodule Livebook.Hubs.TeamClient do
     GenServer.start_link(__MODULE__, team, name: registry_name(team.id))
   end
 
+  @doc """
+  Returns the client pid for the given hub id.
+  """
+  @spec get_pid(String.t()) :: pid() | nil
+  def get_pid(id) do
+    GenServer.whereis(registry_name(id))
+  end
+
   @doc """
   Stops the WebSocket server.
   """
@@ -87,6 +95,25 @@ defmodule Livebook.Hubs.TeamClient do
     GenServer.call(registry_name(id), :get_app_deployments)
   end
 
+  @doc """
+  Returns a list of cached app deployments that should be deployed on
+  this instance.
+  """
+  @spec get_agent_app_deployments(String.t()) :: list(Teams.AppDeployment.t())
+  def get_agent_app_deployments(id) do
+    GenServer.call(registry_name(id), :get_agent_app_deployments)
+  end
+
+  @doc """
+  Returns information necessary to download and decrypt archive for
+  app deployment with the given id.
+  """
+  @spec get_app_deployment_download_info(String.t(), String.t()) ::
+          {:ok, Teams.AppDeployment.t(), derived_key :: binary()} | :error
+  def get_app_deployment_download_info(id, app_deployment_id) do
+    GenServer.call(registry_name(id), {:get_app_deployment_download_info, app_deployment_id})
+  end
+
   @doc """
   Returns if the Team client is connected.
   """
@@ -181,6 +208,28 @@ defmodule Livebook.Hubs.TeamClient do
     {:reply, state.app_deployments, state}
   end
 
+  def handle_call(:get_agent_app_deployments, _caller, state) do
+    if state.deployment_group_id do
+      app_deployments =
+        Enum.filter(state.app_deployments, &(&1.deployment_group_id == state.deployment_group_id))
+
+      {:reply, app_deployments, state}
+    else
+      {:reply, [], state}
+    end
+  end
+
+  def handle_call({:get_app_deployment_download_info, app_deployment_id}, _caller, state) do
+    reply =
+      if app_deployment = Enum.find(state.app_deployments, &(&1.id == app_deployment_id)) do
+        {:ok, app_deployment, state.derived_key}
+      else
+        :error
+      end
+
+    {:reply, reply, state}
+  end
+
   @impl true
   def handle_info(:connected, state) do
     Hubs.Broadcasts.hub_connected(state.hub.id)
@@ -274,7 +323,9 @@ defmodule Livebook.Hubs.TeamClient do
   defp remove_deployment_group(state, deployment_group) do
     %{
       state
-      | deployment_groups: Enum.reject(state.deployment_groups, &(&1.id == deployment_group.id))
+      | deployment_groups: Enum.reject(state.deployment_groups, &(&1.id == deployment_group.id)),
+        app_deployments:
+          Enum.reject(state.app_deployments, &(&1.deployment_group_id == deployment_group.id))
     }
   end
 
@@ -456,9 +507,7 @@ defmodule Livebook.Hubs.TeamClient do
     with {:ok, deployment_group} <- fetch_deployment_group(deployment_group_deleted.id, state) do
       Teams.Broadcasts.deployment_group_deleted(deployment_group)
 
-      state
-      |> undeploy_apps(deployment_group)
-      |> remove_deployment_group(deployment_group)
+      remove_deployment_group(state, deployment_group)
     end
   end
 
@@ -483,12 +532,14 @@ defmodule Livebook.Hubs.TeamClient do
     deployment_group_id = app_deployment.deployment_group_id
 
     with {:ok, deployment_group} <- fetch_deployment_group(deployment_group_id, state) do
+      state = put_app_deployment(state, app_deployment)
+      Teams.Broadcasts.app_deployment_created(app_deployment)
+
       if deployment_group.id == state.deployment_group_id do
-        :ok = download_and_deploy(state.hub, app_deployment, state.derived_key)
+        manager_sync()
       end
 
-      Teams.Broadcasts.app_deployment_created(app_deployment)
-      put_app_deployment(state, app_deployment)
+      state
     end
   end
 
@@ -604,52 +655,10 @@ defmodule Livebook.Hubs.TeamClient do
   defp nullify(""), do: nil
   defp nullify(value), do: value
 
-  defp undeploy_apps(%{deployment_group_id: id} = state, %{id: id}) do
-    fun = &(&1.deployment_group_id == id)
-    app_deployments = Enum.filter(state.app_deployments, fun)
-
-    for %{slug: slug} <- app_deployments do
-      :ok = undeploy_app(slug)
-    end
-
-    %{state | deployment_group_id: nil, app_deployments: Enum.reject(state.app_deployments, fun)}
-  end
-
-  defp undeploy_apps(state, %{id: id}) do
-    %{
-      state
-      | app_deployments: Enum.reject(state.app_deployments, &(&1.deployment_group_id == id))
-    }
-  end
-
-  defp download_and_deploy(team, %Teams.AppDeployment{} = app_deployment, derived_key) do
-    destination_path = app_deployment_path(app_deployment.slug)
-
-    with {:ok, file_content} <- Teams.Requests.download_revision(team, app_deployment),
-         :ok <- undeploy_app(app_deployment.slug),
-         {:ok, decrypted_content} <- Teams.decrypt(file_content, derived_key),
-         :ok <- unzip_app(decrypted_content, destination_path),
-         :ok <- Livebook.Apps.deploy_apps_in_dir(destination_path) do
-      :ok
-    end
-  end
-
-  defp undeploy_app(slug) do
-    with {:ok, app} <- Livebook.Apps.fetch_app(slug) do
-      Livebook.App.close(app.pid)
-    end
-
-    :ok
-  end
-
-  defp app_deployment_path(slug) do
-    Path.join([Livebook.Config.tmp_path(), "apps", slug <> Livebook.Utils.random_short_id()])
-  end
-
-  defp unzip_app(content, destination_path) do
-    case :zip.extract(content, cwd: to_charlist(destination_path)) do
-      {:ok, _} -> :ok
-      {:error, error} -> FileSystem.Utils.posix_error(error)
+  defp manager_sync() do
+    # Each node runs the teams client, but we only need to call sync once
+    if Livebook.Apps.Manager.local?() do
+      Livebook.Apps.Manager.sync_permanent_apps()
     end
   end
 end

lib/livebook/live_markdown.ex

@@ -104,6 +104,7 @@ defmodule Livebook.LiveMarkdown do
   Returns the notebook structure and a list of informative messages/warnings
   related to the imported input.
   """
-  @spec notebook_from_livemd(String.t()) :: {Notebook.t(), list(String.t())}
+  @spec notebook_from_livemd(String.t()) ::
+          {Notebook.t(), %{warnings: list(String.t()), stamp_verified?: boolean()}}
   defdelegate notebook_from_livemd(markdown), to: Livebook.LiveMarkdown.Import
 end

lib/livebook/live_markdown/import.ex

@@ -13,18 +13,18 @@ defmodule Livebook.LiveMarkdown.Import do
     {notebook, valid_hub?, build_messages} = build_notebook(elements)
     {notebook, postprocess_messages} = postprocess_notebook(notebook)
 
-    {notebook, metadata_messages} =
+    {notebook, stamp_verified?, metadata_messages} =
       if stamp_data != nil and valid_hub? do
         postprocess_stamp(notebook, markdown, stamp_data)
       else
-        {notebook, []}
+        {notebook, false, []}
       end
 
     messages =
       earmark_messages ++
         rewrite_messages ++ build_messages ++ postprocess_messages ++ metadata_messages
 
-    {notebook, messages}
+    {notebook, %{warnings: messages, stamp_verified?: stamp_verified?}}
   end
 
   defp earmark_message_to_string({_severity, line_number, message}) do
@@ -630,7 +630,7 @@ defmodule Livebook.LiveMarkdown.Import do
   defp postprocess_stamp(notebook, notebook_source, stamp_data) do
     hub = Hubs.fetch_hub!(notebook.hub_id)
 
-    {valid_stamp?, notebook, messages} =
+    {stamp_verified?, notebook, messages} =
       with %{"offset" => offset, "stamp" => stamp} <- stamp_data,
            {:ok, notebook_source} <- safe_binary_slice(notebook_source, 0, offset),
            {:ok, metadata} <- Livebook.Hubs.verify_notebook_stamp(hub, notebook_source, stamp) do
@@ -653,9 +653,9 @@ defmodule Livebook.LiveMarkdown.Import do
     # we can only enable team features if the stamp is valid
     # (which means the server signed with a private key and we
     # validate it against the public key).
-    teams_enabled = is_struct(hub, Livebook.Hubs.Team) and (hub.offline == nil or valid_stamp?)
+    teams_enabled = is_struct(hub, Livebook.Hubs.Team) and (hub.offline == nil or stamp_verified?)
 
-    {%{notebook | teams_enabled: teams_enabled}, messages}
+    {%{notebook | teams_enabled: teams_enabled}, stamp_verified?, messages}
   end
 
   defp safe_binary_slice(binary, start, size)

lib/livebook/migration.ex

@@ -1,6 +1,4 @@
 defmodule Livebook.Migration do
-  use GenServer, restart: :temporary
-
   alias Livebook.Storage
 
   # We version out storage, so that we know which migrations to run
@@ -10,11 +8,7 @@ defmodule Livebook.Migration do
 
   def migration_version(), do: @migration_version
 
-  def start_link(_opts) do
-    GenServer.start_link(__MODULE__, :ok)
-  end
-
-  def init(:ok) do
+  def run() do
     insert_personal_hub()
     remove_offline_hub()
 
@@ -29,8 +23,6 @@ defmodule Livebook.Migration do
     end
 
     Storage.insert(:system, "global", migration_version: @migration_version)
-
-    :ignore
   end
 
   defp insert_personal_hub() do

lib/livebook/notebook.ex

@@ -30,6 +30,7 @@ defmodule Livebook.Notebook do
   ]
 
   alias Livebook.Notebook.{Section, Cell, AppSettings}
+  alias Livebook.FileSystem
   alias Livebook.Utils.Graph
   import Livebook.Utils, only: [access_by_id: 1]
 
@@ -908,4 +909,31 @@ defmodule Livebook.Notebook do
     )
     |> Ecto.Changeset.validate_format(field, ~r/\.\w+$/, message: "should end with an extension")
   end
+
+  @doc """
+  Copies notebook files from one directory to another.
+
+  Note that the source directory may have more files, only the ones
+  used by the notebook are copied.
+
+  If any of the notebook files does not exist, this function returns
+  an error.
+  """
+  @spec copy_files(t(), FileSystem.File.t(), FileSystem.File.t()) :: :ok | {:error, String.t()}
+  def copy_files(notebook, source_dir, files_dir) do
+    notebook.file_entries
+    |> Enum.filter(&(&1.type == :attachment))
+    |> Enum.reduce_while(:ok, fn file_entry, :ok ->
+      source_file = FileSystem.File.resolve(source_dir, file_entry.name)
+      destination_file = FileSystem.File.resolve(files_dir, file_entry.name)
+
+      case FileSystem.File.copy(source_file, destination_file) do
+        :ok ->
+          {:cont, :ok}
+
+        {:error, error} ->
+          {:halt, {:error, "failed to copy notebok file #{file_entry.name}, #{error}"}}
+      end
+    end)
+  end
 end

lib/livebook/notebook/learn.ex

@@ -116,7 +116,7 @@ defmodule Livebook.Notebook.Learn do
       markdown = File.read!(path)
       # Parse the file to ensure no warnings and read the title.
       # However, in the info we keep just the file contents to save on memory.
-      {notebook, warnings} = Livebook.LiveMarkdown.notebook_from_livemd(markdown)
+      {notebook, %{warnings: warnings}} = Livebook.LiveMarkdown.notebook_from_livemd(markdown)
 
       if warnings != [] do
         items = Enum.map(warnings, &("- " <> &1))
@@ -193,7 +193,8 @@ defmodule Livebook.Notebook.Learn do
         raise NotFoundError, slug: slug
 
       notebook_info ->
-        {notebook, []} = Livebook.LiveMarkdown.notebook_from_livemd(notebook_info.livemd)
+        {notebook, %{warnings: []}} =
+          Livebook.LiveMarkdown.notebook_from_livemd(notebook_info.livemd)
 
         {notebook, notebook_info.files}
     end

lib/livebook/release.ex

@@ -1,4 +1,6 @@
 defmodule Livebook.Release do
+  require Logger
+
   @doc """
   Runs the setup for all apps deployed from directory on startup.
   """
@@ -17,7 +19,20 @@ defmodule Livebook.Release do
           )
       end
 
-      Livebook.Apps.deploy_apps_in_dir(apps_path, warmup: true, skip_deploy: true)
+      app_specs =
+        Livebook.Apps.build_app_specs_in_dir(apps_path,
+          hub_id: Livebook.Config.apps_path_hub_id()
+        )
+
+      if app_specs != [] do
+        Logger.info("Running app warmups")
+
+        for app_spec <- app_specs do
+          with {:error, message} <- Livebook.Apps.warmup_app(app_spec) do
+            Logger.info("Warmup failed for app #{app_spec.slug}, #{message}")
+          end
+        end
+      end
     end
   end
 

lib/livebook/session.ex

@@ -122,7 +122,13 @@ defmodule Livebook.Session do
           registered_files: %{
             String.t() => %{file_ref: Runtime.file_ref(), linked_client_id: Data.client_id()}
           },
-          client_id_with_assets: %{Data.client_id() => map()}
+          client_id_with_assets: %{Data.client_id() => map()},
+          deployment_ref: reference() | nil,
+          deployed_app_monitor_ref: reference() | nil,
+          app_pid: pid() | nil,
+          auto_shutdown_ms: non_neg_integer() | nil,
+          auto_shutdown_timer_ref: reference() | nil,
+          started_by: Livebook.User.t() | nil
         }
 
   @type memory_usage ::
@@ -131,6 +137,9 @@ defmodule Livebook.Session do
             system: Livebook.SystemResources.memory()
           }
 
+  @type files_source ::
+          {:dir, FileSystem.File.t()} | {:url, String.t()} | {:inline, %{String.t() => binary()}}
+
   @typedoc """
   An id assigned to every running session process.
   """
@@ -838,10 +847,14 @@ defmodule Livebook.Session do
         Process.monitor(app_pid)
       end
 
-      if state.data.mode == :app do
-        {:ok, state, {:continue, :app_init}}
-      else
-        {:ok, state}
+      session = self_from_state(state)
+
+      with :ok <- Livebook.Tracker.track_session(session) do
+        if state.data.mode == :app do
+          {:ok, state, {:continue, :app_init}}
+        else
+          {:ok, state}
+        end
       end
     else
       {:error, error} ->
@@ -867,6 +880,7 @@ defmodule Livebook.Session do
         registered_file_deletion_delay: opts[:registered_file_deletion_delay] || 15_000,
         registered_files: %{},
         client_id_with_assets: %{},
+        deployment_ref: nil,
         deployed_app_monitor_ref: nil,
         app_pid: opts[:app_pid],
         auto_shutdown_ms: opts[:auto_shutdown_ms],
@@ -1371,19 +1385,17 @@ defmodule Livebook.Session do
   def handle_cast({:deploy_app, _client_pid}, state) do
     # In the initial state app settings are empty, hence not valid,
     # so we double-check that we can actually deploy
-    if Notebook.AppSettings.valid?(state.data.notebook.app_settings) do
-      files_dir = files_dir_from_state(state)
-      {:ok, pid} = Livebook.Apps.deploy(state.data.notebook, files_source: {:dir, files_dir})
+    if Notebook.AppSettings.valid?(state.data.notebook.app_settings) and
+         state.deployment_ref == nil do
+      app_spec = %Livebook.Apps.PreviewAppSpec{
+        slug: state.data.notebook.app_settings.slug,
+        session_id: state.session_id
+      }
 
-      if ref = state.deployed_app_monitor_ref do
-        Process.demonitor(ref, [:flush])
-      end
+      deployer_pid = Livebook.Apps.Deployer.local_deployer()
+      deployment_ref = Livebook.Apps.Deployer.deploy_monitor(deployer_pid, app_spec)
 
-      ref = Process.monitor(pid)
-      state = put_in(state.deployed_app_monitor_ref, ref)
-
-      operation = {:set_deployed_app_slug, @client_id, state.data.notebook.app_settings.slug}
-      {:noreply, handle_operation(state, operation)}
+      {:noreply, %{state | deployment_ref: deployment_ref}}
     else
       {:noreply, state}
     end
@@ -1454,6 +1466,15 @@ defmodule Livebook.Session do
     {:noreply, %{state | save_task_ref: nil}}
   end
 
+  def handle_info({:DOWN, ref, :process, _pid, reason}, state) when ref == state.deployment_ref do
+    broadcast_error(
+      state.session_id,
+      "app deployment failed, deployer terminated unexpectedly, reason: #{inspect(reason)}"
+    )
+
+    {:noreply, %{state | deployment_ref: nil}}
+  end
+
   def handle_info({:DOWN, ref, :process, _, _}, state)
       when ref == state.deployed_app_monitor_ref do
     {:noreply,
@@ -1814,6 +1835,29 @@ defmodule Livebook.Session do
     {:noreply, handle_operation(state, operation)}
   end
 
+  def handle_info({:deploy_result, ref, result}, state) do
+    Process.demonitor(ref, [:flush])
+
+    state = %{state | deployment_ref: nil}
+
+    case result do
+      {:ok, pid} ->
+        if ref = state.deployed_app_monitor_ref do
+          Process.demonitor(ref, [:flush])
+        end
+
+        ref = Process.monitor(pid)
+        state = put_in(state.deployed_app_monitor_ref, ref)
+
+        operation = {:set_deployed_app_slug, @client_id, state.data.notebook.app_settings.slug}
+        {:noreply, handle_operation(state, operation)}
+
+      {:error, error} ->
+        broadcast_error(state.session_id, "app deployment failed, #{error}")
+        {:noreply, state}
+    end
+  end
+
   def handle_info(_message, state), do: {:noreply, state}
 
   @impl true
@@ -1950,14 +1994,14 @@ defmodule Livebook.Session do
   end
 
   defp initialize_files_from(state, {:dir, dir}) do
-    copy_files(state, dir)
+    copy_notebook_files(state, dir)
   end
 
-  defp copy_files(state, source) do
-    with {:ok, source_exists?} <- FileSystem.File.exists?(source) do
+  defp copy_notebook_files(state, source_dir) do
+    with {:ok, source_exists?} <- FileSystem.File.exists?(source_dir) do
       if source_exists? do
         write_attachment_file_entries(state, fn destination_file, file_entry ->
-          source_file = FileSystem.File.resolve(source, file_entry.name)
+          source_file = FileSystem.File.resolve(source_dir, file_entry.name)
 
           case FileSystem.File.copy(source_file, destination_file) do
             :ok ->
@@ -1978,9 +2022,10 @@ defmodule Livebook.Session do
   end
 
   defp write_attachment_file_entries(state, write_fun) do
+    notebook = state.data.notebook
     files_dir = files_dir_from_state(state)
 
-    state.data.notebook.file_entries
+    notebook.file_entries
     |> Enum.filter(&(&1.type == :attachment))
     |> Task.async_stream(
       fn file_entry ->
@@ -2120,7 +2165,7 @@ defmodule Livebook.Session do
     prev_files_dir = files_dir_from_state(prev_state)
 
     if prev_state.data.file do
-      copy_files(state, prev_files_dir)
+      copy_notebook_files(state, prev_files_dir)
     else
       move_files(state, prev_files_dir)
     end

lib/livebook/sessions.ex

@@ -10,8 +10,6 @@ defmodule Livebook.Sessions do
 
   @doc """
   Spawns a new `Session` process with the given options.
-
-  Makes the session globally visible within the session tracker.
   """
   @spec create_session(keyword()) :: {:ok, Session.t()} | {:error, any()}
   def create_session(opts \\ []) do
@@ -21,16 +19,7 @@ defmodule Livebook.Sessions do
 
     case DynamicSupervisor.start_child(Livebook.SessionSupervisor, {Session, opts}) do
       {:ok, pid} ->
-        session = Session.get_by_pid(pid)
-
-        case Livebook.Tracker.track_session(session) do
-          :ok ->
-            {:ok, session}
-
-          {:error, reason} ->
-            Session.close(pid)
-            {:error, reason}
-        end
+        {:ok, Session.get_by_pid(pid)}
 
       {:error, reason} ->
         {:error, reason}

lib/livebook/utils/supervision_step.ex

@@ -0,0 +1,24 @@
+defmodule Livebook.Utils.SupervisionStep do
+  use GenServer, restart: :temporary
+
+  @doc """
+  Synchronously runs the given function as part of supervision tree
+  startup.
+  """
+  @spec start_link({atom(), function()}) :: :ignore
+  def start_link({_id, fun}) do
+    GenServer.start_link(__MODULE__, fun)
+  end
+
+  @doc false
+  def child_spec({id, fun}) do
+    child_spec = super({id, fun})
+    update_in(child_spec.id, &{&1, id})
+  end
+
+  @impl true
+  def init(fun) do
+    fun.()
+    :ignore
+  end
+end

lib/livebook_web/helpers/session_helpers.ex

@@ -132,7 +132,7 @@ defmodule LivebookWeb.SessionHelpers do
   @spec fork_notebook(Socket.t(), FileSystem.File.t()) :: Socket.t()
   def fork_notebook(socket, file) do
     case import_notebook(file) do
-      {:ok, {notebook, messages}} ->
+      {:ok, {notebook, %{warnings: messages}}} ->
         notebook = Livebook.Notebook.forked(notebook)
         files_dir = Session.files_dir_for_notebook(file)
 
@@ -155,7 +155,7 @@ defmodule LivebookWeb.SessionHelpers do
   @spec open_notebook(Socket.t(), FileSystem.File.t()) :: Socket.t()
   def open_notebook(socket, file) do
     case import_notebook(file) do
-      {:ok, {notebook, messages}} ->
+      {:ok, {notebook, %{warnings: messages}}} ->
         socket
         |> put_import_warnings(messages)
         |> create_session(notebook: notebook, file: file, origin: {:file, file})

lib/livebook_web/live/apps_dashboard_live.ex

@@ -61,14 +61,19 @@ defmodule LivebookWeb.AppsDashboardLive do
       <div :for={app <- Enum.sort_by(@apps, & &1.slug)} data-app-slug={app.slug}>
         <a
           phx-click={JS.toggle(to: "[data-app-slug=#{app.slug}] .toggle")}
-          class="flex items-center justify-between break-all mb-2 text-gray-800 font-medium text-xl hover:cursor-pointer"
+          class="flex items-center justify-between mb-2 hover:cursor-pointer"
         >
-          <%= "/" <> app.slug %>
-          <.remix_icon icon="arrow-drop-up-line" class="text-3xl text-gray-400 toggle" />
-          <.remix_icon icon="arrow-drop-down-line" class="text-3xl text-gray-400 hidden toggle" />
+          <span class="text-gray-800 font-medium text-xl break-all">
+            <%= "/" <> app.slug %>
+          </span>
+          <div class="flex items-center gap-2">
+            <.app_group_tag app_spec={app.app_spec} />
+            <.remix_icon icon="arrow-drop-up-line" class="text-3xl text-gray-400 toggle" />
+            <.remix_icon icon="arrow-drop-down-line" class="text-3xl text-gray-400 hidden toggle" />
+          </div>
         </a>
         <div class="toggle">
-          <div class="mt-4 flex flex-col gap-3">
+          <div :if={app.warnings != []} class="my-3 flex flex-col gap-3">
             <.message_box :for={warning <- app.warnings} kind={:warning} message={warning} />
           </div>
           <div class="flex-col mb-8">
@@ -97,14 +102,22 @@ defmodule LivebookWeb.AppsDashboardLive do
                   </.labeled_text>
                 </div>
                 <div class="flex flex-col md:flex-row md:items-center justify-start lg:justify-end">
-                  <span class="tooltip top" data-tooltip="Terminate">
-                    <.icon_button
-                      aria-label="terminate app"
-                      phx-click={JS.push("terminate_app", value: %{slug: app.slug})}
-                    >
-                      <.remix_icon icon="delete-bin-6-line" />
-                    </.icon_button>
-                  </span>
+                  <%= if app.permanent do %>
+                    <span class="tooltip top" data-tooltip="Permanent apps cannot be terminated">
+                      <.icon_button disabled>
+                        <.remix_icon icon="delete-bin-6-line" />
+                      </.icon_button>
+                    </span>
+                  <% else %>
+                    <span class="tooltip top" data-tooltip="Terminate">
+                      <.icon_button
+                        aria-label="terminate app"
+                        phx-click={JS.push("terminate_app", value: %{slug: app.slug})}
+                      >
+                        <.remix_icon icon="delete-bin-6-line" />
+                      </.icon_button>
+                    </span>
+                  <% end %>
                 </div>
               </div>
             </div>
@@ -135,7 +148,7 @@ defmodule LivebookWeb.AppsDashboardLive do
                   <:actions :let={app_session}>
                     <span class="tooltip left" data-tooltip="Open">
                       <.icon_button
-                        disabled={app_session.app_status.lifecycle}
+                        disabled={app_session.app_status.lifecycle != :active}
                         aria-label="open app"
                         href={~p"/apps/#{app.slug}/#{app_session.id}"}
                       >
@@ -189,6 +202,62 @@ defmodule LivebookWeb.AppsDashboardLive do
     """
   end
 
+  defp app_group_tag(%{app_spec: %Livebook.Apps.PreviewAppSpec{}} = assigns) do
+    ~H"""
+    <span
+      class="tooltip top"
+      data-tooltip={
+        ~S'''
+        This is an app preview, it has
+        been started manually
+        '''
+      }
+    >
+      <span class="font-medium bg-gray-100 text-gray-800 text-xs px-2.5 py-0.5 rounded cursor-default">
+        Preview
+      </span>
+    </span>
+    """
+  end
+
+  defp app_group_tag(%{app_spec: %Livebook.Apps.PathAppSpec{}} = assigns) do
+    ~H"""
+    <span
+      class="tooltip top"
+      data-tooltip={
+        ~S'''
+        This is a permanent app started
+        from LIVEBOOK_APPS_PATH
+        '''
+      }
+    >
+      <span class="font-medium bg-blue-100 text-blue-800 text-xs px-2.5 py-0.5 rounded cursor-default">
+        Apps directory
+      </span>
+    </span>
+    """
+  end
+
+  defp app_group_tag(%{app_spec: %Livebook.Apps.TeamsAppSpec{}} = assigns) do
+    ~H"""
+    <span
+      class="tooltip top"
+      data-tooltip={
+        ~S'''
+        This is a permanent app started
+        from your Livebook Teams hub
+        '''
+      }
+    >
+      <span class="font-medium bg-green-100 text-green-800 text-xs px-2.5 py-0.5 rounded cursor-default">
+        Livebook Teams
+      </span>
+    </span>
+    """
+  end
+
+  defp app_group_tag(assigns), do: ~H""
+
   defp grid(assigns) do
     ~H"""
     <div class="min-w-[650px]">

lib/livebook_web/live/open_live.ex

@@ -253,7 +253,7 @@ defmodule LivebookWeb.OpenLive do
   defp file_from_params(_params), do: Livebook.Settings.default_dir()
 
   defp import_source(socket, source, session_opts) do
-    {notebook, messages} = Livebook.LiveMarkdown.notebook_from_livemd(source)
+    {notebook, %{warnings: messages}} = Livebook.LiveMarkdown.notebook_from_livemd(source)
 
     socket =
       socket

lib/livebook_web/live/session_live.ex

@@ -1089,7 +1089,7 @@ defmodule LivebookWeb.SessionLive do
   defp open_notebook(socket, origin, fallback_locations, requested_url) do
     case fetch_content_with_fallbacks(origin, fallback_locations) do
       {:ok, content} ->
-        {notebook, messages} = Livebook.LiveMarkdown.notebook_from_livemd(content)
+        {notebook, %{warnings: messages}} = Livebook.LiveMarkdown.notebook_from_livemd(content)
 
         # If the current session has no file, fork the notebook
         fork? = socket.private.data.file == nil

lib/livebook_web/live/session_live/app_settings_component.ex

@@ -166,9 +166,15 @@ defmodule LivebookWeb.SessionLive.AppSettingsComponent do
     app_settings = socket.assigns.settings
     deployed_app_slug = socket.assigns.deployed_app_slug
 
-    with {:ok, settings} <- AppSettings.update(app_settings, params) do
-      Livebook.Session.set_app_settings(socket.assigns.session.pid, settings)
-    end
+    app_settings =
+      case AppSettings.update(app_settings, params) do
+        {:ok, app_settings} ->
+          Livebook.Session.set_app_settings(socket.assigns.session.pid, app_settings)
+          app_settings
+
+        {:error, _changeset} ->
+          app_settings
+      end
 
     case socket.assigns.context do
       "preview" ->
@@ -189,16 +195,35 @@ defmodule LivebookWeb.SessionLive.AppSettingsComponent do
     end
 
     slug = app_settings.slug
-    slug_taken? = slug != deployed_app_slug and Livebook.Apps.exists?(slug)
 
-    if slug_taken? do
-      confirm(socket, on_confirm,
-        title: "Deploy app",
-        description: "An app with this slug already exists, do you want to deploy a new version?",
-        confirm_text: "Replace"
-      )
-    else
-      on_confirm.(socket)
+    app =
+      case Livebook.Apps.fetch_app(slug) do
+        {:ok, app} -> app
+        :error -> nil
+      end
+
+    permanent? = app && app.permanent
+    slug_taken? = slug != deployed_app_slug and app != nil
+
+    cond do
+      permanent? ->
+        socket
+        |> put_flash(
+          :error,
+          "A permanent app with this slug already exists and cannot be replaced."
+        )
+        |> push_patch(to: ~p"/sessions/#{socket.assigns.session.id}")
+
+      slug_taken? ->
+        confirm(socket, on_confirm,
+          title: "Deploy app",
+          description:
+            "An app with this slug already exists, do you want to deploy a new version?",
+          confirm_text: "Replace"
+        )
+
+      true ->
+        on_confirm.(socket)
     end
   end
 end

test/livebook/app_test.exs

@@ -3,13 +3,13 @@ defmodule Livebook.AppTest do
 
   alias Livebook.{App, Notebook, Utils}
 
-  describe "start_link/1" do
+  describe "startup" do
     test "eagerly starts a session in single-session mode" do
       slug = Utils.random_short_id()
       app_settings = %{Notebook.AppSettings.new() | slug: slug}
       notebook = %{Notebook.new() | app_settings: app_settings}
 
-      assert {:ok, app_pid} = App.start_link(notebook: notebook)
+      app_pid = start_app(notebook)
       assert %{sessions: [%{pid: session_pid}]} = App.get_by_pid(app_pid)
 
       assert %{mode: :app} = Livebook.Session.get_by_pid(session_pid)
@@ -20,7 +20,7 @@ defmodule Livebook.AppTest do
       app_settings = %{Notebook.AppSettings.new() | slug: slug, auto_shutdown_ms: 5_000}
       notebook = %{Notebook.new() | app_settings: app_settings}
 
-      assert {:ok, app_pid} = App.start_link(notebook: notebook)
+      app_pid = start_app(notebook)
       assert %{sessions: []} = App.get_by_pid(app_pid)
     end
 
@@ -29,7 +29,7 @@ defmodule Livebook.AppTest do
       app_settings = %{Notebook.AppSettings.new() | slug: slug, multi_session: true}
       notebook = %{Notebook.new() | app_settings: app_settings}
 
-      assert {:ok, app_pid} = App.start_link(notebook: notebook)
+      app_pid = start_app(notebook)
       assert %{sessions: []} = App.get_by_pid(app_pid)
     end
   end
@@ -42,7 +42,7 @@ defmodule Livebook.AppTest do
 
       app_pid = start_app(notebook)
 
-      App.deploy(app_pid, %{notebook | name: "New name"})
+      App.deploy(app_pid, deployment_bundle(%{notebook | name: "New name"}))
       assert %{version: 2, notebook_name: "New name"} = App.get_by_pid(app_pid)
     end
 
@@ -57,7 +57,7 @@ defmodule Livebook.AppTest do
 
       App.subscribe(slug)
 
-      App.deploy(app_pid, notebook)
+      App.deploy(app_pid, deployment_bundle(notebook))
 
       assert_receive {:app_updated,
                       %{
@@ -80,7 +80,7 @@ defmodule Livebook.AppTest do
       assert_receive {:app_updated,
                       %{sessions: [%{app_status: %{execution: :executed}, version: 1}]}}
 
-      App.deploy(app_pid, notebook)
+      App.deploy(app_pid, deployment_bundle(notebook))
 
       assert_receive {:app_updated,
                       %{
@@ -106,7 +106,7 @@ defmodule Livebook.AppTest do
       assert_receive {:app_updated,
                       %{sessions: [%{app_status: %{execution: :executed}, version: 1}]}}
 
-      App.deploy(app_pid, notebook)
+      App.deploy(app_pid, deployment_bundle(notebook))
 
       assert_receive {:app_updated,
                       %{
@@ -149,7 +149,7 @@ defmodule Livebook.AppTest do
       assert_receive {:app_updated,
                       %{sessions: [%{id: session_id1, app_status: %{execution: :executed}}]}}
 
-      App.deploy(app_pid, notebook)
+      App.deploy(app_pid, deployment_bundle(notebook))
 
       assert_receive {:app_updated,
                       %{
@@ -237,8 +237,20 @@ defmodule Livebook.AppTest do
     end
   end
 
+  defp deployment_bundle(notebook) do
+    app_spec = Livebook.Apps.NotebookAppSpec.new(notebook)
+
+    %{
+      notebook: notebook,
+      files_tmp_path: Livebook.Apps.generate_files_tmp_path(app_spec.slug),
+      app_spec: app_spec,
+      permanent: false,
+      warnings: []
+    }
+  end
+
   defp start_app(notebook) do
-    opts = [notebook: notebook]
+    opts = [deployment_bundle: deployment_bundle(notebook)]
     start_supervised!({App, opts})
   end
 end

test/livebook/apps/deployer_test.exs

@@ -0,0 +1,200 @@
+defmodule Livebook.Apps.DeployerTest do
+  use ExUnit.Case, async: true
+
+  alias Livebook.Apps
+  alias Livebook.Notebook
+
+  describe "deploy_monitor/3" do
+    test "deploys and registers an app" do
+      slug = Livebook.Utils.random_short_id()
+      app_settings = %{Notebook.AppSettings.new() | slug: slug}
+      notebook = %{Notebook.new() | app_settings: app_settings}
+      app_spec = Apps.NotebookAppSpec.new(notebook)
+
+      Apps.subscribe()
+
+      deployer_pid = Apps.Deployer.local_deployer()
+      ref = Apps.Deployer.deploy_monitor(deployer_pid, app_spec)
+
+      assert_receive {:deploy_result, ^ref, {:ok, pid}}
+      assert_receive {:app_created, %{pid: ^pid}}
+
+      assert {:ok, %{pid: ^pid}} = Apps.fetch_app(slug)
+
+      Livebook.App.close(pid)
+    end
+
+    @tag :tmp_dir
+    test "deploys an app with files", %{tmp_dir: tmp_dir} do
+      app_path = Path.join(tmp_dir, "app.livemd")
+
+      files_path = Path.join(tmp_dir, "files")
+      File.mkdir_p!(files_path)
+      files_path |> Path.join("image.jpg") |> File.write!("content")
+
+      slug = Livebook.Utils.random_short_id()
+
+      File.write!(app_path, """
+      <!-- livebook:{"app_settings":{"access_type":"public","slug":"#{slug}"},"file_entries":[{"name":"image.jpg","type":"attachment"}]} -->
+
+      # App
+      """)
+
+      app_spec = %Livebook.Apps.PathAppSpec{slug: slug, path: app_path}
+
+      Apps.subscribe()
+
+      deployer_pid = Apps.Deployer.local_deployer()
+      ref = Apps.Deployer.deploy_monitor(deployer_pid, app_spec)
+
+      assert_receive {:deploy_result, ^ref, {:ok, pid}}
+      assert_receive {:app_created, %{pid: ^pid}}
+
+      assert_receive {:app_updated, %{pid: ^pid, sessions: [%{pid: session_pid}]}}
+
+      session = Livebook.Session.get_by_pid(session_pid)
+
+      assert Livebook.FileSystem.File.resolve(session.files_dir, "image.jpg")
+             |> Livebook.FileSystem.File.read() == {:ok, "content"}
+
+      Livebook.App.close(pid)
+    end
+
+    test "deploys a new app version if already running" do
+      slug = Livebook.Utils.random_short_id()
+      app_settings = %{Notebook.AppSettings.new() | slug: slug}
+      notebook = %{Notebook.new() | app_settings: app_settings}
+      app_spec = Apps.NotebookAppSpec.new(notebook)
+
+      Apps.subscribe()
+
+      deployer_pid = Apps.Deployer.local_deployer()
+      ref = Apps.Deployer.deploy_monitor(deployer_pid, app_spec)
+
+      assert_receive {:deploy_result, ^ref, {:ok, pid}}
+      assert_receive {:app_created, %{pid: ^pid, version: 1}}
+
+      ref = Apps.Deployer.deploy_monitor(deployer_pid, app_spec)
+
+      assert_receive {:deploy_result, ^ref, {:ok, ^pid}}
+      assert_receive {:app_updated, %{pid: ^pid, version: 2}}
+
+      Livebook.App.close(pid)
+    end
+
+    test "sends an error when the deployment fails" do
+      slug = Livebook.Utils.random_short_id()
+      app_settings = %{Notebook.AppSettings.new() | slug: slug}
+      notebook = %{Notebook.new() | app_settings: app_settings}
+
+      app_spec = Apps.NotebookAppSpec.new(notebook, load_failures: 1)
+
+      deployer_pid = Apps.Deployer.local_deployer()
+      ref = Apps.Deployer.deploy_monitor(deployer_pid, app_spec)
+
+      assert_receive {:deploy_result, ^ref, {:error, "failed to load"}}
+    end
+
+    test "does not redeploy when :start_only is specified" do
+      slug = Livebook.Utils.random_short_id()
+      app_settings = %{Notebook.AppSettings.new() | slug: slug}
+      notebook = %{Notebook.new() | app_settings: app_settings}
+      app_spec = Apps.NotebookAppSpec.new(notebook)
+
+      Apps.subscribe()
+
+      deployer_pid = Apps.Deployer.local_deployer()
+      ref = Apps.Deployer.deploy_monitor(deployer_pid, app_spec)
+
+      assert_receive {:deploy_result, ^ref, {:ok, pid}}
+      assert_receive {:app_created, %{pid: ^pid, version: 1}}
+
+      ref = Apps.Deployer.deploy_monitor(deployer_pid, app_spec, start_only: true)
+
+      assert_receive {:deploy_result, ^ref, {:error, :already_started}}
+
+      Livebook.App.close(pid)
+    end
+
+    @tag :tmp_dir
+    test "runs warmup if enabled by app spec", %{tmp_dir: tmp_dir} do
+      path = Path.join(tmp_dir, "setup.txt")
+
+      slug = Livebook.Utils.random_short_id()
+      # Multi-session, so that no session is started eagerly
+      app_settings = %{Notebook.AppSettings.new() | slug: slug, multi_session: true}
+
+      notebook =
+        %{Notebook.new() | app_settings: app_settings}
+        |> Notebook.put_setup_cell(%{
+          Notebook.Cell.new(:code)
+          | source: """
+            File.touch!("#{path}")
+            """
+        })
+
+      app_spec = Apps.NotebookAppSpec.new(notebook, should_warmup: true)
+
+      deployer_pid = Apps.Deployer.local_deployer()
+      ref = Apps.Deployer.deploy_monitor(deployer_pid, app_spec)
+
+      assert_receive {:deploy_result, ^ref, {:ok, pid}}
+
+      assert File.exists?(path)
+
+      Livebook.App.close(pid)
+    end
+
+    test "warns if the warmup fails" do
+      slug = Livebook.Utils.random_short_id()
+      # Multi-session, so that no session is started eagerly
+      app_settings = %{Notebook.AppSettings.new() | slug: slug, multi_session: true}
+
+      notebook =
+        %{Notebook.new() | app_settings: app_settings}
+        |> Notebook.put_setup_cell(%{
+          Notebook.Cell.new(:code)
+          | source: """
+            raise "error"
+            """
+        })
+
+      app_spec = Apps.NotebookAppSpec.new(notebook, should_warmup: true)
+
+      deployer_pid = Apps.Deployer.local_deployer()
+
+      assert ExUnit.CaptureLog.capture_log(fn ->
+               ref = Apps.Deployer.deploy_monitor(deployer_pid, app_spec)
+
+               assert_receive {:deploy_result, ^ref, {:ok, pid}}
+
+               Livebook.App.close(pid)
+             end) =~ "[app=#{slug}] App warmup failed, setup cell finished with failure"
+    end
+
+    test "sends monitor message when deployer crashes" do
+      slug = Livebook.Utils.random_short_id()
+      app_settings = %{Notebook.AppSettings.new() | slug: slug}
+
+      notebook =
+        %{Notebook.new() | app_settings: app_settings}
+        |> Notebook.put_setup_cell(%{
+          Notebook.Cell.new(:code)
+          | source: """
+            Process.sleep(:infinity)
+            """
+        })
+
+      app_spec = Apps.NotebookAppSpec.new(notebook, should_warmup: true)
+
+      deployer_pid = start_supervised!(Apps.Deployer)
+
+      ref = Apps.Deployer.deploy_monitor(deployer_pid, app_spec)
+
+      # The deployer is stuck at warmup and we kill it
+      Process.exit(deployer_pid, :kill)
+
+      assert_receive {:DOWN, ^ref, :process, _pid, _reason}
+    end
+  end
+end

test/livebook/apps/manager_test.exs

@@ -0,0 +1,119 @@
+defmodule Livebook.Apps.ManagerTest do
+  # Not async, because we alter global config (permanent apps) and we
+  # also test restarting the global manager process
+  use ExUnit.Case, async: false
+
+  alias Livebook.Apps
+  alias Livebook.Notebook
+
+  setup do
+    on_exit(fn ->
+      Apps.set_startup_app_specs([])
+
+      for app <- Apps.list_apps() do
+        Livebook.App.close(app.pid)
+      end
+    end)
+  end
+
+  @tag :tmp_dir
+  test "restarts permanent apps on crash", %{tmp_dir: tmp_dir} do
+    slug = Livebook.Utils.random_short_id()
+    app_spec = path_app_spec(tmp_dir, slug)
+
+    Apps.subscribe()
+
+    Apps.set_startup_app_specs([app_spec])
+    Apps.Manager.sync_permanent_apps()
+
+    assert_receive {:app_created, %{slug: ^slug} = app}
+    # Make sure the app finished init
+    _ = Livebook.App.get_by_pid(app.pid)
+
+    Process.exit(app.pid, :kill)
+    assert_receive {:app_closed, %{slug: ^slug}}
+
+    # Automatically restarted by the manager
+    assert_receive {:app_created, %{slug: ^slug}}
+  end
+
+  @tag :tmp_dir
+  test "closes no-longer-permanent apps", %{tmp_dir: tmp_dir} do
+    slug1 = Livebook.Utils.random_short_id()
+    app_spec1 = path_app_spec(tmp_dir, slug1)
+    slug2 = Livebook.Utils.random_short_id()
+    app_spec2 = path_app_spec(tmp_dir, slug2)
+
+    Apps.subscribe()
+
+    Apps.set_startup_app_specs([app_spec1])
+    Apps.Manager.sync_permanent_apps()
+
+    assert_receive {:app_created, %{slug: ^slug1}}
+
+    Apps.set_startup_app_specs([app_spec2])
+    Apps.Manager.sync_permanent_apps()
+
+    assert_receive {:app_created, %{slug: ^slug2}}
+    assert_receive {:app_closed, %{slug: ^slug1}}
+  end
+
+  @tag :tmp_dir
+  test "redeploys an app when the spec version changes", %{tmp_dir: tmp_dir} do
+    slug = Livebook.Utils.random_short_id()
+    app_spec = path_app_spec(tmp_dir, slug)
+
+    Apps.subscribe()
+
+    Apps.set_startup_app_specs([app_spec])
+    Apps.Manager.sync_permanent_apps()
+
+    assert_receive {:app_created, %{slug: ^slug, version: 1}}
+
+    app_spec = %{app_spec | version: "2"}
+    Apps.set_startup_app_specs([app_spec])
+    Apps.Manager.sync_permanent_apps()
+
+    # Automatically redeployed by the manager
+    assert_receive {:app_updated, %{slug: ^slug, version: 2, app_spec: ^app_spec}}
+  end
+
+  test "retries deployment on failure" do
+    slug = Livebook.Utils.random_short_id()
+    app_settings = %{Notebook.AppSettings.new() | slug: slug}
+    notebook = %{Notebook.new() | app_settings: app_settings}
+    app_spec = Apps.NotebookAppSpec.new(notebook, load_failures: 1)
+
+    Apps.subscribe()
+
+    assert ExUnit.CaptureLog.capture_log(fn ->
+             Apps.set_startup_app_specs([app_spec])
+             Apps.Manager.sync_permanent_apps()
+
+             assert_receive {:app_created, %{slug: ^slug}}
+           end) =~ "[app=#{slug}] Deployment failed, failed to load"
+  end
+
+  test "is restarted on crash" do
+    pid = :global.whereis_name(Apps.Manager)
+
+    Phoenix.PubSub.subscribe(Livebook.PubSub, "manager_watcher")
+
+    Process.exit(pid, :kill)
+
+    assert_receive {:manager_started, pid}
+    assert :global.whereis_name(Apps.Manager) == pid
+  end
+
+  defp path_app_spec(tmp_dir, slug) do
+    app_path = Path.join(tmp_dir, "app_#{slug}.livemd")
+
+    File.write!(app_path, """
+    <!-- livebook:{"app_settings":{"access_type":"public","slug":"#{slug}"}} -->
+
+    # App
+    """)
+
+    %Livebook.Apps.PathAppSpec{slug: slug, path: app_path}
+  end
+end

test/livebook/apps/path_app_spec_test.exs

@@ -0,0 +1,89 @@
+defmodule Livebook.Apps.PathAppSpecTest do
+  use ExUnit.Case, async: true
+
+  import Livebook.TestHelpers
+
+  describe "load" do
+    @tag :tmp_dir
+    test "returns import warnings", %{tmp_dir: tmp_dir} do
+      [app_dir, file_tmp_path] = create_subdirs!(tmp_dir, 2)
+
+      app_path = Path.join(app_dir, "app.livemd")
+
+      slug = "app"
+
+      File.write!(app_path, """
+      <!-- livebook:{"app_settings":{"access_type":"public","slug":"#{slug}"}} -->
+
+      # App
+
+      ```elixir
+      """)
+
+      app_spec = %Livebook.Apps.PathAppSpec{slug: slug, path: app_path}
+
+      assert {:ok, %{warnings: [warning]}} = Livebook.Apps.AppSpec.load(app_spec, file_tmp_path)
+      assert warning =~ "line 5 - fenced Code Block opened with"
+    end
+
+    @tag :tmp_dir
+    test "copies files use by the notebook", %{tmp_dir: tmp_dir} do
+      [app_dir, file_tmp_path] = create_subdirs!(tmp_dir, 2)
+
+      app_path = Path.join(app_dir, "app.livemd")
+
+      files_path = Path.join(app_dir, "files")
+      File.mkdir_p!(files_path)
+      files_path |> Path.join("image1.jpg") |> File.write!("content")
+      files_path |> Path.join("image2.jpg") |> File.write!("content")
+
+      slug = "app"
+
+      File.write!(app_path, """
+      <!-- livebook:{"app_settings":{"access_type":"public","slug":"#{slug}"},"file_entries":[{"name":"image1.jpg","type":"attachment"}]} -->
+
+      # App
+      """)
+
+      app_spec = %Livebook.Apps.PathAppSpec{slug: slug, path: app_path}
+
+      assert {:ok, %{notebook: _}} = Livebook.Apps.AppSpec.load(app_spec, file_tmp_path)
+
+      assert File.ls!(file_tmp_path) == ["image1.jpg"]
+    end
+
+    @tag :tmp_dir
+    test "returns error when file does not exist", %{tmp_dir: tmp_dir} do
+      [app_dir, file_tmp_path] = create_subdirs!(tmp_dir, 2)
+
+      app_path = Path.join(app_dir, "app.livemd")
+
+      slug = "app"
+
+      app_spec = %Livebook.Apps.PathAppSpec{slug: slug, path: app_path}
+
+      assert {:error, message} = Livebook.Apps.AppSpec.load(app_spec, file_tmp_path)
+      assert message =~ "no such file or directory"
+    end
+
+    @tag :tmp_dir
+    test "returns error when one of the notebook files is missing", %{tmp_dir: tmp_dir} do
+      [app_dir, file_tmp_path] = create_subdirs!(tmp_dir, 2)
+
+      app_path = Path.join(app_dir, "app.livemd")
+
+      slug = "app"
+
+      File.write!(app_path, """
+      <!-- livebook:{"app_settings":{"access_type":"public","slug":"#{slug}"},"file_entries":[{"name":"image.jpg","type":"attachment"}]} -->
+
+      # App
+      """)
+
+      app_spec = %Livebook.Apps.PathAppSpec{slug: slug, path: app_path}
+
+      assert {:error, message} = Livebook.Apps.AppSpec.load(app_spec, file_tmp_path)
+      assert message == "failed to copy notebok file image.jpg, no such file or directory"
+    end
+  end
+end

test/livebook/apps_test.exs

@@ -4,156 +4,10 @@ defmodule Livebook.AppsTest do
   import ExUnit.CaptureLog
   import Livebook.HubHelpers
 
+  alias Livebook.Apps
   alias Livebook.Utils
 
-  describe "deploy_apps_in_dir/1" do
-    @tag :tmp_dir
-    test "deploys apps", %{tmp_dir: tmp_dir} do
-      app1_path = Path.join(tmp_dir, "app1.livemd")
-      app2_path = Path.join(tmp_dir, "app2.livemd")
-
-      slug1 = Utils.random_short_id()
-      slug2 = Utils.random_short_id()
-
-      File.write!(app1_path, """
-      <!-- livebook:{"app_settings":{"access_type":"public","slug":"#{slug1}"}} -->
-
-      # App 1
-      """)
-
-      File.write!(app2_path, """
-      <!-- livebook:{"app_settings":{"access_type":"public","slug":"#{slug2}"}} -->
-
-      # App 2
-      """)
-
-      Livebook.Apps.subscribe()
-
-      Livebook.Apps.deploy_apps_in_dir(tmp_dir)
-
-      assert_receive {:app_created, %{slug: ^slug1} = app1}
-
-      assert_receive {:app_updated,
-                      %{slug: ^slug1, sessions: [%{app_status: %{execution: :executed}}]}}
-
-      assert_receive {:app_created, %{slug: ^slug2} = app2}
-
-      assert_receive {:app_updated,
-                      %{slug: ^slug2, sessions: [%{app_status: %{execution: :executed}}]}}
-
-      Livebook.App.close(app1.pid)
-      Livebook.App.close(app2.pid)
-    end
-
-    @tag :capture_log
-    @tag :tmp_dir
-    test "deploys apps with offline hub stamp", %{tmp_dir: tmp_dir} do
-      app1_path = Path.join(tmp_dir, "app1.livemd")
-      app2_path = Path.join(tmp_dir, "app2.livemd")
-      app3_path = Path.join(tmp_dir, "app3.livemd")
-
-      File.write!(app1_path, """
-      <!-- livebook:{"app_settings":{"slug":"offline_hub_app1"}} -->
-
-      # App 1
-      """)
-
-      File.write!(app2_path, """
-      <!-- livebook:{"app_settings":{"slug":"offline_hub_app2"},"hub_id":"team-org-number-3079"} -->
-
-      # App 2
-
-      ## Section 1
-
-      ```elixir
-      IO.puts("hey")
-      ```
-
-      <!-- livebook:{"offset":148,"stamp":{"token":"QTEyOEdDTQ.M486X5ISDg_wVwvndrMYJKIkfXa5qAwggh5LzkV41wVOY8SNQvfA4EFx4Tk.RrcteIrzJVrqQdhH.mtmu5KFj.bibmp2rxZoniYX1xY8dTvw","token_signature":"28ucTCoDXxahOIMg7SvdYIoLpGIUSahEa7mchH0jKncKeZH8-w-vOaL1F1uj_94lqQJFkmHWv988__1bPmYCorw7F1wAvAaprt3o2vSitWWmBszuF5JaimkFqOFcK3mc4NHuswQKuBjSL0W_yR-viiwlx6zPNsTpftVKjRI2Cri1PsMeZgahfdR2gy1OEgavzU6J6YWsNQHIMWgt5gwT6fIua1zaV7K8-TA6-6NRgcfG-pSJqRIm-3-vnbH5lkXRCgXCo_S9zWa6Jrcl5AbLObSr5DUueiwac1RobH7jNghCm1F-o1cUk9W-BJRZ7igVMwaYqLaOnKO8ya9CrkIiMg","version":1}} -->
-      """)
-
-      File.write!(app3_path, """
-      <!-- livebook:{"app_settings":{"slug":"offline_hub_app3"}} -->
-
-      # App 3
-
-      ## Section 1
-
-      ```elixir
-      IO.puts("hey")
-      ```
-
-      <!-- livebook:{"offset":111,"stamp":{"token":"invalid","token_signature":"invalid","version":1}} -->
-      """)
-
-      Livebook.Apps.subscribe()
-
-      Application.put_env(:livebook, :apps_path_hub_id, offline_hub().id, persistent: true)
-      Livebook.Apps.deploy_apps_in_dir(tmp_dir)
-      Application.delete_env(:livebook, :apps_path_hub_id, persistent: true)
-
-      refute_receive {:app_created, %{slug: "offline_hub_app1"}}
-      assert_receive {:app_created, %{pid: pid, slug: "offline_hub_app2"}}
-      refute_receive {:app_created, %{slug: "offline_hub_app3"}}
-
-      assert_receive {:app_updated,
-                      %{
-                        slug: "offline_hub_app2",
-                        sessions: [%{app_status: %{execution: :executed}}]
-                      }}
-
-      Livebook.App.close(pid)
-    end
-
-    @tag :capture_log
-    @tag :tmp_dir
-    test "deploys apps with offline hub secrets", %{tmp_dir: tmp_dir} do
-      app_path = Path.join(tmp_dir, "app1.livemd")
-      hub = offline_hub()
-
-      File.write!(app_path, """
-      <!-- livebook:{"app_settings":{"slug":"offline_hub_app2"},"hub_id":"team-org-number-3079"} -->
-
-      # App 2
-
-      ## Section 1
-
-      ```elixir
-      IO.puts("hey")
-      ```
-
-      <!-- livebook:{"offset":148,"stamp":{"token":"QTEyOEdDTQ.M486X5ISDg_wVwvndrMYJKIkfXa5qAwggh5LzkV41wVOY8SNQvfA4EFx4Tk.RrcteIrzJVrqQdhH.mtmu5KFj.bibmp2rxZoniYX1xY8dTvw","token_signature":"28ucTCoDXxahOIMg7SvdYIoLpGIUSahEa7mchH0jKncKeZH8-w-vOaL1F1uj_94lqQJFkmHWv988__1bPmYCorw7F1wAvAaprt3o2vSitWWmBszuF5JaimkFqOFcK3mc4NHuswQKuBjSL0W_yR-viiwlx6zPNsTpftVKjRI2Cri1PsMeZgahfdR2gy1OEgavzU6J6YWsNQHIMWgt5gwT6fIua1zaV7K8-TA6-6NRgcfG-pSJqRIm-3-vnbH5lkXRCgXCo_S9zWa6Jrcl5AbLObSr5DUueiwac1RobH7jNghCm1F-o1cUk9W-BJRZ7igVMwaYqLaOnKO8ya9CrkIiMg","version":1}} -->
-      """)
-
-      Livebook.Apps.subscribe()
-
-      secret = %Livebook.Secrets.Secret{
-        name: "DB_PASSWORD",
-        value: "postgres",
-        hub_id: hub.id
-      }
-
-      put_offline_hub_secret(secret)
-      assert secret in Livebook.Hubs.get_secrets(hub)
-
-      Livebook.Apps.deploy_apps_in_dir(tmp_dir)
-
-      assert_receive {:app_created, %{pid: pid, slug: "offline_hub_app2"}}
-
-      assert_receive {:app_updated,
-                      %{
-                        slug: "offline_hub_app2",
-                        sessions: [%{app_status: %{execution: :executed}}]
-                      }}
-
-      session_id = Livebook.App.get_session_id(pid)
-      {:ok, session} = Livebook.Sessions.fetch_session(session_id)
-      assert %{hub_secrets: [^secret]} = Livebook.Session.get_data(session.pid)
-
-      Livebook.App.close(pid)
-      remove_offline_hub_secret(secret)
-    end
-
+  describe "build_app_specs_in_dir/1" do
     @tag :tmp_dir
     test "skips apps with incomplete config and warns", %{tmp_dir: tmp_dir} do
       app_path = Path.join(tmp_dir, "app.livemd")
@@ -163,9 +17,52 @@ defmodule Livebook.AppsTest do
       """)
 
       assert capture_log(fn ->
-               Livebook.Apps.deploy_apps_in_dir(tmp_dir)
+               assert [] = Apps.build_app_specs_in_dir(tmp_dir)
              end) =~
-               "Skipping deployment for app at app.livemd. The deployment settings are missing or invalid. Please configure them under the notebook deploy panel."
+               "Ignoring app at app.livemd. The deployment settings are missing or invalid. Please configure them under the notebook deploy panel."
+    end
+
+    @tag :tmp_dir
+    test "skips apps with a duplicate slug and warns", %{tmp_dir: tmp_dir} do
+      app1_path = Path.join(tmp_dir, "app1.livemd")
+      app2_path = Path.join(tmp_dir, "app2.livemd")
+
+      slug = Utils.random_short_id()
+
+      File.write!(app1_path, """
+      <!-- livebook:{"app_settings":{"access_type":"public","slug":"#{slug}"}} -->
+
+      # App 1
+      """)
+
+      File.write!(app2_path, """
+      <!-- livebook:{"app_settings":{"access_type":"public","slug":"#{slug}"}} -->
+
+      # App 2
+      """)
+
+      assert capture_log(fn ->
+               assert [_] = Apps.build_app_specs_in_dir(tmp_dir)
+             end) =~ "App with the same slug (#{slug}) is already present at"
+    end
+
+    @tag :tmp_dir
+    test "warns when there are import warnings", %{tmp_dir: tmp_dir} do
+      app_path = Path.join(tmp_dir, "app.livemd")
+
+      slug = Utils.random_short_id()
+
+      File.write!(app_path, """
+      <!-- livebook:{"app_settings":{"access_type":"public","slug":"#{slug}"}} -->
+
+      # App
+
+      ```elixir
+      """)
+
+      assert capture_log(fn ->
+               assert [_] = Apps.build_app_specs_in_dir(tmp_dir)
+             end) =~ "line 5 - fenced Code Block opened with"
     end
 
     @tag :tmp_dir
@@ -180,16 +77,10 @@ defmodule Livebook.AppsTest do
       # App
       """)
 
-      Livebook.Apps.subscribe()
-
-      assert ExUnit.CaptureLog.capture_log(fn ->
-               Livebook.Apps.deploy_apps_in_dir(tmp_dir)
+      assert capture_log(fn ->
+               assert [_] = Apps.build_app_specs_in_dir(tmp_dir)
              end) =~
                "The app at app.livemd will use a random password. You may want to set LIVEBOOK_APPS_PATH_PASSWORD or make the app public."
-
-      assert_receive {:app_created, %{slug: ^slug} = app}
-
-      Livebook.App.close(app.pid)
     end
 
     @tag :tmp_dir
@@ -204,101 +95,117 @@ defmodule Livebook.AppsTest do
       # App
       """)
 
-      Livebook.Apps.subscribe()
+      Apps.subscribe()
 
-      Livebook.Apps.deploy_apps_in_dir(tmp_dir, password: "verylongpass")
+      assert [app_spec] = Apps.build_app_specs_in_dir(tmp_dir, password: "verylongpass")
 
-      assert_receive {:app_created, %{slug: ^slug} = app}
+      {:ok, %{notebook: notebook}} = Apps.AppSpec.load(app_spec, tmp_dir)
 
-      %{access_type: :protected, password: "verylongpass"} = Livebook.App.get_settings(app.pid)
-
-      Livebook.App.close(app.pid)
+      assert %{access_type: :protected, password: "verylongpass"} = notebook.app_settings
     end
 
     @tag :tmp_dir
-    test "deploys with import warnings", %{tmp_dir: tmp_dir} do
-      app_path = Path.join(tmp_dir, "app.livemd")
+    test "imports apps with valid offline hub stamp", %{tmp_dir: tmp_dir} do
+      app1_path = Path.join(tmp_dir, "app1.livemd")
+      app2_path = Path.join(tmp_dir, "app2.livemd")
+      app3_path = Path.join(tmp_dir, "app3.livemd")
 
-      slug = Utils.random_short_id()
-
-      File.write!(app_path, """
-      <!-- livebook:{"app_settings":{"access_type":"public","slug":"#{slug}"}} -->
+      # Personal hub
+      File.write!(app1_path, """
+      <!-- livebook:{"app_settings":{"slug":"app1"}} -->
 
       # App
+      """)
+
+      # Offline hub with valid stamp
+      File.write!(app2_path, """
+      <!-- livebook:{"app_settings":{"slug":"app2"},"hub_id":"team-org-number-3079"} -->
+
+      # App
+
+      ## Section 1
 
       ```elixir
+      IO.puts("hey")
+      ```
+
+      <!-- livebook:{"offset":134,"stamp":{"token":"XCP.0UjwZQFcT3wejYozhtkGIt6b-6Rvc42RBPkjVfrAnagq9wgDS2q7OhBwxLYncA","token_signature":"l2mD7sDTUOoaVgfjElpO5YyUcbeKO6udu9ReA5NI1ehV1cD90dXfo9EvuxbRK7D8t9qf8EEkUlKY17pNTdKIqJ5xD_Flxwq_Lh_qAfw7tmRJ--rujO8MpzRCC6kTIW9YB78KDQE-Yl-Hq49Rsp5aQQTNwmLvRdDVFIKjxI5s6Vb_npS8zzR_-YpgkalwxVPsobdpfLkIs4rxkQcNzeD3gifzjxF_izVrnWQmACcbNBIxZfmJmSdMvwnYQc4bTKjhPSZ25MHG-6W7CSR1G48x9DYdNcHTVqQoKvesxu2IQhVl8wd-7VzLf6T6OEEMGPozq4tYkMPKE4IRlI4XfQrDzA","version":1}} -->
       """)
 
-      Livebook.Apps.subscribe()
-
-      assert ExUnit.CaptureLog.capture_log(fn ->
-               Livebook.Apps.deploy_apps_in_dir(tmp_dir)
-             end) =~ "line 5 - fenced Code Block opened with"
-
-      assert_receive {:app_created, %{slug: ^slug, warnings: warnings} = app}
-
-      assert warnings == [
-               "Import: line 5 - fenced Code Block opened with ``` not closed at end of input"
-             ]
-
-      Livebook.App.close(app.pid)
-    end
-
-    @tag :tmp_dir
-    test "deploys notebook with attachment files", %{tmp_dir: tmp_dir} do
-      app_path = Path.join(tmp_dir, "app.livemd")
-      files_path = Path.join(tmp_dir, "files")
-      File.mkdir_p!(files_path)
-      image_path = Path.join(files_path, "image.jpg")
-      File.write!(image_path, "content")
-
-      slug = Utils.random_short_id()
-
-      File.write!(app_path, """
-      <!-- livebook:{"app_settings":{"access_type":"public","slug":"#{slug}"},"file_entries":[{"name":"image.jpg","type":"attachment"}]} -->
+      # Offline hub with invalid stamp
+      File.write!(app3_path, """
+      <!-- livebook:{"app_settings":{"slug":"app3"},"hub_id":"team-org-number-3079"} -->
 
       # App
+
+      ## Section 1
+
+      ```elixir
+      IO.puts("hey")
+      ```
+
+      <!-- livebook:{"offset":111,"stamp":{"token":"invalid","token_signature":"invalid","version":1}} -->
       """)
 
-      Livebook.Apps.subscribe()
+      log =
+        assert capture_log(fn ->
+                 assert [%{slug: "app2"}] =
+                          Apps.build_app_specs_in_dir(tmp_dir, hub_id: offline_hub().id)
+               end)
 
-      Livebook.Apps.deploy_apps_in_dir(tmp_dir)
+      assert log =~
+               "Ignoring app at app1.livemd. The notebook does not come from hub #{offline_hub().id}"
 
-      assert_receive {:app_created, %{slug: ^slug} = app}
-
-      session_id = Livebook.App.get_session_id(app.pid)
-      {:ok, session} = Livebook.Sessions.fetch_session(session_id)
-
-      assert Livebook.FileSystem.File.resolve(session.files_dir, "image.jpg")
-             |> Livebook.FileSystem.File.read() == {:ok, "content"}
-
-      Livebook.App.close(app.pid)
+      assert log =~
+               "Ignoring app at app3.livemd. The notebook does not have a valid stamp"
     end
+  end
 
+  describe "integration" do
     @tag :tmp_dir
-    test "skips existing apps when :start_only is enabled", %{tmp_dir: tmp_dir} do
+    test "deploying apps with offline hub secrets", %{tmp_dir: tmp_dir} do
       app_path = Path.join(tmp_dir, "app.livemd")
 
-      slug = Utils.random_short_id()
+      slug = "6kgwrh2d"
 
       File.write!(app_path, """
-      <!-- livebook:{"app_settings":{"access_type":"public","slug":"#{slug}"}} -->
+      <!-- livebook:{"app_settings":{"access_type":"public","slug":"6kgwrh2d"},"hub_id":"team-org-number-3079"} -->
 
       # App
+
+      ## Section 1
+
+      ```elixir
+      IO.puts("hey")
+      ```
+
+      <!-- livebook:{"offset":161,"stamp":{"token":"XCP.SHa_YbuqbAQXVSwGyrgskid5maz8KEdX1XCTGzGcaBGsieVkK50uw4QKfVLi1Wb9wwKM9yLxHxdxlsNOiKtn-kaqmBlTJWPUuYJWICFmIIVSsOi3r-g","token_signature":"UzYy7I-CIbxaZoKsV30ipdjhotJPOR4tCP6ZH5v6cnCe2F7kednP4aNUouTnoxUYwd4AZ59wGz1cCM7PYd8rE3IbECIbT4ixUpftI-hkW2OoymLRv5sjsfGeTPS8PvV9SXiZIG2320G3Kc1Spf4dToZfpNxAimD9xOpZLpRkI9MUH3nKG99yz1mZHuTgLNVS5yvHgAV_xRYtKwPnfwMLvQkD5Z9NacBPnqURVia90j1ueo7tEw8H1qH4VQU2Uh1XWIODIuTuLh55oe4MydGK5NgUhDMg2Zs9QaAN3ejoXHqWSub6k_VrHxIuke8T_xMIem6P25wdHGUZInSJX5x4Xw","version":1}} -->
       """)
 
-      Livebook.Apps.subscribe()
+      Apps.subscribe()
 
-      Livebook.Apps.deploy_apps_in_dir(tmp_dir)
-      assert_receive {:app_created, %{slug: ^slug} = app}
+      hub = offline_hub()
 
-      Livebook.Apps.deploy_apps_in_dir(tmp_dir)
-      assert %{version: 2} = Livebook.App.get_by_pid(app.pid)
+      secret = %Livebook.Secrets.Secret{
+        name: "DB_PASSWORD",
+        value: "postgres",
+        hub_id: hub.id
+      }
 
-      Livebook.Apps.deploy_apps_in_dir(tmp_dir, start_only: true)
-      assert %{version: 2} = Livebook.App.get_by_pid(app.pid)
+      put_offline_hub_secret(secret)
 
-      Livebook.App.close(app.pid)
+      assert [app_spec] = Apps.build_app_specs_in_dir(tmp_dir)
+
+      deployer_pid = Apps.Deployer.local_deployer()
+      Apps.Deployer.deploy_monitor(deployer_pid, app_spec)
+
+      assert_receive {:app_created, %{pid: pid, slug: ^slug}}
+      assert_receive {:app_updated, %{slug: ^slug, sessions: [session]}}
+
+      assert %{secrets: %{"DB_PASSWORD" => ^secret}} = Livebook.Session.get_data(session.pid)
+
+      remove_offline_hub_secret(secret)
+      Livebook.App.close(pid)
     end
   end
 end

test/livebook/live_markdown/import_test.exs

@@ -62,7 +62,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
     ```
     """
 
-    {notebook, []} = Import.notebook_from_livemd(markdown)
+    {notebook, %{warnings: []}} = Import.notebook_from_livemd(markdown)
 
     # Match only on the relevant fields as some may be generated (ids).
 
@@ -159,7 +159,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
     | Maine | ME | Augusta |
     """
 
-    {notebook, []} = Import.notebook_from_livemd(markdown)
+    {notebook, %{warnings: []}} = Import.notebook_from_livemd(markdown)
 
     assert %Notebook{
              name: "My Notebook",
@@ -188,7 +188,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
     Some markdown.
     """
 
-    {notebook, []} = Import.notebook_from_livemd(markdown)
+    {notebook, %{warnings: []}} = Import.notebook_from_livemd(markdown)
 
     assert %Notebook{
              name: "Untitled notebook",
@@ -213,7 +213,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
     ###### Tiny heading
     """
 
-    {notebook, messages} = Import.notebook_from_livemd(markdown)
+    {notebook, %{warnings: messages}} = Import.notebook_from_livemd(markdown)
 
     assert %Notebook{
              name: "Untitled notebook",
@@ -257,7 +257,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
     ## # Section
     """
 
-    {notebook, []} = Import.notebook_from_livemd(markdown)
+    {notebook, %{warnings: []}} = Import.notebook_from_livemd(markdown)
 
     assert %Notebook{
              name: "My *Notebook*",
@@ -278,7 +278,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
     ## Actual section
     """
 
-    {notebook, []} = Import.notebook_from_livemd(markdown)
+    {notebook, %{warnings: []}} = Import.notebook_from_livemd(markdown)
 
     assert %Notebook{
              name: "My Notebook",
@@ -307,7 +307,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
     ```
     """
 
-    {notebook, []} = Import.notebook_from_livemd(markdown)
+    {notebook, %{warnings: []}} = Import.notebook_from_livemd(markdown)
 
     assert %Notebook{
              name: "Untitled notebook",
@@ -337,7 +337,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
     Some markdown.
     """
 
-    {notebook, messages} = Import.notebook_from_livemd(markdown)
+    {notebook, %{warnings: messages}} = Import.notebook_from_livemd(markdown)
 
     assert %Notebook{
              name: "My Notebook",
@@ -369,7 +369,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
     Some markdown.
     """
 
-    {_notebook, messages} = Import.notebook_from_livemd(markdown)
+    {_notebook, %{warnings: messages}} = Import.notebook_from_livemd(markdown)
 
     assert ["line 3 - closing unclosed backquotes ` at end of input"] == messages
   end
@@ -395,7 +395,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
     ```
     """
 
-    {notebook, []} = Import.notebook_from_livemd(markdown)
+    {notebook, %{warnings: []}} = Import.notebook_from_livemd(markdown)
 
     assert %Notebook{
              name: "My Notebook",
@@ -451,7 +451,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
     ```
     """
 
-    {notebook, []} = Import.notebook_from_livemd(markdown)
+    {notebook, %{warnings: []}} = Import.notebook_from_livemd(markdown)
 
     assert %Notebook{
              name: "My Notebook",
@@ -499,7 +499,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
     Cell 2
     """
 
-    {notebook, []} = Import.notebook_from_livemd(markdown)
+    {notebook, %{warnings: []}} = Import.notebook_from_livemd(markdown)
 
     assert %Notebook{
              name: "My Notebook",
@@ -543,7 +543,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
     Cell 1
     """
 
-    {notebook, []} = Import.notebook_from_livemd(markdown)
+    {notebook, %{warnings: []}} = Import.notebook_from_livemd(markdown)
 
     assert %Notebook{
              name: "My Notebook",
@@ -581,7 +581,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
     Cell 1
     """
 
-    {notebook, messages} = Import.notebook_from_livemd(markdown)
+    {notebook, %{warnings: messages}} = Import.notebook_from_livemd(markdown)
 
     assert %Notebook{
              name: "My Notebook",
@@ -630,7 +630,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
       ```
       """
 
-      {notebook, []} = Import.notebook_from_livemd(markdown)
+      {notebook, %{warnings: []}} = Import.notebook_from_livemd(markdown)
 
       assert %Notebook{
                name: "My Notebook",
@@ -679,7 +679,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
       ```
       """
 
-      {notebook, []} = Import.notebook_from_livemd(markdown)
+      {notebook, %{warnings: []}} = Import.notebook_from_livemd(markdown)
 
       assert %Notebook{
                name: "My Notebook",
@@ -718,7 +718,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
       # My Notebook
       """
 
-      {notebook, []} = Import.notebook_from_livemd(markdown)
+      {notebook, %{warnings: []}} = Import.notebook_from_livemd(markdown)
 
       assert %Notebook{name: "My Notebook", persist_outputs: true} = notebook
     end
@@ -731,7 +731,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
     # My Notebook
     """
 
-    {notebook, []} = Import.notebook_from_livemd(markdown)
+    {notebook, %{warnings: []}} = Import.notebook_from_livemd(markdown)
 
     assert %Notebook{name: "My Notebook", autosave_interval_s: 10} = notebook
   end
@@ -743,7 +743,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
     # My Notebook
     """
 
-    {notebook, []} = Import.notebook_from_livemd(markdown)
+    {notebook, %{warnings: []}} = Import.notebook_from_livemd(markdown)
 
     assert %Notebook{name: "My Notebook", default_language: :erlang} = notebook
   end
@@ -757,7 +757,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
     # My Notebook
     """
 
-    {notebook, []} = Import.notebook_from_livemd(markdown)
+    {notebook, %{warnings: []}} = Import.notebook_from_livemd(markdown)
 
     assert %Notebook{name: "My Notebook", hub_id: ^hub_id} = notebook
   end
@@ -769,7 +769,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
     # My Notebook
     """
 
-    {notebook, messages} = Import.notebook_from_livemd(markdown)
+    {notebook, %{warnings: messages}} = Import.notebook_from_livemd(markdown)
 
     assert notebook.hub_id != "nonexistent"
     assert ["this notebook belongs to an Organization you don't have access to" <> _] = messages
@@ -783,7 +783,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
       # My Notebook
       """
 
-      {notebook, []} = Import.notebook_from_livemd(markdown)
+      {notebook, %{warnings: []}} = Import.notebook_from_livemd(markdown)
 
       assert %Notebook{
                name: "My Notebook",
@@ -807,7 +807,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
       # My Notebook
       """
 
-      {notebook, []} = Import.notebook_from_livemd(markdown)
+      {notebook, %{warnings: []}} = Import.notebook_from_livemd(markdown)
 
       assert %Notebook{
                name: "My Notebook",
@@ -826,7 +826,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
       <!-- livebook:{"livebook_object":"cell_input","name":"length","type":"text","value":"100"} -->
       """
 
-      {_notebook, messages} = Import.notebook_from_livemd(markdown)
+      {_notebook, %{warnings: messages}} = Import.notebook_from_livemd(markdown)
 
       assert [
                "found an input cell, but those are no longer supported, please use Kino.Input instead"
@@ -842,7 +842,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
       <!-- livebook:{"livebook_object":"cell_input","name":"length","reactive":true,"type":"text","value":"100"} -->
       """
 
-      {_notebook, messages} = Import.notebook_from_livemd(markdown)
+      {_notebook, %{warnings: messages}} = Import.notebook_from_livemd(markdown)
 
       assert [
                "found an input cell, but those are no longer supported, please use Kino.Input instead." <>
@@ -873,7 +873,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
       ```
       """
 
-      {notebook, []} = Import.notebook_from_livemd(markdown)
+      {notebook, %{warnings: []}} = Import.notebook_from_livemd(markdown)
 
       assert %Notebook{
                name: "My Notebook",
@@ -908,7 +908,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
       ![](images/dog.jpeg)
       """
 
-      {_notebook, messages} = Import.notebook_from_livemd(markdown)
+      {_notebook, %{warnings: messages}} = Import.notebook_from_livemd(markdown)
 
       assert [
                "found Markdown images pointing to the images/ directory. Using this directory has been deprecated, please use notebook files instead"
@@ -928,7 +928,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
       ```
       """
 
-      {notebook, []} = Import.notebook_from_livemd(markdown)
+      {notebook, %{warnings: []}} = Import.notebook_from_livemd(markdown)
 
       assert %Notebook{
                name: "My Notebook",
@@ -969,7 +969,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
     ```
     """
 
-    {notebook, messages} = Import.notebook_from_livemd(markdown)
+    {notebook, %{warnings: messages}} = Import.notebook_from_livemd(markdown)
 
     assert %Notebook{
              name: "My Notebook",
@@ -1001,7 +1001,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
     ```
     """
 
-    {notebook, messages} = Import.notebook_from_livemd(markdown)
+    {notebook, %{warnings: messages}} = Import.notebook_from_livemd(markdown)
 
     assert %Notebook{
              name: "My Notebook",
@@ -1043,7 +1043,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
     ```
     """
 
-    {notebook, messages} = Import.notebook_from_livemd(markdown)
+    {notebook, %{warnings: messages}} = Import.notebook_from_livemd(markdown)
 
     assert %Notebook{
              name: "My Notebook",
@@ -1080,7 +1080,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
       ## Section 1
       """
 
-      {notebook, []} = Import.notebook_from_livemd(markdown)
+      {notebook, %{warnings: []}} = Import.notebook_from_livemd(markdown)
 
       assert %Notebook{
                name: "My Notebook",
@@ -1107,7 +1107,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
       ```
       """
 
-      {notebook, []} = Import.notebook_from_livemd(markdown)
+      {notebook, %{warnings: []}} = Import.notebook_from_livemd(markdown)
 
       assert %Notebook{
                name: "My Notebook",
@@ -1160,7 +1160,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
       <!-- livebook:{"offset":58,"stamp":{"token":"XCP.XcdH6x1x9B90SIKObuM8NWuEN7Tg2nyGWV3YhYtw6M0h8c4K0N5EFa8krthkrIqdIj6aEpUcsbEm4klRkSIh_W2YV1PXuMRQA0vCYU042IVFDbz1gq4","version":2}} -->
       """
 
-      {notebook, []} = Import.notebook_from_livemd(markdown)
+      {notebook, %{warnings: [], stamp_verified?: true}} = Import.notebook_from_livemd(markdown)
 
       assert %Notebook{hub_secret_names: ["DB_PASSWORD"]} = notebook
     end
@@ -1178,7 +1178,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
       <!-- livebook:{"offset":58,"stamp":{"token":"QTEyOEdDTQ.LF8LTeMYrtq8S7wsKMmk2YgOQzMAkEKT2d8fq1Gz3Ot1mydOgEZ1B4hcEZc.Wec6NwBQ584kE661.a_N-5jDiWrjhHha9zxHQ6JJOmxeqgiya3m6YlKt1Na_DPnEfXyLnengaUzQSrf8.ZoD5r6-H87RpTyvFkvEOQw","version":1}} -->
       """
 
-      {notebook, []} = Import.notebook_from_livemd(markdown)
+      {notebook, %{warnings: [], stamp_verified?: true}} = Import.notebook_from_livemd(markdown)
 
       assert %Notebook{hub_secret_names: ["DB_PASSWORD"]} = notebook
     end
@@ -1196,7 +1196,8 @@ defmodule Livebook.LiveMarkdown.ImportTest do
       <!-- livebook:{"offset":58,"stamp":{"token":"invalid","version":2}} -->
       """
 
-      {notebook, messages} = Import.notebook_from_livemd(markdown)
+      {notebook, %{warnings: messages, stamp_verified?: false}} =
+        Import.notebook_from_livemd(markdown)
 
       assert %Notebook{hub_secret_names: []} = notebook
 
@@ -1219,7 +1220,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
       <!-- livebook:{"offset":111,"stamp":{"token":"XCP.PgQjafbthVPSi0vQHsWBVoxWjs2V-IWPR4ADSbNGEuePE0uneqtT1rDJHKkJs9W__Q5cflYclSPEyIQwkGWw6IKHlLsy56PBDH2CiHvvy5GfVEpq0vA","token_signature":"KPp0SdwSfEFubAml93UnBH06Yvd4OKULmtF4FmxZ_iE9qR_o2OwCiMQH_MX7A6yTiXeKCrwlZEV-8m6AhX-t6FXc177m8RL5FmXVqrRZw57V7FuxrGacZjYDCTwpGhBQmIAynhfDt6nVmeQyof8bsiW3sskii9171Fa_XFAoSqJqC1J_o2MFRk16o607N-xwTadGsCVyYSl4FUhmEXraOr0krIEe8bdSQOcpXxaDmRJNAUAJkJd3LRJDt8ZkwgmMm4UJopWozQIk2fZGfSO-cepEHoy9HlrgBGWuNL7_J6z7nLxB4p_vF_mOX7fMhIOfzVRxqmzUmzlXZkEPcKhhgQ","version":1}} -->
       """
 
-      {notebook, []} = Import.notebook_from_livemd(markdown)
+      {notebook, %{warnings: [], stamp_verified?: true}} = Import.notebook_from_livemd(markdown)
 
       assert %Notebook{
                hub_id: "team-org-number-3079",
@@ -1243,7 +1244,8 @@ defmodule Livebook.LiveMarkdown.ImportTest do
       <!-- livebook:{"offset":58,"stamp":{"token":"invalid","token_signature":"invalid","version":1}} -->
       """
 
-      {notebook, messages} = Import.notebook_from_livemd(markdown)
+      {notebook, %{warnings: messages, stamp_verified?: false}} =
+        Import.notebook_from_livemd(markdown)
 
       assert %Notebook{hub_id: "team-org-number-3079", teams_enabled: false} = notebook
       assert ["invalid notebook stamp" <> _] = messages
@@ -1266,7 +1268,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
       <!-- livebook:{"offset":58,"stamp":{"token":"invalid","token_signature":"invalid","version":1}} -->
       """
 
-      {notebook, [_]} = Import.notebook_from_livemd(markdown)
+      {notebook, %{warnings: [_], stamp_verified?: false}} = Import.notebook_from_livemd(markdown)
 
       assert %Notebook{hub_id: ^hub_id, teams_enabled: true} = notebook
 
@@ -1282,7 +1284,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
       # My Notebook
       """
 
-      {notebook, []} = Import.notebook_from_livemd(markdown)
+      {notebook, %{warnings: []}} = Import.notebook_from_livemd(markdown)
 
       assert %Notebook{
                file_entries: [
@@ -1307,7 +1309,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
       # My Notebook
       """
 
-      {notebook, []} = Import.notebook_from_livemd(markdown)
+      {notebook, %{warnings: []}} = Import.notebook_from_livemd(markdown)
 
       assert %Notebook{
                file_entries: [
@@ -1342,7 +1344,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
       # Change file path in the document
       markdown = String.replace(markdown, p("/document.pdf"), p("/other.pdf"))
 
-      {notebook, _} = Import.notebook_from_livemd(markdown)
+      {notebook, %{warnings: _}} = Import.notebook_from_livemd(markdown)
 
       assert %Notebook{
                file_entries: [
@@ -1375,7 +1377,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do
         }
         |> Livebook.LiveMarkdown.Export.notebook_to_livemd()
 
-      {notebook, []} = Import.notebook_from_livemd(markdown)
+      {notebook, %{warnings: []}} = Import.notebook_from_livemd(markdown)
 
       assert %Notebook{
                file_entries: [

test/livebook/session_test.exs

@@ -2,6 +2,7 @@ defmodule Livebook.SessionTest do
   use ExUnit.Case, async: true
 
   import Livebook.HubHelpers
+  import Livebook.AppHelpers
   import Livebook.TestHelpers
 
   alias Livebook.{Session, Text, Runtime, Utils, Notebook, FileSystem, Apps, App}
@@ -1352,9 +1353,10 @@ defmodule Livebook.SessionTest do
       notebook = %{Notebook.new() | app_settings: app_settings}
 
       Apps.subscribe()
-      {:ok, app_pid} = Apps.deploy(notebook)
+      app_pid = deploy_notebook_sync(notebook)
 
-      assert_receive {:app_created, %{pid: ^app_pid, sessions: [%{pid: session_pid}]}}
+      assert_receive {:app_created, %{pid: ^app_pid}}
+      assert_receive {:app_updated, %{pid: ^app_pid, sessions: [%{pid: session_pid}]}}
 
       ref = Process.monitor(session_pid)
 
@@ -1369,9 +1371,10 @@ defmodule Livebook.SessionTest do
       notebook = %{Notebook.new() | app_settings: app_settings}
 
       Apps.subscribe()
-      {:ok, app_pid} = Apps.deploy(notebook)
+      app_pid = deploy_notebook_sync(notebook)
 
-      assert_receive {:app_created, %{pid: ^app_pid, sessions: [%{pid: session_pid}]}}
+      assert_receive {:app_created, %{pid: ^app_pid}}
+      assert_receive {:app_updated, %{pid: ^app_pid, sessions: [%{pid: session_pid}]}}
 
       client_pid = spawn_link(fn -> receive do: (:stop -> :ok) end)
 
@@ -1407,7 +1410,7 @@ defmodule Livebook.SessionTest do
       notebook = %{Notebook.new() | sections: [section], app_settings: app_settings}
 
       Apps.subscribe()
-      {:ok, app_pid} = Apps.deploy(notebook)
+      app_pid = deploy_notebook_sync(notebook)
 
       assert_receive {:app_created, %{pid: ^app_pid} = app}
 
@@ -1437,7 +1440,7 @@ defmodule Livebook.SessionTest do
 
       # Multi-session
 
-      {:ok, app_pid} = Apps.deploy(notebook)
+      app_pid = deploy_notebook_sync(notebook)
       session_id = App.get_session_id(app_pid, user: user)
       {:ok, session} = Livebook.Sessions.fetch_session(session_id)
 
@@ -1449,7 +1452,7 @@ defmodule Livebook.SessionTest do
       # Single-session
 
       notebook = put_in(notebook.app_settings.multi_session, false)
-      {:ok, app_pid} = Apps.deploy(notebook)
+      app_pid = deploy_notebook_sync(notebook)
       session_id = App.get_session_id(app_pid, user: user)
       {:ok, session} = Livebook.Sessions.fetch_session(session_id)
 
@@ -1475,7 +1478,7 @@ defmodule Livebook.SessionTest do
 
       # Multi-session
 
-      {:ok, app_pid} = Apps.deploy(notebook)
+      app_pid = deploy_notebook_sync(notebook)
       session_id = App.get_session_id(app_pid, user: user)
       {:ok, session} = Livebook.Sessions.fetch_session(session_id)
 
@@ -1495,7 +1498,7 @@ defmodule Livebook.SessionTest do
       # Single-session
 
       notebook = put_in(notebook.app_settings.multi_session, false)
-      {:ok, app_pid} = Apps.deploy(notebook)
+      app_pid = deploy_notebook_sync(notebook)
       session_id = App.get_session_id(app_pid, user: user)
       {:ok, session} = Livebook.Sessions.fetch_session(session_id)
 
@@ -1943,6 +1946,7 @@ defmodule Livebook.SessionTest do
   end
 
   defmodule Global do
+    # Not async, because we alter global config (default hub)
     use ExUnit.Case, async: false
 
     describe "default hub for new notebooks" do
@@ -1980,9 +1984,13 @@ defmodule Livebook.SessionTest do
   end
 
   defp start_session(opts \\ []) do
-    opts = Keyword.merge([id: Utils.random_id()], opts)
-    pid = start_supervised!({Session, opts}, id: opts[:id])
-    Session.get_by_pid(pid)
+    {:ok, session} = Livebook.Sessions.create_session(opts)
+
+    on_exit(fn ->
+      Session.close(session.pid)
+    end)
+
+    session
   end
 
   defp insert_section_and_cell(session_pid) do

test/livebook_teams/apps_test.exs

@@ -1,22 +1,25 @@
 defmodule Livebook.Integration.AppsTest do
   use Livebook.TeamsIntegrationCase, async: true
 
-  describe "deploy_apps_in_dir/1" do
+  alias Livebook.Apps
+
+  describe "integration" do
     @tag :tmp_dir
-    test "deploys apps with hub secrets", %{user: user, node: node, tmp_dir: tmp_dir} do
+    test "deploying apps with hub secrets", %{user: user, node: node, tmp_dir: tmp_dir} do
       Livebook.Hubs.Broadcasts.subscribe([:secrets])
 
       hub = create_team_hub(user, node)
       hub_id = hub.id
       secret = insert_secret(name: "DB_PASSWORD", value: "postgres", hub_id: hub.id)
       secret_name = secret.name
+      slug = Livebook.Utils.random_short_id()
 
       assert_receive {:secret_created, %{hub_id: ^hub_id, name: ^secret_name}}
 
       app_path = Path.join(tmp_dir, "app.livemd")
 
-      markdown = """
-      <!-- livebook:{"app_settings":{"slug":"#{hub_id}"},"hub_id":"#{hub_id}"} -->
+      source = """
+      <!-- livebook:{"app_settings":{"access_type":"public","slug":"#{slug}"},"hub_id":"#{hub_id}"} -->
 
       # Team Hub
 
@@ -27,25 +30,22 @@ defmodule Livebook.Integration.AppsTest do
       ```
       """
 
-      {notebook, []} = Livebook.LiveMarkdown.notebook_from_livemd(markdown)
+      {notebook, %{warnings: []}} = Livebook.LiveMarkdown.notebook_from_livemd(source)
       notebook = Map.replace!(notebook, :hub_secret_names, [secret_name])
       {source, []} = Livebook.LiveMarkdown.notebook_to_livemd(notebook)
-
       File.write!(app_path, source)
 
-      Livebook.Apps.subscribe()
-      Livebook.Apps.deploy_apps_in_dir(tmp_dir)
+      Apps.subscribe()
 
-      assert_receive {:app_created, %{pid: pid, slug: ^hub_id}}
+      assert [app_spec] = Apps.build_app_specs_in_dir(tmp_dir)
 
-      assert_receive {:app_updated,
-                      %{slug: ^hub_id, sessions: [%{app_status: %{execution: :executed}}]}}
+      deployer_pid = Apps.Deployer.local_deployer()
+      Apps.Deployer.deploy_monitor(deployer_pid, app_spec)
 
-      session_id = Livebook.App.get_session_id(pid)
-      {:ok, session} = Livebook.Sessions.fetch_session(session_id)
+      assert_receive {:app_created, %{pid: pid, slug: ^slug}}
+      assert_receive {:app_updated, %{slug: ^slug, sessions: [session]}}
 
-      assert %{notebook: %{hub_secret_names: [^secret_name]}, hub_secrets: [^secret]} =
-               Livebook.Session.get_data(session.pid)
+      assert %{secrets: %{^secret_name => ^secret}} = Livebook.Session.get_data(session.pid)
 
       Livebook.App.close(pid)
     end

test/livebook_teams/hubs/team_client_test.exs

@@ -3,8 +3,6 @@ defmodule Livebook.Hubs.TeamClientTest do
 
   alias Livebook.Hubs.TeamClient
 
-  @moduletag :capture_log
-
   setup do
     Livebook.Hubs.Broadcasts.subscribe([:connection, :file_systems, :secrets])
     Livebook.Teams.Broadcasts.subscribe([:deployment_groups, :app_deployments])
@@ -682,6 +680,7 @@ defmodule Livebook.Hubs.TeamClientTest do
         }
 
       agent_connected = %{agent_connected | deployment_groups: [livebook_proto_deployment_group]}
+
       pid = connect_to_teams(team)
 
       # Since we're connecting as Agent, we should receive the
@@ -745,14 +744,6 @@ defmodule Livebook.Hubs.TeamClientTest do
 
       agent_connected = %{agent_connected | app_deployments: [livebook_proto_app_deployment]}
 
-      apps_path = Path.join(tmp_dir, "apps")
-      app_path = Path.join(apps_path, slug)
-      Application.put_env(:livebook, :apps_path, apps_path)
-
-      # To avoid having extracting to the same folder from the original notebook,
-      # we need to create the ./apps/{slug} folder before sending the event
-      File.mkdir_p!(app_path)
-
       Livebook.Apps.subscribe()
 
       send(pid, {:event, :agent_connected, agent_connected})
@@ -764,15 +755,14 @@ defmodule Livebook.Hubs.TeamClientTest do
 
       assert app_deployment in TeamClient.get_app_deployments(team.id)
 
+      Livebook.Hubs.delete_hub(team.id)
       Livebook.App.close(app_pid)
-      Application.put_env(:livebook, :apps_path, nil)
     end
   end
 
   defp connect_to_teams(%{id: id} = team) do
-    {:ok, pid} = TeamClient.start_link(team)
+    Livebook.Hubs.save_hub(team)
     assert_receive {:hub_connected, ^id}
-
-    pid
+    TeamClient.get_pid(team.id)
   end
 end

test/livebook_teams/hubs/team_test.exs

@@ -3,8 +3,6 @@ defmodule Livebook.Hubs.TeamTest do
 
   alias Livebook.Hubs.Provider
 
-  @moduletag :capture_log
-
   describe "stamping" do
     test "generates and verifies stamp for a notebook", %{user: user, node: node} do
       team = create_team_hub(user, node)

test/livebook_teams/teams/connection_test.exs

@@ -2,8 +2,6 @@ defmodule Livebook.Teams.ConnectionTest do
   alias Livebook.FileSystem
   use Livebook.TeamsIntegrationCase, async: true
 
-  @moduletag :capture_log
-
   alias Livebook.Teams.Connection
 
   describe "connect" do

test/livebook_teams/web/hub/edit_live_test.exs

@@ -465,7 +465,7 @@ defmodule LivebookWeb.Integration.Hub.EditLiveTest do
       refute_receive {:secret_created, ^secret}
 
       assert render(view) =~
-               "You are not authorized to perform this action, make sure you have the access or you are not in a Livebook Agent instance"
+               "You are not authorized to perform this action, make sure you have the access or you are not in a Livebook Agent/Offline instance"
 
       refute secret in Livebook.Hubs.get_secrets(hub)
     end
@@ -504,7 +504,7 @@ defmodule LivebookWeb.Integration.Hub.EditLiveTest do
       refute_receive {:file_system_created, %{id: ^id}}
 
       assert render(view) =~
-               "You are not authorized to perform this action, make sure you have the access or you are not in a Livebook Agent instance"
+               "You are not authorized to perform this action, make sure you have the access or you are not in a Livebook Agent/Offline instance"
 
       refute file_system in Livebook.Hubs.get_file_systems(hub)
     end
@@ -523,6 +523,7 @@ defmodule LivebookWeb.Integration.Hub.EditLiveTest do
   end
 
   defmodule Global do
+    # Not async, because we alter global config (default hub)
     use Livebook.TeamsIntegrationCase, async: false
 
     setup %{user: user, node: node} do

test/livebook_web/live/app_auth_live_test.exs

@@ -1,4 +1,5 @@
 defmodule LivebookWeb.AppAuthLiveTest do
+  # Not async, because we alter global config (auth mode)
   use LivebookWeb.ConnCase, async: false
 
   import Phoenix.LiveViewTest
@@ -30,8 +31,15 @@ defmodule LivebookWeb.AppAuthLiveTest do
       |> Map.merge(app_settings_attrs)
 
     notebook = %{Livebook.Notebook.new() | app_settings: app_settings}
+    app_spec = Livebook.Apps.NotebookAppSpec.new(notebook)
 
-    {:ok, app_pid} = Livebook.Apps.deploy(notebook)
+    deployer_pid = Livebook.Apps.Deployer.local_deployer()
+    ref = Livebook.Apps.Deployer.deploy_monitor(deployer_pid, app_spec)
+
+    app_pid =
+      receive do
+        {:deploy_result, ^ref, {:ok, pid}} -> pid
+      end
 
     {slug, app_pid}
   end

test/livebook_web/live/app_live_test.exs

@@ -2,6 +2,7 @@ defmodule LivebookWeb.AppLiveTest do
   use LivebookWeb.ConnCase, async: true
 
   import Phoenix.LiveViewTest
+  import Livebook.AppHelpers
 
   alias Livebook.{App, Apps, Notebook, Utils}
 
@@ -12,9 +13,10 @@ defmodule LivebookWeb.AppLiveTest do
       notebook = %{Notebook.new() | app_settings: app_settings}
 
       Apps.subscribe()
-      {:ok, app_pid} = Apps.deploy(notebook)
+      app_pid = deploy_notebook_sync(notebook)
 
-      assert_receive {:app_created, %{pid: ^app_pid, sessions: [%{id: session_id}]}}
+      assert_receive {:app_created, %{pid: ^app_pid}}
+      assert_receive {:app_updated, %{pid: ^app_pid, sessions: [%{id: session_id}]}}
 
       {:error, {:live_redirect, %{to: to}}} = live(conn, ~p"/apps/#{slug}")
       assert to == ~p"/apps/#{slug}/#{session_id}"
@@ -30,7 +32,7 @@ defmodule LivebookWeb.AppLiveTest do
       notebook = %{Notebook.new() | app_settings: app_settings}
 
       Apps.subscribe()
-      {:ok, app_pid} = Apps.deploy(notebook)
+      app_pid = deploy_notebook_sync(notebook)
 
       assert_receive {:app_created, %{pid: ^app_pid, sessions: []}}
 
@@ -82,7 +84,7 @@ defmodule LivebookWeb.AppLiveTest do
       notebook = %{Notebook.new() | app_settings: app_settings}
 
       Apps.subscribe()
-      {:ok, app_pid} = Apps.deploy(notebook)
+      app_pid = deploy_notebook_sync(notebook)
 
       assert_receive {:app_created, %{pid: ^app_pid, sessions: []}}
 
@@ -114,7 +116,7 @@ defmodule LivebookWeb.AppLiveTest do
 
       Apps.subscribe()
 
-      {:ok, app_pid} = Apps.deploy(notebook)
+      app_pid = deploy_notebook_sync(notebook)
       session_id = App.get_session_id(app_pid)
       assert_receive {:app_updated, %{pid: ^app_pid, sessions: [%{id: ^session_id}]}}
 

test/livebook_web/live/app_session_live_test.exs

@@ -3,6 +3,7 @@ defmodule LivebookWeb.AppSessionLiveTest do
 
   import Phoenix.LiveViewTest
   import Livebook.TestHelpers
+  import Livebook.AppHelpers
 
   alias Livebook.{App, Apps, Notebook, Utils}
 
@@ -11,7 +12,7 @@ defmodule LivebookWeb.AppSessionLiveTest do
     app_settings = %{Notebook.AppSettings.new() | slug: slug}
     notebook = %{Notebook.new() | app_settings: app_settings}
 
-    {:ok, app_pid} = Apps.deploy(notebook)
+    app_pid = deploy_notebook_sync(notebook)
 
     {:ok, view, _} = live(conn, ~p"/apps/#{slug}/nonexistent")
     assert render(view) =~ "This app session does not exist"
@@ -26,7 +27,7 @@ defmodule LivebookWeb.AppSessionLiveTest do
     notebook = %{Notebook.new() | app_settings: app_settings}
 
     Apps.subscribe()
-    {:ok, app_pid} = Apps.deploy(notebook)
+    app_pid = deploy_notebook_sync(notebook)
 
     assert_receive {:app_created, %{pid: ^app_pid}}
 
@@ -51,7 +52,7 @@ defmodule LivebookWeb.AppSessionLiveTest do
     notebook = %{Notebook.new() | app_settings: app_settings}
 
     Apps.subscribe()
-    {:ok, app_pid} = Apps.deploy(notebook)
+    app_pid = deploy_notebook_sync(notebook)
 
     assert_receive {:app_created, %{pid: ^app_pid}}
 
@@ -99,7 +100,7 @@ defmodule LivebookWeb.AppSessionLiveTest do
     }
 
     Livebook.Apps.subscribe()
-    {:ok, app_pid} = Apps.deploy(notebook)
+    app_pid = deploy_notebook_sync(notebook)
 
     assert_receive {:app_created, %{pid: ^app_pid} = app}
 
@@ -151,7 +152,7 @@ defmodule LivebookWeb.AppSessionLiveTest do
     }
 
     Livebook.Apps.subscribe()
-    {:ok, app_pid} = Apps.deploy(notebook)
+    app_pid = deploy_notebook_sync(notebook)
 
     assert_receive {:app_created, %{pid: ^app_pid} = app}
 
@@ -224,7 +225,7 @@ defmodule LivebookWeb.AppSessionLiveTest do
     }
 
     Livebook.Apps.subscribe()
-    {:ok, app_pid} = Apps.deploy(notebook)
+    app_pid = deploy_notebook_sync(notebook)
 
     assert_receive {:app_created, %{pid: ^app_pid} = app}
 

test/livebook_web/live/apps_dashboard_live_test.exs

@@ -3,6 +3,7 @@ defmodule LivebookWeb.AppsDashboardLiveTest do
 
   import Phoenix.LiveViewTest
   import Livebook.TestHelpers
+  import Livebook.AppHelpers
 
   alias Livebook.{App, Apps, Notebook, Utils}
 
@@ -16,7 +17,7 @@ defmodule LivebookWeb.AppsDashboardLiveTest do
     refute render(view) =~ slug
 
     Apps.subscribe()
-    {:ok, app_pid} = Apps.deploy(notebook)
+    app_pid = deploy_notebook_sync(notebook)
 
     assert_receive {:app_created, %{pid: ^app_pid}}
     assert render(view) =~ "My app #{slug}"
@@ -35,7 +36,7 @@ defmodule LivebookWeb.AppsDashboardLiveTest do
     {:ok, view, _} = live(conn, ~p"/apps-dashboard")
 
     Apps.subscribe()
-    {:ok, app_pid} = Apps.deploy(notebook)
+    app_pid = deploy_notebook_sync(notebook)
 
     assert_receive {:app_created, %{pid: ^app_pid}}
 
@@ -56,7 +57,7 @@ defmodule LivebookWeb.AppsDashboardLiveTest do
     {:ok, view, _} = live(conn, ~p"/apps-dashboard")
 
     Apps.subscribe()
-    {:ok, app_pid} = Apps.deploy(notebook)
+    app_pid = deploy_notebook_sync(notebook)
 
     assert_receive {:app_created, %{pid: ^app_pid}}
 

test/livebook_web/plugs/auth_plug_test.exs

@@ -1,4 +1,5 @@
 defmodule LivebookWeb.AuthPlugTest do
+  # Not async, because we alter global config (auth mode)
   use LivebookWeb.ConnCase, async: false
 
   setup context do

test/support/app_helpers.ex

@@ -0,0 +1,21 @@
+defmodule Livebook.AppHelpers do
+  def deploy_notebook_sync(notebook) do
+    app_spec = Livebook.Apps.NotebookAppSpec.new(notebook)
+
+    deployer_pid = Livebook.Apps.Deployer.local_deployer()
+    ref = Livebook.Apps.Deployer.deploy_monitor(deployer_pid, app_spec)
+
+    receive do
+      {:deploy_result, ^ref, {:ok, pid}} ->
+        Process.demonitor(ref, [:flush])
+
+        ExUnit.Callbacks.on_exit(fn ->
+          if Process.alive?(pid) do
+            Livebook.App.close(pid)
+          end
+        end)
+
+        pid
+    end
+  end
+end

test/support/notebook_app_spec.ex

@@ -0,0 +1,38 @@
+defmodule Livebook.Apps.NotebookAppSpec do
+  # App spec carrying an in-memory notebook.
+
+  defstruct [:slug, :notebook, :load_failures, :should_warmup, version: "1"]
+
+  def new(notebook, opts \\ []) do
+    opts = Keyword.validate!(opts, load_failures: 0, should_warmup: false)
+
+    slug = notebook.app_settings.slug
+
+    :persistent_term.put({slug, :failures}, 0)
+
+    %__MODULE__{
+      slug: slug,
+      notebook: notebook,
+      load_failures: opts[:load_failures],
+      should_warmup: opts[:should_warmup]
+    }
+  end
+end
+
+defimpl Livebook.Apps.AppSpec, for: Livebook.Apps.NotebookAppSpec do
+  def load(app_spec, _files_tmp_path) do
+    key = {app_spec.slug, :failures}
+    num_failures = :persistent_term.get(key)
+
+    if num_failures < app_spec.load_failures do
+      :persistent_term.put(key, num_failures + 1)
+      {:error, "failed to load"}
+    else
+      {:ok, %{notebook: app_spec.notebook, warnings: []}}
+    end
+  end
+
+  def should_warmup?(app_spec) do
+    app_spec.should_warmup
+  end
+end

test/support/test_helpers.ex

@@ -21,6 +21,17 @@ defmodule Livebook.TestHelpers do
     end
   end
 
+  @doc """
+  Creates the given number of subdirectories and returns their paths.
+  """
+  def create_subdirs!(path, number) do
+    for n <- 1..number//1 do
+      child_path = Path.join(path, "subdir#{n}")
+      File.mkdir_p!(child_path)
+      child_path
+    end
+  end
+
   @doc """
   Applies the given list of operations to `Livebook.Session.Data`.