From 797844223a4caec759506370e2f9300beb851d33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Tue, 17 Oct 2023 15:04:47 +0200 Subject: [PATCH] Add docker deployment instructions to app panel (#2276) --- lib/livebook/config.ex | 6 +- lib/livebook/hubs/dockerfile.ex | 284 +++++++++++++++++ .../components/core_components.ex | 21 +- lib/livebook_web/live/app_helpers.ex | 146 +++++++++ .../live/hub/edit/team_component.ex | 286 +++--------------- lib/livebook_web/live/session_live.ex | 20 ++ .../live/session_live/app_docker_component.ex | 188 ++++++++++++ .../live/session_live/app_info_component.ex | 57 ++-- lib/livebook_web/router.ex | 1 + test/livebook/hubs/dockerfile_test.exs | 281 +++++++++++++++++ test/livebook_web/live/session_live_test.exs | 47 +++ 11 files changed, 1062 insertions(+), 275 deletions(-) create mode 100644 lib/livebook/hubs/dockerfile.ex create mode 100644 lib/livebook_web/live/session_live/app_docker_component.ex create mode 100644 test/livebook/hubs/dockerfile_test.exs diff --git a/lib/livebook/config.ex b/lib/livebook/config.ex index 3ff8a7ad1..5972d9ea0 100644 --- a/lib/livebook/config.ex +++ b/lib/livebook/config.ex @@ -45,10 +45,10 @@ defmodule Livebook.Config do @identity_provider_read_only Enum.filter(@identity_providers, & &1.read_only) @doc """ - Returns docker tags to be used when generating sample Dockerfiles. + Returns docker images to be used when generating sample Dockerfiles. """ - @spec docker_tags() :: list(%{tag: String.t(), name: String.t(), env: keyword()}) - def docker_tags do + @spec docker_images() :: list(%{tag: String.t(), name: String.t(), env: keyword()}) + def docker_images() do version = app_version() base = if version =~ "dev", do: "latest", else: version diff --git a/lib/livebook/hubs/dockerfile.ex b/lib/livebook/hubs/dockerfile.ex new file mode 100644 index 000000000..18c544c08 --- /dev/null +++ b/lib/livebook/hubs/dockerfile.ex @@ -0,0 +1,284 @@ +defmodule Livebook.Hubs.Dockerfile do + # This module is responsible for building Dockerfile to deploy apps. + + import Ecto.Changeset + + alias Livebook.Hubs + + @type config :: %{ + deploy_all: boolean(), + docker_tag: String.t(), + zta_provider: atom() | nil, + zta_key: String.t() | nil + } + + @doc """ + Builds a changeset for app Dockerfile configuration. + """ + @spec config_changeset(map()) :: Ecto.Changeset.t() + def config_changeset(attrs \\ %{}) do + default_image = Livebook.Config.docker_images() |> hd() + + data = %{deploy_all: false, docker_tag: default_image.tag, zta_provider: nil, zta_key: nil} + + zta_types = + for provider <- Livebook.Config.identity_providers(), + not provider.read_only, + do: provider.type + + types = %{ + deploy_all: :boolean, + docker_tag: :string, + zta_provider: Ecto.ParameterizedType.init(Ecto.Enum, values: zta_types), + zta_key: :string + } + + cast({data, types}, attrs, [:deploy_all, :docker_tag, :zta_provider, :zta_key]) + |> validate_required([:deploy_all, :docker_tag]) + end + + @doc """ + Builds Dockerfile definition for app deployment. + """ + @spec build_dockerfile( + config(), + Hubs.Provider.t(), + list(Livebook.Secrets.Secret.t()), + list(Livebook.FileSystem.t()), + Livebook.FileSystem.File.t() | nil, + list(Livebook.Notebook.file_entry()), + list(Livebook.Session.Data.secrets()) + ) :: String.t() + def build_dockerfile(config, hub, hub_secrets, hub_file_systems, file, file_entries, secrets) do + base_image = Enum.find(Livebook.Config.docker_images(), &(&1.tag == config.docker_tag)) + + image = """ + FROM ghcr.io/livebook-dev/livebook:#{base_image.tag} + """ + + image_envs = format_envs(base_image.env) + + hub_type = Hubs.Provider.type(hub) + used_secrets = used_secrets(config, hub, secrets, hub_secrets) |> Enum.sort_by(& &1.name) + hub_config = format_hub_config(hub_type, config, hub, hub_file_systems, used_secrets) + + apps_config = """ + # Apps configuration + ENV LIVEBOOK_APPS_PATH "/apps" + ENV LIVEBOOK_APPS_PATH_WARMUP "manual" + ENV LIVEBOOK_APPS_PATH_HUB_ID "#{hub.id}" + """ + + notebook = + if config.deploy_all do + """ + # Notebooks and files + COPY . /apps + """ + else + notebook_file_name = Livebook.FileSystem.File.name(file) + + notebook = + """ + # Notebook + COPY #{notebook_file_name} /apps/ + """ + + attachments = + file_entries + |> Enum.filter(&(&1.type == :attachment)) + |> Enum.sort_by(& &1.name) + + if attachments == [] do + notebook + else + list = Enum.map_join(attachments, " ", &"files/#{&1.name}") + + """ + # Files + COPY #{list} /apps/files/ + + #{notebook}\ + """ + end + end + + apps_warmup = """ + # Cache apps setup at build time + RUN /app/bin/warmup_apps.sh + """ + + [ + image, + image_envs, + hub_config, + apps_config, + notebook, + apps_warmup + ] + |> Enum.reject(&is_nil/1) + |> Enum.join("\n") + end + + defp format_hub_config("team", config, hub, hub_file_systems, used_secrets) do + base_env = + """ + ARG TEAMS_KEY="#{hub.teams_key}" + + # Teams Hub configuration for airgapped deployment + ENV LIVEBOOK_TEAMS_KEY ${TEAMS_KEY} + ENV LIVEBOOK_TEAMS_NAME "#{hub.hub_name}" + ENV LIVEBOOK_TEAMS_OFFLINE_KEY "#{hub.org_public_key}" + """ + + secrets = + if used_secrets != [] do + """ + ENV LIVEBOOK_TEAMS_SECRETS "#{encrypt_secrets_to_dockerfile(used_secrets, hub)}" + """ + end + + file_systems = + if hub_file_systems != [] do + """ + ENV LIVEBOOK_TEAMS_FS "#{encrypt_file_systems_to_dockerfile(hub_file_systems, hub)}" + """ + end + + zta = + if zta_configured?(config) do + """ + ENV LIVEBOOK_IDENTITY_PROVIDER "#{config.zta_provider}:#{config.zta_key}" + """ + end + + [base_env, secrets, file_systems, zta] + |> Enum.reject(&is_nil/1) + |> Enum.join() + end + + defp format_hub_config("personal", _config, _hub, _hub_file_systems, used_secrets) do + if used_secrets != [] do + envs = used_secrets |> Enum.map(&{"LB_" <> &1.name, &1.value}) |> format_envs() + + """ + # Personal Hub secrets + #{envs}\ + """ + end + end + + defp format_envs([]), do: nil + + defp format_envs(list) do + Enum.map_join(list, fn {key, value} -> ~s/ENV #{key} "#{value}"\n/ end) + end + + defp encrypt_secrets_to_dockerfile(secrets, hub) do + secrets_map = + for %{name: name, value: value} <- secrets, + into: %{}, + do: {name, value} + + encrypt_to_dockerfile(hub, secrets_map) + end + + defp encrypt_file_systems_to_dockerfile(file_systems, hub) do + file_systems = + for file_system <- file_systems do + file_system + |> Livebook.FileSystem.dump() + |> Map.put_new(:type, Livebook.FileSystems.type(file_system)) + end + + encrypt_to_dockerfile(hub, file_systems) + end + + defp encrypt_to_dockerfile(hub, data) do + secret_key = Livebook.Teams.derive_key(hub.teams_key) + + data + |> Jason.encode!() + |> Livebook.Teams.encrypt(secret_key) + end + + defp used_secrets(config, hub, secrets, hub_secrets) do + if config.deploy_all do + hub_secrets + else + for {_, secret} <- secrets, secret.hub_id == hub.id, do: secret + end + end + + defp zta_configured?(config) do + config.zta_provider != nil and config.zta_key != nil + end + + @doc """ + Returns a list of Dockerfile-related warnings. + + The returned messages may include HTML. + """ + @spec warnings( + config(), + Hubs.Provider.t(), + list(Livebook.Secrets.Secret.t()), + Livebook.Notebook.AppSettings.t(), + list(Livebook.Notebook.file_entry()), + list(Livebook.Session.Data.secrets()) + ) :: list(String.t()) + def warnings(config, hub, hub_secrets, app_settings, file_entries, secrets) do + common_warnings = + [ + if Livebook.Session.Data.session_secrets(secrets, hub.id) != [] do + "The notebook uses session secrets, but those are not available to deployed apps." <> + " Convert them to Hub secrets instead." + end + ] + + hub_warnings = + case Hubs.Provider.type(hub) do + "personal" -> + [ + if used_secrets(config, hub, secrets, hub_secrets) != [] do + "You are deploying an app with secrets and the secrets are included in the Dockerfile" <> + " as environment variables. If someone else deploys this app, they must also set the" <> + " same secrets. Use Livebook Teams to automatically encrypt and synchronize secrets" <> + " across your team and deployments." + end, + if module = find_hub_file_system(file_entries) do + name = LivebookWeb.FileSystemHelpers.file_system_name(module) + + "The #{name} file storage, defined in your personal hub, will not be available in the Docker image." <> + " You must either download all references as attachments or use Livebook Teams to automatically" <> + " encrypt and synchronize file storages across your team and deployments." + end, + if app_settings.access_type == :public do + teams_link = + ~s{Livebook Teams} + + "This app has no password configuration and anyone with access to the server will be able" <> + " to use it. You may either configure a password or use #{teams_link} to add Zero Trust Authentication" <> + " to your deployed notebooks." + end + ] + + "team" -> + [ + if app_settings.access_type == :public and not zta_configured?(config) do + "This app has no password configuration and anyone with access to the server will be able" <> + " to use it. You may either configure a password or configure Zero Trust Authentication." + end + ] + end + + Enum.reject(common_warnings ++ hub_warnings, &is_nil/1) + end + + defp find_hub_file_system(file_entries) do + Enum.find_value(file_entries, fn entry -> + entry.type == :file && entry.file.file_system_module != FileSystem.Local && + entry.file.file_system_module + end) + end +end diff --git a/lib/livebook_web/components/core_components.ex b/lib/livebook_web/components/core_components.ex index 19e7a0d1e..293e76e4f 100644 --- a/lib/livebook_web/components/core_components.ex +++ b/lib/livebook_web/components/core_components.ex @@ -94,11 +94,17 @@ defmodule LivebookWeb.CoreComponents do <.message_box kind={:info} message="🦊 in a 📦" /> + <.message_box kind={:info}> + 🦊 in a 📦 + + """ - attr :message, :string, required: true + attr :message, :string, default: nil attr :kind, :atom, values: [:info, :success, :warning, :error] + slot :inner_block + def message_box(assigns) do ~H"""
-
<%= @message %>
+
<%= @message %>
+
+ <%= render_slot(@inner_block) %> +
""" end @@ -478,11 +491,13 @@ defmodule LivebookWeb.CoreComponents do default: false, doc: "whether to force the text into a single scrollable line" + attr :class, :string, default: nil + slot :inner_block, required: true def labeled_text(assigns) do ~H""" -
+
<%= @label %> diff --git a/lib/livebook_web/live/app_helpers.ex b/lib/livebook_web/live/app_helpers.ex index bc9aca7e9..fb83fc078 100644 --- a/lib/livebook_web/live/app_helpers.ex +++ b/lib/livebook_web/live/app_helpers.ex @@ -1,6 +1,8 @@ defmodule LivebookWeb.AppHelpers do use LivebookWeb, :html + alias Livebook.Hubs + @doc """ Renders page placeholder on unauthenticated dead render. """ @@ -79,4 +81,148 @@ defmodule LivebookWeb.AppHelpers do confirm_icon: "delete-bin-6-line" ) end + + @doc """ + Renders form fields for Dockerfile configuration. + """ + attr :form, Phoenix.HTML.Form, required: true + attr :hub, :map, required: true + attr :show_deploy_all, :boolean, default: true + + def docker_config_form_content(assigns) do + ~H""" +
+ <.radio_field + :if={@show_deploy_all} + label="Deploy" + field={@form[:deploy_all]} + options={[ + {"false", "Only this notebook"}, + {"true", "All notebooks in the current directory"} + ]} + /> + <.radio_field label="Base image" field={@form[:docker_tag]} options={docker_tag_options()} /> + <%= if Hubs.Provider.type(@hub) == "team" do %> +
+
+ <.select_field + label="Zero Trust Authentication provider" + field={@form[:zta_provider]} + help="Enable this option if you want to deploy your notebooks behind an authentication proxy" + prompt="None" + options={zta_options()} + /> + <.text_field + :if={zta_metadata = zta_metadata(@form[:zta_provider].value)} + field={@form[:zta_key]} + label={zta_metadata.value} + phx-debounce + /> +
+
+ See the + + Authentication with <%= zta_metadata.name %> docs + + for more information. +
+
+ <% end %> +
+ """ + end + + @zta_options for provider <- Livebook.Config.identity_providers(), + not provider.read_only, + do: {provider.name, provider.type} + + defp zta_options(), do: @zta_options + + defp docker_tag_options() do + for image <- Livebook.Config.docker_images(), do: {image.tag, image.name} + end + + @doc """ + Renders Docker deployment instruction for an app. + """ + attr :hub, :map, required: true + attr :dockerfile, :string, required: true + + slot :dockerfile_actions, default: nil + + def docker_instructions(assigns) do + ~H""" +
+
+
+ Dockerfile +
+ <%= render_slot(@dockerfile_actions) %> + +
+ + <.code_preview source_id="dockerfile-source" source={@dockerfile} language="dockerfile" /> +
+ +
+ To test the deployment locally, go the the notebook directory, save the Dockerfile, then run: +
+ + <.code_preview + source_id="dockerfile-cmd" + source={ + ~s''' + docker build -t my-app . + docker run --rm -p 8080:8080 -p 8081:8081 my-app + ''' + } + language="text" + /> + +

+ You may additionally perform the following optional steps: +

+ +
    +
  • +
    <.remix_icon icon="arrow-right-line" class="text-gray-900" />
    + + you may remove the default value for TEAMS_KEY + from your Dockerfile and set it as a build argument in your deployment + platform + +
    • +
  • +
    <.remix_icon icon="arrow-right-line" class="text-gray-900" />
    + + if you want to debug your deployed notebooks in production, you may + set the LIVEBOOK_PASSWORD environment variable with a + value of at least 12 characters of your choice + +
    • +
+
+ """ + end + + defp zta_metadata(nil), do: nil + + defp zta_metadata(zta_provider) do + Enum.find(Livebook.Config.identity_providers(), &(&1.type == zta_provider)) + end end diff --git a/lib/livebook_web/live/hub/edit/team_component.ex b/lib/livebook_web/live/hub/edit/team_component.ex index a876d41dc..9f5ff83d6 100644 --- a/lib/livebook_web/live/hub/edit/team_component.ex +++ b/lib/livebook_web/live/hub/edit/team_component.ex @@ -12,7 +12,7 @@ defmodule LivebookWeb.Hub.Edit.TeamComponent do socket = assign(socket, assigns) changeset = Team.change_hub(assigns.hub) show_key? = assigns.params["show-key"] == "true" - secrets = Livebook.Hubs.get_secrets(assigns.hub) + secrets = Hubs.get_secrets(assigns.hub) file_systems = Hubs.get_file_systems(assigns.hub, hub_only: true) secret_name = assigns.params["secret_name"] file_system_id = assigns.params["file_system_id"] @@ -30,9 +30,6 @@ defmodule LivebookWeb.Hub.Edit.TeamComponent do raise(NotFoundError, "could not find file system matching #{inspect(file_system_id)}") end - docker_tags = Livebook.Config.docker_tags() - [%{tag: default_base_image} | _] = docker_tags - {:ok, socket |> assign( @@ -44,13 +41,10 @@ defmodule LivebookWeb.Hub.Edit.TeamComponent do secret_name: secret_name, secret_value: secret_value, hub_metadata: Provider.to_metadata(assigns.hub), - is_default: is_default?, - zta: %{"provider" => "", "key" => ""}, - zta_metadata: nil, - base_image: default_base_image, - docker_tags: docker_tags + is_default: is_default? ) - |> assign_dockerfile() + |> assign_new(:config_changeset, fn -> Hubs.Dockerfile.config_changeset() end) + |> update_dockerfile() |> assign_form(changeset)} end @@ -184,7 +178,7 @@ defmodule LivebookWeb.Hub.Edit.TeamComponent do

- File Storages + File storages

@@ -202,129 +196,36 @@ defmodule LivebookWeb.Hub.Edit.TeamComponent do

- Airgapped Deployment + Airgapped deployment

It is possible to deploy notebooks that belong to this Hub in an airgapped - deployment, without connecting back to Livebook Teams server, by following - the steps below. First, configure your deployment: + deployment, without connecting back to Livebook Teams server. Configure the + deployment below and use the generated Dockerfile in a directory with notebooks + that belong to your Organization.

-
-
- <.radio_field - name="base_image" - label="Base image" - value={@base_image} - options={for tag <- @docker_tags, do: {tag.tag, tag.name}} - /> -
-
- - <.form :let={f} class="py-2" for={@zta} phx-change="change_zta" phx-target={@myself}> -
- <.select_field - name="provider" - label="Zero Trust Authentication provider" - value={@zta["provider"]} - help="Enable this option if you want to deploy your notebooks behind an authentication proxy" - prompt="None" - options={zta_options()} - /> - <.text_field - :if={@zta_metadata} - field={f[:key]} - label={@zta_metadata.value} - phx-debounce - /> -
- -
- - See the - - Authentication with <%= @zta_metadata.name %> docs - - for more information. - -
+ <.form + :let={f} + for={@config_changeset} + as={:data} + phx-change="validate_dockerfile" + phx-target={@myself} + > + -

- Then save the Dockerfile below in a repository with the Livebook notebooks - that belong to your Organization. You must change - the value of the APPS_PATH - argument in the template below to point to a directory with all .livemd - files you want to deploy. -

- -
-
- Dockerfile - -
- - <.code_preview - source_id={"offline-deployment-#{@hub.id}-source"} - source={@dockerfile} - language="dockerfile" - /> -
- -

- You may additionally perform the following optional steps: -

- -
    -
  • -
    <.remix_icon icon="arrow-right-line" class="text-gray-900" />
    - - you may remove the default value for TEAMS_KEY - from your Dockerfile and set it as a build argument in your deployment - platform - -
    • -
  • -
    <.remix_icon icon="arrow-right-line" class="text-gray-900" />
    - - if you want to debug your deployed notebooks in production, you may - set the LIVEBOOK_PASSWORD environment variable with a - value of at least 12 characters of your choice - -
    • -
+

- Danger Zone + Danger zone

@@ -420,23 +321,12 @@ defmodule LivebookWeb.Hub.Edit.TeamComponent do
+ + +
+ """ + end + + defp save_result(%{save_result: {:ok, file}}) do + assigns = %{path: file.path} + + ~H""" + <.message_box kind={:info} message={"File saved at #{@path}"} /> + """ + end + + defp save_result(%{save_result: {:error, message}}) do + assigns = %{message: message} + + ~H""" + <.message_box kind={:error} message={@message} /> + """ + end + + @impl true + def handle_event("validate", %{"data" => data}, socket) do + changeset = + data + |> Hubs.Dockerfile.config_changeset() + |> Map.replace!(:action, :validate) + + {:noreply, assign(socket, changeset: changeset) |> update_dockerfile()} + end + + def handle_event("save_dockerfile", %{}, socket) do + dockerfile_file = FileSystem.File.resolve(socket.assigns.file, "./Dockerfile") + + save_result = + case FileSystem.File.write(dockerfile_file, socket.assigns.dockerfile) do + :ok -> {:ok, dockerfile_file} + {:error, message} -> {:error, message} + end + + {:noreply, assign(socket, save_result: save_result)} + end + + defp update_dockerfile(socket) when socket.assigns.file == nil do + assign(socket, dockerfile: nil, warnings: []) + end + + defp update_dockerfile(socket) do + config = apply_changes(socket.assigns.changeset) + + %{ + hub: hub, + hub_secrets: hub_secrets, + hub_file_systems: hub_file_systems, + file: file, + file_entries: file_entries, + secrets: secrets, + app_settings: app_settings + } = socket.assigns + + dockerfile = + Hubs.Dockerfile.build_dockerfile( + config, + hub, + hub_secrets, + hub_file_systems, + file, + file_entries, + secrets + ) + + warnings = + Hubs.Dockerfile.warnings(config, hub, hub_secrets, app_settings, file_entries, secrets) + + assign(socket, dockerfile: dockerfile, warnings: warnings) + end +end diff --git a/lib/livebook_web/live/session_live/app_info_component.ex b/lib/livebook_web/live/session_live/app_info_component.ex index 2e0d152f8..7e44726bd 100644 --- a/lib/livebook_web/live/session_live/app_info_component.ex +++ b/lib/livebook_web/live/session_live/app_info_component.ex @@ -33,20 +33,29 @@ defmodule LivebookWeb.SessionLive.AppInfoComponent do kind={:warning} message="The notebook uses session secrets, but those are not available to deployed apps. Convert them to Hub secrets instead." /> -
- +
+
+ + <.link + patch={~p"/sessions/#{@session.id}/settings/app"} + class="button-base button-outlined-gray bg-transparent" + > + Configure + +
<.link - patch={~p"/sessions/#{@session.id}/settings/app"} - class="button-base button-outlined-gray bg-transparent" + class="text-sm text-gray-700 hover:text-blue-600" + patch={~p"/sessions/#{@session.id}/app-docker"} > - Configure + <.remix_icon icon="arrow-right-line" /> + Deploy with Docker
@@ -55,18 +64,20 @@ defmodule LivebookWeb.SessionLive.AppInfoComponent do Latest deployment
-
+
<.labeled_text label="URL" one_line> <%= ~p"/apps/#{@app.slug}" %> - <.labeled_text label="Version" one_line> - v<%= @app.version %> - - <.labeled_text label="Session type" one_line> - <%= if(@app.multi_session, do: "Multi", else: "Single") %> - +
+ <.labeled_text label="Session type" one_line class="grow"> + <%= if(@app.multi_session, do: "Multi", else: "Single") %> + + <.labeled_text label="Version" one_line class="grow"> + v<%= @app.version %> + +
@@ -85,9 +96,9 @@ defmodule LivebookWeb.SessionLive.AppInfoComponent do Running sessions
-
-
- <.labeled_text label="Status"> +
+
+ <.labeled_text label="Status" class="grow"> - <.labeled_text label="Version"> + <.labeled_text label="Version" class="grow"> v<%= app_session.version %>
diff --git a/lib/livebook_web/router.ex b/lib/livebook_web/router.ex index f5c077581..14e6f95e4 100644 --- a/lib/livebook_web/router.ex +++ b/lib/livebook_web/router.ex @@ -88,6 +88,7 @@ defmodule LivebookWeb.Router do live "/sessions/:id/settings/runtime", SessionLive, :runtime_settings live "/sessions/:id/settings/file", SessionLive, :file_settings live "/sessions/:id/settings/app", SessionLive, :app_settings + live "/sessions/:id/app-docker", SessionLive, :app_docker live "/sessions/:id/add-file/:tab", SessionLive, :add_file_entry live "/sessions/:id/rename-file/:name", SessionLive, :rename_file_entry live "/sessions/:id/bin", SessionLive, :bin diff --git a/test/livebook/hubs/dockerfile_test.exs b/test/livebook/hubs/dockerfile_test.exs new file mode 100644 index 000000000..5fae62a6a --- /dev/null +++ b/test/livebook/hubs/dockerfile_test.exs @@ -0,0 +1,281 @@ +defmodule Livebook.Hubs.DockerfileTest do + use ExUnit.Case, async: true + + import Livebook.TestHelpers + + alias Livebook.Hubs.Dockerfile + alias Livebook.Hubs + alias Livebook.Secrets.Secret + + describe "build_dockerfile/7" do + test "deploying a single notebook in personal hub" do + config = dockerfile_config() + hub = personal_hub() + file = Livebook.FileSystem.File.local(p("/notebook.livemd")) + + dockerfile = Dockerfile.build_dockerfile(config, hub, [], [], file, [], %{}) + + assert dockerfile == """ + FROM ghcr.io/livebook-dev/livebook:latest + + # Apps configuration + ENV LIVEBOOK_APPS_PATH "/apps" + ENV LIVEBOOK_APPS_PATH_WARMUP "manual" + ENV LIVEBOOK_APPS_PATH_HUB_ID "personal-hub" + + # Notebook + COPY notebook.livemd /apps/ + + # Cache apps setup at build time + RUN /app/bin/warmup_apps.sh + """ + + # With secrets + + secret = %Secret{name: "TEST", value: "test", hub_id: hub.id} + unused_secret = %Secret{name: "TEST2", value: "test", hub_id: hub.id} + session_secret = %Secret{name: "SESSION", value: "test", hub_id: nil} + + hub_secrets = [secret, unused_secret] + secrets = %{"TEST" => secret, "SESSION" => session_secret} + + dockerfile = Dockerfile.build_dockerfile(config, hub, hub_secrets, [], file, [], secrets) + + assert dockerfile =~ + """ + # Personal Hub secrets + ENV LB_TEST "test" + """ + + refute dockerfile =~ "ENV LB_SESSION" + end + + test "deploying a directory in personal hub" do + config = dockerfile_config(%{deploy_all: true}) + hub = personal_hub() + file = Livebook.FileSystem.File.local(p("/notebook.livemd")) + + dockerfile = Dockerfile.build_dockerfile(config, hub, [], [], file, [], %{}) + + assert dockerfile =~ """ + # Notebooks and files + COPY . /apps + """ + + # With secrets + + secret = %Secret{name: "TEST", value: "test", hub_id: hub.id} + unused_secret = %Secret{name: "TEST2", value: "test", hub_id: hub.id} + session_secret = %Secret{name: "SESSION", value: "test", hub_id: nil} + + hub_secrets = [secret, unused_secret] + secrets = %{"TEST" => secret, "SESSION" => session_secret} + + dockerfile = Dockerfile.build_dockerfile(config, hub, hub_secrets, [], file, [], secrets) + + assert dockerfile =~ + """ + # Personal Hub secrets + ENV LB_TEST "test" + ENV LB_TEST2 "test" + """ + + refute dockerfile =~ "ENV LB_SESSION" + end + + test "deploying a single notebook in teams hub" do + config = dockerfile_config() + hub = team_hub() + file = Livebook.FileSystem.File.local(p("/notebook.livemd")) + + dockerfile = Dockerfile.build_dockerfile(config, hub, [], [], file, [], %{}) + + assert dockerfile == """ + FROM ghcr.io/livebook-dev/livebook:latest + + ARG TEAMS_KEY="lb_tk_fn0pL3YLWzPoPFWuHeV3kd0o7_SFuIOoU4C_k6OWDYg" + + # Teams Hub configuration for airgapped deployment + ENV LIVEBOOK_TEAMS_KEY ${TEAMS_KEY} + ENV LIVEBOOK_TEAMS_NAME "org-name-387" + ENV LIVEBOOK_TEAMS_OFFLINE_KEY "lb_opk_fpxnp3r5djwxnmirx3tu276hialoivf3" + + # Apps configuration + ENV LIVEBOOK_APPS_PATH "/apps" + ENV LIVEBOOK_APPS_PATH_WARMUP "manual" + ENV LIVEBOOK_APPS_PATH_HUB_ID "team-org-name-387" + + # Notebook + COPY notebook.livemd /apps/ + + # Cache apps setup at build time + RUN /app/bin/warmup_apps.sh + """ + + # With secrets + + secret = %Secret{name: "TEST", value: "test", hub_id: hub.id} + session_secret = %Secret{name: "SESSION", value: "test", hub_id: nil} + + hub_secrets = [secret] + secrets = %{"TEST" => secret, "SESSION" => session_secret} + + dockerfile = Dockerfile.build_dockerfile(config, hub, hub_secrets, [], file, [], secrets) + + assert dockerfile =~ "ENV LIVEBOOK_TEAMS_SECRETS" + refute dockerfile =~ "ENV TEST" + refute dockerfile =~ "ENV LB_SESSION" + + # With file systems + + file_system = Livebook.Factory.build(:fs_s3) + file_systems = [file_system] + + dockerfile = Dockerfile.build_dockerfile(config, hub, [], file_systems, file, [], %{}) + + assert dockerfile =~ "ENV LIVEBOOK_TEAMS_FS" + end + + test "deploying with ZTA in teams hub" do + config = dockerfile_config(%{zta_provider: :cloudflare, zta_key: "cloudflare_key"}) + hub = team_hub() + file = Livebook.FileSystem.File.local(p("/notebook.livemd")) + + dockerfile = Dockerfile.build_dockerfile(config, hub, [], [], file, [], %{}) + + assert dockerfile =~ ~S/ENV LIVEBOOK_IDENTITY_PROVIDER "cloudflare:cloudflare_key"/ + end + + test "deploying a directory in teams hub" do + config = dockerfile_config(%{deploy_all: true}) + hub = team_hub() + file = Livebook.FileSystem.File.local(p("/notebook.livemd")) + + dockerfile = Dockerfile.build_dockerfile(config, hub, [], [], file, [], %{}) + + assert dockerfile =~ """ + # Notebooks and files + COPY . /apps + """ + end + + test "deploying with different base image" do + config = dockerfile_config(%{docker_tag: "latest-cuda11.8"}) + hub = personal_hub() + file = Livebook.FileSystem.File.local(p("/notebook.livemd")) + + dockerfile = Dockerfile.build_dockerfile(config, hub, [], [], file, [], %{}) + + assert dockerfile =~ """ + FROM ghcr.io/livebook-dev/livebook:latest-cuda11.8 + + ENV XLA_TARGET "cuda118" + """ + end + + test "deploying with file entries" do + config = dockerfile_config() + hub = personal_hub() + file = Livebook.FileSystem.File.local(p("/notebook.livemd")) + + file_entries = [ + %{type: :attachment, name: "image.jpeg"}, + %{type: :attachment, name: "data.csv"} + ] + + dockerfile = Dockerfile.build_dockerfile(config, hub, [], [], file, file_entries, %{}) + + assert dockerfile =~ + """ + # Files + COPY files/data.csv files/image.jpeg /apps/files/ + """ + end + end + + describe "warnings/6" do + test "warns when session secrets are used" do + config = dockerfile_config() + hub = personal_hub() + app_settings = Livebook.Notebook.AppSettings.new() + + session_secret = %Secret{name: "SESSION", value: "test", hub_id: nil} + secrets = %{"SESSION" => session_secret} + + assert [warning] = Dockerfile.warnings(config, hub, [], app_settings, [], secrets) + assert warning =~ "The notebook uses session secrets" + end + + test "warns when hub secrets are used from personal hub" do + config = dockerfile_config() + hub = personal_hub() + app_settings = Livebook.Notebook.AppSettings.new() + + secret = %Secret{name: "TEST", value: "test", hub_id: hub.id} + + hub_secrets = [secret] + secrets = %{"TEST" => secret} + + assert [warning] = Dockerfile.warnings(config, hub, hub_secrets, app_settings, [], secrets) + assert warning =~ "secrets are included in the Dockerfile" + end + + test "warns when there is a reference to external file system from personal hub" do + config = dockerfile_config() + hub = personal_hub() + app_settings = Livebook.Notebook.AppSettings.new() + + file_system = Livebook.Factory.build(:fs_s3) + + file_entries = [ + %{type: :file, file: Livebook.FileSystem.File.new(file_system, "/data.csv")} + ] + + assert [warning] = Dockerfile.warnings(config, hub, [], app_settings, file_entries, %{}) + + assert warning =~ + "The S3 file storage, defined in your personal hub, will not be available in the Docker image" + end + + test "warns when the app has no password in personal hub" do + config = dockerfile_config() + hub = personal_hub() + app_settings = %{Livebook.Notebook.AppSettings.new() | access_type: :public} + + assert [warning] = Dockerfile.warnings(config, hub, [], app_settings, [], %{}) + assert warning =~ "This app has no password configuration" + end + + test "warns when the app has no password and no ZTA in teams hub" do + config = dockerfile_config() + hub = team_hub() + app_settings = %{Livebook.Notebook.AppSettings.new() | access_type: :public} + + assert [warning] = Dockerfile.warnings(config, hub, [], app_settings, [], %{}) + assert warning =~ "This app has no password configuration" + + config = %{config | zta_provider: :cloudflare, zta_key: "key"} + + assert [] = Dockerfile.warnings(config, hub, [], app_settings, [], %{}) + end + end + + defp dockerfile_config(attrs \\ %{}) do + attrs + |> Dockerfile.config_changeset() + |> Ecto.Changeset.apply_changes() + end + + defp personal_hub() do + Hubs.fetch_hub!(Hubs.Personal.id()) + end + + defp team_hub() do + Livebook.Factory.build(:team, + id: "team-org-name-387", + hub_name: "org-name-387", + teams_key: "lb_tk_fn0pL3YLWzPoPFWuHeV3kd0o7_SFuIOoU4C_k6OWDYg", + org_public_key: "lb_opk_fpxnp3r5djwxnmirx3tu276hialoivf3" + ) + end +end diff --git a/test/livebook_web/live/session_live_test.exs b/test/livebook_web/live/session_live_test.exs index 57db84c77..eadbf3f8d 100644 --- a/test/livebook_web/live/session_live_test.exs +++ b/test/livebook_web/live/session_live_test.exs @@ -2169,4 +2169,51 @@ defmodule LivebookWeb.SessionLiveTest do "The notebook uses session secrets, but those are not available to deployed apps. Convert them to Hub secrets instead." end end + + describe "docker deployment" do + test "instructs to choose a file when the notebook is not persisted", + %{conn: conn, session: session} do + {:ok, view, _} = live(conn, ~p"/sessions/#{session.id}/app-docker") + + assert render(view) =~ "To deploy this app, make sure to save the notebook first." + assert render(view) =~ ~p"/sessions/#{session.id}/settings/file" + end + + @tag :tmp_dir + test "instructs to change app settings when invalid", + %{conn: conn, session: session, tmp_dir: tmp_dir} do + notebook_path = Path.join(tmp_dir, "notebook.livemd") + file = Livebook.FileSystem.File.local(notebook_path) + Session.set_file(session.pid, file) + + {:ok, view, _} = live(conn, ~p"/sessions/#{session.id}/app-docker") + + assert render(view) =~ "To deploy this app, make sure to specify valid settings." + assert render(view) =~ ~p"/sessions/#{session.id}/settings/app" + end + + @tag :tmp_dir + test "shows dockerfile and allows saving it", + %{conn: conn, session: session, tmp_dir: tmp_dir} do + notebook_path = Path.join(tmp_dir, "notebook.livemd") + file = Livebook.FileSystem.File.local(notebook_path) + Session.set_file(session.pid, file) + + slug = Livebook.Utils.random_short_id() + app_settings = %{Livebook.Notebook.AppSettings.new() | slug: slug} + Session.set_app_settings(session.pid, app_settings) + + {:ok, view, _} = live(conn, ~p"/sessions/#{session.id}/app-docker") + + assert render(view) =~ "FROM ghcr.io/livebook-dev/livebook:" + + view + |> element("button", "Save alongside notebook") + |> render_click() + + dockerfile_path = Path.join(tmp_dir, "Dockerfile") + + assert File.read!(dockerfile_path) =~ "COPY notebook.livemd /apps" + end + end end