diff --git a/lib/livebook/config.ex b/lib/livebook/config.ex index 3ff8a7ad1..5972d9ea0 100644 --- a/lib/livebook/config.ex +++ b/lib/livebook/config.ex @@ -45,10 +45,10 @@ defmodule Livebook.Config do @identity_provider_read_only Enum.filter(@identity_providers, & &1.read_only) @doc """ - Returns docker tags to be used when generating sample Dockerfiles. + Returns docker images to be used when generating sample Dockerfiles. """ - @spec docker_tags() :: list(%{tag: String.t(), name: String.t(), env: keyword()}) - def docker_tags do + @spec docker_images() :: list(%{tag: String.t(), name: String.t(), env: keyword()}) + def docker_images() do version = app_version() base = if version =~ "dev", do: "latest", else: version diff --git a/lib/livebook/hubs/dockerfile.ex b/lib/livebook/hubs/dockerfile.ex new file mode 100644 index 000000000..18c544c08 --- /dev/null +++ b/lib/livebook/hubs/dockerfile.ex @@ -0,0 +1,284 @@ +defmodule Livebook.Hubs.Dockerfile do + # This module is responsible for building Dockerfile to deploy apps. + + import Ecto.Changeset + + alias Livebook.Hubs + + @type config :: %{ + deploy_all: boolean(), + docker_tag: String.t(), + zta_provider: atom() | nil, + zta_key: String.t() | nil + } + + @doc """ + Builds a changeset for app Dockerfile configuration. + """ + @spec config_changeset(map()) :: Ecto.Changeset.t() + def config_changeset(attrs \\ %{}) do + default_image = Livebook.Config.docker_images() |> hd() + + data = %{deploy_all: false, docker_tag: default_image.tag, zta_provider: nil, zta_key: nil} + + zta_types = + for provider <- Livebook.Config.identity_providers(), + not provider.read_only, + do: provider.type + + types = %{ + deploy_all: :boolean, + docker_tag: :string, + zta_provider: Ecto.ParameterizedType.init(Ecto.Enum, values: zta_types), + zta_key: :string + } + + cast({data, types}, attrs, [:deploy_all, :docker_tag, :zta_provider, :zta_key]) + |> validate_required([:deploy_all, :docker_tag]) + end + + @doc """ + Builds Dockerfile definition for app deployment. + """ + @spec build_dockerfile( + config(), + Hubs.Provider.t(), + list(Livebook.Secrets.Secret.t()), + list(Livebook.FileSystem.t()), + Livebook.FileSystem.File.t() | nil, + list(Livebook.Notebook.file_entry()), + list(Livebook.Session.Data.secrets()) + ) :: String.t() + def build_dockerfile(config, hub, hub_secrets, hub_file_systems, file, file_entries, secrets) do + base_image = Enum.find(Livebook.Config.docker_images(), &(&1.tag == config.docker_tag)) + + image = """ + FROM ghcr.io/livebook-dev/livebook:#{base_image.tag} + """ + + image_envs = format_envs(base_image.env) + + hub_type = Hubs.Provider.type(hub) + used_secrets = used_secrets(config, hub, secrets, hub_secrets) |> Enum.sort_by(& &1.name) + hub_config = format_hub_config(hub_type, config, hub, hub_file_systems, used_secrets) + + apps_config = """ + # Apps configuration + ENV LIVEBOOK_APPS_PATH "/apps" + ENV LIVEBOOK_APPS_PATH_WARMUP "manual" + ENV LIVEBOOK_APPS_PATH_HUB_ID "#{hub.id}" + """ + + notebook = + if config.deploy_all do + """ + # Notebooks and files + COPY . /apps + """ + else + notebook_file_name = Livebook.FileSystem.File.name(file) + + notebook = + """ + # Notebook + COPY #{notebook_file_name} /apps/ + """ + + attachments = + file_entries + |> Enum.filter(&(&1.type == :attachment)) + |> Enum.sort_by(& &1.name) + + if attachments == [] do + notebook + else + list = Enum.map_join(attachments, " ", &"files/#{&1.name}") + + """ + # Files + COPY #{list} /apps/files/ + + #{notebook}\ + """ + end + end + + apps_warmup = """ + # Cache apps setup at build time + RUN /app/bin/warmup_apps.sh + """ + + [ + image, + image_envs, + hub_config, + apps_config, + notebook, + apps_warmup + ] + |> Enum.reject(&is_nil/1) + |> Enum.join("\n") + end + + defp format_hub_config("team", config, hub, hub_file_systems, used_secrets) do + base_env = + """ + ARG TEAMS_KEY="#{hub.teams_key}" + + # Teams Hub configuration for airgapped deployment + ENV LIVEBOOK_TEAMS_KEY ${TEAMS_KEY} + ENV LIVEBOOK_TEAMS_NAME "#{hub.hub_name}" + ENV LIVEBOOK_TEAMS_OFFLINE_KEY "#{hub.org_public_key}" + """ + + secrets = + if used_secrets != [] do + """ + ENV LIVEBOOK_TEAMS_SECRETS "#{encrypt_secrets_to_dockerfile(used_secrets, hub)}" + """ + end + + file_systems = + if hub_file_systems != [] do + """ + ENV LIVEBOOK_TEAMS_FS "#{encrypt_file_systems_to_dockerfile(hub_file_systems, hub)}" + """ + end + + zta = + if zta_configured?(config) do + """ + ENV LIVEBOOK_IDENTITY_PROVIDER "#{config.zta_provider}:#{config.zta_key}" + """ + end + + [base_env, secrets, file_systems, zta] + |> Enum.reject(&is_nil/1) + |> Enum.join() + end + + defp format_hub_config("personal", _config, _hub, _hub_file_systems, used_secrets) do + if used_secrets != [] do + envs = used_secrets |> Enum.map(&{"LB_" <> &1.name, &1.value}) |> format_envs() + + """ + # Personal Hub secrets + #{envs}\ + """ + end + end + + defp format_envs([]), do: nil + + defp format_envs(list) do + Enum.map_join(list, fn {key, value} -> ~s/ENV #{key} "#{value}"\n/ end) + end + + defp encrypt_secrets_to_dockerfile(secrets, hub) do + secrets_map = + for %{name: name, value: value} <- secrets, + into: %{}, + do: {name, value} + + encrypt_to_dockerfile(hub, secrets_map) + end + + defp encrypt_file_systems_to_dockerfile(file_systems, hub) do + file_systems = + for file_system <- file_systems do + file_system + |> Livebook.FileSystem.dump() + |> Map.put_new(:type, Livebook.FileSystems.type(file_system)) + end + + encrypt_to_dockerfile(hub, file_systems) + end + + defp encrypt_to_dockerfile(hub, data) do + secret_key = Livebook.Teams.derive_key(hub.teams_key) + + data + |> Jason.encode!() + |> Livebook.Teams.encrypt(secret_key) + end + + defp used_secrets(config, hub, secrets, hub_secrets) do + if config.deploy_all do + hub_secrets + else + for {_, secret} <- secrets, secret.hub_id == hub.id, do: secret + end + end + + defp zta_configured?(config) do + config.zta_provider != nil and config.zta_key != nil + end + + @doc """ + Returns a list of Dockerfile-related warnings. + + The returned messages may include HTML. + """ + @spec warnings( + config(), + Hubs.Provider.t(), + list(Livebook.Secrets.Secret.t()), + Livebook.Notebook.AppSettings.t(), + list(Livebook.Notebook.file_entry()), + list(Livebook.Session.Data.secrets()) + ) :: list(String.t()) + def warnings(config, hub, hub_secrets, app_settings, file_entries, secrets) do + common_warnings = + [ + if Livebook.Session.Data.session_secrets(secrets, hub.id) != [] do + "The notebook uses session secrets, but those are not available to deployed apps." <> + " Convert them to Hub secrets instead." + end + ] + + hub_warnings = + case Hubs.Provider.type(hub) do + "personal" -> + [ + if used_secrets(config, hub, secrets, hub_secrets) != [] do + "You are deploying an app with secrets and the secrets are included in the Dockerfile" <> + " as environment variables. If someone else deploys this app, they must also set the" <> + " same secrets. Use Livebook Teams to automatically encrypt and synchronize secrets" <> + " across your team and deployments." + end, + if module = find_hub_file_system(file_entries) do + name = LivebookWeb.FileSystemHelpers.file_system_name(module) + + "The #{name} file storage, defined in your personal hub, will not be available in the Docker image." <> + " You must either download all references as attachments or use Livebook Teams to automatically" <> + " encrypt and synchronize file storages across your team and deployments." + end, + if app_settings.access_type == :public do + teams_link = + ~s{Livebook Teams} + + "This app has no password configuration and anyone with access to the server will be able" <> + " to use it. You may either configure a password or use #{teams_link} to add Zero Trust Authentication" <> + " to your deployed notebooks." + end + ] + + "team" -> + [ + if app_settings.access_type == :public and not zta_configured?(config) do + "This app has no password configuration and anyone with access to the server will be able" <> + " to use it. You may either configure a password or configure Zero Trust Authentication." + end + ] + end + + Enum.reject(common_warnings ++ hub_warnings, &is_nil/1) + end + + defp find_hub_file_system(file_entries) do + Enum.find_value(file_entries, fn entry -> + entry.type == :file && entry.file.file_system_module != FileSystem.Local && + entry.file.file_system_module + end) + end +end diff --git a/lib/livebook_web/components/core_components.ex b/lib/livebook_web/components/core_components.ex index 19e7a0d1e..293e76e4f 100644 --- a/lib/livebook_web/components/core_components.ex +++ b/lib/livebook_web/components/core_components.ex @@ -94,11 +94,17 @@ defmodule LivebookWeb.CoreComponents do <.message_box kind={:info} message="🦊 in a 📦" /> + <.message_box kind={:info}> + 🦊 in a 📦 + + """ - attr :message, :string, required: true + attr :message, :string, default: nil attr :kind, :atom, values: [:info, :success, :warning, :error] + slot :inner_block + def message_box(assigns) do ~H"""
+ You may additionally perform the following optional steps: +
+ +TEAMS_KEY
+ from your Dockerfile and set it as a build argument in your deployment
+ platform
+
+ LIVEBOOK_PASSWORD
environment variable with a
+ value of at least 12 characters of your choice
+
+ @@ -202,129 +196,36 @@ defmodule LivebookWeb.Hub.Edit.TeamComponent do
It is possible to deploy notebooks that belong to this Hub in an airgapped - deployment, without connecting back to Livebook Teams server, by following - the steps below. First, configure your deployment: + deployment, without connecting back to Livebook Teams server. Configure the + deployment below and use the generated Dockerfile in a directory with notebooks + that belong to your Organization.
-
- Then save the Dockerfile below in a repository with the Livebook notebooks
- that belong to your Organization. You must change
- the value of the APPS_PATH
- argument in the template below to point to a directory with all .livemd
- files you want to deploy.
-
- You may additionally perform the following optional steps: -
- -TEAMS_KEY
- from your Dockerfile and set it as a build argument in your deployment
- platform
-
- LIVEBOOK_PASSWORD
environment variable with a
- value of at least 12 characters of your choice
-
-