List all environment variables from deployment group in dockerfile (#2858)

lib/livebook/hubs/dockerfile.ex

@@ -9,14 +9,13 @@ defmodule Livebook.Hubs.Dockerfile do
           deploy_all: boolean(),
           docker_tag: String.t(),
           clustering: nil | :auto | :dns,
-          zta_provider: atom() | nil
+          environment_variables: list({String.t(), String.t()})
         }
 
   @types %{
     deploy_all: :boolean,
     docker_tag: :string,
-    clustering: Ecto.ParameterizedType.init(Ecto.Enum, values: [:auto, :dns]),
-    zta_provider: :atom
+    clustering: Ecto.ParameterizedType.init(Ecto.Enum, values: [:auto, :dns])
   }
 
   @doc """
@@ -30,7 +29,7 @@ defmodule Livebook.Hubs.Dockerfile do
       deploy_all: false,
       docker_tag: default_image.tag,
       clustering: nil,
-      zta_provider: nil
+      environment_variables: []
     }
   end
 
@@ -39,10 +38,14 @@ defmodule Livebook.Hubs.Dockerfile do
   """
   @spec from_deployment_group(Livebook.Teams.DeploymentGroup.t()) :: config()
   def from_deployment_group(deployment_group) do
+    environment_variables =
+      for environment_variable <- deployment_group.environment_variables,
+          do: {environment_variable.name, environment_variable.value}
+
     %{
       config_new()
       | clustering: deployment_group.clustering,
-        zta_provider: deployment_group.zta_provider
+        environment_variables: environment_variables
     }
   end
 
@@ -52,7 +55,7 @@ defmodule Livebook.Hubs.Dockerfile do
   @spec config_changeset(config(), map()) :: Ecto.Changeset.t()
   def config_changeset(config, attrs \\ %{}) do
     {config, @types}
-    |> cast(attrs, [:deploy_all, :docker_tag, :clustering, :zta_provider])
+    |> cast(attrs, [:deploy_all, :docker_tag, :clustering])
     |> validate_required([:deploy_all, :docker_tag])
   end
 
@@ -169,6 +172,16 @@ defmodule Livebook.Hubs.Dockerfile do
           nil
       end
 
+    environment_variables =
+      if config.environment_variables != [] do
+        envs = config.environment_variables |> Enum.sort() |> format_envs()
+
+        """
+        # Deployment group environment variables
+        #{envs}\
+        """
+      end
+
     [
       image,
       image_envs,
@@ -176,7 +189,8 @@ defmodule Livebook.Hubs.Dockerfile do
       apps_config,
       notebook,
       apps_warmup,
-      startup
+      startup,
+      environment_variables
     ]
     |> Enum.reject(&is_nil/1)
     |> Enum.join("\n")
@@ -336,7 +350,9 @@ defmodule Livebook.Hubs.Dockerfile do
           []
       end
 
-    %{image: image, env: base_image.env ++ env ++ clustering_env}
+    deployment_group_env = Enum.sort(config.environment_variables)
+
+    %{image: image, env: base_image.env ++ env ++ clustering_env ++ deployment_group_env}
   end
 
   @doc """
@@ -402,9 +418,9 @@ defmodule Livebook.Hubs.Dockerfile do
 
         "team" ->
           [
-            if app_settings.access_type == :public and config.zta_provider != :livebook_teams do
+            if app_settings.access_type == :public do
               "This app has no password configuration and anyone with access to the server will be able" <>
-                " to use it. You may either configure a password or enable authentication with Livebook Teams."
+                " to use it. You may either configure a password or configure an Identity Provider."
             end
           ]
       end

lib/livebook/hubs/team_client.ex

@@ -134,6 +134,14 @@ defmodule Livebook.Hubs.TeamClient do
     GenServer.call(registry_name(id), :identity_enabled?)
   end
 
+  @doc """
+  Returns a list of cached environment variables.
+  """
+  @spec get_environment_variables(String.t()) :: list(Teams.Agent.t())
+  def get_environment_variables(id) do
+    GenServer.call(registry_name(id), :get_environment_variables)
+  end
+
   @doc """
   Returns if the Team client is connected.
   """
@@ -256,6 +264,11 @@ defmodule Livebook.Hubs.TeamClient do
     {:reply, state.agents, state}
   end
 
+  def handle_call(:get_environment_variables, _caller, state) do
+    environment_variables = Enum.flat_map(state.deployment_groups, & &1.environment_variables)
+    {:reply, environment_variables, state}
+  end
+
   def handle_call(:identity_enabled?, _caller, %{deployment_group_id: nil} = state) do
     {:reply, false, state}
   end
@@ -426,6 +439,7 @@ defmodule Livebook.Hubs.TeamClient do
   defp build_deployment_group(state, %LivebookProto.DeploymentGroup{} = deployment_group) do
     secrets = Enum.map(deployment_group.secrets, &build_secret(state, &1))
     agent_keys = Enum.map(deployment_group.agent_keys, &build_agent_key/1)
+    environment_variables = build_environment_variables(state, deployment_group)
 
     %Teams.DeploymentGroup{
       id: deployment_group.id,
@@ -434,6 +448,7 @@ defmodule Livebook.Hubs.TeamClient do
       hub_id: state.hub.id,
       secrets: secrets,
       agent_keys: agent_keys,
+      environment_variables: environment_variables,
       clustering: nullify(deployment_group.clustering),
       zta_provider: atomize(deployment_group.zta_provider),
       url: nullify(deployment_group.url)
@@ -450,6 +465,7 @@ defmodule Livebook.Hubs.TeamClient do
       hub_id: state.hub.id,
       secrets: [],
       agent_keys: agent_keys,
+      environment_variables: [],
       clustering: nullify(deployment_group_created.clustering),
       zta_provider: atomize(deployment_group_created.zta_provider),
       url: nullify(deployment_group_created.url)
@@ -459,6 +475,8 @@ defmodule Livebook.Hubs.TeamClient do
   defp build_deployment_group(state, deployment_group_updated) do
     secrets = Enum.map(deployment_group_updated.secrets, &build_secret(state, &1))
     agent_keys = Enum.map(deployment_group_updated.agent_keys, &build_agent_key/1)
+    environment_variables = build_environment_variables(state, deployment_group_updated)
+
     {:ok, deployment_group} = fetch_deployment_group(deployment_group_updated.id, state)
 
     %{
@@ -466,6 +484,7 @@ defmodule Livebook.Hubs.TeamClient do
       | name: deployment_group_updated.name,
         secrets: secrets,
         agent_keys: agent_keys,
+        environment_variables: environment_variables,
         clustering: atomize(deployment_group_updated.clustering),
         zta_provider: atomize(deployment_group_updated.zta_provider),
         url: nullify(deployment_group_updated.url)
@@ -489,6 +508,17 @@ defmodule Livebook.Hubs.TeamClient do
     }
   end
 
+  defp build_environment_variables(state, deployment_group_updated) do
+    for environment_variable <- deployment_group_updated.environment_variables do
+      %Teams.EnvironmentVariable{
+        name: environment_variable.name,
+        value: environment_variable.value,
+        hub_id: state.hub.id,
+        deployment_group_id: deployment_group_updated.id
+      }
+    end
+  end
+
   defp put_agent(state, agent) do
     state = remove_agent(state, agent)
 

lib/livebook/teams.ex

@@ -233,6 +233,14 @@ defmodule Livebook.Teams do
     TeamClient.get_agents(team.id)
   end
 
+  @doc """
+  Gets a list of environment variables for a given Hub.
+  """
+  @spec get_environment_variables(Team.t()) :: list(Agent.t())
+  def get_environment_variables(team) do
+    TeamClient.get_environment_variables(team.id)
+  end
+
   defp map_teams_field_to_livebook_field(map, teams_field, livebook_field) do
     if value = map[teams_field] do
       Map.put_new(map, livebook_field, value)

lib/livebook/teams/deployment_group.ex

@@ -3,7 +3,7 @@ defmodule Livebook.Teams.DeploymentGroup do
   import Ecto.Changeset
 
   alias Livebook.Secrets.Secret
-  alias Livebook.Teams.AgentKey
+  alias Livebook.Teams.{AgentKey, EnvironmentVariable}
 
   @type t :: %__MODULE__{
           id: String.t() | nil,
@@ -14,6 +14,7 @@ defmodule Livebook.Teams.DeploymentGroup do
           hub_id: String.t() | nil,
           secrets: Ecto.Schema.has_many(Secret.t()),
           agent_keys: Ecto.Schema.has_many(AgentKey.t()),
+          environment_variables: Ecto.Schema.has_many(EnvironmentVariable.t()),
           zta_provider:
             :basic_auth
             | :cloudflare
@@ -37,6 +38,7 @@ defmodule Livebook.Teams.DeploymentGroup do
 
     has_many :secrets, Secret
     has_many :agent_keys, AgentKey
+    has_many :environment_variables, EnvironmentVariable
   end
 
   def changeset(deployment_group, attrs \\ %{}) do

lib/livebook/teams/environment_variable.ex

@@ -0,0 +1,20 @@
+defmodule Livebook.Teams.EnvironmentVariable do
+  use Ecto.Schema
+
+  @type t :: %__MODULE__{
+          name: String.t(),
+          value: String.t(),
+          hub_id: String.t(),
+          deployment_group_id: String.t()
+        }
+
+  @enforce_keys [:name, :value, :hub_id, :deployment_group_id]
+
+  @primary_key false
+  embedded_schema do
+    field :name, :string
+    field :value, :string
+    field :hub_id, :string
+    field :deployment_group_id, :string
+  end
+end

lib/livebook_web/live/hub/edit/team_component.ex

@@ -17,6 +17,7 @@ defmodule LivebookWeb.Hub.Edit.TeamComponent do
     deployment_groups = Teams.get_deployment_groups(assigns.hub)
     app_deployments = Teams.get_app_deployments(assigns.hub)
     agents = Teams.get_agents(assigns.hub)
+    environment_variables = Teams.get_environment_variables(assigns.hub)
     secret_name = assigns.params["secret_name"]
     file_system_id = assigns.params["file_system_id"]
     default? = default_hub?(assigns.hub)
@@ -43,6 +44,8 @@ defmodule LivebookWeb.Hub.Edit.TeamComponent do
        deployment_groups: Enum.sort_by(deployment_groups, & &1.name),
        app_deployments: Enum.frequencies_by(app_deployments, & &1.deployment_group_id),
        agents: Enum.frequencies_by(agents, & &1.deployment_group_id),
+       environment_variables:
+         Enum.frequencies_by(environment_variables, & &1.deployment_group_id),
        show_key: show_key,
        secret_name: secret_name,
        secret_value: secret_value,
@@ -220,6 +223,9 @@ defmodule LivebookWeb.Hub.Edit.TeamComponent do
                   hub={@hub}
                   deployment_group={deployment_group}
                   app_deployments_count={Map.get(@app_deployments, deployment_group.id, 0)}
+                  environment_variables_count={
+                    Map.get(@environment_variables, deployment_group.id, 0)
+                  }
                   agents_count={Map.get(@agents, deployment_group.id, 0)}
                   live_action={@live_action}
                   params={@params}

lib/livebook_web/live/hub/teams/deployment_group_component.ex

@@ -51,7 +51,7 @@ defmodule LivebookWeb.Hub.Teams.DeploymentGroupComponent do
             </.link>
           </div>
           <.link
-            href={Livebook.Config.teams_url() <> "/orgs/#{@hub.org_id}/deployments/groups/#{@deployment_group.id}"}
+            href={org_url(@hub, "/deployments/groups/#{@deployment_group.id}")}
             class="text-sm font-medium text-blue-600"
             target="_blank"
           >
@@ -82,10 +82,22 @@ defmodule LivebookWeb.Hub.Teams.DeploymentGroupComponent do
               + Add new
             </.link>
           </.labeled_text>
-          <.labeled_text class="grow mt-6 lg:border-l border-gray-200 lg:pl-4" label="Authentication">
-            <span class="text-lg font-normal">
-              <%= provider_name(@deployment_group.zta_provider) %>
+          <.labeled_text
+            class="grow mt-6 lg:border-l border-gray-200 lg:pl-4"
+            label="Environment variables"
+          >
+            <span class="text-lg font-normal" aria-label="environment variables">
+              <%= @environment_variables_count %>
             </span>
+            <.link
+              href={
+                org_url(@hub, "/deployments/groups/#{@deployment_group.id}/environment-variables")
+              }
+              target="_blank"
+              class="pl-2 text-blue-600 font-medium"
+            >
+              + Manage
+            </.link>
           </.labeled_text>
         </div>
         <!-- Additional Secrets -->
@@ -188,6 +200,7 @@ defmodule LivebookWeb.Hub.Teams.DeploymentGroupComponent do
     """
   end
 
-  defp provider_name(:livebook_teams), do: "Livebook Teams"
-  defp provider_name(_), do: "None"
+  defp org_url(hub, path) do
+    Livebook.Config.teams_url() <> "/orgs/#{hub.org_id}" <> path
+  end
 end

test/livebook/hubs/dockerfile_test.exs

@@ -204,6 +204,27 @@ defmodule Livebook.Hubs.DockerfileTest do
       assert dockerfile =~ ~s/ENV LIVEBOOK_NODE "livebook_server@MACHINE_IP"/
       assert dockerfile =~ ~s/ENV LIVEBOOK_CLUSTER "dns:QUERY"/
     end
+
+    test "deploying with deployment group environment variables" do
+      config = %{
+        dockerfile_config()
+        | environment_variables: [
+            {"LIVEBOOK_IDENTITY_PROVIDER", "cloudflare:foobar"},
+            {"LIVEBOOK_TEAMS_URL", "http://localhost:8000"}
+          ]
+      }
+
+      hub = team_hub()
+      file = Livebook.FileSystem.File.local(p("/notebook.livemd"))
+
+      dockerfile = Dockerfile.airgapped_dockerfile(config, hub, [], [], file, [], %{})
+
+      assert dockerfile =~ """
+             # Deployment group environment variables
+             ENV LIVEBOOK_IDENTITY_PROVIDER "cloudflare:foobar"
+             ENV LIVEBOOK_TEAMS_URL "http://localhost:8000"\
+             """
+    end
   end
 
   describe "online_docker_info/3" do
@@ -252,6 +273,24 @@ defmodule Livebook.Hubs.DockerfileTest do
       assert {"LIVEBOOK_NODE", "livebook_server@MACHINE_IP"} in env
       assert {"LIVEBOOK_CLUSTER", "dns:QUERY"} in env
     end
+
+    test "deploying with deployment group environment variables" do
+      config = %{
+        dockerfile_config()
+        | environment_variables: %{
+            "LIVEBOOK_IDENTITY_PROVIDER" => "cloudflare:foobar",
+            "LIVEBOOK_TEAMS_URL" => "http://localhost:8000"
+          }
+      }
+
+      hub = team_hub()
+      agent_key = Livebook.Factory.build(:agent_key)
+
+      %{env: env} = Dockerfile.online_docker_info(config, hub, agent_key)
+
+      assert {"LIVEBOOK_IDENTITY_PROVIDER", "cloudflare:foobar"} in env
+      assert {"LIVEBOOK_TEAMS_URL", "http://localhost:8000"} in env
+    end
   end
 
   describe "warnings/6" do
@@ -344,19 +383,6 @@ defmodule Livebook.Hubs.DockerfileTest do
       assert warning =~ "This app has no password configuration"
     end
 
-    test "warns when the app has no password and no ZTA in teams hub" do
-      config = dockerfile_config(%{clustering: :auto})
-      hub = team_hub()
-      app_settings = %{Livebook.Notebook.AppSettings.new() | access_type: :public}
-
-      assert [warning] = Dockerfile.airgapped_warnings(config, hub, [], [], app_settings, [], %{})
-      assert warning =~ "This app has no password configuration"
-
-      config = %{config | zta_provider: :livebook_teams}
-
-      assert [] = Dockerfile.airgapped_warnings(config, hub, [], [], app_settings, [], %{})
-    end
-
     test "warns when no clustering is configured" do
       config = dockerfile_config()
       hub = team_hub()

test/livebook_teams/hubs/team_client_test.exs

@@ -473,7 +473,8 @@ defmodule Livebook.Hubs.TeamClientTest do
           mode: to_string(deployment_group.mode),
           zta_provider: to_string(deployment_group.zta_provider),
           agent_keys: [livebook_proto_agent_key],
-          secrets: []
+          secrets: [],
+          environment_variables: []
         }
 
       # creates the deployment group

test/livebook_teams/web/hub/deployment_group_test.exs

@@ -284,4 +284,31 @@ defmodule LivebookWeb.Integration.Hub.DeploymentGroupTest do
            |> Floki.text()
            |> String.trim() == "1"
   end
+
+  test "shows the environment variables count", %{conn: conn, node: node, hub: hub} do
+    %{id: id} = insert_deployment_group(mode: :online, hub_id: hub.id)
+    deployment_group = erpc_call(node, :get_deployment_group!, [id])
+    id = to_string(deployment_group.id)
+
+    assert_receive {:deployment_group_created, %{id: ^id, environment_variables: []}}
+
+    {:ok, view, _html} = live(conn, ~p"/hub/#{hub.id}")
+
+    assert view
+           |> element("#hub-deployment-group-#{id} [aria-label=\"environment variables\"]")
+           |> render()
+           |> Floki.parse_fragment!()
+           |> Floki.text()
+           |> String.trim() == "0"
+
+    erpc_call(node, :create_environment_variable, [[deployment_group: deployment_group]])
+    assert_receive {:deployment_group_updated, %{id: ^id, environment_variables: [_]}}
+
+    assert view
+           |> element("#hub-deployment-group-#{id} [aria-label=\"environment variables\"]")
+           |> render()
+           |> Floki.parse_fragment!()
+           |> Floki.text()
+           |> String.trim() == "1"
+  end
 end

test/support/factory.ex

@@ -61,7 +61,8 @@ defmodule Livebook.Factory do
       name: unique_value("FOO_"),
       mode: :offline,
       agent_keys: [],
-      secrets: []
+      secrets: [],
+      environment_variables: []
     }
   end