Change fly clustering to auto (#2672)

CHANGELOG.md

@@ -11,7 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 * Improved latency for showing completions in Remote execution cell ([#2447](https://github.com/livebook-dev/livebook/pull/2447))
 * Reduced the time of `Mix.install/2` when adding a new dependency ([#2499](https://github.com/livebook-dev/livebook/pull/2499))
 * Support for Web Bluetooth and Web Serial APIs within iframes ([#2514](https://github.com/livebook-dev/livebook/pull/2514))
-* Support for `LIVEBOOK_CLUSTER=fly` when deploying a Livebook cluster on Fly ([#2519](https://github.com/livebook-dev/livebook/pull/2519))
+* Added `LIVEBOOK_CLUSTER=auto` to automatically configure the cluster on Fly.io ([#2519](https://github.com/livebook-dev/livebook/pull/2519))
 * Healthcheck definition to the Livebook Docker image ([#2517](https://github.com/livebook-dev/livebook/pull/2517))
 * Persistence of app password in the notebook, encrypted in the stamp metadata ([#2550](https://github.com/livebook-dev/livebook/pull/2550))
 * Support for Basic Authentication ZTA ([#2564](https://github.com/livebook-dev/livebook/pull/2564))

docs/deployment/clustering.md

@@ -1,14 +1,14 @@
 # Clustering
 
-If you plan to run several Livebook instances behind a load balancer, you need to enable clustering via the `LIVEBOOK_CLUSTER` environment variable in your Docker image.
+If you plan to run several Livebook instances behind a load balancer, you need to enable clustering via the `LIVEBOOK_CLUSTER` environment variable. In addition you must set `LIVEBOOK_SECRET_KEY_BASE` and `LIVEBOOK_COOKIE` to different random values (use `openssl rand -base64 48` to generate said values).
 
 Depending on the clustering strategy of your choice, you must set additional environment variables, oftentimes, at runtime. When using the Livebook Docker image, you can create a file at `/app/user/env.sh` that exports the necessary environment variables. This file is invoked right before booting Livebook.
 
 You may set `LIVEBOOK_CLUSTER` to one of the following values.
 
-## `fly`
+## `auto`
 
-It automatically sets up a cluster to run on Fly using DNS configuration. Behind the scenes, it sets the relevant environment variables and enables IPv6 support.
+Detects the hosting platform and automatically sets up a cluster using DNS configuration. Currently the only supported platform is Fly.io.
 
 ## `dns:QUERY`
 
@@ -16,6 +16,4 @@ Sets up a cluster using DNS for queries for A/AAAA records to discover new nodes
 
   * `LIVEBOOK_NODE=livebook_server@MACHINE_IP`, where `MACHINE_IP` is the machine IP of each deployed node
 
-  * You must set `LIVEBOOK_SECRET_KEY_BASE` and `LIVEBOOK_COOKIE` to different random values (use `openssl rand -base64 48` to generate said values)
-
   * If your cloud requires IPv6, also set `ERL_AFLAGS="-proto_dist inet6_tcp"`

lib/livebook/hubs/dockerfile.ex

@@ -8,7 +8,7 @@ defmodule Livebook.Hubs.Dockerfile do
   @type config :: %{
           deploy_all: boolean(),
           docker_tag: String.t(),
-          clustering: nil | :fly_io,
+          clustering: nil | :auto | :dns,
           zta_provider: atom() | nil,
           zta_key: String.t() | nil
         }
@@ -54,7 +54,7 @@ defmodule Livebook.Hubs.Dockerfile do
     types = %{
       deploy_all: :boolean,
       docker_tag: :string,
-      clustering: Ecto.ParameterizedType.init(Ecto.Enum, values: [:fly_io, :dns]),
+      clustering: Ecto.ParameterizedType.init(Ecto.Enum, values: [:auto, :dns]),
       zta_provider: Ecto.ParameterizedType.init(Ecto.Enum, values: zta_types),
       zta_key: :string
     }
@@ -152,14 +152,14 @@ defmodule Livebook.Hubs.Dockerfile do
 
     startup =
       case to_string(config.clustering) do
-        "fly_io" ->
+        "auto" ->
           """
           # --- Clustering ---
 
           # Set the same Livebook secrets across all nodes
           ENV LIVEBOOK_SECRET_KEY_BASE "#{secret_key_base}"
           ENV LIVEBOOK_COOKIE "#{cookie}"
-          ENV LIVEBOOK_CLUSTER "fly"
+          ENV LIVEBOOK_CLUSTER "auto"
           """
 
         "dns" ->
@@ -343,9 +343,9 @@ defmodule Livebook.Hubs.Dockerfile do
 
     clustering_env =
       case to_string(config.clustering) do
-        "fly_io" ->
+        "auto" ->
           [
-            {"LIVEBOOK_CLUSTER", "fly"},
+            {"LIVEBOOK_CLUSTER", "auto"},
             {"LIVEBOOK_SECRET_KEY_BASE", secret_key_base},
             {"LIVEBOOK_COOKIE", cookie}
           ]

lib/livebook/teams/deployment_group.ex

@@ -12,7 +12,7 @@ defmodule Livebook.Teams.DeploymentGroup do
     field :name, :string
     field :mode, Ecto.Enum, values: [:online, :offline], default: :online
     field :hub_id, :string
-    field :clustering, Ecto.Enum, values: [:fly_io, :dns]
+    field :clustering, Ecto.Enum, values: [:auto, :dns]
     field :zta_provider, Ecto.Enum, values: @zta_providers
     field :zta_key, :string
     field :url, :string

lib/livebook_web/components/app_components.ex

@@ -107,6 +107,7 @@ defmodule LivebookWeb.AppComponents do
           field={@form[:url]}
         />
 
+        <div>
           <.select_field
             label="Clustering"
             help={
@@ -117,29 +118,32 @@ defmodule LivebookWeb.AppComponents do
               into a single cluster.
               You must either deploy
               it as a single instance
-            or choose a platform to
+              or choose a strategy to
-            enable clustering on.
+              connect the instances.
               '''
             }
             field={@form[:clustering]}
             options={[
-            {"Single instance", ""},
+              {"Automatic", "auto"},
-            {"Fly.io", "fly_io"},
+              {"DNS", "dns"},
-            {"DNS", "dns"}
+              {"Single instance", ""}
             ]}
             disabled={@disabled}
           />
-
+          <div :if={to_string(@form[:clustering].value) == "dns"} class="mt-1 text-sm">
-        <div :if={to_string(@form[:clustering].value) == "dns"} class="text-sm mt-1">
             See the
             <a
-            class="text-blue-800 hover:text-blue-600"
+              class="text-blue-600 hover:text-blue-700"
               href="https://hexdocs.pm/livebook/docker.html#clustering"
             >
               Clustering docs
             </a>
             for more information.
           </div>
+          <p class="mt-1 text-sm">
+            Automatic clustering is available when deploying to Fly.io.
+          </p>
+        </div>
       </div>
     </div>
 

lib/livebook_web/components/form_components.ex

@@ -494,7 +494,9 @@ defmodule LivebookWeb.FormComponents do
           {@rest}
         >
           <option :if={@prompt} value=""><%= @prompt %></option>
-          <%= Phoenix.HTML.Form.options_for_select(@options, @value) %>
+          <%!-- TODO: we use to_string to normalize nil and "", remove
+                this once fixed upstream https://github.com/phoenixframework/phoenix_html/issues/444 --%>
+          <%= Phoenix.HTML.Form.options_for_select(@options, to_string(@value)) %>
         </select>
         <div class="pointer-events-none absolute inset-y-0 right-0 flex items-center px-2 text-gray-500">
           <.remix_icon icon="arrow-down-s-line" />

lib/livebook_web/live/hub/teams/deployment_group_agent_component.ex

@@ -93,7 +93,7 @@ defmodule LivebookWeb.Hub.Teams.DeploymentGroupAgentComponent do
 
       <%= if @agent_key_id do %>
         <div class="mt-5">
-          <.tabs id="deployment-instruction" default={default_tab(@deployment_group)}>
+          <.tabs id="deployment-instruction" default="docker">
             <:tab id="docker" label="Docker">
               <div class="flex flex-col gap-3">
                 <p class="text-gray-700">
@@ -145,9 +145,6 @@ defmodule LivebookWeb.Hub.Teams.DeploymentGroupAgentComponent do
     String.slice(string, 0, preview_length) <> String.duplicate("•", length - preview_length)
   end
 
-  defp default_tab(%{clustering: :fly_io}), do: "fly_io"
-  defp default_tab(_deloyment_group), do: "docker"
-
   @impl true
   def handle_event("select_agent_key", %{"agent_key" => %{"id" => id}}, socket) do
     id = if(id != "", do: id)
@@ -216,7 +213,7 @@ defmodule LivebookWeb.Hub.Teams.DeploymentGroupAgentComponent do
     fly secrets set \\
     #{envs}
 
-    fly deploy
+    fly deploy --ha=false
     """
   end
 end

lib/livebook_web/live/hub/teams/deployment_group_form_component.ex

@@ -23,7 +23,11 @@ defmodule LivebookWeb.Hub.Teams.DeploymentGroupFormComponent do
           %{}
         end
 
-      {:ok, assign_form(socket, Teams.change_deployment_group(%DeploymentGroup{}, attrs))}
+      {:ok,
+       assign_form(
+         socket,
+         Teams.change_deployment_group(%DeploymentGroup{clustering: :auto}, attrs)
+       )}
     end
   end
 

rel/server/env.sh.eex

@@ -1,4 +1,4 @@
-if [ "$LIVEBOOK_CLUSTER" = "fly" ]; then
+if [ "$LIVEBOOK_CLUSTER" = "auto" ] && [ ! -z "$FLY_APP_NAME" ]; then
   export ERL_AFLAGS="-proto_dist inet6_tcp"
   export LIVEBOOK_CLUSTER="dns:${FLY_APP_NAME}.internal"
   if [ ! -z "${LIVEBOOK_NODE}" ];

test/livebook/hubs/dockerfile_test.exs

@@ -198,14 +198,14 @@ defmodule Livebook.Hubs.DockerfileTest do
                """
     end
 
-    test "deploying with fly.io cluster setup" do
+    test "deploying with auto cluster setup" do
-      config = dockerfile_config(%{clustering: :fly_io})
+      config = dockerfile_config(%{clustering: :auto})
       hub = personal_hub()
       file = Livebook.FileSystem.File.local(p("/notebook.livemd"))
 
       dockerfile = Dockerfile.airgapped_dockerfile(config, hub, [], [], file, [], %{})
 
-      assert dockerfile =~ ~s/ENV LIVEBOOK_CLUSTER "fly"/
+      assert dockerfile =~ ~s/ENV LIVEBOOK_CLUSTER "auto"/
     end
 
     test "deploying with dns cluster setup" do
@@ -258,13 +258,13 @@ defmodule Livebook.Hubs.DockerfileTest do
     end
 
     test "deploying with fly.io cluster setup" do
-      config = dockerfile_config(%{clustering: :fly_io})
+      config = dockerfile_config(%{clustering: :auto})
       hub = team_hub()
       agent_key = Livebook.Factory.build(:agent_key)
 
       %{env: env} = Dockerfile.online_docker_info(config, hub, agent_key)
 
-      assert {"LIVEBOOK_CLUSTER", "fly"} in env
+      assert {"LIVEBOOK_CLUSTER", "auto"} in env
     end
 
     test "deploying with dns cluster setup" do
@@ -281,7 +281,7 @@ defmodule Livebook.Hubs.DockerfileTest do
 
   describe "warnings/6" do
     test "warns when session secrets are used" do
-      config = dockerfile_config(%{clustering: :fly_io})
+      config = dockerfile_config(%{clustering: :auto})
       hub = personal_hub()
       app_settings = Livebook.Notebook.AppSettings.new()
 
@@ -295,7 +295,7 @@ defmodule Livebook.Hubs.DockerfileTest do
     end
 
     test "warns when hub secrets are used from personal hub" do
-      config = dockerfile_config(%{clustering: :fly_io})
+      config = dockerfile_config(%{clustering: :auto})
       hub = personal_hub()
       app_settings = Livebook.Notebook.AppSettings.new()
 
@@ -319,7 +319,7 @@ defmodule Livebook.Hubs.DockerfileTest do
     end
 
     test "warns when there is a reference to external file system from personal hub" do
-      config = dockerfile_config(%{clustering: :fly_io})
+      config = dockerfile_config(%{clustering: :auto})
       hub = personal_hub()
       app_settings = Livebook.Notebook.AppSettings.new()
 
@@ -346,7 +346,7 @@ defmodule Livebook.Hubs.DockerfileTest do
     end
 
     test "warns when deploying a directory in personal hub and it has any file systems" do
-      config = dockerfile_config(%{clustering: :fly_io, deploy_all: true})
+      config = dockerfile_config(%{clustering: :auto, deploy_all: true})
       hub = personal_hub()
       app_settings = Livebook.Notebook.AppSettings.new()
 
@@ -361,7 +361,7 @@ defmodule Livebook.Hubs.DockerfileTest do
     end
 
     test "warns when the app has no password in personal hub" do
-      config = dockerfile_config(%{clustering: :fly_io})
+      config = dockerfile_config(%{clustering: :auto})
       hub = personal_hub()
       app_settings = %{Livebook.Notebook.AppSettings.new() | access_type: :public}
 
@@ -370,7 +370,7 @@ defmodule Livebook.Hubs.DockerfileTest do
     end
 
     test "warns when the app has no password and no ZTA in teams hub" do
-      config = dockerfile_config(%{clustering: :fly_io})
+      config = dockerfile_config(%{clustering: :auto})
       hub = team_hub()
       app_settings = %{Livebook.Notebook.AppSettings.new() | access_type: :public}
 
@@ -390,7 +390,7 @@ defmodule Livebook.Hubs.DockerfileTest do
       assert [warning] = Dockerfile.airgapped_warnings(config, hub, [], [], app_settings, [], %{})
       assert warning =~ "The deployment is not configured for clustering"
 
-      config = %{config | clustering: :fly_io}
+      config = %{config | clustering: :auto}
 
       assert [] = Dockerfile.airgapped_warnings(config, hub, [], [], app_settings, [], %{})
     end