livebook-dev/livebook
1
1
Fork 0
mirror of https://github.com/livebook-dev/livebook.git synced 2025-09-05 20:44:30 +08:00
Code Issues Projects Releases Packages Wiki Activity

Change fly clustering to auto (#2672)

Browse source
This commit is contained in:
Jonatan Kłosko 2024-06-24 18:36:49 +02:00 • committed by GitHub
parent ce7d6814a6
commit 272dff78ad
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 70 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
CHANGELOG.md
View file

@ -11,7 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
* Improved latency for showing completions in Remote execution cell ([#2447](https://github.com/livebook-dev/livebook/pull/2447))
* Reduced the time of `Mix.install/2` when adding a new dependency ([#2499](https://github.com/livebook-dev/livebook/pull/2499))
* Support for Web Bluetooth and Web Serial APIs within iframes ([#2514](https://github.com/livebook-dev/livebook/pull/2514))
* Support for `LIVEBOOK_CLUSTER=fly` when deploying a Livebook cluster on Fly ([#2519](https://github.com/livebook-dev/livebook/pull/2519))
* Added `LIVEBOOK_CLUSTER=auto` to automatically configure the cluster on Fly.io ([#2519](https://github.com/livebook-dev/livebook/pull/2519))
* Healthcheck definition to the Livebook Docker image ([#2517](https://github.com/livebook-dev/livebook/pull/2517))
* Persistence of app password in the notebook, encrypted in the stamp metadata ([#2550](https://github.com/livebook-dev/livebook/pull/2550))
* Support for Basic Authentication ZTA ([#2564](https://github.com/livebook-dev/livebook/pull/2564))

8
docs/deployment/clustering.md
View file

@ -1,14 +1,14 @@
# Clustering
If you plan to run several Livebook instances behind a load balancer, you need to enable clustering via the `LIVEBOOK_CLUSTER` environment variable in your Docker image.
If you plan to run several Livebook instances behind a load balancer, you need to enable clustering via the `LIVEBOOK_CLUSTER` environment variable. In addition you must set `LIVEBOOK_SECRET_KEY_BASE` and `LIVEBOOK_COOKIE` to different random values (use `openssl rand -base64 48` to generate said values).
Depending on the clustering strategy of your choice, you must set additional environment variables, oftentimes, at runtime. When using the Livebook Docker image, you can create a file at `/app/user/env.sh` that exports the necessary environment variables. This file is invoked right before booting Livebook.
You may set `LIVEBOOK_CLUSTER` to one of the following values.
## `fly`
## `auto`
It automatically sets up a cluster to run on Fly using DNS configuration. Behind the scenes, it sets the relevant environment variables and enables IPv6 support.
Detects the hosting platform and automatically sets up a cluster using DNS configuration. Currently the only supported platform is Fly.io.
## `dns:QUERY`
@ -16,6 +16,4 @@ Sets up a cluster using DNS for queries for A/AAAA records to discover new nodes
* `LIVEBOOK_NODE=livebook_server@MACHINE_IP`, where `MACHINE_IP` is the machine IP of each deployed node
* You must set `LIVEBOOK_SECRET_KEY_BASE` and `LIVEBOOK_COOKIE` to different random values (use `openssl rand -base64 48` to generate said values)
* If your cloud requires IPv6, also set `ERL_AFLAGS="-proto_dist inet6_tcp"`

12
lib/livebook/hubs/dockerfile.ex
View file

@ -8,7 +8,7 @@ defmodule Livebook.Hubs.Dockerfile do
@type config :: %{
deploy_all: boolean(),
docker_tag: String.t(),
clustering: nil | :fly_io,
clustering: nil | :auto | :dns,
zta_provider: atom() | nil,
zta_key: String.t() | nil
}
@ -54,7 +54,7 @@ defmodule Livebook.Hubs.Dockerfile do
types = %{
deploy_all: :boolean,
docker_tag: :string,
clustering: Ecto.ParameterizedType.init(Ecto.Enum, values: [:fly_io, :dns]),
clustering: Ecto.ParameterizedType.init(Ecto.Enum, values: [:auto, :dns]),
zta_provider: Ecto.ParameterizedType.init(Ecto.Enum, values: zta_types),
zta_key: :string
}
@ -152,14 +152,14 @@ defmodule Livebook.Hubs.Dockerfile do
startup =
case to_string(config.clustering) do
"fly_io" ->
"auto" ->
"""
# --- Clustering ---
# Set the same Livebook secrets across all nodes
ENV LIVEBOOK_SECRET_KEY_BASE "#{secret_key_base}"
ENV LIVEBOOK_COOKIE "#{cookie}"
ENV LIVEBOOK_CLUSTER "fly"
ENV LIVEBOOK_CLUSTER "auto"
"""
"dns" ->
@ -343,9 +343,9 @@ defmodule Livebook.Hubs.Dockerfile do
clustering_env =
case to_string(config.clustering) do
"fly_io" ->
"auto" ->
[
{"LIVEBOOK_CLUSTER", "fly"},
{"LIVEBOOK_CLUSTER", "auto"},
{"LIVEBOOK_SECRET_KEY_BASE", secret_key_base},
{"LIVEBOOK_COOKIE", cookie}
]

2
lib/livebook/teams/deployment_group.ex
View file

@ -12,7 +12,7 @@ defmodule Livebook.Teams.DeploymentGroup do
field :name, :string
field :mode, Ecto.Enum, values: [:online, :offline], default: :online
field :hub_id, :string
field :clustering, Ecto.Enum, values: [:fly_io, :dns]
field :clustering, Ecto.Enum, values: [:auto, :dns]
field :zta_provider, Ecto.Enum, values: @zta_providers
field :zta_key, :string
field :url, :string

68
lib/livebook_web/components/app_components.ex
View file

@ -107,38 +107,42 @@ defmodule LivebookWeb.AppComponents do
field={@form[:url]}
/>
<.select_field
label="Clustering"
help={
~S'''
When running multiple
instances of Livebook,
they need to be connected
into a single cluster.
You must either deploy
it as a single instance
or choose a platform to
enable clustering on.
'''
}
field={@form[:clustering]}
options={[
{"Single instance", ""},
{"Fly.io", "fly_io"},
{"DNS", "dns"}
]}
disabled={@disabled}
/>
<div :if={to_string(@form[:clustering].value) == "dns"} class="text-sm mt-1">
See the
<a
class="text-blue-800 hover:text-blue-600"
href="https://hexdocs.pm/livebook/docker.html#clustering"
>
Clustering docs
</a>
for more information.
<div>
<.select_field
label="Clustering"
help={
~S'''
When running multiple
instances of Livebook,
they need to be connected
into a single cluster.
You must either deploy
it as a single instance
or choose a strategy to
connect the instances.
'''
}
field={@form[:clustering]}
options={[
{"Automatic", "auto"},
{"DNS", "dns"},
{"Single instance", ""}
]}
disabled={@disabled}
/>
<div :if={to_string(@form[:clustering].value) == "dns"} class="mt-1 text-sm">
See the
<a
class="text-blue-600 hover:text-blue-700"
href="https://hexdocs.pm/livebook/docker.html#clustering"
>
Clustering docs
</a>
for more information.
</div>
<p class="mt-1 text-sm">
Automatic clustering is available when deploying to Fly.io.
</p>
</div>
</div>
</div>

4
lib/livebook_web/components/form_components.ex
View file

@ -494,7 +494,9 @@ defmodule LivebookWeb.FormComponents do
{@rest}
>
<option :if={@prompt} value=""><%= @prompt %></option>
<%= Phoenix.HTML.Form.options_for_select(@options, @value) %>
<%!-- TODO: we use to_string to normalize nil and "", remove
this once fixed upstream https://github.com/phoenixframework/phoenix_html/issues/444 --%>
<%= Phoenix.HTML.Form.options_for_select(@options, to_string(@value)) %>
</select>
<div class="pointer-events-none absolute inset-y-0 right-0 flex items-center px-2 text-gray-500">
<.remix_icon icon="arrow-down-s-line" />

7
lib/livebook_web/live/hub/teams/deployment_group_agent_component.ex
View file

@ -93,7 +93,7 @@ defmodule LivebookWeb.Hub.Teams.DeploymentGroupAgentComponent do
<%= if @agent_key_id do %>
<div class="mt-5">
<.tabs id="deployment-instruction" default={default_tab(@deployment_group)}>
<.tabs id="deployment-instruction" default="docker">
<:tab id="docker" label="Docker">
<div class="flex flex-col gap-3">
<p class="text-gray-700">
@ -145,9 +145,6 @@ defmodule LivebookWeb.Hub.Teams.DeploymentGroupAgentComponent do
String.slice(string, 0, preview_length) <> String.duplicate("•", length - preview_length)
end
defp default_tab(%{clustering: :fly_io}), do: "fly_io"
defp default_tab(_deloyment_group), do: "docker"
@impl true
def handle_event("select_agent_key", %{"agent_key" => %{"id" => id}}, socket) do
id = if(id != "", do: id)
@ -216,7 +213,7 @@ defmodule LivebookWeb.Hub.Teams.DeploymentGroupAgentComponent do
fly secrets set \\
#{envs}
fly deploy
fly deploy --ha=false
"""
end
end

6
lib/livebook_web/live/hub/teams/deployment_group_form_component.ex
View file

@ -23,7 +23,11 @@ defmodule LivebookWeb.Hub.Teams.DeploymentGroupFormComponent do
%{}
end
{:ok, assign_form(socket, Teams.change_deployment_group(%DeploymentGroup{}, attrs))}
{:ok,
assign_form(
socket,
Teams.change_deployment_group(%DeploymentGroup{clustering: :auto}, attrs)
)}
end
end

2
rel/server/env.sh.eex
View file

@ -1,4 +1,4 @@
if [ "$LIVEBOOK_CLUSTER" = "fly" ]; then
if [ "$LIVEBOOK_CLUSTER" = "auto" ] && [ ! -z "$FLY_APP_NAME" ]; then
export ERL_AFLAGS="-proto_dist inet6_tcp"
export LIVEBOOK_CLUSTER="dns:${FLY_APP_NAME}.internal"
if [ ! -z "${LIVEBOOK_NODE}" ];

24
test/livebook/hubs/dockerfile_test.exs
View file

@ -198,14 +198,14 @@ defmodule Livebook.Hubs.DockerfileTest do
"""
end
test "deploying with fly.io cluster setup" do
config = dockerfile_config(%{clustering: :fly_io})
test "deploying with auto cluster setup" do
config = dockerfile_config(%{clustering: :auto})
hub = personal_hub()
file = Livebook.FileSystem.File.local(p("/notebook.livemd"))
dockerfile = Dockerfile.airgapped_dockerfile(config, hub, [], [], file, [], %{})
assert dockerfile =~ ~s/ENV LIVEBOOK_CLUSTER "fly"/
assert dockerfile =~ ~s/ENV LIVEBOOK_CLUSTER "auto"/
end
test "deploying with dns cluster setup" do
@ -258,13 +258,13 @@ defmodule Livebook.Hubs.DockerfileTest do
end
test "deploying with fly.io cluster setup" do
config = dockerfile_config(%{clustering: :fly_io})
config = dockerfile_config(%{clustering: :auto})
hub = team_hub()
agent_key = Livebook.Factory.build(:agent_key)
%{env: env} = Dockerfile.online_docker_info(config, hub, agent_key)
assert {"LIVEBOOK_CLUSTER", "fly"} in env
assert {"LIVEBOOK_CLUSTER", "auto"} in env
end
test "deploying with dns cluster setup" do
@ -281,7 +281,7 @@ defmodule Livebook.Hubs.DockerfileTest do
describe "warnings/6" do
test "warns when session secrets are used" do
config = dockerfile_config(%{clustering: :fly_io})
config = dockerfile_config(%{clustering: :auto})
hub = personal_hub()
app_settings = Livebook.Notebook.AppSettings.new()
@ -295,7 +295,7 @@ defmodule Livebook.Hubs.DockerfileTest do
end
test "warns when hub secrets are used from personal hub" do
config = dockerfile_config(%{clustering: :fly_io})
config = dockerfile_config(%{clustering: :auto})
hub = personal_hub()
app_settings = Livebook.Notebook.AppSettings.new()
@ -319,7 +319,7 @@ defmodule Livebook.Hubs.DockerfileTest do
end
test "warns when there is a reference to external file system from personal hub" do
config = dockerfile_config(%{clustering: :fly_io})
config = dockerfile_config(%{clustering: :auto})
hub = personal_hub()
app_settings = Livebook.Notebook.AppSettings.new()
@ -346,7 +346,7 @@ defmodule Livebook.Hubs.DockerfileTest do
end
test "warns when deploying a directory in personal hub and it has any file systems" do
config = dockerfile_config(%{clustering: :fly_io, deploy_all: true})
config = dockerfile_config(%{clustering: :auto, deploy_all: true})
hub = personal_hub()
app_settings = Livebook.Notebook.AppSettings.new()
@ -361,7 +361,7 @@ defmodule Livebook.Hubs.DockerfileTest do
end
test "warns when the app has no password in personal hub" do
config = dockerfile_config(%{clustering: :fly_io})
config = dockerfile_config(%{clustering: :auto})
hub = personal_hub()
app_settings = %{Livebook.Notebook.AppSettings.new() | access_type: :public}
@ -370,7 +370,7 @@ defmodule Livebook.Hubs.DockerfileTest do
end
test "warns when the app has no password and no ZTA in teams hub" do
config = dockerfile_config(%{clustering: :fly_io})
config = dockerfile_config(%{clustering: :auto})
hub = team_hub()
app_settings = %{Livebook.Notebook.AppSettings.new() | access_type: :public}
@ -390,7 +390,7 @@ defmodule Livebook.Hubs.DockerfileTest do
assert [warning] = Dockerfile.airgapped_warnings(config, hub, [], [], app_settings, [], %{})
assert warning =~ "The deployment is not configured for clustering"
config = %{config | clustering: :fly_io}
config = %{config | clustering: :auto}
assert [] = Dockerfile.airgapped_warnings(config, hub, [], [], app_settings, [], %{})
end