Add support for configuring file systems using env variables (#498)

* Add support for configuring file systems using env variables

* Add UI for copying file systems env configuration
This commit is contained in:
Jonatan Kłosko 2021-08-18 14:41:57 +02:00 committed by GitHub
parent 1afcd7030e
commit bf05fb0a50
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 119 additions and 7 deletions

View file

@ -132,6 +132,10 @@ The following environment variables configure Livebook:
"attached:NODE:COOKIE" (Attached node) or "embedded" (Embedded).
Defaults to "standalone".
* LIVEBOOK_FILE_SYSTEM_1, LIVEBOOK_FILE_SYSTEM_2, ... - configures additional
file systems. Each variable should hold a configuration string, which must
be of the form: "s3 BUCKET_URL ACCESS_KEY_ID SECRET_ACCESS_KEY".
* LIVEBOOK_IP - sets the ip address to start the web application on.
Must be a valid IPv4 or IPv6 address.
@ -139,7 +143,7 @@ The following environment variables configure Livebook:
Must be at least 12 characters. Defaults to token authentication.
* LIVEBOOK_PORT - sets the port Livebook runs on. If you want multiple instances
to run on the same domain but different ports, you also need to set 'LIVEBOOK_SECRET_KEY_BASE'.
to run on the same domain but different ports, you also need to set LIVEBOOK_SECRET_KEY_BASE.
Defaults to 8080.
* LIVEBOOK_ROOT_PATH - sets the root path to use for file selection.

View file

@ -61,6 +61,10 @@
@apply bg-gray-100;
}
.icon-button:disabled {
@apply cursor-default pointer-events-none text-gray-300;
}
.icon-button i {
line-height: 1;
}

View file

@ -36,4 +36,6 @@ root_path =
|> Livebook.FileSystem.Utils.ensure_dir_path()
local_file_system = Livebook.FileSystem.Local.new(default_path: root_path)
config :livebook, :file_systems, [local_file_system]
configured_file_systems = Livebook.Config.file_systems!("LIVEBOOK_FILE_SYSTEM_")
config :livebook, :file_systems, [local_file_system | configured_file_systems]

View file

@ -252,6 +252,61 @@ defmodule Livebook.Config do
}
end
@doc """
Parses file systems list.
Appends subsequent numbers to the given env prefix (starting from 1)
and parses the env variables until `nil` is encountered.
"""
def file_systems!(env_prefix) do
Stream.iterate(1, &(&1 + 1))
|> Stream.map(fn n ->
env = env_prefix <> Integer.to_string(n)
System.get_env(env)
end)
|> Stream.take_while(& &1)
|> Enum.map(&parse_file_system!/1)
end
defp parse_file_system!(string) do
case string do
"s3 " <> config ->
FileSystem.S3.from_config_string(config)
_ ->
abort!(
~s{unrecognised file system, expected "s3 BUCKET_URL ACCESS_KEY_ID SECRET_ACCESS_KEY", got: #{inspect(string)}}
)
end
|> case do
{:ok, file_system} -> file_system
{:error, message} -> abort!(message)
end
end
@doc """
Returns environment variables configuration corresponding
to the given file systems.
The first (default) file system is ignored.
"""
def file_systems_as_env(file_systems)
def file_systems_as_env([_ | additional_file_systems]) do
additional_file_systems
|> Enum.with_index(1)
|> Enum.map(fn {file_system, n} ->
config = file_system_to_config_string(file_system)
["LIVEBOOK_FILE_SYSTEM_", Integer.to_string(n), "=", ?", config, ?"]
end)
|> Enum.intersperse(" ")
|> IO.iodata_to_binary()
end
defp file_system_to_config_string(%FileSystem.S3{} = file_system) do
["s3 ", FileSystem.S3.to_config_string(file_system)]
end
@doc """
Aborts booting due to a configuration error.
"""

View file

@ -24,6 +24,34 @@ defmodule Livebook.FileSystem.S3 do
secret_access_key: secret_access_key
}
end
@doc """
Parses file system from a configuration string.
The expected format is `"BUCKET_URL ACCESS_KEY_ID SECRET_ACCESS_KEY"`.
## Examples
Livebook.FileSystem.S3.from_config_string("https://s3.eu-central-1.amazonaws.com/mybucket myaccesskeyid mysecret")
"""
@spec from_config_string(String.t()) :: {:ok, t()} | {:error, String.t()}
def from_config_string(string) do
case String.split(string) do
[bucket_url, access_key_id, secret_access_key] ->
{:ok, new(bucket_url, access_key_id, secret_access_key)}
args ->
{:error, "S3 filesystem configuration expects 3 arguments, but got #{length(args)}"}
end
end
@doc """
Formats the given file system into an equivalent configuration string.
"""
@spec to_config_string(t()) :: String.t()
def to_config_string(file_system) do
"#{file_system.bucket_url} #{file_system.access_key_id} #{file_system.secret_access_key}"
end
end
defimpl Livebook.FileSystem, for: Livebook.FileSystem.S3 do

View file

@ -14,8 +14,14 @@ defmodule LivebookWeb.SettingsLive do
current_user = build_current_user(session, socket)
file_systems = Livebook.Config.file_systems()
file_systems_env = Livebook.Config.file_systems_as_env(file_systems)
{:ok, assign(socket, current_user: current_user, file_systems: file_systems)}
{:ok,
assign(socket,
current_user: current_user,
file_systems: file_systems,
file_systems_env: file_systems_env
)}
end
@impl true
@ -44,9 +50,21 @@ defmodule LivebookWeb.SettingsLive do
</div>
<!-- File systems configuration -->
<div class="flex flex-col space-y-4">
<h2 class="text-xl text-gray-800 font-semibold">
File systems
</h2>
<div class="flex justify-between items-center">
<h2 class="text-xl text-gray-800 font-semibold">
File systems
</h2>
<span class="tooltip top" aria-label="Copy as environment variables">
<button class="icon-button"
id={"file-systems-env-clipcopy"}
phx-hook="ClipCopy"
data-target-id={"file-systems-env-source"}
disabled={@file_systems_env == ""}>
<.remix_icon icon="clipboard-line" class="text-lg" />
</button>
<span class="hidden" id="file-systems-env-source"><%= @file_systems_env %></span>
</span>
</div>
<%= live_component LivebookWeb.SettingsLive.FileSystemsComponent,
file_systems: @file_systems %>
</div>
@ -97,6 +115,7 @@ defmodule LivebookWeb.SettingsLive do
end
def handle_info({:file_systems_updated, file_systems}, socket) do
{:noreply, assign(socket, :file_systems, file_systems)}
file_systems_env = Livebook.Config.file_systems_as_env(file_systems)
{:noreply, assign(socket, file_systems: file_systems, file_systems_env: file_systems_env)}
end
end