From bf05fb0a50543f7d54dc6e6f9fd4712ec0c8a95e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Wed, 18 Aug 2021 14:41:57 +0200 Subject: [PATCH] Add support for configuring file systems using env variables (#498) * Add support for configuring file systems using env variables * Add UI for copying file systems env configuration --- README.md | 6 ++- assets/css/components.css | 4 ++ config/runtime.exs | 4 +- lib/livebook/config.ex | 55 ++++++++++++++++++++++++++ lib/livebook/file_system/s3.ex | 28 +++++++++++++ lib/livebook_web/live/settings_live.ex | 29 +++++++++++--- 6 files changed, 119 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 23a4783e6..4c57fde5c 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,10 @@ The following environment variables configure Livebook: "attached:NODE:COOKIE" (Attached node) or "embedded" (Embedded). Defaults to "standalone". + * LIVEBOOK_FILE_SYSTEM_1, LIVEBOOK_FILE_SYSTEM_2, ... - configures additional + file systems. Each variable should hold a configuration string, which must + be of the form: "s3 BUCKET_URL ACCESS_KEY_ID SECRET_ACCESS_KEY". + * LIVEBOOK_IP - sets the ip address to start the web application on. Must be a valid IPv4 or IPv6 address. @@ -139,7 +143,7 @@ The following environment variables configure Livebook: Must be at least 12 characters. Defaults to token authentication. * LIVEBOOK_PORT - sets the port Livebook runs on. If you want multiple instances - to run on the same domain but different ports, you also need to set 'LIVEBOOK_SECRET_KEY_BASE'. + to run on the same domain but different ports, you also need to set LIVEBOOK_SECRET_KEY_BASE. Defaults to 8080. * LIVEBOOK_ROOT_PATH - sets the root path to use for file selection. diff --git a/assets/css/components.css b/assets/css/components.css index f3290a0c3..de70291cf 100644 --- a/assets/css/components.css +++ b/assets/css/components.css @@ -61,6 +61,10 @@ @apply bg-gray-100; } + .icon-button:disabled { + @apply cursor-default pointer-events-none text-gray-300; + } + .icon-button i { line-height: 1; } diff --git a/config/runtime.exs b/config/runtime.exs index cca81536a..0d3158270 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -36,4 +36,6 @@ root_path = |> Livebook.FileSystem.Utils.ensure_dir_path() local_file_system = Livebook.FileSystem.Local.new(default_path: root_path) -config :livebook, :file_systems, [local_file_system] +configured_file_systems = Livebook.Config.file_systems!("LIVEBOOK_FILE_SYSTEM_") + +config :livebook, :file_systems, [local_file_system | configured_file_systems] diff --git a/lib/livebook/config.ex b/lib/livebook/config.ex index b7e9d8f1d..f379542ca 100644 --- a/lib/livebook/config.ex +++ b/lib/livebook/config.ex @@ -252,6 +252,61 @@ defmodule Livebook.Config do } end + @doc """ + Parses file systems list. + + Appends subsequent numbers to the given env prefix (starting from 1) + and parses the env variables until `nil` is encountered. + """ + def file_systems!(env_prefix) do + Stream.iterate(1, &(&1 + 1)) + |> Stream.map(fn n -> + env = env_prefix <> Integer.to_string(n) + System.get_env(env) + end) + |> Stream.take_while(& &1) + |> Enum.map(&parse_file_system!/1) + end + + defp parse_file_system!(string) do + case string do + "s3 " <> config -> + FileSystem.S3.from_config_string(config) + + _ -> + abort!( + ~s{unrecognised file system, expected "s3 BUCKET_URL ACCESS_KEY_ID SECRET_ACCESS_KEY", got: #{inspect(string)}} + ) + end + |> case do + {:ok, file_system} -> file_system + {:error, message} -> abort!(message) + end + end + + @doc """ + Returns environment variables configuration corresponding + to the given file systems. + + The first (default) file system is ignored. + """ + def file_systems_as_env(file_systems) + + def file_systems_as_env([_ | additional_file_systems]) do + additional_file_systems + |> Enum.with_index(1) + |> Enum.map(fn {file_system, n} -> + config = file_system_to_config_string(file_system) + ["LIVEBOOK_FILE_SYSTEM_", Integer.to_string(n), "=", ?", config, ?"] + end) + |> Enum.intersperse(" ") + |> IO.iodata_to_binary() + end + + defp file_system_to_config_string(%FileSystem.S3{} = file_system) do + ["s3 ", FileSystem.S3.to_config_string(file_system)] + end + @doc """ Aborts booting due to a configuration error. """ diff --git a/lib/livebook/file_system/s3.ex b/lib/livebook/file_system/s3.ex index 149b0c633..705537e15 100644 --- a/lib/livebook/file_system/s3.ex +++ b/lib/livebook/file_system/s3.ex @@ -24,6 +24,34 @@ defmodule Livebook.FileSystem.S3 do secret_access_key: secret_access_key } end + + @doc """ + Parses file system from a configuration string. + + The expected format is `"BUCKET_URL ACCESS_KEY_ID SECRET_ACCESS_KEY"`. + + ## Examples + + Livebook.FileSystem.S3.from_config_string("https://s3.eu-central-1.amazonaws.com/mybucket myaccesskeyid mysecret") + """ + @spec from_config_string(String.t()) :: {:ok, t()} | {:error, String.t()} + def from_config_string(string) do + case String.split(string) do + [bucket_url, access_key_id, secret_access_key] -> + {:ok, new(bucket_url, access_key_id, secret_access_key)} + + args -> + {:error, "S3 filesystem configuration expects 3 arguments, but got #{length(args)}"} + end + end + + @doc """ + Formats the given file system into an equivalent configuration string. + """ + @spec to_config_string(t()) :: String.t() + def to_config_string(file_system) do + "#{file_system.bucket_url} #{file_system.access_key_id} #{file_system.secret_access_key}" + end end defimpl Livebook.FileSystem, for: Livebook.FileSystem.S3 do diff --git a/lib/livebook_web/live/settings_live.ex b/lib/livebook_web/live/settings_live.ex index c0e32ee99..eb0e4bbb0 100644 --- a/lib/livebook_web/live/settings_live.ex +++ b/lib/livebook_web/live/settings_live.ex @@ -14,8 +14,14 @@ defmodule LivebookWeb.SettingsLive do current_user = build_current_user(session, socket) file_systems = Livebook.Config.file_systems() + file_systems_env = Livebook.Config.file_systems_as_env(file_systems) - {:ok, assign(socket, current_user: current_user, file_systems: file_systems)} + {:ok, + assign(socket, + current_user: current_user, + file_systems: file_systems, + file_systems_env: file_systems_env + )} end @impl true @@ -44,9 +50,21 @@ defmodule LivebookWeb.SettingsLive do
-

- File systems -

+
+

+ File systems +

+ + + + +
<%= live_component LivebookWeb.SettingsLive.FileSystemsComponent, file_systems: @file_systems %>
@@ -97,6 +115,7 @@ defmodule LivebookWeb.SettingsLive do end def handle_info({:file_systems_updated, file_systems}, socket) do - {:noreply, assign(socket, :file_systems, file_systems)} + file_systems_env = Livebook.Config.file_systems_as_env(file_systems) + {:noreply, assign(socket, file_systems: file_systems, file_systems_env: file_systems_env)} end end