Add support for configuring file systems using env variables (#498)

* Add support for configuring file systems using env variables

* Add UI for copying file systems env configuration

README.md

@@ -132,6 +132,10 @@ The following environment variables configure Livebook:
     "attached:NODE:COOKIE" (Attached node) or "embedded" (Embedded).
     Defaults to "standalone".
 
+  * LIVEBOOK_FILE_SYSTEM_1, LIVEBOOK_FILE_SYSTEM_2, ... - configures additional
+    file systems. Each variable should hold a configuration string, which must
+    be of the form: "s3 BUCKET_URL ACCESS_KEY_ID SECRET_ACCESS_KEY".
+
   * LIVEBOOK_IP - sets the ip address to start the web application on.
     Must be a valid IPv4 or IPv6 address.
 
@@ -139,7 +143,7 @@ The following environment variables configure Livebook:
     Must be at least 12 characters. Defaults to token authentication.
 
   * LIVEBOOK_PORT - sets the port Livebook runs on. If you want multiple instances
-    to run on the same domain but different ports, you also need to set 'LIVEBOOK_SECRET_KEY_BASE'.
+    to run on the same domain but different ports, you also need to set LIVEBOOK_SECRET_KEY_BASE.
     Defaults to 8080.
 
   * LIVEBOOK_ROOT_PATH - sets the root path to use for file selection.

assets/css/components.css

@@ -61,6 +61,10 @@
     @apply bg-gray-100;
   }
 
+  .icon-button:disabled {
+    @apply cursor-default pointer-events-none text-gray-300;
+  }
+
   .icon-button i {
     line-height: 1;
   }

config/runtime.exs

@@ -36,4 +36,6 @@ root_path =
   |> Livebook.FileSystem.Utils.ensure_dir_path()
 
 local_file_system = Livebook.FileSystem.Local.new(default_path: root_path)
-config :livebook, :file_systems, [local_file_system]
+configured_file_systems = Livebook.Config.file_systems!("LIVEBOOK_FILE_SYSTEM_")
+
+config :livebook, :file_systems, [local_file_system | configured_file_systems]

lib/livebook/config.ex

@@ -252,6 +252,61 @@ defmodule Livebook.Config do
     }
   end
 
+  @doc """
+  Parses file systems list.
+
+  Appends subsequent numbers to the given env prefix (starting from 1)
+  and parses the env variables until `nil` is encountered.
+  """
+  def file_systems!(env_prefix) do
+    Stream.iterate(1, &(&1 + 1))
+    |> Stream.map(fn n ->
+      env = env_prefix <> Integer.to_string(n)
+      System.get_env(env)
+    end)
+    |> Stream.take_while(& &1)
+    |> Enum.map(&parse_file_system!/1)
+  end
+
+  defp parse_file_system!(string) do
+    case string do
+      "s3 " <> config ->
+        FileSystem.S3.from_config_string(config)
+
+      _ ->
+        abort!(
+          ~s{unrecognised file system, expected "s3 BUCKET_URL ACCESS_KEY_ID SECRET_ACCESS_KEY", got: #{inspect(string)}}
+        )
+    end
+    |> case do
+      {:ok, file_system} -> file_system
+      {:error, message} -> abort!(message)
+    end
+  end
+
+  @doc """
+  Returns environment variables configuration corresponding
+  to the given file systems.
+
+  The first (default) file system is ignored.
+  """
+  def file_systems_as_env(file_systems)
+
+  def file_systems_as_env([_ | additional_file_systems]) do
+    additional_file_systems
+    |> Enum.with_index(1)
+    |> Enum.map(fn {file_system, n} ->
+      config = file_system_to_config_string(file_system)
+      ["LIVEBOOK_FILE_SYSTEM_", Integer.to_string(n), "=", ?", config, ?"]
+    end)
+    |> Enum.intersperse(" ")
+    |> IO.iodata_to_binary()
+  end
+
+  defp file_system_to_config_string(%FileSystem.S3{} = file_system) do
+    ["s3 ", FileSystem.S3.to_config_string(file_system)]
+  end
+
   @doc """
   Aborts booting due to a configuration error.
   """

lib/livebook/file_system/s3.ex

@@ -24,6 +24,34 @@ defmodule Livebook.FileSystem.S3 do
       secret_access_key: secret_access_key
     }
   end
+
+  @doc """
+  Parses file system from a configuration string.
+
+  The expected format is `"BUCKET_URL ACCESS_KEY_ID SECRET_ACCESS_KEY"`.
+
+  ## Examples
+
+      Livebook.FileSystem.S3.from_config_string("https://s3.eu-central-1.amazonaws.com/mybucket myaccesskeyid mysecret")
+  """
+  @spec from_config_string(String.t()) :: {:ok, t()} | {:error, String.t()}
+  def from_config_string(string) do
+    case String.split(string) do
+      [bucket_url, access_key_id, secret_access_key] ->
+        {:ok, new(bucket_url, access_key_id, secret_access_key)}
+
+      args ->
+        {:error, "S3 filesystem configuration expects 3 arguments, but got #{length(args)}"}
+    end
+  end
+
+  @doc """
+  Formats the given file system into an equivalent configuration string.
+  """
+  @spec to_config_string(t()) :: String.t()
+  def to_config_string(file_system) do
+    "#{file_system.bucket_url} #{file_system.access_key_id} #{file_system.secret_access_key}"
+  end
 end
 
 defimpl Livebook.FileSystem, for: Livebook.FileSystem.S3 do

lib/livebook_web/live/settings_live.ex

@@ -14,8 +14,14 @@ defmodule LivebookWeb.SettingsLive do
     current_user = build_current_user(session, socket)
 
     file_systems = Livebook.Config.file_systems()
+    file_systems_env = Livebook.Config.file_systems_as_env(file_systems)
 
-    {:ok, assign(socket, current_user: current_user, file_systems: file_systems)}
+    {:ok,
+     assign(socket,
+       current_user: current_user,
+       file_systems: file_systems,
+       file_systems_env: file_systems_env
+     )}
   end
 
   @impl true
@@ -44,9 +50,21 @@ defmodule LivebookWeb.SettingsLive do
           </div>
           <!-- File systems configuration -->
           <div class="flex flex-col space-y-4">
-            <h2 class="text-xl text-gray-800 font-semibold">
-              File systems
-            </h2>
+            <div class="flex justify-between items-center">
+              <h2 class="text-xl text-gray-800 font-semibold">
+                File systems
+              </h2>
+              <span class="tooltip top" aria-label="Copy as environment variables">
+                <button class="icon-button"
+                  id={"file-systems-env-clipcopy"}
+                  phx-hook="ClipCopy"
+                  data-target-id={"file-systems-env-source"}
+                  disabled={@file_systems_env == ""}>
+                  <.remix_icon icon="clipboard-line" class="text-lg" />
+                </button>
+                <span class="hidden" id="file-systems-env-source"><%= @file_systems_env %></span>
+              </span>
+            </div>
             <%= live_component LivebookWeb.SettingsLive.FileSystemsComponent,
                   file_systems: @file_systems %>
           </div>
@@ -97,6 +115,7 @@ defmodule LivebookWeb.SettingsLive do
   end
 
   def handle_info({:file_systems_updated, file_systems}, socket) do
-    {:noreply, assign(socket, :file_systems, file_systems)}
+    file_systems_env = Livebook.Config.file_systems_as_env(file_systems)
+    {:noreply, assign(socket, file_systems: file_systems, file_systems_env: file_systems_env)}
   end
 end