livebook-dev/livebook
1
1
Fork 0
mirror of https://github.com/livebook-dev/livebook.git synced 2024-09-20 10:05:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

Implement File Storage (#2212)

Browse source
This commit is contained in:
Alexandre de Souza 2023-09-25 11:18:30 -03:00 committed by GitHub
parent 393915da7e
commit 96bf5ddbcc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
29 changed files with 838 additions and 343 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

73
lib/livebook/file_system/s3.ex
View file

@ -7,10 +7,11 @@ defmodule Livebook.FileSystem.S3 do
@type t :: %__MODULE__{
id: String.t(),
bucket_url: String.t(),
external_id: String.t(),
external_id: String.t() | nil,
region: String.t(),
access_key_id: String.t(),
secret_access_key: String.t()
secret_access_key: String.t(),
hub_id: String.t()
}
embedded_schema do
@ -19,6 +20,7 @@ defmodule Livebook.FileSystem.S3 do
field :region, :string
field :access_key_id, :string
field :secret_access_key, :string
field :hub_id, :string
end
@doc """
@ -32,22 +34,20 @@ defmodule Livebook.FileSystem.S3 do
* `:external_id` - the external id from Teams.
* `:prefix` - the id prefix.
* `:hub_id` - the Hub id.
* `:id` - the file system id.
"""
@spec new(String.t(), String.t(), String.t(), keyword()) :: t()
def new(bucket_url, access_key_id, secret_access_key, opts \\ []) do
opts = Keyword.validate!(opts, [:region, :external_id, :prefix])
opts = Keyword.validate!(opts, [:region, :external_id, :hub_id, :id])
bucket_url = String.trim_trailing(bucket_url, "/")
region = opts[:region] || region_from_uri(bucket_url)
hash = :crypto.hash(:sha256, bucket_url) |> Base.url_encode64(padding: false)
id =
if prefix = opts[:prefix],
do: "#{prefix}-s3-#{hash}",
else: "s3-#{hash}"
hub_id = Keyword.get(opts, :hub_id, Livebook.Hubs.Personal.id())
id = opts[:id] || id(hub_id, bucket_url)
%__MODULE__{
id: id,
@ -55,14 +55,21 @@ defmodule Livebook.FileSystem.S3 do
external_id: opts[:external_id],
region: region,
access_key_id: access_key_id,
secret_access_key: secret_access_key
secret_access_key: secret_access_key,
hub_id: hub_id
}
end
defp region_from_uri(uri) do
# For many services the API host is of the form *.[region].[rootdomain].com
%{host: host} = URI.parse(uri)
host |> String.split(".") |> Enum.reverse() |> Enum.at(2, "auto")
splitted_host = host |> String.split(".") |> Enum.reverse()
case Enum.at(splitted_host, 2, "auto") do
"s3" -> "us-east-1"
"r2" -> "auto"
region -> region
end
end
@doc """
@ -110,10 +117,42 @@ defmodule Livebook.FileSystem.S3 do
:external_id,
:region,
:access_key_id,
:secret_access_key
:secret_access_key,
:hub_id
])
|> put_region_from_uri()
|> validate_required([:bucket_url, :access_key_id, :secret_access_key])
|> Livebook.Utils.validate_url(:bucket_url)
|> put_id()
end
defp put_region_from_uri(changeset) do
case get_field(changeset, :bucket_url) do
nil -> changeset
bucket_url -> put_change(changeset, :region, region_from_uri(bucket_url))
end
end
defp put_id(changeset) do
hub_id = get_field(changeset, :hub_id)
bucket_url = get_field(changeset, :bucket_url)
if get_field(changeset, :id) do
changeset
else
put_change(changeset, :id, id(hub_id, bucket_url))
end
end
def id(_, nil), do: nil
def id(nil, bucket_url), do: hashed_id(bucket_url)
def id(hub_id, bucket_url), do: "#{hub_id}-#{hashed_id(bucket_url)}"
defp hashed_id(bucket_url) do
hash = :crypto.hash(:sha256, bucket_url)
encrypted_hash = Base.url_encode64(hash, padding: false)
"s3-#{encrypted_hash}"
end
end
@ -377,7 +416,8 @@ defimpl Livebook.FileSystem, for: Livebook.FileSystem.S3 do
region: fields["region"],
access_key_id: fields["access_key_id"],
secret_access_key: fields["secret_access_key"],
prefix: fields["prefix"]
id: fields["id"],
hub_id: fields["hub_id"]
})
end
@ -385,14 +425,15 @@ defimpl Livebook.FileSystem, for: Livebook.FileSystem.S3 do
S3.new(fields.bucket_url, fields.access_key_id, fields.secret_access_key,
region: fields[:region],
external_id: fields[:external_id],
prefix: fields[:prefix]
id: fields[:id],
hub_id: fields[:hub_id]
)
end
def dump(file_system) do
file_system
|> Map.from_struct()
|> Map.take([:bucket_url, :region, :access_key_id, :secret_access_key])
|> Map.take([:id, :bucket_url, :region, :access_key_id, :secret_access_key, :hub_id])
end
def external_metadata(file_system) do

27
lib/livebook/file_systems.ex
View file

@ -7,6 +7,33 @@ defmodule Livebook.FileSystems do
@spec type(FileSystem.t()) :: String.t()
def type(%FileSystem.S3{}), do: "s3"
@doc """
Updates file system with the given changes.
"""
@spec update_file_system(FileSystem.t(), map()) ::
{:ok, FileSystem.t()} | {:error, Ecto.Changeset.t()}
def update_file_system(file_system, attrs) do
file_system
|> change_file_system(attrs)
|> Ecto.Changeset.apply_action(:update)
end
@doc """
Returns an `%Ecto.Changeset{}` for tracking file system changes.
"""
@spec change_file_system(FileSystem.t()) :: Ecto.Changeset.t()
def change_file_system(file_system) do
change_file_system(file_system, %{})
end
@doc """
Returns an `%Ecto.Changeset{}` for tracking file system changes.
"""
@spec change_file_system(FileSystem.t(), map()) :: Ecto.Changeset.t()
def change_file_system(%FileSystem.S3{} = file_system, attrs) do
FileSystem.S3.change_file_system(file_system, attrs)
end
@doc """
Loads the file system from given type and dumped data.
"""

29
lib/livebook/hubs.ex
View file

@ -157,6 +157,12 @@ defmodule Livebook.Hubs do
* `{:secret_updated, %Secret{}}`
* `{:secret_deleted, %Secret{}}`
Topic `hubs:file_systems`:
* `{:file_system_created, FileSystem.t()}`
* `{:file_system_updated, FileSystem.t()}`
* `{:file_system_deleted, FileSystem.t()}`
"""
@spec subscribe(atom() | list(atom())) :: :ok | {:error, term()}
def subscribe(topics) when is_list(topics) do
@ -295,14 +301,27 @@ defmodule Livebook.Hubs do
end
@doc """
Gets a list of file systems for given hub.
Gets a list of file systems from all hubs.
"""
@spec get_file_systems(Provider.t()) :: list(FileSystem.t())
def get_file_systems(hub) do
hub_file_systems = Provider.get_file_systems(hub)
@spec get_file_systems() :: list(FileSystem.t())
def get_file_systems() do
file_systems = Enum.flat_map(get_hubs(), &Provider.get_file_systems/1)
local_file_system = Livebook.Config.local_file_system()
[local_file_system | Enum.sort_by(hub_file_systems, & &1.id)]
[local_file_system | Enum.sort_by(file_systems, & &1.id)]
end
@doc """
Gets a list of file systems for given hub.
"""
@spec get_file_systems(Provider.t(), keyword()) :: list(FileSystem.t())
def get_file_systems(hub, opts \\ []) do
hub_file_systems = Provider.get_file_systems(hub)
sorted_hub_file_systems = Enum.sort_by(hub_file_systems, & &1.id)
if opts[:hub_only],
do: sorted_hub_file_systems,
else: [Livebook.Config.local_file_system() | sorted_hub_file_systems]
end
@doc """

7
lib/livebook/hubs/team_client.ex
View file

@ -187,10 +187,9 @@ defmodule Livebook.Hubs.TeamClient do
{:ok, decrypted_value} = Teams.decrypt(file_system.value, secret_key, sign_secret)
dumped_data =
Map.merge(Jason.decode!(decrypted_value), %{
"external_id" => file_system.id,
"prefix" => state.hub.id
})
decrypted_value
|> Jason.decode!()
|> Map.put("external_id", file_system.id)
FileSystems.load(file_system.type, dumped_data)
end

6
lib/livebook/users/user.ex
View file

@ -31,10 +31,10 @@ defmodule Livebook.Users.User do
@doc """
Generates a new user.
"""
@spec new() :: t()
def new() do
@spec new(String.t()) :: t()
def new(id \\ Utils.random_id()) do
%__MODULE__{
id: Utils.random_id(),
id: id,
name: nil,
email: nil,
hex_color: Livebook.EctoTypes.HexColor.random()

18
lib/livebook_web/live/file_select_component.ex
View file

@ -45,7 +45,8 @@ defmodule LivebookWeb.FileSelectComponent do
renaming_file: nil,
renamed_name: nil,
error_message: nil,
file_systems: Livebook.Settings.file_systems()
configure_path: nil,
file_systems: []
)
|> allow_upload(:folder,
accept: :any,
@ -70,7 +71,14 @@ defmodule LivebookWeb.FileSelectComponent do
|> assign(assigns)
|> update_file_infos(force_reload? or running_files_changed?)
{:ok, socket}
{file_systems, configure_hub_id} =
if hub = socket.assigns[:hub],
do: {Livebook.Hubs.get_file_systems(hub), hub.id},
else: {Livebook.Hubs.get_file_systems(), Livebook.Hubs.Personal.id()}
configure_path = ~p"/hub/#{configure_hub_id}/file-systems/new"
{:ok, assign(socket, file_systems: file_systems, configure_path: configure_path)}
end
@impl true
@ -83,6 +91,7 @@ defmodule LivebookWeb.FileSelectComponent do
<.file_system_menu_button
file={@file}
file_systems={@file_systems}
configure_path={@configure_path}
file_system_select_disabled={@file_system_select_disabled}
myself={@myself}
/>
@ -281,7 +290,7 @@ defmodule LivebookWeb.FileSelectComponent do
<%= for file_system <- @file_systems do %>
<%= if file_system == @file.file_system do %>
<.menu_item variant={:selected}>
<button role="menuitem">
<button id={"file-system-#{file_system.id}"} role="menuitem">
<.file_system_icon file_system={file_system} />
<span><%= file_system_label(file_system) %></span>
</button>
@ -289,6 +298,7 @@ defmodule LivebookWeb.FileSelectComponent do
<% else %>
<.menu_item>
<button
id={"file-system-#{file_system.id}"}
role="menuitem"
phx-target={@myself}
phx-click="set_file_system"
@ -301,7 +311,7 @@ defmodule LivebookWeb.FileSelectComponent do
<% end %>
<% end %>
<.menu_item>
<.link navigate={~p"/settings"} class="border-t border-gray-200" role="menuitem">
<.link navigate={@configure_path} class="border-t border-gray-200" role="menuitem">
<.remix_icon icon="settings-3-line" />
<span>Configure</span>
</.link>

4
lib/livebook_web/live/hooks/user_hook.ex
View file

@ -33,9 +33,7 @@ defmodule LivebookWeb.UserHook do
# attributes if the socket is connected. Otherwise uses
# `user_data` from session.
defp build_current_user(session, socket) do
user = User.new()
identity_data = Map.new(session["identity_data"], fn {k, v} -> {Atom.to_string(k), v} end)
connect_params = get_connect_params(socket) || %{}
attrs = connect_params["user_data"] || session["user_data"] || %{}
@ -45,6 +43,8 @@ defmodule LivebookWeb.UserHook do
attrs -> attrs
end
user = User.new(attrs["id"])
case Livebook.Users.update_user(user, attrs) do
{:ok, user} -> user
{:error, _changeset} -> user

65
lib/livebook_web/live/hub/edit/personal_component.ex
View file

@ -11,7 +11,9 @@ defmodule LivebookWeb.Hub.Edit.PersonalComponent do
socket = assign(socket, assigns)
changeset = Personal.change_hub(assigns.hub)
secrets = Hubs.get_secrets(assigns.hub)
file_systems = Hubs.get_file_systems(assigns.hub, hub_only: true)
secret_name = assigns.params["secret_name"]
file_system_id = assigns.params["file_system_id"]
secret_value =
if assigns.live_action == :edit_secret do
@ -19,9 +21,18 @@ defmodule LivebookWeb.Hub.Edit.PersonalComponent do
raise(NotFoundError, "could not find secret matching #{inspect(secret_name)}")
end
file_system =
if assigns.live_action == :edit_file_system do
Enum.find_value(file_systems, &(&1.id == file_system_id && &1)) ||
raise(NotFoundError, "could not find file system matching #{inspect(file_system_id)}")
end
{:ok,
assign(socket,
secrets: secrets,
file_system: file_system,
file_system_id: file_system_id,
file_systems: file_systems,
changeset: changeset,
stamp_changeset: changeset,
secret_name: secret_name,
@ -90,7 +101,23 @@ defmodule LivebookWeb.Hub.Edit.PersonalComponent do
id="hub-secrets-list"
hub={@hub}
secrets={@secrets}
target={@myself}
/>
</div>
<div class="flex flex-col space-y-4">
<h2 class="text-xl text-gray-800 font-medium pb-2 border-b border-gray-200">
File Storages
</h2>
<p class="text-gray-700">
File storages are used to store notebooks and their files.
</p>
<.live_component
module={LivebookWeb.Hub.FileSystemListComponent}
id="hub-file-systems-list"
hub_id={@hub.id}
file_systems={@file_systems}
/>
</div>
@ -167,6 +194,23 @@ defmodule LivebookWeb.Hub.Edit.PersonalComponent do
return_to={~p"/hub/#{@hub.id}"}
/>
</.modal>
<.modal
:if={@live_action in [:new_file_system, :edit_file_system]}
id="file-systems-modal"
show
width={:medium}
patch={~p"/hub/#{@hub.id}"}
>
<.live_component
module={LivebookWeb.Hub.FileSystemFormComponent}
id="file-systems"
hub={@hub}
file_system={@file_system}
file_system_id={@file_system_id}
return_to={~p"/hub/#{@hub.id}"}
/>
</.modal>
</div>
"""
end
@ -193,25 +237,6 @@ defmodule LivebookWeb.Hub.Edit.PersonalComponent do
{:noreply, validate(params, :stamp_changeset, socket)}
end
def handle_event("delete_hub_secret", attrs, socket) do
%{hub: hub} = socket.assigns
on_confirm = fn socket ->
{:ok, secret} = Livebook.Secrets.update_secret(%Livebook.Secrets.Secret{}, attrs)
_ = Livebook.Hubs.delete_secret(hub, secret)
socket
end
{:noreply,
confirm(socket, on_confirm,
title: "Delete hub secret - #{attrs["name"]}",
description: "Are you sure you want to delete this hub secret?",
confirm_text: "Delete",
confirm_icon: "delete-bin-6-line"
)}
end
defp save(params, changeset_name, socket) do
case Personal.update_hub(socket.assigns.hub, params) do
{:ok, hub} ->

46
lib/livebook_web/live/hub/edit/team_component.ex
View file

@ -13,7 +13,9 @@ defmodule LivebookWeb.Hub.Edit.TeamComponent do
changeset = Team.change_hub(assigns.hub)
show_key? = assigns.params["show-key"] == "true"
secrets = Livebook.Hubs.get_secrets(assigns.hub)
file_systems = Hubs.get_file_systems(assigns.hub, hub_only: true)
secret_name = assigns.params["secret_name"]
file_system_id = assigns.params["file_system_id"]
is_default? = is_default?(assigns.hub)
secret_value =
@ -22,10 +24,19 @@ defmodule LivebookWeb.Hub.Edit.TeamComponent do
raise(NotFoundError, "could not find secret matching #{inspect(secret_name)}")
end
file_system =
if assigns.live_action == :edit_file_system do
Enum.find_value(file_systems, &(&1.id == file_system_id && &1)) ||
raise(NotFoundError, "could not find file system matching #{inspect(file_system_id)}")
end
{:ok,
socket
|> assign(
secrets: secrets,
file_system: file_system,
file_system_id: file_system_id,
file_systems: file_systems,
show_key: show_key?,
secret_name: secret_name,
secret_value: secret_value,
@ -163,6 +174,24 @@ defmodule LivebookWeb.Hub.Edit.TeamComponent do
/>
</div>
<div class="flex flex-col space-y-4">
<h2 class="text-xl text-gray-800 font-medium pb-2 border-b border-gray-200">
File Storages
</h2>
<p class="text-gray-700">
File storages are used to store notebooks and their files.
</p>
<.live_component
module={LivebookWeb.Hub.FileSystemListComponent}
id="hub-file-systems-list"
hub_id={@hub.id}
file_systems={@file_systems}
target={@myself}
/>
</div>
<div class="flex flex-col space-y-4">
<h2 class="text-xl text-gray-800 font-medium pb-2 border-b border-gray-200">
Airgapped Deployment
@ -314,6 +343,23 @@ defmodule LivebookWeb.Hub.Edit.TeamComponent do
return_to={~p"/hub/#{@hub.id}"}
/>
</.modal>
<.modal
:if={@live_action in [:new_file_system, :edit_file_system]}
id="file-systems-modal"
show
width={:medium}
patch={~p"/hub/#{@hub.id}"}
>
<.live_component
module={LivebookWeb.Hub.FileSystemFormComponent}
id="file-systems"
hub={@hub}
file_system={@file_system}
file_system_id={@file_system_id}
return_to={~p"/hub/#{@hub.id}"}
/>
</.modal>
</div>
</div>
</div>

33
lib/livebook_web/live/hub/edit_live.ex
View file

@ -9,12 +9,13 @@ defmodule LivebookWeb.Hub.EditLive do
@impl true
def mount(_params, _session, socket) do
Hubs.subscribe([:connection])
{:ok, assign(socket, hub: nil, type: nil, page_title: "Hub - Livebook", params: %{})}
end
@impl true
def handle_params(params, _url, socket) do
Hubs.subscribe([:connection, :secrets])
hub = Hubs.fetch_hub!(params["id"])
type = Provider.type(hub)
@ -85,36 +86,6 @@ defmodule LivebookWeb.Hub.EditLive do
end
@impl true
def handle_info(
{:secret_created, %{name: name, hub_id: id}},
%{assigns: %{hub: %{id: id}}} = socket
) do
{:noreply,
socket
|> push_navigate(to: ~p"/hub/#{id}")
|> put_flash(:success, "Secret #{name} created successfully")}
end
def handle_info(
{:secret_updated, %{name: name, hub_id: id}},
%{assigns: %{hub: %{id: id}}} = socket
) do
{:noreply,
socket
|> push_navigate(to: ~p"/hub/#{id}")
|> put_flash(:success, "Secret #{name} updated successfully")}
end
def handle_info(
{:secret_deleted, %{name: name, hub_id: id}},
%{assigns: %{hub: %{id: id}}} = socket
) do
{:noreply,
socket
|> push_navigate(to: ~p"/hub/#{id}")
|> put_flash(:success, "Secret #{name} deleted successfully")}
end
def handle_info({:hub_connected, id}, %{assigns: %{hub: %{id: id}}} = socket) do
{:noreply, push_navigate(socket, to: ~p"/hub/#{id}")}
end

134
lib/livebook_web/live/hub/file_system_form_component.ex Normal file
View file

@ -0,0 +1,134 @@
defmodule LivebookWeb.Hub.FileSystemFormComponent do
use LivebookWeb, :live_component
alias Livebook.FileSystem
alias Livebook.FileSystems
@impl true
def update(assigns, socket) do
{file_system, assigns} = Map.pop!(assigns, :file_system)
mode = mode(file_system)
button = button(file_system)
title = title(file_system)
file_system = file_system || %FileSystem.S3{}
changeset = FileSystems.change_file_system(file_system)
socket = assign(socket, assigns)
{:ok,
assign(socket,
file_system: file_system,
changeset: changeset,
mode: mode,
title: title,
button: button,
error_message: nil
)}
end
@impl true
def render(assigns) do
~H"""
<div class="p-6 flex flex-col space-y-5">
<h3 class="text-2xl font-semibold text-gray-800">
<%= @title %>
</h3>
<p class="text-gray-700">
Configure an AWS S3 bucket as a Livebook file system.
Many storage services offer an S3-compatible API and
those work as well.
</p>
<div :if={@error_message} class="error-box">
<%= @error_message %>
</div>
<.form
:let={f}
id="file-systems-form"
for={to_form(@changeset, as: :file_system)}
phx-target={@myself}
phx-submit="save"
phx-change="validate"
autocomplete="off"
spellcheck="false"
>
<div class="flex flex-col space-y-4">
<.text_field
field={f[:bucket_url]}
label="Bucket URL"
placeholder="https://s3.[region].amazonaws.com/[bucket]"
/>
<.text_field field={f[:region]} label="Region (optional)" />
<.password_field field={f[:access_key_id]} label="Access Key ID" />
<.password_field field={f[:secret_access_key]} label="Secret Access Key" />
<div class="flex space-x-2">
<button class="button-base button-blue" type="submit" disabled={not @changeset.valid?}>
<.remix_icon icon={@button.icon} class="align-middle mr-1" />
<span class="font-normal"><%= @button.label %></span>
</button>
<.link patch={@return_to} class="button-base button-outlined-gray">
Cancel
</.link>
</div>
</div>
</.form>
</div>
"""
end
@impl true
def handle_event("validate", %{"file_system" => attrs}, socket) do
changeset =
socket.assigns.file_system
|> FileSystems.change_file_system(attrs)
|> Map.replace!(:action, :validate)
{:noreply, assign(socket, changeset: changeset)}
end
def handle_event("save", %{"file_system" => attrs}, socket) do
with {:ok, file_system} <- FileSystems.update_file_system(socket.assigns.file_system, attrs),
:ok <- check_file_system_conectivity(file_system),
:ok <- save_file_system(file_system, socket) do
message =
case socket.assigns.mode do
:new -> "File storage added successfully"
:edit -> "File storage updated successfully"
end
{:noreply,
socket
|> put_flash(:success, message)
|> push_redirect(to: socket.assigns.return_to)}
else
{:error, %Ecto.Changeset{} = changeset} -> {:noreply, assign(socket, changeset: changeset)}
{:transport_error, message} -> {:noreply, put_flash(socket, :error, message)}
{:error, message} -> {:noreply, assign(socket, error_message: message)}
end
end
defp check_file_system_conectivity(file_system) do
default_dir = FileSystem.File.new(file_system)
case FileSystem.File.list(default_dir) do
{:ok, _} -> :ok
{:error, message} -> {:error, "Connection test failed: " <> message}
end
end
defp save_file_system(file_system, socket) do
case socket.assigns.mode do
:new -> Livebook.Hubs.create_file_system(socket.assigns.hub, file_system)
:edit -> Livebook.Hubs.update_file_system(socket.assigns.hub, file_system)
end
end
defp mode(nil), do: :new
defp mode(_), do: :edit
defp title(nil), do: "Add file storage"
defp title(_), do: "Edit file storage"
defp button(nil), do: %{icon: "add-line", label: "Add"}
defp button(_), do: %{icon: "save-line", label: "Save"}
end

104
lib/livebook_web/live/hub/file_system_list_component.ex Normal file
View file

@ -0,0 +1,104 @@
defmodule LivebookWeb.Hub.FileSystemListComponent do
use LivebookWeb, :live_component
alias Livebook.FileSystem
alias Livebook.FileSystems
@impl true
def render(assigns) do
~H"""
<div id={@id} class="flex flex-col space-y-4">
<div class="flex flex-col space-y-4">
<.no_entries :if={@file_systems == []}>
No file storages in this Hub yet.
</.no_entries>
<div
:for={file_system <- @file_systems}
class="flex items-center justify-between border border-gray-200 rounded-lg p-4"
>
<div class="flex items-center space-x-12">
<.labeled_text label="Type"><%= type(file_system) %></.labeled_text>
<.labeled_text label="Bucket URL"><%= name(file_system) %></.labeled_text>
</div>
<div class="flex items-center space-x-2">
<.menu id={"hub-file-system-#{file_system.id}-menu"}>
<:toggle>
<button class="icon-button" aria-label="open file system menu" type="button">
<.remix_icon icon="more-2-fill" class="text-xl" />
</button>
</:toggle>
<.menu_item>
<.link
id={"hub-file-system-#{file_system.id}-edit"}
patch={~p"/hub/#{@hub_id}/file-systems/edit/#{file_system.id}"}
type="button"
role="menuitem"
>
<.remix_icon icon="file-edit-line" />
<span>Edit</span>
</.link>
</.menu_item>
<.menu_item variant={:danger}>
<button
id={"hub-file-system-#{file_system.id}-detach"}
type="button"
role="menuitem"
class="text-red-600"
phx-click={
JS.push("detach_file_system",
value: %{id: file_system.id, name: name(file_system)}
)
}
phx-target={@myself}
>
<.remix_icon icon="delete-bin-line" />
<span>Detach</span>
</button>
</.menu_item>
</.menu>
</div>
</div>
</div>
<div class="flex">
<.link
patch={~p"/hub/#{@hub_id}/file-systems/new"}
class="button-base button-blue"
id="add-file-system"
>
Add file storage
</.link>
</div>
</div>
"""
end
@impl true
def handle_event("detach_file_system", %{"id" => id, "name" => name}, socket) do
on_confirm = fn socket ->
hub = Livebook.Hubs.fetch_hub!(socket.assigns.hub.id)
file_systems = Livebook.Hubs.get_file_systems(hub)
file_system = Enum.find(file_systems, &(&1.id == id))
case Livebook.Hubs.delete_file_system(hub, file_system) do
:ok ->
socket
|> put_flash(:success, "File storage deleted successfully")
|> push_navigate(to: ~p"/hub/#{hub.id}")
{:transport_error, reason} ->
put_flash(socket, :error, reason)
end
end
{:noreply,
confirm(socket, on_confirm,
title: "Detach hub file storage",
description: "Are you sure you want to detach #{name}?",
confirm_text: "Detach",
confirm_icon: "delete-bin-6-line"
)}
end
defp type(file_system), do: file_system |> FileSystems.type() |> String.upcase()
defp name(file_system), do: file_system |> FileSystem.external_metadata() |> Map.get(:name)
end

2
lib/livebook_web/live/hub/secret_form_component.ex
View file

@ -80,7 +80,7 @@ defmodule LivebookWeb.Hub.SecretFormComponent do
message =
if socket.assigns.secret_name,
do: "Secret #{secret.name} updated successfully",
else: "Secret #{secret.name} created successfully"
else: "Secret #{secret.name} added successfully"
{:noreply,
socket

13
lib/livebook_web/live/hub/secret_list_component.ex
View file

@ -64,7 +64,7 @@ defmodule LivebookWeb.Hub.SecretListComponent do
}
)
}
phx-target={@target}
phx-target={@myself}
role="menuitem"
>
<.remix_icon icon="delete-bin-line" />
@ -89,11 +89,16 @@ defmodule LivebookWeb.Hub.SecretListComponent do
def handle_event("delete_hub_secret", attrs, socket) do
on_confirm = fn socket ->
{:ok, secret} = Secrets.update_secret(%Secret{}, attrs)
hub = Livebook.Hubs.fetch_hub!(secret.hub_id)
hub = Hubs.fetch_hub!(secret.hub_id)
case Hubs.delete_secret(hub, secret) do
:ok -> socket
{:transport_error, reason} -> put_flash(socket, :error, reason)
:ok ->
socket
|> put_flash(:success, "Secret #{secret.name} deleted successfully")
|> push_navigate(to: ~p"/hub/#{hub.id}")
{:transport_error, reason} ->
put_flash(socket, :error, reason)
end
end

12
lib/livebook_web/live/session_live.ex
View file

@ -461,6 +461,7 @@ defmodule LivebookWeb.SessionLive do
id="persistence"
session={@session}
file={@data_view.file}
hub={@data_view.hub}
persist_outputs={@data_view.persist_outputs}
autosave_interval_s={@data_view.autosave_interval_s}
/>
@ -488,7 +489,12 @@ defmodule LivebookWeb.SessionLive do
width={:big}
patch={@self_path}
>
<.add_file_entry_content session={@session} file_entries={@data_view.file_entries} tab={@tab} />
<.add_file_entry_content
session={@session}
hub={@data_view.hub}
file_entries={@data_view.file_entries}
tab={@tab}
/>
</.modal>
<.modal
@ -947,24 +953,28 @@ defmodule LivebookWeb.SessionLive do
:if={@tab == "storage"}
module={LivebookWeb.SessionLive.AddFileEntryFileComponent}
id="add-file-entry-from-file"
hub={@hub}
session={@session}
/>
<.live_component
:if={@tab == "url"}
module={LivebookWeb.SessionLive.AddFileEntryUrlComponent}
id="add-file-entry-from-url"
hub={@hub}
session={@session}
/>
<.live_component
:if={@tab == "upload"}
module={LivebookWeb.SessionLive.AddFileEntryUploadComponent}
id="add-file-entry-from-upload"
hub={@hub}
session={@session}
/>
<.live_component
:if={@tab == "unlisted"}
module={LivebookWeb.SessionLive.AddFileEntryUnlistedComponent}
id="add-file-entry-from-unlisted"
hub={@hub}
session={@session}
file_entries={@file_entries}
/>

1
lib/livebook_web/live/session_live/add_file_entry_file_component.ex
View file

@ -63,6 +63,7 @@ defmodule LivebookWeb.SessionLive.AddFileEntryFileComponent do
module={LivebookWeb.FileSelectComponent}
id="add-file-entry-select"
file={@file}
hub={@hub}
extnames={:any}
running_files={[]}
target={{__MODULE__, @id}}

1
lib/livebook_web/live/session_live/persistence_component.ex
View file

@ -77,6 +77,7 @@ defmodule LivebookWeb.SessionLive.PersistenceComponent do
module={LivebookWeb.FileSelectComponent}
id="persistence_file_select"
file={@draft_file}
hub={@hub}
extnames={[LiveMarkdown.extension()]}
running_files={@running_files}
submit_event={:confirm_file}

52
lib/livebook_web/live/settings_live.ex
View file

@ -13,7 +13,6 @@ defmodule LivebookWeb.SettingsLive do
{:ok,
assign(socket,
file_systems: Livebook.Settings.file_systems(),
env_vars: Livebook.Settings.fetch_env_vars() |> Enum.sort(),
env_var: nil,
autosave_path_state: %{
@ -99,18 +98,6 @@ defmodule LivebookWeb.SettingsLive do
</p>
<.autosave_path_select state={@autosave_path_state} />
</div>
<!-- File systems configuration -->
<div class="flex flex-col space-y-4">
<h2 class="text-xl text-gray-800 font-medium pb-2 border-b border-gray-200">
File systems
</h2>
<p class="mt-4 text-gray-700">
File systems are used to store notebooks. The local disk file system
is visible only to the current machine, but alternative file systems
are available, such as S3-based storages.
</p>
<LivebookWeb.SettingsLive.FileSystemsComponent.render file_systems={@file_systems} />
</div>
<!-- Environment variables configuration -->
<div class="flex flex-col space-y-4">
<h2 class="text-xl text-gray-800 font-medium pb-2 border-b border-gray-200">
@ -190,20 +177,6 @@ defmodule LivebookWeb.SettingsLive do
</div>
</LayoutHelpers.layout>
<.modal
:if={@live_action == :add_file_system}
id="add-file-system-modal"
show
width={:medium}
patch={~p"/settings"}
>
<.live_component
module={LivebookWeb.SettingsLive.AddFileSystemComponent}
id="add-file-system"
return_to={~p"/settings"}
/>
</.modal>
<.modal
:if={@live_action in [:add_env_var, :edit_env_var]}
id="env-var-modal"
@ -273,10 +246,6 @@ defmodule LivebookWeb.SettingsLive do
{:noreply, assign(socket, env_var: env_var)}
end
def handle_params(%{"file_system_id" => file_system_id}, _url, socket) do
{:noreply, assign(socket, file_system_id: file_system_id)}
end
def handle_params(_params, _url, socket), do: {:noreply, assign(socket, env_var: nil)}
@impl true
@ -316,23 +285,6 @@ defmodule LivebookWeb.SettingsLive do
{:noreply, update(socket, :autosave_path_state, &%{&1 | dialog_opened?: true})}
end
def handle_event("detach_file_system", %{"id" => file_system_id}, socket) do
on_confirm = fn socket ->
Livebook.Settings.remove_file_system(file_system_id)
file_systems = Livebook.Settings.file_systems()
assign(socket, file_systems: file_systems)
end
{:noreply,
confirm(socket, on_confirm,
title: "Detach file system",
description:
"Are you sure you want to detach this file system? Any sessions using it will keep the access until they get closed.",
confirm_text: "Detach",
confirm_icon: "close-circle-line"
)}
end
def handle_event("save", %{"update_check_enabled" => enabled}, socket) do
enabled = enabled == "true"
Livebook.UpdateCheck.set_enabled(enabled)
@ -371,10 +323,6 @@ defmodule LivebookWeb.SettingsLive do
end
@impl true
def handle_info({:file_systems_updated, file_systems}, socket) do
{:noreply, assign(socket, file_systems: file_systems)}
end
def handle_info({:set_file, file, _info}, socket) do
{:noreply, update(socket, :autosave_path_state, &%{&1 | file: file})}
end

113
lib/livebook_web/live/settings_live/add_file_system_component.ex
View file

@ -1,113 +0,0 @@
defmodule LivebookWeb.SettingsLive.AddFileSystemComponent do
use LivebookWeb, :live_component
import Ecto.Changeset
alias Livebook.FileSystem
@impl true
def mount(socket) do
{:ok, assign(socket, changeset: changeset(), error_message: nil)}
end
defp changeset(attrs \\ %{}) do
data = %{bucket_url: nil, region: nil, access_key_id: nil, secret_access_key: nil}
types = %{
bucket_url: :string,
region: :string,
access_key_id: :string,
secret_access_key: :string
}
cast({data, types}, attrs, [:bucket_url, :region, :access_key_id, :secret_access_key])
|> validate_required([:bucket_url, :access_key_id, :secret_access_key])
|> Livebook.Utils.validate_url(:bucket_url)
end
@impl true
def render(assigns) do
~H"""
<div class="p-6 flex flex-col space-y-5">
<h3 class="text-2xl font-semibold text-gray-800">
Add file system
</h3>
<p class="text-gray-700">
Configure an AWS S3 bucket as a Livebook file system.
Many storage services offer an S3-compatible API and
those work as well.
</p>
<div :if={@error_message} class="error-box">
<%= @error_message %>
</div>
<.form
:let={f}
for={@changeset}
as={:data}
phx-target={@myself}
phx-submit="add"
phx-change="validate"
autocomplete="off"
spellcheck="false"
>
<div class="flex flex-col space-y-4">
<.text_field
field={f[:bucket_url]}
label="Bucket URL"
placeholder="https://s3.[region].amazonaws.com/[bucket]"
/>
<.text_field field={f[:region]} label="Region (optional)" />
<.password_field field={f[:access_key_id]} label="Access Key ID" />
<.password_field field={f[:secret_access_key]} label="Secret Access Key" />
<div class="flex space-x-2">
<button class="button-base button-blue" type="submit" disabled={not @changeset.valid?}>
Add
</button>
<.link patch={@return_to} class="button-base button-outlined-gray">
Cancel
</.link>
</div>
</div>
</.form>
</div>
"""
end
@impl true
def handle_event("validate", %{"data" => data}, socket) do
changeset = data |> changeset() |> Map.replace!(:action, :validate)
{:noreply, assign(socket, changeset: changeset)}
end
def handle_event("add", %{"data" => data}, socket) do
data
|> changeset()
|> apply_action(:insert)
|> case do
{:ok, data} ->
file_system =
FileSystem.S3.new(data.bucket_url, data.access_key_id, data.secret_access_key,
region: data.region
)
default_dir = FileSystem.File.new(file_system)
case FileSystem.File.list(default_dir) do
{:ok, _} ->
Livebook.Settings.save_file_system(file_system)
send(self(), {:file_systems_updated, Livebook.Settings.file_systems()})
{:noreply, push_patch(socket, to: socket.assigns.return_to)}
{:error, message} ->
{:noreply,
assign(socket,
changeset: changeset(data),
error_message: "Connection test failed: " <> message
)}
end
{:error, changeset} ->
{:noreply, assign(socket, changeset: changeset)}
end
end
end

68
lib/livebook_web/live/settings_live/file_systems_component.ex
View file

@ -1,68 +0,0 @@
defmodule LivebookWeb.SettingsLive.FileSystemsComponent do
use LivebookWeb, :live_component
alias Livebook.FileSystem
@impl true
def render(assigns) do
~H"""
<div class="flex flex-col space-y-4">
<div class="flex flex-col space-y-4">
<div
:for={file_system <- @file_systems}
class="flex items-center justify-between border border-gray-200 rounded-lg p-4"
>
<div class="flex items-center space-x-12">
<.file_system_info file_system={file_system} />
</div>
<.file_system_actions file_system_id={file_system.id} />
</div>
</div>
<div class="flex">
<.link patch={~p"/settings/add-file-system"} class="button-base button-blue">
Add file system
</.link>
</div>
</div>
"""
end
defp file_system_info(%{file_system: %FileSystem.Local{}} = assigns) do
~H"""
<.labeled_text label="Type">Local disk</.labeled_text>
"""
end
defp file_system_info(%{file_system: %FileSystem.S3{}} = assigns) do
~H"""
<.labeled_text label="Type">S3</.labeled_text>
<.labeled_text label="Bucket URL"><%= @file_system.bucket_url %></.labeled_text>
"""
end
defp file_system_actions(assigns) do
~H"""
<div class="flex items-center space-x-2">
<.menu :if={@file_system_id != "local"} id={"file-system-#{@file_system_id}-menu"}>
<:toggle>
<button class="icon-button" aria-label="open file system menu" type="button">
<.remix_icon icon="more-2-fill" class="text-xl" />
</button>
</:toggle>
<.menu_item variant={:danger}>
<button
:if={@file_system_id != "local"}
type="button"
role="menuitem"
class="text-red-600"
phx-click={JS.push("detach_file_system", value: %{id: @file_system_id})}
>
<.remix_icon icon="delete-bin-line" />
<span>Detach</span>
</button>
</.menu_item>
</.menu>
</div>
"""
end
end

2
lib/livebook_web/router.ex
View file

@ -79,6 +79,8 @@ defmodule LivebookWeb.Router do
live "/hub/:id/env-var/edit/:env_var_id", Hub.EditLive, :edit_env_var, as: :hub
live "/hub/:id/secrets/new", Hub.EditLive, :new_secret, as: :hub
live "/hub/:id/secrets/edit/:secret_name", Hub.EditLive, :edit_secret, as: :hub
live "/hub/:id/file-systems/new", Hub.EditLive, :new_file_system, as: :hub
live "/hub/:id/file-systems/edit/:file_system_id", Hub.EditLive, :edit_file_system, as: :hub
live "/sessions/:id", SessionLive, :page
live "/sessions/:id/shortcuts", SessionLive, :shortcuts

4
test/livebook/file_system/s3_test.exs
View file

@ -1057,10 +1057,12 @@ defmodule Livebook.FileSystem.S3Test do
file_system = build(:fs_s3)
assert FileSystem.dump(file_system) == %{
id: "personal-hub-s3-86IzUeRugmgK2-X2FmlkurFD0UPsr4Qs1IwieDqfQpA",
bucket_url: "https://mybucket.s3.amazonaws.com",
region: "us-east-1",
access_key_id: "key",
secret_access_key: "secret"
secret_access_key: "secret",
hub_id: "personal-hub"
}
end
end

9
test/livebook_teams/hubs/team_client_test.exs
View file

@ -311,7 +311,8 @@ defmodule Livebook.Hubs.TeamClientTest do
hash = :crypto.hash(:sha256, bucket_url)
fs_id = "#{id}-s3-#{Base.url_encode64(hash, padding: false)}"
file_system = build(:fs_s3, id: fs_id, bucket_url: bucket_url, external_id: "123456")
file_system =
build(:fs_s3, id: fs_id, bucket_url: bucket_url, external_id: "123456", hub_id: id)
type = Livebook.FileSystems.type(file_system)
%{name: name} = Livebook.FileSystem.external_metadata(file_system)
@ -353,7 +354,8 @@ defmodule Livebook.Hubs.TeamClientTest do
hash = :crypto.hash(:sha256, bucket_url)
fs_id = "#{id}-s3-#{Base.url_encode64(hash, padding: false)}"
file_system = build(:fs_s3, id: fs_id, bucket_url: bucket_url, external_id: "994641")
file_system =
build(:fs_s3, id: fs_id, bucket_url: bucket_url, external_id: "994641", hub_id: id)
type = Livebook.FileSystems.type(file_system)
%{name: name} = Livebook.FileSystem.external_metadata(file_system)
@ -425,7 +427,8 @@ defmodule Livebook.Hubs.TeamClientTest do
hash = :crypto.hash(:sha256, bucket_url)
fs_id = "#{id}-s3-#{Base.url_encode64(hash, padding: false)}"
file_system = build(:fs_s3, id: fs_id, bucket_url: bucket_url, external_id: "45465641")
file_system =
build(:fs_s3, id: fs_id, bucket_url: bucket_url, external_id: "45465641", hub_id: id)
type = Livebook.FileSystems.type(file_system)
%{name: name} = Livebook.FileSystem.external_metadata(file_system)

13
test/livebook_teams/teams_test.exs
View file

@ -295,7 +295,12 @@ defmodule Livebook.TeamsTest do
test "returns transport errors when file system doesn't exists", %{user: user, node: node} do
hub = create_team_hub(user, node)
file_system = build(:fs_s3, bucket_url: "https://i_cant_exist.s3.amazonaws.com")
file_system =
build(:fs_s3,
bucket_url: "https://i_cant_exist.s3.amazonaws.com",
external_id: "123456789"
)
# Guarantee it doesn't exists and will return HTTP status 404
assert Teams.delete_file_system(hub, file_system) ==
@ -303,10 +308,4 @@ defmodule Livebook.TeamsTest do
"Something went wrong, try again later or please file a bug if it persists"}
end
end
defp create_teams_file_system(hub, node) do
org_key = :erpc.call(node, Hub.Integration, :get_org_key!, [hub.org_key_id])
:erpc.call(node, Hub.Integration, :create_file_system, [[org_key: org_key]])
end
end

134
test/livebook_teams/web/hub/edit_live_test.exs
View file

@ -7,7 +7,7 @@ defmodule LivebookWeb.Integration.Hub.EditLiveTest do
alias Livebook.Hubs
setup %{user: user, node: node} do
Livebook.Hubs.subscribe([:crud, :connection, :secrets])
Livebook.Hubs.subscribe([:crud, :connection, :secrets, :file_systems])
hub = create_team_hub(user, node)
id = hub.id
@ -99,7 +99,7 @@ defmodule LivebookWeb.Integration.Hub.EditLiveTest do
assert_receive {:secret_created, ^secret}
%{"success" => "Secret TEAM_ADD_SECRET created successfully"} =
%{"success" => "Secret TEAM_ADD_SECRET added successfully"} =
assert_redirect(view, "/hub/#{hub.id}")
{:ok, view, _html} = live(conn, ~p"/hub/#{hub.id}")
@ -252,5 +252,135 @@ defmodule LivebookWeb.Integration.Hub.EditLiveTest do
|> element("span", "Default")
|> has_element?()
end
test "creates a file system", %{conn: conn, hub: hub} do
{:ok, view, _html} = live(conn, ~p"/hub/#{hub.id}")
bypass = Bypass.open()
file_system = build_bypass_file_system(bypass, hub.id)
id = file_system.id
attrs = %{file_system: Livebook.FileSystem.dump(file_system)}
expect_s3_listing(bypass)
refute render(view) =~ file_system.bucket_url
view
|> element("#add-file-system")
|> render_click(%{})
assert_patch(view, ~p"/hub/#{hub.id}/file-systems/new")
assert render(view) =~ "Add file storage"
view
|> element("#file-systems-form")
|> render_change(attrs)
refute view
|> element("#file-systems-form button[disabled]")
|> has_element?()
view
|> element("#file-systems-form")
|> render_submit(attrs)
assert_receive {:file_system_created, %{id: ^id} = file_system}
%{"success" => "File storage added successfully"} =
assert_redirect(view, "/hub/#{hub.id}")
{:ok, view, _html} = live(conn, ~p"/hub/#{hub.id}")
assert render(element(view, "#hub-file-systems-list")) =~ file_system.bucket_url
assert file_system in Livebook.Hubs.get_file_systems(hub)
end
test "updates existing file system", %{conn: conn, hub: hub} do
bypass = Bypass.open()
file_system = build_bypass_file_system(bypass, hub.id)
id = file_system.id
:ok = Hubs.create_file_system(hub, file_system)
assert_receive {:file_system_created, %{id: ^id} = file_system}
{:ok, view, _html} = live(conn, ~p"/hub/#{hub.id}")
attrs = %{file_system: Livebook.FileSystem.dump(file_system)}
expect_s3_listing(bypass)
view
|> element("#hub-file-system-#{file_system.id}-edit")
|> render_click(%{"file_system" => file_system})
assert_patch(view, ~p"/hub/#{hub.id}/file-systems/edit/#{file_system.id}")
assert render(view) =~ "Edit file storage"
view
|> element("#file-systems-form")
|> render_change(attrs)
refute view
|> element("#file-systems-form button[disabled]")
|> has_element?()
view
|> element("#file-systems-form")
|> render_submit(put_in(attrs.file_system.access_key_id, "new key"))
updated_file_system = %{file_system | access_key_id: "new key"}
assert_receive {:file_system_updated, ^updated_file_system}
%{"success" => "File storage updated successfully"} =
assert_redirect(view, "/hub/#{hub.id}")
{:ok, view, _html} = live(conn, ~p"/hub/#{hub.id}")
assert render(element(view, "#hub-file-systems-list")) =~ file_system.bucket_url
assert updated_file_system in Livebook.Hubs.get_file_systems(hub)
end
test "detaches existing file system", %{conn: conn, hub: hub} do
bypass = Bypass.open()
file_system = build_bypass_file_system(bypass, hub.id)
id = file_system.id
:ok = Hubs.create_file_system(hub, file_system)
assert_receive {:file_system_created, %{id: ^id} = file_system}
{:ok, view, _html} = live(conn, ~p"/hub/#{hub.id}")
refute view
|> element("#file-systems-form button[disabled]")
|> has_element?()
view
|> element("#hub-file-system-#{file_system.id}-detach", "Detach")
|> render_click()
render_confirm(view)
assert_receive {:file_system_deleted, ^file_system}
%{"success" => "File storage deleted successfully"} =
assert_redirect(view, "/hub/#{hub.id}")
{:ok, view, _html} = live(conn, ~p"/hub/#{hub.id}")
refute render(element(view, "#hub-file-systems-list")) =~ file_system.bucket_url
refute file_system in Livebook.Hubs.get_file_systems(hub)
end
end
defp expect_s3_listing(bypass) do
Bypass.expect_once(bypass, "GET", "/", fn conn ->
conn
|> Plug.Conn.put_resp_content_type("application/xml")
|> Plug.Conn.resp(200, """
<ListBucketResult>
<Name>mybucket</Name>
</ListBucketResult>
""")
end)
end
end

55
test/livebook_teams/web/session_live_test.exs
View file

@ -278,4 +278,59 @@ defmodule LivebookWeb.Integration.SessionLiveTest do
assert output == "\e[32m\"#{secret.value}\"\e[0m"
end
end
describe "files" do
test "shows only hub's file systems",
%{conn: conn, user: user, node: node, session: session} do
Session.subscribe(session.id)
Livebook.Hubs.subscribe([:file_systems])
personal_id = Livebook.Hubs.Personal.id()
personal_file_system = build(:fs_s3)
Livebook.Hubs.Personal.save_file_system(personal_file_system)
team = create_team_hub(user, node)
team_id = team.id
bucket_url = "https://my-own-bucket.s3.amazonaws.com"
file_system =
build(:fs_s3,
id: Livebook.FileSystem.S3.id(team_id, bucket_url),
bucket_url: bucket_url,
hub_id: team_id
)
Livebook.Hubs.create_file_system(team, file_system)
assert_receive {:file_system_created, team_file_system}
# loads the session page
{:ok, view, _html} = live(conn, ~p"/sessions/#{session.id}/add-file/storage")
# change the hub to Personal
# and checks the file systems from Personal
Session.set_notebook_hub(session.pid, personal_id)
assert_receive {:operation, {:set_notebook_hub, _client, ^personal_id}}
# targets the file system dropdown menu
file_system_menu = with_target(view, "#add-file-entry-select #file-system-menu-content")
# checks the file systems from Personal
assert has_element?(file_system_menu, "#file-system-local")
assert has_element?(file_system_menu, "#file-system-#{personal_file_system.id}")
refute has_element?(file_system_menu, "#file-system-#{team_file_system.id}")
# change the hub to Team
# and checks the file systems from Team
Session.set_notebook_hub(session.pid, team.id)
assert_receive {:operation, {:set_notebook_hub, _client, ^team_id}}
# targets the file system dropdown menu
file_system_menu = with_target(view, "#file-system-menu")
assert has_element?(file_system_menu, "#file-system-local")
refute has_element?(file_system_menu, "#file-system-#{personal_file_system.id}")
assert has_element?(file_system_menu, "#file-system-#{team_file_system.id}")
end
end
end

128
test/livebook_web/live/hub/edit_live_test.exs
View file

@ -9,7 +9,7 @@ defmodule LivebookWeb.Hub.EditLiveTest do
describe "personal" do
setup do
Livebook.Hubs.subscribe([:crud, :secrets])
Livebook.Hubs.subscribe([:crud, :secrets, :file_systems])
{:ok, hub: Hubs.fetch_hub!(Hubs.Personal.id())}
end
@ -82,7 +82,7 @@ defmodule LivebookWeb.Hub.EditLiveTest do
assert_receive {:secret_created, ^secret}
%{"success" => "Secret PERSONAL_ADD_SECRET created successfully"} =
%{"success" => "Secret PERSONAL_ADD_SECRET added successfully"} =
assert_redirect(view, "/hub/#{hub.id}")
{:ok, view, _html} = live(conn, ~p"/hub/#{hub.id}")
@ -161,5 +161,129 @@ defmodule LivebookWeb.Hub.EditLiveTest do
refute render(element(view, "#hub-secrets-list")) =~ secret.name
refute secret in Livebook.Hubs.get_secrets(hub)
end
test "creates file system", %{conn: conn, hub: hub} do
{:ok, view, _html} = live(conn, ~p"/hub/#{hub.id}")
bypass = Bypass.open()
file_system = build_bypass_file_system(bypass)
attrs = %{file_system: Livebook.FileSystem.dump(file_system)}
expect_s3_listing(bypass)
refute render(view) =~ file_system.bucket_url
view
|> element("#add-file-system")
|> render_click(%{})
assert_patch(view, ~p"/hub/#{hub.id}/file-systems/new")
assert render(view) =~ "Add file storage"
view
|> element("#file-systems-form")
|> render_change(attrs)
refute view
|> element("#file-systems-form button[disabled]")
|> has_element?()
view
|> element("#file-systems-form")
|> render_submit(attrs)
assert_receive {:file_system_created, ^file_system}
%{"success" => "File storage added successfully"} =
assert_redirect(view, "/hub/#{hub.id}")
{:ok, view, _html} = live(conn, ~p"/hub/#{hub.id}")
assert render(element(view, "#hub-file-systems-list")) =~ file_system.bucket_url
assert file_system in Livebook.Hubs.get_file_systems(hub)
end
test "updates file system", %{conn: conn, hub: hub} do
bypass = Bypass.open()
file_system = build_bypass_file_system(bypass)
:ok = Hubs.create_file_system(hub, file_system)
{:ok, view, _html} = live(conn, ~p"/hub/#{hub.id}")
attrs = %{file_system: Livebook.FileSystem.dump(file_system)}
expect_s3_listing(bypass)
view
|> element("#hub-file-system-#{file_system.id}-edit")
|> render_click(%{"file_system" => file_system})
assert_patch(view, ~p"/hub/#{hub.id}/file-systems/edit/#{file_system.id}")
assert render(view) =~ "Edit file storage"
view
|> element("#file-systems-form")
|> render_change(attrs)
refute view
|> element("#file-systems-form button[disabled]")
|> has_element?()
view
|> element("#file-systems-form")
|> render_submit(put_in(attrs.file_system.access_key_id, "new key"))
updated_file_system = %{file_system | access_key_id: "new key"}
assert_receive {:file_system_updated, ^updated_file_system}
%{"success" => "File storage updated successfully"} =
assert_redirect(view, "/hub/#{hub.id}")
{:ok, view, _html} = live(conn, ~p"/hub/#{hub.id}")
assert render(element(view, "#hub-file-systems-list")) =~ file_system.bucket_url
assert updated_file_system in Livebook.Hubs.get_file_systems(hub)
end
test "detaches file system", %{conn: conn, hub: hub} do
bypass = Bypass.open()
file_system = build_bypass_file_system(bypass)
:ok = Hubs.create_file_system(hub, file_system)
{:ok, view, _html} = live(conn, ~p"/hub/#{hub.id}")
refute view
|> element("#file-systems-form button[disabled]")
|> has_element?()
view
|> element("#hub-file-system-#{file_system.id}-detach", "Detach")
|> render_click()
render_confirm(view)
assert_receive {:file_system_deleted, ^file_system}
%{"success" => "File storage deleted successfully"} =
assert_redirect(view, "/hub/#{hub.id}")
{:ok, view, _html} = live(conn, ~p"/hub/#{hub.id}")
refute render(element(view, "#hub-file-systems-list")) =~ file_system.bucket_url
refute file_system in Livebook.Hubs.get_file_systems(hub)
end
end
defp expect_s3_listing(bypass) do
Bypass.expect_once(bypass, "GET", "/", fn conn ->
conn
|> Plug.Conn.put_resp_content_type("application/xml")
|> Plug.Conn.resp(200, """
<ListBucketResult>
<Name>mybucket</Name>
</ListBucketResult>
""")
end)
end
end

9
test/support/factory.ex
View file

@ -68,15 +68,16 @@ defmodule Livebook.Factory do
def build(:fs_s3) do
bucket_url = "https://mybucket.s3.amazonaws.com"
hash = :crypto.hash(:sha256, bucket_url)
id = "s3-#{Base.url_encode64(hash, padding: false)}"
hub_id = Livebook.Hubs.Personal.id()
%Livebook.FileSystem.S3{
id: id,
id: "#{hub_id}-s3-#{Base.url_encode64(hash, padding: false)}",
bucket_url: bucket_url,
external_id: id,
external_id: nil,
region: "us-east-1",
access_key_id: "key",
secret_access_key: "secret"
secret_access_key: "secret",
hub_id: hub_id
}
end

19
test/support/hub_helpers.ex
View file

@ -100,6 +100,25 @@ defmodule Livebook.HubHelpers do
send(pid, {:event, :secret_deleted, secret_deleted})
end
def create_teams_file_system(hub, node) do
org_key = erpc_call(node, :get_org_key!, [hub.org_key_id])
erpc_call(node, :create_file_system, [[org_key: org_key]])
end
def build_bypass_file_system(bypass, hub_id \\ nil) do
bucket_url = "http://localhost:#{bypass.port}"
file_system =
build(:fs_s3,
id: Livebook.FileSystem.S3.id(hub_id, bucket_url),
bucket_url: bucket_url,
region: "auto",
hub_id: hub_id
)
file_system
end
defp hub_pid(hub) do
if pid = GenServer.whereis({:via, Registry, {Livebook.HubsRegistry, hub.id}}) do
{:ok, pid}