Implements Secrets features inside hub's Provider (#1712)

lib/livebook/application.ex

@@ -190,7 +190,7 @@ defmodule Livebook.Application do
         %Livebook.Secrets.Secret{name: name, value: value, origin: :startup}
       end
 
-    Livebook.Secrets.set_temporary_secrets(secrets)
+    Livebook.Secrets.set_startup_secrets(secrets)
   end
 
   defp config_env_var?("LIVEBOOK_" <> _), do: true

lib/livebook/hubs.ex

@@ -3,7 +3,6 @@ defmodule Livebook.Hubs do
 
   alias Livebook.Storage
   alias Livebook.Hubs.{Broadcasts, Enterprise, Fly, Metadata, Personal, Provider}
-  alias Livebook.Secrets
   alias Livebook.Secrets.Secret
 
   @namespace :hubs
@@ -181,9 +180,7 @@ defmodule Livebook.Hubs do
   """
   @spec connect_hubs() :: :ok
   def connect_hubs do
-    for hub <- get_hubs(),
+    for hub <- get_hubs([:connect]), do: connect_hub(hub)
-        capability?(hub, [:connect]),
-        do: connect_hub(hub)
 
     :ok
   end
@@ -211,19 +208,34 @@ defmodule Livebook.Hubs do
   @doc """
   Creates a secret for given hub.
   """
-  @spec create_secret(Secret.t()) :: :ok | {:error, list({String.t(), list(String.t())})}
+  @spec create_secret(Secret.t()) :: :ok | {:error, list({atom(), list(String.t())})}
   def create_secret(%Secret{origin: {:hub, id}} = secret) do
-    case get_hub(id) do
+    {:ok, hub} = get_hub(id)
-      {:ok, hub} ->
+    true = capability?(hub, [:create_secret])
-        if capability?(hub, [:secrets]) do
-          Provider.create_secret(hub, secret)
-        else
-          {:error, Secrets.add_secret_error(secret, :origin, "is invalid")}
-        end
 
-      :error ->
+    Provider.create_secret(hub, secret)
-        {:error, Secrets.add_secret_error(secret, :origin, "is invalid")}
+  end
-    end
+
+  @doc """
+  Updates a secret for given hub.
+  """
+  @spec update_secret(Secret.t()) :: :ok | {:error, list({atom(), list(String.t())})}
+  def update_secret(%Secret{origin: {:hub, id}} = secret) do
+    {:ok, hub} = get_hub(id)
+    true = capability?(hub, [:update_secret])
+
+    Provider.update_secret(hub, secret)
+  end
+
+  @doc """
+  Deletes a secret for given hub.
+  """
+  @spec delete_secret(Secret.t()) :: :ok | {:error, list({atom(), list(String.t())})}
+  def delete_secret(%Secret{origin: {:hub, id}} = secret) do
+    {:ok, hub} = get_hub(id)
+    true = capability?(hub, [:delete_secret])
+
+    Provider.delete_secret(hub, secret)
   end
 
   defp capability?(hub, capabilities) do

lib/livebook/hubs/broadcasts.ex

@@ -10,7 +10,7 @@ defmodule Livebook.Hubs.Broadcasts do
   @secrets_topic "hubs:secrets"
 
   @doc """
-  Broadcasts when hubs changed under `hubs:crud` topic
+  Broadcasts under `hubs:crud` topic when hubs changed.
   """
   @spec hub_changed() :: broadcast()
   def hub_changed do
@@ -18,7 +18,7 @@ defmodule Livebook.Hubs.Broadcasts do
   end
 
   @doc """
-  Broadcasts when hub connected under `hubs:connection` topic
+  Broadcasts under `hubs:connection` topic when hub connected.
   """
   @spec hub_connected() :: broadcast()
   def hub_connected do
@@ -26,7 +26,7 @@ defmodule Livebook.Hubs.Broadcasts do
   end
 
   @doc """
-  Broadcasts when hub disconnected under `hubs:connection` topic
+  Broadcasts under `hubs:connection` topic when hub disconnected.
   """
   @spec hub_disconnected() :: broadcast()
   def hub_disconnected do
@@ -34,7 +34,7 @@ defmodule Livebook.Hubs.Broadcasts do
   end
 
   @doc """
-  Broadcasts when hub had an error when connecting under `hubs:connection` topic
+  Broadcasts under `hubs:connection` topic when hub received a connection error.
   """
   @spec hub_connection_failed(String.t()) :: broadcast()
   def hub_connection_failed(reason) when is_binary(reason) do
@@ -42,15 +42,7 @@ defmodule Livebook.Hubs.Broadcasts do
   end
 
   @doc """
-  Broadcasts when hub had an error when disconnecting under `hubs:connection` topic
+  Broadcasts under `hubs:secrets` topic when hub received a new secret.
-  """
-  @spec hub_disconnection_failed(String.t()) :: broadcast()
-  def hub_disconnection_failed(reason) when is_binary(reason) do
-    broadcast(@connection_topic, {:hub_disconnection_failed, reason})
-  end
-
-  @doc """
-  Broadcasts when hub received a new secret under `hubs:secrets` topic
   """
   @spec secret_created(Secret.t()) :: broadcast()
   def secret_created(%Secret{} = secret) do
@@ -58,13 +50,21 @@ defmodule Livebook.Hubs.Broadcasts do
   end
 
   @doc """
-  Broadcasts when hub received an updated secret under `hubs:secrets` topic
+  Broadcasts under `hubs:secrets` topic when hub received an updated secret.
   """
   @spec secret_updated(Secret.t()) :: broadcast()
   def secret_updated(%Secret{} = secret) do
     broadcast(@secrets_topic, {:secret_updated, secret})
   end
 
+  @doc """
+  Broadcasts under `hubs:secrets` topic when hub received a deleted secret.
+  """
+  @spec secret_deleted(Secret.t()) :: broadcast()
+  def secret_deleted(%Secret{} = secret) do
+    broadcast(@secrets_topic, {:secret_deleted, secret})
+  end
+
   defp broadcast(topic, message) do
     Phoenix.PubSub.broadcast(Livebook.PubSub, topic, message)
   end

lib/livebook/hubs/enterprise.ex

@@ -145,17 +145,16 @@ defimpl Livebook.Hubs.Provider, for: Livebook.Hubs.Enterprise do
     EnterpriseClient.stop(enterprise.id)
   end
 
-  def capabilities(_enterprise), do: [:connect, :secrets]
+  def capabilities(_enterprise), do: ~w(connect secrets list_secrets create_secret)a
 
   def get_secrets(enterprise) do
     EnterpriseClient.get_secrets(enterprise.id)
   end
 
   def create_secret(enterprise, secret) do
-    create_secret_request =
+    data = LivebookProto.build_create_secret_request(name: secret.name, value: secret.value)
-      LivebookProto.CreateSecretRequest.new!(name: secret.name, value: secret.value)
 
-    case EnterpriseClient.send_request(enterprise.id, create_secret_request) do
+    case EnterpriseClient.send_request(enterprise.id, data) do
       {:create_secret, _} ->
         :ok
 
@@ -178,6 +177,10 @@ defimpl Livebook.Hubs.Provider, for: Livebook.Hubs.Enterprise do
     end
   end
 
+  def update_secret(_enterprise, _secret), do: raise("not implemented")
+
+  def delete_secret(_enterprise, _secret), do: raise("not implemented")
+
   def connection_error(enterprise) do
     reason = EnterpriseClient.get_connection_error(enterprise.id)
     "Cannot connect to Hub: #{reason}. Will attempt to reconnect automatically..."

lib/livebook/hubs/enterprise_client.ex

@@ -1,6 +1,7 @@
 defmodule Livebook.Hubs.EnterpriseClient do
   @moduledoc false
   use GenServer
+  require Logger
 
   alias Livebook.Hubs.Broadcasts
   alias Livebook.Hubs.Enterprise
@@ -86,6 +87,14 @@ defmodule Livebook.Hubs.EnterpriseClient do
     {:ok, %__MODULE__{hub: enterprise, server: pid}}
   end
 
+  @impl true
+  def handle_continue(:synchronize_user, state) do
+    data = LivebookProto.build_handshake_request(app_version: Livebook.Config.app_version())
+    {:handshake, _} = ClientConnection.send_request(state.server, data)
+
+    {:noreply, state}
+  end
+
   @impl true
   def handle_call(:fetch_server, _caller, state) do
     if state.connected? do
@@ -110,7 +119,7 @@ defmodule Livebook.Hubs.EnterpriseClient do
   @impl true
   def handle_info({:connect, :ok, _}, state) do
     Broadcasts.hub_connected()
-    {:noreply, %{state | connected?: true, connection_error: nil}}
+    {:noreply, %{state | connected?: true, connection_error: nil}, {:continue, :synchronize_user}}
   end
 
   def handle_info({:connect, :error, reason}, state) do
@@ -118,25 +127,17 @@ defmodule Livebook.Hubs.EnterpriseClient do
     {:noreply, %{state | connected?: false, connection_error: reason}}
   end
 
-  def handle_info({:event, :secret_created, %{name: name, value: value}}, state) do
-    secret = %Secret{name: name, value: value, origin: {:hub, state.hub.id}}
-    Broadcasts.secret_created(secret)
-
-    {:noreply, put_secret(state, secret)}
-  end
-
-  def handle_info({:event, :secret_updated, %{name: name, value: value}}, state) do
-    secret = %Secret{name: name, value: value, origin: {:hub, state.hub.id}}
-    Broadcasts.secret_updated(secret)
-
-    {:noreply, put_secret(state, secret)}
-  end
-
   def handle_info({:disconnect, :ok, :disconnected}, state) do
     Broadcasts.hub_disconnected()
     {:stop, :normal, state}
   end
 
+  def handle_info({:event, topic, data}, state) do
+    Logger.debug("Received event #{topic} with data: #{inspect(data)}")
+
+    {:noreply, handle_event(topic, data, state)}
+  end
+
   # Private
 
   defp registry_name(id) do
@@ -144,6 +145,79 @@ defmodule Livebook.Hubs.EnterpriseClient do
   end
 
   defp put_secret(state, secret) do
-    %{state | secrets: [secret | Enum.reject(state.secrets, &(&1.name == secret.name))]}
+    state = remove_secret(state, secret)
+    %{state | secrets: [secret | state.secrets]}
+  end
+
+  defp remove_secret(state, secret) do
+    %{state | secrets: Enum.reject(state.secrets, &(&1.name == secret.name))}
+  end
+
+  defp build_secret(state, %{name: name, value: value}),
+    do: %Secret{name: name, value: value, origin: {:hub, state.hub.id}}
+
+  defp update_hub(state, name) do
+    case Enterprise.update_hub(state.hub, %{hub_name: name}) do
+      {:ok, hub} -> %{state | hub: hub}
+      {:error, _} -> state
+    end
+  end
+
+  defp handle_event(:secret_created, secret_created, state) do
+    secret = build_secret(state, secret_created)
+    Broadcasts.secret_created(secret)
+
+    put_secret(state, secret)
+  end
+
+  defp handle_event(:secret_updated, secret_updated, state) do
+    secret = build_secret(state, secret_updated)
+    Broadcasts.secret_updated(secret)
+
+    put_secret(state, secret)
+  end
+
+  defp handle_event(:secret_deleted, secret_deleted, state) do
+    secret = build_secret(state, secret_deleted)
+    Broadcasts.secret_deleted(secret)
+
+    remove_secret(state, secret)
+  end
+
+  defp handle_event(:user_synchronized, user_synchronized, %{secrets: []} = state) do
+    state = update_hub(state, user_synchronized.name)
+    secrets = for secret <- user_synchronized.secrets, do: build_secret(state, secret)
+
+    %{state | secrets: secrets}
+  end
+
+  defp handle_event(:user_synchronized, user_synchronized, state) do
+    state = update_hub(state, user_synchronized.name)
+    secrets = for secret <- user_synchronized.secrets, do: build_secret(state, secret)
+
+    created_secrets =
+      Enum.reject(secrets, fn secret ->
+        Enum.find(state.secrets, &(&1.name == secret.name and &1.value == secret.value))
+      end)
+
+    deleted_secrets =
+      Enum.reject(state.secrets, fn secret ->
+        Enum.find(secrets, &(&1.name == secret.name))
+      end)
+
+    updated_secrets =
+      Enum.filter(secrets, fn secret ->
+        Enum.find(state.secrets, &(&1.name == secret.name and &1.value != secret.value))
+      end)
+
+    events_by_type = [
+      secret_deleted: deleted_secrets,
+      secret_created: created_secrets,
+      secret_updated: updated_secrets
+    ]
+
+    for {type, events} <- events_by_type, event <- events, reduce: state do
+      state -> handle_event(type, event, state)
+    end
   end
 end

lib/livebook/hubs/fly.ex

@@ -146,13 +146,20 @@ defimpl Livebook.Hubs.Provider, for: Livebook.Hubs.Fly do
 
   def connection_spec(_fly), do: nil
 
-  def disconnect(_fly), do: :ok
+  def disconnect(_fly), do: raise("not implemented")
 
   def capabilities(_fly), do: []
 
   def get_secrets(_fly), do: []
 
+  # TODO: Implement the FlyClient.set_secrets/2
   def create_secret(_fly, _secret), do: :ok
 
-  def connection_error(_fly), do: nil
+  # TODO: Implement the FlyClient.set_secrets/2
+  def update_secret(_fly, _secret), do: :ok
+
+  # TODO: Implement the FlyClient.unset_secrets/2
+  def delete_secret(_fly, _secret), do: :ok
+
+  def connection_error(_fly), do: raise("not implemented")
 end

lib/livebook/hubs/personal.ex

@@ -62,6 +62,9 @@ defmodule Livebook.Hubs.Personal do
 end
 
 defimpl Livebook.Hubs.Provider, for: Livebook.Hubs.Personal do
+  alias Livebook.Hubs.Broadcasts
+  alias Livebook.Secrets
+
   def load(personal, fields) do
     %{personal | id: fields.id, hub_name: fields.hub_name, hub_emoji: fields.hub_emoji}
   end
@@ -80,13 +83,28 @@ defimpl Livebook.Hubs.Provider, for: Livebook.Hubs.Personal do
 
   def connection_spec(_personal), do: nil
 
-  def disconnect(_personal), do: :ok
+  def disconnect(_personal), do: raise("not implemented")
 
-  def capabilities(_personal), do: []
+  def capabilities(_personal), do: ~w(list_secrets create_secret update_secret delete_secret)a
 
-  def get_secrets(_personal), do: []
+  def get_secrets(_personal) do
+    Secrets.get_secrets()
+  end
 
-  def create_secret(_personal, _secret), do: :ok
+  def create_secret(_personal, secret) do
+    Secrets.set_secret(secret)
+    :ok = Broadcasts.secret_created(secret)
+  end
 
-  def connection_error(_personal), do: nil
+  def update_secret(_personal, secret) do
+    Secrets.set_secret(secret)
+    :ok = Broadcasts.secret_updated(secret)
+  end
+
+  def delete_secret(_personal, secret) do
+    :ok = Secrets.unset_secret(secret.name)
+    :ok = Broadcasts.secret_deleted(secret)
+  end
+
+  def connection_error(_personal), do: raise("not implemented")
 end

lib/livebook/hubs/provider.ex

@@ -3,8 +3,9 @@ defprotocol Livebook.Hubs.Provider do
 
   alias Livebook.Secrets.Secret
 
-  @type t :: Livebook.Hubs.Enterprise.t() | Livebook.Hubs.Fly.t() | Livebook.Hubs.Local.t()
+  @type t :: Livebook.Hubs.Enterprise.t() | Livebook.Hubs.Fly.t() | Livebook.Hubs.Personal.t()
-  @type capability :: :connect | :secrets
+  @type capability ::
+          :connect | :secrets | :list_secrets | :create_secret | :update_secret | :delete_secret
   @type capabilities :: list(capability())
   @type changeset_errors :: %{required(:errors) => list({String.t(), {Stirng.t(), list()}})}
 
@@ -51,11 +52,23 @@ defprotocol Livebook.Hubs.Provider do
   def get_secrets(hub)
 
   @doc """
-  Creates a secret of  the given hub.
+  Creates a secret of the given hub.
   """
   @spec create_secret(t(), Secret.t()) :: :ok | {:error, changeset_errors()}
   def create_secret(hub, secret)
 
+  @doc """
+  Updates a secret of the given hub.
+  """
+  @spec update_secret(t(), Secret.t()) :: :ok | {:error, changeset_errors()}
+  def update_secret(hub, secret)
+
+  @doc """
+  Deletes a secret of the given hub.
+  """
+  @spec delete_secret(t(), Secret.t()) :: :ok | {:error, changeset_errors()}
+  def delete_secret(hub, secret)
+
   @doc """
   Gets the connection error from hub.
   """

lib/livebook/secrets.ex

@@ -4,19 +4,17 @@ defmodule Livebook.Secrets do
   alias Livebook.Storage
   alias Livebook.Secrets.Secret
 
-  @temporary_key :livebook_temporary_secrets
+  @secret_startup_key :livebook_startup_secrets
 
   @doc """
   Get the secrets list from storage.
   """
   @spec get_secrets() :: list(Secret.t())
   def get_secrets do
-    temporary_secrets = :persistent_term.get(@temporary_key, [])
+    startup_secrets = :persistent_term.get(@secret_startup_key, [])
+    storage_secrets = for fields <- Storage.all(:secrets), do: to_struct(fields)
 
-    for fields <- Storage.all(:secrets) do
+    Enum.concat(storage_secrets, startup_secrets)
-      to_struct(fields)
-    end
-    |> Enum.concat(temporary_secrets)
   end
 
   @doc """
@@ -109,9 +107,9 @@ defmodule Livebook.Secrets do
   @doc """
   Sets additional secrets that are kept only in memory.
   """
-  @spec set_temporary_secrets(list(Secret.t())) :: :ok
+  @spec set_startup_secrets(list(Secret.t())) :: :ok
-  def set_temporary_secrets(secrets) do
+  def set_startup_secrets(secrets) do
-    :persistent_term.put(@temporary_key, secrets)
+    :persistent_term.put(@secret_startup_key, secrets)
   end
 
   @doc """

lib/livebook_web/live/hub/edit/enterprise_component.ex

@@ -31,7 +31,6 @@ defmodule LivebookWeb.Hub.Edit.EnterpriseComponent do
             phx-submit="save"
             phx-change="validate"
             phx-target={@myself}
-            phx-debounce="blur"
           >
             <div class="grid grid-cols-1 md:grid-cols-1 gap-3">
               <.emoji_field field={f[:hub_emoji]} label="Emoji" />

lib/livebook_web/live/hub/edit/fly_component.ex

@@ -67,7 +67,6 @@ defmodule LivebookWeb.Hub.Edit.FlyComponent do
             phx-submit="save"
             phx-change="validate"
             phx-target={@myself}
-            phx-debounce="blur"
           >
             <div class="grid grid-cols-1 md:grid-cols-2 gap-3">
               <.text_field field={f[:hub_name]} label="Name" />

lib/livebook_web/live/hub/edit/personal_component.ex

@@ -31,7 +31,6 @@ defmodule LivebookWeb.Hub.Edit.PersonalComponent do
             phx-submit="save"
             phx-change="validate"
             phx-target={@myself}
-            phx-debounce="blur"
           >
             <div class="grid grid-cols-1 md:grid-cols-2 gap-3">
               <.text_field field={f[:hub_name]} label="Name" />

lib/livebook_web/live/hub/new/enterprise_component.ex

@@ -117,12 +117,11 @@ defmodule LivebookWeb.Hub.New.EnterpriseComponent do
          |> push_patch(to: ~p"/hub")}
 
       :hub_connected ->
-        session_request =
+        data = LivebookProto.build_handshake_request(app_version: Livebook.Config.app_version())
-          LivebookProto.SessionRequest.new!(app_version: Livebook.Config.app_version())
 
-        case EnterpriseClient.send_request(pid, session_request) do
+        case EnterpriseClient.send_request(pid, data) do
-          {:session, session_response} ->
+          {:handshake, handshake_response} ->
-            base = %{base | external_id: session_response.id}
+            base = %{base | external_id: handshake_response.id}
             changeset = Enterprise.validate_hub(base)
 
             {:noreply, assign(socket, pid: pid, changeset: changeset, base: base)}

lib/livebook_web/live/hub/new/fly_component.ex

@@ -31,7 +31,6 @@ defmodule LivebookWeb.Hub.New.FlyComponent do
         phx-submit="save"
         phx-change="validate"
         phx-target={@myself}
-        phx-debounce="blur"
       >
         <.password_field
           type="password"

lib/livebook_web/live/session_live/secrets_component.ex

@@ -109,6 +109,7 @@ defmodule LivebookWeb.SessionLive.SecretsComponent do
             />
             <.radio_field
               field={f[:origin]}
+              value={SecretOrigin.encode(f[:origin].value)}
               label="Storage"
               options={
                 [{"session", "only this session"}, {"app", "in the Livebook app"}] ++

lib/livebook_web/live/session_live/secrets_list_component.ex

@@ -111,6 +111,7 @@ defmodule LivebookWeb.SessionLive.SecretsListComponent do
             prefix={"hub-#{id}"}
             data_secrets={@secrets}
             hubs={@hubs}
+            myself={@myself}
           />
         </div>
       </div>

proto/lib/livebook_proto.ex

@@ -1,7 +1,14 @@
 defmodule LivebookProto do
   @moduledoc false
 
-  alias LivebookProto.{Request, Response}
+  alias LivebookProto.{
+    CreateSecretRequest,
+    CreateSecretResponse,
+    HandshakeRequest,
+    HandshakeResponse,
+    Request,
+    Response
+  }
 
   @request_mapping (for {_id, field_prop} <- Request.__message_props__().field_props,
                         into: %{} do
@@ -13,6 +20,13 @@ defmodule LivebookProto do
                        {field_prop.type, field_prop.name_atom}
                      end)
 
+  @type request_proto :: HandshakeRequest.t() | CreateSecretRequest.t()
+  @type response_proto :: HandshakeResponse.t() | CreateSecretResponse.t()
+
+  @doc """
+  Builds a request frame with given data and id.
+  """
+  @spec build_request_frame(request_proto(), integer()) :: {:binary, iodata()}
   def build_request_frame(%struct{} = data, id \\ -1) do
     type = request_type(struct)
     message = Request.new!(id: id, type: {type, data})
@@ -20,6 +34,22 @@ defmodule LivebookProto do
     {:binary, Request.encode(message)}
   end
 
+  @doc """
+  Builds a create secret request struct.
+  """
+  @spec build_create_secret_request(keyword()) :: CreateSecretRequest.t()
+  defdelegate build_create_secret_request(fields), to: CreateSecretRequest, as: :new!
+
+  @doc """
+  Builds a handshake request struct.
+  """
+  @spec build_handshake_request(keyword()) :: HandshakeRequest.t()
+  defdelegate build_handshake_request(fields), to: HandshakeRequest, as: :new!
+
+  @doc """
+  Builds a response with given data and id.
+  """
+  @spec build_response(response_proto(), integer()) :: Response.t()
   def build_response(%struct{} = data, id \\ -1) do
     type = response_type(struct)
     Response.new!(id: id, type: {type, data})

proto/lib/livebook_proto/event.pb.ex

@@ -13,4 +13,14 @@ defmodule LivebookProto.Event do
     type: LivebookProto.SecretUpdated,
     json_name: "secretUpdated",
     oneof: 0
+
+  field :secret_deleted, 102,
+    type: LivebookProto.SecretDeleted,
+    json_name: "secretDeleted",
+    oneof: 0
+
+  field :user_synchronized, 103,
+    type: LivebookProto.UserSynchronized,
+    json_name: "userSynchronized",
+    oneof: 0
 end

proto/lib/livebook_proto/handshake_request.pb.ex

@@ -1,4 +1,4 @@
-defmodule LivebookProto.SessionRequest do
+defmodule LivebookProto.HandshakeRequest do
   @moduledoc false
   use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
 

proto/lib/livebook_proto/handshake_response.pb.ex

@@ -1,7 +1,8 @@
-defmodule LivebookProto.SessionResponse do
+defmodule LivebookProto.HandshakeResponse do
   @moduledoc false
   use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
 
   field :id, 1, type: :string
-  field :user, 2, type: LivebookProto.User
+  field :name, 2, type: :string
+  field :user, 3, type: LivebookProto.User
 end

proto/lib/livebook_proto/request.pb.ex

@@ -5,7 +5,7 @@ defmodule LivebookProto.Request do
   oneof :type, 0
 
   field :id, 1, type: :int32
-  field :session, 2, type: LivebookProto.SessionRequest, oneof: 0
+  field :handshake, 2, type: LivebookProto.HandshakeRequest, oneof: 0
 
   field :create_secret, 3,
     type: LivebookProto.CreateSecretRequest,

proto/lib/livebook_proto/response.pb.ex

@@ -7,7 +7,7 @@ defmodule LivebookProto.Response do
   field :id, 1, type: :int32
   field :error, 2, type: LivebookProto.Error, oneof: 0
   field :changeset, 3, type: LivebookProto.ChangesetError, oneof: 0
-  field :session, 4, type: LivebookProto.SessionResponse, oneof: 0
+  field :handshake, 4, type: LivebookProto.HandshakeResponse, oneof: 0
 
   field :create_secret, 5,
     type: LivebookProto.CreateSecretResponse,

proto/lib/livebook_proto/secret.pb.ex

@@ -0,0 +1,7 @@
+defmodule LivebookProto.Secret do
+  @moduledoc false
+  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  field :name, 1, type: :string
+  field :value, 2, type: :string
+end

proto/lib/livebook_proto/secret_deleted.pb.ex

@@ -0,0 +1,7 @@
+defmodule LivebookProto.SecretDeleted do
+  @moduledoc false
+  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  field :name, 1, type: :string
+  field :value, 2, type: :string
+end

proto/lib/livebook_proto/user_synchronized.pb.ex

@@ -0,0 +1,8 @@
+defmodule LivebookProto.UserSynchronized do
+  @moduledoc false
+  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  field :id, 1, type: :string
+  field :name, 2, type: :string
+  field :secrets, 3, repeated: true, type: LivebookProto.Secret
+end

proto/messages.proto

@@ -5,6 +5,11 @@ message User {
   string email = 2;
 }
 
+message Secret {
+  string name = 1;
+  string value = 2;
+}
+
 message Error {
   string details = 1;
 }
@@ -28,13 +33,25 @@ message SecretUpdated {
   string value = 2;
 }
 
-message SessionRequest {
+message SecretDeleted {
+  string name = 1;
+  string value = 2;
+}
+
+message UserSynchronized {
+  string id = 1;
+  string name = 2;
+  repeated Secret secrets = 3;
+}
+
+message HandshakeRequest {
   string app_version = 1;
 }
 
-message SessionResponse {
+message HandshakeResponse {
   string id = 1;
-  User user = 2;
+  string name = 2;
+  User user = 3;
 }
 
 message CreateSecretRequest {
@@ -49,7 +66,7 @@ message Request {
   int32 id = 1;
 
   oneof type {
-    SessionRequest session = 2;
+    HandshakeRequest handshake = 2;
     CreateSecretRequest create_secret = 3;
   }
 }
@@ -61,7 +78,7 @@ message Response {
     Error error = 2;
     ChangesetError changeset = 3;
 
-    SessionResponse session = 4;
+    HandshakeResponse handshake = 4;
     CreateSecretResponse create_secret = 5;
   }
 }
@@ -70,5 +87,7 @@ message Event {
   oneof type {
     SecretCreated secret_created = 100;
     SecretUpdated secret_updated = 101;
+    SecretDeleted secret_deleted = 102;
+    UserSynchronized user_synchronized = 103;
   }
 }

test/livebook/hubs/enterprise_client_test.exs

@@ -55,22 +55,45 @@ defmodule Livebook.Hubs.EnterpriseClientTest do
       name = "API_TOKEN_ID"
       value = Livebook.Utils.random_id()
       :erpc.call(node, Enterprise.Integration, :create_secret, [name, value])
+      secret = %Secret{name: name, value: value, origin: {:hub, id}}
 
-      assert_receive {:secret_created, %Secret{name: ^name, value: ^value, origin: {:hub, ^id}}}
+      assert_receive {:secret_created, ^secret}
+      assert secret in EnterpriseClient.get_secrets(id)
     end
 
     test "receives a secret_updated event", %{node: node, hub_id: id} do
       name = "SUPER_SUDO_USER"
       value = "JakePeralta"
-      secret = :erpc.call(node, Enterprise.Integration, :create_secret, [name, value])
-
-      assert_receive {:secret_created, %Secret{name: ^name, value: ^value, origin: {:hub, ^id}}}
-
       new_value = "ChonkyCat"
-      :erpc.call(node, Enterprise.Integration, :update_secret, [secret, new_value])
+      enterprise_secret = :erpc.call(node, Enterprise.Integration, :create_secret, [name, value])
+      secret = %Secret{name: name, value: value, origin: {:hub, id}}
+      updated_secret = %Secret{name: name, value: new_value, origin: {:hub, id}}
 
-      assert_receive {:secret_updated,
+      assert_receive {:secret_created, ^secret}
-                      %Secret{name: ^name, value: ^new_value, origin: {:hub, ^id}}}
+      assert secret in EnterpriseClient.get_secrets(id)
+      refute updated_secret in EnterpriseClient.get_secrets(id)
+
+      :erpc.call(node, Enterprise.Integration, :update_secret, [enterprise_secret, new_value])
+
+      assert_receive {:secret_updated, ^updated_secret}
+
+      assert updated_secret in EnterpriseClient.get_secrets(id)
+      refute secret in EnterpriseClient.get_secrets(id)
+    end
+
+    test "receives a secret_deleted event", %{node: node, hub_id: id} do
+      name = "SUPER_DELETE"
+      value = "JakePeralta"
+      enteprise_secret = :erpc.call(node, Enterprise.Integration, :create_secret, [name, value])
+      secret = %Secret{name: name, value: value, origin: {:hub, id}}
+
+      assert_receive {:secret_created, ^secret}
+      assert secret in EnterpriseClient.get_secrets(id)
+
+      :erpc.call(node, Enterprise.Integration, :delete_secret, [enteprise_secret])
+
+      assert_receive {:secret_deleted, ^secret}
+      refute secret in EnterpriseClient.get_secrets(id)
     end
   end
 end

test/livebook/hubs/provider_test.exs

@@ -1,30 +1,77 @@
 defmodule Livebook.Hubs.ProviderTest do
   use Livebook.DataCase
 
-  alias Livebook.Hubs.{Fly, Metadata, Provider}
+  alias Livebook.Hubs.Provider
+  alias Livebook.Secrets
 
-  describe "Fly" do
+  describe "personal" do
-    test "to_metadata/1" do
+    setup do
-      fly = build(:fly)
+      {:ok, hub: build(:personal)}
-
-      assert Provider.to_metadata(fly) == %Metadata{
-               id: fly.id,
-               name: fly.hub_name,
-               emoji: fly.hub_emoji,
-               provider: fly,
-               connected?: false
-             }
     end
 
-    test "load/2" do
+    test "load/2", %{hub: hub} do
-      fly = build(:fly)
+      assert Provider.load(hub, Map.from_struct(hub)) == hub
-      fields = Map.from_struct(fly)
-
-      assert Provider.load(%Fly{}, fields) == fly
     end
 
-    test "type/1" do
+    test "type/1", %{hub: hub} do
-      assert Provider.type(%Fly{}) == "fly"
+      assert Provider.type(hub) == "personal"
+    end
+
+    test "connection_spec/1", %{hub: hub} do
+      refute Provider.connection_spec(hub)
+    end
+
+    test "disconnect/1", %{hub: hub} do
+      assert_raise RuntimeError, "not implemented", fn -> Provider.disconnect(hub) end
+    end
+
+    test "capabilities/1", %{hub: hub} do
+      assert Provider.capabilities(hub) == [
+               :list_secrets,
+               :create_secret,
+               :update_secret,
+               :delete_secret
+             ]
+    end
+
+    test "get_secrets/1 without startup secrets", %{hub: hub} do
+      secret = insert_secret(name: "GET_PERSONAL_SECRET", origin: {:hub, "personal-hub"})
+      assert secret in Provider.get_secrets(hub)
+    end
+
+    test "get_secrets/1 with startup secrets", %{hub: hub} do
+      secret = build(:secret, name: "GET_PERSONAL_SECRET", origin: :startup)
+      Livebook.Secrets.set_startup_secrets([secret])
+
+      assert secret in Provider.get_secrets(hub)
+    end
+
+    test "create_secret/1", %{hub: hub} do
+      secret = build(:secret, name: "CREATE_PERSONAL_SECRET", origin: {:hub, "personal-hub"})
+      assert Provider.create_secret(hub, secret) == :ok
+    end
+
+    test "update_secret/1", %{hub: hub} do
+      secret = insert_secret(name: "UPDATE_PERSONAL_SECRET", origin: {:hub, "personal-hub"})
+      assert secret in Secrets.get_secrets()
+
+      updated_secret = %{secret | value: "123321"}
+
+      assert Provider.update_secret(hub, updated_secret) == :ok
+      assert updated_secret in Secrets.get_secrets()
+      refute secret in Secrets.get_secrets()
+    end
+
+    test "delete_secret/1", %{hub: hub} do
+      secret = insert_secret(name: "DELETE_PERSONAL_SECRET", origin: {:hub, "personal-hub"})
+      assert secret in Secrets.get_secrets()
+
+      assert Provider.delete_secret(hub, secret) == :ok
+      refute secret in Secrets.get_secrets()
+    end
+
+    test "connection_error/1", %{hub: hub} do
+      assert_raise RuntimeError, "not implemented", fn -> Provider.connection_error(hub) end
     end
   end
 end

test/livebook/secrets_test.exs

@@ -19,7 +19,7 @@ defmodule Livebook.SecretsTest do
     test "returns a list of secrets from temporary storage" do
       secret = build(:secret, name: "FOO", value: "222", origin: :startup)
 
-      Secrets.set_temporary_secrets([secret])
+      Secrets.set_startup_secrets([secret])
       assert secret in Secrets.get_secrets()
 
       # We can't delete from temporary storage, since it will be deleted

test/livebook/web_socket/client_connection_test.exs

@@ -46,33 +46,20 @@ defmodule Livebook.WebSocket.ClientConnectionTest do
     end
 
     test "successfully sends a session request", %{conn: conn, user: %{id: id, email: email}} do
-      session_request =
+      data = LivebookProto.build_handshake_request(app_version: Livebook.Config.app_version())
-        LivebookProto.SessionRequest.new!(app_version: Livebook.Config.app_version())
 
-      assert {:session, session_response} = ClientConnection.send_request(conn, session_request)
+      assert {:handshake, handshake_response} = ClientConnection.send_request(conn, data)
-      assert %{id: _, user: %{id: ^id, email: ^email}} = session_response
+      assert %{id: _, user: %{id: ^id, email: ^email}} = handshake_response
     end
 
     test "successfully sends a create secret message", %{conn: conn} do
-      create_secret_request =
+      data = LivebookProto.build_create_secret_request(name: "MY_USERNAME", value: "Jake Peralta")
-        LivebookProto.CreateSecretRequest.new!(
+      assert {:create_secret, _} = ClientConnection.send_request(conn, data)
-          name: "MY_USERNAME",
-          value: "Jake Peralta"
-        )
-
-      assert {:create_secret, _} = ClientConnection.send_request(conn, create_secret_request)
     end
 
     test "sends a create secret message, but receive a changeset error", %{conn: conn} do
-      create_secret_request =
+      data = LivebookProto.build_create_secret_request(name: "MY_USERNAME", value: "")
-        LivebookProto.CreateSecretRequest.new!(
+      assert {:changeset_error, errors} = ClientConnection.send_request(conn, data)
-          name: "MY_USERNAME",
-          value: ""
-        )
-
-      assert {:changeset_error, errors} =
-               ClientConnection.send_request(conn, create_secret_request)
-
       assert "can't be blank" in errors.value
     end
   end
@@ -121,25 +108,23 @@ defmodule Livebook.WebSocket.ClientConnectionTest do
     setup %{url: url, token: token} do
       headers = [{"X-Auth-Token", token}]
 
-      {:ok, _conn} = ClientConnection.start_link(self(), url, headers)
+      {:ok, conn} = ClientConnection.start_link(self(), url, headers)
       assert_receive {:connect, :ok, :connected}
 
-      :ok
+      {:ok, conn: conn}
     end
 
-    test "receives a secret_created event" do
+    test "receives a secret_created event", %{node: node} do
       name = "MY_SECRET_ID"
       value = Livebook.Utils.random_id()
-      node = EnterpriseServer.get_node()
       :erpc.call(node, Enterprise.Integration, :create_secret, [name, value])
 
       assert_receive {:event, :secret_created, %{name: ^name, value: ^value}}
     end
 
-    test "receives a secret_updated event" do
+    test "receives a secret_updated event", %{node: node} do
       name = "API_USERNAME"
       value = "JakePeralta"
-      node = EnterpriseServer.get_node()
       secret = :erpc.call(node, Enterprise.Integration, :create_secret, [name, value])
 
       assert_receive {:event, :secret_created, %{name: ^name, value: ^value}}
@@ -149,5 +134,35 @@ defmodule Livebook.WebSocket.ClientConnectionTest do
 
       assert_receive {:event, :secret_updated, %{name: ^name, value: ^new_value}}
     end
+
+    test "receives a secret_deleted event", %{node: node} do
+      name = "DELETE_ME"
+      value = "JakePeralta"
+      secret = :erpc.call(node, Enterprise.Integration, :create_secret, [name, value])
+
+      assert_receive {:event, :secret_created, %{name: ^name, value: ^value}}
+
+      :erpc.call(node, Enterprise.Integration, :delete_secret, [secret])
+
+      assert_receive {:event, :secret_deleted, %{name: ^name, value: ^value}}
+    end
+
+    test "receives a session_created event", %{conn: conn, node: node} do
+      data = LivebookProto.build_handshake_request(app_version: Livebook.Config.app_version())
+      assert {:handshake, _} = ClientConnection.send_request(conn, data)
+
+      id = :erpc.call(node, Enterprise.Integration, :fetch_env!, ["ENTERPRISE_ID"])
+      name = :erpc.call(node, Enterprise.Integration, :fetch_env!, ["ENTERPRISE_NAME"])
+
+      assert_receive {:event, :user_synchronized, %{id: ^id, name: ^name, secrets: []}}
+
+      secret = :erpc.call(node, Enterprise.Integration, :create_secret, ["SESSION", "123"])
+      assert_receive {:event, :secret_created, %{name: "SESSION", value: "123"}}
+
+      assert {:handshake, _} = ClientConnection.send_request(conn, data)
+
+      assert_receive {:event, :user_synchronized, %{id: ^id, name: ^name, secrets: secrets}}
+      assert LivebookProto.Secret.new!(name: secret.name, value: secret.value) in secrets
+    end
   end
 end

test/livebook_web/live/session_live/secrets_component_test.exs

@@ -60,13 +60,8 @@ defmodule LivebookWeb.SessionLive.SecretsComponentTest do
 
       assert_receive {:secret_created, ^secret}
       assert render(view) =~ "A new secret has been created on your Livebook Enterprise"
-      assert has_element?(view, "#hub-#{enterprise.id}-secret-#{secret.name}-title")
+      assert has_element?(view, "#hub-#{enterprise.id}-secret-#{secret.name}-wrapper")
-
+      assert has_element?(view, ~s/[data-tooltip="#{enterprise.hub_name}"]/, enterprise.hub_emoji)
-      assert has_element?(
-               view,
-               "#hub-#{enterprise.id}-secret-#{secret.name}-title span",
-               enterprise.hub_emoji
-             )
     end
 
     test "toggle a secret from Enterprise hub",

test/livebook_web/live/session_live_test.exs

@@ -1217,7 +1217,7 @@ defmodule LivebookWeb.SessionLiveTest do
 
     test "loads secret from temporary storage", %{conn: conn, session: session} do
       secret = build(:secret, name: "FOOBARBAZ", value: "ChonkyCat", origin: :startup)
-      Livebook.Secrets.set_temporary_secrets([secret])
+      Livebook.Secrets.set_startup_secrets([secret])
 
       {:ok, view, _} = live(conn, ~p"/sessions/#{session.id}")
 
@@ -1246,7 +1246,7 @@ defmodule LivebookWeb.SessionLiveTest do
     test "granting access for unavailable startup secret using 'Add secret' button",
          %{conn: conn, session: session} do
       secret = build(:secret, name: "MYSTARTUPSECRET", value: "ChonkyCat", origin: :startup)
-      Livebook.Secrets.set_temporary_secrets([secret])
+      Livebook.Secrets.set_startup_secrets([secret])
 
       # Subscribe and executes the code to trigger
       # the `System.EnvError` exception and outputs the 'Add secret' button

test/support/factory.ex

@@ -43,6 +43,18 @@ defmodule Livebook.Factory do
     }
   end
 
+  def build(:personal_metadata) do
+    :personal |> build() |> Livebook.Hubs.Provider.to_metadata()
+  end
+
+  def build(:personal) do
+    %Livebook.Hubs.Personal{
+      id: "personal-hub",
+      hub_name: "My Hub",
+      hub_emoji: "🏠"
+    }
+  end
+
   def build(:env_var) do
     %Livebook.Settings.EnvVar{
       name: "BAR",
@@ -58,11 +70,11 @@ defmodule Livebook.Factory do
     }
   end
 
-  def build(factory_name, attrs \\ %{}) do
+  def build(factory_name, attrs) do
     factory_name |> build() |> struct!(attrs)
   end
 
-  def params_for(factory_name, attrs \\ %{}) do
+  def params_for(factory_name, attrs) do
     factory_name |> build() |> struct!(attrs) |> Map.from_struct()
   end
 

test/support/integration/enterprise_server.ex

@@ -220,6 +220,10 @@ defmodule Livebook.EnterpriseServer do
     System.get_env("ENTERPRISE_DEBUG", "false")
   end
 
+  defp proto do
+    System.get_env("ENTERPRISE_LIVEBOOK_PROTO_PATH")
+  end
+
   defp wait_on_start(state, port) do
     url = state.url || fetch_url(state)
 
@@ -265,6 +269,8 @@ defmodule Livebook.EnterpriseServer do
       "DEBUG" => debug()
     }
 
+    env = if proto(), do: Map.merge(env, %{"LIVEBOOK_PROTO_PATH" => proto()}), else: env
+
     if state_env do
       Map.merge(env, state_env)
     else