Add Team hub from environment variables (#2020)

lib/livebook/application.ex

@@ -57,6 +57,7 @@ defmodule Livebook.Application do
       {:ok, _} = result ->
         Livebook.Migration.migrate()
         load_lb_env_vars()
+        create_offline_hub()
         clear_env_vars()
         display_startup_info()
         Livebook.Hubs.connect_hubs()
@@ -189,7 +190,7 @@ defmodule Livebook.Application do
     end
   end
 
-  defp load_lb_env_vars do
+  defp load_lb_env_vars() do
     secrets =
       for {"LB_" <> name = var, value} <- System.get_env() do
         System.delete_env(var)
@@ -204,6 +205,22 @@ defmodule Livebook.Application do
     Livebook.Secrets.set_startup_secrets(secrets)
   end
 
+  def create_offline_hub() do
+    name = System.get_env("LIVEBOOK_TEAMS_NAME")
+    teams_key = System.get_env("LIVEBOOK_TEAMS_KEY")
+    public_key = System.get_env("LIVEBOOK_TEAMS_OFFLINE_KEY")
+
+    if name && teams_key && public_key do
+      Livebook.Hubs.set_offline_hub(%Livebook.Hubs.Team{
+        id: "team-#{name}",
+        hub_name: name,
+        hub_emoji: "💡",
+        teams_key: teams_key,
+        org_public_key: public_key
+      })
+    end
+  end
+
   defp config_env_var?("LIVEBOOK_" <> _), do: true
   defp config_env_var?("RELEASE_" <> _), do: true
   defp config_env_var?("MIX_ENV"), do: true

lib/livebook/hubs.ex

@@ -272,4 +272,25 @@ defmodule Livebook.Hubs do
   def capability?(hub, capabilities) do
     capabilities -- Provider.capabilities(hub) == []
   end
+
+  @offline_hub_key :livebook_offline_hub
+
+  @doc """
+  Get the offline hub.
+  """
+  @spec get_offline_hub(String.t()) :: Provider.t() | nil
+  def get_offline_hub(id) do
+    case :persistent_term.get(@offline_hub_key, nil) do
+      %{id: ^id} = hub -> hub
+      _ -> nil
+    end
+  end
+
+  @doc """
+  Sets a offline hub that will be kept only in memory.
+  """
+  @spec set_offline_hub(Provider.t()) :: :ok
+  def set_offline_hub(hub) do
+    :persistent_term.put(@offline_hub_key, hub)
+  end
 end

lib/livebook/hubs/provider.ex

@@ -100,7 +100,8 @@ defprotocol Livebook.Hubs.Provider do
 
   See `t:notebook_stamp/0` for more details.
   """
-  @spec notebook_stamp(t(), iodata(), map()) :: {:ok, notebook_stamp()} | :skip | :error
+  @spec notebook_stamp(t(), iodata(), map()) ::
+          {:ok, notebook_stamp()} | :skip | {:error, String.t()}
   def notebook_stamp(hub, notebook_source, metadata)
 
   @doc """

lib/livebook/hubs/team.ex

@@ -67,18 +67,7 @@ defimpl Livebook.Hubs.Provider, for: Livebook.Hubs.Team do
   alias Livebook.Teams
 
   def load(team, fields) do
-    %{
+    struct(team, fields)
-      team
-      | id: fields.id,
-        session_token: fields.session_token,
-        teams_key: fields.teams_key,
-        org_public_key: fields.org_public_key,
-        org_id: fields.org_id,
-        user_id: fields.user_id,
-        org_key_id: fields.org_key_id,
-        hub_name: fields.hub_name,
-        hub_emoji: fields.hub_emoji
-    }
   end
 
   def to_metadata(team) do

lib/livebook/live_markdown/import.ex

@@ -1,4 +1,5 @@
 defmodule Livebook.LiveMarkdown.Import do
+  alias Livebook.Hubs
   alias Livebook.Notebook
   alias Livebook.LiveMarkdown.MarkdownHelpers
 
@@ -9,9 +10,11 @@ defmodule Livebook.LiveMarkdown.Import do
     {ast, rewrite_messages} = rewrite_ast(ast)
     elements = group_elements(ast)
     {stamp_data, elements} = take_stamp_data(elements)
-    {notebook, build_messages} = build_notebook(elements)
+    {notebook, stamp_hub_id, build_messages} = build_notebook(elements)
     {notebook, postprocess_messages} = postprocess_notebook(notebook)
-    {notebook, metadata_messages} = postprocess_stamp(notebook, markdown, stamp_data)
+
+    {notebook, metadata_messages} =
+      postprocess_stamp(notebook, markdown, stamp_data, stamp_hub_id)
 
     messages =
       earmark_messages ++
@@ -326,7 +329,7 @@ defmodule Livebook.LiveMarkdown.Import do
           ]
       end
 
-    {attrs, metadata_messages} = notebook_metadata_to_attrs(metadata)
+    {attrs, stamp_hub_id, metadata_messages} = notebook_metadata_to_attrs(metadata)
     messages = messages ++ metadata_messages
 
     # We identify a single leading cell as the setup cell, in any
@@ -349,7 +352,7 @@ defmodule Livebook.LiveMarkdown.Import do
       |> maybe_put_setup_cell(setup_cell)
       |> Map.merge(attrs)
 
-    {notebook, messages}
+    {notebook, stamp_hub_id, messages}
   end
 
   defp maybe_put_name(notebook, nil), do: notebook
@@ -377,36 +380,36 @@ defmodule Livebook.LiveMarkdown.Import do
   defp grab_leading_comments(elems), do: {[], elems}
 
   defp notebook_metadata_to_attrs(metadata) do
-    Enum.reduce(metadata, {%{}, []}, fn
+    Enum.reduce(metadata, {%{}, Livebook.Hubs.Personal.id(), []}, fn
-      {"persist_outputs", persist_outputs}, {attrs, messages} ->
+      {"persist_outputs", persist_outputs}, {attrs, id, messages} ->
-        {Map.put(attrs, :persist_outputs, persist_outputs), messages}
+        {Map.put(attrs, :persist_outputs, persist_outputs), id, messages}
 
-      {"autosave_interval_s", autosave_interval_s}, {attrs, messages} ->
+      {"autosave_interval_s", autosave_interval_s}, {attrs, id, messages} ->
-        {Map.put(attrs, :autosave_interval_s, autosave_interval_s), messages}
+        {Map.put(attrs, :autosave_interval_s, autosave_interval_s), id, messages}
 
-      {"default_language", default_language}, {attrs, messages}
+      {"default_language", default_language}, {attrs, id, messages}
       when default_language in ["elixir", "erlang"] ->
         default_language = String.to_atom(default_language)
-        {Map.put(attrs, :default_language, default_language), messages}
+        {Map.put(attrs, :default_language, default_language), id, messages}
 
-      {"hub_id", hub_id}, {attrs, messages} ->
+      {"hub_id", hub_id}, {attrs, id, messages} ->
-        if Livebook.Hubs.hub_exists?(hub_id) do
+        cond do
-          {Map.put(attrs, :hub_id, hub_id), messages}
+          Hubs.hub_exists?(hub_id) -> {Map.put(attrs, :hub_id, hub_id), hub_id, messages}
-        else
+          Hubs.get_offline_hub(hub_id) -> {attrs, hub_id, messages}
-          {attrs, messages ++ ["ignoring notebook Hub with unknown id"]}
+          true -> {attrs, id, messages ++ ["ignoring notebook Hub with unknown id"]}
         end
 
-      {"app_settings", app_settings_metadata}, {attrs, messages} ->
+      {"app_settings", app_settings_metadata}, {attrs, id, messages} ->
         app_settings =
           Map.merge(
             Notebook.AppSettings.new(),
             app_settings_metadata_to_attrs(app_settings_metadata)
           )
 
-        {Map.put(attrs, :app_settings, app_settings), messages}
+        {Map.put(attrs, :app_settings, app_settings), id, messages}
 
-      _entry, {attrs, messages} ->
+      _entry, {attrs, id, messages} ->
-        {attrs, messages}
+        {attrs, id, messages}
     end)
   end
 
@@ -521,10 +524,10 @@ defmodule Livebook.LiveMarkdown.Import do
   defp take_stamp_data([{:stamp, data} | elements]), do: {data, elements}
   defp take_stamp_data(elements), do: {nil, elements}
 
-  defp postprocess_stamp(notebook, _notebook_source, nil), do: {notebook, []}
+  defp postprocess_stamp(notebook, _notebook_source, nil, _), do: {notebook, []}
 
-  defp postprocess_stamp(notebook, notebook_source, stamp_data) do
+  defp postprocess_stamp(notebook, notebook_source, stamp_data, stamp_hub_id) do
-    hub = Livebook.Hubs.fetch_hub!(notebook.hub_id)
+    hub = Hubs.get_offline_hub(stamp_hub_id) || Hubs.fetch_hub!(stamp_hub_id)
 
     with %{"offset" => offset, "stamp" => stamp} <- stamp_data,
          {:ok, notebook_source} <- safe_binary_slice(notebook_source, 0, offset),

lib/livebook/secrets/secret.ex

@@ -4,8 +4,8 @@ defmodule Livebook.Secrets.Secret do
   import Ecto.Changeset
 
   @type t :: %__MODULE__{
-          name: String.t() | nil,
+          name: String.t(),
-          value: String.t() | nil,
+          value: String.t(),
           hub_id: String.t() | nil
         }
 

lib/livebook/session/data.ex

@@ -884,8 +884,6 @@ defmodule Livebook.Session.Data do
       |> update_notebook_hub_secret_names()
       |> set_dirty()
       |> wrap_ok()
-    else
-      _ -> :error
     end
   end
 

lib/livebook_web/live/hub/edit/team_component.ex

@@ -47,6 +47,7 @@ defmodule LivebookWeb.Hub.Edit.TeamComponent do
               <.remix_icon icon="key-2-fill" class="text-xl sm:hidden" />
             </button>
             <button
+              id="delete-hub"
               phx-click={JS.push("delete_hub", value: %{id: @hub.id})}
               class="button-base button-red"
             >

lib/livebook_web/live/session_live.ex

@@ -549,7 +549,7 @@ defmodule LivebookWeb.SessionLive do
       ) %>
     </.modal>
 
-    <.modal :if={@live_action == :secrets} id="secrets-modal" show width={:medium} patch={@self_path}>
+    <.modal :if={@live_action == :secrets} id="secrets-modal" show width={:large} patch={@self_path}>
       <.live_component
         module={LivebookWeb.SessionLive.SecretsComponent}
         id="secrets"

lib/livebook_web/live/session_live/secrets_component.ex

@@ -36,7 +36,7 @@ defmodule LivebookWeb.SessionLive.SecretsComponent do
   @impl true
   def render(assigns) do
     ~H"""
-    <div class="p-6 max-w-4xl flex flex-col space-y-5">
+    <div class="p-6 w-full flex flex-col space-y-5">
       <h3 class="text-2xl font-semibold text-gray-800">
         <%= @title %>
       </h3>

test/livebook/live_markdown/import_test.exs

@@ -1120,5 +1120,66 @@ defmodule Livebook.LiveMarkdown.ImportTest do
 
       assert messages == ["failed to verify notebook stamp"]
     end
+
+    test "restores hub secret names from notebook stamp using offline hub" do
+      hub =
+        Livebook.Factory.build(:team,
+          id: "team-org-number-2946",
+          teams_key: "AleIxOFlwSiOS78WXtVU01ySmitjzy-5pAuCh4i1wZE",
+          org_public_key:
+            "MIIBCgKCAQEA2uRttEa6UvtiAUhv-MhPZvvlrCNeeL5n6oP4pliqoMBD7vsi4EvwnrqjCCicwHeT4y8Pu1kmzTelDAHEyO8alllBtfnZnQkPOqo1Y6c6qBHhcioc2FrNvdAydMiByhyn_aqNbFNeMMgy9ogHerAQ6XPrGSaXEvIcWn3myz-zxYdeEDW5G5W95o7Q0x7lokdVBUwXbazH0JVu_-1FUr7aOSjjuNHX6rXMRA3wr4n2SuhGOvihrX5IYRb733pae2aTOfJZGD_83eUPHTu_cPoUflcvIPtnVlGTxBgSX9Ayl1X3uDOnJsk2pxawFF6GxBMUKjMGyGDTg_lL45cgsWovXQIDAQAB",
+          hub_name: "org-number-2946"
+        )
+
+      Livebook.Hubs.set_offline_hub(hub)
+
+      markdown = """
+      <!-- livebook:{"hub_id":"team-org-number-2946"} -->
+
+      # My Notebook
+
+      ## Section 1
+
+      ```elixir
+      IO.puts("hey")
+      ```
+
+      <!-- livebook:{"offset":111,"stamp":{"token":"QTEyOEdDTQ.yw3drh2WcwU8K6jS9Wp0HPupyX3qoc8iBmUXrMVKvSPnIOGEYMmu160e89E.xyzsr7PxSBrA8Elt.N3KyvcuTrFyMYpSl8WB1Sctv-1YjSjv_DCZoOVje_zXPYpm4iV_Ss5tVUSA7IWE.lV7grc6HYOYJrf0YYScPwQ","token_signature":"KSd-EhXw2CrmS9m4aZnPhTgWzlNdQNJ0wvYmuNvi8Pxaqb-prKO0FN_BTcPHtk4ZDHJaIFac-8dyefkCHpIElAc_N7vExgO9_7wSOJ8Hagip7DOxOBfqcR6iC17ejiw-2wWFJu0p6deaXpm2RWkWJU--wiU1cAHoKoJGqIsMMxNmgAkT44Pok0ni5BtnTfZjq_c2iPTYfP-8uU2WFIDmzEeOL-He5iWNUlixnf5Aj1YSVNldi6vTtR70xBRvlUxPCkWbt1x6XjanspY15j43PgVTo0EPM4kGCkS2HcWBZB_XscxZ4-V-WdpQ0pkv1goPdfDGDcAbjP7z8oum9_ZKNA","version":1}} -->
+      """
+
+      {notebook, []} = Import.notebook_from_livemd(markdown)
+
+      assert %Notebook{hub_id: "personal-hub", hub_secret_names: ["DB_PASSWORD"]} = notebook
+    end
+
+    test "returns a warning when notebook stamp is invalid using offline hub" do
+      hub =
+        Livebook.Factory.build(:team,
+          id: "team-org-number-2946",
+          teams_key: "AleIxOFlwSiOS78WXtVU01ySmitjzy-5pAuCh4i1wZE",
+          org_public_key:
+            "MIIBCgKCAQEA2uRttEa6UvtiAUhv-MhPZvvlrCNeeL5n6oP4pliqoMBD7vsi4EvwnrqjCCicwHeT4y8Pu1kmzTelDAHEyO8alllBtfnZnQkPOqo1Y6c6qBHhcioc2FrNvdAydMiByhyn_aqNbFNeMMgy9ogHerAQ6XPrGSaXEvIcWn3myz-zxYdeEDW5G5W95o7Q0x7lokdVBUwXbazH0JVu_-1FUr7aOSjjuNHX6rXMRA3wr4n2SuhGOvihrX5IYRb733pae2aTOfJZGD_83eUPHTu_cPoUflcvIPtnVlGTxBgSX9Ayl1X3uDOnJsk2pxawFF6GxBMUKjMGyGDTg_lL45cgsWovXQIDAQAB",
+          hub_name: "org-number-2946"
+        )
+
+      Livebook.Hubs.set_offline_hub(hub)
+
+      markdown = """
+      <!-- livebook:{"hub_id":"team-org-number-2946"} -->
+
+      # My Notebook
+
+      ## Section 1
+
+      ```elixir
+      IO.puts("hey")
+      ```
+
+      <!-- livebook:{"offset":58,"stamp":{"token":"invalid","token_signature":"invalid","version":1}} -->
+      """
+
+      assert {%Notebook{hub_secret_names: []}, ["failed to verify notebook stamp"]} =
+               Import.notebook_from_livemd(markdown)
+    end
   end
 end

test/livebook_web/live/hub/new_live_test.exs

@@ -53,11 +53,11 @@ defmodule LivebookWeb.Hub.NewLiveTest do
 
       # access the page and shows the teams key modal
       {:ok, view, _html} = live(conn, "/hub/team-#{name}?show-key=true")
-      assert has_element?(view, "#show-key-modal")
+      refute has_element?(view, "#show-key-modal.hidden")
 
       # access the page when closes the modal
       assert {:ok, view, _html} = live(conn, "/hub/team-#{name}")
-      refute has_element?(view, "#show-key-modal")
+      assert has_element?(view, "#show-key-modal.hidden")
 
       # checks if the hub is in the sidebar
       assert_sidebar_hub(view, "team-#{name}", name)
@@ -114,11 +114,11 @@ defmodule LivebookWeb.Hub.NewLiveTest do
 
       # access the page and shows the teams key modal
       {:ok, view, _html} = live(conn, "/hub/team-#{name}?show-key=true")
-      assert has_element?(view, "#show-key-modal")
+      refute has_element?(view, "#show-key-modal.hidden")
 
       # access the page when closes the modal
       assert {:ok, view, _html} = live(conn, "/hub/team-#{name}")
-      refute has_element?(view, "#show-key-modal")
+      assert has_element?(view, "#show-key-modal.hidden")
 
       # checks if the hub is in the sidebar
       assert_sidebar_hub(view, "team-#{name}", name)

test/support/factory.ex

@@ -23,6 +23,7 @@ defmodule Livebook.Factory do
       org_id: 1,
       user_id: 1,
       org_key_id: 1,
+      org_public_key: Livebook.Utils.random_id(),
       teams_key: org.teams_key,
       session_token: Livebook.Utils.random_short_id()
     }