diff --git a/lib/livebook/hubs.ex b/lib/livebook/hubs.ex index 2277ea2a6..52fbd6068 100644 --- a/lib/livebook/hubs.ex +++ b/lib/livebook/hubs.ex @@ -248,7 +248,7 @@ defmodule Livebook.Hubs do Generates a notebook stamp. """ @spec notebook_stamp(Provider.t(), iodata(), map()) :: - {:ok, Provider.notebook_stamp()} | :skip | :error + {:ok, Provider.notebook_stamp()} | :skip | {:error, String.t()} def notebook_stamp(hub, notebook_source, metadata) do Provider.notebook_stamp(hub, notebook_source, metadata) end diff --git a/lib/livebook/hubs/personal.ex b/lib/livebook/hubs/personal.ex index 84f744e60..fb91768d6 100644 --- a/lib/livebook/hubs/personal.ex +++ b/lib/livebook/hubs/personal.ex @@ -143,15 +143,7 @@ defimpl Livebook.Hubs.Provider, for: Livebook.Hubs.Personal do end def notebook_stamp(hub, notebook_source, metadata) do - # We use AES-GCM-128 to encrypt metadata and generate signature - # for both metadata and the notebook source. We make use of the - # implementation in MessageEncryptor, which conveniently returns - # a single token - - {secret, sign_secret} = derive_keys(hub.secret_key) - - payload = :erlang.term_to_binary(metadata) - token = Plug.Crypto.MessageEncryptor.encrypt(payload, notebook_source, secret, sign_secret) + token = Livebook.Stamping.aead_encrypt(metadata, notebook_source, hub.secret_key) stamp = %{"version" => 1, "token" => token} @@ -161,27 +153,6 @@ defimpl Livebook.Hubs.Provider, for: Livebook.Hubs.Personal do def verify_notebook_stamp(hub, notebook_source, stamp) do %{"version" => 1, "token" => token} = stamp - {secret, sign_secret} = derive_keys(hub.secret_key) - - case Plug.Crypto.MessageEncryptor.decrypt(token, notebook_source, secret, sign_secret) do - {:ok, payload} -> - metadata = Plug.Crypto.non_executable_binary_to_term(payload) - {:ok, metadata} - - :error -> - :error - end - end - - defp derive_keys(secret_key) do - binary_key = Base.url_decode64!(secret_key, padding: false) - - <> = - Plug.Crypto.KeyGenerator.generate(binary_key, "notebook signing", - length: 32, - cache: Plug.Crypto.Keys - ) - - {secret, sign_secret} + Livebook.Stamping.aead_decrypt(token, notebook_source, hub.secret_key) end end diff --git a/lib/livebook/hubs/team.ex b/lib/livebook/hubs/team.ex index 4e3a4163d..2b79392a3 100644 --- a/lib/livebook/hubs/team.ex +++ b/lib/livebook/hubs/team.ex @@ -10,6 +10,7 @@ defmodule Livebook.Hubs.Team do user_id: pos_integer() | nil, org_key_id: pos_integer() | nil, teams_key: String.t() | nil, + org_public_key: String.t() | nil, session_token: String.t() | nil, hub_name: String.t() | nil, hub_emoji: String.t() | nil @@ -20,6 +21,7 @@ defmodule Livebook.Hubs.Team do field :user_id, :integer field :org_key_id, :integer field :teams_key, :string + field :org_public_key, :string field :session_token, :string field :hub_name, :string field :hub_emoji, :string @@ -30,6 +32,7 @@ defmodule Livebook.Hubs.Team do user_id org_key_id teams_key + org_public_key session_token hub_name hub_emoji @@ -68,6 +71,7 @@ defimpl Livebook.Hubs.Provider, for: Livebook.Hubs.Team do | id: fields.id, session_token: fields.session_token, teams_key: fields.teams_key, + org_public_key: fields.org_public_key, org_id: fields.org_id, user_id: fields.user_id, org_key_id: fields.org_key_id, @@ -107,9 +111,35 @@ defimpl Livebook.Hubs.Provider, for: Livebook.Hubs.Team do "Cannot connect to Hub: #{reason}. Will attempt to reconnect automatically..." end - def notebook_stamp(_hub, _notebook_source, _metadata) do - :skip + def notebook_stamp(hub, notebook_source, metadata) do + # We apply authenticated encryption using the shared teams key, + # just as for the personal hub, but we additionally sign the token + # with a private organization key stored on the Teams server. We + # then validate the signature using the corresponding public key. + # + # This results in a two factor mechanism, where creating a valid + # stamp requires access to the shared local key and an authenticated + # request to the Teams server (which ensures team membership). + + token = Livebook.Stamping.aead_encrypt(metadata, notebook_source, hub.teams_key) + + case Livebook.Teams.org_sign(hub, token) do + {:ok, token_signature} -> + stamp = %{"version" => 1, "token" => token, "token_signature" => token_signature} + {:ok, stamp} + + _ -> + {:error, "request to Livebook Teams failed"} + end end - def verify_notebook_stamp(_hub, _notebook_source, _stamp), do: raise("not implemented") + def verify_notebook_stamp(hub, notebook_source, stamp) do + %{"version" => 1, "token" => token, "token_signature" => token_signature} = stamp + + if Livebook.Stamping.rsa_verify?(token_signature, token, hub.org_public_key) do + Livebook.Stamping.aead_decrypt(token, notebook_source, hub.teams_key) + else + :error + end + end end diff --git a/lib/livebook/live_markdown.ex b/lib/livebook/live_markdown.ex index 2d8c6db51..64353b434 100644 --- a/lib/livebook/live_markdown.ex +++ b/lib/livebook/live_markdown.ex @@ -92,7 +92,7 @@ defmodule Livebook.LiveMarkdown do value of `:persist_outputs` notebook attribute. """ - @spec notebook_to_livemd(Notebook.t(), keyword()) :: String.t() + @spec notebook_to_livemd(Notebook.t(), keyword()) :: {String.t(), list(String.t())} defdelegate notebook_to_livemd(notebook, opts \\ []), to: Livebook.LiveMarkdown.Export @doc """ diff --git a/lib/livebook/live_markdown/export.ex b/lib/livebook/live_markdown/export.ex index 28a32c8c1..beec0b5c2 100644 --- a/lib/livebook/live_markdown/export.ex +++ b/lib/livebook/live_markdown/export.ex @@ -5,6 +5,7 @@ defmodule Livebook.LiveMarkdown.Export do def notebook_to_livemd(notebook, opts \\ []) do include_outputs? = Keyword.get(opts, :include_outputs, notebook.persist_outputs) + include_stamp? = Keyword.get(opts, :include_stamp, true) js_ref_with_data = if include_outputs?, do: collect_js_output_data(notebook), else: %{} @@ -15,9 +16,12 @@ defmodule Livebook.LiveMarkdown.Export do # Add trailing newline notebook_source = [iodata, "\n"] - notebook_footer = render_notebook_footer(notebook, notebook_source) + {notebook_footer, footer_warnings} = + render_notebook_footer(notebook, notebook_source, include_stamp?) - IO.iodata_to_binary([notebook_source, notebook_footer]) + source = IO.iodata_to_binary([notebook_source, notebook_footer]) + + {source, footer_warnings} end defp collect_js_output_data(notebook) do @@ -327,16 +331,29 @@ defmodule Livebook.LiveMarkdown.Export do |> Enum.map(fn {_modifiers, string} -> string end) end - defp render_notebook_footer(notebook, notebook_source) do + defp render_notebook_footer(_notebook, _notebook_source, _include_stamp? = false), do: {[], []} + + defp render_notebook_footer(notebook, notebook_source, true) do metadata = notebook_stamp_metadata(notebook) - with {:ok, hub} <- Livebook.Hubs.fetch_hub(notebook.hub_id), - {:ok, stamp} <- Livebook.Hubs.notebook_stamp(hub, notebook_source, metadata) do - offset = IO.iodata_length(notebook_source) - json = %{"offset" => offset, "stamp" => stamp} |> ensure_order() |> Jason.encode!() - ["\n", "", "\n"] - else - _ -> [] + case Livebook.Hubs.fetch_hub(notebook.hub_id) do + {:ok, hub} -> + case Livebook.Hubs.notebook_stamp(hub, notebook_source, metadata) do + {:ok, stamp} -> + offset = IO.iodata_length(notebook_source) + json = %{"offset" => offset, "stamp" => stamp} |> ensure_order() |> Jason.encode!() + footer = ["\n", "", "\n"] + {footer, []} + + :skip -> + {[], []} + + {:error, message} -> + {[], ["failed to stamp the notebook, #{message}"]} + end + + :error -> + {[], []} end end diff --git a/lib/livebook/session.ex b/lib/livebook/session.ex index ecad8a494..d252659ea 100644 --- a/lib/livebook/session.ex +++ b/lib/livebook/session.ex @@ -1413,9 +1413,12 @@ defmodule Livebook.Session do {:noreply, handle_operation(state, operation)} end - def handle_info({:save_finished, pid, result, file, default?}, %{save_task_pid: pid} = state) do + def handle_info( + {:save_finished, pid, result, warnings, file, default?}, + %{save_task_pid: pid} = state + ) do state = %{state | save_task_pid: nil} - {:noreply, handle_save_finished(state, result, file, default?)} + {:noreply, handle_save_finished(state, result, warnings, file, default?)} end def handle_info({:runtime_memory_usage, runtime_memory}, state) do @@ -2118,9 +2121,9 @@ defmodule Livebook.Session do {:ok, pid} = Task.Supervisor.start_child(Livebook.TaskSupervisor, fn -> - content = LiveMarkdown.notebook_to_livemd(notebook) + {content, warnings} = LiveMarkdown.notebook_to_livemd(notebook) result = FileSystem.File.write(file, content) - send(pid, {:save_finished, self(), result, file, default?}) + send(pid, {:save_finished, self(), result, warnings, file, default?}) end) %{state | save_task_pid: pid} @@ -2135,9 +2138,9 @@ defmodule Livebook.Session do {file, default?} = notebook_autosave_file(state) if file && should_save_notebook?(state) do - content = LiveMarkdown.notebook_to_livemd(state.data.notebook) + {content, warnings} = LiveMarkdown.notebook_to_livemd(state.data.notebook) result = FileSystem.File.write(file, content) - handle_save_finished(state, result, file, default?) + handle_save_finished(state, result, warnings, file, default?) else state end @@ -2146,7 +2149,7 @@ defmodule Livebook.Session do defp maybe_save_notebook_sync(state), do: state defp should_save_notebook?(state) do - state.data.dirty and state.save_task_pid == nil + (state.data.dirty or state.data.persistence_warnings != []) and state.save_task_pid == nil end defp notebook_autosave_file(state) do @@ -2192,7 +2195,7 @@ defmodule Livebook.Session do end end - defp handle_save_finished(state, result, file, default?) do + defp handle_save_finished(state, result, warnings, file, default?) do state = if default? do if state.saved_default_file && state.saved_default_file != file do @@ -2206,7 +2209,7 @@ defmodule Livebook.Session do case result do :ok -> - handle_operation(state, {:mark_as_not_dirty, @client_id}) + handle_operation(state, {:notebook_saved, @client_id, warnings}) {:error, message} -> broadcast_error(state.session_id, "failed to save notebook - #{message}") diff --git a/lib/livebook/session/data.ex b/lib/livebook/session/data.ex index 432fd44a2..0205af12b 100644 --- a/lib/livebook/session/data.ex +++ b/lib/livebook/session/data.ex @@ -23,6 +23,7 @@ defmodule Livebook.Session.Data do :origin, :file, :dirty, + :persistence_warnings, :section_infos, :cell_infos, :input_values, @@ -210,7 +211,7 @@ defmodule Livebook.Session.Data do | {:set_smart_cell_definitions, client_id(), list(Runtime.smart_cell_definition())} | {:set_file, client_id(), FileSystem.File.t() | nil} | {:set_autosave_interval, client_id(), non_neg_integer() | nil} - | {:mark_as_not_dirty, client_id()} + | {:notebook_saved, client_id(), persistence_warnings :: list(String.t())} | {:set_secret, client_id(), Secret.t()} | {:unset_secret, client_id(), String.t()} | {:set_notebook_hub, client_id(), String.t()} @@ -284,6 +285,7 @@ defmodule Livebook.Session.Data do origin: opts[:origin], file: nil, dirty: true, + persistence_warnings: [], section_infos: initial_section_infos(notebook), cell_infos: initial_cell_infos(notebook), input_values: initial_input_values(notebook), @@ -850,9 +852,10 @@ defmodule Livebook.Session.Data do |> wrap_ok() end - def apply_operation(data, {:mark_as_not_dirty, _client_id}) do + def apply_operation(data, {:notebook_saved, _client_id, persistence_warnings}) do data |> with_actions() + |> set!(persistence_warnings: persistence_warnings) |> set_dirty(false) |> wrap_ok() end diff --git a/lib/livebook/stamping.ex b/lib/livebook/stamping.ex new file mode 100644 index 000000000..f80ea303b --- /dev/null +++ b/lib/livebook/stamping.ex @@ -0,0 +1,68 @@ +defmodule Livebook.Stamping do + @moduledoc false + + # Cryptographic functions used to implement notebook stamping. + + @doc """ + Performs authenticated encryption with associated data (AEAD) [1]. + + Uses AES-GCM-128 [2]. Returns a single token which carries encrypted + `payload` and signature for both `payload` and `additional_data`. + + [1]: https://en.wikipedia.org/wiki/Authenticated_encryption#Authenticated_encryption_with_associated_data_(AEAD) + [2]: https://www.rfc-editor.org/rfc/rfc5116#section-5 + """ + @spec aead_encrypt(term(), String.t(), String.t()) :: String.t() + def aead_encrypt(payload, additional_data, secret_key) do + {secret, sign_secret} = derive_keys(secret_key) + + payload = :erlang.term_to_binary(payload) + Plug.Crypto.MessageEncryptor.encrypt(payload, additional_data, secret, sign_secret) + end + + @doc """ + Decrypts and verifies data obtained from `aead_encrypt/3`. + """ + @spec aead_decrypt(String.t(), String.t(), String.t()) :: {:ok, term()} | :error + def aead_decrypt(encrypted, additional_data, secret_key) do + {secret, sign_secret} = derive_keys(secret_key) + + case Plug.Crypto.MessageEncryptor.decrypt(encrypted, additional_data, secret, sign_secret) do + {:ok, payload} -> + payload = Plug.Crypto.non_executable_binary_to_term(payload) + {:ok, payload} + + :error -> + :error + end + end + + defp derive_keys(secret_key) do + binary_key = Base.url_decode64!(secret_key, padding: false) + + <> = + Plug.Crypto.KeyGenerator.generate(binary_key, "notebook signing", + length: 32, + cache: Plug.Crypto.Keys + ) + + {secret, sign_secret} + end + + @doc """ + Verifies RSA `signature` of the given `payload` using the public key. + """ + @spec rsa_verify?(String.t(), String.t(), String.t()) :: boolean() + def rsa_verify?(signature, payload, public_key) do + der_key = Base.url_decode64!(public_key, padding: false) + raw_key = :public_key.der_decode(:RSAPublicKey, der_key) + + case Base.url_decode64(signature, padding: false) do + {:ok, raw_signature} -> + :public_key.verify(payload, :sha256, raw_signature, raw_key) + + :error -> + false + end + end +end diff --git a/lib/livebook/teams.ex b/lib/livebook/teams.ex index 752ba16eb..db00f6564 100644 --- a/lib/livebook/teams.ex +++ b/lib/livebook/teams.ex @@ -3,7 +3,7 @@ defmodule Livebook.Teams do alias Livebook.Hubs alias Livebook.Hubs.Team - alias Livebook.Teams.{HTTP, Org} + alias Livebook.Teams.{Requests, Org} import Ecto.Changeset, only: [add_error: 3, apply_action: 2, apply_action!: 2, get_field: 2] @@ -18,7 +18,7 @@ defmodule Livebook.Teams do | {:error, Ecto.Changeset.t()} | {:transport_error, String.t()} def create_org(%Org{} = org, attrs) do - create_org_request(org, attrs, &HTTP.create_org/1) + create_org_request(org, attrs, &Requests.create_org/1) end @doc """ @@ -32,7 +32,7 @@ defmodule Livebook.Teams do | {:error, Ecto.Changeset.t()} | {:transport_error, String.t()} def join_org(%Org{} = org, attrs) do - create_org_request(org, attrs, &HTTP.join_org/1) + create_org_request(org, attrs, &Requests.join_org/1) end defp create_org_request(%Org{} = org, attrs, callback) when is_function(callback, 1) do @@ -74,7 +74,7 @@ defmodule Livebook.Teams do | {:error, :expired} | {:transport_error, String.t()} def get_org_request_completion_data(%Org{id: id}, device_code) do - case HTTP.get_org_request_completion_data(id, device_code) do + case Requests.get_org_request_completion_data(id, device_code) do {:ok, %{"status" => "awaiting_confirmation"}} -> {:ok, :awaiting_confirmation} {:ok, completion_data} -> {:ok, completion_data} {:error, %{"status" => "expired"}} -> {:error, :expired} @@ -82,6 +82,19 @@ defmodule Livebook.Teams do end end + @doc """ + Send a request to Livebook Teams API to get an org request. + """ + @spec org_sign(Team.t(), String.t()) :: + {:ok, String.t()} + | {:transport_error, String.t()} + def org_sign(team, payload) do + case Requests.org_sign(team, payload) do + {:ok, %{"signature" => signature}} -> {:ok, signature} + any -> any + end + end + @doc """ Creates a Hub. diff --git a/lib/livebook/teams/http.ex b/lib/livebook/teams/requests.ex similarity index 70% rename from lib/livebook/teams/http.ex rename to lib/livebook/teams/requests.ex index 8a3a3014c..d80f3310b 100644 --- a/lib/livebook/teams/http.ex +++ b/lib/livebook/teams/requests.ex @@ -1,7 +1,8 @@ -defmodule Livebook.Teams.HTTP do +defmodule Livebook.Teams.Requests do @moduledoc false alias Livebook.Teams.Org + alias Livebook.Hubs.Team alias Livebook.Utils.HTTP @doc """ @@ -10,7 +11,7 @@ defmodule Livebook.Teams.HTTP do @spec create_org(Org.t()) :: {:ok, map()} | {:error, map() | String.t()} | {:transport_error, String.t()} def create_org(org) do - post("/api/org-request", %{name: org.name, key_hash: Org.key_hash(org)}) + post("/api/v1/org-request", %{name: org.name, key_hash: Org.key_hash(org)}) end @doc """ @@ -19,7 +20,7 @@ defmodule Livebook.Teams.HTTP do @spec join_org(Org.t()) :: {:ok, map()} | {:error, map() | String.t()} | {:transport_error, String.t()} def join_org(org) do - post("/api/org-request/join", %{name: org.name, key_hash: Org.key_hash(org)}) + post("/api/v1/org-request/join", %{name: org.name, key_hash: Org.key_hash(org)}) end @doc """ @@ -28,7 +29,22 @@ defmodule Livebook.Teams.HTTP do @spec get_org_request_completion_data(pos_integer(), binary) :: {:ok, map()} | {:error, map() | String.t()} | {:transport_error, String.t()} def get_org_request_completion_data(id, device_code) do - get("/api/org-request/#{id}?device_code=#{device_code}") + get("/api/v1/org-request/#{id}?device_code=#{device_code}") + end + + @doc """ + Send a request to Livebook Team API to sign the given payload. + """ + @spec org_sign(Team.t(), String.t()) :: + {:ok, map()} | {:error, map() | String.t()} | {:transport_error, String.t()} + def org_sign(team, payload) do + headers = auth_headers(team) + post("/api/v1/org/sign", %{payload: payload}, headers) + end + + defp auth_headers(team) do + token = "#{team.user_id}:#{team.org_id}:#{team.org_key_id}:#{team.session_token}" + [{"authorization", "Bearer " <> token}] end defp post(path, json, headers \\ []) do diff --git a/lib/livebook_web/controllers/session_controller.ex b/lib/livebook_web/controllers/session_controller.ex index 2e5ae378b..6ad719541 100644 --- a/lib/livebook_web/controllers/session_controller.ex +++ b/lib/livebook_web/controllers/session_controller.ex @@ -29,7 +29,7 @@ defmodule LivebookWeb.SessionController do defp send_notebook_source(conn, notebook, file_name, "livemd" = format) do opts = [include_outputs: conn.params["include_outputs"] == "true"] - source = Livebook.LiveMarkdown.notebook_to_livemd(notebook, opts) + {source, _warnings} = Livebook.LiveMarkdown.notebook_to_livemd(notebook, opts) send_download(conn, {:binary, source}, filename: file_name <> "." <> format, diff --git a/lib/livebook_web/live/app_session_live/source_component.ex b/lib/livebook_web/live/app_session_live/source_component.ex index 02e9704b6..0d54547d6 100644 --- a/lib/livebook_web/live/app_session_live/source_component.ex +++ b/lib/livebook_web/live/app_session_live/source_component.ex @@ -13,7 +13,16 @@ defmodule LivebookWeb.AppSessionLive.SourceComponent do # the whole notebook in assigns notebook = Session.get_notebook(socket.assigns.session.pid) - Livebook.LiveMarkdown.notebook_to_livemd(notebook, include_outputs: false) + # We ignore the stamp, since it's not relevant for end-users, + # and this way we don't generate the stamp every time they + # look at the source + {source, _warnings} = + Livebook.LiveMarkdown.notebook_to_livemd(notebook, + include_outputs: false, + include_stamp: false + ) + + source end) {:ok, socket} diff --git a/lib/livebook_web/live/file_select_component.ex b/lib/livebook_web/live/file_select_component.ex index a2fe6d153..9a124ec80 100644 --- a/lib/livebook_web/live/file_select_component.ex +++ b/lib/livebook_web/live/file_select_component.ex @@ -668,7 +668,8 @@ defmodule LivebookWeb.FileSelectComponent do defp create_notebook(_parent_dir, ""), do: {:error, :ignore} defp create_notebook(parent_dir, name) do - source = Livebook.Session.default_notebook() |> Livebook.LiveMarkdown.notebook_to_livemd() + {source, _warnings} = + Livebook.Session.default_notebook() |> Livebook.LiveMarkdown.notebook_to_livemd() new_file = parent_dir diff --git a/lib/livebook_web/live/hub/edit/team_component.ex b/lib/livebook_web/live/hub/edit/team_component.ex index d738e6e8f..93398cfea 100644 --- a/lib/livebook_web/live/hub/edit/team_component.ex +++ b/lib/livebook_web/live/hub/edit/team_component.ex @@ -137,9 +137,11 @@ defmodule LivebookWeb.Hub.Edit.TeamComponent do <.emoji_field field={f[:hub_emoji]} label="Emoji" /> - +
+ +
diff --git a/lib/livebook_web/live/hub/new_live.ex b/lib/livebook_web/live/hub/new_live.ex index 1bd408796..ac08b5b70 100644 --- a/lib/livebook_web/live/hub/new_live.ex +++ b/lib/livebook_web/live/hub/new_live.ex @@ -318,6 +318,7 @@ defmodule LivebookWeb.Hub.NewLive do org_id: response["id"], user_id: response["user_id"], org_key_id: response["org_key_id"], + org_public_key: response["org_public_key"], session_token: response["session_token"], teams_key: org.teams_key, hub_name: org.name, diff --git a/lib/livebook_web/live/session_live.ex b/lib/livebook_web/live/session_live.ex index 9be6deb7e..2705cdd7d 100644 --- a/lib/livebook_web/live/session_live.ex +++ b/lib/livebook_web/live/session_live.ex @@ -236,6 +236,7 @@ defmodule LivebookWeb.SessionLive do session_id={@session.id} file={@data_view.file} dirty={@data_view.dirty} + persistence_warnings={@data_view.persistence_warnings} autosave_interval_s={@data_view.autosave_interval_s} runtime={@data_view.runtime} global_status={@data_view.global_status} @@ -2141,6 +2142,7 @@ defmodule LivebookWeb.SessionLive do autosave_interval_s: data.notebook.autosave_interval_s, default_language: data.notebook.default_language, dirty: data.dirty, + persistence_warnings: data.persistence_warnings, runtime: data.runtime, smart_cell_definitions: data.smart_cell_definitions, code_block_definitions: Livebook.Runtime.code_block_definitions(data.runtime), diff --git a/lib/livebook_web/live/session_live/export_live_markdown_component.ex b/lib/livebook_web/live/session_live/export_live_markdown_component.ex index a07bb4181..9aaebcca3 100644 --- a/lib/livebook_web/live/session_live/export_live_markdown_component.ex +++ b/lib/livebook_web/live/session_live/export_live_markdown_component.ex @@ -14,7 +14,7 @@ defmodule LivebookWeb.SessionLive.ExportLiveMarkdownComponent do end defp assign_source(%{assigns: assigns} = socket) do - source = + {source, _warnings} = Livebook.LiveMarkdown.notebook_to_livemd(assigns.notebook, include_outputs: assigns.include_outputs ) diff --git a/lib/livebook_web/live/session_live/indicators_component.ex b/lib/livebook_web/live/session_live/indicators_component.ex index bb60882eb..eb41f4eb5 100644 --- a/lib/livebook_web/live/session_live/indicators_component.ex +++ b/lib/livebook_web/live/session_live/indicators_component.ex @@ -37,6 +37,7 @@ defmodule LivebookWeb.SessionLive.IndicatorsComponent do <.persistence_indicator file={@file} dirty={@dirty} + persistence_warnings={@persistence_warnings} autosave_interval_s={@autosave_interval_s} session_id={@session_id} /> @@ -105,13 +106,29 @@ defmodule LivebookWeb.SessionLive.IndicatorsComponent do defp persistence_indicator(%{dirty: false} = assigns) do ~H""" - + + "Notebook saved" + + warnings -> + "Notebook saved with warnings:\n" <> Enum.map_join(warnings, "\n", &("- " <> &1)) + end + } + > <.link patch={~p"/sessions/#{@session_id}/settings/file"} - class="icon-button icon-outlined-button border-green-bright-300 hover:bg-green-bright-50 focus:bg-green-bright-50" + class="icon-button icon-outlined-button border-green-bright-300 hover:bg-green-bright-50 focus:bg-green-bright-50 relative" aria-label="notebook saved, click to open file settings" > <.remix_icon icon="save-line" class="text-xl text-green-bright-400" /> + <.remix_icon + :if={@persistence_warnings != []} + icon="error-warning-fill" + class="text-lg text-red-400 absolute -top-1.5 -right-2" + /> """ diff --git a/lib/livebook_web/live/session_live/secrets_list_component.ex b/lib/livebook_web/live/session_live/secrets_list_component.ex index 2f19e73f6..343c5dfbc 100644 --- a/lib/livebook_web/live/session_live/secrets_list_component.ex +++ b/lib/livebook_web/live/session_live/secrets_list_component.ex @@ -74,7 +74,7 @@ defmodule LivebookWeb.SessionLive.SecretsListComponent do <%= if @hub_secrets == [] do %> - No secrets stored in Livebook so far + No secrets stored in this hub so far <% else %> Toggle to share with this session <% end %> diff --git a/mix.exs b/mix.exs index 5f7523923..0db4e7386 100644 --- a/mix.exs +++ b/mix.exs @@ -30,7 +30,16 @@ defmodule Livebook.MixProject do def application do [ mod: {Livebook.Application, []}, - extra_applications: [:logger, :runtime_tools, :os_mon, :inets, :ssl, :xmerl], + extra_applications: [ + :logger, + :runtime_tools, + :os_mon, + :inets, + :ssl, + :xmerl, + :crypto, + :public_key + ], env: Application.get_all_env(:livebook) ] end diff --git a/test/livebook/hubs/team_client_test.exs b/test/livebook/hubs/team_client_test.exs index 7a7bf5d75..81540ca2d 100644 --- a/test/livebook/hubs/team_client_test.exs +++ b/test/livebook/hubs/team_client_test.exs @@ -1,9 +1,10 @@ defmodule Livebook.Hubs.TeamClientTest do use Livebook.TeamsIntegrationCase, async: true - @moduletag :capture_log alias Livebook.Hubs.TeamClient + @moduletag :capture_log + setup do Livebook.Hubs.subscribe([:connection]) :ok @@ -13,6 +14,7 @@ defmodule Livebook.Hubs.TeamClientTest do test "successfully authenticates the web socket connection", %{user: user, node: node} do org = :erpc.call(node, Hub.Integration, :create_org, []) org_key = :erpc.call(node, Hub.Integration, :create_org_key, [[org: org]]) + org_key_pair = :erpc.call(node, Hub.Integration, :create_org_key_pair, [[org: org]]) token = :erpc.call(node, Hub.Integration, :associate_user_with_org, [user, org]) team = @@ -22,6 +24,7 @@ defmodule Livebook.Hubs.TeamClientTest do user_id: user.id, org_id: org.id, org_key_id: org_key.id, + org_public_key: org_key_pair.public_key, session_token: token ) diff --git a/test/livebook/hubs/team_test.exs b/test/livebook/hubs/team_test.exs new file mode 100644 index 000000000..130f5effe --- /dev/null +++ b/test/livebook/hubs/team_test.exs @@ -0,0 +1,45 @@ +defmodule Livebook.Hubs.TeamTest do + use Livebook.TeamsIntegrationCase, async: true + + alias Livebook.Hubs.Provider + + @moduletag :capture_log + + describe "stamping" do + test "generates and verifies stamp for a notebook", %{user: user, node: node} do + org = :erpc.call(node, Hub.Integration, :create_org, []) + org_key = :erpc.call(node, Hub.Integration, :create_org_key, [[org: org]]) + org_key_pair = :erpc.call(node, Hub.Integration, :create_org_key_pair, [[org: org]]) + token = :erpc.call(node, Hub.Integration, :associate_user_with_org, [user, org]) + + team = + build(:team, + id: "team-#{org.name}", + hub_name: org.name, + user_id: user.id, + org_id: org.id, + org_key_id: org_key.id, + org_public_key: org_key_pair.public_key, + session_token: token + ) + + notebook_source = """ + # Team notebook + + # Intro + + ```elixir + IO.puts("Hello!") + ``` + """ + + metadata = %{"key" => "value"} + + assert {:ok, stamp} = Provider.notebook_stamp(team, notebook_source, metadata) + + assert {:ok, ^metadata} = Provider.verify_notebook_stamp(team, notebook_source, stamp) + + assert :error = Provider.verify_notebook_stamp(team, notebook_source <> "change\n", stamp) + end + end +end diff --git a/test/livebook/live_markdown/export_test.exs b/test/livebook/live_markdown/export_test.exs index 6df9e092c..280c6678a 100644 --- a/test/livebook/live_markdown/export_test.exs +++ b/test/livebook/live_markdown/export_test.exs @@ -150,7 +150,7 @@ defmodule Livebook.LiveMarkdown.ExportTest do ``` """ - document = Export.notebook_to_livemd(notebook) + {document, []} = Export.notebook_to_livemd(notebook) assert expected_document == document end @@ -189,7 +189,7 @@ defmodule Livebook.LiveMarkdown.ExportTest do | Maine | ME | Augusta | """ - document = Export.notebook_to_livemd(notebook) + {document, []} = Export.notebook_to_livemd(notebook) assert expected_document == document end @@ -226,7 +226,7 @@ defmodule Livebook.LiveMarkdown.ExportTest do ### Heading 3 """ - document = Export.notebook_to_livemd(notebook) + {document, []} = Export.notebook_to_livemd(notebook) assert expected_document == document end @@ -271,7 +271,7 @@ defmodule Livebook.LiveMarkdown.ExportTest do ``` """ - document = Export.notebook_to_livemd(notebook) + {document, []} = Export.notebook_to_livemd(notebook) assert expected_document == document end @@ -336,7 +336,7 @@ defmodule Livebook.LiveMarkdown.ExportTest do ``` """ - document = Export.notebook_to_livemd(notebook) + {document, []} = Export.notebook_to_livemd(notebook) assert expected_document == document end @@ -372,7 +372,7 @@ defmodule Livebook.LiveMarkdown.ExportTest do ``` """ - document = Export.notebook_to_livemd(notebook) + {document, []} = Export.notebook_to_livemd(notebook) assert expected_document == document end @@ -410,7 +410,7 @@ defmodule Livebook.LiveMarkdown.ExportTest do ``` """ - document = Export.notebook_to_livemd(notebook) + {document, []} = Export.notebook_to_livemd(notebook) assert expected_document == document end @@ -461,7 +461,7 @@ defmodule Livebook.LiveMarkdown.ExportTest do ````` """ - document = Export.notebook_to_livemd(notebook) + {document, []} = Export.notebook_to_livemd(notebook) assert expected_document == document end @@ -504,7 +504,7 @@ defmodule Livebook.LiveMarkdown.ExportTest do Cell 2 """ - document = Export.notebook_to_livemd(notebook) + {document, []} = Export.notebook_to_livemd(notebook) assert expected_document == document end @@ -554,7 +554,7 @@ defmodule Livebook.LiveMarkdown.ExportTest do Cell 1 """ - document = Export.notebook_to_livemd(notebook) + {document, []} = Export.notebook_to_livemd(notebook) assert expected_document == document end @@ -593,7 +593,7 @@ defmodule Livebook.LiveMarkdown.ExportTest do ``` """ - document = Export.notebook_to_livemd(notebook) + {document, []} = Export.notebook_to_livemd(notebook) assert expected_document == document end @@ -635,7 +635,7 @@ defmodule Livebook.LiveMarkdown.ExportTest do ``` """ - document = Export.notebook_to_livemd(notebook, include_outputs: true) + {document, []} = Export.notebook_to_livemd(notebook, include_outputs: true) assert expected_document == document end @@ -683,7 +683,7 @@ defmodule Livebook.LiveMarkdown.ExportTest do ``` """ - document = Export.notebook_to_livemd(notebook, include_outputs: true) + {document, []} = Export.notebook_to_livemd(notebook, include_outputs: true) assert expected_document == document end @@ -719,7 +719,7 @@ defmodule Livebook.LiveMarkdown.ExportTest do ``` """ - document = Export.notebook_to_livemd(notebook, include_outputs: true) + {document, []} = Export.notebook_to_livemd(notebook, include_outputs: true) assert expected_document == document end @@ -764,7 +764,7 @@ defmodule Livebook.LiveMarkdown.ExportTest do ``` """ - document = Export.notebook_to_livemd(notebook, include_outputs: true) + {document, []} = Export.notebook_to_livemd(notebook, include_outputs: true) assert expected_document == document end @@ -816,7 +816,7 @@ defmodule Livebook.LiveMarkdown.ExportTest do ``` """ - document = Export.notebook_to_livemd(notebook, include_outputs: true) + {document, []} = Export.notebook_to_livemd(notebook, include_outputs: true) assert expected_document == document end @@ -867,7 +867,7 @@ defmodule Livebook.LiveMarkdown.ExportTest do ``` """ - document = Export.notebook_to_livemd(notebook, include_outputs: true) + {document, []} = Export.notebook_to_livemd(notebook, include_outputs: true) assert expected_document == document end @@ -922,7 +922,7 @@ defmodule Livebook.LiveMarkdown.ExportTest do ``` """ - document = Export.notebook_to_livemd(notebook, include_outputs: true) + {document, []} = Export.notebook_to_livemd(notebook, include_outputs: true) assert expected_document == document end @@ -971,7 +971,7 @@ defmodule Livebook.LiveMarkdown.ExportTest do ``` """ - document = Export.notebook_to_livemd(notebook, include_outputs: true) + {document, []} = Export.notebook_to_livemd(notebook, include_outputs: true) assert expected_document == document end @@ -1025,7 +1025,7 @@ defmodule Livebook.LiveMarkdown.ExportTest do ``` """ - document = Export.notebook_to_livemd(notebook, include_outputs: true) + {document, []} = Export.notebook_to_livemd(notebook, include_outputs: true) assert expected_document == document end @@ -1070,7 +1070,7 @@ defmodule Livebook.LiveMarkdown.ExportTest do ``` """ - document = Export.notebook_to_livemd(notebook) + {document, []} = Export.notebook_to_livemd(notebook) assert expected_document == document end @@ -1109,7 +1109,7 @@ defmodule Livebook.LiveMarkdown.ExportTest do ``` """ - document = Export.notebook_to_livemd(notebook, include_outputs: false) + {document, []} = Export.notebook_to_livemd(notebook, include_outputs: false) assert expected_document == document end @@ -1127,7 +1127,7 @@ defmodule Livebook.LiveMarkdown.ExportTest do # My Notebook """ - document = Export.notebook_to_livemd(notebook) + {document, []} = Export.notebook_to_livemd(notebook) assert expected_document == document end @@ -1145,27 +1145,27 @@ defmodule Livebook.LiveMarkdown.ExportTest do # My Notebook """ - document = Export.notebook_to_livemd(notebook) + {document, []} = Export.notebook_to_livemd(notebook) assert expected_document == document end test "persists hub id when not default" do - Livebook.Factory.build(:team, id: "team-persisted-id") + %{id: hub_id} = Livebook.Factory.build(:team) notebook = %{ Notebook.new() | name: "My Notebook", - hub_id: "team-persisted-id" + hub_id: hub_id } expected_document = """ - + # My Notebook """ - document = Export.notebook_to_livemd(notebook) + {document, []} = Export.notebook_to_livemd(notebook) assert expected_document == document end @@ -1194,7 +1194,7 @@ defmodule Livebook.LiveMarkdown.ExportTest do # My Notebook """ - document = Export.notebook_to_livemd(notebook) + {document, []} = Export.notebook_to_livemd(notebook) assert expected_document == document end @@ -1217,7 +1217,7 @@ defmodule Livebook.LiveMarkdown.ExportTest do # My Notebook """ - document = Export.notebook_to_livemd(notebook) + {document, []} = Export.notebook_to_livemd(notebook) assert expected_document == document end @@ -1243,7 +1243,7 @@ defmodule Livebook.LiveMarkdown.ExportTest do ## Section 1 """ - document = Export.notebook_to_livemd(notebook) + {document, []} = Export.notebook_to_livemd(notebook) assert expected_document == document end @@ -1282,11 +1282,47 @@ defmodule Livebook.LiveMarkdown.ExportTest do """ - document = Export.notebook_to_livemd(notebook) + {document, []} = Export.notebook_to_livemd(notebook) assert document =~ expected_document end + test "skips notebook stamp if :include_stamp is false" do + notebook = %{ + Notebook.new() + | name: "My Notebook", + sections: [ + %{ + Notebook.Section.new() + | name: "Section 1", + cells: [ + %{ + Notebook.Cell.new(:code) + | source: """ + IO.puts("hey") + """ + } + ] + } + ], + hub_secret_names: ["DB_PASSWORD"] + } + + expected_document = """ + # My Notebook + + ## Section 1 + + ```elixir + IO.puts("hey") + ``` + """ + + {document, []} = Export.notebook_to_livemd(notebook, include_stamp: false) + + assert expected_document == document + end + defp spawn_widget_with_data(ref, data) do spawn(fn -> receive do diff --git a/test/livebook/live_markdown/import_test.exs b/test/livebook/live_markdown/import_test.exs index 0713941a8..79b38b04d 100644 --- a/test/livebook/live_markdown/import_test.exs +++ b/test/livebook/live_markdown/import_test.exs @@ -743,17 +743,17 @@ defmodule Livebook.LiveMarkdown.ImportTest do end test "imports notebook hub id when exists" do - Livebook.Factory.insert_hub(:team, id: "team-persisted-id") + %{id: hub_id} = Livebook.Factory.insert_hub(:team) markdown = """ - + # My Notebook """ {notebook, []} = Import.notebook_from_livemd(markdown) - assert %Notebook{name: "My Notebook", hub_id: "team-persisted-id"} = notebook + assert %Notebook{name: "My Notebook", hub_id: ^hub_id} = notebook end test "imports ignores hub id when does not exist" do @@ -1081,6 +1081,7 @@ defmodule Livebook.LiveMarkdown.ImportTest do # hub_secret_names: ["DB_PASSWORD"] # } # |> Livebook.LiveMarkdown.Export.notebook_to_livemd() + # |> elem(0) # |> IO.puts() markdown = """ diff --git a/test/livebook/session/data_test.exs b/test/livebook/session/data_test.exs index 22785cb5e..488d96e71 100644 --- a/test/livebook/session/data_test.exs +++ b/test/livebook/session/data_test.exs @@ -1843,7 +1843,7 @@ defmodule Livebook.Session.DataTest do evaluate_cells_operations(["setup"]), {:queue_cells_evaluation, @cid, ["c1"]}, {:set_notebook_attributes, @cid, %{persist_outputs: true}}, - {:mark_as_not_dirty, @cid} + {:notebook_saved, @cid, []} ]) operation = {:add_cell_evaluation_output, @cid, "c1", {:stdout, "Hello!"}} @@ -2333,7 +2333,7 @@ defmodule Livebook.Session.DataTest do evaluate_cells_operations(["setup"]), {:queue_cells_evaluation, @cid, ["c1"]}, {:set_notebook_attributes, @cid, %{persist_outputs: true}}, - {:mark_as_not_dirty, @cid} + {:notebook_saved, @cid, []} ]) operation = {:add_cell_evaluation_response, @cid, "c1", {:ok, [1, 2, 3]}, eval_meta()} @@ -3785,14 +3785,14 @@ defmodule Livebook.Session.DataTest do end end - describe "apply_operation/2 given :mark_as_not_dirty" do + describe "apply_operation/2 given :notebook_saved" do test "sets dirty flag to false" do data = data_after_operations!([ {:insert_section, @cid, 0, "s1"} ]) - operation = {:mark_as_not_dirty, @cid} + operation = {:notebook_saved, @cid, []} assert {:ok, %{dirty: false}, []} = Data.apply_operation(data, operation) end diff --git a/test/livebook/session_test.exs b/test/livebook/session_test.exs index 1ee217fca..38fe44a7b 100644 --- a/test/livebook/session_test.exs +++ b/test/livebook/session_test.exs @@ -553,7 +553,7 @@ defmodule Livebook.SessionTest do Session.save(session.pid) - assert_receive {:operation, {:mark_as_not_dirty, _}} + assert_receive {:operation, {:notebook_saved, _, []}} assert {:ok, "# My notebook\n" <> _rest} = FileSystem.File.read(file) end @@ -573,7 +573,7 @@ defmodule Livebook.SessionTest do Session.save(session.pid) - assert_receive {:operation, {:mark_as_not_dirty, _}} + assert_receive {:operation, {:notebook_saved, _, []}} assert {:ok, "# My notebook\n" <> _rest} = FileSystem.File.read(file) end end @@ -1154,7 +1154,7 @@ defmodule Livebook.SessionTest do Session.subscribe(session.id) Session.save(session.pid) - assert_receive {:operation, {:mark_as_not_dirty, _}} + assert_receive {:operation, {:notebook_saved, _, []}} assert [notebook_path] = Path.wildcard(notebook_glob) assert Path.basename(notebook_path) =~ "untitled_notebook" @@ -1163,7 +1163,7 @@ defmodule Livebook.SessionTest do Session.set_notebook_name(session.pid, "Cat's guide to life") Session.save(session.pid) - assert_receive {:operation, {:mark_as_not_dirty, _}} + assert_receive {:operation, {:notebook_saved, _, []}} assert [notebook_path] = Path.wildcard(notebook_glob) assert Path.basename(notebook_path) =~ "cats_guide_to_life" diff --git a/test/livebook/teams_test.exs b/test/livebook/teams_test.exs index c0338f99f..cef48cdc2 100644 --- a/test/livebook/teams_test.exs +++ b/test/livebook/teams_test.exs @@ -80,7 +80,12 @@ defmodule Livebook.TeamsTest do %{ token: token, user_org: %{ - org: %{id: id, name: name, keys: [%{id: org_key_id}]}, + org: %{ + id: id, + name: name, + keys: [%{id: org_key_id}], + key_pair: %{public_key: org_public_key} + }, user: %{id: user_id} } } = org_request.user_org_session @@ -91,6 +96,7 @@ defmodule Livebook.TeamsTest do "id" => id, "name" => name, "org_key_id" => org_key_id, + "org_public_key" => org_public_key, "session_token" => token, "user_id" => user_id }} diff --git a/test/livebook_web/live/hub/new_live_test.exs b/test/livebook_web/live/hub/new_live_test.exs index 9656008b6..2b1ec632e 100644 --- a/test/livebook_web/live/hub/new_live_test.exs +++ b/test/livebook_web/live/hub/new_live_test.exs @@ -74,6 +74,7 @@ defmodule LivebookWeb.Hub.NewLiveTest do # previously create the org and associate user with org org = :erpc.call(node, Hub.Integration, :create_org, [[name: name]]) :erpc.call(node, Hub.Integration, :create_org_key, [[org: org, key_hash: key_hash]]) + :erpc.call(node, Hub.Integration, :create_org_key_pair, [[org: org]]) :erpc.call(node, Hub.Integration, :create_user_org, [[org: org, user: user]]) # select the new org option