livebook-dev/livebook
1
1
Fork 0
mirror of https://github.com/livebook-dev/livebook.git synced 2024-09-20 10:05:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactor the LivebookProto messages (#2521)

Browse source
This commit is contained in:
Alexandre de Souza 2024-03-25 15:08:40 -03:00 committed by GitHub
parent 84ee13fd1b
commit fdd2d2c2f2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
24 changed files with 369 additions and 362 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

194
lib/livebook/hubs/team_client.ex
View file

@ -19,7 +19,8 @@ defmodule Livebook.Hubs.TeamClient do
connected?: false,
secrets: [],
file_systems: [],
deployment_groups: []
deployment_groups: [],
app_deployments: []
]
@type registry_name :: {:via, Registry, {Livebook.HubsRegistry, String.t()}}
@ -78,6 +79,14 @@ defmodule Livebook.Hubs.TeamClient do
GenServer.call(registry_name(id), :get_deployment_groups)
end
@doc """
Returns a list of cached app deployments.
"""
@spec get_app_deployments(String.t()) :: list(Teams.AppDeployment.t())
def get_app_deployments(id) do
GenServer.call(registry_name(id), :get_app_deployments)
end
@doc """
Returns if the Team client is connected.
"""
@ -168,6 +177,10 @@ defmodule Livebook.Hubs.TeamClient do
{:reply, state.deployment_groups, state}
end
def handle_call(:get_app_deployments, _caller, state) do
{:reply, state.app_deployments, state}
end
@impl true
def handle_info(:connected, state) do
Hubs.Broadcasts.hub_connected(state.hub.id)
@ -265,32 +278,17 @@ defmodule Livebook.Hubs.TeamClient do
}
end
defp put_agent_key(deployment_group, agent_key) do
deployment_group = remove_agent_key(deployment_group, agent_key)
agent_keys = [agent_key | deployment_group.agent_keys]
defp put_app_deployment(state, app_deployment) do
state = remove_app_deployment(state, app_deployment)
app_deployments = [app_deployment | state.app_deployments]
%{deployment_group | agent_keys: Enum.sort_by(agent_keys, & &1.id)}
%{state | app_deployments: Enum.sort_by(app_deployments, & &1.slug)}
end
defp remove_agent_key(deployment_group, agent_key) do
defp remove_app_deployment(state, app_deployment) do
%{
deployment_group
| agent_keys: Enum.reject(deployment_group.agent_keys, &(&1.id == agent_key.id))
}
end
defp put_app_deployment(deployment_group, app_deployment) do
deployment_group = remove_app_deployment(deployment_group, app_deployment)
app_deployments = [app_deployment | deployment_group.app_deployments]
%{deployment_group | app_deployments: Enum.sort_by(app_deployments, & &1.slug)}
end
defp remove_app_deployment(deployment_group, app_deployment) do
%{
deployment_group
| app_deployments:
Enum.reject(deployment_group.app_deployments, &(&1.slug == app_deployment.slug))
state
| app_deployments: Enum.reject(state.app_deployments, &(&1.id == app_deployment.id))
}
end
@ -302,10 +300,9 @@ defmodule Livebook.Hubs.TeamClient do
}
end
defp build_deployment_group(state, deployment_group) do
defp build_deployment_group(state, %LivebookProto.DeploymentGroup{} = deployment_group) do
secrets = Enum.map(deployment_group.secrets, &build_secret(state, &1))
agent_keys = Enum.map(deployment_group.agent_keys, &build_agent_key/1)
app_deployments = Enum.map(deployment_group.deployed_apps, &build_app_deployment/1)
%Teams.DeploymentGroup{
id: deployment_group.id,
@ -314,26 +311,54 @@ defmodule Livebook.Hubs.TeamClient do
hub_id: state.hub.id,
secrets: secrets,
agent_keys: agent_keys,
app_deployments: app_deployments,
clustering: nullify(deployment_group.clustering),
zta_provider: atomize(deployment_group.zta_provider),
zta_key: nullify(deployment_group.zta_key)
}
end
defp build_app_deployment(app_deployment) do
path = URI.parse(app_deployment.archive_url).path
defp build_deployment_group(state, %{mode: _} = deployment_group_created) do
agent_keys = Enum.map(deployment_group_created.agent_keys, &build_agent_key/1)
%Teams.DeploymentGroup{
id: deployment_group_created.id,
name: deployment_group_created.name,
mode: atomize(deployment_group_created.mode),
hub_id: state.hub.id,
secrets: [],
agent_keys: agent_keys,
clustering: nullify(deployment_group_created.clustering),
zta_provider: atomize(deployment_group_created.zta_provider),
zta_key: nullify(deployment_group_created.zta_key)
}
end
defp build_deployment_group(state, deployment_group_updated) do
secrets = Enum.map(deployment_group_updated.secrets, &build_secret(state, &1))
agent_keys = Enum.map(deployment_group_updated.agent_keys, &build_agent_key/1)
{:ok, deployment_group} = fetch_deployment_group(deployment_group_updated.id, state)
%{
deployment_group
| name: deployment_group_updated.name,
secrets: secrets,
agent_keys: agent_keys,
clustering: nullify(deployment_group_updated.clustering),
zta_provider: atomize(deployment_group_updated.zta_provider),
zta_key: nullify(deployment_group_updated.zta_key)
}
end
defp build_app_deployment(%LivebookProto.AppDeployment{} = app_deployment) do
%Teams.AppDeployment{
id: app_deployment.id,
filename: Path.basename(path),
slug: app_deployment.slug,
sha: app_deployment.sha,
title: app_deployment.title,
deployment_group_id: app_deployment.deployment_group_id,
file: {:url, app_deployment.archive_url},
file: nil,
deployed_by: app_deployment.deployed_by,
deployed_at: NaiveDateTime.from_iso8601!(app_deployment.deployed_at)
deployed_at: NaiveDateTime.from_gregorian_seconds(app_deployment.deployed_at)
}
end
@ -415,9 +440,7 @@ defmodule Livebook.Hubs.TeamClient do
end
defp handle_event(:deployment_group_updated, %Teams.DeploymentGroup{} = deployment_group, state) do
deployment_group = deploy_apps(state.deployment_group_id, deployment_group, state.derived_key)
Teams.Broadcasts.deployment_group_updated(deployment_group)
put_deployment_group(state, deployment_group)
end
@ -431,18 +454,20 @@ defmodule Livebook.Hubs.TeamClient do
defp handle_event(:deployment_group_deleted, deployment_group_deleted, state) do
with {:ok, deployment_group} <- fetch_deployment_group(deployment_group_deleted.id, state) do
undeploy_apps(state.deployment_group_id, deployment_group)
Teams.Broadcasts.deployment_group_deleted(deployment_group)
remove_deployment_group(state, deployment_group)
state
|> undeploy_apps(deployment_group)
|> remove_deployment_group(deployment_group)
end
end
defp handle_event(:user_connected, connected, state) do
defp handle_event(:user_connected, user_connected, state) do
state
|> dispatch_secrets(connected)
|> dispatch_file_systems(connected)
|> dispatch_deployment_groups(connected)
|> dispatch_secrets(user_connected)
|> dispatch_file_systems(user_connected)
|> dispatch_deployment_groups(user_connected)
|> dispatch_app_deployments(user_connected)
end
defp handle_event(:agent_connected, agent_connected, state) do
@ -451,42 +476,28 @@ defmodule Livebook.Hubs.TeamClient do
|> dispatch_secrets(agent_connected)
|> dispatch_file_systems(agent_connected)
|> dispatch_deployment_groups(agent_connected)
|> dispatch_app_deployments(agent_connected)
end
defp handle_event(:agent_key_created, agent_key_created, state) do
agent_key = build_agent_key(agent_key_created)
defp handle_event(:app_deployment_created, %Teams.AppDeployment{} = app_deployment, state) do
deployment_group_id = app_deployment.deployment_group_id
with {:ok, deployment_group} <- fetch_deployment_group(agent_key.deployment_group_id, state) do
deployment_group = put_agent_key(deployment_group, agent_key)
handle_event(:deployment_group_updated, deployment_group, state)
end
end
with {:ok, deployment_group} <- fetch_deployment_group(deployment_group_id, state) do
if deployment_group.id == state.deployment_group_id do
:ok = download_and_deploy(state.hub, app_deployment, state.derived_key)
end
defp handle_event(:agent_key_deleted, agent_key_deleted, state) do
agent_key = build_agent_key(agent_key_deleted)
with {:ok, deployment_group} <- fetch_deployment_group(agent_key.deployment_group_id, state) do
deployment_group = remove_agent_key(deployment_group, agent_key)
handle_event(:deployment_group_updated, deployment_group, state)
Teams.Broadcasts.app_deployment_created(app_deployment)
put_app_deployment(state, app_deployment)
end
end
defp handle_event(:app_deployment_created, app_deployment_created, state) do
app_deployment = build_app_deployment(app_deployment_created)
deployment_group_id = app_deployment.deployment_group_id
with {:ok, deployment_group} <- fetch_deployment_group(deployment_group_id, state) do
app_deployment =
if deployment_group_id == state.deployment_group_id do
{:ok, app_deployment} = download_and_deploy(app_deployment, state.derived_key)
app_deployment
else
app_deployment
end
deployment_group = put_app_deployment(deployment_group, app_deployment)
handle_event(:deployment_group_updated, deployment_group, state)
end
handle_event(
:app_deployment_created,
build_app_deployment(app_deployment_created.app_deployment),
state
)
end
defp dispatch_secrets(state, %{secrets: secrets}) do
@ -525,13 +536,11 @@ defmodule Livebook.Hubs.TeamClient do
)
end
defp dispatch_file_systems(state, _), do: state
defp dispatch_deployment_groups(state, %{deployment_groups: deployment_groups}) do
decrypted_deployment_groups = Enum.map(deployment_groups, &build_deployment_group(state, &1))
deployment_groups = Enum.map(deployment_groups, &build_deployment_group(state, &1))
{created, deleted, updated} =
diff(state.deployment_groups, decrypted_deployment_groups, &(&1.id == &2.id))
diff(state.deployment_groups, deployment_groups, &(&1.id == &2.id))
dispatch_events(state,
deployment_group_deleted: deleted,
@ -540,6 +549,13 @@ defmodule Livebook.Hubs.TeamClient do
)
end
defp dispatch_app_deployments(state, %{app_deployments: app_deployments}) do
app_deployments = Enum.map(app_deployments, &build_app_deployment/1)
{created, _, _} = diff(state.app_deployments, app_deployments, &(&1.id == &2.id))
dispatch_events(state, app_deployment_created: created)
end
defp update_hub(state, %{public_key: org_public_key}) do
hub = %{state.hub | org_public_key: org_public_key}
@ -588,39 +604,33 @@ defmodule Livebook.Hubs.TeamClient do
defp nullify(""), do: nil
defp nullify(value), do: value
defp deploy_apps(id, %{id: id} = deployment_group, derived_key) do
app_deployments =
for app_deployment <- deployment_group.app_deployments do
{:ok, app_deployment} = download_and_deploy(app_deployment, derived_key)
app_deployment
end
defp undeploy_apps(%{deployment_group_id: id} = state, %{id: id}) do
fun = &(&1.deployment_group_id == id)
app_deployments = Enum.filter(state.app_deployments, fun)
%{deployment_group | app_deployments: app_deployments}
end
defp deploy_apps(_, deployment_group, _), do: deployment_group
defp undeploy_apps(id, %{id: id} = deployment_group) do
for %{slug: slug} <- deployment_group.app_deployments do
for %{slug: slug} <- app_deployments do
:ok = undeploy_app(slug)
end
%{state | deployment_group_id: nil, app_deployments: Enum.reject(state.app_deployments, fun)}
end
defp undeploy_apps(_, _), do: :noop
defp download_and_deploy(%{file: nil} = app_deployment, _) do
app_deployment
defp undeploy_apps(state, %{id: id}) do
%{
state
| app_deployments: Enum.reject(state.app_deployments, &(&1.deployment_group_id == id))
}
end
defp download_and_deploy(%{file: {:url, archive_url}} = app_deployment, derived_key) do
defp download_and_deploy(team, %Teams.AppDeployment{} = app_deployment, derived_key) do
destination_path = app_deployment_path(app_deployment.slug)
with {:ok, %{status: 200} = response} <- Req.get(archive_url),
with {:ok, file_content} <- Teams.Requests.download_revision(team, app_deployment),
:ok <- undeploy_app(app_deployment.slug),
{:ok, decrypted_content} <- Teams.decrypt(response.body, derived_key),
{:ok, decrypted_content} <- Teams.decrypt(file_content, derived_key),
:ok <- unzip_app(decrypted_content, destination_path),
:ok <- Livebook.Apps.deploy_apps_in_dir(destination_path) do
{:ok, %{app_deployment | file: nil}}
:ok
end
end

8
lib/livebook/teams.ex
View file

@ -272,4 +272,12 @@ defmodule Livebook.Teams do
any
end
end
@doc """
Gets a list of app deployments for a given Hub.
"""
@spec get_app_deployments(Team.t()) :: list(AppDeployment.t())
def get_app_deployments(team) do
TeamClient.get_app_deployments(team.id)
end
end

8
lib/livebook/teams/app_deployment.ex
View file

@ -2,24 +2,21 @@ defmodule Livebook.Teams.AppDeployment do
use Ecto.Schema
alias Livebook.FileSystem
@file_extension ".zip"
@max_size 20 * 1024 * 1024
@type t :: %__MODULE__{
id: String.t() | nil,
filename: String.t() | nil,
slug: String.t() | nil,
sha: String.t() | nil,
title: String.t() | nil,
deployment_group_id: String.t() | nil,
file: {:url, String.t()} | {:content, binary()} | nil,
file: binary() | nil,
deployed_by: String.t() | nil,
deployed_at: NaiveDateTime.t() | nil
}
@primary_key {:id, :string, autogenerate: false}
embedded_schema do
field :filename, :string
field :slug, :string
field :sha, :string
field :title, :string
@ -45,12 +42,11 @@ defmodule Livebook.Teams.AppDeployment do
{:ok,
%__MODULE__{
filename: shasum <> @file_extension,
slug: notebook.app_settings.slug,
sha: shasum,
title: notebook.name,
deployment_group_id: notebook.deployment_group_id,
file: {:content, zip_content}
file: zip_content
}}
end
end

27
lib/livebook/teams/broadcasts.ex
View file

@ -1,26 +1,25 @@
defmodule Livebook.Teams.Broadcasts do
alias Livebook.Teams.{AgentKey, DeploymentGroup}
alias Livebook.Teams.{AppDeployment, DeploymentGroup}
@type broadcast :: :ok | {:error, term()}
@deployment_groups_topic "teams:deployment_groups"
@agent_keys_topic "teams:agent_keys"
@app_deployments_topic "teams:app_deployments"
@doc """
Subscribes to one or more subtopics in `"teams"`.
## Messages
Topic `teams:deployment_groups`:
Topic `#{@deployment_groups_topic}`:
* `{:deployment_group_created, DeploymentGroup.t()}`
* `{:deployment_group_updated, DeploymentGroup.t()}`
* `{:deployment_group_deleted, DeploymentGroup.t()}`
Topic `teams:agent_keys`:
Topic `#{@app_deployments_topic}`:
* `{:agent_key_created, AgentKey.t()}`
* `{:agent_key_deleted, AgentKey.t()}`
* `{:app_deployment_created, AppDeployment.t()}`
"""
@spec subscribe(atom() | list(atom())) :: :ok | {:error, term()}
@ -73,19 +72,11 @@ defmodule Livebook.Teams.Broadcasts do
end
@doc """
Broadcasts under `#{@agent_keys_topic}` topic when hub received a new agent key.
Broadcasts under `#{@app_deployments_topic}` topic when hub received a new app deployment.
"""
@spec agent_key_created(AgentKey.t()) :: broadcast()
def agent_key_created(%AgentKey{} = agent_key) do
broadcast(@agent_keys_topic, {:agent_key_created, agent_key})
end
@doc """
Broadcasts under `#{@agent_keys_topic}` topic when hub received a deleted agent key.
"""
@spec agent_key_deleted(AgentKey.t()) :: broadcast()
def agent_key_deleted(%AgentKey{} = agent_key) do
broadcast(@agent_keys_topic, {:agent_key_deleted, agent_key})
@spec app_deployment_created(AppDeployment.t()) :: broadcast()
def app_deployment_created(%AppDeployment{} = app_deployment) do
broadcast(@app_deployments_topic, {:app_deployment_created, app_deployment})
end
defp broadcast(topic, message) do

6
lib/livebook/teams/deployment_group.ex
View file

@ -3,7 +3,7 @@ defmodule Livebook.Teams.DeploymentGroup do
import Ecto.Changeset
alias Livebook.Secrets.Secret
alias Livebook.Teams.{AgentKey, AppDeployment}
alias Livebook.Teams.AgentKey
# If this list is updated, it must also be mirrored on Livebook Teams Server.
@zta_providers ~w(cloudflare google_iap tailscale teleport)a
@ -17,8 +17,7 @@ defmodule Livebook.Teams.DeploymentGroup do
zta_provider: :cloudflare | :google_iap | :tailscale | :teleport,
zta_key: String.t(),
secrets: [Secret.t()],
agent_keys: [AgentKey.t()],
app_deployments: [AppDeployment.t()]
agent_keys: [AgentKey.t()]
}
@primary_key {:id, :string, autogenerate: false}
@ -32,7 +31,6 @@ defmodule Livebook.Teams.DeploymentGroup do
has_many :secrets, Secret
has_many :agent_keys, AgentKey
has_many :app_deployments, AppDeployment
end
def changeset(deployment_group, attrs \\ %{}) do

22
lib/livebook/teams/requests.ex
View file

@ -223,21 +223,30 @@ defmodule Livebook.Teams.Requests do
"""
@spec deploy_app(Team.t(), AppDeployment.t()) ::
{:ok, map()} | {:error, map() | String.t()} | {:transport_error, String.t()}
def deploy_app(team, %{file: {:content, content}} = app_deployment) do
def deploy_app(team, app_deployment) do
secret_key = Teams.derive_key(team.teams_key)
params = %{
filename: app_deployment.filename <> ".encrypted",
title: app_deployment.title,
slug: app_deployment.slug,
deployment_group_id: app_deployment.deployment_group_id,
sha: app_deployment.sha
}
encrypted_content = Teams.encrypt(content, secret_key)
encrypted_content = Teams.encrypt(app_deployment.file, secret_key)
upload("/api/v1/org/apps", encrypted_content, params, team)
end
@doc """
Send a request to Livebook Team API to download an app revision.
"""
@spec download_revision(Team.t(), AppDeployment.t()) ::
{:ok, binary()} | {:error, map() | String.t()} | {:transport_error, String.t()}
def download_revision(team, app_deployment) do
params = %{id: app_deployment.id, deployment_group_id: app_deployment.deployment_group_id}
get("/api/v1/org/apps", params, team)
end
@doc """
Add requests errors to a `changeset` for the given `fields`.
"""
@ -281,8 +290,9 @@ defmodule Livebook.Teams.Requests do
|> dispatch_messages(team)
end
defp get(path, params \\ %{}) do
defp get(path, params \\ %{}, team \\ nil) do
build_req()
|> add_team_auth(team)
|> request(method: :get, url: path, params: params)
end
@ -320,9 +330,7 @@ defmodule Livebook.Teams.Requests do
{:ok, body}
{:ok, %{status: status} = response} when status in 200..299 ->
if json?(response),
do: {:ok, response.body},
else: {:error, response.body}
{:ok, response.body}
{:ok, %{status: status} = response} when status in [410, 422] ->
if json?(response),

9
lib/livebook_web/live/hub/teams/deployment_group_live.ex
View file

@ -16,6 +16,7 @@ defmodule LivebookWeb.Hub.Teams.DeploymentGroupLive do
deployment_group_id = params["deployment_group_id"]
secret_name = params["secret_name"]
deployment_groups = Teams.get_deployment_groups(hub)
app_deployments = Teams.get_app_deployments(hub)
default? = default_hub?(hub)
deployment_group =
@ -38,9 +39,11 @@ defmodule LivebookWeb.Hub.Teams.DeploymentGroupLive do
else: []
app_deployments =
if socket.assigns.live_action != :new_deployment_group,
do: deployment_group.app_deployments,
else: []
if socket.assigns.live_action != :new_deployment_group do
Enum.filter(app_deployments, &(&1.deployment_group_id == deployment_group.id))
else
[]
end
secret_value =
if socket.assigns.live_action == :edit_secret do

10
lib/livebook_web/live/session_live/app_teams_component.ex
View file

@ -1,12 +1,11 @@
defmodule LivebookWeb.SessionLive.AppTeamsComponent do
use LivebookWeb, :live_component
alias Livebook.Hubs.Provider
@impl true
def update(assigns, socket) do
socket = assign(socket, assigns)
deployment_groups = Provider.deployment_groups(assigns.hub)
deployment_groups = Livebook.Teams.get_deployment_groups(assigns.hub)
app_deployments = Livebook.Teams.get_app_deployments(assigns.hub)
deployment_group =
if assigns.deployment_group_id do
@ -15,7 +14,10 @@ defmodule LivebookWeb.SessionLive.AppTeamsComponent do
app_deployment =
if deployment_group do
Enum.find(deployment_group.app_deployments, &(&1.slug == assigns.app_settings.slug))
Enum.find(
app_deployments,
&(&1.slug == assigns.app_settings.slug and &1.deployment_group_id == deployment_group.id)
)
end
socket =

4
proto/lib/livebook_proto.ex
View file

@ -1,8 +1,6 @@
defmodule LivebookProto do
alias LivebookProto.{
AgentConnected,
AgentKeyCreated,
AgentKeyDeleted,
AppDeploymentCreated,
FileSystemCreated,
FileSystemDeleted,
@ -24,8 +22,6 @@ defmodule LivebookProto do
@type event_proto ::
AgentConnected.t()
| AgentKeyCreated.t()
| AgentKeyDeleted.t()
| AppDeploymentCreated.t()
| FileSystemCreated.t()
| FileSystemDeleted.t()

5
proto/lib/livebook_proto/agent_connected.pb.ex
View file

@ -12,4 +12,9 @@ defmodule LivebookProto.AgentConnected do
repeated: true,
type: LivebookProto.DeploymentGroup,
json_name: "deploymentGroups"
field :app_deployments, 8,
repeated: true,
type: LivebookProto.AppDeployment,
json_name: "appDeployments"
end

7
proto/lib/livebook_proto/agent_key_created.pb.ex
View file

@ -1,7 +0,0 @@
defmodule LivebookProto.AgentKeyCreated do
use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0"
field :id, 1, type: :string
field :key, 2, type: :string
field :deployment_group_id, 3, type: :string, json_name: "deploymentGroupId"
end

7
proto/lib/livebook_proto/agent_key_deleted.pb.ex
View file

@ -1,7 +0,0 @@
defmodule LivebookProto.AgentKeyDeleted do
use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0"
field :id, 1, type: :string
field :key, 2, type: :string
field :deployment_group_id, 3, type: :string, json_name: "deploymentGroupId"
end

12
proto/lib/livebook_proto/app_deployment.pb.ex Normal file
View file

@ -0,0 +1,12 @@
defmodule LivebookProto.AppDeployment do
use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0"
field :id, 1, type: :string
field :title, 2, type: :string
field :sha, 3, type: :string
field :revision_id, 4, type: :string, json_name: "revisionId"
field :slug, 5, type: :string
field :deployment_group_id, 6, type: :string, json_name: "deploymentGroupId"
field :deployed_by, 7, type: :string, json_name: "deployedBy"
field :deployed_at, 8, type: :int64, json_name: "deployedAt"
end

10
proto/lib/livebook_proto/app_deployment_created.pb.ex
View file

@ -1,13 +1,5 @@
defmodule LivebookProto.AppDeploymentCreated do
use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0"
field :id, 1, type: :string
field :title, 2, type: :string
field :sha, 3, type: :string
field :archive_url, 4, type: :string, json_name: "archiveUrl"
field :app_id, 5, type: :string, json_name: "appId"
field :slug, 6, type: :string
field :deployment_group_id, 7, type: :string, json_name: "deploymentGroupId"
field :deployed_by, 8, type: :string, json_name: "deployedBy"
field :deployed_at, 9, type: :string, json_name: "deployedAt"
field :app_deployment, 1, type: LivebookProto.AppDeployment, json_name: "appDeployment"
end

13
proto/lib/livebook_proto/deployed_app.pb.ex
View file

@ -1,13 +0,0 @@
defmodule LivebookProto.DeployedApp do
use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0"
field :id, 1, type: :string
field :title, 2, type: :string
field :sha, 3, type: :string
field :archive_url, 4, type: :string, json_name: "archiveUrl"
field :app_id, 5, type: :string, json_name: "appId"
field :slug, 6, type: :string
field :deployment_group_id, 7, type: :string, json_name: "deploymentGroupId"
field :deployed_by, 8, type: :string, json_name: "deployedBy"
field :deployed_at, 9, type: :string, json_name: "deployedAt"
end

5
proto/lib/livebook_proto/deployment_group.pb.ex
View file

@ -9,9 +9,4 @@ defmodule LivebookProto.DeploymentGroup do
field :zta_provider, 6, type: :string, json_name: "ztaProvider"
field :zta_key, 7, type: :string, json_name: "ztaKey"
field :agent_keys, 8, repeated: true, type: LivebookProto.AgentKey, json_name: "agentKeys"
field :deployed_apps, 9,
repeated: true,
type: LivebookProto.DeployedApp,
json_name: "deployedApps"
end

6
proto/lib/livebook_proto/deployment_group_created.pb.ex
View file

@ -4,14 +4,8 @@ defmodule LivebookProto.DeploymentGroupCreated do
field :id, 1, type: :string
field :name, 2, type: :string
field :mode, 3, type: :string
field :secrets, 4, repeated: true, type: LivebookProto.DeploymentGroupSecret
field :clustering, 5, type: :string
field :zta_provider, 6, type: :string, json_name: "ztaProvider"
field :zta_key, 7, type: :string, json_name: "ztaKey"
field :agent_keys, 8, repeated: true, type: LivebookProto.AgentKey, json_name: "agentKeys"
field :deployed_apps, 9,
repeated: true,
type: LivebookProto.DeployedApp,
json_name: "deployedApps"
end

16
proto/lib/livebook_proto/deployment_group_updated.pb.ex
View file

@ -3,15 +3,9 @@ defmodule LivebookProto.DeploymentGroupUpdated do
field :id, 1, type: :string
field :name, 2, type: :string
field :mode, 3, type: :string
field :secrets, 4, repeated: true, type: LivebookProto.DeploymentGroupSecret
field :clustering, 5, type: :string
field :zta_provider, 6, type: :string, json_name: "ztaProvider"
field :zta_key, 7, type: :string, json_name: "ztaKey"
field :agent_keys, 8, repeated: true, type: LivebookProto.AgentKey, json_name: "agentKeys"
field :deployed_apps, 9,
repeated: true,
type: LivebookProto.DeployedApp,
json_name: "deployedApps"
field :secrets, 3, repeated: true, type: LivebookProto.DeploymentGroupSecret
field :clustering, 4, type: :string
field :zta_provider, 5, type: :string, json_name: "ztaProvider"
field :zta_key, 6, type: :string, json_name: "ztaKey"
field :agent_keys, 7, repeated: true, type: LivebookProto.AgentKey, json_name: "agentKeys"
end

17
proto/lib/livebook_proto/event.pb.ex
View file

@ -58,23 +58,10 @@ defmodule LivebookProto.Event do
json_name: "agentConnected",
oneof: 0
field :agent_key_created, 12,
type: LivebookProto.AgentKeyCreated,
json_name: "agentKeyCreated",
oneof: 0
field :agent_key_deleted, 13,
type: LivebookProto.AgentKeyDeleted,
json_name: "agentKeyDeleted",
oneof: 0
field :app_deployment_created, 14,
field :app_deployment_created, 12,
type: LivebookProto.AppDeploymentCreated,
json_name: "appDeploymentCreated",
oneof: 0
field :user_deleted, 15,
type: LivebookProto.UserDeleted,
json_name: "userDeleted",
oneof: 0
field :user_deleted, 13, type: LivebookProto.UserDeleted, json_name: "userDeleted", oneof: 0
end

5
proto/lib/livebook_proto/user_connected.pb.ex
View file

@ -9,4 +9,9 @@ defmodule LivebookProto.UserConnected do
repeated: true,
type: LivebookProto.DeploymentGroup,
json_name: "deploymentGroups"
field :app_deployments, 5,
repeated: true,
type: LivebookProto.AppDeployment,
json_name: "appDeployments"
end

58
proto/messages.proto
View file

@ -63,31 +63,26 @@ message DeploymentGroup {
string zta_provider = 6;
string zta_key = 7;
repeated AgentKey agent_keys = 8;
repeated DeployedApp deployed_apps = 9;
}
message DeploymentGroupCreated {
string id = 1;
string name = 2;
string mode = 3;
repeated DeploymentGroupSecret secrets = 4;
string clustering = 5;
string zta_provider = 6;
string zta_key = 7;
repeated AgentKey agent_keys = 8;
repeated DeployedApp deployed_apps = 9;
}
message DeploymentGroupUpdated {
string id = 1;
string name = 2;
string mode = 3;
repeated DeploymentGroupSecret secrets = 4;
string clustering = 5;
string zta_provider = 6;
string zta_key = 7;
repeated AgentKey agent_keys = 8;
repeated DeployedApp deployed_apps = 9;
repeated DeploymentGroupSecret secrets = 3;
string clustering = 4;
string zta_provider = 5;
string zta_key = 6;
repeated AgentKey agent_keys = 7;
}
message DeploymentGroupDeleted {
@ -100,23 +95,12 @@ message AgentKey {
string deployment_group_id = 3;
}
message AgentKeyCreated {
string id = 1;
string key = 2;
string deployment_group_id = 3;
}
message AgentKeyDeleted {
string id = 1;
string key = 2;
string deployment_group_id = 3;
}
message UserConnected {
string name = 1;
repeated Secret secrets = 2;
repeated FileSystem file_systems = 3;
repeated DeploymentGroup deployment_groups = 4;
repeated AppDeployment app_deployments = 5;
}
message AgentConnected {
@ -127,30 +111,22 @@ message AgentConnected {
repeated Secret secrets = 5;
repeated FileSystem file_systems = 6;
repeated DeploymentGroup deployment_groups = 7;
repeated AppDeployment app_deployments = 8;
}
message DeployedApp {
message AppDeployment {
string id = 1;
string title = 2;
string sha = 3;
string archive_url = 4;
string app_id = 5;
string slug = 6;
string deployment_group_id = 7;
string deployed_by = 8;
string deployed_at = 9;
string revision_id = 4;
string slug = 5;
string deployment_group_id = 6;
string deployed_by = 7;
int64 deployed_at = 8;
}
message AppDeploymentCreated {
string id = 1;
string title = 2;
string sha = 3;
string archive_url = 4;
string app_id = 5;
string slug = 6;
string deployment_group_id = 7;
string deployed_by = 8;
string deployed_at = 9;
AppDeployment app_deployment = 1;
}
message UserDeleted {
@ -170,9 +146,7 @@ message Event {
DeploymentGroupUpdated deployment_group_updated = 9;
DeploymentGroupDeleted deployment_group_deleted = 10;
AgentConnected agent_connected = 11;
AgentKeyCreated agent_key_created = 12;
AgentKeyDeleted agent_key_deleted = 13;
AppDeploymentCreated app_deployment_created = 14;
UserDeleted user_deleted = 15;
AppDeploymentCreated app_deployment_created = 12;
UserDeleted user_deleted = 13;
}
}

139
test/livebook_teams/hubs/team_client_test.exs
View file

@ -7,7 +7,7 @@ defmodule Livebook.Hubs.TeamClientTest do
setup do
Livebook.Hubs.Broadcasts.subscribe([:connection, :file_systems, :secrets])
Livebook.Teams.Broadcasts.subscribe([:deployment_groups])
Livebook.Teams.Broadcasts.subscribe([:deployment_groups, :app_deployments])
:ok
end
@ -159,8 +159,7 @@ defmodule Livebook.Hubs.TeamClientTest do
id = to_string(id)
# receives `{:event, :deployment_group_created, :deployment_group}` event
assert_receive {:deployment_group_created, %{id: ^id, app_deployments: []}}
assert_receive {:deployment_group_created, %{id: ^id}}
# creates the app deployment
slug = Livebook.Utils.random_short_id()
@ -178,14 +177,14 @@ defmodule Livebook.Hubs.TeamClientTest do
{:ok, app_deployment} = Livebook.Teams.AppDeployment.new(notebook, files_dir)
:ok = Livebook.Teams.deploy_app(team, app_deployment)
# since the `app_deployment` belongs to a deployment group,
# we dispatch the `{:event, :deployment_group_updated, :deployment_group}` event
assert_receive {:deployment_group_updated,
%{
id: ^id,
app_deployments: [
%Livebook.Teams.AppDeployment{title: ^title, slug: ^slug}
]
sha = app_deployment.sha
assert_receive {:app_deployment_created,
%Livebook.Teams.AppDeployment{
slug: ^slug,
sha: ^sha,
title: ^title,
deployment_group_id: ^id
}}
end
end
@ -199,7 +198,8 @@ defmodule Livebook.Hubs.TeamClientTest do
name: team.hub_name,
secrets: [],
file_systems: [],
deployment_groups: []
deployment_groups: [],
app_deployments: []
}
{:ok, team: team, user_connected: user_connected}
@ -319,8 +319,7 @@ defmodule Livebook.Hubs.TeamClientTest do
name: deployment_group.name,
mode: to_string(deployment_group.mode),
secrets: [],
agent_keys: [],
deployed_apps: []
agent_keys: []
}
# creates the deployment group
@ -357,6 +356,8 @@ defmodule Livebook.Hubs.TeamClientTest do
end
test "dispatches the app deployments list", %{team: team, user_connected: user_connected} do
pid = connect_to_teams(team)
deployment_group =
build(:deployment_group,
id: "1",
@ -371,56 +372,33 @@ defmodule Livebook.Hubs.TeamClientTest do
name: deployment_group.name,
mode: to_string(deployment_group.mode),
secrets: [],
agent_keys: [],
deployed_apps: []
agent_keys: []
}
user_connected = %{user_connected | deployment_groups: [livebook_proto_deployment_group]}
pid = connect_to_teams(team)
refute_receive {:deployment_group_created, ^deployment_group}
send(pid, {:event, :user_connected, user_connected})
assert_receive {:deployment_group_created, ^deployment_group}
assert deployment_group in TeamClient.get_deployment_groups(team.id)
url = "http://localhost/123456.zip"
app_deployment =
build(:app_deployment,
filename: "123456.zip",
file: {:url, url},
deployment_group_id: deployment_group.id
)
# updates the deployment group with an app deployment
updated_deployment_group = %{deployment_group | app_deployments: [app_deployment]}
app_deployment = build(:app_deployment, file: nil, deployment_group_id: deployment_group.id)
{seconds, 0} = NaiveDateTime.to_gregorian_seconds(app_deployment.deployed_at)
livebook_proto_app_deployment =
%LivebookProto.DeployedApp{
%LivebookProto.AppDeployment{
id: app_deployment.id,
title: app_deployment.title,
slug: app_deployment.slug,
sha: app_deployment.sha,
archive_url: url,
deployed_by: app_deployment.deployed_by,
deployed_at: to_string(app_deployment.deployed_at),
app_id: "1",
deployed_at: seconds,
revision_id: "1",
deployment_group_id: app_deployment.deployment_group_id
}
updated_livebook_proto_deployment_group = %{
livebook_proto_deployment_group
| deployed_apps: [livebook_proto_app_deployment]
}
user_connected = %{
user_connected
| deployment_groups: [updated_livebook_proto_deployment_group]
| deployment_groups: [livebook_proto_deployment_group],
app_deployments: [livebook_proto_app_deployment]
}
send(pid, {:event, :user_connected, user_connected})
assert_receive {:deployment_group_updated, ^updated_deployment_group}
refute deployment_group in TeamClient.get_deployment_groups(team.id)
assert updated_deployment_group in TeamClient.get_deployment_groups(team.id)
assert_receive {:app_deployment_created, ^app_deployment}, 5000
assert app_deployment in TeamClient.get_app_deployments(team.id)
end
end
@ -437,11 +415,13 @@ defmodule Livebook.Hubs.TeamClientTest do
deployment_group_id: deployment_group.id,
secrets: [],
file_systems: [],
deployment_groups: []
deployment_groups: [],
app_deployments: []
}
{:ok,
team: team,
org: org,
deployment_group: deployment_group,
agent_connected: agent_connected,
agent_key: agent_key}
@ -662,10 +642,12 @@ defmodule Livebook.Hubs.TeamClientTest do
test "dispatches the app deployments list",
%{
team: team,
org: teams_org,
deployment_group: teams_deployment_group,
agent_key: teams_agent_key,
agent_connected: agent_connected,
tmp_dir: tmp_dir
tmp_dir: tmp_dir,
node: node
} do
agent_key =
build(:agent_key,
@ -696,8 +678,7 @@ defmodule Livebook.Hubs.TeamClientTest do
name: deployment_group.name,
mode: to_string(deployment_group.mode),
secrets: [],
agent_keys: [livebook_proto_agent_key],
deployed_apps: []
agent_keys: [livebook_proto_agent_key]
}
agent_connected = %{agent_connected | deployment_groups: [livebook_proto_deployment_group]}
@ -723,60 +704,46 @@ defmodule Livebook.Hubs.TeamClientTest do
files_dir = Livebook.FileSystem.File.local(tmp_dir)
{:ok, %{file: {:content, zip_content}} = app_deployment} =
{:ok, %Livebook.Teams.AppDeployment{file: zip_content} = app_deployment} =
Livebook.Teams.AppDeployment.new(notebook, files_dir)
secret_key = Livebook.Teams.derive_key(team.teams_key)
encrypted_content = Livebook.Teams.encrypt(zip_content, secret_key)
teams_app_deployment =
erpc_call(node, :upload_app_deployment, [
teams_org,
teams_deployment_group,
app_deployment,
encrypted_content
])
# Since the app deployment struct generation is from Livebook side,
# we don't have yet the information about who deployed the app,
# so we need to add it ourselves
app_deployment = %{
app_deployment
| id: "1",
filename: app_deployment.filename <> ".encrypted",
deployed_by: "Jake Peralta",
deployed_at: NaiveDateTime.utc_now()
| id: to_string(teams_app_deployment.id),
file: nil,
deployed_by: teams_app_deployment.app_revision.created_by.name,
deployed_at: teams_app_deployment.updated_at
}
bypass = Bypass.open()
Bypass.expect_once(bypass, "GET", "/#{app_deployment.filename}", fn conn ->
conn
|> Plug.Conn.put_resp_content_type("application/octet-stream")
|> Plug.Conn.resp(200, encrypted_content)
end)
# updates the deployment group with an app deployment
# and after we deploy, the `:file` key turns to `nil` value
updated_deployment_group = %{
deployment_group
| app_deployments: [%{app_deployment | file: nil}]
}
{seconds, 0} = NaiveDateTime.to_gregorian_seconds(app_deployment.deployed_at)
livebook_proto_app_deployment =
%LivebookProto.DeployedApp{
%LivebookProto.AppDeployment{
id: app_deployment.id,
title: app_deployment.title,
slug: app_deployment.slug,
sha: app_deployment.sha,
archive_url: "http://localhost:#{bypass.port}/#{app_deployment.filename}",
deployed_by: app_deployment.deployed_by,
deployed_at: to_string(app_deployment.deployed_at),
app_id: "1",
deployed_at: seconds,
revision_id: to_string(teams_app_deployment.app_revision.id),
deployment_group_id: app_deployment.deployment_group_id
}
updated_livebook_proto_deployment_group = %{
livebook_proto_deployment_group
| deployed_apps: [livebook_proto_app_deployment]
}
agent_connected = %{
agent_connected
| deployment_groups: [updated_livebook_proto_deployment_group]
}
agent_connected = %{agent_connected | app_deployments: [livebook_proto_app_deployment]}
apps_path = Path.join(tmp_dir, "apps")
app_path = Path.join(apps_path, slug)
@ -789,15 +756,13 @@ defmodule Livebook.Hubs.TeamClientTest do
Livebook.Apps.subscribe()
send(pid, {:event, :agent_connected, agent_connected})
assert_receive {:deployment_group_updated, ^updated_deployment_group}
assert_receive {:app_deployment_created, ^app_deployment}
assert_receive {:app_created, %{pid: app_pid, slug: ^slug}}
assert_receive {:app_updated,
%{slug: ^slug, sessions: [%{app_status: %{execution: :executed}}]}}
refute deployment_group in TeamClient.get_deployment_groups(team.id)
assert updated_deployment_group in TeamClient.get_deployment_groups(team.id)
assert app_deployment in TeamClient.get_app_deployments(team.id)
Livebook.App.close(app_pid)
Application.put_env(:livebook, :apps_path, nil)

128
test/livebook_teams/teams/connection_test.exs
View file

@ -79,7 +79,7 @@ defmodule Livebook.Teams.ConnectionTest do
assert {:ok, _conn} = Connection.start_link(self(), headers)
assert_receive :connected
# creates a new deployment group
# creates a new deployment group with offline mode
deployment_group = build(:deployment_group, name: "FOO", mode: :offline)
assert {:ok, _id} =
@ -89,28 +89,132 @@ defmodule Livebook.Teams.ConnectionTest do
assert_receive {:event, :deployment_group_created, deployment_group_created}
assert deployment_group_created.name == deployment_group.name
assert String.to_existing_atom(deployment_group_created.mode) == deployment_group.mode
# since the deployment group is with offline mode, the agent key shouldn't exists
assert deployment_group_created.agent_keys == []
# creates a new deployment group with online mode
deployment_group = build(:deployment_group, name: "BAR", mode: :online)
{:ok, _id} = Livebook.Teams.create_deployment_group(hub, deployment_group)
# deployment_group name and mode are not encrypted
assert_receive {:event, :deployment_group_created, deployment_group_created}
assert deployment_group_created.name == deployment_group.name
assert String.to_existing_atom(deployment_group_created.mode) == deployment_group.mode
# receives the built-in agent key
assert [agent_key] = deployment_group_created.agent_keys
assert is_binary(agent_key.key)
assert agent_key.deployment_group_id == deployment_group_created.id
end
test "receives the agent_key_created event", %{user: user, node: node} do
@tag :tmp_dir
test "receives the app deployments list from user_connected event",
%{user: user, node: node, tmp_dir: tmp_dir} do
{hub, headers} = build_team_headers(user, node)
# creates a new deployment group
deployment_group = build(:deployment_group, name: "BAZ", mode: :online)
{:ok, id} = Livebook.Teams.create_deployment_group(hub, deployment_group)
# creates a new app deployment
deployment_group_id = to_string(id)
slug = Livebook.Utils.random_short_id()
title = "MyNotebook3-#{slug}"
notebook = %{
Livebook.Notebook.new()
| app_settings: %{Livebook.Notebook.AppSettings.new() | slug: slug},
name: title,
hub_id: hub.id,
deployment_group_id: deployment_group_id
}
files_dir = Livebook.FileSystem.File.local(tmp_dir)
{:ok, %Livebook.Teams.AppDeployment{} = app_deployment} =
Livebook.Teams.AppDeployment.new(notebook, files_dir)
# since we want to fetch the app deployment from connection event,
# we need to persist it before we connect to the WebSocket
:ok = Livebook.Teams.deploy_app(hub, app_deployment)
assert {:ok, _conn} = Connection.start_link(self(), headers)
assert_receive :connected
assert_receive {:event, :user_connected, user_connected}
assert [app_deployment2] = user_connected.app_deployments
assert app_deployment2.title == title
assert app_deployment2.slug == slug
assert app_deployment2.sha == app_deployment.sha
assert app_deployment2.deployment_group_id == deployment_group_id
end
@tag :tmp_dir
test "receives the app deployments list from agent_connected event",
%{user: user, node: node, tmp_dir: tmp_dir} do
# To create a new app deployment, we need use the User connection
{hub, _headers} = build_team_headers(user, node)
# creates a new deployment group
deployment_group = build(:deployment_group, name: "FOO", mode: :online)
deployment_group = build(:deployment_group, name: "BAZ", mode: :online)
{:ok, id} = Livebook.Teams.create_deployment_group(hub, deployment_group)
teams_deployment_group = erpc_call(node, :get_deployment_group!, [id])
[teams_agent_key] = teams_deployment_group.agent_keys
assert {:ok, deployment_group_id} =
Livebook.Teams.create_deployment_group(hub, deployment_group)
# creates a new app deployment
slug = Livebook.Utils.random_short_id()
title = "MyNotebook3-#{slug}"
# creates a new agent key
deployment_group = %{deployment_group | id: to_string(deployment_group_id)}
assert Livebook.Teams.create_agent_key(hub, deployment_group) == :ok
notebook = %{
Livebook.Notebook.new()
| app_settings: %{Livebook.Notebook.AppSettings.new() | slug: slug},
name: title,
hub_id: hub.id,
deployment_group_id: to_string(id)
}
# agent_key key is not encrypted
assert_receive {:event, :agent_key_created, agent_key_created}
assert "lb_ak_" <> _ = agent_key_created.key
assert agent_key_created.deployment_group_id == deployment_group.id
files_dir = Livebook.FileSystem.File.local(tmp_dir)
{:ok, %Livebook.Teams.AppDeployment{} = app_deployment} =
Livebook.Teams.AppDeployment.new(notebook, files_dir)
# since we want to fetch the app deployment from connection event,
# we need to persist it before we connect to the WebSocket
:ok = Livebook.Teams.deploy_app(hub, app_deployment)
# As we need to be Agent to receive the app deployments list to be deployed,
# we will create another connection here
public_key = hub.org_public_key
hub = %{
hub
| user_id: nil,
org_public_key: nil,
session_token: teams_agent_key.key
}
agent_name = Livebook.Config.agent_name()
headers = [
{"x-lb-version", Livebook.Config.app_version()},
{"x-org", to_string(hub.org_id)},
{"x-org-key", to_string(hub.org_key_id)},
{"x-agent-name", agent_name},
{"x-agent-key", hub.session_token}
]
assert {:ok, _conn} = Connection.start_link(self(), headers)
assert_receive :connected
assert_receive {:event, :agent_connected, agent_connected}
assert agent_connected.name == agent_name
assert agent_connected.public_key == public_key
assert [app_deployment2] = agent_connected.app_deployments
assert app_deployment2.title == title
assert app_deployment2.slug == slug
assert app_deployment2.sha == app_deployment.sha
assert app_deployment2.deployment_group_id == to_string(id)
end
end
end

15
test/support/factory.ex
View file

@ -61,8 +61,7 @@ defmodule Livebook.Factory do
name: "FOO",
mode: :offline,
agent_keys: [],
secrets: [],
app_deployments: []
secrets: []
}
end
@ -102,24 +101,20 @@ defmodule Livebook.Factory do
def build(:app_deployment) do
slug = Livebook.Utils.random_short_id()
path = Plug.Upload.random_file!(slug)
local = Livebook.FileSystem.Local.new()
file = Livebook.FileSystem.File.new(local, path)
{:ok, content} = Livebook.FileSystem.File.read(file)
content = :crypto.strong_rand_bytes(1024 * 1024)
md5_hash = :crypto.hash(:md5, content)
shasum = Base.encode16(md5_hash, case: :lower)
deployed_at = NaiveDateTime.utc_now()
%Livebook.Teams.AppDeployment{
id: "1",
title: "MyNotebook",
sha: shasum,
slug: slug,
file: file,
filename: Path.basename(file.path),
file: content,
deployment_group_id: "1",
deployed_by: "Ada Lovelace",
deployed_at: NaiveDateTime.utc_now()
deployed_at: NaiveDateTime.truncate(deployed_at, :second)
}
end