From c96ea9f79d41f95d8130afc10dd90f452dda64cb Mon Sep 17 00:00:00 2001 From: Alexandre de Souza Date: Mon, 8 Sep 2025 13:18:40 -0300 Subject: [PATCH] Add tests --- test/livebook/file_system/git_test.exs | 224 ++++++++++++++++++ .../livebook_teams/web/hub/edit_live_test.exs | 144 ++++++++++- test/livebook_teams/web/open_live_test.exs | 73 ++++++ test/livebook_web/live/hub/edit_live_test.exs | 3 + test/support/factory.ex | 18 +- test/support/hub_helpers.ex | 14 +- test/support/integration/teams_rpc.ex | 23 +- test/test_helper.exs | 4 +- 8 files changed, 478 insertions(+), 25 deletions(-) create mode 100644 test/livebook/file_system/git_test.exs create mode 100644 test/livebook_teams/web/open_live_test.exs diff --git a/test/livebook/file_system/git_test.exs b/test/livebook/file_system/git_test.exs new file mode 100644 index 000000000..627947296 --- /dev/null +++ b/test/livebook/file_system/git_test.exs @@ -0,0 +1,224 @@ +defmodule Livebook.FileSystem.GitTest do + use Livebook.DataCase, async: true + + @moduletag :git + + alias Livebook.FileSystem + alias Livebook.FileSystem.Git + + setup %{test: test} do + repo_url = "git@github.com:livebook-dev/test.git" + hub_id = test |> to_string() |> Base.encode32(padding: false) + id = Livebook.FileSystem.Utils.id("git", hub_id, repo_url) + + {:ok, file_system: build(:fs_git, id: id, repo_url: repo_url, hub_id: hub_id)} + end + + describe "FileSystem.default_path/1" do + test "returns the root path", %{file_system: file_system} do + assert FileSystem.default_path(file_system) == "/" + end + end + + describe "common request errors" do + test "authorization failure", %{file_system: file_system} do + file_system = %{file_system | key: "foo"} + + assert {:error, reason} = FileSystem.list(file_system, "/dir/", false) + assert reason =~ "Permission denied (publickey)." + end + end + + describe "FileSystem.list/3" do + test "returns an empty list with invalid path", %{file_system: file_system} do + assert FileSystem.list(file_system, "/path/", false) == + {:error, "no such file or directory"} + end + + test "returns a list of absolute child object paths", %{file_system: file_system} do + assert {:ok, paths} = FileSystem.list(file_system, "/", false) + assert "/notebook_files/" in paths + assert "/file.txt" in paths + end + end + + describe "FileSystem.read/2" do + test "returns an error when a nonexistent key is given", %{file_system: file_system} do + assert FileSystem.read(file_system, "/another_file.txt") == + {:error, "fatal: path 'another_file.txt' does not exist in 'main'"} + end + + test "returns object contents under the given key", %{file_system: file_system} do + assert {:ok, content} = FileSystem.read(file_system, "/file.txt") + assert content =~ "git file storage works" + end + end + + describe "FileSystem.write/3" do + test "not implemented", %{file_system: file_system} do + assert_raise RuntimeError, "not implemented", fn -> + FileSystem.write(file_system, "/file.txt", "") + end + end + end + + describe "FileSystem.create_dir/2" do + test "not implemented", %{file_system: file_system} do + assert_raise RuntimeError, "not implemented", fn -> + FileSystem.create_dir(file_system, "/folder") + end + end + end + + describe "FileSystem.remove/2" do + test "not implemented", %{file_system: file_system} do + assert_raise RuntimeError, "not implemented", fn -> + FileSystem.remove(file_system, "/file.txt") + end + end + end + + describe "FileSystem.copy/3" do + test "not implemented", %{file_system: file_system} do + assert_raise RuntimeError, "not implemented", fn -> + FileSystem.copy(file_system, "/file.txt", "/folder/file.txt") + end + end + end + + describe "FileSystem.rename/3" do + test "not implemented", %{file_system: file_system} do + assert_raise RuntimeError, "not implemented", fn -> + FileSystem.rename(file_system, "/file.txt", "/another_file.txt") + end + end + end + + describe "FileSystem.etag_for/2" do + test "returns an error when a nonexistent key is given", %{file_system: file_system} do + assert {:error, reason} = FileSystem.etag_for(file_system, "/another_file.txt") + assert reason =~ "fatal: path 'another_file.txt' does not exist in 'main'" + end + + test "returns the ETag value received from the server", %{file_system: file_system} do + assert {:ok, _etag} = FileSystem.etag_for(file_system, "/file.txt") + end + end + + describe "FileSystem.exists?/2" do + test "returns valid response", %{file_system: file_system} do + assert {:ok, true} = FileSystem.exists?(file_system, "/file.txt") + assert {:ok, false} = FileSystem.exists?(file_system, "/another_file.txt") + end + + test "returns error with invalid path", %{file_system: file_system} do + assert {:error, "fatal: ../../.bashrc: '../../.bashrc' is outside repository at" <> _} = + FileSystem.exists?(file_system, "../../.bashrc") + end + end + + describe "FileSystem.resolve_path/3" do + test "resolves relative paths", %{file_system: file_system} do + assert "/dir/" = FileSystem.resolve_path(file_system, "/dir/", "") + assert "/dir/file.txt" = FileSystem.resolve_path(file_system, "/dir/", "file.txt") + assert "/dir/nested/" = FileSystem.resolve_path(file_system, "/dir/", "nested/") + assert "/dir/" = FileSystem.resolve_path(file_system, "/dir/", ".") + assert "/" = FileSystem.resolve_path(file_system, "/dir/", "..") + + assert "/file.txt" = + FileSystem.resolve_path(file_system, "/dir/", "nested/../.././file.txt") + end + + test "resolves absolute paths", %{file_system: file_system} do + assert "/" = FileSystem.resolve_path(file_system, "/dir/", "/") + assert "/file.txt" = FileSystem.resolve_path(file_system, "/dir/", "/file.txt") + assert "/nested/" = FileSystem.resolve_path(file_system, "/dir/", "/nested/") + + assert "/nested/file.txt" = + FileSystem.resolve_path(file_system, "/dir/", "///nested///other/..///file.txt") + end + end + + describe "FileSystem chunked write" do + test "not implemented", %{file_system: file_system} do + assert_raise RuntimeError, "not implemented", fn -> + FileSystem.write_stream_init(file_system, "/readme.txt", part_size: 5_000) + end + end + end + + describe "FileSystem.read_stream_into/2" do + test "not implemented", %{file_system: file_system} do + assert_raise RuntimeError, "not implemented", fn -> + FileSystem.read_stream_into(file_system, "/file.txt", <<>>) + end + end + end + + describe "FileSystem.load/2" do + test "loads from atom keys" do + fields = %{ + id: "team-123456-git-Ios91o6sRIRnTmpRlO2jpwHPtFXlZh2FH6rkvxuN_8M", + repo_url: "git@github.com:livebook-dev/test.git", + branch: "main", + key: "foo", + hub_id: "team-123456", + external_id: "1" + } + + assert FileSystem.load(%Git{}, fields) == %Git{ + id: "team-123456-git-Ios91o6sRIRnTmpRlO2jpwHPtFXlZh2FH6rkvxuN_8M", + repo_url: "git@github.com:livebook-dev/test.git", + branch: "main", + key: "foo", + hub_id: "team-123456", + external_id: "1" + } + end + + test "loads from string keys" do + fields = %{ + "id" => "team-123456-git-Ios91o6sRIRnTmpRlO2jpwHPtFXlZh2FH6rkvxuN_8M", + "repo_url" => "git@github.com:livebook-dev/test.git", + "branch" => "main", + "key" => "foo", + "hub_id" => "team-123456", + "external_id" => "1" + } + + assert FileSystem.load(%Git{}, fields) == %Git{ + id: "team-123456-git-Ios91o6sRIRnTmpRlO2jpwHPtFXlZh2FH6rkvxuN_8M", + repo_url: "git@github.com:livebook-dev/test.git", + branch: "main", + key: "foo", + hub_id: "team-123456", + external_id: "1" + } + end + end + + describe "FileSystem.dump/1" do + test "dumps into a map ready to be stored" do + repo_url = "git@github.com:livebook-dev/test.git" + hub_id = "team-123456" + + file_system = + build(:fs_git, + id: Livebook.FileSystem.Utils.id("git", hub_id, repo_url), + repo_url: repo_url, + branch: "main", + key: "foo", + hub_id: hub_id + ) + + assert FileSystem.dump(file_system) == %{ + id: "team-123456-git-Ios91o6sRIRnTmpRlO2jpwHPtFXlZh2FH6rkvxuN_8M", + repo_url: "git@github.com:livebook-dev/test.git", + branch: "main", + key: "foo", + hub_id: "team-123456", + external_id: "1" + } + end + end +end diff --git a/test/livebook_teams/web/hub/edit_live_test.exs b/test/livebook_teams/web/hub/edit_live_test.exs index 6f8444407..eb412a4db 100644 --- a/test/livebook_teams/web/hub/edit_live_test.exs +++ b/test/livebook_teams/web/hub/edit_live_test.exs @@ -179,7 +179,7 @@ defmodule LivebookWeb.Integration.Hub.EditLiveTest do end end - test "creates a file system", %{conn: conn, team: team} do + test "creates a S3 file system", %{conn: conn, team: team} do {:ok, view, _html} = live(conn, ~p"/hub/#{team.id}") bypass = Bypass.open() @@ -188,7 +188,6 @@ defmodule LivebookWeb.Integration.Hub.EditLiveTest do attrs = %{file_system: Livebook.FileSystem.dump(file_system)} expect_s3_listing(bypass) - refute render(view) =~ file_system.bucket_url view @@ -197,6 +196,7 @@ defmodule LivebookWeb.Integration.Hub.EditLiveTest do assert_patch(view, ~p"/hub/#{team.id}/file-systems/new") assert render(view) =~ "Add file storage" + assert has_element?(view, "#file_system_type-s3") view |> element("#file-systems-form") @@ -210,14 +210,55 @@ defmodule LivebookWeb.Integration.Hub.EditLiveTest do |> element("#file-systems-form") |> render_submit(attrs) - assert_receive {:file_system_created, %{id: ^id} = file_system} + assert_receive {:file_system_created, %Livebook.FileSystem.S3{id: ^id} = file_system} assert_patch(view, "/hub/#{team.id}") assert render(view) =~ "File storage added successfully" assert render(element(view, "#hub-file-systems-list")) =~ file_system.bucket_url assert file_system in Livebook.Hubs.get_file_systems(team) end - test "updates existing file system", %{conn: conn, team: team, node: node, org_key: org_key} do + @tag :git + test "creates a Git file system", %{conn: conn, team: team} do + file_system = build(:fs_git) + id = file_system.id + attrs = %{file_system: Livebook.FileSystem.dump(file_system)} + + {:ok, view, _html} = live(conn, ~p"/hub/#{team.id}") + refute render(view) =~ file_system.repo_url + + view + |> element("#add-file-system") + |> render_click() + + assert_patch(view, ~p"/hub/#{team.id}/file-systems/new") + assert render(view) =~ "Add file storage" + + # change the file system type from S3 to git + view + |> element("#file_system_type-git") + |> render_click() + + view + |> element("#file-systems-form") + |> render_change(attrs) + + refute view + |> element("#file-systems-form button[disabled]") + |> has_element?() + + view + |> element("#file-systems-form") + |> render_submit(attrs) + + assert_receive {:file_system_created, %Livebook.FileSystem.Git{id: ^id} = file_system} + assert_patch(view, "/hub/#{team.id}") + assert render(view) =~ "File storage added successfully" + assert render(element(view, "#hub-file-systems-list")) =~ file_system.repo_url + assert file_system in Livebook.Hubs.get_file_systems(team) + end + + test "updates existing S3 file system", + %{conn: conn, team: team, node: node, org_key: org_key} do bypass = Bypass.open() file_system = build_bypass_file_system(bypass, team.id) @@ -248,16 +289,69 @@ defmodule LivebookWeb.Integration.Hub.EditLiveTest do |> element("#file-systems-form") |> render_submit(put_in(attrs.file_system.access_key_id, "new key")) - updated_file_system = %{file_system | access_key_id: "new key"} - - assert_receive {:file_system_updated, ^updated_file_system} assert_patch(view, "/hub/#{team.id}") assert render(view) =~ "File storage updated successfully" + + updated_file_system = %{file_system | access_key_id: "new key"} + assert_receive {:file_system_updated, ^updated_file_system} + assert render(element(view, "#hub-file-systems-list")) =~ file_system.bucket_url assert updated_file_system in Livebook.Hubs.get_file_systems(team) end - test "detaches existing file system", %{conn: conn, team: team, node: node, org_key: org_key} do + @tag :git + test "updates existing Git file system", + %{conn: conn, team: team, node: node, org_key: org_key} do + file_system = build(:fs_git) + file_system = TeamsRPC.create_file_system(node, team, org_key, file_system) + assert_receive {:file_system_created, %Livebook.FileSystem.Git{} = ^file_system} + + # guarantee the branch is "main" and "file.txt" exists + {:ok, paths} = Livebook.FileSystem.list(file_system, "/", false) + assert "/file.txt" in paths + refute "/another_file.txt" in paths + + {:ok, view, _html} = live(conn, ~p"/hub/#{team.id}") + attrs = %{file_system: Livebook.FileSystem.dump(file_system)} + attrs = put_in(attrs.file_system.branch, "test") + + view + |> element("#hub-file-system-#{file_system.id}-edit") + |> render_click(%{"file_system" => file_system}) + + assert_patch(view, ~p"/hub/#{team.id}/file-systems/edit/#{file_system.id}") + assert render(view) =~ "Edit file storage" + assert has_element?(view, "#file_system_type-git") + + view + |> element("#file-systems-form") + |> render_change(attrs) + + refute view + |> element("#file-systems-form button[disabled]") + |> has_element?() + + refute view + |> element("#file-systems-form") + |> render_submit(attrs) =~ "Connection test failed" + + assert_patch(view, "/hub/#{team.id}") + assert render(view) =~ "File storage updated successfully" + + updated_file_system = %{file_system | branch: "test"} + assert_receive {:file_system_updated, ^updated_file_system} + + assert render(element(view, "#hub-file-systems-list")) =~ file_system.repo_url + assert updated_file_system in Livebook.Hubs.get_file_systems(team) + + # guarantee the branch has changed and the repository is updated + {:ok, paths} = Livebook.FileSystem.list(updated_file_system, "/", false) + refute "/file.txt" in paths + assert "/another_file.txt" in paths + end + + test "detaches existing S3 file system", + %{conn: conn, team: team, node: node, org_key: org_key} do bypass = Bypass.open() file_system = build_bypass_file_system(bypass, team.id) @@ -282,6 +376,40 @@ defmodule LivebookWeb.Integration.Hub.EditLiveTest do refute render(element(view, "#hub-file-systems-list")) =~ file_system.bucket_url refute file_system in Livebook.Hubs.get_file_systems(team) end + + @tag :git + test "detaches existing Git file system", + %{conn: conn, team: team, node: node, org_key: org_key} do + file_system = build(:fs_git) + file_system = TeamsRPC.create_file_system(node, team, org_key, file_system) + assert_receive {:file_system_created, %Livebook.FileSystem.Git{} = ^file_system} + + # wait for the repo to be cloned + # TODO: remove this sleep + Process.sleep(100) + + # guarantee the folder exists + repo_dir = Livebook.FileSystem.Git.git_dir(file_system) + assert File.exists?(repo_dir) + + {:ok, view, _html} = live(conn, ~p"/hub/#{team.id}") + + view + |> element("#hub-file-system-#{file_system.id}-detach", "Detach") + |> render_click() + + render_confirm(view) + + assert_receive {:file_system_deleted, ^file_system} + + assert_patch(view, "/hub/#{team.id}") + assert render(view) =~ "File storage deleted successfully" + refute render(element(view, "#hub-file-systems-list")) =~ file_system.repo_url + refute file_system in Livebook.Hubs.get_file_systems(team) + + # guarantee the folder were deleted + refute File.exists?(repo_dir) + end end describe "agent" do diff --git a/test/livebook_teams/web/open_live_test.exs b/test/livebook_teams/web/open_live_test.exs new file mode 100644 index 000000000..5dce4dcfe --- /dev/null +++ b/test/livebook_teams/web/open_live_test.exs @@ -0,0 +1,73 @@ +defmodule LivebookWeb.Integration.OpenLiveTest do + use Livebook.TeamsIntegrationCase, async: true + + import Phoenix.LiveViewTest + + @moduletag teams_for: :agent + setup :teams + + @moduletag subscribe_to_hubs_topics: [:connection, :file_systems] + @moduletag subscribe_to_teams_topics: [ + :clients, + :agents, + :deployment_groups, + :app_deployments, + :app_server + ] + + describe "git file storage" do + @describetag :git + + setup %{team: team, node: node, org_key: org_key} do + repo_url = "git@github.com:livebook-dev/test.git" + + file_system = + build(:fs_git, + id: Livebook.FileSystem.Utils.id("git", team.id, repo_url), + repo_url: repo_url, + hub_id: team.id, + external_id: nil + ) + + file_system = TeamsRPC.create_file_system(node, team, org_key, file_system) + assert_receive {:file_system_created, ^file_system} + + {:ok, file_system: file_system} + end + + test "lists files and folder on read-only mode", %{conn: conn, file_system: file_system} do + {:ok, view, html} = live(conn, ~p"/open/storage") + assert html =~ file_system.repo_url + + # change to Git file system + view + |> element(~s{button[id*="file-system-#{file_system.id}"]}) + |> render_click() + + # guarantee the write functions were disabled + assert has_element?(view, ~s{div[id*="new-item-menu"] button[disabled]}) + assert render(view) =~ "notebook_files" + + # change the path to list the .livemd file + view + |> element(~s{form[id*="path-form"]}) + |> render_change(%{path: "/notebook_files/"}) + + # render the view separately to make sure it received the :set_file event + assert render(view) =~ "notebook.livemd" + + # select the file + file_info_id = Base.url_encode64("/notebook_files/notebook.livemd", padding: false) + + view + |> element(~s{div[id*="file-#{file_info_id}"] button[aria-label="notebook.livemd"]}) + |> render_click() + + # guarantee the open function is disabled + assert has_element?(view, ~s{button[phx-click="open"][disabled]}) + + # only fork is available + assert has_element?(view, ~s{button[phx-click="fork"]:not([disabled])}) + end + end +end diff --git a/test/livebook_web/live/hub/edit_live_test.exs b/test/livebook_web/live/hub/edit_live_test.exs index b1c2a8e5c..3b5058b42 100644 --- a/test/livebook_web/live/hub/edit_live_test.exs +++ b/test/livebook_web/live/hub/edit_live_test.exs @@ -172,6 +172,9 @@ defmodule LivebookWeb.Hub.EditLiveTest do assert_patch(view, ~p"/hub/#{hub.id}/file-systems/new") assert render(view) =~ "Add file storage" + # Guarantee Git isn't available for Personal hub + refute render(view) =~ "Git" + view |> element("#file-systems-form") |> render_change(attrs) diff --git a/test/support/factory.ex b/test/support/factory.ex index 4ce541ecc..264555c89 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -80,11 +80,10 @@ defmodule Livebook.Factory do def build(:fs_s3) do bucket_url = "https://#{unique_value("mybucket-")}.s3.amazonaws.com" - hash = :crypto.hash(:sha256, bucket_url) hub_id = Livebook.Hubs.Personal.id() %Livebook.FileSystem.S3{ - id: "#{hub_id}-s3-#{Base.url_encode64(hash, padding: false)}", + id: Livebook.FileSystem.Utils.id("s3", hub_id, bucket_url), bucket_url: bucket_url, external_id: nil, region: "us-east-1", @@ -94,6 +93,21 @@ defmodule Livebook.Factory do } end + def build(:fs_git) do + repo_url = "git@github.com:livebook-dev/test.git" + hub_id = unique_value("team-") + key = System.get_env("TEST_GIT_SSH_KEY") + + %Livebook.FileSystem.Git{ + id: Livebook.FileSystem.Utils.id("git", hub_id, repo_url), + repo_url: repo_url, + branch: "main", + key: key, + external_id: "1", + hub_id: hub_id + } + end + def build(:agent_key) do %Livebook.Teams.AgentKey{ id: "1", diff --git a/test/support/hub_helpers.ex b/test/support/hub_helpers.ex index 674c2590d..9e347b969 100644 --- a/test/support/hub_helpers.ex +++ b/test/support/hub_helpers.ex @@ -119,18 +119,14 @@ defmodule Livebook.HubHelpers do def put_offline_hub_file_system(file_system) do hub = offline_hub() {:ok, pid} = hub_pid(hub) - secret_key = Livebook.Teams.derive_key(hub.teams_key) %{name: name} = Livebook.FileSystem.external_metadata(file_system) - attrs = Livebook.FileSystem.dump(file_system) - json = JSON.encode!(attrs) - value = Livebook.Teams.encrypt(json, secret_key) file_system_created = %LivebookProto.FileSystemCreated{ id: file_system.external_id, name: name, type: Livebook.FileSystems.type(file_system), - value: value + value: generate_file_system_json(hub, file_system) } send(pid, {:event, :file_system_created, file_system_created}) @@ -194,6 +190,14 @@ defmodule Livebook.HubHelpers do assert_receive {:agent_joined, ^agent} end + def generate_file_system_json(team, file_system) do + secret_key = Livebook.Teams.derive_key(team.teams_key) + attrs = Livebook.FileSystem.dump(file_system) + json = JSON.encode!(attrs) + + Livebook.Teams.encrypt(json, secret_key) + end + defp hub_pid(hub) do if pid = Livebook.Hubs.TeamClient.get_pid(hub.id) do {:ok, pid} diff --git a/test/support/integration/teams_rpc.ex b/test/support/integration/teams_rpc.ex index cc16b59ba..116988727 100644 --- a/test/support/integration/teams_rpc.ex +++ b/test/support/integration/teams_rpc.ex @@ -84,17 +84,22 @@ defmodule Livebook.TeamsRPC do end def create_file_system(node, team, org_key, file_system \\ nil) do - file_system = if file_system, do: file_system, else: Factory.build(:fs_s3) - derived_key = Livebook.Teams.derive_key(team.teams_key) - name = Livebook.FileSystem.external_metadata(file_system).name - type = Livebook.FileSystems.type(file_system) - attrs = Livebook.FileSystem.dump(file_system) - json = JSON.encode!(attrs) - value = Livebook.Teams.encrypt(json, derived_key) + file_system = + if file_system, + do: file_system, + else: Factory.build(:fs_s3) + + type = Livebook.FileSystems.type(file_system) + + attrs = %{ + name: Livebook.FileSystem.external_metadata(file_system).name, + type: String.to_atom(type), + value: Livebook.HubHelpers.generate_file_system_json(team, file_system), + org_key: org_key + } - attrs = %{name: name, type: String.to_atom(type), value: value, org_key: org_key} external_id = :erpc.call(node, TeamsRPC, :create_file_system, [attrs]).id - Map.replace!(file_system, :external_id, external_id) + Map.replace!(file_system, :external_id, to_string(external_id)) end def create_deployment_group(node, attrs \\ []) do diff --git a/test/test_helper.exs b/test/test_helper.exs index 9ba8a0be9..6f8f6446e 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -73,8 +73,10 @@ teams_exclude = end fly_exclude = if System.get_env("TEST_FLY_API_TOKEN"), do: [], else: [:fly] +git_exclude = if System.get_env("TEST_GIT_SSH_KEY"), do: [], else: [:git] ExUnit.start( assert_receive_timeout: if(windows?, do: 5_000, else: 1_500), - exclude: erl_docs_exclude ++ windows_exclude ++ teams_exclude ++ fly_exclude ++ [:k8s] + exclude: + erl_docs_exclude ++ windows_exclude ++ teams_exclude ++ fly_exclude ++ [:k8s] ++ git_exclude )