Add tests

test/livebook/file_system/git_test.exs

@@ -0,0 +1,224 @@
+defmodule Livebook.FileSystem.GitTest do
+  use Livebook.DataCase, async: true
+
+  @moduletag :git
+
+  alias Livebook.FileSystem
+  alias Livebook.FileSystem.Git
+
+  setup %{test: test} do
+    repo_url = "git@github.com:livebook-dev/test.git"
+    hub_id = test |> to_string() |> Base.encode32(padding: false)
+    id = Livebook.FileSystem.Utils.id("git", hub_id, repo_url)
+
+    {:ok, file_system: build(:fs_git, id: id, repo_url: repo_url, hub_id: hub_id)}
+  end
+
+  describe "FileSystem.default_path/1" do
+    test "returns the root path", %{file_system: file_system} do
+      assert FileSystem.default_path(file_system) == "/"
+    end
+  end
+
+  describe "common request errors" do
+    test "authorization failure", %{file_system: file_system} do
+      file_system = %{file_system | key: "foo"}
+
+      assert {:error, reason} = FileSystem.list(file_system, "/dir/", false)
+      assert reason =~ "Permission denied (publickey)."
+    end
+  end
+
+  describe "FileSystem.list/3" do
+    test "returns an empty list with invalid path", %{file_system: file_system} do
+      assert FileSystem.list(file_system, "/path/", false) ==
+               {:error, "no such file or directory"}
+    end
+
+    test "returns a list of absolute child object paths", %{file_system: file_system} do
+      assert {:ok, paths} = FileSystem.list(file_system, "/", false)
+      assert "/notebook_files/" in paths
+      assert "/file.txt" in paths
+    end
+  end
+
+  describe "FileSystem.read/2" do
+    test "returns an error when a nonexistent key is given", %{file_system: file_system} do
+      assert FileSystem.read(file_system, "/another_file.txt") ==
+               {:error, "fatal: path 'another_file.txt' does not exist in 'main'"}
+    end
+
+    test "returns object contents under the given key", %{file_system: file_system} do
+      assert {:ok, content} = FileSystem.read(file_system, "/file.txt")
+      assert content =~ "git file storage works"
+    end
+  end
+
+  describe "FileSystem.write/3" do
+    test "not implemented", %{file_system: file_system} do
+      assert_raise RuntimeError, "not implemented", fn ->
+        FileSystem.write(file_system, "/file.txt", "")
+      end
+    end
+  end
+
+  describe "FileSystem.create_dir/2" do
+    test "not implemented", %{file_system: file_system} do
+      assert_raise RuntimeError, "not implemented", fn ->
+        FileSystem.create_dir(file_system, "/folder")
+      end
+    end
+  end
+
+  describe "FileSystem.remove/2" do
+    test "not implemented", %{file_system: file_system} do
+      assert_raise RuntimeError, "not implemented", fn ->
+        FileSystem.remove(file_system, "/file.txt")
+      end
+    end
+  end
+
+  describe "FileSystem.copy/3" do
+    test "not implemented", %{file_system: file_system} do
+      assert_raise RuntimeError, "not implemented", fn ->
+        FileSystem.copy(file_system, "/file.txt", "/folder/file.txt")
+      end
+    end
+  end
+
+  describe "FileSystem.rename/3" do
+    test "not implemented", %{file_system: file_system} do
+      assert_raise RuntimeError, "not implemented", fn ->
+        FileSystem.rename(file_system, "/file.txt", "/another_file.txt")
+      end
+    end
+  end
+
+  describe "FileSystem.etag_for/2" do
+    test "returns an error when a nonexistent key is given", %{file_system: file_system} do
+      assert {:error, reason} = FileSystem.etag_for(file_system, "/another_file.txt")
+      assert reason =~ "fatal: path 'another_file.txt' does not exist in 'main'"
+    end
+
+    test "returns the ETag value received from the server", %{file_system: file_system} do
+      assert {:ok, _etag} = FileSystem.etag_for(file_system, "/file.txt")
+    end
+  end
+
+  describe "FileSystem.exists?/2" do
+    test "returns valid response", %{file_system: file_system} do
+      assert {:ok, true} = FileSystem.exists?(file_system, "/file.txt")
+      assert {:ok, false} = FileSystem.exists?(file_system, "/another_file.txt")
+    end
+
+    test "returns error with invalid path", %{file_system: file_system} do
+      assert {:error, "fatal: ../../.bashrc: '../../.bashrc' is outside repository at" <> _} =
+               FileSystem.exists?(file_system, "../../.bashrc")
+    end
+  end
+
+  describe "FileSystem.resolve_path/3" do
+    test "resolves relative paths", %{file_system: file_system} do
+      assert "/dir/" = FileSystem.resolve_path(file_system, "/dir/", "")
+      assert "/dir/file.txt" = FileSystem.resolve_path(file_system, "/dir/", "file.txt")
+      assert "/dir/nested/" = FileSystem.resolve_path(file_system, "/dir/", "nested/")
+      assert "/dir/" = FileSystem.resolve_path(file_system, "/dir/", ".")
+      assert "/" = FileSystem.resolve_path(file_system, "/dir/", "..")
+
+      assert "/file.txt" =
+               FileSystem.resolve_path(file_system, "/dir/", "nested/../.././file.txt")
+    end
+
+    test "resolves absolute paths", %{file_system: file_system} do
+      assert "/" = FileSystem.resolve_path(file_system, "/dir/", "/")
+      assert "/file.txt" = FileSystem.resolve_path(file_system, "/dir/", "/file.txt")
+      assert "/nested/" = FileSystem.resolve_path(file_system, "/dir/", "/nested/")
+
+      assert "/nested/file.txt" =
+               FileSystem.resolve_path(file_system, "/dir/", "///nested///other/..///file.txt")
+    end
+  end
+
+  describe "FileSystem chunked write" do
+    test "not implemented", %{file_system: file_system} do
+      assert_raise RuntimeError, "not implemented", fn ->
+        FileSystem.write_stream_init(file_system, "/readme.txt", part_size: 5_000)
+      end
+    end
+  end
+
+  describe "FileSystem.read_stream_into/2" do
+    test "not implemented", %{file_system: file_system} do
+      assert_raise RuntimeError, "not implemented", fn ->
+        FileSystem.read_stream_into(file_system, "/file.txt", <<>>)
+      end
+    end
+  end
+
+  describe "FileSystem.load/2" do
+    test "loads from atom keys" do
+      fields = %{
+        id: "team-123456-git-Ios91o6sRIRnTmpRlO2jpwHPtFXlZh2FH6rkvxuN_8M",
+        repo_url: "git@github.com:livebook-dev/test.git",
+        branch: "main",
+        key: "foo",
+        hub_id: "team-123456",
+        external_id: "1"
+      }
+
+      assert FileSystem.load(%Git{}, fields) == %Git{
+               id: "team-123456-git-Ios91o6sRIRnTmpRlO2jpwHPtFXlZh2FH6rkvxuN_8M",
+               repo_url: "git@github.com:livebook-dev/test.git",
+               branch: "main",
+               key: "foo",
+               hub_id: "team-123456",
+               external_id: "1"
+             }
+    end
+
+    test "loads from string keys" do
+      fields = %{
+        "id" => "team-123456-git-Ios91o6sRIRnTmpRlO2jpwHPtFXlZh2FH6rkvxuN_8M",
+        "repo_url" => "git@github.com:livebook-dev/test.git",
+        "branch" => "main",
+        "key" => "foo",
+        "hub_id" => "team-123456",
+        "external_id" => "1"
+      }
+
+      assert FileSystem.load(%Git{}, fields) == %Git{
+               id: "team-123456-git-Ios91o6sRIRnTmpRlO2jpwHPtFXlZh2FH6rkvxuN_8M",
+               repo_url: "git@github.com:livebook-dev/test.git",
+               branch: "main",
+               key: "foo",
+               hub_id: "team-123456",
+               external_id: "1"
+             }
+    end
+  end
+
+  describe "FileSystem.dump/1" do
+    test "dumps into a map ready to be stored" do
+      repo_url = "git@github.com:livebook-dev/test.git"
+      hub_id = "team-123456"
+
+      file_system =
+        build(:fs_git,
+          id: Livebook.FileSystem.Utils.id("git", hub_id, repo_url),
+          repo_url: repo_url,
+          branch: "main",
+          key: "foo",
+          hub_id: hub_id
+        )
+
+      assert FileSystem.dump(file_system) == %{
+               id: "team-123456-git-Ios91o6sRIRnTmpRlO2jpwHPtFXlZh2FH6rkvxuN_8M",
+               repo_url: "git@github.com:livebook-dev/test.git",
+               branch: "main",
+               key: "foo",
+               hub_id: "team-123456",
+               external_id: "1"
+             }
+    end
+  end
+end

test/livebook_teams/web/hub/edit_live_test.exs

@@ -179,7 +179,7 @@ defmodule LivebookWeb.Integration.Hub.EditLiveTest do
       end
     end
 
-    test "creates a file system", %{conn: conn, team: team} do
+    test "creates a S3 file system", %{conn: conn, team: team} do
       {:ok, view, _html} = live(conn, ~p"/hub/#{team.id}")
 
       bypass = Bypass.open()
@@ -188,7 +188,6 @@ defmodule LivebookWeb.Integration.Hub.EditLiveTest do
       attrs = %{file_system: Livebook.FileSystem.dump(file_system)}
 
       expect_s3_listing(bypass)
-
       refute render(view) =~ file_system.bucket_url
 
       view
@@ -197,6 +196,7 @@ defmodule LivebookWeb.Integration.Hub.EditLiveTest do
 
       assert_patch(view, ~p"/hub/#{team.id}/file-systems/new")
       assert render(view) =~ "Add file storage"
+      assert has_element?(view, "#file_system_type-s3")
 
       view
       |> element("#file-systems-form")
@@ -210,14 +210,55 @@ defmodule LivebookWeb.Integration.Hub.EditLiveTest do
       |> element("#file-systems-form")
       |> render_submit(attrs)
 
-      assert_receive {:file_system_created, %{id: ^id} = file_system}
+      assert_receive {:file_system_created, %Livebook.FileSystem.S3{id: ^id} = file_system}
       assert_patch(view, "/hub/#{team.id}")
       assert render(view) =~ "File storage added successfully"
       assert render(element(view, "#hub-file-systems-list")) =~ file_system.bucket_url
       assert file_system in Livebook.Hubs.get_file_systems(team)
     end
 
-    test "updates existing file system", %{conn: conn, team: team, node: node, org_key: org_key} do
+    @tag :git
+    test "creates a Git file system", %{conn: conn, team: team} do
+      file_system = build(:fs_git)
+      id = file_system.id
+      attrs = %{file_system: Livebook.FileSystem.dump(file_system)}
+
+      {:ok, view, _html} = live(conn, ~p"/hub/#{team.id}")
+      refute render(view) =~ file_system.repo_url
+
+      view
+      |> element("#add-file-system")
+      |> render_click()
+
+      assert_patch(view, ~p"/hub/#{team.id}/file-systems/new")
+      assert render(view) =~ "Add file storage"
+
+      # change the file system type from S3 to git
+      view
+      |> element("#file_system_type-git")
+      |> render_click()
+
+      view
+      |> element("#file-systems-form")
+      |> render_change(attrs)
+
+      refute view
+             |> element("#file-systems-form button[disabled]")
+             |> has_element?()
+
+      view
+      |> element("#file-systems-form")
+      |> render_submit(attrs)
+
+      assert_receive {:file_system_created, %Livebook.FileSystem.Git{id: ^id} = file_system}
+      assert_patch(view, "/hub/#{team.id}")
+      assert render(view) =~ "File storage added successfully"
+      assert render(element(view, "#hub-file-systems-list")) =~ file_system.repo_url
+      assert file_system in Livebook.Hubs.get_file_systems(team)
+    end
+
+    test "updates existing S3 file system",
+         %{conn: conn, team: team, node: node, org_key: org_key} do
       bypass = Bypass.open()
 
       file_system = build_bypass_file_system(bypass, team.id)
@@ -248,16 +289,69 @@ defmodule LivebookWeb.Integration.Hub.EditLiveTest do
       |> element("#file-systems-form")
       |> render_submit(put_in(attrs.file_system.access_key_id, "new key"))
 
-      updated_file_system = %{file_system | access_key_id: "new key"}
-
-      assert_receive {:file_system_updated, ^updated_file_system}
       assert_patch(view, "/hub/#{team.id}")
       assert render(view) =~ "File storage updated successfully"
+
+      updated_file_system = %{file_system | access_key_id: "new key"}
+      assert_receive {:file_system_updated, ^updated_file_system}
+
       assert render(element(view, "#hub-file-systems-list")) =~ file_system.bucket_url
       assert updated_file_system in Livebook.Hubs.get_file_systems(team)
     end
 
-    test "detaches existing file system", %{conn: conn, team: team, node: node, org_key: org_key} do
+    @tag :git
+    test "updates existing Git file system",
+         %{conn: conn, team: team, node: node, org_key: org_key} do
+      file_system = build(:fs_git)
+      file_system = TeamsRPC.create_file_system(node, team, org_key, file_system)
+      assert_receive {:file_system_created, %Livebook.FileSystem.Git{} = ^file_system}
+
+      # guarantee the branch is "main" and "file.txt" exists 
+      {:ok, paths} = Livebook.FileSystem.list(file_system, "/", false)
+      assert "/file.txt" in paths
+      refute "/another_file.txt" in paths
+
+      {:ok, view, _html} = live(conn, ~p"/hub/#{team.id}")
+      attrs = %{file_system: Livebook.FileSystem.dump(file_system)}
+      attrs = put_in(attrs.file_system.branch, "test")
+
+      view
+      |> element("#hub-file-system-#{file_system.id}-edit")
+      |> render_click(%{"file_system" => file_system})
+
+      assert_patch(view, ~p"/hub/#{team.id}/file-systems/edit/#{file_system.id}")
+      assert render(view) =~ "Edit file storage"
+      assert has_element?(view, "#file_system_type-git")
+
+      view
+      |> element("#file-systems-form")
+      |> render_change(attrs)
+
+      refute view
+             |> element("#file-systems-form button[disabled]")
+             |> has_element?()
+
+      refute view
+             |> element("#file-systems-form")
+             |> render_submit(attrs) =~ "Connection test failed"
+
+      assert_patch(view, "/hub/#{team.id}")
+      assert render(view) =~ "File storage updated successfully"
+
+      updated_file_system = %{file_system | branch: "test"}
+      assert_receive {:file_system_updated, ^updated_file_system}
+
+      assert render(element(view, "#hub-file-systems-list")) =~ file_system.repo_url
+      assert updated_file_system in Livebook.Hubs.get_file_systems(team)
+
+      # guarantee the branch has changed and the repository is updated
+      {:ok, paths} = Livebook.FileSystem.list(updated_file_system, "/", false)
+      refute "/file.txt" in paths
+      assert "/another_file.txt" in paths
+    end
+
+    test "detaches existing S3 file system",
+         %{conn: conn, team: team, node: node, org_key: org_key} do
       bypass = Bypass.open()
 
       file_system = build_bypass_file_system(bypass, team.id)
@@ -282,6 +376,40 @@ defmodule LivebookWeb.Integration.Hub.EditLiveTest do
       refute render(element(view, "#hub-file-systems-list")) =~ file_system.bucket_url
       refute file_system in Livebook.Hubs.get_file_systems(team)
     end
+
+    @tag :git
+    test "detaches existing Git file system",
+         %{conn: conn, team: team, node: node, org_key: org_key} do
+      file_system = build(:fs_git)
+      file_system = TeamsRPC.create_file_system(node, team, org_key, file_system)
+      assert_receive {:file_system_created, %Livebook.FileSystem.Git{} = ^file_system}
+
+      # wait for the repo to be cloned
+      # TODO: remove this sleep
+      Process.sleep(100)
+
+      # guarantee the folder exists
+      repo_dir = Livebook.FileSystem.Git.git_dir(file_system)
+      assert File.exists?(repo_dir)
+
+      {:ok, view, _html} = live(conn, ~p"/hub/#{team.id}")
+
+      view
+      |> element("#hub-file-system-#{file_system.id}-detach", "Detach")
+      |> render_click()
+
+      render_confirm(view)
+
+      assert_receive {:file_system_deleted, ^file_system}
+
+      assert_patch(view, "/hub/#{team.id}")
+      assert render(view) =~ "File storage deleted successfully"
+      refute render(element(view, "#hub-file-systems-list")) =~ file_system.repo_url
+      refute file_system in Livebook.Hubs.get_file_systems(team)
+
+      # guarantee the folder were deleted
+      refute File.exists?(repo_dir)
+    end
   end
 
   describe "agent" do

test/livebook_teams/web/open_live_test.exs

@@ -0,0 +1,73 @@
+defmodule LivebookWeb.Integration.OpenLiveTest do
+  use Livebook.TeamsIntegrationCase, async: true
+
+  import Phoenix.LiveViewTest
+
+  @moduletag teams_for: :agent
+  setup :teams
+
+  @moduletag subscribe_to_hubs_topics: [:connection, :file_systems]
+  @moduletag subscribe_to_teams_topics: [
+               :clients,
+               :agents,
+               :deployment_groups,
+               :app_deployments,
+               :app_server
+             ]
+
+  describe "git file storage" do
+    @describetag :git
+
+    setup %{team: team, node: node, org_key: org_key} do
+      repo_url = "git@github.com:livebook-dev/test.git"
+
+      file_system =
+        build(:fs_git,
+          id: Livebook.FileSystem.Utils.id("git", team.id, repo_url),
+          repo_url: repo_url,
+          hub_id: team.id,
+          external_id: nil
+        )
+
+      file_system = TeamsRPC.create_file_system(node, team, org_key, file_system)
+      assert_receive {:file_system_created, ^file_system}
+
+      {:ok, file_system: file_system}
+    end
+
+    test "lists files and folder on read-only mode", %{conn: conn, file_system: file_system} do
+      {:ok, view, html} = live(conn, ~p"/open/storage")
+      assert html =~ file_system.repo_url
+
+      # change to Git file system
+      view
+      |> element(~s{button[id*="file-system-#{file_system.id}"]})
+      |> render_click()
+
+      # guarantee the write functions were disabled
+      assert has_element?(view, ~s{div[id*="new-item-menu"] button[disabled]})
+      assert render(view) =~ "notebook_files"
+
+      # change the path to list the .livemd file
+      view
+      |> element(~s{form[id*="path-form"]})
+      |> render_change(%{path: "/notebook_files/"})
+
+      # render the view separately to make sure it received the :set_file event
+      assert render(view) =~ "notebook.livemd"
+
+      # select the file
+      file_info_id = Base.url_encode64("/notebook_files/notebook.livemd", padding: false)
+
+      view
+      |> element(~s{div[id*="file-#{file_info_id}"] button[aria-label="notebook.livemd"]})
+      |> render_click()
+
+      # guarantee the open function is disabled
+      assert has_element?(view, ~s{button[phx-click="open"][disabled]})
+
+      # only fork is available
+      assert has_element?(view, ~s{button[phx-click="fork"]:not([disabled])})
+    end
+  end
+end

test/livebook_web/live/hub/edit_live_test.exs

@@ -172,6 +172,9 @@ defmodule LivebookWeb.Hub.EditLiveTest do
       assert_patch(view, ~p"/hub/#{hub.id}/file-systems/new")
       assert render(view) =~ "Add file storage"
 
+      # Guarantee Git isn't available for Personal hub
+      refute render(view) =~ "Git"
+
       view
       |> element("#file-systems-form")
       |> render_change(attrs)

test/support/factory.ex

@@ -80,11 +80,10 @@ defmodule Livebook.Factory do
 
   def build(:fs_s3) do
     bucket_url = "https://#{unique_value("mybucket-")}.s3.amazonaws.com"
-    hash = :crypto.hash(:sha256, bucket_url)
     hub_id = Livebook.Hubs.Personal.id()
 
     %Livebook.FileSystem.S3{
-      id: "#{hub_id}-s3-#{Base.url_encode64(hash, padding: false)}",
+      id: Livebook.FileSystem.Utils.id("s3", hub_id, bucket_url),
       bucket_url: bucket_url,
       external_id: nil,
       region: "us-east-1",
@@ -94,6 +93,21 @@ defmodule Livebook.Factory do
     }
   end
 
+  def build(:fs_git) do
+    repo_url = "git@github.com:livebook-dev/test.git"
+    hub_id = unique_value("team-")
+    key = System.get_env("TEST_GIT_SSH_KEY")
+
+    %Livebook.FileSystem.Git{
+      id: Livebook.FileSystem.Utils.id("git", hub_id, repo_url),
+      repo_url: repo_url,
+      branch: "main",
+      key: key,
+      external_id: "1",
+      hub_id: hub_id
+    }
+  end
+
   def build(:agent_key) do
     %Livebook.Teams.AgentKey{
       id: "1",

test/support/hub_helpers.ex

@@ -119,18 +119,14 @@ defmodule Livebook.HubHelpers do
   def put_offline_hub_file_system(file_system) do
     hub = offline_hub()
     {:ok, pid} = hub_pid(hub)
-    secret_key = Livebook.Teams.derive_key(hub.teams_key)
     %{name: name} = Livebook.FileSystem.external_metadata(file_system)
-    attrs = Livebook.FileSystem.dump(file_system)
-    json = JSON.encode!(attrs)
-    value = Livebook.Teams.encrypt(json, secret_key)
 
     file_system_created =
       %LivebookProto.FileSystemCreated{
         id: file_system.external_id,
         name: name,
         type: Livebook.FileSystems.type(file_system),
-        value: value
+        value: generate_file_system_json(hub, file_system)
       }
 
     send(pid, {:event, :file_system_created, file_system_created})
@@ -194,6 +190,14 @@ defmodule Livebook.HubHelpers do
     assert_receive {:agent_joined, ^agent}
   end
 
+  def generate_file_system_json(team, file_system) do
+    secret_key = Livebook.Teams.derive_key(team.teams_key)
+    attrs = Livebook.FileSystem.dump(file_system)
+    json = JSON.encode!(attrs)
+
+    Livebook.Teams.encrypt(json, secret_key)
+  end
+
   defp hub_pid(hub) do
     if pid = Livebook.Hubs.TeamClient.get_pid(hub.id) do
       {:ok, pid}

test/support/integration/teams_rpc.ex

@@ -84,17 +84,22 @@ defmodule Livebook.TeamsRPC do
   end
 
   def create_file_system(node, team, org_key, file_system \\ nil) do
-    file_system = if file_system, do: file_system, else: Factory.build(:fs_s3)
-    derived_key = Livebook.Teams.derive_key(team.teams_key)
-    name = Livebook.FileSystem.external_metadata(file_system).name
-    type = Livebook.FileSystems.type(file_system)
-    attrs = Livebook.FileSystem.dump(file_system)
-    json = JSON.encode!(attrs)
-    value = Livebook.Teams.encrypt(json, derived_key)
+    file_system =
+      if file_system,
+        do: file_system,
+        else: Factory.build(:fs_s3)
+
+    type = Livebook.FileSystems.type(file_system)
+
+    attrs = %{
+      name: Livebook.FileSystem.external_metadata(file_system).name,
+      type: String.to_atom(type),
+      value: Livebook.HubHelpers.generate_file_system_json(team, file_system),
+      org_key: org_key
+    }
 
-    attrs = %{name: name, type: String.to_atom(type), value: value, org_key: org_key}
     external_id = :erpc.call(node, TeamsRPC, :create_file_system, [attrs]).id
-    Map.replace!(file_system, :external_id, external_id)
+    Map.replace!(file_system, :external_id, to_string(external_id))
   end
 
   def create_deployment_group(node, attrs \\ []) do

test/test_helper.exs

@@ -73,8 +73,10 @@ teams_exclude =
   end
 
 fly_exclude = if System.get_env("TEST_FLY_API_TOKEN"), do: [], else: [:fly]
+git_exclude = if System.get_env("TEST_GIT_SSH_KEY"), do: [], else: [:git]
 
 ExUnit.start(
   assert_receive_timeout: if(windows?, do: 5_000, else: 1_500),
-  exclude: erl_docs_exclude ++ windows_exclude ++ teams_exclude ++ fly_exclude ++ [:k8s]
+  exclude:
+    erl_docs_exclude ++ windows_exclude ++ teams_exclude ++ fly_exclude ++ [:k8s] ++ git_exclude
 )