WIP

lib/livebook/file_system/git.ex

@@ -0,0 +1,181 @@
+defmodule Livebook.FileSystem.Git do
+  use Ecto.Schema
+  import Ecto.Changeset
+
+  alias Livebook.FileSystem
+
+  # File system backed by an Git repository.
+
+  @type t :: %__MODULE__{
+          id: String.t(),
+          repo_url: String.t(),
+          external_id: String.t() | nil,
+          hub_id: String.t()
+        }
+
+  embedded_schema do
+    field :repo_url, :string
+    field :external_id, :string
+    field :hub_id, :string
+  end
+
+  @doc """
+  Returns an `%Ecto.Changeset{}` for tracking file system changes.
+  """
+  @spec change_file_system(t(), map()) :: Ecto.Changeset.t()
+  def change_file_system(git, attrs \\ %{}) do
+    changeset(git, attrs)
+  end
+
+  defp changeset(git, attrs) do
+    git
+    |> cast(attrs, [:repo_url, :external_id, :hub_id])
+    |> validate_format(:repo_url, ~r/^git@[\w\.\-]+:[\w\.\-]+\/[\w\.\-]+\.git$/,
+      message: "must be a valid repo URL"
+    )
+    |> validate_required([:repo_url, :hub_id])
+    |> put_id()
+  end
+
+  defp put_id(changeset) do
+    hub_id = get_field(changeset, :hub_id)
+    repo_url = get_field(changeset, :repo_url)
+
+    if get_field(changeset, :id) do
+      changeset
+    else
+      put_change(changeset, :id, FileSystem.Utils.id("git", hub_id, repo_url))
+    end
+  end
+end
+
+defimpl Livebook.FileSystem, for: Livebook.FileSystem.Git do
+  alias Livebook.FileSystem
+  alias Livebook.FileSystem.Git
+
+  def type(_file_system) do
+    :global
+  end
+
+  def default_path(_file_system) do
+    "/"
+  end
+
+  def list(file_system, path, _recursive) do
+    FileSystem.Utils.assert_dir_path!(path)
+    "/" <> path = path
+
+    with {:ok, keys} <- Git.Client.list_files(file_system, path) do
+      if keys == [] do
+        FileSystem.Utils.posix_error(:enoent)
+      else
+        {:ok, Enum.map(keys, &("/" <> &1))}
+      end
+    end
+  end
+
+  def read(file_system, path) do
+    FileSystem.Utils.assert_regular_path!(path)
+    "/" <> path = path
+
+    Git.Client.read_file(file_system, path)
+  end
+
+  def write(_file_system, path, _content) do
+    FileSystem.Utils.assert_regular_path!(path)
+    :ok
+  end
+
+  def access(_file_system, _path) do
+    {:ok, :read}
+  end
+
+  def create_dir(_file_system, path) do
+    FileSystem.Utils.assert_dir_path!(path)
+    :ok
+  end
+
+  def remove(_file_system, _path) do
+    :ok
+  end
+
+  def copy(_file_system, _source_path, _destination_path) do
+    :ok
+  end
+
+  def rename(_file_system, _source_path, _destination_path) do
+    :ok
+  end
+
+  def etag_for(file_system, path) do
+    FileSystem.Utils.assert_regular_path!(path)
+    "/" <> path = path
+
+    Git.Client.etag(file_system, path)
+  end
+
+  def exists?(file_system, path) do
+    "/" <> path = path
+
+    with {:ok, files} <- Git.Client.list_files(file_system, path) do
+      {:ok, files != []}
+    end
+  end
+
+  def resolve_path(_file_system, dir_path, subject) do
+    FileSystem.Utils.resolve_unix_like_path(dir_path, subject)
+  end
+
+  def write_stream_init(_file_system, path, _opts) do
+    FileSystem.Utils.assert_regular_path!(path)
+    "/" <> path = path
+
+    {:ok, %{path: path}}
+  end
+
+  def write_stream_chunk(_file_system, state, chunk) when is_binary(chunk) do
+    {:ok, state}
+  end
+
+  def write_stream_finish(_file_system, _state) do
+    :ok
+  end
+
+  def write_stream_halt(_file_system, _state) do
+    :ok
+  end
+
+  def read_stream_into(_file_system, path, _collectable) do
+    FileSystem.Utils.assert_regular_path!(path)
+    :ok
+  end
+
+  def load(file_system, %{"repo_url" => _} = fields) do
+    load(file_system, %{
+      repo_url: fields["repo_url"],
+      external_id: fields["external_id"],
+      id: fields["id"],
+      hub_id: fields["hub_id"]
+    })
+  end
+
+  def load(file_system, fields) do
+    %{
+      file_system
+      | id: fields.id,
+        repo_url: fields.repo_url,
+        external_id: fields.external_id,
+        hub_id: fields.hub_id
+    }
+  end
+
+  def dump(file_system) do
+    file_system
+    |> Map.from_struct()
+    |> Map.take([:id, :repo_url, :hub_id, :external_id])
+  end
+
+  def external_metadata(file_system) do
+    %{name: file_system.repo_url, error_field: "repo_url"}
+  end
+end

lib/livebook/file_system/git/client.ex

@@ -0,0 +1,98 @@
+defmodule Livebook.FileSystem.Git.Client do
+  alias Livebook.FileSystem
+
+  @doc """
+  Sends a request to the repository to get list of files.
+  """
+  @spec list_files(FileSystem.Git.t(), String.t()) ::
+          {:ok, list(String.t())} | {:error, FileSystem.error()}
+  def list_files(file_system, path)
+
+  def list_files(%FileSystem.Git{} = file_system, "/") do
+    git_dir = git_dir(file_system)
+    ls_tree(git_dir, "HEAD")
+  end
+
+  def list_files(%FileSystem.Git{} = file_system, path) do
+    git_dir = git_dir(file_system)
+    ls_tree(git_dir, "HEAD", [path])
+  end
+
+  @doc """
+  Sends a request to the repository to read a file.
+  """
+  @spec read_file(FileSystem.Git.t(), String.t()) ::
+          {:ok, String.t()} | {:error, FileSystem.error()}
+  def read_file(%FileSystem.Git{} = file_system, path) do
+    git_dir = git_dir(file_system)
+    show(git_dir, "HEAD", path)
+  end
+
+  @doc """
+  Sends a request to the repository to read file ETag.
+  """
+  @spec etag(FileSystem.Git.t(), String.t()) :: {:ok, String.t()} | {:error, FileSystem.error()}
+  def etag(%FileSystem.Git{} = file_system, path) do
+    git_dir = git_dir(file_system)
+    rev_parse(git_dir, "HEAD", path)
+  end
+
+  defp ls_tree(git_dir, branch, args \\ []) do
+    with {:ok, result} <- git(git_dir, ["ls-tree", "--name-only", branch] ++ args) do
+      {:ok, String.split(result, "\n", trim: true)}
+    end
+  end
+
+  defp show(git_dir, branch, path) do
+    git(git_dir, ["show", "#{branch}:#{path}"])
+  end
+
+  defp rev_parse(git_dir, branch, path) do
+    with {:ok, etag} <- git(git_dir, ["rev-parse", "#{branch}:#{path}"]) do
+      {:ok, String.trim(etag)}
+    end
+  end
+
+  defp git(git_dir, args) do
+    if git = System.find_executable("git") do
+      case System.cmd(git, args, cmd_opts(git_dir)) do
+        {result, 0} -> {:ok, result}
+        {error, _} -> {:error, String.trim(error)}
+      end
+    else
+      {:error, "'git' executable not found"}
+    end
+  end
+
+  @cmd_opts [use_stdio: true, stderr_to_stdout: true]
+
+  defp cmd_opts(git_dir) do
+    if File.exists?(git_dir) do
+      Keyword.merge(@cmd_opts, cd: git_dir)
+    else
+      @cmd_opts
+    end
+  end
+
+  defp git_dir(file_system) do
+    git_dir = Path.join(System.tmp_dir!(), file_system.id)
+
+    args = [
+      "clone",
+      "--bare",
+      "--depth=1",
+      "--single-branch",
+      file_system.repo_url,
+      git_dir
+    ]
+
+    if File.exists?(git_dir) do
+      git_dir
+    else
+      case git(git_dir, args) do
+        {:ok, _} -> git_dir
+        {:error, reason} -> raise reason
+      end
+    end
+  end
+end

lib/livebook/file_system/s3.ex

@@ -2,6 +2,8 @@ defmodule Livebook.FileSystem.S3 do
   use Ecto.Schema
   import Ecto.Changeset
 
+  alias Livebook.FileSystem
+
   # File system backed by an S3 bucket.
 
   @type t :: %__MODULE__{
@@ -81,34 +83,17 @@ defmodule Livebook.FileSystem.S3 do
     if get_field(changeset, :id) do
       changeset
     else
-      put_change(changeset, :id, id(hub_id, bucket_url))
+      put_change(changeset, :id, FileSystem.Utils.id("s3", hub_id, bucket_url))
     end
   end
 
-  def id(_, nil), do: nil
-
-  def id(hub_id, bucket_url) do
-    if hub_id == nil or hub_id == Livebook.Hubs.Personal.id() do
-      hashed_id(bucket_url)
-    else
-      "#{hub_id}-#{hashed_id(bucket_url)}"
-    end
-  end
-
-  defp hashed_id(bucket_url) do
-    hash = :crypto.hash(:sha256, bucket_url)
-    encrypted_hash = Base.url_encode64(hash, padding: false)
-
-    "s3-#{encrypted_hash}"
-  end
-
   @doc """
   Retrieves credentials for the given file system.
 
   If the credentials are not specified by the file system, they are
   fetched from environment variables or AWS instance if applicable.
   """
-  @spec credentials(S3.t()) :: S3.credentials()
+  @spec credentials(t()) :: credentials()
   def credentials(%__MODULE__{} = file_system) do
     case {file_system.access_key_id, file_system.secret_access_key} do
       {nil, nil} ->

lib/livebook/file_system/utils.ex

@@ -104,6 +104,26 @@ defmodule Livebook.FileSystem.Utils do
     |> Enum.join("/")
   end
 
+  @doc """
+  Returns the id based on given hub id and data with given prefix.
+  """
+  def id(prefix, hub_id, data)
+
+  def id(_, _, nil), do: nil
+
+  def id(prefix, hub_id, data) do
+    if hub_id == nil or hub_id == Livebook.Hubs.Personal.id() do
+      hashed_id(prefix, data)
+    else
+      "#{hub_id}-#{hashed_id(prefix, data)}"
+    end
+  end
+
+  defp hashed_id(prefix, data) do
+    hash = :crypto.hash(:sha256, data)
+    "#{prefix}-#{Base.url_encode64(hash, padding: false)}"
+  end
+
   defp remove_in_middle([], _elem), do: []
   defp remove_in_middle([head], _elem), do: [head]
   defp remove_in_middle([head | tail], elem), do: remove_in_middle(tail, elem, [head])

lib/livebook/file_systems.ex

@@ -17,6 +17,10 @@ defmodule Livebook.FileSystems do
     FileSystem.S3.change_file_system(file_system, attrs)
   end
 
+  def change_file_system(%FileSystem.Git{} = file_system, attrs) do
+    FileSystem.Git.change_file_system(file_system, attrs)
+  end
+
   @doc """
   Loads the file system from given type and dumped data.
   """
@@ -35,6 +39,7 @@ defmodule Livebook.FileSystems do
   def type_to_module(type)
   def type_to_module("local"), do: FileSystem.Local
   def type_to_module("s3"), do: FileSystem.S3
+  def type_to_module("git"), do: FileSystem.Git
 
   @doc """
   Returns a serializable type for corresponding to the given file
@@ -44,4 +49,5 @@ defmodule Livebook.FileSystems do
   def module_to_type(module)
   def module_to_type(FileSystem.Local), do: "local"
   def module_to_type(FileSystem.S3), do: "s3"
+  def module_to_type(FileSystem.Git), do: "git"
 end

lib/livebook/hubs/broadcasts.ex

@@ -127,7 +127,7 @@ defmodule Livebook.Hubs.Broadcasts do
     broadcast(@secrets_topic, {:secret_deleted, secret})
   end
 
-  @allowed_file_systems [FileSystem.S3]
+  @allowed_file_systems [FileSystem.S3, FileSystem.Git]
 
   @doc """
   Broadcasts under `#{@file_systems_topic}` topic when hub received a new file system.

lib/livebook/hubs/personal.ex

@@ -140,7 +140,10 @@ defmodule Livebook.Hubs.Personal do
   @spec get_file_systems() :: list(FileSystem.t())
   def get_file_systems() do
     Storage.all(@file_systems_namespace)
-    |> Enum.sort_by(& &1.bucket_url)
+    |> Enum.sort_by(fn
+      %{repo_url: repo_url} -> repo_url
+      %{bucket_url: bucket_url} -> bucket_url
+    end)
     |> Enum.map(&to_file_system/1)
   end
 

lib/livebook_web/components/file_system_components.ex

@@ -10,6 +10,7 @@ defmodule LivebookWeb.FileSystemComponents do
 
   def file_system_name(FileSystem.Local), do: "Disk"
   def file_system_name(FileSystem.S3), do: "S3"
+  def file_system_name(FileSystem.Git), do: "Git"
 
   @doc """
   Formats the given file system into a descriptive label.
@@ -18,6 +19,7 @@ defmodule LivebookWeb.FileSystemComponents do
 
   def file_system_label(%FileSystem.Local{}), do: "Disk"
   def file_system_label(%FileSystem.S3{} = fs), do: fs.bucket_url
+  def file_system_label(%FileSystem.Git{} = fs), do: fs.repo_url
 
   @doc """
   Renders an icon representing the given file system.
@@ -37,4 +39,10 @@ defmodule LivebookWeb.FileSystemComponents do
     </i>
     """
   end
+
+  def file_system_icon(%{file_system: %FileSystem.Git{}} = assigns) do
+    ~H"""
+    <.remix_icon icon="git-repository-line leading-none" />
+    """
+  end
 end

mix.lock

@@ -56,7 +56,7 @@
   "telemetry_metrics": {:hex, :telemetry_metrics, "1.1.0", "5bd5f3b5637e0abea0426b947e3ce5dd304f8b3bc6617039e2b5a008adc02f8f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e7b79e8ddfde70adb6db8a6623d1778ec66401f366e9a8f5dd0955c56bc8ce67"},
   "telemetry_poller": {:hex, :telemetry_poller, "1.1.0", "58fa7c216257291caaf8d05678c8d01bd45f4bdbc1286838a28c4bb62ef32999", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9eb9d9cbfd81cbd7cdd24682f8711b6e2b691289a0de6826e58452f28c103c8f"},
   "thousand_island": {:hex, :thousand_island, "1.3.5", "6022b6338f1635b3d32406ff98d68b843ba73b3aa95cfc27154223244f3a6ca5", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2be6954916fdfe4756af3239fb6b6d75d0b8063b5df03ba76fd8a4c87849e180"},
-  "tidewave": {:hex, :tidewave, "0.3.0", "a3b2bf9f6b1aa371b3ea606ef84f7be930aef7bd72e08248cbee6b07f0f174c7", [:mix], [{:circular_buffer, "~> 0.4 or ~> 1.0", [hex: :circular_buffer, repo: "hexpm", optional: false]}, {:igniter, "~> 0.6", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:plug, "~> 1.17", [hex: :plug, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}], "hexpm", "e291537458993f26400f4b82c3d9ceeaac615df099ffdd5caedb2a23e364b972"},
+  "tidewave": {:hex, :tidewave, "0.3.2", "598c7d7ffd5efaff07df97e67052824d0a320951f239efc68fc8b6ce8ba66b5d", [:mix], [{:circular_buffer, "~> 0.4 or ~> 1.0", [hex: :circular_buffer, repo: "hexpm", optional: false]}, {:igniter, "~> 0.6", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:plug, "~> 1.17", [hex: :plug, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}], "hexpm", "e80ec8418a5ae7c46f89797da6e2de26f84868a6cd5ffb793971596cf44462ca"},
   "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"},
   "websock_adapter": {:hex, :websock_adapter, "0.5.7", "65fa74042530064ef0570b75b43f5c49bb8b235d6515671b3d250022cb8a1f9e", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "d0f478ee64deddfec64b800673fd6e0c8888b079d9f3444dd96d2a98383bdbd1"},
   "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"},