diff --git a/lib/livebook/file_system/git.ex b/lib/livebook/file_system/git.ex new file mode 100644 index 000000000..e17bd3eb9 --- /dev/null +++ b/lib/livebook/file_system/git.ex @@ -0,0 +1,181 @@ +defmodule Livebook.FileSystem.Git do + use Ecto.Schema + import Ecto.Changeset + + alias Livebook.FileSystem + + # File system backed by an Git repository. + + @type t :: %__MODULE__{ + id: String.t(), + repo_url: String.t(), + external_id: String.t() | nil, + hub_id: String.t() + } + + embedded_schema do + field :repo_url, :string + field :external_id, :string + field :hub_id, :string + end + + @doc """ + Returns an `%Ecto.Changeset{}` for tracking file system changes. + """ + @spec change_file_system(t(), map()) :: Ecto.Changeset.t() + def change_file_system(git, attrs \\ %{}) do + changeset(git, attrs) + end + + defp changeset(git, attrs) do + git + |> cast(attrs, [:repo_url, :external_id, :hub_id]) + |> validate_format(:repo_url, ~r/^git@[\w\.\-]+:[\w\.\-]+\/[\w\.\-]+\.git$/, + message: "must be a valid repo URL" + ) + |> validate_required([:repo_url, :hub_id]) + |> put_id() + end + + defp put_id(changeset) do + hub_id = get_field(changeset, :hub_id) + repo_url = get_field(changeset, :repo_url) + + if get_field(changeset, :id) do + changeset + else + put_change(changeset, :id, FileSystem.Utils.id("git", hub_id, repo_url)) + end + end +end + +defimpl Livebook.FileSystem, for: Livebook.FileSystem.Git do + alias Livebook.FileSystem + alias Livebook.FileSystem.Git + + def type(_file_system) do + :global + end + + def default_path(_file_system) do + "/" + end + + def list(file_system, path, _recursive) do + FileSystem.Utils.assert_dir_path!(path) + "/" <> path = path + + with {:ok, keys} <- Git.Client.list_files(file_system, path) do + if keys == [] do + FileSystem.Utils.posix_error(:enoent) + else + {:ok, Enum.map(keys, &("/" <> &1))} + end + end + end + + def read(file_system, path) do + FileSystem.Utils.assert_regular_path!(path) + "/" <> path = path + + Git.Client.read_file(file_system, path) + end + + def write(_file_system, path, _content) do + FileSystem.Utils.assert_regular_path!(path) + :ok + end + + def access(_file_system, _path) do + {:ok, :read} + end + + def create_dir(_file_system, path) do + FileSystem.Utils.assert_dir_path!(path) + :ok + end + + def remove(_file_system, _path) do + :ok + end + + def copy(_file_system, _source_path, _destination_path) do + :ok + end + + def rename(_file_system, _source_path, _destination_path) do + :ok + end + + def etag_for(file_system, path) do + FileSystem.Utils.assert_regular_path!(path) + "/" <> path = path + + Git.Client.etag(file_system, path) + end + + def exists?(file_system, path) do + "/" <> path = path + + with {:ok, files} <- Git.Client.list_files(file_system, path) do + {:ok, files != []} + end + end + + def resolve_path(_file_system, dir_path, subject) do + FileSystem.Utils.resolve_unix_like_path(dir_path, subject) + end + + def write_stream_init(_file_system, path, _opts) do + FileSystem.Utils.assert_regular_path!(path) + "/" <> path = path + + {:ok, %{path: path}} + end + + def write_stream_chunk(_file_system, state, chunk) when is_binary(chunk) do + {:ok, state} + end + + def write_stream_finish(_file_system, _state) do + :ok + end + + def write_stream_halt(_file_system, _state) do + :ok + end + + def read_stream_into(_file_system, path, _collectable) do + FileSystem.Utils.assert_regular_path!(path) + :ok + end + + def load(file_system, %{"repo_url" => _} = fields) do + load(file_system, %{ + repo_url: fields["repo_url"], + external_id: fields["external_id"], + id: fields["id"], + hub_id: fields["hub_id"] + }) + end + + def load(file_system, fields) do + %{ + file_system + | id: fields.id, + repo_url: fields.repo_url, + external_id: fields.external_id, + hub_id: fields.hub_id + } + end + + def dump(file_system) do + file_system + |> Map.from_struct() + |> Map.take([:id, :repo_url, :hub_id, :external_id]) + end + + def external_metadata(file_system) do + %{name: file_system.repo_url, error_field: "repo_url"} + end +end diff --git a/lib/livebook/file_system/git/client.ex b/lib/livebook/file_system/git/client.ex new file mode 100644 index 000000000..fba8444ba --- /dev/null +++ b/lib/livebook/file_system/git/client.ex @@ -0,0 +1,98 @@ +defmodule Livebook.FileSystem.Git.Client do + alias Livebook.FileSystem + + @doc """ + Sends a request to the repository to get list of files. + """ + @spec list_files(FileSystem.Git.t(), String.t()) :: + {:ok, list(String.t())} | {:error, FileSystem.error()} + def list_files(file_system, path) + + def list_files(%FileSystem.Git{} = file_system, "/") do + git_dir = git_dir(file_system) + ls_tree(git_dir, "HEAD") + end + + def list_files(%FileSystem.Git{} = file_system, path) do + git_dir = git_dir(file_system) + ls_tree(git_dir, "HEAD", [path]) + end + + @doc """ + Sends a request to the repository to read a file. + """ + @spec read_file(FileSystem.Git.t(), String.t()) :: + {:ok, String.t()} | {:error, FileSystem.error()} + def read_file(%FileSystem.Git{} = file_system, path) do + git_dir = git_dir(file_system) + show(git_dir, "HEAD", path) + end + + @doc """ + Sends a request to the repository to read file ETag. + """ + @spec etag(FileSystem.Git.t(), String.t()) :: {:ok, String.t()} | {:error, FileSystem.error()} + def etag(%FileSystem.Git{} = file_system, path) do + git_dir = git_dir(file_system) + rev_parse(git_dir, "HEAD", path) + end + + defp ls_tree(git_dir, branch, args \\ []) do + with {:ok, result} <- git(git_dir, ["ls-tree", "--name-only", branch] ++ args) do + {:ok, String.split(result, "\n", trim: true)} + end + end + + defp show(git_dir, branch, path) do + git(git_dir, ["show", "#{branch}:#{path}"]) + end + + defp rev_parse(git_dir, branch, path) do + with {:ok, etag} <- git(git_dir, ["rev-parse", "#{branch}:#{path}"]) do + {:ok, String.trim(etag)} + end + end + + defp git(git_dir, args) do + if git = System.find_executable("git") do + case System.cmd(git, args, cmd_opts(git_dir)) do + {result, 0} -> {:ok, result} + {error, _} -> {:error, String.trim(error)} + end + else + {:error, "'git' executable not found"} + end + end + + @cmd_opts [use_stdio: true, stderr_to_stdout: true] + + defp cmd_opts(git_dir) do + if File.exists?(git_dir) do + Keyword.merge(@cmd_opts, cd: git_dir) + else + @cmd_opts + end + end + + defp git_dir(file_system) do + git_dir = Path.join(System.tmp_dir!(), file_system.id) + + args = [ + "clone", + "--bare", + "--depth=1", + "--single-branch", + file_system.repo_url, + git_dir + ] + + if File.exists?(git_dir) do + git_dir + else + case git(git_dir, args) do + {:ok, _} -> git_dir + {:error, reason} -> raise reason + end + end + end +end diff --git a/lib/livebook/file_system/s3.ex b/lib/livebook/file_system/s3.ex index 6051cc672..3bc91087a 100644 --- a/lib/livebook/file_system/s3.ex +++ b/lib/livebook/file_system/s3.ex @@ -2,6 +2,8 @@ defmodule Livebook.FileSystem.S3 do use Ecto.Schema import Ecto.Changeset + alias Livebook.FileSystem + # File system backed by an S3 bucket. @type t :: %__MODULE__{ @@ -81,34 +83,17 @@ defmodule Livebook.FileSystem.S3 do if get_field(changeset, :id) do changeset else - put_change(changeset, :id, id(hub_id, bucket_url)) + put_change(changeset, :id, FileSystem.Utils.id("s3", hub_id, bucket_url)) end end - def id(_, nil), do: nil - - def id(hub_id, bucket_url) do - if hub_id == nil or hub_id == Livebook.Hubs.Personal.id() do - hashed_id(bucket_url) - else - "#{hub_id}-#{hashed_id(bucket_url)}" - end - end - - defp hashed_id(bucket_url) do - hash = :crypto.hash(:sha256, bucket_url) - encrypted_hash = Base.url_encode64(hash, padding: false) - - "s3-#{encrypted_hash}" - end - @doc """ Retrieves credentials for the given file system. If the credentials are not specified by the file system, they are fetched from environment variables or AWS instance if applicable. """ - @spec credentials(S3.t()) :: S3.credentials() + @spec credentials(t()) :: credentials() def credentials(%__MODULE__{} = file_system) do case {file_system.access_key_id, file_system.secret_access_key} do {nil, nil} -> diff --git a/lib/livebook/file_system/utils.ex b/lib/livebook/file_system/utils.ex index 38635898e..a2ae132f2 100644 --- a/lib/livebook/file_system/utils.ex +++ b/lib/livebook/file_system/utils.ex @@ -104,6 +104,26 @@ defmodule Livebook.FileSystem.Utils do |> Enum.join("/") end + @doc """ + Returns the id based on given hub id and data with given prefix. + """ + def id(prefix, hub_id, data) + + def id(_, _, nil), do: nil + + def id(prefix, hub_id, data) do + if hub_id == nil or hub_id == Livebook.Hubs.Personal.id() do + hashed_id(prefix, data) + else + "#{hub_id}-#{hashed_id(prefix, data)}" + end + end + + defp hashed_id(prefix, data) do + hash = :crypto.hash(:sha256, data) + "#{prefix}-#{Base.url_encode64(hash, padding: false)}" + end + defp remove_in_middle([], _elem), do: [] defp remove_in_middle([head], _elem), do: [head] defp remove_in_middle([head | tail], elem), do: remove_in_middle(tail, elem, [head]) diff --git a/lib/livebook/file_systems.ex b/lib/livebook/file_systems.ex index c0b2bf2e4..58a0492b2 100644 --- a/lib/livebook/file_systems.ex +++ b/lib/livebook/file_systems.ex @@ -17,6 +17,10 @@ defmodule Livebook.FileSystems do FileSystem.S3.change_file_system(file_system, attrs) end + def change_file_system(%FileSystem.Git{} = file_system, attrs) do + FileSystem.Git.change_file_system(file_system, attrs) + end + @doc """ Loads the file system from given type and dumped data. """ @@ -35,6 +39,7 @@ defmodule Livebook.FileSystems do def type_to_module(type) def type_to_module("local"), do: FileSystem.Local def type_to_module("s3"), do: FileSystem.S3 + def type_to_module("git"), do: FileSystem.Git @doc """ Returns a serializable type for corresponding to the given file @@ -44,4 +49,5 @@ defmodule Livebook.FileSystems do def module_to_type(module) def module_to_type(FileSystem.Local), do: "local" def module_to_type(FileSystem.S3), do: "s3" + def module_to_type(FileSystem.Git), do: "git" end diff --git a/lib/livebook/hubs/broadcasts.ex b/lib/livebook/hubs/broadcasts.ex index ad0d4ad5d..59e196049 100644 --- a/lib/livebook/hubs/broadcasts.ex +++ b/lib/livebook/hubs/broadcasts.ex @@ -127,7 +127,7 @@ defmodule Livebook.Hubs.Broadcasts do broadcast(@secrets_topic, {:secret_deleted, secret}) end - @allowed_file_systems [FileSystem.S3] + @allowed_file_systems [FileSystem.S3, FileSystem.Git] @doc """ Broadcasts under `#{@file_systems_topic}` topic when hub received a new file system. diff --git a/lib/livebook/hubs/personal.ex b/lib/livebook/hubs/personal.ex index 4101fb971..81a8c66e7 100644 --- a/lib/livebook/hubs/personal.ex +++ b/lib/livebook/hubs/personal.ex @@ -140,7 +140,10 @@ defmodule Livebook.Hubs.Personal do @spec get_file_systems() :: list(FileSystem.t()) def get_file_systems() do Storage.all(@file_systems_namespace) - |> Enum.sort_by(& &1.bucket_url) + |> Enum.sort_by(fn + %{repo_url: repo_url} -> repo_url + %{bucket_url: bucket_url} -> bucket_url + end) |> Enum.map(&to_file_system/1) end diff --git a/lib/livebook_web/components/file_system_components.ex b/lib/livebook_web/components/file_system_components.ex index cb9fac5a4..421d80312 100644 --- a/lib/livebook_web/components/file_system_components.ex +++ b/lib/livebook_web/components/file_system_components.ex @@ -10,6 +10,7 @@ defmodule LivebookWeb.FileSystemComponents do def file_system_name(FileSystem.Local), do: "Disk" def file_system_name(FileSystem.S3), do: "S3" + def file_system_name(FileSystem.Git), do: "Git" @doc """ Formats the given file system into a descriptive label. @@ -18,6 +19,7 @@ defmodule LivebookWeb.FileSystemComponents do def file_system_label(%FileSystem.Local{}), do: "Disk" def file_system_label(%FileSystem.S3{} = fs), do: fs.bucket_url + def file_system_label(%FileSystem.Git{} = fs), do: fs.repo_url @doc """ Renders an icon representing the given file system. @@ -37,4 +39,10 @@ defmodule LivebookWeb.FileSystemComponents do """ end + + def file_system_icon(%{file_system: %FileSystem.Git{}} = assigns) do + ~H""" + <.remix_icon icon="git-repository-line leading-none" /> + """ + end end diff --git a/mix.lock b/mix.lock index 28c89a1a6..6df0289e4 100644 --- a/mix.lock +++ b/mix.lock @@ -56,7 +56,7 @@ "telemetry_metrics": {:hex, :telemetry_metrics, "1.1.0", "5bd5f3b5637e0abea0426b947e3ce5dd304f8b3bc6617039e2b5a008adc02f8f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e7b79e8ddfde70adb6db8a6623d1778ec66401f366e9a8f5dd0955c56bc8ce67"}, "telemetry_poller": {:hex, :telemetry_poller, "1.1.0", "58fa7c216257291caaf8d05678c8d01bd45f4bdbc1286838a28c4bb62ef32999", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9eb9d9cbfd81cbd7cdd24682f8711b6e2b691289a0de6826e58452f28c103c8f"}, "thousand_island": {:hex, :thousand_island, "1.3.5", "6022b6338f1635b3d32406ff98d68b843ba73b3aa95cfc27154223244f3a6ca5", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2be6954916fdfe4756af3239fb6b6d75d0b8063b5df03ba76fd8a4c87849e180"}, - "tidewave": {:hex, :tidewave, "0.3.0", "a3b2bf9f6b1aa371b3ea606ef84f7be930aef7bd72e08248cbee6b07f0f174c7", [:mix], [{:circular_buffer, "~> 0.4 or ~> 1.0", [hex: :circular_buffer, repo: "hexpm", optional: false]}, {:igniter, "~> 0.6", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:plug, "~> 1.17", [hex: :plug, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}], "hexpm", "e291537458993f26400f4b82c3d9ceeaac615df099ffdd5caedb2a23e364b972"}, + "tidewave": {:hex, :tidewave, "0.3.2", "598c7d7ffd5efaff07df97e67052824d0a320951f239efc68fc8b6ce8ba66b5d", [:mix], [{:circular_buffer, "~> 0.4 or ~> 1.0", [hex: :circular_buffer, repo: "hexpm", optional: false]}, {:igniter, "~> 0.6", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:plug, "~> 1.17", [hex: :plug, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}], "hexpm", "e80ec8418a5ae7c46f89797da6e2de26f84868a6cd5ffb793971596cf44462ca"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, "websock_adapter": {:hex, :websock_adapter, "0.5.7", "65fa74042530064ef0570b75b43f5c49bb8b235d6515671b3d250022cb8a1f9e", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "d0f478ee64deddfec64b800673fd6e0c8888b079d9f3444dd96d2a98383bdbd1"}, "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"},