diff --git a/lib/livebook/migration.ex b/lib/livebook/migration.ex index a718134ba..0dde131a1 100644 --- a/lib/livebook/migration.ex +++ b/lib/livebook/migration.ex @@ -1,23 +1,14 @@ defmodule Livebook.Migration do use GenServer, restart: :temporary - # We version our storage so we can remove migrations in the future - # by deleting the whole storage when finding too old versions. - # - # We also document tables and IDs used in previous versions, - # as those must be avoided in the future. - # - # ## v1 (Livebook v0.11, Oct 2023) - # - # * Deleted hubs.local-host - # * Migrated secrets to hub_secrets - # * Migrated filesystems to file_systems - # * Migrated settings.global.default_file_system_id to settings.global.default_dir - # * Added :file_system_type to starred/recent files - # + alias Livebook.Storage + + # We version out storage, so that we know which migrations to run + # when someone upgrades. In the future we can also remove migrations + # by deleting the whole storage when the storage version is too old. @migration_version 1 - alias Livebook.Storage + def migration_version(), do: @migration_version def start_link(_opts) do GenServer.start_link(__MODULE__, :ok) @@ -27,23 +18,19 @@ defmodule Livebook.Migration do insert_personal_hub() remove_offline_hub() - # v1 - add_personal_hub_secret_key() - delete_local_host_hub() - move_app_secrets_to_personal_hub() - add_file_system_type_to_notebook_manager_files() + storage_version = + case Storage.fetch_key(:system, "global", :migration_version) do + {:ok, version} -> version + :error -> 0 + end - # TODO: remove on Livebook v0.12 - update_file_systems_to_deterministic_ids() - ensure_new_file_system_attributes() - move_default_file_system_id_to_default_dir() + for version <- (storage_version + 1)..@migration_version//1 do + migration(version) + end Storage.insert(:system, "global", migration_version: @migration_version) - :ignore - end - defp delete_local_host_hub() do - Storage.delete(:hubs, "local-host") + :ignore end defp insert_personal_hub() do @@ -62,17 +49,25 @@ defmodule Livebook.Migration do # be present if the environment variables are set. Consequently, we # always remove it and insert on startup if applicable. - for %{id: "team-" <> _ = id} = attrs <- Storage.all(:hubs), offline_hub?(attrs) do + for %{id: "team-" <> _ = id, offline: true} <- Storage.all(:hubs) do :ok = Storage.delete(:hubs, id) end end - # TODO: use just :offline on Livebook v0.12 - defp offline_hub?(%{offline: true}), do: true - defp offline_hub?(%{user_id: 0, org_id: 0, org_key_id: 0}), do: true - defp offline_hub?(_attrs), do: false + defp migration(1) do + v1_add_personal_hub_secret_key() + v1_delete_local_host_hub() + v1_move_app_secrets_to_personal_hub() + v1_add_file_system_type_to_notebook_manager_files() + v1_remove_old_filesystems() + v1_remove_default_file_system_id() + end - defp move_app_secrets_to_personal_hub() do + defp v1_delete_local_host_hub() do + Storage.delete(:hubs, "local-host") + end + + defp v1_move_app_secrets_to_personal_hub() do for %{name: name, value: value} <- Storage.all(:secrets) do secret = %Livebook.Secrets.Secret{ name: name, @@ -85,87 +80,15 @@ defmodule Livebook.Migration do end end - defp add_personal_hub_secret_key() do + defp v1_add_personal_hub_secret_key() do with :error <- Storage.fetch_key(:hubs, Livebook.Hubs.Personal.id(), :secret_key) do secret_key = Livebook.Hubs.Personal.generate_secret_key() Storage.insert(:hubs, Livebook.Hubs.Personal.id(), secret_key: secret_key) end end - # We changed S3 file system ids, such that they are deterministic - # for the same bucket, rather than random. We take this opportunity - # to rename the scope from :filesystem to :file_systems, which - # conveniently allows for easy check if there's anything to migrate. - # This migration can be removed in the future (at the cost of discarding - # very old file systems (which can be re-added). - # TODO: remove on Livebook v0.12 - defp update_file_systems_to_deterministic_ids() do - case Storage.all(:filesystem) do - [] -> - :ok - - configs -> - id_mapping = - for config <- configs, into: %{} do - old_id = config.id - - # Ensure new file system fields - new_fields = %{ - hub_id: Livebook.Hubs.Personal.id(), - external_id: nil, - region: Livebook.FileSystem.S3.region_from_url(config.bucket_url) - } - - config = Map.merge(new_fields, config) - - # At this point S3 is the only file system we store - file_system = Livebook.FileSystems.load("s3", config) - Livebook.Hubs.Personal.save_file_system(file_system) - Storage.delete(:filesystem, old_id) - {old_id, file_system.id} - end - - # Remap default file system id - with {:ok, default_file_system_id} <- - Storage.fetch_key(:settings, "global", :default_file_system_id), - {:ok, new_id} <- Map.fetch(id_mapping, default_file_system_id) do - Storage.insert(:settings, "global", default_file_system_id: new_id) - end - end - end - - # Note that this is already handled by update_file_systems_to_deterministic_ids/0, - # we add this migration so it also applies to people using Livebook main. - # TODO: remove on Livebook v0.12 - defp ensure_new_file_system_attributes() do - for attrs <- Storage.all(:file_systems) do - new_attrs = %{ - hub_id: Livebook.Hubs.Personal.id(), - external_id: nil, - region: Livebook.FileSystem.S3.region_from_url(attrs.bucket_url) - } - - attrs = Map.merge(new_attrs, attrs) - - Storage.insert(:file_systems, attrs.id, Map.to_list(attrs)) - end - end - - # TODO: remove on Livebook v0.12 - defp move_default_file_system_id_to_default_dir() do - with {:ok, default_file_system_id} <- - Storage.fetch_key(:settings, "global", :default_file_system_id) do - Livebook.Hubs.get_file_systems() - |> Enum.find(&(&1.id == default_file_system_id)) - |> Livebook.FileSystem.File.new() - |> Livebook.Settings.set_default_dir() - - Storage.delete_key(:settings, "global", :default_file_system_id) - end - end - # In the past, not all files had a file_system_type, so we need to detect one from the id. - defp add_file_system_type_to_notebook_manager_files() do + defp v1_add_file_system_type_to_notebook_manager_files() do with {:ok, %{recent_notebooks: _, starred_notebooks: _} = attrs} <- Storage.fetch(:notebook_manager, "global") do attrs = @@ -188,4 +111,20 @@ defmodule Livebook.Migration do Map.put(file, :file_system_type, file_system_type) end + + defp v1_remove_old_filesystems() do + # For a while we had a migration converting this to the new table. + # Now we just delete the old entries for simplicity. If someone + # upgrateds from an old version, they just need to re-add their + # file systems. This has negligible impact. + for id <- Storage.all(:filesystem), do: Storage.delete(:filesystem, id) + end + + defp v1_remove_default_file_system_id() do + # For a while we had a migration converting this to default dir. + # Now we just delete the old setting for simplicity. If someone + # upgrateds from an old version, they just need to set the default + # fiel system again. This has negligible impact. + Storage.delete_key(:settings, "global", :default_file_system_id) + end end diff --git a/lib/livebook/storage.ex b/lib/livebook/storage.ex index 66a65c8ed..7369c3012 100644 --- a/lib/livebook/storage.ex +++ b/lib/livebook/storage.ex @@ -140,12 +140,44 @@ defmodule Livebook.Storage do GenServer.call(__MODULE__, {:delete_key, namespace, entity_id, key}) end - @doc """ - Returns file path where the data is persisted. - """ - @spec config_file_path() :: Path.t() - def config_file_path() do - Path.join([Livebook.Config.data_path(), "livebook_config.ets"]) + defp config_file_path() do + migration_version = Livebook.Migration.migration_version() + Path.join([Livebook.Config.data_path(), "livebook_config.v#{migration_version}.ets"]) + end + + defp config_file_path_for_restore() do + # We look for a file matching the current expected version, or an + # older one that will get migrated. + # + # There may be files with a more recent version in case the user + # downgrated Livebook, and those files we ignore. + + migration_version = Livebook.Migration.migration_version() + dir = Livebook.Config.data_path() + + names = + case File.ls(dir) do + {:ok, names} -> names + {:error, _} -> [] + end + + candidates = + for name <- names, version = file_version(name), version <= migration_version do + %{name: name, version: version} + end + + if candidates != [] do + %{name: name} = Enum.max_by(candidates, & &1.version) + Path.join([dir, name]) + end + end + + defp file_version(name) do + case String.split(name, ".") do + ["livebook_config", "ets"] -> 0 + ["livebook_config", "v" <> version, "ets"] -> String.to_integer(version) + _ -> nil + end end @impl true @@ -197,24 +229,23 @@ defmodule Livebook.Storage do defp table_name(), do: :persistent_term.get(__MODULE__) defp load_or_create_table() do - path = config_file_path() + tab = + if path = config_file_path_for_restore() do + path + |> String.to_charlist() + |> :ets.file2tab() + |> case do + {:ok, tab} -> + Logger.info("Reading storage from #{path}") + tab - path - |> String.to_charlist() - |> :ets.file2tab() - |> case do - {:ok, tab} -> - Logger.info("Reading storage from #{path}") - tab - - {:error, reason} -> - case reason do - {:read_error, {:file_error, _, :enoent}} -> :ok - _ -> Logger.warning("Could not open up #{config_file_path()}: #{inspect(reason)}") + {:error, reason} -> + Logger.warning("Could not open up #{path}: #{inspect(reason)}") + nil end + end - :ets.new(__MODULE__, [:protected, :duplicate_bag]) - end + tab || :ets.new(__MODULE__, [:protected, :duplicate_bag]) end defp delete_keys(table, namespace, entity_id, keys) do