Introduce LIVEBOOK_HOME and LIVEBOOK_DATA_PATH (#963)

And move the relevant configurations to settings.
This commit is contained in:
José Valim 2022-01-31 11:51:57 +01:00 committed by GitHub
parent 7876887ded
commit efdbf67f59
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 199 additions and 191 deletions

View file

@ -144,19 +144,22 @@ Livebook if said token is supplied as part of the URL.
The following environment variables configure Livebook:
* LIVEBOOK_AUTOSAVE_PATH - sets the directory where notebooks with no file are
saved. Defaults to livebook/notebooks/ under the default user cache location.
You can pass "none" to disable this behaviour.
* LIVEBOOK_COOKIE - sets the cookie for running Livebook in a cluster.
Defaults to a random string that is generated on boot.
* LIVEBOOK_DATA_PATH - the directory to store Livebook configuration.
Defaults to "livebook" under the default user data directory.
* LIVEBOOK_DEFAULT_RUNTIME - sets the runtime type that is used by default
when none is started explicitly for the given notebook. Must be either
"standalone" (Elixir standalone), "mix[:PATH]" (Mix standalone),
"attached:NODE:COOKIE" (Attached node) or "embedded" (Embedded).
Defaults to "standalone".
* LIVEBOOK_HOME - sets the home path for the Livebook instance. This is the
default path used on file selection screens and others. Defaults to the
user's operating system home.
* LIVEBOOK_IP - sets the ip address to start the web application on.
Must be a valid IPv4 or IPv6 address.
@ -168,10 +171,6 @@ The following environment variables configure Livebook:
you also need to set LIVEBOOK_SECRET_KEY_BASE. Defaults to 8080. If set to 0,
a random port will be picked.
* LIVEBOOK_ROOT_PATH - sets the root path to use for file selection. This does
not restrict access to upper directories unless the operating system user is
also restricted.
* LIVEBOOK_SECRET_KEY_BASE - sets a secret key that is used to sign and encrypt
the session and other payloads used by Livebook. Must be at least 64 characters
long and it can be generated by commands such as: 'openssl rand -base64 48'.
@ -231,7 +230,7 @@ Livebook development is sponsored by:
## Continuous Integration
Our CI servers and desktop app for macOS are powered by:
Our CI server and desktop app for macOS are powered by:
<a href="https://www.macstadium.com"><img src="https://user-images.githubusercontent.com/9582/151619816-b7794798-8261-46c8-bb88-d12108e3ff12.png" width="320" /></a>

View file

@ -46,28 +46,19 @@ defmodule Livebook do
config :livebook, :default_runtime, runtime
end
if home = Livebook.Config.writable_dir!("LIVEBOOK_HOME") do
config :livebook, :home, home
end
if data_path = Livebook.Config.writable_dir!("LIVEBOOK_DATA_PATH") do
config :livebook, :data_path, data_path
end
config :livebook,
:cookie,
Livebook.Config.cookie!("LIVEBOOK_COOKIE") ||
Livebook.Config.cookie!("RELEASE_COOKIE") ||
Livebook.Utils.random_cookie()
root_path =
Livebook.Config.root_path!("LIVEBOOK_ROOT_PATH")
|> Livebook.FileSystem.Utils.ensure_dir_path()
local_file_system = Livebook.FileSystem.Local.new(default_path: root_path)
config :livebook, :default_file_systems, [local_file_system]
autosave_path =
if config_env() == :test do
nil
else
Livebook.Config.autosave_path!("LIVEBOOK_AUTOSAVE_PATH")
end
config :livebook, :autosave_path, autosave_path
end
@doc """

View file

@ -6,6 +6,7 @@ defmodule Livebook.Application do
use Application
def start(_type, _args) do
ensure_local_filesystem!()
ensure_distribution!()
validate_hostname_resolution!()
set_cookie()
@ -52,6 +53,15 @@ defmodule Livebook.Application do
:ok
end
defp ensure_local_filesystem!() do
home =
Livebook.Config.home()
|> Livebook.FileSystem.Utils.ensure_dir_path()
local_filesystem = Livebook.FileSystem.Local.new(default_path: home)
:persistent_term.put(:livebook_local_filesystem, local_filesystem)
end
defp ensure_distribution!() do
unless Node.alive?() do
case System.cmd("epmd", ["-daemon"]) do

View file

@ -35,128 +35,60 @@ defmodule Livebook.Config do
end
@doc """
Returns the list of currently available file systems.
Returns the local filesystem.
"""
@spec file_systems() :: list(FileSystem.t())
def file_systems() do
Application.fetch_env!(:livebook, :default_file_systems) ++
Enum.map(storage().all(:filesystem), &storage_to_fs/1)
@spec local_filesystem() :: FileSystem.t()
def local_filesystem do
:persistent_term.get(:livebook_local_filesystem)
end
@doc """
Appends a new file system to the configured ones.
Returns the local filesystem home.
"""
@spec append_file_system(FileSystem.t()) :: list(FileSystem.t())
def append_file_system(%FileSystem.S3{} = file_system) do
attributes =
file_system
|> FileSystem.S3.to_config()
|> Map.to_list()
storage().insert(:filesystem, generate_filesystem_id(), [{:type, "s3"} | attributes])
file_systems()
@spec local_filesystem_home() :: FileSystem.File.t()
def local_filesystem_home do
FileSystem.File.new(local_filesystem())
end
@doc """
Removes the given file system from the configured ones.
Returns the home path.
"""
@spec remove_file_system(FileSystem.t()) :: list(FileSystem.t())
def remove_file_system(file_system) do
storage().all(:filesystem)
|> Enum.find(&(storage_to_fs(&1) == file_system))
|> case do
%{id: id} -> storage().delete(:filesystem, id)
end
file_systems()
@spec home() :: String.t()
def home do
Application.get_env(:livebook, :home) || System.user_home() || File.cwd!()
end
@doc """
Returns the default directory.
Returns the configuration path.
"""
@spec default_dir() :: FileSystem.File.t()
def default_dir() do
[file_system | _] = Livebook.Config.file_systems()
FileSystem.File.new(file_system)
end
@doc """
Returns the directory where notebooks with no file should be persisted.
"""
@spec autosave_path() :: String.t() | nil
def autosave_path() do
Application.fetch_env!(:livebook, :autosave_path)
@spec data_path() :: String.t()
def data_path() do
Application.get_env(:livebook, :data_path) || :filename.basedir(:user_data, "livebook")
end
## Parsing
@doc """
Parses and validates the root path from env.
Parses and validates dir from env.
"""
def root_path!(env) do
if root_path = System.get_env(env) do
root_path!(env, root_path)
else
File.cwd!()
def writable_dir!(env) do
if dir = System.get_env(env) do
writable_dir!(env, dir)
end
end
@doc """
Validates `root_path` within context.
Validates `dir` within context.
"""
def root_path!(context, root_path) do
if File.dir?(root_path) do
Path.expand(root_path)
def writable_dir!(context, dir) do
if writable_dir?(dir) do
Path.expand(dir)
else
IO.warn("ignoring #{context} because it doesn't point to a directory: #{root_path}")
File.cwd!()
abort!("expected #{context} to be a writable directory: #{dir}")
end
end
@doc """
Parses and validates the autosave directory from env.
"""
def autosave_path!(env) do
if path = System.get_env(env) do
autosave_path!(env, path)
else
default_autosave_path!()
end
end
@doc """
Validates `autosave_path` within context.
"""
def autosave_path!(context, path)
def autosave_path!(_context, "none"), do: nil
def autosave_path!(context, path) do
if writable_directory?(path) do
Path.expand(path)
else
IO.warn("ignoring #{context} because it doesn't point to a writable directory: #{path}")
default_autosave_path!()
end
end
defp default_autosave_path!() do
cache_path = :filename.basedir(:user_cache, "livebook")
path =
if writable_directory?(cache_path) do
cache_path
else
System.tmp_dir!() |> Path.expand() |> Path.join("livebook")
end
notebooks_path = Path.join(path, "notebooks")
File.mkdir_p!(notebooks_path)
notebooks_path
end
defp writable_directory?(path) do
defp writable_dir?(path) do
case File.stat(path) do
{:ok, %{type: :directory, access: access}} when access in [:read_write, :write] -> true
_ -> false
@ -326,26 +258,6 @@ defmodule Livebook.Config do
}
end
defp storage() do
Livebook.Storage.current()
end
defp storage_to_fs(%{type: "s3"} = config) do
case FileSystem.S3.from_config(config) do
{:ok, fs} ->
fs
{:error, message} ->
abort!(
~s{unrecognised file system, expected "s3 BUCKET_URL ACCESS_KEY_ID SECRET_ACCESS_KEY", got: #{inspect(message)}}
)
end
end
defp generate_filesystem_id() do
:crypto.strong_rand_bytes(6) |> Base.url_encode64()
end
@doc """
Aborts booting due to a configuration error.
"""

View file

@ -112,7 +112,7 @@ defmodule Livebook.Session do
to `:copy_images_from` when the images are in memory
* `:autosave_path` - a local directory to save notebooks without a file into.
Defaults to `Livebook.Config.autosave_path/1`
Defaults to `Livebook.Settings.autosave_path/0`
"""
@spec start_link(keyword()) :: {:ok, pid} | {:error, any()}
def start_link(opts) do
@ -1229,7 +1229,7 @@ defmodule Livebook.Session do
end
defp default_notebook_file(state) do
if path = state.autosave_path || Livebook.Config.autosave_path() do
if path = state.autosave_path || Livebook.Settings.autosave_path() do
dir = path |> FileSystem.Utils.ensure_dir_path() |> FileSystem.File.local()
notebook_rel_path = default_notebook_path(state)
FileSystem.File.resolve(dir, notebook_rel_path)

69
lib/livebook/settings.ex Normal file
View file

@ -0,0 +1,69 @@
defmodule Livebook.Settings do
# Keeps all Livebook settings that are backed by storage.
@moduledoc false
alias Livebook.FileSystem
@doc """
Returns the autosave path.
TODO: Make this configurable in the UI.
"""
@spec autosave_path() :: String.t() | nil
def autosave_path() do
case storage().fetch_key(:settings, "global", :autosave_path) do
{:ok, value} -> value
:error -> Path.join(Livebook.Config.data_path(), "autosaved")
end
end
@doc """
Returns all known filesystems.
"""
@spec file_systems() :: list(FileSystem.t())
def file_systems() do
[Livebook.Config.local_filesystem() | Enum.map(storage().all(:filesystem), &storage_to_fs/1)]
end
@doc """
Appends a new file system to the configured ones.
TODO: Refactor to receive settings submission parameters.
"""
@spec append_file_system(FileSystem.t()) :: :ok
def append_file_system(%FileSystem.S3{} = file_system) do
attributes =
file_system
|> FileSystem.S3.to_config()
|> Map.to_list()
storage().insert(:filesystem, generate_filesystem_id(), [{:type, "s3"} | attributes])
end
@doc """
Removes the given file system from the configured ones.
TODO: Refactor to receive the filesystem id.
"""
@spec remove_file_system(FileSystem.t()) :: :ok
def remove_file_system(file_system) do
storage().all(:filesystem)
|> Enum.find(&(storage_to_fs(&1) == file_system))
|> then(fn %{id: id} -> storage().delete(:filesystem, id) end)
end
defp storage() do
Livebook.Storage.current()
end
defp storage_to_fs(%{type: "s3"} = config) do
case FileSystem.S3.from_config(config) do
{:ok, fs} -> fs
{:error, message} -> raise ArgumentError, "invalid S3 filesystem: #{message}"
end
end
defp generate_filesystem_id() do
:crypto.strong_rand_bytes(6) |> Base.url_encode64()
end
end

View file

@ -7,11 +7,20 @@ defmodule Livebook.Storage do
@type namespace :: atom()
@type entity_id :: binary()
@type attribute :: atom()
@type value :: binary()
@type value :: binary() | nil
@type timestamp :: non_neg_integer()
@type entity :: %{required(:id) => entity_id(), optional(attribute()) => value()}
@doc """
Returns all values in namespace.
all(:filesystem)
[%{id: "rand-id", type: "s3", bucket_url: "/...", secret: "abc", access_key: "xyz"}]
"""
@callback all(namespace()) :: [entity()]
@doc """
Returns a map identified by `entity_id` in `namespace`.
@ -22,13 +31,13 @@ defmodule Livebook.Storage do
@callback fetch(namespace(), entity_id()) :: {:ok, entity()} | :error
@doc """
Returns all values in namespace.
Returns the value for a given `namespace`-`entity_id`-`attribute`.
all(:filesystem)
[%{id: "rand-id", type: "s3", bucket_url: "/...", secret: "abc", access_key: "xyz"}]
fetch_key(:filesystem, "rand-id", :type)
#=> {:ok, "s3"}
"""
@callback all(namespace()) :: [entity()]
@callback fetch_key(namespace(), entity_id(), attribute()) :: {:ok, value()} | :error
@doc """
Inserts given list of attribute-value paris to a entity belonging to specified namespace.

View file

@ -14,6 +14,21 @@ defmodule Livebook.Storage.Ets do
use GenServer
@impl Livebook.Storage
def all(namespace) do
@table_name
|> :ets.match({{namespace, :"$1"}, :"$2", :"$3", :_})
|> Enum.group_by(
fn [entity_id, _attr, _val] -> entity_id end,
fn [_id, attr, val] -> {attr, val} end
)
|> Enum.map(fn {entity_id, attributes} ->
attributes
|> Map.new()
|> Map.put(:id, entity_id)
end)
end
@impl Livebook.Storage
def fetch(namespace, entity_id) do
@table_name
@ -32,18 +47,13 @@ defmodule Livebook.Storage.Ets do
end
@impl Livebook.Storage
def all(namespace) do
def fetch_key(namespace, entity_id, key) do
@table_name
|> :ets.match({{namespace, :"$1"}, :"$2", :"$3", :_})
|> Enum.group_by(
fn [entity_id, _attr, _val] -> entity_id end,
fn [_id, attr, val] -> {attr, val} end
)
|> Enum.map(fn {entity_id, attributes} ->
attributes
|> Map.new()
|> Map.put(:id, entity_id)
end)
|> :ets.match({{namespace, entity_id}, key, :"$1", :_})
|> case do
[[value]] -> {:ok, value}
[] -> :error
end
end
@impl Livebook.Storage

View file

@ -32,10 +32,9 @@ defmodule LivebookCLI.Server do
## Available options
--autosave-path The directory where notebooks with no file are persisted.
Defaults to livebook/notebooks/ under the default user cache
location. You can pass "none" to disable this behaviour
--cookie Sets a cookie for the app distributed node
--data-path The directory to store Livebook configuration,
defaults to "livebook" under the default user data directory
--default-runtime Sets the runtime type that is used by default when none is started
explicitly for the given notebook, defaults to standalone
Supported options:
@ -43,6 +42,7 @@ defmodule LivebookCLI.Server do
* mix[:PATH] - Mix standalone
* attached:NODE:COOKIE - Attached
* embedded - Embedded
--home The home path for the Livebook instance
--ip The ip address to start the web application on, defaults to 127.0.0.1
Must be a valid IPv4 or IPv6 address
--name Set a name for the app distributed node
@ -50,7 +50,6 @@ defmodule LivebookCLI.Server do
If LIVEBOOK_PASSWORD is set, it takes precedence over token auth
--open Open browser window pointing to the application
-p, --port The port to start the web application on, defaults to 8080
--root-path The root path to use for file selection
--sname Set a short name for the app distributed node
The --help option can be given to print this notice.
@ -167,14 +166,14 @@ defmodule LivebookCLI.Server do
end
@switches [
autosave_path: :string,
data_path: :string,
cookie: :string,
default_runtime: :string,
ip: :string,
name: :string,
open: :boolean,
port: :integer,
root_path: :string,
home: :string,
sname: :string,
token: :boolean
]
@ -214,13 +213,9 @@ defmodule LivebookCLI.Server do
opts_to_config(opts, [{:livebook, LivebookWeb.Endpoint, http: [ip: ip]} | config])
end
defp opts_to_config([{:root_path, root_path} | opts], config) do
root_path =
Livebook.Config.root_path!("--root-path", root_path)
|> Livebook.FileSystem.Utils.ensure_dir_path()
local_file_system = Livebook.FileSystem.Local.new(default_path: root_path)
opts_to_config(opts, [{:livebook, :default_file_systems, [local_file_system]} | config])
defp opts_to_config([{:home, home} | opts], config) do
home = Livebook.Config.writable_dir!("--home", home)
opts_to_config(opts, [{:livebook, :home, home} | config])
end
defp opts_to_config([{:sname, sname} | opts], config) do
@ -243,9 +238,9 @@ defmodule LivebookCLI.Server do
opts_to_config(opts, [{:livebook, :default_runtime, default_runtime} | config])
end
defp opts_to_config([{:autosave_path, path} | opts], config) do
autosave_path = Livebook.Config.autosave_path!("--autosave-path", path)
opts_to_config(opts, [{:livebook, :autosave_path, autosave_path} | config])
defp opts_to_config([{:data_path, path} | opts], config) do
data_path = Livebook.Config.writable_dir!("--data-path", path)
opts_to_config(opts, [{:livebook, :data_path, data_path} | config])
end
defp opts_to_config([_opt | opts], config), do: opts_to_config(opts, config)

View file

@ -40,7 +40,7 @@ defmodule LivebookWeb.FileSelectComponent do
renaming_file: nil,
renamed_name: nil,
error_message: nil,
file_systems: Livebook.Config.file_systems()
file_systems: Livebook.Settings.file_systems()
)}
end

View file

@ -20,7 +20,7 @@ defmodule LivebookWeb.HomeLive do
socket
|> SidebarHelpers.shared_home_handlers()
|> assign(
file: Livebook.Config.default_dir(),
file: Livebook.Config.local_filesystem_home(),
file_info: %{exists: true, access: :read_write},
sessions: sessions,
notebook_infos: notebook_infos,
@ -279,7 +279,7 @@ defmodule LivebookWeb.HomeLive do
def handle_event("open_autosave_directory", %{}, socket) do
file =
Livebook.Config.autosave_path()
Livebook.Settings.autosave_path()
|> FileSystem.Utils.ensure_dir_path()
|> FileSystem.File.local()

View file

@ -16,7 +16,7 @@ defmodule LivebookWeb.HomeLive.SessionListComponent do
sessions = sort_sessions(sessions, socket.assigns.order_by)
show_autosave_note? =
case Livebook.Config.autosave_path() do
case Livebook.Settings.autosave_path() do
nil -> false
path -> File.ls!(path) != []
end

View file

@ -113,9 +113,7 @@ defmodule LivebookWeb.SessionLive.MixStandaloneLive do
end
defp initial_file(_runtime) do
Livebook.Config.file_systems()
|> Enum.find(&is_struct(&1, FileSystem.Local))
|> FileSystem.File.new()
Livebook.Config.local_filesystem_home()
end
defp matching_runtime?(%Runtime.MixStandalone{} = runtime, path) do

View file

@ -155,7 +155,7 @@ defmodule LivebookWeb.SessionLive.PersistenceLive do
@impl true
def handle_event("open_file_select", %{}, socket) do
file = socket.assigns.new_attrs.file || Livebook.Config.default_dir()
file = socket.assigns.new_attrs.file || Livebook.Config.local_filesystem_home()
{:noreply, assign(socket, draft_file: file)}
end

View file

@ -7,7 +7,7 @@ defmodule LivebookWeb.SettingsLive do
@impl true
def mount(_params, _session, socket) do
file_systems = Livebook.Config.file_systems()
file_systems = Livebook.Settings.file_systems()
{:ok,
socket

View file

@ -82,8 +82,8 @@ defmodule LivebookWeb.SettingsLive.AddFileSystemComponent do
case FileSystem.File.list(default_dir) do
{:ok, _} ->
file_systems = Livebook.Config.append_file_system(file_system)
send(self(), {:file_systems_updated, file_systems})
Livebook.Settings.append_file_system(file_system)
send(self(), {:file_systems_updated, Livebook.Settings.file_systems()})
{:noreply, push_patch(socket, to: socket.assigns.return_to)}
{:error, message} ->

View file

@ -26,8 +26,8 @@ defmodule LivebookWeb.SettingsLive.RemoveFileSystemComponent do
@impl true
def handle_event("detach", %{}, socket) do
file_systems = Livebook.Config.remove_file_system(socket.assigns.file_system)
send(self(), {:file_systems_updated, file_systems})
Livebook.Settings.remove_file_system(socket.assigns.file_system)
send(self(), {:file_systems_updated, Livebook.Settings.file_systems()})
{:noreply, push_patch(socket, to: socket.assigns.return_to)}
end
end

View file

@ -1,4 +1,3 @@
set LIVEBOOK_ROOT_PATH=%USERPROFILE%
set RELEASE_MODE=interactive
if not defined RELEASE_COOKIE (
for /f "skip=1" %%X in ('wmic os get localdatetime') do if not defined TIMESTAMP set TIMESTAMP=%%X

View file

@ -1,3 +1,2 @@
export LIVEBOOK_ROOT_PATH=$HOME
export RELEASE_MODE=interactive
export RELEASE_COOKIE="${RELEASE_COOKIE:-$(cat /dev/urandom | env LC_ALL=C tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)}"

View file

@ -1,5 +1,5 @@
defmodule Livebook.Storage.EtsTest do
use ExUnit.Case, async: false
use ExUnit.Case, async: true
alias Livebook.Storage.Ets
@ -36,6 +36,20 @@ defmodule Livebook.Storage.EtsTest do
end
end
describe "fetch_key/3" do
test "reads a given key" do
:ok = Ets.insert(:fetch_key, "test", key1: "val1")
assert Ets.fetch_key(:fetch_key, "test", :key1) == {:ok, "val1"}
assert Ets.fetch_key(:fetch_key, "test", :key2) == :error
end
test "handles nil accordingly" do
assert Ets.fetch_key(:fetch_key, "test_nil", :key1) == :error
:ok = Ets.insert(:fetch_key, "test_nil", key1: nil)
assert Ets.fetch_key(:fetch_key, "test_nil", :key1) == {:ok, nil}
end
end
test "fetch/2" do
:ok = Ets.insert(:fetch, "test", key1: "val1")
@ -66,7 +80,7 @@ defmodule Livebook.Storage.EtsTest do
{:ok, entity1} = Ets.fetch(:all, "test1")
{:ok, entity2} = Ets.fetch(:all, "test2")
assert [^entity1, ^entity2] = Ets.all(:all)
assert [^entity1, ^entity2] = Enum.sort(Ets.all(:all))
end
test "returns an empty list if no entities exist for given namespace" do

View file

@ -6,6 +6,9 @@ Livebook.Runtime.ErlDist.NodeManager.start(
unload_modules_on_termination: false
)
# Disable autosaving
Livebook.Storage.current().insert(:settings, "global", autosave_path: nil)
erl_docs_available? = Code.fetch_docs(:gen_server) != {:error, :chunk_not_found}
exclude = []