From c5ba8f8f818e651021e288cdfe625700e810d658 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jonatan=20K=C5=82osko?=
Date: Mon, 15 Jul 2024 06:19:04 +0200
Subject: [PATCH] Introduce Fly.io runtime (#2708)
---
.formatter.exs | 2 +-
.github/workflows/test.yml | 36 +-
.gitignore | 3 -
README.md | 5 +-
assets/js/events.js | 3 +
config/config.exs | 1 -
lib/livebook.ex | 13 +-
lib/livebook/application.ex | 79 +-
lib/livebook/config.ex | 33 +-
lib/livebook/epmd.ex | 95 +--
lib/livebook/epmd/node_pool.ex | 10 +-
lib/livebook/fly_api.ex | 274 ++++++
lib/livebook/runtime.ex | 47 +-
lib/livebook/runtime/attached.ex | 95 ++-
lib/livebook/runtime/embedded.ex | 53 +-
lib/livebook/runtime/epmd.ex | 87 ++
lib/livebook/runtime/erl_dist.ex | 31 +-
.../runtime/erl_dist/logger_gl_handler.ex | 4 +-
lib/livebook/runtime/erl_dist/node_manager.ex | 82 +-
.../runtime/erl_dist/runtime_server.ex | 23 +-
lib/livebook/runtime/erl_dist/sink.ex | 33 -
lib/livebook/runtime/evaluator.ex | 11 +-
lib/livebook/runtime/evaluator/io_proxy.ex | 32 +-
lib/livebook/runtime/fly.ex | 467 ++++++++++
.../{elixir_standalone.ex => standalone.ex} | 212 ++---
lib/livebook/session.ex | 229 ++---
lib/livebook/session/data.ex | 293 +++++--
.../components/form_components.ex | 7 +-
lib/livebook_web/live/session_live.ex | 245 +++---
..._live.ex => attached_runtime_component.ex} | 98 +--
.../session_live/elixir_standalone_live.ex | 65 --
..._live.ex => embedded_runtime_component.ex} | 39 +-
.../session_live/fly_runtime_component.ex | 748 ++++++++++++++++
.../session_live/insert_buttons_component.ex | 4 +-
lib/livebook_web/live/session_live/render.ex | 57 +-
.../live/session_live/runtime_component.ex | 66 +-
.../live/session_live/section_component.ex | 6 +-
.../standalone_runtime_component.ex | 41 +
mix.exs | 30 +-
mix.lock | 4 +-
rel/app/env.bat.eex | 8 -
rel/app/env.sh.eex | 5 -
rel/app/vm.args.eex | 3 +-
rel/server/env.bat.eex | 9 +-
rel/server/env.sh.eex | 6 +-
rel/server/overlays/bin/server | 18 +-
rel/server/overlays/bin/start_flame.exs | 2 +
rel/server/overlays/bin/start_runtime.exs | 42 +
rel/server/vm.args.eex | 3 +-
test/livebook/epmd_test.exs | 15 +-
test/livebook/hubs/dockerfile_test.exs | 18 +-
test/livebook/intellisense_test.exs | 3 +-
test/livebook/runtime/attached_test.exs | 5 +-
.../runtime/erl_dist/node_manager_test.exs | 7 +-
.../runtime/erl_dist/runtime_server_test.exs | 4 +-
test/livebook/runtime/fly_test.exs | 93 ++
...tandalone_test.exs => standalone_test.exs} | 14 +-
test/livebook/session/data_test.exs | 797 +++++++++++-------
test/livebook/session_test.exs | 173 ++--
.../controllers/session_controller_test.exs | 6 +-
test/livebook_web/live/session_live_test.exs | 291 ++++++-
test/livebook_web/plugs/proxy_plug_test.exs | 4 +-
test/support/noop_runtime.ex | 20 +-
test/support/session_helpers.ex | 5 +
test/test_helper.exs | 26 +-
65 files changed, 3737 insertions(+), 1503 deletions(-)
create mode 100644 lib/livebook/fly_api.ex
create mode 100644 lib/livebook/runtime/epmd.ex
delete mode 100644 lib/livebook/runtime/erl_dist/sink.ex
create mode 100644 lib/livebook/runtime/fly.ex
rename lib/livebook/runtime/{elixir_standalone.ex => standalone.ex} (58%)
rename lib/livebook_web/live/session_live/{attached_live.ex => attached_runtime_component.ex} (56%)
delete mode 100644 lib/livebook_web/live/session_live/elixir_standalone_live.ex
rename lib/livebook_web/live/session_live/{embedded_live.ex => embedded_runtime_component.ex} (51%)
create mode 100644 lib/livebook_web/live/session_live/fly_runtime_component.ex
create mode 100644 lib/livebook_web/live/session_live/standalone_runtime_component.ex
create mode 100644 rel/server/overlays/bin/start_runtime.exs
create mode 100644 test/livebook/runtime/fly_test.exs
rename test/livebook/runtime/{elixir_standalone_test.exs => standalone_test.exs} (72%)
diff --git a/.formatter.exs b/.formatter.exs
index 05a9ef928..be74b1a8a 100644
--- a/.formatter.exs
+++ b/.formatter.exs
@@ -1,5 +1,5 @@
[
import_deps: [:phoenix, :ecto],
plugins: [Phoenix.LiveView.HTMLFormatter],
- inputs: ["*.{heex,ex,exs}", "{config,lib,test}/**/*.{heex,ex,exs}"]
+ inputs: ["*.{heex,ex,exs}", "{config,lib,test}/**/*.{heex,ex,exs}", "rel/*/overlays/**/*.exs"]
]
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index d77cbda90..a92c058a2 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -10,6 +10,7 @@ jobs:
runs-on: ubuntu-latest
env:
MIX_ENV: test
+ ELIXIR_ERL_OPTIONS: "-epmd_module Elixir.Livebook.EPMD"
steps:
- name: Checkout git repo
uses: actions/checkout@v3
@@ -59,41 +60,6 @@ jobs:
- name: Run assets tests
run: npm test --prefix assets
- epmdless:
- runs-on: ubuntu-latest
- if: github.event_name == 'push'
- env:
- MIX_ENV: test
- LIVEBOOK_EPMDLESS: true
- ELIXIR_ERL_OPTIONS: "-epmd_module Elixir.Livebook.EPMD -start_epmd false -erl_epmd_port 0"
- steps:
- - name: Checkout git repo
- uses: actions/checkout@v3
- - name: Read ./versions
- run: |
- . versions
- echo "elixir=$elixir" >> $GITHUB_ENV
- echo "otp=$otp" >> $GITHUB_ENV
- echo "openssl=$openssl" >> $GITHUB_ENV
- - name: Install Erlang & Elixir
- uses: erlef/setup-beam@v1
- with:
- otp-version: ${{ env.otp }}
- elixir-version: ${{ env.elixir }}
- - name: Cache Mix
- uses: actions/cache@v3
- with:
- path: |
- deps
- _build
- key: ${{ runner.os }}-mix-${{ env.elixir }}-${{ env.otp }}-${{ hashFiles('**/mix.lock') }}
- restore-keys: |
- ${{ runner.os }}-mix-${{ env.elixir }}-${{ env.otp }}-
- - name: Install mix dependencies
- run: mix deps.get
- - name: Run tests
- run: mix test
-
windows:
runs-on: windows-latest
if: github.event_name == 'push'
diff --git a/.gitignore b/.gitignore
index 64bb3abe0..8bdd34dd8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -33,6 +33,3 @@ npm-debug.log
# The built Escript
/livebook
-
-# The priv directory with the EPMD file
-/priv/epmd
diff --git a/README.md b/README.md
index 5cca5fa62..e6ebdd74e 100644
--- a/README.md
+++ b/README.md
@@ -217,12 +217,9 @@ The following environment variables can be used to configure Livebook on boot:
* `LIVEBOOK_DEFAULT_RUNTIME` - sets the runtime type that is used by default
when none is started explicitly for the given notebook. Must be either
- "standalone" (Elixir standalone), "attached:NODE:COOKIE" (Attached node)
+ "standalone" (Standalone), "attached:NODE:COOKIE" (Attached node)
or "embedded" (Embedded). Defaults to "standalone".
- * `LIVEBOOK_EPMDLESS` - if set to "true", it disables the usage of EPMD. This is
- only supported within releases and defaults to true for the Desktop app.
-
* `LIVEBOOK_FIPS` - if set to "true", it enables the FIPS mode on startup.
See more details in [the documentation](https://hexdocs.pm/livebook/fips.html).
diff --git a/assets/js/events.js b/assets/js/events.js
index f3f14f938..53ebfa5e7 100644
--- a/assets/js/events.js
+++ b/assets/js/events.js
@@ -61,6 +61,8 @@ export function registerGlobalEventHandlers() {
});
window.addEventListener("lb:scroll_into_view", (event) => {
+ const options = event.detail || {};
+
// If the element is going to be shown, we want to wait for that
waitUntilVisible(event.target).then(() => {
scrollIntoView(event.target, {
@@ -68,6 +70,7 @@ export function registerGlobalEventHandlers() {
behavior: "smooth",
block: "nearest",
inline: "nearest",
+ ...options,
});
});
});
diff --git a/config/config.exs b/config/config.exs
index 1ad0e4b44..f4b36afab 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -30,7 +30,6 @@ config :livebook,
app_service_url: nil,
authentication: :token,
aws_credentials: false,
- epmdless: false,
feature_flags: [],
force_ssl_host: nil,
learn_notebooks: [],
diff --git a/lib/livebook.ex b/lib/livebook.ex
index 0a3656d74..9b8639df0 100644
--- a/lib/livebook.ex
+++ b/lib/livebook.ex
@@ -149,22 +149,19 @@ defmodule Livebook do
config :livebook, :aws_credentials, true
end
- if Livebook.Config.boolean!("LIVEBOOK_EPMDLESS", false) do
- config :livebook, :epmdless, true
- end
-
config :livebook,
:default_runtime,
Livebook.Config.default_runtime!("LIVEBOOK_DEFAULT_RUNTIME") ||
- Livebook.Runtime.ElixirStandalone.new()
+ Livebook.Runtime.Standalone.new()
- config :livebook, :default_app_runtime, Livebook.Runtime.ElixirStandalone.new()
+ config :livebook, :default_app_runtime, Livebook.Runtime.Standalone.new()
config :livebook,
:runtime_modules,
[
- Livebook.Runtime.ElixirStandalone,
- Livebook.Runtime.Attached
+ Livebook.Runtime.Standalone,
+ Livebook.Runtime.Attached,
+ Livebook.Runtime.Fly
]
if home = Livebook.Config.writable_dir!("LIVEBOOK_HOME") do
diff --git a/lib/livebook/application.ex b/lib/livebook/application.ex
index f5e98df59..046362bef 100644
--- a/lib/livebook/application.ex
+++ b/lib/livebook/application.ex
@@ -7,14 +7,8 @@ defmodule Livebook.Application do
ensure_directories!()
set_local_file_system!()
- if Livebook.Config.epmdless?() do
- validate_epmdless!()
- ensure_distribution!()
- else
- ensure_epmd!()
- ensure_distribution!()
- end
-
+ validate_epmd_module!()
+ start_distribution!()
set_cookie()
children =
@@ -48,6 +42,8 @@ defmodule Livebook.Application do
Livebook.EPMD.NodePool,
# Start the server responsible for associating files with sessions
Livebook.Session.FileGuard,
+ # Start the supervisor dynamically managing runtimes
+ {DynamicSupervisor, name: Livebook.RuntimeSupervisor, strategy: :one_for_one},
# Start the supervisor dynamically managing sessions
{DynamicSupervisor, name: Livebook.SessionSupervisor, strategy: :one_for_one},
# Start the registry for managing unique connections
@@ -124,60 +120,33 @@ defmodule Livebook.Application do
:persistent_term.put(:livebook_local_file_system, local_file_system)
end
- defp validate_epmdless!() do
- with {:ok, [[~c"Elixir.Livebook.EPMD"]]} <- :init.get_argument(:epmd_module),
- {:ok, [[~c"false"]]} <- :init.get_argument(:start_epmd),
- {:ok, [[~c"0"]]} <- :init.get_argument(:erl_epmd_port) do
- :ok
- else
+ defp validate_epmd_module!() do
+ # We use a custom EPMD module. In releases and Escript, we make
+ # sure the necessary erl flags are set. When running from source,
+ # those need to be passed explicitly.
+ case :init.get_argument(:epmd_module) do
+ {:ok, [[~c"Elixir.Livebook.EPMD"]]} ->
+ :ok
+
_ ->
Livebook.Config.abort!("""
- You must specify ELIXIR_ERL_OPTIONS=\"-epmd_module Elixir.Livebook.EPMD -start_epmd false -erl_epmd_port 0\" with LIVEBOOK_EPMDLESS. \
- The epmd module can be found inside #{Application.app_dir(:livebook, "priv/ebin")}.
+ You must set the environment variable ELIXIR_ERL_OPTIONS="-epmd_module Elixir.Livebook.EPMD"
""")
end
end
- defp ensure_epmd!() do
- unless Node.alive?() do
- case System.cmd("epmd", ["-daemon"]) do
- {_, 0} ->
- :ok
+ defp start_distribution!() do
+ node = get_node_name()
- _ ->
- Livebook.Config.abort!("""
- Could not start epmd (Erlang Port Mapper Daemon). Livebook uses epmd to \
- talk to different runtimes. You may have to start epmd explicitly by calling:
+ case Node.start(node, :longnames) do
+ {:ok, _} ->
+ :ok
- epmd -daemon
-
- Or by calling:
-
- elixir --sname test -e "IO.puts node()"
-
- Then you can try booting Livebook again
- """)
- end
+ {:error, reason} ->
+ Livebook.Config.abort!("Could not start distributed node: #{inspect(reason)}")
end
end
- defp ensure_distribution!() do
- unless Node.alive?() do
- node = get_node_name()
-
- case Node.start(node, :longnames) do
- {:ok, _} ->
- :ok
-
- {:error, reason} ->
- Livebook.Config.abort!("Could not start distributed node: #{inspect(reason)}")
- end
- end
- end
-
- import Record
- defrecordp :hostent, Record.extract(:hostent, from_lib: "kernel/include/inet.hrl")
-
defp set_cookie() do
cookie = Application.fetch_env!(:livebook, :cookie)
Node.set_cookie(cookie)
@@ -356,10 +325,10 @@ defmodule Livebook.Application do
})
end
- # We set ELIXIR_ERL_OPTIONS when LIVEBOOK_EPMDLESS is set to true.
- # By design, we don't allow ELIXIR_ERL_OPTIONS to pass through.
- # Use ERL_AFLAGS and ERL_ZFLAGS if you want to configure both
- # Livebook and spawned runtimes.
+ # We set ELIXIR_ERL_OPTIONS to set our custom EPMD module when
+ # running from source. By design, we don't allow ELIXIR_ERL_OPTIONS
+ # to pass through. Use ERL_AFLAGS and ERL_ZFLAGS if you want to
+ # configure both Livebook and spawned runtimes.
defp config_env_var?("ELIXIR_ERL_OPTIONS"), do: true
defp config_env_var?("LIVEBOOK_" <> _), do: true
defp config_env_var?("RELEASE_" <> _), do: true
diff --git a/lib/livebook/config.ex b/lib/livebook/config.ex
index 45d5f7bf0..036db32bb 100644
--- a/lib/livebook/config.ex
+++ b/lib/livebook/config.ex
@@ -60,12 +60,26 @@ defmodule Livebook.Config do
})
def docker_images() do
version = app_version()
- base = if version =~ "dev", do: "latest", else: version
+
+ {version, version_cuda} =
+ if version =~ "dev" do
+ {"edge", "latest"}
+ else
+ {version, version}
+ end
[
- %{tag: base, name: "Livebook", env: []},
- %{tag: "#{base}-cuda11.8", name: "Livebook + CUDA 11.8", env: [{"XLA_TARGET", "cuda118"}]},
- %{tag: "#{base}-cuda12.1", name: "Livebook + CUDA 12.1", env: [{"XLA_TARGET", "cuda120"}]}
+ %{tag: version, name: "Livebook", env: []},
+ %{
+ tag: "#{version_cuda}-cuda11.8",
+ name: "Livebook + CUDA 11.8",
+ env: [{"XLA_TARGET", "cuda118"}]
+ },
+ %{
+ tag: "#{version_cuda}-cuda12.1",
+ name: "Livebook + CUDA 12.1",
+ env: [{"XLA_TARGET", "cuda120"}]
+ }
]
end
@@ -158,7 +172,7 @@ defmodule Livebook.Config do
@spec tmp_path() :: String.t()
def tmp_path() do
tmp_dir = System.tmp_dir!() |> Path.expand()
- Path.join(tmp_dir, "livebook")
+ Path.join([tmp_dir, "livebook", app_version()])
end
@doc """
@@ -353,13 +367,6 @@ defmodule Livebook.Config do
Application.fetch_env!(:livebook, :update_instructions_url)
end
- @doc """
- Returns a boolean if epmdless mode is configured.
- """
- def epmdless? do
- Application.fetch_env!(:livebook, :epmdless)
- end
-
@doc """
Returns the force ssl host if any.
"""
@@ -673,7 +680,7 @@ defmodule Livebook.Config do
nil
"standalone" ->
- Livebook.Runtime.ElixirStandalone.new()
+ Livebook.Runtime.Standalone.new()
"embedded" ->
Livebook.Runtime.Embedded.new()
diff --git a/lib/livebook/epmd.ex b/lib/livebook/epmd.ex
index 4af7f96ea..a25d99958 100644
--- a/lib/livebook/epmd.ex
+++ b/lib/livebook/epmd.ex
@@ -1,16 +1,16 @@
defmodule Livebook.EPMD do
- # A custom EPMD module used to bypass the epmd OS daemon
- # on both Livebook and the runtimes.
- @after_compile __MODULE__
+ # A custom EPMD module used to bypass the epmd OS daemon on Livebook.
+ #
+ # We also use it for the Fly runtime, such that we connect to the
+ # remote node via a local proxy port.
# From Erlang/OTP 23+
@epmd_dist_version 6
- @external_resource "priv/epmd/Elixir.Livebook.EPMD.beam"
@doc """
Gets a random child node name.
"""
- def random_child_node do
+ def random_child_node() do
String.to_atom(Livebook.EPMD.NodePool.get_name())
end
@@ -30,82 +30,51 @@ defmodule Livebook.EPMD do
# Custom EPMD callbacks
- # Custom callback that registers the parent information.
- # We read this information when trying to connect to the parent.
- def start_link() do
- with {:ok, [[node, port]]} <- :init.get_argument(:livebook_parent) do
- [name, host] = :string.split(node, ~c"@")
-
- :persistent_term.put(
- :livebook_parent,
- {name, host, List.to_atom(node), List.to_integer(port)}
- )
- end
-
- :erl_epmd.start_link()
- end
-
# Custom callback to register our current node port.
def register_node(name, port), do: register_node(name, port, :inet)
def register_node(name, port, family) do
:persistent_term.put(:livebook_dist_port, port)
- :erl_epmd.register_node(name, port, family)
+
+ case :erl_epmd.register_node(name, port, family) do
+ {:ok, creation} -> {:ok, creation}
+ {:error, :already_registered} -> {:error, :already_registered}
+ # If registration fails because EPMD is not running, we ignore
+ # that, because we do not rely on EPMD
+ _ -> {:ok, -1}
+ end
end
# Custom callback that accesses the parent information.
def port_please(name, host), do: port_please(name, host, :infinity)
+ def port_please(~c"remote_runtime_" ++ port, _host, _timeout) do
+ # The node name includes the local port proxied to a remote machine
+ port = List.to_integer(port)
+ {:port, port, @epmd_dist_version}
+ end
+
def port_please(name, host, timeout) do
- case livebook_port(name) do
- 0 -> :erl_epmd.port_please(name, host, timeout)
- port -> {:port, port, @epmd_dist_version}
+ :erl_epmd.port_please(name, host, timeout)
+ end
+
+ # Custom callback for resolving remote runtime node domain, such as
+ # Fly .internal, to loopback, because we communicate via a local
+ # proxied port
+ def address_please(~c"remote_runtime_" ++ _, _host, address_family) do
+ case address_family do
+ :inet -> {:ok, {127, 0, 0, 1}}
+ :inet6 -> {:ok, {0, 0, 0, 0, 0, 0, 0, 1}}
end
end
- # If we are running inside a Livebook Runtime,
- # we should be able to reach the parent directly
- # or reach siblings through the parent.
- defp livebook_port(name) do
- case :persistent_term.get(:livebook_parent, nil) do
- {parent_name, parent_host, parent_node, parent_port} ->
- case match_name(name, parent_name) do
- :parent -> parent_port
- :sibling -> sibling_port(parent_node, name, parent_host)
- :none -> 0
- end
-
- _ ->
- 0
- end
- end
-
- defp match_name([x | name], [x | parent_name]), do: match_name(name, parent_name)
- defp match_name([?-, ?- | _name], _parent), do: :sibling
- defp match_name([], []), do: :parent
- defp match_name(_name, _parent), do: :none
-
- defp sibling_port(parent_node, name, host) do
- :gen_server.call(
- {Livebook.EPMD.NodePool, parent_node},
- {:get_port, :erlang.list_to_binary(name ++ [?@] ++ host)},
- 5000
- )
- catch
- _, _ -> 0
+ def address_please(name, host, address_family) do
+ :erl_epmd.address_please(name, host, address_family)
end
# Default EPMD callbacks
+ defdelegate start_link(), to: :erl_epmd
defdelegate listen_port_please(name, host), to: :erl_epmd
defdelegate names(host_name), to: :erl_epmd
- defdelegate address_please(name, host, address_family), to: :erl_epmd
-
- # Store .beam file in priv as well
-
- def __after_compile__(_env, binary) do
- File.mkdir_p!("priv/epmd")
- File.write!("priv/epmd/Elixir.Livebook.EPMD.beam", binary)
- Mix.Project.build_structure()
- end
end
diff --git a/lib/livebook/epmd/node_pool.ex b/lib/livebook/epmd/node_pool.ex
index 38536ca1a..e50e296a1 100644
--- a/lib/livebook/epmd/node_pool.ex
+++ b/lib/livebook/epmd/node_pool.ex
@@ -58,7 +58,7 @@ defmodule Livebook.EPMD.NodePool do
# Server side code
- @impl GenServer
+ @impl true
def init(opts) do
:net_kernel.monitor_nodes(true, node_type: :all)
[name, host] = node() |> Atom.to_string() |> :binary.split("@")
@@ -74,23 +74,21 @@ defmodule Livebook.EPMD.NodePool do
{:ok, state}
end
- @impl GenServer
+ @impl true
def handle_call(:get_name, _, state) do
{name, state} = server_get_name(state)
{:reply, name, put_in(state.active_names[name], 0)}
end
- @impl GenServer
def handle_call({:get_port, name}, _, state) do
{:reply, Map.get(state.active_names, name, 0), state}
end
- @impl GenServer
def handle_call({:update_name, name, port}, _, state) do
{:reply, :ok, server_update_name(name, port, state)}
end
- @impl GenServer
+ @impl true
def handle_info({:nodedown, node, _info}, state) do
case state.buffer_time do
0 -> send(self(), {:release_node, node})
@@ -100,12 +98,10 @@ defmodule Livebook.EPMD.NodePool do
{:noreply, state}
end
- @impl GenServer
def handle_info({:nodeup, _node, _info}, state) do
{:noreply, state}
end
- @impl GenServer
def handle_info({:release_node, node}, state) do
{:noreply, server_release_name(Atom.to_string(node), state)}
end
diff --git a/lib/livebook/fly_api.ex b/lib/livebook/fly_api.ex
new file mode 100644
index 000000000..e2b29175b
--- /dev/null
+++ b/lib/livebook/fly_api.ex
@@ -0,0 +1,274 @@
+defmodule Livebook.FlyAPI do
+ # Calls to the Fly API.
+ #
+ # Note that Fly currently exposes both a REST Machines API [1] and
+ # a more elaborate GraphQL API [2]. The Machines API should be
+ # preferred whenever possible. The Go client [3] serves as a good
+ # reference for various operations.
+ #
+ # [1]: https://fly.io/docs/machines/api
+ # [2]: https://github.com/superfly/fly-go/blob/v0.1.18/schema.graphql
+ # [3]: https://github.com/superfly/fly-go
+
+ # See https://github.com/superfly/fly-go/blob/ea7601fc38ba5e9786155711471646dcb0bf63b8/flaps/flaps_volumes.go#L12
+ @destroyed_volume_states ~w(scheduling_destroy fork_cleanup waiting_for_detach pending_destroy destroying)
+
+ @api_url "https://api.fly.io/graphql"
+ @flaps_url "https://api.machines.dev"
+
+ @type error :: %{message: String.t(), status: pos_integer() | nil}
+
+ @doc """
+ The valid values for CPU kind.
+ """
+ @spec cpu_kinds() :: list(String.t())
+ def cpu_kinds(), do: ~w(shared performance)
+
+ @doc """
+ The valid values for GPU kind.
+ """
+ @spec gpu_kinds() :: list(String.t())
+ def gpu_kinds(), do: ~w(a10 a100-pcie-40gb a100-sxm4-80gb l40s)
+
+ @doc """
+ Fetches information about organizations visible to the given token
+ and also regions data.
+ """
+ @spec get_orgs_and_regions(String.t()) :: {:ok, data} | {:error, error}
+ when data: %{
+ orgs: list(%{name: String.t(), slug: String.t()}),
+ regions: %{name: String.t(), code: String.t()},
+ closest_region: String.t()
+ }
+ def get_orgs_and_regions(token) do
+ query = """
+ query {
+ organizations {
+ nodes {
+ rawSlug
+ name
+ }
+ }
+ platform {
+ requestRegion
+ regions {
+ name
+ code
+ }
+ }
+ }
+ """
+
+ with {:ok, data} <- api_request(token, query) do
+ {:ok,
+ %{
+ orgs: Enum.map(data["organizations"]["nodes"], &parse_org/1),
+ regions: Enum.map(data["platform"]["regions"], &parse_region/1),
+ closest_region: data["platform"]["requestRegion"]
+ }}
+ end
+ end
+
+ defp parse_org(org) do
+ %{name: org["name"], slug: org["rawSlug"]}
+ end
+
+ defp parse_region(region) do
+ %{code: region["code"], name: region["name"]}
+ end
+
+ @doc """
+ Fetches volumes in the given app.
+
+ Note that destroyed volumes are ignored.
+ """
+ @spec get_app_volumes(String.t(), String.t()) :: {:ok, data} | {:error, error}
+ when data:
+ list(%{
+ id: String.t(),
+ name: String.t(),
+ region: String.t(),
+ size_gb: pos_integer()
+ })
+ def get_app_volumes(token, app_name) do
+ with {:ok, data} <- flaps_request(token, "/v1/apps/#{app_name}/volumes") do
+ volumes =
+ for volume <- data,
+ volume["state"] not in @destroyed_volume_states,
+ do: parse_volume(volume)
+
+ {:ok, volumes}
+ end
+ end
+
+ defp parse_volume(volume) do
+ %{
+ id: volume["id"],
+ name: volume["name"],
+ region: volume["region"],
+ size_gb: volume["size_gb"]
+ }
+ end
+
+ @doc """
+ Creates an app under the given organization.
+ """
+ @spec create_app(String.t(), String.t(), String.t()) :: :ok | {:error, error}
+ def create_app(token, app_name, org_slug) do
+ with {:ok, _data} <-
+ flaps_request(token, "/v1/apps",
+ method: :post,
+ json: %{app_name: app_name, org_slug: org_slug}
+ ) do
+ :ok
+ end
+ end
+
+ @doc """
+ Creates a new volume in the given app.
+
+ The `compute` attributes hint the expected machine specs that this
+ volume will be attached to. This helps to ensure that the volume is
+ placed on the right hardware (e.g. GPU-enabled).
+ """
+ @spec create_volume(String.t(), String.t(), String.t(), String.t(), pos_integer(), map()) ::
+ {:ok, data} | {:error, error}
+ when data: %{
+ id: String.t(),
+ name: String.t(),
+ region: String.t(),
+ size_gb: pos_integer()
+ }
+ def create_volume(token, app_name, name, region, size_gb, compute) do
+ with {:ok, data} <-
+ flaps_request(token, "/v1/apps/#{app_name}/volumes",
+ method: :post,
+ json: %{
+ name: name,
+ size_gb: size_gb,
+ region: region,
+ compute: compute
+ }
+ ) do
+ {:ok, parse_volume(data)}
+ end
+ end
+
+ @doc """
+ Deletes the given volume.
+ """
+ @spec delete_volume(String.t(), String.t(), String.t()) :: :ok | {:error, error}
+ def delete_volume(token, app_name, volume_id) do
+ with {:ok, _data} <-
+ flaps_request(token, "/v1/apps/#{app_name}/volumes/#{volume_id}", method: :delete) do
+ :ok
+ end
+ end
+
+ @doc """
+ Creates a new machine in the given app.
+ """
+ @spec create_machine(String.t(), String.t(), String.t(), String.t(), map()) ::
+ {:ok, data} | {:error, error}
+ when data: %{id: String.t(), private_ip: String.t()}
+ def create_machine(token, app_name, name, region, config) do
+ boot_timeout = 30_000
+
+ with {:ok, data} <-
+ flaps_request(token, "/v1/apps/#{app_name}/machines",
+ method: :post,
+ json: %{name: name, region: region, config: config},
+ receive_timeout: boot_timeout
+ ) do
+ {:ok, parse_machine(data)}
+ end
+ end
+
+ defp parse_machine(machine) do
+ %{id: machine["id"], private_ip: machine["private_ip"]}
+ end
+
+ defp flaps_request(token, path, opts \\ []) do
+ opts =
+ [base_url: @flaps_url, url: path, auth: {:bearer, token}]
+ |> Keyword.merge(opts)
+ |> Keyword.merge(test_options())
+
+ case Req.request(opts) do
+ {:ok, %{status: status, body: body}} when status in 200..299 ->
+ {:ok, body}
+
+ {:ok, %{status: status, body: body}} ->
+ message =
+ case body do
+ %{"error" => error} when is_binary(error) ->
+ Livebook.Utils.downcase_first(error)
+
+ _ ->
+ "HTTP status #{status}"
+ end
+
+ {:error, %{message: message, status: status}}
+
+ {:error, exception} ->
+ {:error, %{message: "reason: #{Exception.message(exception)}", status: nil}}
+ end
+ end
+
+ defp api_request(token, query) do
+ opts =
+ [
+ base_url: @api_url,
+ method: :post,
+ auth: {:bearer, token},
+ json: %{query: query}
+ ]
+ |> Keyword.merge(test_options())
+
+ case Req.request(opts) do
+ {:ok, %{status: 200, body: body}} ->
+ case body do
+ %{"errors" => [%{"extensions" => %{"code" => "UNAUTHORIZED"}} | _]} ->
+ {:error, %{message: "could not authorize with the given token", status: 401}}
+
+ %{"errors" => [%{"extensions" => %{"message" => message}} | _]} ->
+ {:error, %{message: Livebook.Utils.downcase_first(message), status: nil}}
+
+ %{"data" => data} ->
+ {:ok, data}
+ end
+
+ {:ok, %{status: status}} ->
+ {:error, %{message: "HTTP status #{status}", status: status}}
+
+ {:error, exception} ->
+ {:error, %{message: "reason: #{Exception.message(exception)}", status: nil}}
+ end
+ end
+
+ # TODO: do not rely on private APIs. Also, ideally we should still
+ # be able to use Req.Test.expect/2
+ if Mix.env() == :test do
+ defp test_options() do
+ case Req.Test.__fetch_plug__(__MODULE__) do
+ :passthrough ->
+ []
+
+ _plug ->
+ [plug: {Req.Test, __MODULE__}]
+ end
+ end
+
+ @doc false
+ def stub(plug) do
+ Req.Test.stub(__MODULE__, plug)
+ end
+
+ @doc false
+ def passthrough() do
+ Req.Test.stub(__MODULE__, :passthrough)
+ end
+ else
+ defp test_options(), do: []
+ end
+end
diff --git a/lib/livebook/runtime.ex b/lib/livebook/runtime.ex
index e1fb8e2bb..c6e128cec 100644
--- a/lib/livebook/runtime.ex
+++ b/lib/livebook/runtime.ex
@@ -785,19 +785,32 @@ defprotocol Livebook.Runtime do
def describe(runtime)
@doc """
- Synchronously initializes the given runtime.
+ Asynchronously initializes the given runtime.
- This function starts the necessary resources and processes.
+ The initialization should take care of starting any OS processes
+ necessary, setting up resources and communication.
+
+ Since the initialization may take time, it should always happen in
+ a separate process. This function should return the `pid` of that
+ process. Once the initialization is finished, the process should
+ send the following message to the caller:
+
+ * `{:runtime_connect_done, pid, {:ok, runtime} | {:error, message}}`
+
+ The `runtime` should be the struct updated with all information
+ necessary for further communication.
+
+ In case the initialization is a particularly involved process, the
+ process may send updates to the caller:
+
+ * `{:runtime_connect_info, pid, info}`
+
+ Where `info` is a few word text describing the current initialization
+ step.
"""
- @spec connect(t()) :: {:ok, t()} | {:error, String.t()}
+ @spec connect(t()) :: pid()
def connect(runtime)
- @doc """
- Checks if the given runtime is in a connected state.
- """
- @spec connected?(t()) :: boolean()
- def connected?(runtime)
-
@doc """
Sets the caller as the runtime owner.
@@ -824,13 +837,15 @@ defprotocol Livebook.Runtime do
Synchronously disconnects the runtime and cleans up the underlying
resources.
"""
- @spec disconnect(t()) :: {:ok, t()}
+ @spec disconnect(t()) :: :ok
def disconnect(runtime)
@doc """
Returns a fresh runtime of the same type with the same configuration.
- Note that the runtime is in a stopped state.
+ This function is expected to only modify the runtime struct, unsetting
+ any information added by `connect/1`. It should not have any side
+ effects.
"""
@spec duplicate(Runtime.t()) :: Runtime.t()
def duplicate(runtime)
@@ -889,6 +904,9 @@ defprotocol Livebook.Runtime do
* `:smart_cell_ref` - a reference of the smart cell which code is
to be evaluated, if applicable
+ * `:disable_dependencies_cache` - disables dependencies cache, so
+ they are fetched and compiled from scratch
+
"""
@spec evaluate_code(t(), atom(), String.t(), locator(), parent_locators(), keyword()) :: :ok
def evaluate_code(runtime, language, code, locator, parent_locators, opts \\ [])
@@ -1076,13 +1094,6 @@ defprotocol Livebook.Runtime do
@spec search_packages(t(), pid(), String.t()) :: reference()
def search_packages(runtime, send_to, search)
- @doc """
- Disables dependencies cache, so they are fetched and compiled from
- scratch.
- """
- @spec disable_dependencies_cache(t()) :: :ok
- def disable_dependencies_cache(runtime)
-
@doc """
Sets the given environment variables.
"""
diff --git a/lib/livebook/runtime/attached.ex b/lib/livebook/runtime/attached.ex
index eac182b71..c8a765cd4 100644
--- a/lib/livebook/runtime/attached.ex
+++ b/lib/livebook/runtime/attached.ex
@@ -1,10 +1,10 @@
defmodule Livebook.Runtime.Attached do
# A runtime backed by an Elixir node managed externally.
#
- # Such node must be already started and available, Livebook doesn't
- # manage its lifetime in any way and only loads/unloads the
- # necessary elements. The node can be an ordinary Elixir runtime,
- # a Mix project shell, a running release or anything else.
+ # Such node must be already started and accessible. Livebook doesn't
+ # manage the node's lifetime in any way and only loads/unloads the
+ # necessary modules and processes. The node can be an ordinary Elixir
+ # runtime, a Mix project shell, a running release or anything else.
defstruct [:node, :cookie, :server_pid]
@@ -22,17 +22,20 @@ defmodule Livebook.Runtime.Attached do
%__MODULE__{node: node, cookie: cookie}
end
- @doc """
- Checks if the given node is available for use and initializes
- it with Livebook-specific modules and processes.
- """
- @spec connect(t()) :: {:ok, t()} | {:error, String.t()}
- def connect(runtime) do
- %{node: node, cookie: cookie} = runtime
+ def __connect__(runtime) do
+ caller = self()
- # We need to append the hostname on connect because
- # net_kernel has not yet started during new/2.
- node = append_hostname(node)
+ {:ok, pid} =
+ DynamicSupervisor.start_child(
+ Livebook.RuntimeSupervisor,
+ {Task, fn -> do_connect(runtime, caller) end}
+ )
+
+ pid
+ end
+
+ defp do_connect(runtime, caller) do
+ %{node: node, cookie: cookie} = runtime
# Set cookie for connecting to this specific node
Node.set_cookie(node, cookie)
@@ -44,7 +47,11 @@ defmodule Livebook.Runtime.Attached do
node_manager_opts: [parent_node: node(), capture_orphan_logs: false]
)
- {:ok, %{runtime | node: node, server_pid: server_pid}}
+ runtime = %{runtime | node: node, server_pid: server_pid}
+ send(caller, {:runtime_connect_done, self(), {:ok, runtime}})
+ else
+ {:error, error} ->
+ send(caller, {:runtime_connect_done, self(), {:error, error}})
end
end
@@ -57,26 +64,45 @@ defmodule Livebook.Runtime.Attached do
end
end
- @elixir_version_requirement Keyword.fetch!(Mix.Project.config(), :elixir)
-
defp check_attached_node_version(node) do
attached_node_version = :erpc.call(node, System, :version, [])
- if Version.match?(attached_node_version, @elixir_version_requirement) do
+ requirement = elixir_version_requirement()
+
+ if Version.match?(attached_node_version, requirement) do
:ok
else
- {:error,
- "the node uses Elixir #{attached_node_version}, but #{@elixir_version_requirement} is required"}
+ {:error, "the node uses Elixir #{attached_node_version}, but #{requirement} is required"}
end
end
- defp append_hostname(node) do
- with :nomatch <- :string.find(Atom.to_string(node), "@"),
- <> <- :string.find(Atom.to_string(:net_kernel.nodename()), "@") do
- :"#{node}#{suffix}"
- else
- _ -> node
- end
+ @elixir_version_requirement Keyword.fetch!(Mix.Project.config(), :elixir)
+
+ @doc """
+ Returns requirement for the attached node Elixir version.
+ """
+ @spec elixir_version_requirement() :: String.t()
+ def elixir_version_requirement() do
+ # We load compiled modules binary into the remote node. Erlang
+ # provides rather good compatibility of the binary format, and
+ # in case loading fails we show an appropriate message. However,
+ # it is more likely that the Elixir core functions used in the
+ # compiled module differ across versions. We assume that such
+ # changes are unlikely within the same minor version, so that's
+ # the requirement we enforce.
+
+ current = Version.parse!(System.version())
+ same_minor = "#{current.major}.#{current.minor}.0"
+
+ # Make sure Livebook does not enforce a higher patch version
+ min_version =
+ if Version.match?(same_minor, @elixir_version_requirement) do
+ same_minor
+ else
+ current
+ end
+
+ "~> " <> min_version
end
end
@@ -85,17 +111,13 @@ defimpl Livebook.Runtime, for: Livebook.Runtime.Attached do
def describe(runtime) do
[
- {"Type", "Attached"},
+ {"Type", "Attached node"},
{"Node name", Atom.to_string(runtime.node)}
]
end
def connect(runtime) do
- Livebook.Runtime.Attached.connect(runtime)
- end
-
- def connected?(runtime) do
- runtime.server_pid != nil
+ Livebook.Runtime.Attached.__connect__(runtime)
end
def take_ownership(runtime, opts \\ []) do
@@ -105,7 +127,8 @@ defimpl Livebook.Runtime, for: Livebook.Runtime.Attached do
def disconnect(runtime) do
RuntimeServer.stop(runtime.server_pid)
- {:ok, %{runtime | server_pid: nil}}
+ Node.disconnect(runtime.node)
+ :ok
end
def duplicate(runtime) do
@@ -181,10 +204,6 @@ defimpl Livebook.Runtime, for: Livebook.Runtime.Attached do
raise "not supported"
end
- def disable_dependencies_cache(runtime) do
- RuntimeServer.disable_dependencies_cache(runtime.server_pid)
- end
-
def put_system_envs(runtime, envs) do
RuntimeServer.put_system_envs(runtime.server_pid, envs)
end
diff --git a/lib/livebook/runtime/embedded.ex b/lib/livebook/runtime/embedded.ex
index 3f24940dc..872824a85 100644
--- a/lib/livebook/runtime/embedded.ex
+++ b/lib/livebook/runtime/embedded.ex
@@ -2,7 +2,13 @@ defmodule Livebook.Runtime.Embedded do
# A runtime backed by the same node Livebook is running in.
#
# This runtime is reserved for specific use cases, where there is
- # no option of starting a separate Elixir runtime.
+ # no option of starting a separate Elixir OS process.
+ #
+ # As we run in the Livebook node, all the necessary modules are in
+ # place, so we just ensure the node manager process is running and
+ # we start a new runtime server. We also disable modules cleanup
+ # on termination, since we don't want to unload any modules from
+ # the current node.
defstruct [:server_pid]
@@ -18,30 +24,26 @@ defmodule Livebook.Runtime.Embedded do
%__MODULE__{}
end
- @doc """
- Initializes new runtime by starting the necessary processes within
- the current node.
- """
- @spec connect(t()) :: {:ok, t()}
- def connect(runtime) do
- # As we run in the Livebook node, all the necessary modules
- # are in place, so we just start the manager process.
- # We make it anonymous, so that multiple embedded runtimes
- # can be started (for different notebooks).
- # We also disable cleanup, as we don't want to unload any
- # modules or revert the configuration (because other runtimes
- # may rely on it). If someone uses embedded runtimes,
- # this cleanup is not particularly important anyway.
- # We tell manager to not override :standard_error,
- # as we already do it for the Livebook application globally
- # (see Livebook.Application.start/2).
+ def __connect__(runtime) do
+ caller = self()
+ {:ok, pid} =
+ DynamicSupervisor.start_child(
+ Livebook.RuntimeSupervisor,
+ {Task, fn -> do_connect(runtime, caller) end}
+ )
+
+ pid
+ end
+
+ defp do_connect(runtime, caller) do
server_pid =
ErlDist.initialize(node(),
node_manager_opts: [unload_modules_on_termination: false]
)
- {:ok, %{runtime | server_pid: server_pid}}
+ runtime = %{runtime | server_pid: server_pid}
+ send(caller, {:runtime_connect_done, self(), {:ok, runtime}})
end
end
@@ -53,11 +55,7 @@ defimpl Livebook.Runtime, for: Livebook.Runtime.Embedded do
end
def connect(runtime) do
- Livebook.Runtime.Embedded.connect(runtime)
- end
-
- def connected?(runtime) do
- runtime.server_pid != nil
+ Livebook.Runtime.Embedded.__connect__(runtime)
end
def take_ownership(runtime, opts \\ []) do
@@ -66,8 +64,7 @@ defimpl Livebook.Runtime, for: Livebook.Runtime.Embedded do
end
def disconnect(runtime) do
- RuntimeServer.stop(runtime.server_pid)
- {:ok, %{runtime | server_pid: nil}}
+ :ok = RuntimeServer.stop(runtime.server_pid)
end
def duplicate(_runtime) do
@@ -147,10 +144,6 @@ defimpl Livebook.Runtime, for: Livebook.Runtime.Embedded do
Livebook.Runtime.Dependencies.search_packages_in_list(packages, send_to, search)
end
- def disable_dependencies_cache(runtime) do
- RuntimeServer.disable_dependencies_cache(runtime.server_pid)
- end
-
def put_system_envs(runtime, envs) do
RuntimeServer.put_system_envs(runtime.server_pid, envs)
end
diff --git a/lib/livebook/runtime/epmd.ex b/lib/livebook/runtime/epmd.ex
new file mode 100644
index 000000000..f5f43b78a
--- /dev/null
+++ b/lib/livebook/runtime/epmd.ex
@@ -0,0 +1,87 @@
+defmodule Livebook.Runtime.EPMD do
+ # A custom EPMD module used to bypass the epmd OS daemon in the
+ # standalone runtime.
+ #
+ # We used to start epmd on application boot, however sometimes it
+ # would fail. In particular, on Windows starting epmd may require
+ # accepting a firewall pop up, and the first boot could still fail.
+ # To avoid this, we use a custom port resolution that does not rely
+ # on the epmd OS daemon running.
+
+ # From Erlang/OTP 23+
+ @epmd_dist_version 6
+
+ @doc """
+ Persists parent information, used when connecting to the parent.
+ """
+ def register_parent(parent_node, parent_port) do
+ [name, host] = parent_node |> Atom.to_charlist() |> :string.split(~c"@")
+ :persistent_term.put(:livebook_parent, {name, host, parent_node, parent_port})
+ end
+
+ @doc """
+ Returns the current distribution port.
+ """
+ def dist_port do
+ :persistent_term.get(:livebook_dist_port)
+ end
+
+ # Custom EPMD callbacks
+
+ # Custom callback to register our current node port.
+ def register_node(name, port), do: register_node(name, port, :inet)
+
+ def register_node(name, port, family) do
+ :persistent_term.put(:livebook_dist_port, port)
+
+ case :erl_epmd.register_node(name, port, family) do
+ {:ok, creation} -> {:ok, creation}
+ {:error, :already_registered} -> {:error, :already_registered}
+ # If registration fails because EPMD is not running, we ignore
+ # that, because we do not rely on EPMD
+ _ -> {:ok, -1}
+ end
+ end
+
+ # Custom callback for resolving parent and sibling node ports.
+ def port_please(name, host), do: port_please(name, host, :infinity)
+
+ def port_please(name, host, timeout) do
+ case livebook_port(name) do
+ 0 -> :erl_epmd.port_please(name, host, timeout)
+ port -> {:port, port, @epmd_dist_version}
+ end
+ end
+
+ defp livebook_port(name) do
+ {parent_name, parent_host, parent_node, parent_port} = :persistent_term.get(:livebook_parent)
+
+ case match_name(name, parent_name) do
+ :parent -> parent_port
+ :sibling -> sibling_port(parent_node, name, parent_host)
+ :none -> 0
+ end
+ end
+
+ defp match_name([x | name], [x | parent_name]), do: match_name(name, parent_name)
+ defp match_name([?-, ?- | _name], _parent), do: :sibling
+ defp match_name([], []), do: :parent
+ defp match_name(_name, _parent), do: :none
+
+ defp sibling_port(parent_node, name, host) do
+ :gen_server.call(
+ {Livebook.EPMD.NodePool, parent_node},
+ {:get_port, :erlang.list_to_binary(name ++ [?@] ++ host)},
+ 5000
+ )
+ catch
+ _, _ -> 0
+ end
+
+ # Default EPMD callbacks
+
+ defdelegate start_link(), to: :erl_epmd
+ defdelegate address_please(name, host, address_family), to: :erl_epmd
+ defdelegate listen_port_please(name, host), to: :erl_epmd
+ defdelegate names(host_name), to: :erl_epmd
+end
diff --git a/lib/livebook/runtime/erl_dist.ex b/lib/livebook/runtime/erl_dist.ex
index 73661894b..0c59bd4bc 100644
--- a/lib/livebook/runtime/erl_dist.ex
+++ b/lib/livebook/runtime/erl_dist.ex
@@ -5,7 +5,7 @@ defmodule Livebook.Runtime.ErlDist do
# To ensure proper isolation between sessions, code evaluation may
# take place in a separate Elixir runtime, which also makes it easy
# to terminate the whole evaluation environment without stopping
- # Livebook. Both `Runtime.ElixirStandalone` and `Runtime.Attached`
+ # Livebook. Both `Runtime.Standalone` and `Runtime.Attached`
# do that and this module contains the shared functionality.
#
# To work with a separate node, we have to inject the necessary
@@ -40,7 +40,6 @@ defmodule Livebook.Runtime.ErlDist do
Livebook.Runtime.ErlDist.EvaluatorSupervisor,
Livebook.Runtime.ErlDist.IOForwardGL,
Livebook.Runtime.ErlDist.LoggerGLHandler,
- Livebook.Runtime.ErlDist.Sink,
Livebook.Runtime.ErlDist.SmartCellGL,
Livebook.Proxy.Adapter,
Livebook.Proxy.Handler
@@ -62,15 +61,22 @@ defmodule Livebook.Runtime.ErlDist do
"""
@spec initialize(node(), keyword()) :: pid()
def initialize(node, opts \\ []) do
- unless modules_loaded?(node) do
- load_required_modules(node)
- end
+ # First, we attempt to communicate with the node manager, in case
+ # there is one running. Otherwise, the node is not initialized,
+ # so we need to initialize it and try again
+ case start_runtime_server(node, opts[:runtime_server_opts] || []) do
+ {:ok, pid} ->
+ pid
- unless node_manager_started?(node) do
- start_node_manager(node, opts[:node_manager_opts] || [])
- end
+ {:error, :down} ->
+ unless modules_loaded?(node) do
+ load_required_modules(node)
+ end
- start_runtime_server(node, opts[:runtime_server_opts] || [])
+ {:ok, _} = start_node_manager(node, opts[:node_manager_opts] || [])
+ {:ok, pid} = start_runtime_server(node, opts[:runtime_server_opts] || [])
+ pid
+ end
end
defp load_required_modules(node) do
@@ -109,13 +115,6 @@ defmodule Livebook.Runtime.ErlDist do
:rpc.call(node, Code, :ensure_loaded?, [Livebook.Runtime.ErlDist.NodeManager])
end
- defp node_manager_started?(node) do
- case :rpc.call(node, Process, :whereis, [Livebook.Runtime.ErlDist.NodeManager]) do
- nil -> false
- _pid -> true
- end
- end
-
@doc """
Unloads the previously loaded Livebook modules from the caller node.
"""
diff --git a/lib/livebook/runtime/erl_dist/logger_gl_handler.ex b/lib/livebook/runtime/erl_dist/logger_gl_handler.ex
index 4c9e16b0f..e314853f0 100644
--- a/lib/livebook/runtime/erl_dist/logger_gl_handler.ex
+++ b/lib/livebook/runtime/erl_dist/logger_gl_handler.ex
@@ -3,7 +3,7 @@ defmodule Livebook.Runtime.ErlDist.LoggerGLHandler do
def log(%{meta: meta} = event, %{formatter: {formatter_module, formatter_config}}) do
message = apply(formatter_module, :format, [event, formatter_config])
- if Livebook.Runtime.ErlDist.NodeManager.known_io_proxy?(meta.gl) do
+ if Livebook.Runtime.Evaluator.IOProxy.io_proxy?(meta.gl) do
async_io(meta.gl, message)
else
send(Livebook.Runtime.ErlDist.NodeManager, {:orphan_log, message})
@@ -11,7 +11,7 @@ defmodule Livebook.Runtime.ErlDist.LoggerGLHandler do
end
def async_io(device, output) when is_pid(device) do
- reply_to = Livebook.Runtime.ErlDist.Sink.pid()
+ reply_to = Livebook.Runtime.ErlDist.NodeManager.sink_pid()
send(device, {:io_request, reply_to, make_ref(), {:put_chars, :unicode, output}})
end
diff --git a/lib/livebook/runtime/erl_dist/node_manager.ex b/lib/livebook/runtime/erl_dist/node_manager.ex
index 39efd1359..110db1009 100644
--- a/lib/livebook/runtime/erl_dist/node_manager.ex
+++ b/lib/livebook/runtime/erl_dist/node_manager.ex
@@ -15,7 +15,6 @@ defmodule Livebook.Runtime.ErlDist.NodeManager do
alias Livebook.Runtime.ErlDist
@name __MODULE__
- @io_proxy_registry_name __MODULE__.IOProxyRegistry
@doc """
Starts the node manager.
@@ -52,21 +51,44 @@ defmodule Livebook.Runtime.ErlDist.NodeManager do
@doc """
Starts a new `Livebook.Runtime.ErlDist.RuntimeServer` for evaluation.
- """
- @spec start_runtime_server(node(), keyword()) :: pid()
- def start_runtime_server(node, opts \\ []) do
- GenServer.call(server(node), {:start_runtime_server, opts})
- end
- @doc false
- def known_io_proxy?(pid) do
- case Registry.keys(@io_proxy_registry_name, pid) do
- [_] -> true
- [] -> false
+ This function fails gracefully when the node manager is not running
+ or is about to terminate. This is why we do not use `GenServer.call/2`.
+
+ To start a runtime server we could check if the node manager is alive
+ and then try to call it, however it could terminate between these
+ operations (if the last runtime server terminated). This race condition
+ could happen when reconnecting to the same runtime node. To avoid
+ this, we combine the check and start into an atomic operation.
+ """
+ @spec start_runtime_server(node(), keyword()) :: {:ok, pid()} | {:error, :down}
+ def start_runtime_server(node, opts \\ []) do
+ if pid = :rpc.call(node, Process, :whereis, [@name]) do
+ ref = Process.monitor(pid)
+ send(pid, {:start_runtime_server, self(), ref, opts})
+
+ receive do
+ {:reply, ^ref, pid} ->
+ Process.demonitor(ref, [:flush])
+ {:ok, pid}
+
+ {:DOWN, ^ref, :process, _, _} ->
+ {:error, :down}
+ end
+ else
+ {:error, :down}
end
end
- defp server(node) when is_atom(node), do: {@name, node}
+ @sink_key {__MODULE__, :sink}
+
+ @doc """
+ Returns a process that ignores all incoming messages.
+ """
+ @spec sink_pid() :: pid()
+ def sink_pid() do
+ :persistent_term.get(@sink_key)
+ end
@impl true
def init(opts) do
@@ -77,13 +99,17 @@ defmodule Livebook.Runtime.ErlDist.NodeManager do
## Initialize the node
+ # Note that we intentionally do not name any processes other than
+ # the manager itself. This way, when the manager terminates, another
+ # one can be started immediately without the possibility of the
+ # linked processes to be still around and cause name conflicts.
+ # This scenario could be the case when reconnecting to the same
+ # runtime node.
+
Process.flag(:trap_exit, true)
{:ok, server_supervisor} = DynamicSupervisor.start_link(strategy: :one_for_one)
- {:ok, io_proxy_registry} =
- Registry.start_link(name: @io_proxy_registry_name, keys: :duplicate)
-
# Register our own standard error IO device that proxies to
# sender's group leader.
original_standard_error = Process.whereis(:standard_error)
@@ -91,7 +117,7 @@ defmodule Livebook.Runtime.ErlDist.NodeManager do
Process.unregister(:standard_error)
Process.register(io_forward_gl_pid, :standard_error)
- {:ok, _pid} = Livebook.Runtime.ErlDist.Sink.start_link()
+ :persistent_term.put(@sink_key, spawn_link(&sink_loop/0))
:logger.add_handler(:livebook_gl_handler, Livebook.Runtime.ErlDist.LoggerGLHandler, %{
formatter: Logger.Formatter.new(),
@@ -131,8 +157,7 @@ defmodule Livebook.Runtime.ErlDist.NodeManager do
original_standard_error: original_standard_error,
parent_node: parent_node,
capture_orphan_logs: capture_orphan_logs,
- tmp_dir: tmp_dir,
- io_proxy_registry: io_proxy_registry
+ tmp_dir: tmp_dir
}}
end
@@ -151,6 +176,8 @@ defmodule Livebook.Runtime.ErlDist.NodeManager do
:logger.remove_handler(:livebook_gl_handler)
+ :persistent_term.erase(@sink_key)
+
if state.unload_modules_on_termination do
ErlDist.unload_required_modules()
end
@@ -193,25 +220,26 @@ defmodule Livebook.Runtime.ErlDist.NodeManager do
{:noreply, state}
end
- def handle_info(_message, state), do: {:noreply, state}
-
- @impl true
- def handle_call({:start_runtime_server, opts}, _from, state) do
+ def handle_info({:start_runtime_server, pid, ref, opts}, state) do
opts =
opts
|> Keyword.put_new(:ebin_path, ebin_path(state.tmp_dir))
|> Keyword.put_new(:tmp_dir, child_tmp_dir(state.tmp_dir))
|> Keyword.put_new(:base_path_env, System.get_env("PATH", ""))
- |> Keyword.put_new(:io_proxy_registry, @io_proxy_registry_name)
{:ok, server_pid} =
DynamicSupervisor.start_child(state.server_supervisor, {ErlDist.RuntimeServer, opts})
Process.monitor(server_pid)
state = update_in(state.runtime_servers, &[server_pid | &1])
- {:reply, server_pid, state}
+
+ send(pid, {:reply, ref, server_pid})
+
+ {:noreply, state}
end
+ def handle_info(_message, state), do: {:noreply, state}
+
defp make_tmp_dir() do
path = Path.join([System.tmp_dir!(), "livebook_runtime", random_long_id()])
@@ -229,4 +257,10 @@ defmodule Livebook.Runtime.ErlDist.NodeManager do
defp random_long_id() do
:crypto.strong_rand_bytes(20) |> Base.encode32(case: :lower)
end
+
+ defp sink_loop() do
+ receive do
+ _ -> sink_loop()
+ end
+ end
end
diff --git a/lib/livebook/runtime/erl_dist/runtime_server.ex b/lib/livebook/runtime/erl_dist/runtime_server.ex
index 2cb358a90..8e1800e40 100644
--- a/lib/livebook/runtime/erl_dist/runtime_server.ex
+++ b/lib/livebook/runtime/erl_dist/runtime_server.ex
@@ -49,9 +49,6 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
to merge new values into when setting environment variables.
Defaults to `System.get_env("PATH", "")`
- * `:io_proxy_registry` - the registry to register IO proxy
- processes in
-
"""
def start_link(opts \\ []) do
GenServer.start_link(__MODULE__, opts)
@@ -269,14 +266,6 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
GenServer.call(pid, {:has_dependencies?, dependencies})
end
- @doc """
- Disables dependencies cache globally.
- """
- @spec disable_dependencies_cache(pid()) :: :ok
- def disable_dependencies_cache(pid) do
- GenServer.cast(pid, :disable_dependencies_cache)
- end
-
@doc """
Sets the given environment variables.
"""
@@ -378,7 +367,6 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
base_env_path:
Keyword.get_lazy(opts, :base_env_path, fn -> System.get_env("PATH", "") end),
ebin_path: Keyword.get(opts, :ebin_path),
- io_proxy_registry: Keyword.get(opts, :io_proxy_registry),
tmp_dir: Keyword.get(opts, :tmp_dir),
mix_install_project_dir: nil
}}
@@ -391,7 +379,7 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
if state.owner do
{:noreply, state}
else
- {:stop, :no_owner, state}
+ {:stop, {:shutdown, :no_owner}, state}
end
end
@@ -656,12 +644,6 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
{:noreply, state}
end
- def handle_cast(:disable_dependencies_cache, state) do
- System.put_env("MIX_INSTALL_FORCE", "true")
-
- {:noreply, state}
- end
-
def handle_cast({:put_system_envs, envs}, state) do
envs
|> Enum.map(fn
@@ -799,8 +781,7 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
object_tracker: state.object_tracker,
client_tracker: state.client_tracker,
ebin_path: state.ebin_path,
- tmp_dir: evaluator_tmp_dir(state),
- io_proxy_registry: state.io_proxy_registry
+ tmp_dir: evaluator_tmp_dir(state)
)
Process.monitor(evaluator.pid)
diff --git a/lib/livebook/runtime/erl_dist/sink.ex b/lib/livebook/runtime/erl_dist/sink.ex
deleted file mode 100644
index d50a853b0..000000000
--- a/lib/livebook/runtime/erl_dist/sink.ex
+++ /dev/null
@@ -1,33 +0,0 @@
-defmodule Livebook.Runtime.ErlDist.Sink do
- # An idle process that ignores all incoming messages.
-
- use GenServer
-
- @name __MODULE__
-
- @doc """
- Starts the process.
- """
- @spec start_link() :: GenServer.on_start()
- def start_link() do
- GenServer.start_link(__MODULE__, {}, name: @name)
- end
-
- @doc """
- Returns pid of the global sink process.
- """
- @spec pid() :: pid()
- def pid() do
- Process.whereis(@name)
- end
-
- @impl true
- def init({}) do
- {:ok, {}}
- end
-
- @impl true
- def handle_info(_message, state) do
- {:noreply, state}
- end
-end
diff --git a/lib/livebook/runtime/evaluator.ex b/lib/livebook/runtime/evaluator.ex
index 6e1a83eca..fd99c3747 100644
--- a/lib/livebook/runtime/evaluator.ex
+++ b/lib/livebook/runtime/evaluator.ex
@@ -88,9 +88,6 @@ defmodule Livebook.Runtime.Evaluator do
* `:tmp_dir` - a temporary directory for arbitrary use during
evaluation
- * `:io_proxy_registry` - the registry to register IO proxy
- processes in
-
"""
@spec start_link(keyword()) :: {:ok, pid(), t()} | {:error, term()}
def start_link(opts \\ []) do
@@ -273,7 +270,6 @@ defmodule Livebook.Runtime.Evaluator do
client_tracker = Keyword.fetch!(opts, :client_tracker)
ebin_path = Keyword.get(opts, :ebin_path)
tmp_dir = Keyword.get(opts, :tmp_dir)
- io_proxy_registry = Keyword.get(opts, :io_proxy_registry)
{:ok, io_proxy} =
Evaluator.IOProxy.start(%{
@@ -283,8 +279,7 @@ defmodule Livebook.Runtime.Evaluator do
object_tracker: object_tracker,
client_tracker: client_tracker,
ebin_path: ebin_path,
- tmp_dir: tmp_dir,
- registry: io_proxy_registry
+ tmp_dir: tmp_dir
})
io_proxy_monitor = Process.monitor(io_proxy)
@@ -430,6 +425,10 @@ defmodule Livebook.Runtime.Evaluator do
set_pdict(context, state.ignored_pdict_keys)
+ if opts[:disable_dependencies_cache] do
+ System.put_env("MIX_INSTALL_FORCE", "true")
+ end
+
start_time = System.monotonic_time()
{eval_result, code_markers} = eval(language, code, context.binding, context.env)
evaluation_time_ms = time_diff_ms(start_time)
diff --git a/lib/livebook/runtime/evaluator/io_proxy.ex b/lib/livebook/runtime/evaluator/io_proxy.ex
index d3b85e8e5..b768b2844 100644
--- a/lib/livebook/runtime/evaluator/io_proxy.ex
+++ b/lib/livebook/runtime/evaluator/io_proxy.ex
@@ -31,8 +31,7 @@ defmodule Livebook.Runtime.Evaluator.IOProxy do
object_tracker: pid(),
client_tracker: pid(),
ebin_path: String.t() | nil,
- tmp_dir: String.t() | nil,
- registry: atom() | nil
+ tmp_dir: String.t() | nil
}) :: GenServer.on_start()
def start(args) do
GenServer.start(__MODULE__, args)
@@ -71,6 +70,28 @@ defmodule Livebook.Runtime.Evaluator.IOProxy do
GenServer.cast(pid, {:tracer_updates, updates})
end
+ @doc """
+ Checks if the given process is a IO proxy.
+
+ The check happens against the process dictionary.
+ """
+ def io_proxy?(pid) do
+ process_get_key(pid, :io_proxy) == true
+ end
+
+ defp process_get_key(pid, key) do
+ try do
+ case Process.info(pid, {:dictionary, key}) do
+ {{:dictionary, ^key}, :undefined} -> nil
+ {{:dictionary, ^key}, value} -> value
+ nil -> nil
+ end
+ rescue
+ # TODO: remove error handler once we require OTP 26.2
+ _ -> Process.info(pid, [:dictionary])[:dictionary][key]
+ end
+ end
+
@impl true
def init(args) do
%{
@@ -80,15 +101,12 @@ defmodule Livebook.Runtime.Evaluator.IOProxy do
object_tracker: object_tracker,
client_tracker: client_tracker,
ebin_path: ebin_path,
- tmp_dir: tmp_dir,
- registry: registry
+ tmp_dir: tmp_dir
} = args
evaluator_monitor = Process.monitor(evaluator)
- if registry do
- Registry.register(registry, nil, nil)
- end
+ Process.put(:io_proxy, true)
{:ok,
%{
diff --git a/lib/livebook/runtime/fly.ex b/lib/livebook/runtime/fly.ex
new file mode 100644
index 000000000..0fc2a1563
--- /dev/null
+++ b/lib/livebook/runtime/fly.ex
@@ -0,0 +1,467 @@
+defmodule Livebook.Runtime.Fly do
+ # A runtime backed by a Fly.io machine managed by Livebook.
+ #
+ # This runtime uses a Livebook-managed Elixir node, similarly to
+ # the Standalone runtime, however it runs on a temporary Fly.io
+ # machine. The machine is configured to automatically shutdown
+ # as soon as the runtime is disconnected.
+ #
+ # Note: this runtime requires `flyctl` executable to be available
+ # in the system.
+ #
+ # ## Communication
+ #
+ # The machine runs the Livebook Docker image and we configure it to
+ # invoke the start_runtime.exs script, by setting LIVEBOOK_RUNTIME.
+ # This environment variable also includes encoded information passed
+ # from the parent. Once the Elixir node starts on the machine, it
+ # waits for the parent to connect and finish the initialization.
+ #
+ # Now, we want to establish a distribution connection from the local
+ # Livebook node to the node on the Fly.io machine. We could reach
+ # the node directly, by requiring the user to set up WireGuard.
+ # However, that would require the user to install WireGuard and go
+ # through a few configuration steps. Instead, we use flyctl proxy
+ # feature and only require flyctl to be installed.
+ #
+ # With flyctl proxy, we proxy a local port to the the distribution
+ # port of the Fly.io node. Then, in our EPMD module (`Livebook.EPMD`)
+ # we special case those nodes in two ways: (1) we infer the
+ # distribution port from the node name; (2) we resolve the node
+ # address to loopback, ignoring its hostname.
+ #
+ # ### Distribution protocol
+ #
+ # Usually, nodes need to be configured to use the same distribution
+ # protocol (`-proto_dist`). We configure the Fly.io node to use IPv6
+ # distribution (`-proto_dist inet6_tcp`). However, depending whether
+ # the local node runs IPv4 or IPv6 distribution, we configure the
+ # flyctl proxy to bind to a IPv4 or IPv6 loopback respectively. The
+ # proxy always communicates with the Fly.io machine over IPv6, as
+ # is the case with all internal networking. Consequently, regardless
+ # of the protocol used by the local node, the remote node perceives
+ # it as IPv6.
+ #
+ # Sidenote, a node using IPv6 distribution may accept connections
+ # from a node using IPv4, depending on the `:kernel` application
+ # configuration `inet_dist_listen_options` -> `ipv6_v6only`, which
+ # has OS-specific value. However, we don't rely on this here.
+
+ defstruct [:config, :node, :server_pid]
+
+ use GenServer, restart: :temporary
+
+ require Logger
+
+ @type t :: %__MODULE__{
+ config: config(),
+ node: node() | nil,
+ server_pid: pid() | nil
+ }
+
+ @type config :: %{
+ token: String.t(),
+ app_name: String.t(),
+ region: String.t(),
+ cpu_kind: String.t(),
+ cpus: pos_integer(),
+ memory_gb: pos_integer(),
+ gpu_kind: String.t() | nil,
+ gpus: pos_integer() | nil,
+ volume_id: String.t() | nil,
+ docker_tag: String.t()
+ }
+
+ @doc """
+ Returns a new runtime instance.
+ """
+ @spec new(config()) :: t()
+ def new(config) do
+ %__MODULE__{config: config}
+ end
+
+ def __connect__(runtime) do
+ {:ok, pid} =
+ DynamicSupervisor.start_child(Livebook.RuntimeSupervisor, {__MODULE__, {runtime, self()}})
+
+ pid
+ end
+
+ @doc false
+ def start_link({runtime, caller}) do
+ GenServer.start_link(__MODULE__, {runtime, caller})
+ end
+
+ @impl true
+ def init({runtime, caller}) do
+ state = %{primary_ref: nil, proxy_port: nil}
+ {:ok, state, {:continue, {:init, runtime, caller}}}
+ end
+
+ @impl true
+ def handle_continue({:init, runtime, caller}, state) do
+ config = runtime.config
+ local_port = get_free_port!()
+ remote_port = 44444
+ node_base = "remote_runtime_#{local_port}"
+
+ runtime_data =
+ %{
+ node_base: node_base,
+ cookie: Node.get_cookie(),
+ dist_port: remote_port
+ }
+ |> :erlang.term_to_binary()
+ |> Base.encode64()
+
+ with {:ok, machine_id, machine_ip} <-
+ with_log(caller, "create machine", fn ->
+ create_machine(config, runtime_data)
+ end),
+ child_node <- :"#{node_base}@#{machine_id}.vm.#{config.app_name}.internal",
+ {:ok, proxy_port} <-
+ with_log(caller, "start proxy", fn ->
+ start_fly_proxy(config.app_name, machine_ip, local_port, remote_port, config.token)
+ end),
+ :ok <-
+ with_log(caller, "connect to node", fn ->
+ connect_loop(child_node, 40, 250)
+ end),
+ {:ok, primary_pid} <- fetch_runtime_info(child_node) do
+ primary_ref = Process.monitor(primary_pid)
+
+ server_pid =
+ with_log(caller, "initialize node", fn ->
+ initialize_node(child_node)
+ end)
+
+ send(primary_pid, :node_initialized)
+
+ runtime = %{runtime | node: child_node, server_pid: server_pid}
+ send(caller, {:runtime_connect_done, self(), {:ok, runtime}})
+
+ {:noreply, %{state | primary_ref: primary_ref, proxy_port: proxy_port}}
+ else
+ {:error, error} ->
+ send(caller, {:runtime_connect_done, self(), {:error, error}})
+
+ {:stop, :shutdown, state}
+ end
+ end
+
+ @impl true
+ def handle_info({:DOWN, ref, :process, _pid, _reason}, state) when ref == state.primary_ref do
+ {:stop, :shutdown, state}
+ end
+
+ def handle_info({port, _message}, state) when state.proxy_port == port do
+ {:noreply, state}
+ end
+
+ defp create_machine(config, runtime_data) do
+ base_image = Enum.find(Livebook.Config.docker_images(), &(&1.tag == config.docker_tag))
+ image = "ghcr.io/livebook-dev/livebook:#{base_image.tag}"
+
+ env =
+ Map.merge(
+ Map.new(base_image.env),
+ %{
+ "LIVEBOOK_RUNTIME" => runtime_data,
+ "ERL_AFLAGS" => "-proto_dist inet6_tcp"
+ }
+ )
+
+ name = "#{config.app_name}-livebook-runtime-#{Livebook.Utils.random_id()}"
+
+ machine_config = %{
+ image: image,
+ guest: %{
+ cpu_kind: config.cpu_kind,
+ cpus: config.cpus,
+ memory_mb: config.memory_gb * 1024,
+ gpu_kind: config.gpu_kind,
+ gpus: config.gpus
+ },
+ mounts: config.volume_id && [%{volume: config.volume_id, path: "/home/livebook"}],
+ auto_destroy: true,
+ restart: %{policy: "no"},
+ env: env
+ }
+
+ case Livebook.FlyAPI.create_machine(
+ config.token,
+ config.app_name,
+ name,
+ config.region,
+ machine_config
+ ) do
+ {:ok, %{id: machine_id, private_ip: machine_ip}} ->
+ {:ok, machine_id, machine_ip}
+
+ {:error, %{message: message}} ->
+ {:error, "could not create machine, reason: #{message}"}
+ end
+ end
+
+ defp connect_loop(_node, 0, _interval) do
+ {:error, "could not establish connection with the node"}
+ end
+
+ defp connect_loop(node, attempts, interval) do
+ if Node.connect(node) do
+ :ok
+ else
+ Process.sleep(interval)
+ connect_loop(node, attempts - 1, interval)
+ end
+ end
+
+ defp get_free_port!() do
+ {:ok, socket} = :gen_tcp.listen(0, active: false, reuseaddr: true)
+ {:ok, port} = :inet.port(socket)
+ :gen_tcp.close(socket)
+ port
+ end
+
+ defp start_fly_proxy(app_name, host, local_port, remote_port, token) do
+ with {:ok, flyctl_path} <- find_fly_executable() do
+ ports = "#{local_port}:#{remote_port}"
+
+ # We want the proxy to accept the same protocol that we are
+ # going to use for distribution
+ bind_addr =
+ if Livebook.Utils.proto_dist() == :inet6_tcp do
+ "[::1]"
+ else
+ "127.0.0.1"
+ end
+
+ args = [
+ "proxy",
+ ports,
+ host,
+ "--app",
+ app_name,
+ "--bind-addr",
+ bind_addr,
+ "--access-token",
+ token,
+ "--watch-stdin"
+ ]
+
+ port =
+ Port.open(
+ {:spawn_executable, flyctl_path},
+ [:binary, :hide, :stderr_to_stdout, args: args]
+ )
+
+ port_ref = Port.monitor(port)
+
+ result =
+ receive do
+ {^port, {:data, "Proxying " <> _}} ->
+ {:ok, port}
+
+ {^port, {:data, "Error: unknown flag: --watch-stdin\n"}} ->
+ {:error,
+ "failed to open fly proxy, because the current version " <>
+ "is missing a required feature. Please update flyctl"}
+
+ {^port, {:data, "Error: " <> error}} ->
+ {:error, "failed to open fly proxy. Error: #{String.trim(error)}"}
+
+ {:DOWN, ^port_ref, :port, _object, reason} ->
+ {:error, "failed to open fly proxy. Process terminated, reason: #{inspect(reason)}"}
+ after
+ 30_000 ->
+ {:error, "failed to open fly proxy. Timed out after 30s"}
+ end
+
+ Port.demonitor(port_ref, [:flush])
+
+ result
+ end
+ end
+
+ defp find_fly_executable() do
+ if path = System.find_executable("flyctl") do
+ {:ok, path}
+ else
+ {:error,
+ "no flyctl executable found in PATH. For installation instructions" <>
+ " refer to https://fly.io/docs/flyctl/install"}
+ end
+ end
+
+ defp fetch_runtime_info(child_node) do
+ # Note: it is Livebook that starts the runtime node, so we know
+ # that the node runs Livebook release of the exact same version
+
+ %{
+ pid: pid,
+ elixir_version: elixir_version
+ } = :erpc.call(child_node, :persistent_term, :get, [:livebook_runtime_info])
+
+ if elixir_version != System.version() do
+ {:error,
+ "the local Elixir version (#{inspect(System.version())}) does not" <>
+ " match the one used by the runtime (#{elixir_version})"}
+ else
+ {:ok, pid}
+ end
+ end
+
+ defp initialize_node(child_node) do
+ init_opts = [
+ runtime_server_opts: [
+ extra_smart_cell_definitions: Livebook.Runtime.Definitions.smart_cell_definitions()
+ ]
+ ]
+
+ Livebook.Runtime.ErlDist.initialize(child_node, init_opts)
+ end
+
+ defp with_log(caller, name, fun) do
+ send(caller, {:runtime_connect_info, self(), name})
+
+ {microseconds, result} = :timer.tc(fun)
+ milliseconds = div(microseconds, 1000)
+
+ case result do
+ {:error, error} ->
+ Logger.debug("[fly runtime] #{name} FAILED in #{milliseconds}ms, error: #{error}")
+
+ _ ->
+ Logger.debug("[fly runtime] #{name} finished in #{milliseconds}ms")
+ end
+
+ result
+ end
+end
+
+defimpl Livebook.Runtime, for: Livebook.Runtime.Fly do
+ alias Livebook.Runtime.ErlDist.RuntimeServer
+
+ def describe(runtime) do
+ [{"Type", "Fly.io machine"}] ++
+ if runtime.node do
+ [{"Node name", Atom.to_string(runtime.node)}]
+ else
+ []
+ end
+ end
+
+ def connect(runtime) do
+ Livebook.Runtime.Fly.__connect__(runtime)
+ end
+
+ def take_ownership(runtime, opts \\ []) do
+ RuntimeServer.attach(runtime.server_pid, self(), opts)
+ Process.monitor(runtime.server_pid)
+ end
+
+ def disconnect(runtime) do
+ :ok = RuntimeServer.stop(runtime.server_pid)
+ end
+
+ def duplicate(runtime) do
+ Livebook.Runtime.Fly.new(runtime.config)
+ end
+
+ def evaluate_code(runtime, language, code, locator, parent_locators, opts \\ []) do
+ RuntimeServer.evaluate_code(
+ runtime.server_pid,
+ language,
+ code,
+ locator,
+ parent_locators,
+ opts
+ )
+ end
+
+ def forget_evaluation(runtime, locator) do
+ RuntimeServer.forget_evaluation(runtime.server_pid, locator)
+ end
+
+ def drop_container(runtime, container_ref) do
+ RuntimeServer.drop_container(runtime.server_pid, container_ref)
+ end
+
+ def handle_intellisense(runtime, send_to, request, parent_locators, node) do
+ RuntimeServer.handle_intellisense(runtime.server_pid, send_to, request, parent_locators, node)
+ end
+
+ def read_file(runtime, path) do
+ RuntimeServer.read_file(runtime.server_pid, path)
+ end
+
+ def transfer_file(runtime, path, file_id, callback) do
+ RuntimeServer.transfer_file(runtime.server_pid, path, file_id, callback)
+ end
+
+ def relabel_file(runtime, file_id, new_file_id) do
+ RuntimeServer.relabel_file(runtime.server_pid, file_id, new_file_id)
+ end
+
+ def revoke_file(runtime, file_id) do
+ RuntimeServer.revoke_file(runtime.server_pid, file_id)
+ end
+
+ def start_smart_cell(runtime, kind, ref, attrs, parent_locators) do
+ RuntimeServer.start_smart_cell(runtime.server_pid, kind, ref, attrs, parent_locators)
+ end
+
+ def set_smart_cell_parent_locators(runtime, ref, parent_locators) do
+ RuntimeServer.set_smart_cell_parent_locators(runtime.server_pid, ref, parent_locators)
+ end
+
+ def stop_smart_cell(runtime, ref) do
+ RuntimeServer.stop_smart_cell(runtime.server_pid, ref)
+ end
+
+ def fixed_dependencies?(_runtime), do: false
+
+ def add_dependencies(_runtime, code, dependencies) do
+ Livebook.Runtime.Dependencies.add_dependencies(code, dependencies)
+ end
+
+ def has_dependencies?(runtime, dependencies) do
+ RuntimeServer.has_dependencies?(runtime.server_pid, dependencies)
+ end
+
+ def snippet_definitions(_runtime) do
+ Livebook.Runtime.Definitions.snippet_definitions()
+ end
+
+ def search_packages(_runtime, send_to, search) do
+ Livebook.Runtime.Dependencies.search_packages_on_hex(send_to, search)
+ end
+
+ def put_system_envs(runtime, envs) do
+ RuntimeServer.put_system_envs(runtime.server_pid, envs)
+ end
+
+ def delete_system_envs(runtime, names) do
+ RuntimeServer.delete_system_envs(runtime.server_pid, names)
+ end
+
+ def restore_transient_state(runtime, transient_state) do
+ RuntimeServer.restore_transient_state(runtime.server_pid, transient_state)
+ end
+
+ def register_clients(runtime, clients) do
+ RuntimeServer.register_clients(runtime.server_pid, clients)
+ end
+
+ def unregister_clients(runtime, client_ids) do
+ RuntimeServer.unregister_clients(runtime.server_pid, client_ids)
+ end
+
+ def fetch_proxy_handler_spec(runtime) do
+ RuntimeServer.fetch_proxy_handler_spec(runtime.server_pid)
+ end
+
+ def disconnect_node(runtime, node) do
+ RuntimeServer.disconnect_node(runtime.server_pid, node)
+ end
+end
diff --git a/lib/livebook/runtime/elixir_standalone.ex b/lib/livebook/runtime/standalone.ex
similarity index 58%
rename from lib/livebook/runtime/elixir_standalone.ex
rename to lib/livebook/runtime/standalone.ex
index f45ba4134..1fa1c7dd3 100644
--- a/lib/livebook/runtime/elixir_standalone.ex
+++ b/lib/livebook/runtime/standalone.ex
@@ -1,4 +1,4 @@
-defmodule Livebook.Runtime.ElixirStandalone do
+defmodule Livebook.Runtime.Standalone do
defstruct [:node, :server_pid]
# A runtime backed by a standalone Elixir node managed by Livebook.
@@ -7,6 +7,26 @@ defmodule Livebook.Runtime.ElixirStandalone do
# Most importantly we have to make sure the started node doesn't
# stay in the system when the session or the entire Livebook
# terminates.
+ #
+ # Note: this runtime requires `elixir` executable to be available in
+ # the system.
+ #
+ # ## Connecting
+ #
+ # Connecting the runtime starts a new Elixir node (a system process).
+ # That child node connects back to the parent and notifies that it
+ # is ready by sending a `:node_started` message. Next, the parent
+ # initializes the child node by loading the necessary modules and
+ # starting processes, in particular the node manager and one runtime
+ # server. Once done, the parent sends a `:node_initialized` message
+ # to the child, and the child starts monitoring the node manager.
+ # Once the node manager terminates, the node shuts down.
+ #
+ # If no process calls `Livebook.Runtime.take_ownership/1` for a
+ # period of time, the node automatically terminates. Whoever takes
+ # the ownership, becomes the owner and as soon as it terminates,
+ # the node shuts down. The node may also be shut down by calling
+ # `Livebook.Runtime.disconnect/1`.
alias Livebook.Utils
@@ -23,20 +43,19 @@ defmodule Livebook.Runtime.ElixirStandalone do
%__MODULE__{}
end
- @doc """
- Starts a new Elixir node (a system process) and initializes it with
- Livebook-specific modules and processes.
+ def __connect__(runtime) do
+ caller = self()
- If no process calls `Runtime.take_ownership/1` for a period of time,
- the node automatically terminates. Whoever takes the ownersihp,
- becomes the owner and as soon as it terminates, the node terminates
- as well. The node may also be terminated by calling `Runtime.disconnect/1`.
+ {:ok, pid} =
+ DynamicSupervisor.start_child(
+ Livebook.RuntimeSupervisor,
+ {Task, fn -> do_connect(runtime, caller) end}
+ )
- Note: to start the node it is required that `elixir` is a recognised
- executable within the system.
- """
- @spec connect(t()) :: {:ok, t()} | {:error, String.t()}
- def connect(runtime) do
+ pid
+ end
+
+ defp do_connect(runtime, caller) do
child_node = Livebook.EPMD.random_child_node()
Utils.temporarily_register(self(), child_node, fn ->
@@ -50,9 +69,10 @@ defmodule Livebook.Runtime.ElixirStandalone do
port = start_elixir_node(elixir_path, child_node),
{:ok, server_pid} <- parent_init_sequence(child_node, port, init_opts) do
runtime = %{runtime | node: child_node, server_pid: server_pid}
- {:ok, runtime}
+ send(caller, {:runtime_connect_done, self(), {:ok, runtime}})
else
- {:error, error} -> {:error, error}
+ {:error, error} ->
+ send(caller, {:runtime_connect_done, self(), {:error, error}})
end
end)
end
@@ -77,31 +97,6 @@ defmodule Livebook.Runtime.ElixirStandalone do
])
end
- # ---
- #
- # Once the new node is spawned we need to establish a connection,
- # initialize it and make sure it correctly reacts to the parent node terminating.
- #
- # The procedure goes as follows:
- #
- # 1. The child sends {:node_initialized, ref} message to the parent
- # to communicate it's ready for initialization.
- #
- # 2. The parent initializes the child node - loads necessary modules,
- # starts the NodeManager process and a single RuntimeServer process.
- #
- # 3. The parent sends {:node_initialized, ref} message back to the child,
- # to communicate successful initialization.
- #
- # 4. The child starts monitoring the NodeManager process and freezes
- # until the NodeManager process terminates. The NodeManager process
- # serves as the leading remote process and represents the node from now on.
- #
- # The nodes either successfully go through this flow or return an error,
- # either if the other node dies or is not responding for too long.
- #
- # ---
-
defp parent_init_sequence(child_node, port, init_opts) do
port_ref = Port.monitor(port)
@@ -131,76 +126,108 @@ defmodule Livebook.Runtime.ElixirStandalone do
loop.(loop)
end
- # Note Windows does not handle escaped quotes and newlines the same way as Unix,
- # so the string cannot have constructs newlines nor strings. That's why we pass
- # the parent node name as ARGV and write the code avoiding newlines.
- #
- # This boot script must be kept in sync with Livebook.EPMD.
- #
- # Also note that we explicitly halt, just in case `System.no_halt(true)` is
- # called within the runtime.
- @child_node_eval_string """
- {:ok, [[node]]} = :init.get_argument(:livebook_current);\
- {:ok, _} = :net_kernel.start(List.to_atom(node), %{name_domain: :longnames});\
- {:ok, [[parent_node, _port]]} = :init.get_argument(:livebook_parent);\
- dist_port = :persistent_term.get(:livebook_dist_port, 0);\
- init_ref = make_ref();\
- parent_process = {node(), List.to_atom(parent_node)};\
- send(parent_process, {:node_started, init_ref, node(), dist_port, self()});\
- receive do {:node_initialized, ^init_ref} ->\
- manager_ref = Process.monitor(Livebook.Runtime.ErlDist.NodeManager);\
- receive do {:DOWN, ^manager_ref, :process, _object, _reason} -> :ok end;\
- after 10_000 ->\
- :timeout;\
- end;\
- System.halt()\
- """
+ defp child_node_eval_string(node, parent_node, parent_port) do
+ # We pass the child node code as --eval argument. Windows handles
+ # escaped quotes and newlines differently from Unix, so to avoid
+ # those kind of issues, we encode the string in base 64 and pass
+ # as positional argument. Then, we use a simple --eval that decodes
+ # and evaluates the string.
- if @child_node_eval_string =~ "\n" do
- raise "invalid @child_node_eval_string, newline found: #{inspect(@child_node_eval_string)}"
+ quote do
+ node = unquote(node)
+ parent_node = unquote(parent_node)
+ parent_port = unquote(parent_port)
+
+ # We start distribution here, rather than on node boot, so that
+ # -pa takes effect and Livebook.EPMD is available
+ {:ok, _} = :net_kernel.start(node, %{name_domain: :longnames})
+ Livebook.Runtime.EPMD.register_parent(parent_node, parent_port)
+ dist_port = Livebook.Runtime.EPMD.dist_port()
+
+ init_ref = make_ref()
+ parent_process = {node(), parent_node}
+ send(parent_process, {:node_started, init_ref, node(), dist_port, self()})
+
+ receive do
+ {:node_initialized, ^init_ref} ->
+ manager_ref = Process.monitor(Livebook.Runtime.ErlDist.NodeManager)
+
+ receive do
+ {:DOWN, ^manager_ref, :process, _object, _reason} -> :ok
+ end
+ after
+ 10_000 -> :timeout
+ end
+
+ # We explicitly halt at the end, just in case `System.no_halt(true)`
+ # is called within the runtime
+ System.halt()
+ end
+ |> Macro.to_string()
+ |> Base.encode64()
end
defp elixir_flags(node_name) do
parent_name = node()
parent_port = Livebook.EPMD.dist_port()
- epmdless_flags =
- if parent_port != 0 do
- "-epmd_module Elixir.Livebook.EPMD -start_epmd false -erl_epmd_port 0 "
- else
- ""
- end
-
[
"--erl",
- # Minimize schedulers busy wait threshold,
- # so that they go to sleep immediately after evaluation.
- # Increase the default stack for dirty io threads (cuda requires it).
- # Enable ANSI escape codes as we handle them with HTML.
- # Disable stdin, so that the system process never tries to read terminal input.
+ # Note: keep these flags in sync with the remote runtime.
+ #
+ # * minimize schedulers busy wait threshold, so that they go
+ # to sleep immediately after evaluation
+ #
+ # * increase the default stack for dirty IO threads, necessary
+ # for CUDA
+ #
+ # * enable ANSI escape codes as we handle them with HTML
+ #
+ # * disable stdin, so that the system process never tries to
+ # read terminal input
+ #
+ # * specify a custom EPMD module and disable automatic EPMD
+ # startup
+ #
"+sbwt none +sbwtdcpu none +sbwtdio none +sssdio 128 -elixir ansi_enabled true -noinput " <>
- epmdless_flags <>
- "-livebook_parent #{parent_name} #{parent_port} -livebook_current #{node_name}",
- # Add the location of Livebook.EPMD
+ "-epmd_module Elixir.Livebook.Runtime.EPMD",
+ # Add the location of Livebook.Runtime.EPMD
"-pa",
- Application.app_dir(:livebook, "priv/epmd"),
+ epmd_module_path!(),
# Make the node hidden, so it doesn't automatically join the cluster
"--hidden",
# Use the cookie in Livebook
"--cookie",
Atom.to_string(Node.get_cookie()),
"--eval",
- @child_node_eval_string
+ "System.argv() |> hd() |> Base.decode64!() |> Code.eval_string()",
+ child_node_eval_string(node_name, parent_name, parent_port)
]
end
+
+ defp epmd_module_path!() do
+ # We need to make the custom Livebook.Runtime.EPMD module available
+ # before the child node starts distrubtion. We persist the module
+ # into a temporary directory and add to the code paths. Note that
+ # we could persist it to priv/ at build time, however for Escript
+ # priv/ is packaged into the archive, so it is not accessible in
+ # the file system.
+
+ epmd_path = Path.join(Livebook.Config.tmp_path(), "epmd")
+ File.rm_rf!(epmd_path)
+ File.mkdir_p!(epmd_path)
+ {_module, binary, path} = :code.get_object_code(Livebook.Runtime.EPMD)
+ File.write!(Path.join(epmd_path, Path.basename(path)), binary)
+ epmd_path
+ end
end
-defimpl Livebook.Runtime, for: Livebook.Runtime.ElixirStandalone do
+defimpl Livebook.Runtime, for: Livebook.Runtime.Standalone do
alias Livebook.Runtime.ErlDist.RuntimeServer
def describe(runtime) do
- [{"Type", "Elixir standalone"}] ++
- if connected?(runtime) do
+ [{"Type", "Standalone"}] ++
+ if runtime.node do
[{"Node name", Atom.to_string(runtime.node)}]
else
[]
@@ -208,11 +235,7 @@ defimpl Livebook.Runtime, for: Livebook.Runtime.ElixirStandalone do
end
def connect(runtime) do
- Livebook.Runtime.ElixirStandalone.connect(runtime)
- end
-
- def connected?(runtime) do
- runtime.server_pid != nil
+ Livebook.Runtime.Standalone.__connect__(runtime)
end
def take_ownership(runtime, opts \\ []) do
@@ -222,11 +245,10 @@ defimpl Livebook.Runtime, for: Livebook.Runtime.ElixirStandalone do
def disconnect(runtime) do
:ok = RuntimeServer.stop(runtime.server_pid)
- {:ok, %{runtime | node: nil, server_pid: nil}}
end
def duplicate(_runtime) do
- Livebook.Runtime.ElixirStandalone.new()
+ Livebook.Runtime.Standalone.new()
end
def evaluate_code(runtime, language, code, locator, parent_locators, opts \\ []) do
@@ -298,10 +320,6 @@ defimpl Livebook.Runtime, for: Livebook.Runtime.ElixirStandalone do
Livebook.Runtime.Dependencies.search_packages_on_hex(send_to, search)
end
- def disable_dependencies_cache(runtime) do
- RuntimeServer.disable_dependencies_cache(runtime.server_pid)
- end
-
def put_system_envs(runtime, envs) do
RuntimeServer.put_system_envs(runtime.server_pid, envs)
end
diff --git a/lib/livebook/session.ex b/lib/livebook/session.ex
index 36b3e3644..f13215921 100644
--- a/lib/livebook/session.ex
+++ b/lib/livebook/session.ex
@@ -111,6 +111,7 @@ defmodule Livebook.Session do
data: Data.t(),
client_pids_with_id: %{pid() => Data.client_id()},
created_at: DateTime.t(),
+ runtime_connect: %{ref: reference(), pid: pid()} | nil,
runtime_monitor_ref: reference() | nil,
autosave_timer_ref: reference() | nil,
autosave_path: String.t() | nil,
@@ -452,20 +453,12 @@ defmodule Livebook.Session do
GenServer.cast(pid, {:add_dependencies, dependencies})
end
- @doc """
- Sends disable dependencies cache request to the server.
- """
- @spec disable_dependencies_cache(pid()) :: :ok
- def disable_dependencies_cache(pid) do
- GenServer.cast(pid, :disable_dependencies_cache)
- end
-
@doc """
Sends cell evaluation request to the server.
"""
- @spec queue_cell_evaluation(pid(), Cell.id()) :: :ok
- def queue_cell_evaluation(pid, cell_id) do
- GenServer.cast(pid, {:queue_cell_evaluation, self(), cell_id})
+ @spec queue_cell_evaluation(pid(), Cell.id(), keyword()) :: :ok
+ def queue_cell_evaluation(pid, cell_id, evaluation_opts \\ []) do
+ GenServer.cast(pid, {:queue_cell_evaluation, self(), cell_id, evaluation_opts})
end
@doc """
@@ -586,14 +579,22 @@ defmodule Livebook.Session do
@doc """
Sends runtime update to the server.
-
- If the runtime is connected, the session takes the ownership.
"""
@spec set_runtime(pid(), Runtime.t()) :: :ok
def set_runtime(pid, runtime) do
GenServer.cast(pid, {:set_runtime, self(), runtime})
end
+ @doc """
+ Sends request to connect to the configured runtime.
+
+ Once the runtime is connected, the session takes the ownership.
+ """
+ @spec connect_runtime(pid()) :: :ok
+ def connect_runtime(pid) do
+ GenServer.cast(pid, {:connect_runtime, self()})
+ end
+
@doc """
Sends file location update request to the server.
"""
@@ -890,6 +891,7 @@ defmodule Livebook.Session do
data: data,
client_pids_with_id: %{},
created_at: DateTime.utc_now(),
+ runtime_connect: nil,
runtime_monitor_ref: nil,
autosave_timer_ref: nil,
autosave_path: opts[:autosave_path],
@@ -997,7 +999,7 @@ defmodule Livebook.Session do
@impl true
def handle_continue(:app_init, state) do
cell_ids = Data.cell_ids_for_full_evaluation(state.data, [])
- operation = {:queue_cells_evaluation, @client_id, cell_ids}
+ operation = {:queue_cells_evaluation, @client_id, cell_ids, []}
{:noreply, handle_operation(state, operation)}
end
@@ -1031,18 +1033,16 @@ defmodule Livebook.Session do
Notebook.find_asset_info(state.data.notebook, hash) ||
Enum.find_value(state.client_id_with_assets, fn {_client_id, assets} -> assets[hash] end)
- runtime = state.data.runtime
-
reply =
cond do
assets_info == nil ->
{:error, "unknown hash"}
- not Runtime.connected?(runtime) ->
+ state.data.runtime_status != :connected ->
{:error, "runtime not started"}
true ->
- {:ok, runtime, assets_info.archive_path}
+ {:ok, state.data.runtime, assets_info.archive_path}
end
{:reply, reply, state}
@@ -1076,17 +1076,7 @@ defmodule Livebook.Session do
def handle_call({:disconnect_runtime, client_pid}, _from, state) do
client_id = client_id(state, client_pid)
-
- state =
- if Runtime.connected?(state.data.runtime) do
- {:ok, runtime} = Runtime.disconnect(state.data.runtime)
-
- %{state | runtime_monitor_ref: nil}
- |> handle_operation({:set_runtime, client_id, runtime})
- else
- state
- end
-
+ state = handle_operation(state, {:disconnect_runtime, client_id})
{:reply, :ok, state}
end
@@ -1099,7 +1089,7 @@ defmodule Livebook.Session do
end
def handle_call(:fetch_proxy_handler_spec, _from, state) do
- if Runtime.connected?(state.data.runtime) do
+ if state.data.runtime_status == :connected do
{:reply, Runtime.fetch_proxy_handler_spec(state.data.runtime), state}
else
{:reply, {:error, :disconnected}, state}
@@ -1233,17 +1223,9 @@ defmodule Livebook.Session do
{:noreply, do_add_dependencies(state, dependencies)}
end
- def handle_cast(:disable_dependencies_cache, state) do
- if Runtime.connected?(state.data.runtime) do
- Runtime.disable_dependencies_cache(state.data.runtime)
- end
-
- {:noreply, state}
- end
-
- def handle_cast({:queue_cell_evaluation, client_pid, cell_id}, state) do
+ def handle_cast({:queue_cell_evaluation, client_pid, cell_id, evaluation_opts}, state) do
client_id = client_id(state, client_pid)
- operation = {:queue_cells_evaluation, client_id, [cell_id]}
+ operation = {:queue_cells_evaluation, client_id, [cell_id], evaluation_opts}
{:noreply, handle_operation(state, operation)}
end
@@ -1253,7 +1235,7 @@ defmodule Livebook.Session do
case Notebook.fetch_section(state.data.notebook, section_id) do
{:ok, section} ->
cell_ids = for cell <- section.cells, Cell.evaluable?(cell), do: cell.id
- operation = {:queue_cells_evaluation, client_id, cell_ids}
+ operation = {:queue_cells_evaluation, client_id, cell_ids, []}
{:noreply, handle_operation(state, operation)}
:error ->
@@ -1268,7 +1250,7 @@ defmodule Livebook.Session do
for {bound_cell, _} <- Data.bound_cells_with_section(state.data, input_id),
do: bound_cell.id
- operation = {:queue_cells_evaluation, client_id, cell_ids}
+ operation = {:queue_cells_evaluation, client_id, cell_ids, []}
{:noreply, handle_operation(state, operation)}
end
@@ -1277,7 +1259,7 @@ defmodule Livebook.Session do
cell_ids = Data.cell_ids_for_full_evaluation(state.data, forced_cell_ids)
- operation = {:queue_cells_evaluation, client_id, cell_ids}
+ operation = {:queue_cells_evaluation, client_id, cell_ids, []}
{:noreply, handle_operation(state, operation)}
end
@@ -1286,7 +1268,7 @@ defmodule Livebook.Session do
cell_ids = Data.cell_ids_for_reevaluation(state.data)
- operation = {:queue_cells_evaluation, client_id, cell_ids}
+ operation = {:queue_cells_evaluation, client_id, cell_ids, []}
{:noreply, handle_operation(state, operation)}
end
@@ -1343,21 +1325,14 @@ defmodule Livebook.Session do
def handle_cast({:set_runtime, client_pid, runtime}, state) do
client_id = client_id(state, client_pid)
-
- if Runtime.connected?(state.data.runtime) do
- {:ok, _} = Runtime.disconnect(state.data.runtime)
- end
-
- state =
- if Runtime.connected?(runtime) do
- own_runtime(runtime, state)
- else
- state
- end
-
{:noreply, handle_operation(state, {:set_runtime, client_id, runtime})}
end
+ def handle_cast({:connect_runtime, client_pid}, state) do
+ client_id = client_id(state, client_pid)
+ {:noreply, handle_operation(state, {:connect_runtime, client_id})}
+ end
+
def handle_cast({:set_file, client_pid, file}, state) do
client_id = client_id(state, client_pid)
@@ -1489,18 +1464,28 @@ defmodule Livebook.Session do
end
@impl true
+ def handle_info({:DOWN, ref, :process, _, reason}, state)
+ when ref == state.runtime_connect.ref do
+ broadcast_error(
+ state.session_id,
+ "connecting runtime failed unexpectedly - #{Exception.format_exit(reason)}"
+ )
+
+ {:noreply,
+ %{state | runtime_connect: nil}
+ |> handle_operation({:runtime_down, @client_id})}
+ end
+
def handle_info({:DOWN, ref, :process, _, reason}, state)
when ref == state.runtime_monitor_ref do
broadcast_error(
state.session_id,
- "runtime node terminated unexpectedly - #{Exception.format_exit(reason)}"
+ "runtime terminated unexpectedly - #{Exception.format_exit(reason)}"
)
{:noreply,
%{state | runtime_monitor_ref: nil}
- |> handle_operation(
- {:set_runtime, @client_id, Livebook.Runtime.duplicate(state.data.runtime)}
- )}
+ |> handle_operation({:runtime_down, @client_id})}
end
def handle_info({:DOWN, ref, :process, _, _}, state) when ref == state.save_task_ref do
@@ -1540,6 +1525,30 @@ defmodule Livebook.Session do
{:noreply, state}
end
+ def handle_info({:runtime_connect_info, pid, info}, state)
+ when pid == state.runtime_connect.pid do
+ state = handle_operation(state, {:set_runtime_connect_info, @client_id, info})
+ {:noreply, state}
+ end
+
+ def handle_info({:runtime_connect_done, pid, result}, state)
+ when pid == state.runtime_connect.pid do
+ Process.demonitor(state.runtime_connect.ref, [:flush])
+
+ state =
+ case result do
+ {:ok, runtime} ->
+ state = own_runtime(runtime, state)
+ handle_operation(state, {:runtime_connected, @client_id, runtime})
+
+ {:error, message} ->
+ broadcast_error(state.session_id, "connecting runtime failed - #{message}")
+ handle_operation(state, {:runtime_down, @client_id})
+ end
+
+ {:noreply, %{state | runtime_connect: nil}}
+ end
+
def handle_info({:runtime_evaluation_output, cell_id, output}, state) do
output = normalize_runtime_output(output)
operation = {:add_cell_evaluation_output, @client_id, cell_id, output}
@@ -1840,7 +1849,8 @@ defmodule Livebook.Session do
state =
with {:ok, cell, section} <- Notebook.fetch_cell_and_section(state.data.notebook, cell_id),
:evaluating <- state.data.cell_infos[cell.id].eval.status do
- start_evaluation(state, cell, section)
+ evaluation_opts = state.data.cell_infos[cell.id].eval.evaluation_opts
+ start_evaluation(state, cell, section, evaluation_opts)
else
_ -> state
end
@@ -1854,7 +1864,7 @@ defmodule Livebook.Session do
end
def handle_info({:env_var_set, env_var}, state) do
- if Runtime.connected?(state.data.runtime) do
+ if state.data.runtime_status == :connected do
Runtime.put_system_envs(state.data.runtime, [{env_var.name, env_var.value}])
end
@@ -1862,7 +1872,7 @@ defmodule Livebook.Session do
end
def handle_info({:env_var_unset, env_var}, state) do
- if Runtime.connected?(state.data.runtime) do
+ if state.data.runtime_status == :connected do
Runtime.delete_system_envs(state.data.runtime, [env_var.name])
end
@@ -1874,7 +1884,7 @@ defmodule Livebook.Session do
case File.rm_rf(path) do
{:ok, _} ->
- if Runtime.connected?(state.data.runtime) do
+ if state.data.runtime_status == :connected do
{:file, file_id} = file_ref
Runtime.revoke_file(state.data.runtime, file_id)
end
@@ -2223,17 +2233,14 @@ defmodule Livebook.Session do
notify_update(state)
end
- defp after_operation(state, _prev_state, {:set_runtime, _client_id, runtime}) do
- if Runtime.connected?(runtime) do
- set_runtime_secrets(state, state.data.secrets)
- set_runtime_env_vars(state)
+ defp after_operation(state, _prev_state, {:runtime_connected, _client_id, _runtime}) do
+ set_runtime_secrets(state, state.data.secrets)
+ set_runtime_env_vars(state)
+ state
+ end
- state
- else
- state
- |> put_memory_usage(nil)
- |> notify_update()
- end
+ defp after_operation(state, _prev_state, {:runtime_down, _client_id}) do
+ after_runtime_disconnected(state)
end
defp after_operation(state, prev_state, {:set_file, _client_id, _file}) do
@@ -2273,7 +2280,7 @@ defmodule Livebook.Session do
state = put_in(state.client_id_with_assets[client_id], %{})
- if Runtime.connected?(state.data.runtime) do
+ if state.data.runtime_status == :connected do
Runtime.register_clients(state.data.runtime, [client_id])
end
@@ -2292,7 +2299,7 @@ defmodule Livebook.Session do
state = delete_client_files(state, client_id)
{_, state} = pop_in(state.client_id_with_assets[client_id])
- if Runtime.connected?(state.data.runtime) do
+ if state.data.runtime_status == :connected do
Runtime.unregister_clients(state.data.runtime, [client_id])
end
@@ -2357,12 +2364,18 @@ defmodule Livebook.Session do
end
defp after_operation(state, _prev_state, {:set_secret, _client_id, secret}) do
- if Runtime.connected?(state.data.runtime), do: set_runtime_secret(state, secret)
+ if state.data.runtime_status == :connected do
+ set_runtime_secret(state, secret)
+ end
+
state
end
defp after_operation(state, _prev_state, {:unset_secret, _client_id, secret_name}) do
- if Runtime.connected?(state.data.runtime), do: delete_runtime_secrets(state, [secret_name])
+ if state.data.runtime_status == :connected do
+ delete_runtime_secrets(state, [secret_name])
+ end
+
state
end
@@ -2405,18 +2418,18 @@ defmodule Livebook.Session do
end
defp handle_action(state, :connect_runtime) do
- case Runtime.connect(state.data.runtime) do
- {:ok, runtime} ->
- state = own_runtime(runtime, state)
- handle_operation(state, {:set_runtime, @client_id, runtime})
-
- {:error, error} ->
- broadcast_error(state.session_id, "failed to connect runtime - #{error}")
- handle_operation(state, {:set_runtime, @client_id, state.data.runtime})
- end
+ pid = Runtime.connect(state.data.runtime)
+ ref = Process.monitor(pid)
+ %{state | runtime_connect: %{pid: pid, ref: ref}}
end
- defp handle_action(state, {:start_evaluation, cell, section}) do
+ defp handle_action(state, {:disconnect_runtime, runtime}) do
+ Runtime.disconnect(runtime)
+ state = %{state | runtime_monitor_ref: nil}
+ after_runtime_disconnected(state)
+ end
+
+ defp handle_action(state, {:start_evaluation, cell, section, evaluation_opts}) do
info = state.data.cell_infos[cell.id]
if is_struct(cell, Cell.Smart) and info.status == :started do
@@ -2429,12 +2442,12 @@ defmodule Livebook.Session do
state
else
- start_evaluation(state, cell, section)
+ start_evaluation(state, cell, section, evaluation_opts)
end
end
defp handle_action(state, {:stop_evaluation, section}) do
- if Runtime.connected?(state.data.runtime) do
+ if state.data.runtime_status == :connected do
Runtime.drop_container(state.data.runtime, container_ref_for_section(section))
end
@@ -2442,7 +2455,7 @@ defmodule Livebook.Session do
end
defp handle_action(state, {:forget_evaluation, cell, section}) do
- if Runtime.connected?(state.data.runtime) do
+ if state.data.runtime_status == :connected do
Runtime.forget_evaluation(state.data.runtime, {container_ref_for_section(section), cell.id})
end
@@ -2450,7 +2463,7 @@ defmodule Livebook.Session do
end
defp handle_action(state, {:start_smart_cell, cell, _section}) do
- if Runtime.connected?(state.data.runtime) do
+ if state.data.runtime_status == :connected do
parent_locators = parent_locators_for_cell(state.data, cell)
Runtime.start_smart_cell(
@@ -2466,7 +2479,7 @@ defmodule Livebook.Session do
end
defp handle_action(state, {:set_smart_cell_parents, cell, _section, parents}) do
- if Runtime.connected?(state.data.runtime) do
+ if state.data.runtime_status == :connected do
parent_locators = evaluation_parents_to_locators(parents)
Runtime.set_smart_cell_parent_locators(state.data.runtime, cell.id, parent_locators)
end
@@ -2475,7 +2488,7 @@ defmodule Livebook.Session do
end
defp handle_action(state, {:stop_smart_cell, cell}) do
- if Runtime.connected?(state.data.runtime) do
+ if state.data.runtime_status == :connected do
Runtime.stop_smart_cell(state.data.runtime, cell.id)
end
@@ -2497,20 +2510,6 @@ defmodule Livebook.Session do
state
end
- defp handle_action(state, :app_recover) do
- if Runtime.connected?(state.data.runtime) do
- {:ok, _} = Runtime.disconnect(state.data.runtime)
- end
-
- new_runtime = Livebook.Runtime.duplicate(state.data.runtime)
- cell_ids = Data.cell_ids_for_full_evaluation(state.data, [])
-
- state
- |> handle_operation({:erase_outputs, @client_id})
- |> handle_operation({:set_runtime, @client_id, new_runtime})
- |> handle_operation({:queue_cells_evaluation, @client_id, cell_ids})
- end
-
defp handle_action(state, :app_terminate) do
send(self(), :close)
@@ -2519,7 +2518,7 @@ defmodule Livebook.Session do
defp handle_action(state, _action), do: state
- defp start_evaluation(state, cell, section) do
+ defp start_evaluation(state, cell, section, evaluation_opts) do
path =
case state.data.file || default_notebook_file(state) do
nil -> ""
@@ -2534,7 +2533,7 @@ defmodule Livebook.Session do
_ -> nil
end
- opts = [file: file, smart_cell_ref: smart_cell_ref]
+ opts = evaluation_opts ++ [file: file, smart_cell_ref: smart_cell_ref]
locator = {container_ref_for_section(section), cell.id}
parent_locators = parent_locators_for_cell(state.data, cell)
@@ -2601,6 +2600,12 @@ defmodule Livebook.Session do
Runtime.put_system_envs(state.data.runtime, env_vars)
end
+ defp after_runtime_disconnected(state) do
+ state
+ |> put_memory_usage(nil)
+ |> notify_update()
+ end
+
defp notify_update(state) do
session = self_from_state(state)
Livebook.Sessions.update_session(session)
@@ -2880,7 +2885,7 @@ defmodule Livebook.Session do
cache_file = file_entry_cache_file(state.session_id, name)
FileSystem.File.remove(cache_file)
- if Runtime.connected?(state.data.runtime) do
+ if state.data.runtime_status == :connected do
file_id = file_entry_file_id(name)
Runtime.revoke_file(state.data.runtime, file_id)
end
@@ -2901,7 +2906,7 @@ defmodule Livebook.Session do
FileSystem.File.rename(file, new_file)
end
- if Runtime.connected?(state.data.runtime) do
+ if state.data.runtime_status == :connected do
file_id = file_entry_file_id(name)
new_file_id = file_entry_file_id(new_name)
Runtime.relabel_file(state.data.runtime, file_id, new_file_id)
diff --git a/lib/livebook/session/data.ex b/lib/livebook/session/data.ex
index e07946d93..b96060b91 100644
--- a/lib/livebook/session/data.ex
+++ b/lib/livebook/session/data.ex
@@ -27,6 +27,8 @@ defmodule Livebook.Session.Data do
:input_infos,
:bin_entries,
:runtime,
+ :runtime_status,
+ :runtime_connect_info,
:runtime_transient_state,
:runtime_connected_nodes,
:smart_cell_definitions,
@@ -55,6 +57,8 @@ defmodule Livebook.Session.Data do
input_infos: %{input_id() => input_info()},
bin_entries: list(cell_bin_entry()),
runtime: Runtime.t(),
+ runtime_status: runtime_status(),
+ runtime_connect_info: String.t() | nil,
runtime_transient_state: Runtime.transient_state(),
runtime_connected_nodes: list(node()),
smart_cell_definitions: list(Runtime.smart_cell_definition()),
@@ -125,6 +129,8 @@ defmodule Livebook.Session.Data do
deleted_at: DateTime.t()
}
+ @type runtime_status :: :disconnected | :connecting | :connected
+
@type cell_revision :: non_neg_integer()
@type cell_evaluation_validity :: :fresh | :evaluated | :stale | :aborted
@@ -188,7 +194,7 @@ defmodule Livebook.Session.Data do
| {:restore_cell, client_id(), Cell.id()}
| {:move_cell, client_id(), Cell.id(), offset :: integer()}
| {:move_section, client_id(), Section.id(), offset :: integer()}
- | {:queue_cells_evaluation, client_id(), list(Cell.id())}
+ | {:queue_cells_evaluation, client_id(), list(Cell.id()), evaluation_opts :: keyword()}
| {:add_cell_doctest_report, client_id(), Cell.id(), Runtime.doctest_report()}
| {:add_cell_evaluation_output, client_id(), Cell.id(), term()}
| {:add_cell_evaluation_response, client_id(), Cell.id(), term(), metadata :: map()}
@@ -215,6 +221,11 @@ defmodule Livebook.Session.Data do
| {:set_cell_attributes, client_id(), Cell.id(), map()}
| {:set_input_value, client_id(), input_id(), value :: term()}
| {:set_runtime, client_id(), Runtime.t()}
+ | {:connect_runtime, client_id()}
+ | {:set_runtime_connect_info, client_id(), String.t()}
+ | {:runtime_connected, client_id(), Runtime.t()}
+ | {:disconnect_runtime, client_id()}
+ | {:runtime_down, client_id()}
| {:set_runtime_transient_state, client_id(), Runtime.transient_state()}
| {:set_runtime_connected_nodes, client_id(), list(node())}
| {:set_smart_cell_definitions, client_id(), list(Runtime.smart_cell_definition())}
@@ -237,6 +248,7 @@ defmodule Livebook.Session.Data do
@type action ::
:connect_runtime
+ | {:disconnect_runtime, Runtime.t()}
| {:start_evaluation, Cell.t(), Section.t()}
| {:stop_evaluation, Section.t()}
| {:forget_evaluation, Cell.t(), Section.t()}
@@ -246,7 +258,6 @@ defmodule Livebook.Session.Data do
| {:report_delta, client_id(), Cell.t(), cell_source_tag(), Text.Delta.t()}
| {:clean_up_input_values, %{input_id() => input_info()}}
| :app_report_status
- | :app_recover
| :app_terminate
@doc """
@@ -305,6 +316,8 @@ defmodule Livebook.Session.Data do
input_infos: initial_input_infos(notebook),
bin_entries: [],
runtime: default_runtime,
+ runtime_status: :disconnected,
+ runtime_connect_info: nil,
runtime_transient_state: %{},
runtime_connected_nodes: [],
smart_cell_definitions: [],
@@ -552,7 +565,7 @@ defmodule Livebook.Session.Data do
end
end
- def apply_operation(data, {:queue_cells_evaluation, _client_id, cell_ids}) do
+ def apply_operation(data, {:queue_cells_evaluation, _client_id, cell_ids, evaluation_opts}) do
cells_with_section =
data.notebook
|> Notebook.evaluable_cells_with_section()
@@ -568,7 +581,7 @@ defmodule Livebook.Session.Data do
|> with_actions()
|> queue_prerequisite_cells_evaluation(cell_ids)
|> reduce(cells_with_section, fn data_actions, {cell, section} ->
- queue_cell_evaluation(data_actions, cell, section)
+ queue_cell_evaluation(data_actions, cell, section, evaluation_opts)
end)
|> maybe_connect_runtime(data)
|> update_validity_and_evaluation()
@@ -862,10 +875,71 @@ defmodule Livebook.Session.Data do
end
def apply_operation(data, {:set_runtime, _client_id, runtime}) do
- data
- |> with_actions()
- |> set_runtime(data, runtime)
- |> wrap_ok()
+ with true <- data.runtime_status in [:connected, :disconnected] do
+ data
+ |> with_actions()
+ |> set_runtime(runtime)
+ |> wrap_ok()
+ else
+ _ -> :error
+ end
+ end
+
+ def apply_operation(data, {:connect_runtime, _client_id}) do
+ with :disconnected <- data.runtime_status do
+ data
+ |> with_actions()
+ |> connect_runtime()
+ |> wrap_ok()
+ else
+ _ -> :error
+ end
+ end
+
+ def apply_operation(data, {:set_runtime_connect_info, _client_id, info}) do
+ with :connecting <- data.runtime_status do
+ data
+ |> with_actions()
+ |> set_runtime_connect_info(info)
+ |> wrap_ok()
+ else
+ _ -> :error
+ end
+ end
+
+ def apply_operation(data, {:runtime_connected, _client_id, runtime}) do
+ with :connecting <- data.runtime_status do
+ data
+ |> with_actions()
+ |> runtime_connected(runtime)
+ |> wrap_ok()
+ else
+ _ -> :error
+ end
+ end
+
+ def apply_operation(data, {:disconnect_runtime, _client_id}) do
+ with :connected <- data.runtime_status do
+ data
+ |> with_actions()
+ |> disconnect_runtime()
+ |> app_update_execution_status()
+ |> wrap_ok()
+ else
+ _ -> :error
+ end
+ end
+
+ def apply_operation(data, {:runtime_down, _client_id}) do
+ with true <- data.runtime_status in [:connecting, :connected] do
+ data
+ |> with_actions()
+ |> clear_runtime()
+ |> app_update_execution_status()
+ |> wrap_ok()
+ else
+ _ -> :error
+ end
end
def apply_operation(data, {:set_runtime_transient_state, _client_id, transient_state}) do
@@ -1261,16 +1335,17 @@ defmodule Livebook.Session.Data do
end
end
- defp queue_cell_evaluation(data_actions, cell, section) do
+ defp queue_cell_evaluation(data_actions, cell, section, evaluation_opts \\ []) do
data_actions
|> update_section_info!(section.id, fn section ->
update_in(section.evaluation_queue, &MapSet.put(&1, cell.id))
end)
|> update_cell_eval_info!(cell.id, fn eval_info ->
- update_in(eval_info.status, fn
- :ready -> :queued
- other -> other
- end)
+ if eval_info.status == :ready do
+ %{eval_info | status: :queued, evaluation_opts: evaluation_opts}
+ else
+ eval_info
+ end
end)
end
@@ -1374,9 +1449,9 @@ defmodule Livebook.Session.Data do
end
defp maybe_connect_runtime({data, _} = data_actions, prev_data) do
- if not Runtime.connected?(data.runtime) and not any_cell_queued?(prev_data) and
+ if data.runtime_status == :disconnected and not any_cell_queued?(prev_data) and
any_cell_queued?(data) do
- add_action(data_actions, :connect_runtime)
+ connect_runtime(data_actions)
else
data_actions
end
@@ -1403,8 +1478,10 @@ defmodule Livebook.Session.Data do
queue_prerequisite_cells_evaluation(data_actions, trailing_queued_cell_ids)
end
- defp maybe_evaluate_queued({data, _} = data_actions) do
- if Runtime.connected?(data.runtime) do
+ defp maybe_evaluate_queued(data_actions) do
+ {data, _} = data_actions = check_setup_cell_for_reevaluation(data_actions)
+
+ if data.runtime_status == :connected do
main_flow_evaluating? = main_flow_evaluating?(data)
{awaiting_branch_sections, awaiting_regular_sections} =
@@ -1453,6 +1530,43 @@ defmodule Livebook.Session.Data do
end
end
+ defp check_setup_cell_for_reevaluation({data, _} = data_actions) do
+ # When setup cell has been evaluated and is queued again, we need
+ # to reconnect the runtime to get a fresh evaluation environment
+ # for setup. We subsequently queue all cells that are currently
+ # queued
+
+ case data.cell_infos[Cell.setup_cell_id()].eval do
+ %{status: :queued, validity: :evaluated} when data.runtime_status == :connected ->
+ queued_cells_with_section =
+ data.notebook
+ |> Notebook.evaluable_cells_with_section()
+ |> Enum.filter(fn {cell, _} ->
+ data.cell_infos[cell.id].eval.status == :queued
+ end)
+ |> Enum.map(fn {cell, section} ->
+ {cell, section, data.cell_infos[cell.id].eval.evaluation_opts}
+ end)
+
+ cell_ids =
+ for {cell, _section, _evaluation_opts} <- queued_cells_with_section, do: cell.id
+
+ data_actions
+ |> disconnect_runtime()
+ |> connect_runtime()
+ |> queue_prerequisite_cells_evaluation(cell_ids)
+ |> reduce(
+ queued_cells_with_section,
+ fn data_actions, {cell, section, evaluation_opts} ->
+ queue_cell_evaluation(data_actions, cell, section, evaluation_opts)
+ end
+ )
+
+ _ ->
+ data_actions
+ end
+ end
+
defp first_queued_cell(data, section) do
find_queued_cell(data, section.cells)
end
@@ -1533,7 +1647,9 @@ defmodule Livebook.Session.Data do
evaluating_cell_id: cell.id,
evaluation_queue: MapSet.delete(section_info.evaluation_queue, cell.id)
)
- |> add_action({:start_evaluation, cell, section})
+ |> add_action(
+ {:start_evaluation, cell, section, data.cell_infos[cell.id].eval.evaluation_opts}
+ )
else
data_actions
end
@@ -1596,7 +1712,7 @@ defmodule Livebook.Session.Data do
|> Notebook.parent_cells_with_section(cell_ids)
|> Enum.filter(fn {cell, _section} ->
info = data.cell_infos[cell.id]
- Cell.evaluable?(cell) and cell_outdated?(data, cell) and info.eval.status == :ready
+ Cell.evaluable?(cell) and cell_outdated?(data, cell.id) and info.eval.status == :ready
end)
|> Enum.reverse()
@@ -1709,7 +1825,7 @@ defmodule Livebook.Session.Data do
end
defp recover_smart_cell({data, _} = data_actions, cell, section) do
- if Runtime.connected?(data.runtime) do
+ if data.runtime_status == :connected do
start_smart_cell(data_actions, cell, section)
else
data_actions
@@ -1965,24 +2081,53 @@ defmodule Livebook.Session.Data do
|> set!(input_infos: Map.put(data.input_infos, input_id, input_info(value)))
end
- defp set_runtime(data_actions, prev_data, runtime) do
- {data, _} =
- data_actions =
- set!(data_actions,
- runtime: runtime,
- runtime_connected_nodes: [],
- smart_cell_definitions: []
- )
+ defp set_runtime({data, _} = data_actions, runtime) do
+ data_actions =
+ case data.runtime_status do
+ :connected ->
+ disconnect_runtime(data_actions)
- if not Runtime.connected?(prev_data.runtime) and Runtime.connected?(data.runtime) do
- data_actions
- |> maybe_evaluate_queued()
- else
- data_actions
- |> clear_all_evaluation()
- |> clear_smart_cells()
- |> app_update_execution_status()
- end
+ :disconnected ->
+ data_actions
+ end
+
+ set!(data_actions, runtime: runtime)
+ end
+
+ defp connect_runtime(data_actions) do
+ data_actions
+ |> set!(runtime_status: :connecting)
+ |> add_action(:connect_runtime)
+ end
+
+ defp set_runtime_connect_info(data_actions, info) do
+ data_actions
+ |> set!(runtime_connect_info: info)
+ end
+
+ defp runtime_connected(data_actions, runtime) do
+ data_actions
+ |> set!(runtime: runtime, runtime_status: :connected, runtime_connect_info: nil)
+ |> maybe_evaluate_queued()
+ end
+
+ defp disconnect_runtime({data, _} = data_actions) do
+ data_actions
+ |> add_action({:disconnect_runtime, data.runtime})
+ |> clear_runtime()
+ end
+
+ defp clear_runtime({data, _} = data_actions) do
+ data_actions
+ |> set!(
+ runtime: Runtime.duplicate(data.runtime),
+ runtime_status: :disconnected,
+ runtime_connect_info: nil,
+ runtime_connected_nodes: [],
+ smart_cell_definitions: []
+ )
+ |> clear_all_evaluation()
+ |> clear_smart_cells()
end
defp set_secret({data, _} = data_actions, secret) do
@@ -2037,7 +2182,7 @@ defmodule Livebook.Session.Data do
end
defp maybe_start_smart_cells({data, _} = data_actions) do
- if Runtime.connected?(data.runtime) do
+ if data.runtime_status == :connected do
dead_cells = dead_smart_cells_with_section(data)
kinds =
@@ -2219,6 +2364,7 @@ defmodule Livebook.Session.Data do
status: :ready,
errored: false,
interrupted: false,
+ evaluation_opts: [],
evaluation_digest: nil,
evaluation_time_ms: nil,
evaluation_start: nil,
@@ -2619,25 +2765,38 @@ defmodule Livebook.Session.Data do
# If everything was executed and an error happened, it means it
# was a runtime crash and everything is aborted
- data_actions =
+ {data_actions, execution_status} =
if data.app_data.status.execution == :executed and execution_status == :error do
- add_action(data_actions, :app_recover)
+ {app_recover(data_actions), :executing}
else
- data_actions
+ {data_actions, execution_status}
end
update_app_data!(data_actions, &put_in(&1.status.execution, execution_status))
end
+ defp app_recover({data, _} = data_actions) do
+ evaluable_cells_with_section = Notebook.evaluable_cells_with_section(data.notebook)
+
+ data_actions
+ |> disconnect_runtime()
+ |> connect_runtime()
+ |> erase_outputs()
+ |> garbage_collect_input_infos()
+ |> reduce(evaluable_cells_with_section, fn data_actions, {cell, section} ->
+ queue_cell_evaluation(data_actions, cell, section)
+ end)
+ end
+
@doc """
Checks if the given cell is outdated.
- A cell is considered outdated if its new/fresh or its content
- has changed since the last evaluation.
+ A cell is considered outdated if its fresh/stale or its content has
+ changed since the last evaluation.
"""
- @spec cell_outdated?(t(), Cell.t()) :: boolean()
- def cell_outdated?(data, cell) do
- info = data.cell_infos[cell.id]
+ @spec cell_outdated?(t(), Cell.id()) :: boolean()
+ def cell_outdated?(data, cell_id) do
+ info = data.cell_infos[cell_id]
info.eval.validity != :evaluated or info.eval.evaluation_digest != info.sources.primary.digest
end
@@ -2649,28 +2808,36 @@ defmodule Livebook.Session.Data do
"""
@spec cell_ids_for_full_evaluation(t(), list(Cell.id())) :: list(Cell.id())
def cell_ids_for_full_evaluation(data, forced_cell_ids) do
+ requires_reconnect? =
+ data.cell_infos[Cell.setup_cell_id()].eval.validity == :evaluated and
+ cell_outdated?(data, Cell.setup_cell_id())
+
evaluable_cells_with_section = Notebook.evaluable_cells_with_section(data.notebook)
- evaluable_cell_ids =
- for {cell, _} <- evaluable_cells_with_section,
- cell_outdated?(data, cell) or cell.id in forced_cell_ids,
- do: cell.id,
- into: MapSet.new()
+ if requires_reconnect? do
+ for {cell, _} <- evaluable_cells_with_section, do: cell.id
+ else
+ evaluable_cell_ids =
+ for {cell, _} <- evaluable_cells_with_section,
+ cell_outdated?(data, cell.id) or cell.id in forced_cell_ids,
+ do: cell.id,
+ into: MapSet.new()
- cell_identifier_parents = cell_identifier_parents(data)
+ cell_identifier_parents = cell_identifier_parents(data)
- child_ids =
- for {cell_id, cell_identifier_parents} <- cell_identifier_parents,
- Enum.any?(cell_identifier_parents, &(&1 in evaluable_cell_ids)),
- do: cell_id
+ child_ids =
+ for {cell_id, cell_identifier_parents} <- cell_identifier_parents,
+ Enum.any?(cell_identifier_parents, &(&1 in evaluable_cell_ids)),
+ do: cell_id
- child_ids
- |> Enum.into(evaluable_cell_ids)
- |> Enum.to_list()
- |> Enum.filter(fn cell_id ->
- info = data.cell_infos[cell_id]
- info.eval.status == :ready
- end)
+ child_ids
+ |> Enum.into(evaluable_cell_ids)
+ |> Enum.to_list()
+ |> Enum.filter(fn cell_id ->
+ info = data.cell_infos[cell_id]
+ info.eval.status == :ready
+ end)
+ end
end
# Builds identifier parent list for every evaluable cell.
diff --git a/lib/livebook_web/components/form_components.ex b/lib/livebook_web/components/form_components.ex
index e628d567e..254505fc8 100644
--- a/lib/livebook_web/components/form_components.ex
+++ b/lib/livebook_web/components/form_components.ex
@@ -487,8 +487,11 @@ defmodule LivebookWeb.FormComponents do
id={@id}
name={@name}
class={[
- "w-full px-3 py-2 pr-7 appearance-none bg-gray-50 text-sm border rounded-lg placeholder-gray-400 text-gray-600 disabled:opacity-70 disabled:cursor-not-allowed",
- if(@errors == [], do: "border-gray-200", else: "border-red-300"),
+ "w-full px-3 py-2 pr-7 appearance-none text-sm border rounded-lg placeholder-gray-400 disabled:opacity-70 disabled:cursor-not-allowed",
+ if(@errors == [],
+ do: "bg-gray-50 border-gray-200 text-gray-600",
+ else: "bg-red-50 border-red-600 text-red-600"
+ ),
@class
]}
{@rest}
diff --git a/lib/livebook_web/live/session_live.ex b/lib/livebook_web/live/session_live.ex
index 0a4455f2b..6f03331c4 100644
--- a/lib/livebook_web/live/session_live.ex
+++ b/lib/livebook_web/live/session_live.ex
@@ -263,30 +263,38 @@ defmodule LivebookWeb.SessionLive do
data = socket.private.data
%{"section_id" => section_id, "cell_id" => cell_id} = params
- if Livebook.Runtime.connected?(socket.private.data.runtime) do
- case example_snippet_definition_by_name(data, params["definition_name"]) do
- {:ok, definition} ->
- variant = Enum.fetch!(definition.variants, params["variant_idx"])
+ socket =
+ case socket.private.data.runtime_status do
+ :disconnected ->
+ reason = "To insert this block, you need a connected runtime."
+ confirm_setup_runtime(socket, reason)
- socket =
- ensure_packages_then(socket, variant.packages, definition.name, "block", fn socket ->
- with {:ok, section, index} <-
- section_with_next_index(socket.private.data.notebook, section_id, cell_id) do
- attrs = %{source: variant.source}
- Session.insert_cell(socket.assigns.session.pid, section.id, index, :code, attrs)
- {:ok, socket}
+ :connecting ->
+ message = "To insert this block, wait for the runtime to finish connecting."
+ {:noreply, put_flash(socket, :info, message)}
+
+ :connected ->
+ case example_snippet_definition_by_name(data, params["definition_name"]) do
+ {:ok, definition} ->
+ variant = Enum.fetch!(definition.variants, params["variant_idx"])
+
+ fun = fn socket ->
+ with {:ok, section, index} <-
+ section_with_next_index(socket.private.data.notebook, section_id, cell_id) do
+ attrs = %{source: variant.source}
+ Session.insert_cell(socket.assigns.session.pid, section.id, index, :code, attrs)
+ {:ok, socket}
+ end
end
- end)
- {:noreply, socket}
+ ensure_packages_then(socket, variant.packages, definition.name, "block", fun)
- _ ->
- {:noreply, socket}
+ _ ->
+ socket
+ end
end
- else
- reason = "To insert this block, you need a connected runtime."
- {:noreply, confirm_setup_default_runtime(socket, reason)}
- end
+
+ {:noreply, socket}
end
def handle_event("insert_smart_cell_below", params, socket) do
@@ -486,24 +494,14 @@ defmodule LivebookWeb.SessionLive do
end
def handle_event("queue_cell_evaluation", %{"cell_id" => cell_id} = params, socket) do
- data = socket.private.data
-
- {status, socket} =
- with {:ok, cell, _section} <- Notebook.fetch_cell_and_section(data.notebook, cell_id),
- true <- Cell.setup?(cell),
- false <- data.cell_infos[cell.id].eval.validity == :fresh do
- maybe_reconnect_runtime(socket)
+ opts =
+ if params["disable_dependencies_cache"] do
+ [disable_dependencies_cache: true]
else
- _ -> {:ok, socket}
+ []
end
- if params["disable_dependencies_cache"] do
- Session.disable_dependencies_cache(socket.assigns.session.pid)
- end
-
- if status == :ok do
- Session.queue_cell_evaluation(socket.assigns.session.pid, cell_id)
- end
+ Session.queue_cell_evaluation(socket.assigns.session.pid, cell_id, opts)
{:noreply, socket}
end
@@ -559,18 +557,15 @@ defmodule LivebookWeb.SessionLive do
end
end
- def handle_event("reconnect_runtime", %{}, socket) do
- {_, socket} = maybe_reconnect_runtime(socket)
- {:noreply, socket}
- end
-
def handle_event("connect_runtime", %{}, socket) do
- {_, socket} = connect_runtime(socket)
+ Session.connect_runtime(socket.assigns.session.pid)
{:noreply, socket}
end
- def handle_event("setup_default_runtime", %{"reason" => reason}, socket) do
- {:noreply, confirm_setup_default_runtime(socket, reason)}
+ def handle_event("reconnect_runtime", %{}, socket) do
+ Session.disconnect_runtime(socket.assigns.session.pid)
+ Session.connect_runtime(socket.assigns.session.pid)
+ {:noreply, socket}
end
def handle_event("disconnect_runtime", %{}, socket) do
@@ -578,12 +573,15 @@ defmodule LivebookWeb.SessionLive do
{:noreply, socket}
end
+ def handle_event("setup_runtime", %{"reason" => reason}, socket) do
+ {:noreply, confirm_setup_runtime(socket, reason)}
+ end
+
def handle_event("runtime_disconnect_node", %{"node" => node}, socket) do
node = Enum.find(socket.private.data.runtime_connected_nodes, &(Atom.to_string(&1) == node))
- runtime = socket.private.data.runtime
- if node && Runtime.connected?(runtime) do
- Runtime.disconnect_node(runtime, node)
+ if node do
+ Runtime.disconnect_node(socket.private.data.runtime, node)
end
{:noreply, socket}
@@ -628,7 +626,7 @@ defmodule LivebookWeb.SessionLive do
data = socket.private.data
with {:ok, cell, _section} <- Notebook.fetch_cell_and_section(data.notebook, cell_id) do
- if Runtime.connected?(data.runtime) do
+ if data.runtime_status == :connected do
parent_locators = Session.parent_locators_for_cell(data, cell)
node = intellisense_node(cell)
@@ -636,19 +634,20 @@ defmodule LivebookWeb.SessionLive do
{:reply, %{"ref" => inspect(ref)}, socket}
else
- info =
+ reason =
cond do
params["type"] == "completion" and not params["editor_auto_completion"] ->
- "You need to start a runtime (or evaluate a cell) for code completion"
+ "You need a connected runtime to enable code completion."
params["type"] == "format" ->
- "You need to start a runtime (or evaluate a cell) to enable code formatting"
+ "You need a connected runtime to enable code formatting."
true ->
nil
end
- socket = if info, do: put_flash(socket, :info, info), else: socket
+ socket = if reason, do: confirm_setup_runtime(socket, reason), else: socket
+
{:reply, %{"ref" => nil}, socket}
end
else
@@ -782,21 +781,27 @@ defmodule LivebookWeb.SessionLive do
socket
) do
if file_entry = find_file_entry(socket, file_entry_name) do
- if Livebook.Runtime.connected?(socket.private.data.runtime) do
- {:noreply,
- socket
- |> assign(
- insert_file_metadata: %{
- section_id: section_id,
- cell_id: cell_id,
- file_entry: file_entry,
- handlers: handlers_for_file_entry(file_entry, socket.private.data.runtime)
- }
- )
- |> push_patch(to: ~p"/sessions/#{socket.assigns.session.id}/insert-file")}
- else
- reason = "To see the available options, you need a connected runtime."
- {:noreply, confirm_setup_default_runtime(socket, reason)}
+ case socket.private.data.runtime_status do
+ :connected ->
+ {:noreply,
+ socket
+ |> assign(
+ insert_file_metadata: %{
+ section_id: section_id,
+ cell_id: cell_id,
+ file_entry: file_entry,
+ handlers: handlers_for_file_entry(file_entry, socket.private.data.runtime)
+ }
+ )
+ |> push_patch(to: ~p"/sessions/#{socket.assigns.session.id}/insert-file")}
+
+ :connecting ->
+ message = "To see the available options, wait for the runtime to finish connecting."
+ {:noreply, put_flash(socket, :info, message)}
+
+ :disconnected ->
+ reason = "To see the available options, you need a connected runtime."
+ {:noreply, confirm_setup_runtime(socket, reason)}
end
else
{:noreply, socket}
@@ -843,15 +848,21 @@ defmodule LivebookWeb.SessionLive do
%{"section_id" => section_id, "cell_id" => cell_id},
socket
) do
- if Livebook.Runtime.connected?(socket.private.data.runtime) do
- {:noreply,
- socket
- |> assign(file_drop_metadata: %{section_id: section_id, cell_id: cell_id})
- |> push_patch(to: ~p"/sessions/#{socket.assigns.session.id}/add-file/upload")
- |> push_event("finish_file_drop", %{})}
- else
- reason = "To see the available options, you need a connected runtime."
- {:noreply, confirm_setup_default_runtime(socket, reason)}
+ case socket.private.data.runtime_status do
+ :disconnected ->
+ reason = "To see the available options, you need a connected runtime."
+ {:noreply, confirm_setup_runtime(socket, reason)}
+
+ :connecting ->
+ message = "To see the available options, wait for the runtime to finish connecting."
+ {:noreply, put_flash(socket, :info, message)}
+
+ :connected ->
+ {:noreply,
+ socket
+ |> assign(file_drop_metadata: %{section_id: section_id, cell_id: cell_id})
+ |> push_patch(to: ~p"/sessions/#{socket.assigns.session.id}/add-file/upload")
+ |> push_event("finish_file_drop", %{})}
end
end
@@ -883,6 +894,20 @@ defmodule LivebookWeb.SessionLive do
{:noreply, handle_operation(socket, operation)}
end
+ def handle_info({:error, error}, socket) when socket.assigns.live_action == :runtime_settings do
+ # When the runtime settings modal is open we assume the error is
+ # related to connecting the runtime and we show it dirrectly there
+
+ message = error |> to_string() |> upcase_first()
+
+ send_update(LivebookWeb.SessionLive.RuntimeComponent,
+ id: "runtime-settings",
+ event: {:error, message}
+ )
+
+ {:noreply, socket}
+ end
+
def handle_info({:error, error}, socket) do
message = error |> to_string() |> upcase_first()
socket = put_flash(socket, :error, message)
@@ -1527,49 +1552,15 @@ defmodule LivebookWeb.SessionLive do
defp autofocus_cell_id(%Notebook{sections: [%{cells: [%{id: id, source: ""}]}]}), do: id
defp autofocus_cell_id(_notebook), do: nil
- defp connect_runtime(socket) do
- case Runtime.connect(socket.private.data.runtime) do
- {:ok, runtime} ->
- Session.set_runtime(socket.assigns.session.pid, runtime)
- {:ok, socket}
-
- {:error, message} ->
- {:error, put_flash(socket, :error, "Failed to connect runtime - #{message}")}
- end
- end
-
- defp maybe_reconnect_runtime(%{private: %{data: data}} = socket) do
- if Runtime.connected?(data.runtime) do
- data.runtime
- |> Runtime.duplicate()
- |> Runtime.connect()
- |> case do
- {:ok, new_runtime} ->
- Session.set_runtime(socket.assigns.session.pid, new_runtime)
- {:ok, clear_flash(socket, :error)}
-
- {:error, message} ->
- {:error, put_flash(socket, :error, "Failed to connect runtime - #{message}")}
- end
- else
- {:ok, socket}
- end
- end
-
- defp confirm_setup_default_runtime(socket, reason) do
+ defp confirm_setup_runtime(socket, reason) do
on_confirm = fn socket ->
- {status, socket} = connect_runtime(socket)
-
- if status == :ok do
- Session.queue_cell_evaluation(socket.assigns.session.pid, Cell.setup_cell_id())
- end
-
+ Session.queue_cell_evaluation(socket.assigns.session.pid, Cell.setup_cell_id())
socket
end
confirm(socket, on_confirm,
title: "Setup runtime",
- description: "#{reason} Do you want to connect and setup the default one?",
+ description: "#{reason} Do you want to connect and setup the current one?",
confirm_text: "Setup runtime",
confirm_icon: "play-line",
danger: false
@@ -1582,7 +1573,7 @@ defmodule LivebookWeb.SessionLive do
defp example_snippet_definition_by_name(data, name) do
data.runtime
- |> Livebook.Runtime.snippet_definitions()
+ |> Runtime.snippet_definitions()
|> Enum.find_value(:error, &(&1.type == :example && &1.name == name && {:ok, &1}))
end
@@ -1590,25 +1581,12 @@ defmodule LivebookWeb.SessionLive do
Enum.find_value(data.smart_cell_definitions, :error, &(&1.kind == kind && {:ok, &1}))
end
- defp add_dependencies_and_reevaluate(socket, dependencies) do
- Session.add_dependencies(socket.assigns.session.pid, dependencies)
-
- {status, socket} = maybe_reconnect_runtime(socket)
-
- if status == :ok do
- Session.queue_cell_evaluation(socket.assigns.session.pid, Cell.setup_cell_id())
- Session.queue_cells_reevaluation(socket.assigns.session.pid)
- end
-
- socket
- end
-
defp ensure_packages_then(socket, packages, target_name, target_type, fun) do
dependencies = Enum.map(packages, & &1.dependency)
has_dependencies? =
dependencies == [] or
- Livebook.Runtime.has_dependencies?(socket.private.data.runtime, dependencies)
+ Runtime.has_dependencies?(socket.private.data.runtime, dependencies)
cond do
has_dependencies? ->
@@ -1617,7 +1595,7 @@ defmodule LivebookWeb.SessionLive do
:error -> socket
end
- Livebook.Runtime.fixed_dependencies?(socket.private.data.runtime) ->
+ Runtime.fixed_dependencies?(socket.private.data.runtime) ->
put_flash(socket, :error, "This runtime doesn't support adding dependencies")
true ->
@@ -1632,6 +1610,13 @@ defmodule LivebookWeb.SessionLive do
end
end
+ defp add_dependencies_and_reevaluate(socket, dependencies) do
+ Session.add_dependencies(socket.assigns.session.pid, dependencies)
+ Session.queue_cell_evaluation(socket.assigns.session.pid, Cell.setup_cell_id())
+ Session.queue_cells_reevaluation(socket.assigns.session.pid)
+ socket
+ end
+
defp confirm_add_packages(socket, on_confirm, packages, target_name, target_type) do
assigns = %{packages: packages, target_name: target_name, target_type: target_type}
@@ -1728,7 +1713,7 @@ defmodule LivebookWeb.SessionLive do
defp handlers_for_file_entry(file_entry, runtime) do
handlers =
- for definition <- Livebook.Runtime.snippet_definitions(runtime),
+ for definition <- Runtime.snippet_definitions(runtime),
definition.type == :file_action,
do: %{definition: definition, cell_type: :code}
@@ -1789,11 +1774,13 @@ defmodule LivebookWeb.SessionLive do
dirty: data.dirty,
persistence_warnings: data.persistence_warnings,
runtime: data.runtime,
+ runtime_status: data.runtime_status,
+ runtime_connect_info: data.runtime_connect_info,
runtime_connected_nodes: Enum.sort(data.runtime_connected_nodes),
smart_cell_definitions: Enum.sort_by(data.smart_cell_definitions, & &1.name),
example_snippet_definitions:
data.runtime
- |> Livebook.Runtime.snippet_definitions()
+ |> Runtime.snippet_definitions()
|> Enum.filter(&(&1.type == :example))
|> Enum.sort_by(& &1.name),
global_status: global_status(data),
diff --git a/lib/livebook_web/live/session_live/attached_live.ex b/lib/livebook_web/live/session_live/attached_runtime_component.ex
similarity index 56%
rename from lib/livebook_web/live/session_live/attached_live.ex
rename to lib/livebook_web/live/session_live/attached_runtime_component.ex
index f8241fe29..ba5e55828 100644
--- a/lib/livebook_web/live/session_live/attached_live.ex
+++ b/lib/livebook_web/live/session_live/attached_runtime_component.ex
@@ -1,33 +1,39 @@
-defmodule LivebookWeb.SessionLive.AttachedLive do
- use LivebookWeb, :live_view
+defmodule LivebookWeb.SessionLive.AttachedRuntimeComponent do
+ use LivebookWeb, :live_component
import Ecto.Changeset
alias Livebook.{Session, Runtime}
@impl true
- def mount(
- _params,
- %{"session_pid" => session_pid, "current_runtime" => current_runtime},
- socket
- ) do
- session = Session.get_by_pid(session_pid)
-
+ def mount(socket) do
unless Livebook.Config.runtime_enabled?(Livebook.Runtime.Attached) do
raise "runtime module not allowed"
end
- if connected?(socket) do
- Session.subscribe(session.id)
- end
+ {:ok, socket}
+ end
- {:ok,
- assign(socket,
- session: session,
- current_runtime: current_runtime,
- error_message: nil,
- changeset: changeset(current_runtime)
- )}
+ @impl true
+ def update(assigns, socket) do
+ changeset =
+ case socket.assigns[:changeset] do
+ nil ->
+ changeset(assigns.runtime)
+
+ changeset when socket.assigns.runtime == assigns.runtime ->
+ changeset
+
+ changeset ->
+ changeset(assigns.runtime, changeset.params)
+ end
+
+ socket =
+ socket
+ |> assign(assigns)
+ |> assign(:changeset, changeset)
+
+ {:ok, socket}
end
defp changeset(runtime, attrs \\ %{}) do
@@ -50,13 +56,10 @@ defmodule LivebookWeb.SessionLive.AttachedLive do
def render(assigns) do
~H"""
-
- <%= @error_message %>
-
Connect the session to an already running node
and evaluate code in the context of that node.
- The node must run Erlang/OTP <%= :erlang.system_info(:otp_release) %> and Elixir <%= System.version() %> (or later).
+ The node must run Elixir <%= Livebook.Runtime.Attached.elixir_version_requirement() %>.
Make sure to give the node a name and a cookie, for example:
@@ -71,6 +74,7 @@ defmodule LivebookWeb.SessionLive.AttachedLive do
as={:data}
phx-submit="init"
phx-change="validate"
+ phx-target={@myself}
autocomplete="off"
spellcheck="false"
>
@@ -78,62 +82,52 @@ defmodule LivebookWeb.SessionLive.AttachedLive do
<.text_field field={f[:name]} label="Name" placeholder={test_node()} />
<.text_field field={f[:cookie]} label="Cookie" placeholder="mycookie" />
- <.button type="submit" disabled={not @changeset.valid?}>
- <%= if(reconnecting?(@changeset), do: "Reconnect", else: "Connect") %>
+ <.button type="submit" disabled={@runtime_status == :connecting or not @changeset.valid?}>
+ <%= label(@changeset, @runtime_status) %>
"""
end
+ defp label(changeset, runtime_status) do
+ reconnecting? = changeset.valid? and changeset.data == apply_changes(changeset)
+
+ case {reconnecting?, runtime_status} do
+ {true, :connected} -> "Reconnect"
+ {true, :connecting} -> "Connecting..."
+ _ -> "Connect"
+ end
+ end
+
@impl true
def handle_event("validate", %{"data" => data}, socket) do
changeset =
- socket.assigns.current_runtime |> changeset(data) |> Map.replace!(:action, :validate)
+ socket.assigns.runtime
+ |> changeset(data)
+ |> Map.replace!(:action, :validate)
{:noreply, assign(socket, changeset: changeset)}
end
def handle_event("init", %{"data" => data}, socket) do
- socket.assigns.current_runtime
+ socket.assigns.runtime
|> changeset(data)
|> apply_action(:insert)
|> case do
{:ok, data} ->
node = String.to_atom(data.name)
cookie = String.to_atom(data.cookie)
-
runtime = Runtime.Attached.new(node, cookie)
-
- case Runtime.connect(runtime) do
- {:ok, runtime} ->
- Session.set_runtime(socket.assigns.session.pid, runtime)
- {:noreply, assign(socket, changeset: changeset(runtime), error_message: nil)}
-
- {:error, message} ->
- {:noreply,
- assign(socket,
- changeset: changeset(socket.assigns.current_runtime, data),
- error_message: Livebook.Utils.upcase_first(message)
- )}
- end
+ Session.set_runtime(socket.assigns.session.pid, runtime)
+ Session.connect_runtime(socket.assigns.session.pid)
+ {:noreply, socket}
{:error, changeset} ->
{:noreply, assign(socket, changeset: changeset)}
end
end
- @impl true
- def handle_info({:operation, {:set_runtime, _pid, runtime}}, socket) do
- {:noreply, assign(socket, current_runtime: runtime)}
- end
-
- def handle_info(_message, socket), do: {:noreply, socket}
-
- defp reconnecting?(changeset) do
- changeset.valid? and changeset.data == apply_changes(changeset)
- end
-
defp test_node() do
"test@#{Livebook.Utils.node_host()}"
end
diff --git a/lib/livebook_web/live/session_live/elixir_standalone_live.ex b/lib/livebook_web/live/session_live/elixir_standalone_live.ex
deleted file mode 100644
index 71f04a13e..000000000
--- a/lib/livebook_web/live/session_live/elixir_standalone_live.ex
+++ /dev/null
@@ -1,65 +0,0 @@
-defmodule LivebookWeb.SessionLive.ElixirStandaloneLive do
- use LivebookWeb, :live_view
-
- alias Livebook.{Session, Runtime}
-
- @impl true
- def mount(
- _params,
- %{"session_pid" => session_pid, "current_runtime" => current_runtime},
- socket
- ) do
- session = Session.get_by_pid(session_pid)
-
- unless Livebook.Config.runtime_enabled?(Livebook.Runtime.ElixirStandalone) do
- raise "runtime module not allowed"
- end
-
- if connected?(socket) do
- Session.subscribe(session.id)
- end
-
- {:ok, assign(socket, session: session, current_runtime: current_runtime, error_message: nil)}
- end
-
- @impl true
- def render(assigns) do
- ~H"""
-
-
- <%= @error_message %>
-
-
- Start a new local node to evaluate code.
-
- <.button phx-click="init">
- <%= if(matching_runtime?(@current_runtime), do: "Reconnect", else: "Connect") %>
-
-
- """
- end
-
- defp matching_runtime?(%Runtime.ElixirStandalone{} = runtime), do: Runtime.connected?(runtime)
- defp matching_runtime?(_runtime), do: false
-
- @impl true
- def handle_event("init", _params, socket) do
- Runtime.ElixirStandalone.new()
- |> Runtime.connect()
- |> case do
- {:ok, runtime} ->
- Session.set_runtime(socket.assigns.session.pid, runtime)
- {:noreply, assign(socket, error_message: nil)}
-
- {:error, message} ->
- {:noreply, assign(socket, error_message: message)}
- end
- end
-
- @impl true
- def handle_info({:operation, {:set_runtime, _pid, runtime}}, socket) do
- {:noreply, assign(socket, current_runtime: runtime)}
- end
-
- def handle_info(_message, socket), do: {:noreply, socket}
-end
diff --git a/lib/livebook_web/live/session_live/embedded_live.ex b/lib/livebook_web/live/session_live/embedded_runtime_component.ex
similarity index 51%
rename from lib/livebook_web/live/session_live/embedded_live.ex
rename to lib/livebook_web/live/session_live/embedded_runtime_component.ex
index c5f7cfa6e..8ae8ed0ea 100644
--- a/lib/livebook_web/live/session_live/embedded_live.ex
+++ b/lib/livebook_web/live/session_live/embedded_runtime_component.ex
@@ -1,25 +1,15 @@
-defmodule LivebookWeb.SessionLive.EmbeddedLive do
- use LivebookWeb, :live_view
+defmodule LivebookWeb.SessionLive.EmbeddedRuntimeComponent do
+ use LivebookWeb, :live_component
alias Livebook.{Session, Runtime}
@impl true
- def mount(
- _params,
- %{"session_pid" => session_pid, "current_runtime" => current_runtime},
- socket
- ) do
- session = Session.get_by_pid(session_pid)
-
+ def mount(socket) do
unless Livebook.Config.runtime_enabled?(Livebook.Runtime.Embedded) do
raise "runtime module not allowed"
end
- if connected?(socket) do
- Session.subscribe(session.id)
- end
-
- {:ok, assign(socket, session: session, current_runtime: current_runtime)}
+ {:ok, socket}
end
@impl true
@@ -31,7 +21,7 @@ defmodule LivebookWeb.SessionLive.EmbeddedLive do
This is reserved for specific cases where there is no option
of starting a separate Elixir runtime (for example, on embedded
devices or cases where the amount of memory available is
- limited). Prefer the "Elixir standalone" runtime whenever possible.
+ limited). Prefer the "Standalone" runtime whenever possible.
- <%= if Runtime.connected?(@data_view.runtime) do %>
- <.button phx-click="reconnect_runtime">
- <.remix_icon icon="wireless-charging-line" />
- Reconnect
-
- <% else %>
- <.button phx-click="connect_runtime">
- <.remix_icon icon="wireless-charging-line" />
- Connect
-
- <% end %>
+ <.button :if={@data_view.runtime_status == :disconnected} phx-click="connect_runtime">
+ <.remix_icon icon="wireless-charging-line" />
+ Connect
+
+ <.button :if={@data_view.runtime_status == :connecting} disabled>
+ <.remix_icon icon="wireless-charging-line" />
+ Connecting...
+
+ <.button :if={@data_view.runtime_status == :connected} phx-click="reconnect_runtime">
+ <.remix_icon icon="wireless-charging-line" />
+ Reconnect
+
<.button color="gray" outlined patch={~p"/sessions/#{@session.id}/settings/runtime"}>
Configure
<.button
- :if={Runtime.connected?(@data_view.runtime)}
+ :if={@data_view.runtime_status == :connected}
color="red"
outlined
type="button"
@@ -679,6 +682,15 @@ defmodule LivebookWeb.SessionLive.Render do
+
-
-
- Memory
-
-
- <%= format_bytes(@memory_usage.system.free) %> available
-
+
+ Memory
<%= if uses_memory?(@memory_usage) do %>
<.runtime_memory_info memory_usage={@memory_usage} />
@@ -1003,7 +1010,7 @@ defmodule LivebookWeb.SessionLive.Render do
session_id={@session_id}
/>
<.runtime_indicator
- runtime={@runtime}
+ runtime_status={@runtime_status}
global_status={@global_status}
session_id={@session_id}
/>
@@ -1133,9 +1140,7 @@ defmodule LivebookWeb.SessionLive.Render do
defp runtime_indicator(assigns) do
~H"""
- <%= if Livebook.Runtime.connected?(@runtime) do %>
- <.global_status status={elem(@global_status, 0)} cell_id={elem(@global_status, 1)} />
- <% else %>
+ <%= if @runtime_status == :disconnected do %>
<.link
patch={~p"/sessions/#{@session_id}/settings/runtime"}
@@ -1145,6 +1150,8 @@ defmodule LivebookWeb.SessionLive.Render do
<.remix_icon icon="loader-3-line" />
+ <% else %>
+ <.global_status status={elem(@global_status, 0)} cell_id={elem(@global_status, 1)} />
<% end %>
"""
end
@@ -1344,7 +1351,7 @@ defmodule LivebookWeb.SessionLive.Render do
session_id={@session.id}
session_pid={@session.pid}
client_id={@client_id}
- runtime={@data_view.runtime}
+ runtime_status={@data_view.runtime_status}
smart_cell_definitions={@data_view.smart_cell_definitions}
example_snippet_definitions={@data_view.example_snippet_definitions}
installing?={@data_view.installing?}
diff --git a/lib/livebook_web/live/session_live/runtime_component.ex b/lib/livebook_web/live/session_live/runtime_component.ex
index a60ab014d..f6985d9c9 100644
--- a/lib/livebook_web/live/session_live/runtime_component.ex
+++ b/lib/livebook_web/live/session_live/runtime_component.ex
@@ -5,20 +5,21 @@ defmodule LivebookWeb.SessionLive.RuntimeComponent do
@impl true
def mount(socket) do
- {:ok, assign(socket, type: nil)}
+ {:ok, assign(socket, error_message: nil)}
end
@impl true
- def update(assigns, socket) do
- assigns =
- if socket.assigns.type == nil do
- type = runtime_type(assigns.runtime)
- Map.put(assigns, :type, type)
- else
- assigns
- end
+ def update(%{event: {:error, message}}, socket) do
+ {:ok, assign(socket, error_message: message)}
+ end
- {:ok, assign(socket, assigns)}
+ def update(assigns, socket) do
+ socket =
+ socket
+ |> assign(assigns)
+ |> assign_new(:type, fn -> runtime_type(assigns.runtime) end)
+
+ {:ok, socket}
end
@impl true
@@ -31,13 +32,13 @@ defmodule LivebookWeb.SessionLive.RuntimeComponent do
<.choice_button
- :if={Livebook.Config.runtime_enabled?(Livebook.Runtime.ElixirStandalone)}
- active={@type == "elixir_standalone"}
+ :if={Livebook.Config.runtime_enabled?(Livebook.Runtime.Standalone)}
+ active={@type == "standalone"}
phx-click="set_runtime_type"
- phx-value-type="elixir_standalone"
+ phx-value-type="standalone"
phx-target={@myself}
>
- Elixir standalone
+ Standalone
<.choice_button
:if={Livebook.Config.runtime_enabled?(Livebook.Runtime.Attached)}
@@ -57,25 +58,46 @@ defmodule LivebookWeb.SessionLive.RuntimeComponent do
>
Embedded
+ <.choice_button
+ :if={Livebook.Config.runtime_enabled?(Livebook.Runtime.Fly)}
+ active={@type == "fly"}
+ phx-click="set_runtime_type"
+ phx-value-type="fly"
+ phx-target={@myself}
+ >
+ Fly.io machine
+
+
+
+ <%= @error_message %>
- <%= live_render(@socket, live_view_for_type(@type),
- id: "runtime-config-#{@type}",
- session: %{"session_pid" => @session.pid, "current_runtime" => @runtime}
- ) %>
+ <.live_component
+ id={"runtime-config-#{@type}"}
+ module={component_for_type(@type)}
+ session={@session}
+ runtime={@runtime}
+ runtime_status={@runtime_status}
+ runtime_connect_info={@runtime_connect_info}
+ />
"""
end
- defp runtime_type(%Runtime.ElixirStandalone{}), do: "elixir_standalone"
+ defp runtime_type(%Runtime.Standalone{}), do: "standalone"
defp runtime_type(%Runtime.Attached{}), do: "attached"
defp runtime_type(%Runtime.Embedded{}), do: "embedded"
+ defp runtime_type(%Runtime.Fly{}), do: "fly"
- defp live_view_for_type("elixir_standalone"), do: LivebookWeb.SessionLive.ElixirStandaloneLive
- defp live_view_for_type("attached"), do: LivebookWeb.SessionLive.AttachedLive
- defp live_view_for_type("embedded"), do: LivebookWeb.SessionLive.EmbeddedLive
+ defp component_for_type("standalone"), do: LivebookWeb.SessionLive.StandaloneRuntimeComponent
+ defp component_for_type("attached"), do: LivebookWeb.SessionLive.AttachedRuntimeComponent
+ defp component_for_type("embedded"), do: LivebookWeb.SessionLive.EmbeddedRuntimeComponent
+ defp component_for_type("fly"), do: LivebookWeb.SessionLive.FlyRuntimeComponent
@impl true
def handle_event("set_runtime_type", %{"type" => type}, socket) do
diff --git a/lib/livebook_web/live/session_live/section_component.ex b/lib/livebook_web/live/session_live/section_component.ex
index b7e30e720..d6da38a79 100644
--- a/lib/livebook_web/live/session_live/section_component.ex
+++ b/lib/livebook_web/live/session_live/section_component.ex
@@ -147,7 +147,7 @@ defmodule LivebookWeb.SessionLive.SectionComponent do
persistent={@section_view.cell_views == []}
smart_cell_definitions={@smart_cell_definitions}
example_snippet_definitions={@example_snippet_definitions}
- runtime={@runtime}
+ runtime_status={@runtime_status}
section_id={@section_view.id}
cell_id={nil}
session_id={@session_id}
@@ -160,7 +160,7 @@ defmodule LivebookWeb.SessionLive.SectionComponent do
session_id={@session_id}
session_pid={@session_pid}
client_id={@client_id}
- runtime={@runtime}
+ runtime_status={@runtime_status}
installing?={@installing?}
allowed_uri_schemes={@allowed_uri_schemes}
cell_view={cell_view}
@@ -171,7 +171,7 @@ defmodule LivebookWeb.SessionLive.SectionComponent do
persistent={false}
smart_cell_definitions={@smart_cell_definitions}
example_snippet_definitions={@example_snippet_definitions}
- runtime={@runtime}
+ runtime_status={@runtime_status}
section_id={@section_view.id}
cell_id={cell_view.id}
session_id={@session_id}
diff --git a/lib/livebook_web/live/session_live/standalone_runtime_component.ex b/lib/livebook_web/live/session_live/standalone_runtime_component.ex
new file mode 100644
index 000000000..d0ed848d0
--- /dev/null
+++ b/lib/livebook_web/live/session_live/standalone_runtime_component.ex
@@ -0,0 +1,41 @@
+defmodule LivebookWeb.SessionLive.StandaloneRuntimeComponent do
+ use LivebookWeb, :live_component
+
+ alias Livebook.{Session, Runtime}
+
+ @impl true
+ def mount(socket) do
+ unless Livebook.Config.runtime_enabled?(Livebook.Runtime.Standalone) do
+ raise "runtime module not allowed"
+ end
+
+ {:ok, socket}
+ end
+
+ @impl true
+ def render(assigns) do
+ ~H"""
+
+
+ Start a new local Elixir node to evaluate code. Whenever you reconnect this runtime,
+ a fresh node is started.
+
+ <.button phx-click="init" phx-target={@myself} disabled={@runtime_status == :connecting}>
+ <%= label(@runtime, @runtime_status) %>
+
+
+ """
+ end
+
+ defp label(%Runtime.Standalone{}, :connecting), do: "Connecting..."
+ defp label(%Runtime.Standalone{}, :connected), do: "Reconnect"
+ defp label(_runtime, _runtime_status), do: "Connect"
+
+ @impl true
+ def handle_event("init", _params, socket) do
+ runtime = Runtime.Standalone.new()
+ Session.set_runtime(socket.assigns.session.pid, runtime)
+ Session.connect_runtime(socket.assigns.session.pid)
+ {:noreply, socket}
+ end
+end
diff --git a/mix.exs b/mix.exs
index 920237137..9987f1f4e 100644
--- a/mix.exs
+++ b/mix.exs
@@ -75,7 +75,8 @@ defmodule Livebook.MixProject do
defp escript do
[
main_module: LivebookCLI,
- app: nil
+ app: nil,
+ emu_args: "-epmd_module Elixir.Livebook.EPMD"
]
end
@@ -121,7 +122,7 @@ defmodule Livebook.MixProject do
{:bypass, "~> 2.1", only: :test},
# ZTA deps
{:jose, "~> 1.11.5"},
- {:req, "~> 0.4.4"},
+ {:req, "~> 0.5.2"},
# Docs
{:ex_doc, "~> 0.30", only: :dev, runtime: false}
]
@@ -163,7 +164,7 @@ defmodule Livebook.MixProject do
include_executables_for: [:unix, :windows],
include_erts: false,
rel_templates_path: "rel/server",
- steps: [:assemble, &remove_cookie/1]
+ steps: [:assemble, &remove_cookie/1, &write_runtime_modules/1]
],
app: [
applications: @release_apps,
@@ -179,10 +180,33 @@ defmodule Livebook.MixProject do
end
defp remove_cookie(release) do
+ # We remove the COOKIE file when assembling the release, because we
+ # don't want to share the same cookie across users.
+
File.rm!(Path.join(release.path, "releases/COOKIE"))
release
end
+ defp write_runtime_modules(release) do
+ # We copy the subset of Livebook modules that are injected into
+ # the runtime node. See overlays/bin/server for more details
+
+ app = release.applications[:livebook]
+
+ source = Path.join([release.path, "lib", "livebook-#{app[:vsn]}", "ebin"])
+ destination = Path.join([release.path, "lib", "livebook_runtime_ebin"])
+
+ File.mkdir_p!(destination)
+
+ for module <- Livebook.Runtime.ErlDist.required_modules() do
+ from = Path.join(source, "#{module}.beam")
+ to = Path.join(destination, "#{module}.beam")
+ File.cp!(from, to)
+ end
+
+ release
+ end
+
@compile {:no_warn_undefined, Standalone}
defp standalone_erlang_elixir(release) do
diff --git a/mix.lock b/mix.lock
index b05bd2e85..4158e446b 100644
--- a/mix.lock
+++ b/mix.lock
@@ -24,7 +24,7 @@
"makeup": {:hex, :makeup, "1.1.1", "fa0bc768698053b2b3869fa8a62616501ff9d11a562f3ce39580d60860c3a55e", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5dc62fbdd0de44de194898b6710692490be74baa02d9d108bc29f007783b0b48"},
"makeup_elixir": {:hex, :makeup_elixir, "0.16.2", "627e84b8e8bf22e60a2579dad15067c755531fea049ae26ef1020cad58fe9578", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "41193978704763f6bbe6cc2758b84909e62984c7752b3784bd3c218bb341706b"},
"makeup_erlang": {:hex, :makeup_erlang, "1.0.0", "6f0eff9c9c489f26b69b61440bf1b238d95badae49adac77973cbacae87e3c2e", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "ea7a9307de9d1548d2a72d299058d1fd2339e3d398560a0e46c27dab4891e4d2"},
- "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"},
+ "mime": {:hex, :mime, "2.0.6", "8f18486773d9b15f95f4f4f1e39b710045fa1de891fada4516559967276e4dc2", [:mix], [], "hexpm", "c9945363a6b26d747389aac3643f8e0e09d30499a138ad64fe8fd1d13d9b153e"},
"mint": {:hex, :mint, "1.6.1", "065e8a5bc9bbd46a41099dfea3e0656436c5cbcb6e741c80bd2bad5cd872446f", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "4fc518dcc191d02f433393a72a7ba3f6f94b101d094cb6bf532ea54c89423780"},
"mint_web_socket": {:hex, :mint_web_socket, "1.0.4", "0b539116dbb3d3f861cdf5e15e269a933cb501c113a14db7001a3157d96ffafd", [:mix], [{:mint, ">= 1.4.1 and < 2.0.0-0", [hex: :mint, repo: "hexpm", optional: false]}], "hexpm", "027d4c5529c45a4ba0ce27a01c0f35f284a5468519c045ca15f43decb360a991"},
"nimble_options": {:hex, :nimble_options, "1.1.0", "3b31a57ede9cb1502071fade751ab0c7b8dbe75a9a4c2b5bbb0943a690b63172", [:mix], [], "hexpm", "8bbbb3941af3ca9acc7835f5655ea062111c9c27bcac53e004460dfd19008a99"},
@@ -44,7 +44,7 @@
"plug_crypto": {:hex, :plug_crypto, "2.1.0", "f44309c2b06d249c27c8d3f65cfe08158ade08418cf540fd4f72d4d6863abb7b", [:mix], [], "hexpm", "131216a4b030b8f8ce0f26038bc4421ae60e4bb95c5cf5395e1421437824c4fa"},
"protobuf": {:hex, :protobuf, "0.12.0", "58c0dfea5f929b96b5aa54ec02b7130688f09d2de5ddc521d696eec2a015b223", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "75fa6cbf262062073dd51be44dd0ab940500e18386a6c4e87d5819a58964dc45"},
"ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"},
- "req": {:hex, :req, "0.4.14", "103de133a076a31044e5458e0f850d5681eef23dfabf3ea34af63212e3b902e2", [:mix], [{:aws_signature, "~> 0.3.2", [hex: :aws_signature, repo: "hexpm", optional: true]}, {:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:nimble_ownership, "~> 0.2.0 or ~> 0.3.0", [hex: :nimble_ownership, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "2ddd3d33f9ab714ced8d3c15fd03db40c14dbf129003c4a3eb80fac2cc0b1b08"},
+ "req": {:hex, :req, "0.5.2", "70b4976e5fbefe84e5a57fd3eea49d4e9aa0ac015301275490eafeaec380f97f", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "0c63539ab4c2d6ced6114d2684276cef18ac185ee00674ee9af4b1febba1f986"},
"telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"},
"telemetry_metrics": {:hex, :telemetry_metrics, "1.0.0", "29f5f84991ca98b8eb02fc208b2e6de7c95f8bb2294ef244a176675adc7775df", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f23713b3847286a534e005126d4c959ebcca68ae9582118ce436b521d1d47d5d"},
"telemetry_poller": {:hex, :telemetry_poller, "1.1.0", "58fa7c216257291caaf8d05678c8d01bd45f4bdbc1286838a28c4bb62ef32999", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9eb9d9cbfd81cbd7cdd24682f8711b6e2b691289a0de6826e58452f28c103c8f"},
diff --git a/rel/app/env.bat.eex b/rel/app/env.bat.eex
index 0844f5a6e..a48c90989 100644
--- a/rel/app/env.bat.eex
+++ b/rel/app/env.bat.eex
@@ -2,14 +2,6 @@ if exist "!USERPROFILE!\.livebookdesktop.bat" (
call "!USERPROFILE!\.livebookdesktop.bat"
)
-if not defined LIVEBOOK_EPMDLESS set LIVEBOOK_EPMDLESS=true
-if "!LIVEBOOK_EPMDLESS!"=="1" goto epmdless
-if "!LIVEBOOK_EPMDLESS!"=="true" goto epmdless
-goto continue
-:epmdless
-set ELIXIR_ERL_OPTIONS=!ELIXIR_ERL_OPTIONS! -epmd_module Elixir.Livebook.EPMD -start_epmd false -erl_epmd_port 0
-:continue
-
set RELEASE_MODE=interactive
set RELEASE_DISTRIBUTION=none
diff --git a/rel/app/env.sh.eex b/rel/app/env.sh.eex
index 51a377f52..9e6c8506b 100644
--- a/rel/app/env.sh.eex
+++ b/rel/app/env.sh.eex
@@ -2,11 +2,6 @@ if [ -f "$HOME/.livebookdesktop.sh" ]; then
. "$HOME/.livebookdesktop.sh"
fi
-export LIVEBOOK_EPMDLESS=${LIVEBOOK_EPMDLESS:-true}
-if [ "$LIVEBOOK_EPMDLESS" = "true" ] || [ "$LIVEBOOK_EPMDLESS" = "1" ]; then
- export ELIXIR_ERL_OPTIONS="${ELIXIR_ERL_OPTIONS} -epmd_module Elixir.Livebook.EPMD -start_epmd false -erl_epmd_port 0"
-fi
-
export RELEASE_MODE="interactive"
export RELEASE_DISTRIBUTION="none"
diff --git a/rel/app/vm.args.eex b/rel/app/vm.args.eex
index abd39f5ab..904264a59 100644
--- a/rel/app/vm.args.eex
+++ b/rel/app/vm.args.eex
@@ -1,3 +1,4 @@
# Disable busy waiting so that we don't waste resources
# Limit the maximal number of ports for the same reason
-+sbwt none +sbwtdcpu none +sbwtdio none +Q 65536
+# Set the custom EPMD module
++sbwt none +sbwtdcpu none +sbwtdio none +Q 65536 -epmd_module Elixir.Livebook.EPMD
diff --git a/rel/server/env.bat.eex b/rel/server/env.bat.eex
index bc8541736..56501aace 100644
--- a/rel/server/env.bat.eex
+++ b/rel/server/env.bat.eex
@@ -2,13 +2,6 @@ if exist "!RELEASE_ROOT!\user\env.bat" (
call "!RELEASE_ROOT!\user\env.bat"
)
-if "!LIVEBOOK_EPMDLESS!"=="1" goto epmdless
-if "!LIVEBOOK_EPMDLESS!"=="true" goto epmdless
-goto continue
-:epmdless
-set ELIXIR_ERL_OPTIONS=!ELIXIR_ERL_OPTIONS! -epmd_module Elixir.Livebook.EPMD -start_epmd false -erl_epmd_port 0
-:continue
-
set RELEASE_MODE=interactive
set RELEASE_DISTRIBUTION=none
@@ -19,3 +12,5 @@ if not defined RELEASE_COOKIE (
for /f "skip=1" %%X in ('wmic os get localdatetime') do if not defined TIMESTAMP set TIMESTAMP=%%X
set RELEASE_COOKIE=cookie-!TIMESTAMP:~0,11!-!RANDOM!
)
+
+cd !HOMEDRIVE!!HOMEPATH!
diff --git a/rel/server/env.sh.eex b/rel/server/env.sh.eex
index a077aeeeb..2ab9e1e87 100644
--- a/rel/server/env.sh.eex
+++ b/rel/server/env.sh.eex
@@ -18,10 +18,6 @@ if [ -f "${RELEASE_ROOT}/user/env.sh" ]; then
. "${RELEASE_ROOT}/user/env.sh"
fi
-if [ "$LIVEBOOK_EPMDLESS" = "true" ] || [ "$LIVEBOOK_EPMDLESS" = "1" ]; then
- export ELIXIR_ERL_OPTIONS="${ELIXIR_ERL_OPTIONS} -epmd_module Elixir.Livebook.EPMD -start_epmd false -erl_epmd_port 0"
-fi
-
export RELEASE_MODE="interactive"
export RELEASE_DISTRIBUTION="none"
@@ -39,3 +35,5 @@ if [ ! -z "${LIVEBOOK_COOKIE}" ]; then export RELEASE_COOKIE=${LIVEBOOK_COOKIE};
# a fixed value. Note that this value is overriden on boot, so other
# than being the initial node cookie, we don't really use it.
export RELEASE_COOKIE="${RELEASE_COOKIE:-$(cat /dev/urandom | env LC_ALL=C tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)}"
+
+cd $HOME
diff --git a/rel/server/overlays/bin/server b/rel/server/overlays/bin/server
index 0c5d34ff2..8e0d70b81 100755
--- a/rel/server/overlays/bin/server
+++ b/rel/server/overlays/bin/server
@@ -3,9 +3,23 @@ set -e
cd -P -- "$(dirname -- "$0")"
+# Livebook does not start EPMD automatically, but we want to start it
+# here, becasue we need it for clustering
+epmd -daemon
+
if [ -n "${FLAME_PARENT}" ]; then
- epmd -daemon
- elixir ./start_flame.exs
+ exec elixir ./start_flame.exs
+elif [ -n "${LIVEBOOK_RUNTIME}" ]; then
+ # Note: keep the flags in sync with the standalone runtime
+ erl_flags="+sbwt none +sbwtdcpu none +sbwtdio none +sssdio 128 -elixir ansi_enabled true -noinput"
+
+ # We add Livebook modules to the path, so that they are loaded from
+ # from disk, rather than having module binaries sent from the parent
+ # node. This cuts down the initialization time.
+ livebook_beams="$(dirname -- "$(pwd)")/lib/livebook_runtime_ebin"
+ erl_flags="$erl_flags -pa $livebook_beams"
+
+ exec elixir --erl "$erl_flags" ./start_runtime.exs
else
exec ./livebook start
fi
diff --git a/rel/server/overlays/bin/start_flame.exs b/rel/server/overlays/bin/start_flame.exs
index d4135529b..be0d85319 100644
--- a/rel/server/overlays/bin/start_flame.exs
+++ b/rel/server/overlays/bin/start_flame.exs
@@ -1,3 +1,5 @@
+File.cd!(System.fetch_env!("HOME"))
+
flame_parent = System.fetch_env!("FLAME_PARENT") |> Base.decode64!() |> :erlang.binary_to_term()
%{
diff --git a/rel/server/overlays/bin/start_runtime.exs b/rel/server/overlays/bin/start_runtime.exs
new file mode 100644
index 000000000..c4e44e623
--- /dev/null
+++ b/rel/server/overlays/bin/start_runtime.exs
@@ -0,0 +1,42 @@
+File.cd!(System.fetch_env!("HOME"))
+
+%{
+ node_base: node_base,
+ cookie: cookie,
+ dist_port: dist_port
+} = System.fetch_env!("LIVEBOOK_RUNTIME") |> Base.decode64!() |> :erlang.binary_to_term()
+
+# This is the only Fly-specific part of starting Livebook as runtime
+app = System.fetch_env!("FLY_APP_NAME")
+machine_id = System.fetch_env!("FLY_MACHINE_ID")
+node = :"#{node_base}@#{machine_id}.vm.#{app}.internal"
+
+# We persist the information before the node is reachable
+:persistent_term.put(:livebook_runtime_info, %{
+ pid: self(),
+ elixir_version: System.version()
+})
+
+Application.put_env(:kernel, :inet_dist_listen_min, dist_port)
+Application.put_env(:kernel, :inet_dist_listen_max, dist_port)
+
+{:ok, _} = :net_kernel.start(node, %{name_domain: :longnames, hidden: true})
+Node.set_cookie(cookie)
+
+IO.puts("Runtime node started, waiting for the parent finish initialization")
+
+receive do
+ :node_initialized ->
+ manager_ref = Process.monitor(Livebook.Runtime.ErlDist.NodeManager)
+
+ receive do
+ {:DOWN, ^manager_ref, :process, _object, _reason} -> :ok
+ end
+
+ IO.puts("The owner disconnected from the runtime, shutting down")
+after
+ 20_000 ->
+ IO.puts(:stderr, "No node initialization within 20s, shutting down")
+end
+
+System.halt()
diff --git a/rel/server/vm.args.eex b/rel/server/vm.args.eex
index abd39f5ab..904264a59 100644
--- a/rel/server/vm.args.eex
+++ b/rel/server/vm.args.eex
@@ -1,3 +1,4 @@
# Disable busy waiting so that we don't waste resources
# Limit the maximal number of ports for the same reason
-+sbwt none +sbwtdcpu none +sbwtdio none +Q 65536
+# Set the custom EPMD module
++sbwt none +sbwtdcpu none +sbwtdio none +Q 65536 -epmd_module Elixir.Livebook.EPMD
diff --git a/test/livebook/epmd_test.exs b/test/livebook/epmd_test.exs
index 2508b49e3..bf25e2261 100644
--- a/test/livebook/epmd_test.exs
+++ b/test/livebook/epmd_test.exs
@@ -1,18 +1,7 @@
defmodule Livebook.EPMDTest do
use ExUnit.Case, async: true
- describe "with epmd" do
- @describetag :with_epmd
- test "has a random dist port" do
- assert Livebook.EPMD.dist_port() == 0
- end
- end
-
- describe "without epmd" do
- @describetag :without_epmd
-
- test "has a custom dist port" do
- assert Livebook.EPMD.dist_port() != 0
- end
+ test "has a custom dist port" do
+ assert Livebook.EPMD.dist_port() != 0
end
end
diff --git a/test/livebook/hubs/dockerfile_test.exs b/test/livebook/hubs/dockerfile_test.exs
index 2507d3a1e..559016f22 100644
--- a/test/livebook/hubs/dockerfile_test.exs
+++ b/test/livebook/hubs/dockerfile_test.exs
@@ -7,9 +7,9 @@ defmodule Livebook.Hubs.DockerfileTest do
alias Livebook.Hubs
alias Livebook.Secrets.Secret
- @docker_tag if Livebook.Config.app_version() =~ "-dev",
- do: "latest",
- else: Livebook.Config.app_version()
+ @versions if Livebook.Config.app_version() =~ "-dev",
+ do: %{base: "edge", cuda: "latest"},
+ else: %{base: Livebook.Config.app_version(), cuda: Livebook.Config.app_version()}
describe "airgapped_dockerfile/7" do
test "deploying a single notebook in personal hub" do
@@ -20,7 +20,7 @@ defmodule Livebook.Hubs.DockerfileTest do
dockerfile = Dockerfile.airgapped_dockerfile(config, hub, [], [], file, [], %{})
assert dockerfile == """
- FROM ghcr.io/livebook-dev/livebook:#{@docker_tag}
+ FROM ghcr.io/livebook-dev/livebook:#{@versions.base}
# Apps configuration
ENV LIVEBOOK_APPS_PATH "/apps"
@@ -97,7 +97,7 @@ defmodule Livebook.Hubs.DockerfileTest do
dockerfile = Dockerfile.airgapped_dockerfile(config, hub, [], [], file, [], %{})
assert dockerfile == """
- FROM ghcr.io/livebook-dev/livebook:#{@docker_tag}
+ FROM ghcr.io/livebook-dev/livebook:#{@versions.base}
ARG TEAMS_KEY="lb_tk_fn0pL3YLWzPoPFWuHeV3kd0o7_SFuIOoU4C_k6OWDYg"
@@ -166,14 +166,14 @@ defmodule Livebook.Hubs.DockerfileTest do
end
test "deploying with different base image" do
- config = dockerfile_config(%{docker_tag: "#{@docker_tag}-cuda11.8"})
+ config = dockerfile_config(%{docker_tag: "#{@versions.cuda}-cuda11.8"})
hub = personal_hub()
file = Livebook.FileSystem.File.local(p("/notebook.livemd"))
dockerfile = Dockerfile.airgapped_dockerfile(config, hub, [], [], file, [], %{})
assert dockerfile =~ """
- FROM ghcr.io/livebook-dev/livebook:#{@docker_tag}-cuda11.8
+ FROM ghcr.io/livebook-dev/livebook:#{@versions.cuda}-cuda11.8
ENV XLA_TARGET "cuda118"
"""
@@ -247,13 +247,13 @@ defmodule Livebook.Hubs.DockerfileTest do
end
test "deploying with different base image" do
- config = dockerfile_config(%{docker_tag: "#{@docker_tag}-cuda11.8"})
+ config = dockerfile_config(%{docker_tag: "#{@versions.cuda}-cuda11.8"})
hub = team_hub()
agent_key = Livebook.Factory.build(:agent_key)
%{image: image, env: env} = Dockerfile.online_docker_info(config, hub, agent_key)
- assert image == "ghcr.io/livebook-dev/livebook:#{@docker_tag}-cuda11.8"
+ assert image == "ghcr.io/livebook-dev/livebook:#{@versions.cuda}-cuda11.8"
assert {"XLA_TARGET", "cuda118"} in env
end
diff --git a/test/livebook/intellisense_test.exs b/test/livebook/intellisense_test.exs
index 87a861ff2..9b3f3b360 100644
--- a/test/livebook/intellisense_test.exs
+++ b/test/livebook/intellisense_test.exs
@@ -1863,7 +1863,8 @@ defmodule Livebook.IntellisenseTest do
# in the past we used :peer.start, but it was often failing on CI
# (the start was timing out)
- {:ok, runtime} = Livebook.Runtime.ElixirStandalone.new() |> Livebook.Runtime.connect()
+ pid = Livebook.Runtime.Standalone.new() |> Livebook.Runtime.connect()
+ assert_receive {:runtime_connect_done, ^pid, {:ok, runtime}}
parent = self()
diff --git a/test/livebook/runtime/attached_test.exs b/test/livebook/runtime/attached_test.exs
index 9b6af61fa..8fe171084 100644
--- a/test/livebook/runtime/attached_test.exs
+++ b/test/livebook/runtime/attached_test.exs
@@ -6,7 +6,10 @@ defmodule Livebook.Runtime.AttachedTest do
describe "Runtime.connect/1" do
test "given an invalid node returns an error" do
runtime = Runtime.Attached.new(:nonexistent@node)
- assert {:error, "node :nonexistent@node is unreachable"} = Runtime.connect(runtime)
+ pid = Runtime.connect(runtime)
+
+ assert_receive {:runtime_connect_done, ^pid,
+ {:error, "node :nonexistent@node is unreachable"}}
end
end
end
diff --git a/test/livebook/runtime/erl_dist/node_manager_test.exs b/test/livebook/runtime/erl_dist/node_manager_test.exs
index 85d285c85..b2606d575 100644
--- a/test/livebook/runtime/erl_dist/node_manager_test.exs
+++ b/test/livebook/runtime/erl_dist/node_manager_test.exs
@@ -7,15 +7,16 @@ defmodule Livebook.Runtime.ErlDist.NodeManagerTest do
test "terminates when the last runtime server terminates" do
# We use a standalone runtime, so that we have an isolated node
# with its own node manager
- assert {:ok, %{node: node, server_pid: server1} = runtime} =
- Runtime.ElixirStandalone.new() |> Runtime.connect()
+ pid = Runtime.Standalone.new() |> Runtime.connect()
+ assert_receive {:runtime_connect_done, ^pid, {:ok, runtime}}
+ %{node: node, server_pid: server1} = runtime
Runtime.take_ownership(runtime)
manager_pid = :erpc.call(node, Process, :whereis, [Livebook.Runtime.ErlDist.NodeManager])
ref = Process.monitor(manager_pid)
- server2 = NodeManager.start_runtime_server(node)
+ {:ok, server2} = NodeManager.start_runtime_server(node)
RuntimeServer.stop(server1)
RuntimeServer.stop(server2)
diff --git a/test/livebook/runtime/erl_dist/runtime_server_test.exs b/test/livebook/runtime/erl_dist/runtime_server_test.exs
index 0f8a8d159..a41ce8846 100644
--- a/test/livebook/runtime/erl_dist/runtime_server_test.exs
+++ b/test/livebook/runtime/erl_dist/runtime_server_test.exs
@@ -4,7 +4,7 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServerTest do
alias Livebook.Runtime.ErlDist.{NodeManager, RuntimeServer}
setup ctx do
- runtime_server_pid = NodeManager.start_runtime_server(node(), ctx[:opts] || [])
+ {:ok, runtime_server_pid} = NodeManager.start_runtime_server(node(), ctx[:opts] || [])
RuntimeServer.attach(runtime_server_pid, self())
{:ok, %{pid: runtime_server_pid}}
end
@@ -24,7 +24,7 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServerTest do
end
end)
- pid = NodeManager.start_runtime_server(node())
+ {:ok, pid} = NodeManager.start_runtime_server(node())
RuntimeServer.attach(pid, owner)
# Make sure the node is running.
diff --git a/test/livebook/runtime/fly_test.exs b/test/livebook/runtime/fly_test.exs
new file mode 100644
index 000000000..d4c9e750c
--- /dev/null
+++ b/test/livebook/runtime/fly_test.exs
@@ -0,0 +1,93 @@
+defmodule Livebook.Runtime.FlyTest do
+ use ExUnit.Case, async: true
+
+ # To run these tests create a Fly app, generate deployment token,
+ # then set TEST_FLY_APP_NAME and TEST_FLY_API_TOKEN
+ @moduletag :fly
+
+ alias Livebook.Runtime
+
+ @assert_receive_timeout 10_000
+
+ setup do
+ Livebook.FlyAPI.passthrough()
+ :ok
+ end
+
+ test "connecting flow" do
+ fly = fly!()
+ config = config(%{token: fly.token, app_name: fly.app_name})
+
+ assert [] = fly_run(fly, ~w(machine list))
+
+ pid = Runtime.Fly.new(config) |> Runtime.connect()
+
+ Req.Test.allow(Livebook.FlyAPI, self(), pid)
+
+ assert_receive {:runtime_connect_info, ^pid, "create machine"}, @assert_receive_timeout
+ assert_receive {:runtime_connect_info, ^pid, "start proxy"}, @assert_receive_timeout
+ assert_receive {:runtime_connect_info, ^pid, "connect to node"}, @assert_receive_timeout
+ assert_receive {:runtime_connect_info, ^pid, "initialize node"}, @assert_receive_timeout
+ assert_receive {:runtime_connect_done, ^pid, {:ok, runtime}}, @assert_receive_timeout
+
+ Runtime.take_ownership(runtime)
+
+ assert [_] = fly_run(fly, ~w(machine list))
+
+ # Verify that we can actually evaluate code on the Fly machine
+ Runtime.evaluate_code(runtime, :elixir, ~s/System.fetch_env!("FLY_APP_NAME")/, {:c1, :e1}, [])
+ assert_receive {:runtime_evaluation_response, :e1, %{type: :terminal_text, text: text}, _meta}
+ assert text =~ fly.app_name
+
+ Runtime.disconnect(runtime)
+
+ # The machine should be automatically destroyed. Blocking in tests
+ # is bad, but this test suit is inherently time-consuming and it
+ # is opt-in anyway, so it is fine in this case.
+ Process.sleep(2000)
+
+ assert [] = fly_run(fly, ~w(machine list))
+ end
+
+ test "connecting fails with invalid token" do
+ fly = fly!()
+ config = config(%{token: "invalid", app_name: fly.app_name})
+
+ pid = Runtime.Fly.new(config) |> Runtime.connect()
+
+ Req.Test.allow(Livebook.FlyAPI, self(), pid)
+
+ assert_receive {:runtime_connect_done, ^pid, {:error, error}}, @assert_receive_timeout
+ assert error == "could not create machine, reason: authenticate: token validation error"
+ end
+
+ defp config(attrs) do
+ defaults = %{
+ token: nil,
+ app_name: nil,
+ region: "fra",
+ cpu_kind: "shared",
+ cpus: 1,
+ memory_gb: 1,
+ gpu_kind: nil,
+ gpus: nil,
+ volume_id: nil,
+ docker_tag: "edge"
+ }
+
+ Map.merge(defaults, attrs)
+ end
+
+ defp fly_run(fly, args) do
+ {output, 0} =
+ System.cmd("fly", args ++ ["--app", fly.app_name, "--access-token", fly.token, "--json"])
+
+ Jason.decode!(output)
+ end
+
+ defp fly!() do
+ token = System.fetch_env!("TEST_FLY_API_TOKEN")
+ app_name = System.fetch_env!("TEST_FLY_APP_NAME")
+ %{token: token, app_name: app_name}
+ end
+end
diff --git a/test/livebook/runtime/elixir_standalone_test.exs b/test/livebook/runtime/standalone_test.exs
similarity index 72%
rename from test/livebook/runtime/elixir_standalone_test.exs
rename to test/livebook/runtime/standalone_test.exs
index e03a8394b..83098fc4e 100644
--- a/test/livebook/runtime/elixir_standalone_test.exs
+++ b/test/livebook/runtime/standalone_test.exs
@@ -1,11 +1,13 @@
-defmodule Livebook.Runtime.ElixirStandaloneTest do
+defmodule Livebook.Runtime.StandaloneTest do
use ExUnit.Case, async: true
alias Livebook.Runtime
describe "Runtime.connect/1" do
test "starts a new Elixir runtime in distribution mode and ties its lifetime to the NodeManager process" do
- assert {:ok, %{node: node} = runtime} = Runtime.ElixirStandalone.new() |> Runtime.connect()
+ pid = Runtime.Standalone.new() |> Runtime.connect()
+ assert_receive {:runtime_connect_done, ^pid, {:ok, runtime}}
+ %{node: node} = runtime
Runtime.take_ownership(runtime)
# Make sure the node is running.
@@ -21,7 +23,9 @@ defmodule Livebook.Runtime.ElixirStandaloneTest do
end
test "loads necessary modules and starts manager process" do
- assert {:ok, %{node: node} = runtime} = Runtime.ElixirStandalone.new() |> Runtime.connect()
+ pid = Runtime.Standalone.new() |> Runtime.connect()
+ assert_receive {:runtime_connect_done, ^pid, {:ok, runtime}}
+ %{node: node} = runtime
Runtime.take_ownership(runtime)
assert evaluator_module_loaded?(node)
@@ -30,7 +34,9 @@ defmodule Livebook.Runtime.ElixirStandaloneTest do
end
test "Runtime.disconnect/1 makes the node terminate" do
- assert {:ok, %{node: node} = runtime} = Runtime.ElixirStandalone.new() |> Runtime.connect()
+ pid = Runtime.Standalone.new() |> Runtime.connect()
+ assert_receive {:runtime_connect_done, ^pid, {:ok, runtime}}
+ %{node: node} = runtime
Runtime.take_ownership(runtime)
# Make sure the node is running.
diff --git a/test/livebook/session/data_test.exs b/test/livebook/session/data_test.exs
index 7fa68795c..8118f2ec8 100644
--- a/test/livebook/session/data_test.exs
+++ b/test/livebook/session/data_test.exs
@@ -315,7 +315,7 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 2, "s3"},
{:insert_cell, @cid, "s3", 0, :code, "c3", %{}},
{:insert_cell, @cid, "s3", 1, :code, "c4", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2", "c3", "c4"],
uses: %{"c3" => ["c2"]}
)
@@ -343,9 +343,9 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s2", 0, :code, "c2", %{}},
{:insert_section, @cid, 2, "s3"},
{:insert_cell, @cid, "s3", 0, :code, "c3", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1"]),
- {:queue_cells_evaluation, @cid, ["c2", "c3"]}
+ {:queue_cells_evaluation, @cid, ["c2", "c3"], []}
])
operation = {:set_section_parent, @cid, "s2", "s1"}
@@ -431,7 +431,7 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 2, "s3"},
{:insert_cell, @cid, "s3", 0, :code, "c3", %{}},
{:set_section_parent, @cid, "s2", "s1"},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2", "c3"])
])
@@ -457,9 +457,9 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 2, "s3"},
{:insert_cell, @cid, "s3", 0, :code, "c3", %{}},
{:set_section_parent, @cid, "s2", "s1"},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1"]),
- {:queue_cells_evaluation, @cid, ["c2", "c3"]}
+ {:queue_cells_evaluation, @cid, ["c2", "c3"], []}
])
operation = {:unset_section_parent, @cid, "s2"}
@@ -524,7 +524,7 @@ defmodule Livebook.Session.DataTest do
data =
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
- {:set_runtime, @cid, connected_noop_runtime()}
+ connect_noop_runtime_operations()
])
operation = {:insert_cell, @cid, "s1", 0, :smart, "c1", %{kind: "text"}}
@@ -537,7 +537,7 @@ defmodule Livebook.Session.DataTest do
data =
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
{:set_smart_cell_definitions, @cid, @smart_cell_definitions}
])
@@ -554,7 +554,7 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :smart, "c2", %{kind: "text"}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
{:set_smart_cell_definitions, @cid, @smart_cell_definitions},
{:smart_cell_started, @cid, "c2", Delta.new(), nil, %{}, nil}
])
@@ -629,7 +629,7 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_section, @cid, 1, "s2"},
{:insert_cell, @cid, "s2", 0, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2"])
])
@@ -662,7 +662,7 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 1, "s2"},
{:insert_cell, @cid, "s2", 0, :code, "c2", %{}},
{:insert_cell, @cid, "s2", 1, :code, "c3", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2", "c3"], uses: %{"c2" => ["c1"]})
])
@@ -688,7 +688,7 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s3", 0, :code, "c3", %{}},
{:insert_cell, @cid, "s3", 1, :code, "c4", %{}},
{:set_section_parent, @cid, "s2", "s1"},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2", "c3", "c4"],
uses: %{"c3" => ["c2"]}
)
@@ -734,9 +734,9 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1", "c2"]}
+ {:queue_cells_evaluation, @cid, ["c1", "c2"], []}
])
operation = {:delete_cell, @cid, "c1"}
@@ -786,7 +786,7 @@ defmodule Livebook.Session.DataTest do
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1"])
])
@@ -802,9 +802,9 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1", "c2"]}
+ {:queue_cells_evaluation, @cid, ["c1", "c2"], []}
])
operation = {:delete_cell, @cid, "c2"}
@@ -821,7 +821,7 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
{:insert_cell, @cid, "s1", 2, :code, "c3", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2", "c3"],
uses: %{"c2" => ["c1"]}
)
@@ -845,7 +845,7 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
{:set_cell_attributes, @cid, "c2", %{reevaluate_automatically: true}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2"],
uses: %{"c2" => ["c1"]}
)
@@ -865,7 +865,7 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :markdown, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c2"])
])
@@ -882,7 +882,7 @@ defmodule Livebook.Session.DataTest do
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1"])
])
@@ -896,7 +896,7 @@ defmodule Livebook.Session.DataTest do
data =
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
{:set_smart_cell_definitions, @cid, @smart_cell_definitions},
{:insert_cell, @cid, "s1", 0, :smart, "c1", %{kind: "text"}},
{:smart_cell_started, @cid, "c1", Delta.new(), nil, %{}, nil}
@@ -914,11 +914,11 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :smart, "c2", %{kind: "text"}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
{:set_smart_cell_definitions, @cid, @smart_cell_definitions},
{:smart_cell_started, @cid, "c2", Delta.new(), nil, %{}, nil},
- {:queue_cells_evaluation, @cid, ["c1"]},
+ {:queue_cells_evaluation, @cid, ["c1"], []},
{:add_cell_evaluation_response, @cid, "c1", @eval_resp, eval_meta()}
])
@@ -937,9 +937,9 @@ defmodule Livebook.Session.DataTest do
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1"]},
+ {:queue_cells_evaluation, @cid, ["c1"], []},
{:add_cell_evaluation_response, @cid, "c1", @input, eval_meta()},
{:set_input_value, @cid, "i1", "value"}
])
@@ -959,9 +959,9 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1"]},
+ {:queue_cells_evaluation, @cid, ["c1"], []},
{:add_cell_evaluation_response, @cid, "c1", @input, eval_meta()},
evaluate_cells_operations(["c2"], %{bind_inputs: %{"c2" => ["i1"]}})
])
@@ -1047,7 +1047,7 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :smart, "c1", %{kind: "text"}},
{:delete_cell, @cid, "c1"},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
{:set_smart_cell_definitions, @cid, @smart_cell_definitions}
])
@@ -1087,9 +1087,9 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_section, @cid, 1, "s2"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1"]}
+ {:queue_cells_evaluation, @cid, ["c1"], []}
])
operation = {:move_cell, @cid, "c1", 1}
@@ -1174,7 +1174,7 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
{:insert_cell, @cid, "s1", 2, :code, "c3", %{}},
{:insert_cell, @cid, "s1", 3, :code, "c4", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2", "c3", "c4"],
uses: %{"c2" => ["c1"], "c3" => ["c2"], "c4" => ["c1"]}
)
@@ -1200,7 +1200,7 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
{:insert_cell, @cid, "s1", 2, :code, "c3", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2", "c3"],
uses: %{"c2" => :unknown}
)
@@ -1224,7 +1224,7 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :markdown, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1"])
])
@@ -1245,9 +1245,9 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
{:insert_cell, @cid, "s1", 2, :code, "c3", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1", "c2", "c3"]}
+ {:queue_cells_evaluation, @cid, ["c1", "c2", "c3"], []}
])
operation = {:move_cell, @cid, "c2", -1}
@@ -1272,7 +1272,7 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 2, "s3"},
{:insert_cell, @cid, "s3", 0, :code, "c4", %{}},
{:set_section_parent, @cid, "s2", "s1"},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2", "c3", "c4"],
uses: %{"c4" => ["c2"]}
)
@@ -1298,7 +1298,7 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
{:insert_cell, @cid, "s1", 2, :code, "c3", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2", "c3"],
uses: %{"c2" => ["c1"], "c3" => ["c1"]}
)
@@ -1416,7 +1416,7 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s3", 0, :code, "c3", %{}},
{:insert_cell, @cid, "s3", 1, :code, "c4", %{}},
{:insert_cell, @cid, "s3", 2, :code, "c5", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2", "c3", "c4", "c5"],
uses: %{"c2" => ["c1"], "c3" => ["c1"], "c4" => ["c2"]}
)
@@ -1445,9 +1445,9 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s2", 0, :code, "c2", %{}},
{:insert_section, @cid, 2, "s3"},
{:insert_cell, @cid, "s3", 0, :code, "c3", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1", "c2", "c3"]}
+ {:queue_cells_evaluation, @cid, ["c1", "c2", "c3"], []}
])
operation = {:move_section, @cid, "s2", -1}
@@ -1477,7 +1477,7 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 3, "s4"},
{:insert_cell, @cid, "s4", 0, :code, "c4", %{}},
{:set_section_parent, @cid, "s2", "s1"},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2", "c3", "c4"],
uses: %{"c2" => ["c3"]}
)
@@ -1500,13 +1500,13 @@ defmodule Livebook.Session.DataTest do
describe "apply_operation/2 given :queue_cells_evaluation" do
test "returns an error given an empty list of cells" do
data = Data.new()
- operation = {:queue_cells_evaluation, @cid, []}
+ operation = {:queue_cells_evaluation, @cid, [], []}
assert :error = Data.apply_operation(data, operation)
end
test "returns an error given invalid cell id" do
data = Data.new()
- operation = {:queue_cells_evaluation, @cid, ["nonexistent"]}
+ operation = {:queue_cells_evaluation, @cid, ["nonexistent"], []}
assert :error = Data.apply_operation(data, operation)
end
@@ -1517,7 +1517,7 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s1", 0, :markdown, "c1", %{}}
])
- operation = {:queue_cells_evaluation, @cid, ["c1"]}
+ operation = {:queue_cells_evaluation, @cid, ["c1"], []}
assert :error = Data.apply_operation(data, operation)
end
@@ -1526,12 +1526,12 @@ defmodule Livebook.Session.DataTest do
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1"]}
+ {:queue_cells_evaluation, @cid, ["c1"], []}
])
- operation = {:queue_cells_evaluation, @cid, ["c1"]}
+ operation = {:queue_cells_evaluation, @cid, ["c1"], []}
assert :error = Data.apply_operation(data, operation)
end
@@ -1542,7 +1542,7 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}}
])
- operation = {:queue_cells_evaluation, @cid, ["c1"]}
+ operation = {:queue_cells_evaluation, @cid, ["c1"], []}
assert {:ok,
%{
@@ -1561,10 +1561,10 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
- {:queue_cells_evaluation, @cid, ["c1"]}
+ {:queue_cells_evaluation, @cid, ["c1"], []}
])
- operation = {:queue_cells_evaluation, @cid, ["c2"]}
+ operation = {:queue_cells_evaluation, @cid, ["c2"], []}
assert {:ok,
%{
@@ -1585,11 +1585,11 @@ defmodule Livebook.Session.DataTest do
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"])
])
- operation = {:queue_cells_evaluation, @cid, ["c1"]}
+ operation = {:queue_cells_evaluation, @cid, ["c1"], []}
assert {:ok,
%{
@@ -1607,13 +1607,13 @@ defmodule Livebook.Session.DataTest do
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"])
])
- operation = {:queue_cells_evaluation, @cid, ["c1"]}
+ operation = {:queue_cells_evaluation, @cid, ["c1"], []}
- assert {:ok, _data, [{:start_evaluation, %{id: "c1"}, %{id: "s1"}}]} =
+ assert {:ok, _data, [{:start_evaluation, %{id: "c1"}, %{id: "s1"}, []}]} =
Data.apply_operation(data, operation)
end
@@ -1623,12 +1623,12 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1"]}
+ {:queue_cells_evaluation, @cid, ["c1"], []}
])
- operation = {:queue_cells_evaluation, @cid, ["c2"]}
+ operation = {:queue_cells_evaluation, @cid, ["c2"], []}
assert {:ok,
%{
@@ -1648,12 +1648,12 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_section, @cid, 1, "s2"},
{:insert_cell, @cid, "s2", 0, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1"]}
+ {:queue_cells_evaluation, @cid, ["c1"], []}
])
- operation = {:queue_cells_evaluation, @cid, ["c2"]}
+ operation = {:queue_cells_evaluation, @cid, ["c2"], []}
assert {:ok,
%{
@@ -1678,7 +1678,7 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s2", 0, :code, "c3", %{}},
{:insert_cell, @cid, "s2", 1, :code, "c4", %{}},
# Evaluate first 2 cells
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2"], uses: %{"c2" => ["c1"]}),
# Evaluate the first cell, so the second becomes stale
evaluate_cells_operations(["c1"], versions: %{"c1" => 1})
@@ -1696,7 +1696,7 @@ defmodule Livebook.Session.DataTest do
# Queuing cell 4 should also queue cell 3 and cell 2, so that
# they all become evaluated.
- operation = {:queue_cells_evaluation, @cid, ["c4"]}
+ operation = {:queue_cells_evaluation, @cid, ["c4"], []}
assert {:ok,
%{
@@ -1725,11 +1725,11 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 2, "s3"},
{:insert_cell, @cid, "s3", 0, :code, "c3", %{}},
{:set_section_parent, @cid, "s3", "s1"},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"])
])
- operation = {:queue_cells_evaluation, @cid, ["c3"]}
+ operation = {:queue_cells_evaluation, @cid, ["c3"], []}
# Cell 3 depends directly on cell 1, so cell 2 shouldn't be queued
@@ -1762,12 +1762,12 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 2, "s3"},
{:insert_cell, @cid, "s3", 0, :code, "c3", %{}},
{:set_section_parent, @cid, "s2", "s1"},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1"]),
- {:queue_cells_evaluation, @cid, ["c3"]}
+ {:queue_cells_evaluation, @cid, ["c3"], []}
])
- operation = {:queue_cells_evaluation, @cid, ["c2"]}
+ operation = {:queue_cells_evaluation, @cid, ["c2"], []}
assert {:ok,
%{
@@ -1793,11 +1793,11 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 1, "s2"},
{:insert_cell, @cid, "s2", 0, :code, "c2", %{}},
{:set_section_parent, @cid, "s2", "s1"},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1"])
])
- operation = {:queue_cells_evaluation, @cid, ["c2"]}
+ operation = {:queue_cells_evaluation, @cid, ["c2"], []}
assert {:ok,
%{
@@ -1806,7 +1806,7 @@ defmodule Livebook.Session.DataTest do
"s2" => %{evaluating_cell_id: "c2"}
}
} = new_data,
- [{:start_evaluation, %{id: "c2"}, %{id: "s2"}}]} =
+ [{:start_evaluation, %{id: "c2"}, %{id: "s2"}, []}]} =
Data.apply_operation(data, operation)
assert new_data.section_infos["s2"].evaluation_queue == MapSet.new([])
@@ -1823,12 +1823,12 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 2, "s3"},
{:insert_cell, @cid, "s3", 0, :code, "c4", %{}},
{:set_section_parent, @cid, "s2", "s1"},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2"]),
- {:queue_cells_evaluation, @cid, ["c4"]}
+ {:queue_cells_evaluation, @cid, ["c4"], []}
])
- operation = {:queue_cells_evaluation, @cid, ["c3"]}
+ operation = {:queue_cells_evaluation, @cid, ["c3"], []}
assert {:ok,
%{
@@ -1856,12 +1856,12 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 2, "s3"},
{:insert_cell, @cid, "s3", 0, :code, "c3", %{}},
{:set_section_parent, @cid, "s2", "s1"},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1"]),
- {:queue_cells_evaluation, @cid, ["c2"]}
+ {:queue_cells_evaluation, @cid, ["c2"], []}
])
- operation = {:queue_cells_evaluation, @cid, ["c3"]}
+ operation = {:queue_cells_evaluation, @cid, ["c3"], []}
assert {:ok,
%{
@@ -1873,6 +1873,57 @@ defmodule Livebook.Session.DataTest do
assert new_data.section_infos["s3"].evaluation_queue == MapSet.new([])
end
+
+ test "includes evaluation options in the evaluation action" do
+ data =
+ data_after_operations!([
+ {:insert_section, @cid, 0, "s1"},
+ {:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
+ connect_noop_runtime_operations(),
+ evaluate_cells_operations(["setup"])
+ ])
+
+ evaluation_opts = [disable_dependencies_cache: true]
+ operation = {:queue_cells_evaluation, @cid, ["c1"], evaluation_opts}
+
+ assert {:ok,
+ %{
+ cell_infos: %{
+ "c1" => %{eval: %{status: :evaluating, evaluation_opts: ^evaluation_opts}}
+ }
+ } = new_data,
+ [{:start_evaluation, %{id: "c1"}, %{id: "s1"}, ^evaluation_opts}]} =
+ Data.apply_operation(data, operation)
+
+ assert new_data.section_infos["s1"].evaluation_queue == MapSet.new([])
+ end
+
+ test "reconnects the runtime when the setup cell is reevaluated" do
+ data =
+ data_after_operations!([
+ {:insert_section, @cid, 0, "s1"},
+ {:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
+ connect_noop_runtime_operations(),
+ evaluate_cells_operations(["setup"])
+ ])
+
+ runtime = data.runtime
+
+ evaluation_opts = [disable_dependencies_cache: true]
+ operation = {:queue_cells_evaluation, @cid, ["setup"], evaluation_opts}
+
+ assert {:ok,
+ %{
+ runtime_status: :connecting,
+ cell_infos: %{
+ "setup" => %{eval: %{status: :queued, evaluation_opts: ^evaluation_opts}}
+ }
+ } = new_data,
+ [{:disconnect_runtime, ^runtime}, :connect_runtime]} =
+ Data.apply_operation(data, operation)
+
+ assert new_data.section_infos["s1"].evaluation_queue == MapSet.new([])
+ end
end
describe "apply_operation/2 given :add_cell_evaluation_output" do
@@ -1881,9 +1932,9 @@ defmodule Livebook.Session.DataTest do
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1"]}
+ {:queue_cells_evaluation, @cid, ["c1"], []}
])
operation = {:add_cell_evaluation_output, @cid, "c1", @stdout}
@@ -1905,7 +1956,7 @@ defmodule Livebook.Session.DataTest do
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1"])
])
@@ -1928,9 +1979,9 @@ defmodule Livebook.Session.DataTest do
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1"]},
+ {:queue_cells_evaluation, @cid, ["c1"], []},
{:set_notebook_attributes, @cid, %{persist_outputs: true}},
{:notebook_saved, @cid, []}
])
@@ -1945,9 +1996,9 @@ defmodule Livebook.Session.DataTest do
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1"]}
+ {:queue_cells_evaluation, @cid, ["c1"], []}
])
operation = {:add_cell_evaluation_output, @cid, "c1", @input}
@@ -1969,9 +2020,9 @@ defmodule Livebook.Session.DataTest do
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1"]}
+ {:queue_cells_evaluation, @cid, ["c1"], []}
])
operation = {:add_cell_evaluation_response, @cid, "c1", @eval_resp, eval_meta()}
@@ -1993,9 +2044,9 @@ defmodule Livebook.Session.DataTest do
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1"]}
+ {:queue_cells_evaluation, @cid, ["c1"], []}
])
operation = {:add_cell_evaluation_response, @cid, "c1", @eval_resp, eval_meta()}
@@ -2017,12 +2068,12 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
# Evaluate the first cell
evaluate_cells_operations(["c1"]),
# Start evaluating the second cell
- {:queue_cells_evaluation, @cid, ["c2"]},
+ {:queue_cells_evaluation, @cid, ["c2"], []},
# Remove the first cell, this should make the second cell stale
{:delete_cell, @cid, "c1"}
])
@@ -2041,9 +2092,9 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1", "c2"]}
+ {:queue_cells_evaluation, @cid, ["c1", "c2"], []}
])
operation = {:add_cell_evaluation_response, @cid, "c1", @eval_resp, eval_meta()}
@@ -2055,7 +2106,7 @@ defmodule Livebook.Session.DataTest do
"s1" => %{evaluating_cell_id: "c2"}
}
} = new_data,
- [{:start_evaluation, %{id: "c2"}, %{id: "s1"}}]} =
+ [{:start_evaluation, %{id: "c2"}, %{id: "s1"}, []}]} =
Data.apply_operation(data, operation)
assert new_data.section_infos["s1"].evaluation_queue == MapSet.new([])
@@ -2068,9 +2119,9 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_section, @cid, 1, "s2"},
{:insert_cell, @cid, "s2", 0, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1", "c2"]}
+ {:queue_cells_evaluation, @cid, ["c1", "c2"], []}
])
operation = {:add_cell_evaluation_response, @cid, "c1", @eval_resp, eval_meta()}
@@ -2083,7 +2134,7 @@ defmodule Livebook.Session.DataTest do
"s2" => %{evaluating_cell_id: "c2"}
}
} = new_data,
- [{:start_evaluation, %{id: "c2"}, %{id: "s2"}}]} =
+ [{:start_evaluation, %{id: "c2"}, %{id: "s2"}, []}]} =
Data.apply_operation(data, operation)
assert new_data.section_infos["s1"].evaluation_queue == MapSet.new([])
@@ -2099,11 +2150,11 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 1, "s2"},
{:insert_cell, @cid, "s2", 0, :code, "c3", %{}},
{:insert_cell, @cid, "s2", 1, :code, "c4", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2", "c3", "c4"],
uses: %{"c2" => ["c1"], "c4" => ["c2"]}
),
- {:queue_cells_evaluation, @cid, ["c1"]}
+ {:queue_cells_evaluation, @cid, ["c1"], []}
])
operation =
@@ -2126,11 +2177,11 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
{:insert_cell, @cid, "s1", 2, :code, "c3", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2", "c3"],
uses: %{"c2" => :unknown}
),
- {:queue_cells_evaluation, @cid, ["c1"]}
+ {:queue_cells_evaluation, @cid, ["c1"], []}
])
operation =
@@ -2155,11 +2206,11 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s1", 2, :code, "c3", %{}},
{:insert_cell, @cid, "s1", 3, :code, "c4", %{}},
{:insert_cell, @cid, "s1", 4, :code, "c5", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2", "c3", "c4", "c5"],
uses: %{"c2" => ["c1"], "c3" => ["c2"]}
),
- {:queue_cells_evaluation, @cid, ["c1", "c5"]}
+ {:queue_cells_evaluation, @cid, ["c1", "c5"], []}
])
operation =
@@ -2195,11 +2246,11 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s4", 0, :code, "c4", %{}},
{:set_section_parent, @cid, "s3", "s2"},
{:set_section_parent, @cid, "s4", "s1"},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2", "c3", "c4"],
uses: %{"c3" => ["c2"], "c4" => ["c1", "c2"]}
),
- {:queue_cells_evaluation, @cid, ["c2"]}
+ {:queue_cells_evaluation, @cid, ["c2"], []}
])
operation =
@@ -2226,11 +2277,11 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
{:insert_cell, @cid, "s1", 2, :code, "c3", %{}},
{:set_cell_attributes, @cid, "c3", %{reevaluate_automatically: true}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2", "c3"],
uses: %{"c2" => ["c1"], "c3" => ["c2"]}
),
- {:queue_cells_evaluation, @cid, ["c1"]}
+ {:queue_cells_evaluation, @cid, ["c1"], []}
])
operation =
@@ -2252,9 +2303,9 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
{:set_cell_attributes, @cid, "c2", %{reevaluate_automatically: true}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1"]}
+ {:queue_cells_evaluation, @cid, ["c1"], []}
])
operation = {:add_cell_evaluation_response, @cid, "c1", @eval_resp, eval_meta()}
@@ -2277,9 +2328,9 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 1, "s2"},
{:insert_cell, @cid, "s2", 0, :code, "c3", %{}},
{:insert_cell, @cid, "s2", 1, :code, "c4", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1", "c2", "c3", "c4"]}
+ {:queue_cells_evaluation, @cid, ["c1", "c2", "c3", "c4"], []}
])
operation =
@@ -2307,11 +2358,11 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
{:insert_cell, @cid, "s1", 2, :code, "c3", %{}},
{:set_cell_attributes, @cid, "c3", %{reevaluate_automatically: true}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2", "c3"],
uses: %{"c2" => ["c1"], "c3" => ["c2"]}
),
- {:queue_cells_evaluation, @cid, ["c1"]}
+ {:queue_cells_evaluation, @cid, ["c1"], []}
])
operation =
@@ -2337,13 +2388,13 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
{:insert_cell, @cid, "s1", 2, :code, "c3", %{}},
{:set_cell_attributes, @cid, "c3", %{reevaluate_automatically: true}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2", "c3"],
uses: %{"c2" => ["c1"], "c3" => ["c2"]}
),
- {:queue_cells_evaluation, @cid, ["c1"]},
+ {:queue_cells_evaluation, @cid, ["c1"], []},
{:add_cell_evaluation_response, @cid, "c1", @eval_resp, eval_meta(errored: true)},
- {:queue_cells_evaluation, @cid, ["c1"]}
+ {:queue_cells_evaluation, @cid, ["c1"], []}
])
operation = {:add_cell_evaluation_response, @cid, "c1", @eval_resp, eval_meta()}
@@ -2366,13 +2417,13 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1", "c2"]},
+ {:queue_cells_evaluation, @cid, ["c1", "c2"], []},
{:add_cell_evaluation_response, @cid, "c1", @input, eval_meta()},
{:add_cell_evaluation_response, @cid, "c2", @eval_resp, eval_meta()},
# Make the code cell evaluating
- {:queue_cells_evaluation, @cid, ["c2"]},
+ {:queue_cells_evaluation, @cid, ["c2"], []},
# Bind the input (effectively read the current value)
{:bind_input, @cid, "c2", "i1"},
# Change the input value, while the cell is evaluating
@@ -2394,9 +2445,9 @@ defmodule Livebook.Session.DataTest do
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1"]}
+ {:queue_cells_evaluation, @cid, ["c1"], []}
])
operation = {:add_cell_evaluation_response, @cid, "c1", {:ok, [1, 2, 3]}, eval_meta()}
@@ -2414,9 +2465,9 @@ defmodule Livebook.Session.DataTest do
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1"]},
+ {:queue_cells_evaluation, @cid, ["c1"], []},
{:set_notebook_attributes, @cid, %{persist_outputs: true}},
{:notebook_saved, @cid, []}
])
@@ -2431,9 +2482,9 @@ defmodule Livebook.Session.DataTest do
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1"]}
+ {:queue_cells_evaluation, @cid, ["c1"], []}
])
operation = {:add_cell_evaluation_response, @cid, "c1", @input, eval_meta()}
@@ -2449,9 +2500,9 @@ defmodule Livebook.Session.DataTest do
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1"]}
+ {:queue_cells_evaluation, @cid, ["c1"], []}
])
operation = {:add_cell_evaluation_response, @cid, "c1", output, eval_meta()}
@@ -2465,12 +2516,12 @@ defmodule Livebook.Session.DataTest do
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1"]},
+ {:queue_cells_evaluation, @cid, ["c1"], []},
{:add_cell_evaluation_response, @cid, "c1", @input, eval_meta()},
{:set_input_value, @cid, "i1", "value"},
- {:queue_cells_evaluation, @cid, ["c1"]}
+ {:queue_cells_evaluation, @cid, ["c1"], []}
])
# Output the same input again
@@ -2485,12 +2536,12 @@ defmodule Livebook.Session.DataTest do
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1"]},
+ {:queue_cells_evaluation, @cid, ["c1"], []},
{:add_cell_evaluation_response, @cid, "c1", @input, eval_meta()},
{:set_input_value, @cid, "i1", "value"},
- {:queue_cells_evaluation, @cid, ["c1"]}
+ {:queue_cells_evaluation, @cid, ["c1"], []}
])
# This time w don't output the input
@@ -2509,13 +2560,13 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1", "c2"]},
+ {:queue_cells_evaluation, @cid, ["c1", "c2"], []},
{:add_cell_evaluation_response, @cid, "c1", @input, eval_meta()},
{:add_cell_evaluation_response, @cid, "c2", @input, eval_meta()},
{:set_input_value, @cid, "i1", "value"},
- {:queue_cells_evaluation, @cid, ["c1"]}
+ {:queue_cells_evaluation, @cid, ["c1"], []}
])
# This time w don't output the input
@@ -2535,13 +2586,13 @@ defmodule Livebook.Session.DataTest do
{:set_section_parent, @cid, "s3", "s1"},
{:insert_cell, @cid, "s2", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s3", 0, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1"]},
+ {:queue_cells_evaluation, @cid, ["c1"], []},
{:add_cell_evaluation_response, @cid, "c1", @input, eval_meta()},
{:set_input_value, @cid, "i1", "value"},
- {:queue_cells_evaluation, @cid, ["c1"]},
- {:queue_cells_evaluation, @cid, ["c2"]}
+ {:queue_cells_evaluation, @cid, ["c1"], []},
+ {:queue_cells_evaluation, @cid, ["c2"], []}
])
# This time w don't output the input
@@ -2557,11 +2608,11 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :smart, "c2", %{kind: "text"}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
{:set_smart_cell_definitions, @cid, @smart_cell_definitions},
{:smart_cell_started, @cid, "c2", Delta.new(), nil, %{}, nil},
- {:queue_cells_evaluation, @cid, ["c1"]}
+ {:queue_cells_evaluation, @cid, ["c1"], []}
])
operation = {:add_cell_evaluation_response, @cid, "c1", @eval_resp, eval_meta()}
@@ -2586,9 +2637,9 @@ defmodule Livebook.Session.DataTest do
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1"]}
+ {:queue_cells_evaluation, @cid, ["c1"], []}
])
doctest_report = %{status: :running, line: 5}
@@ -2608,9 +2659,9 @@ defmodule Livebook.Session.DataTest do
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1"]},
+ {:queue_cells_evaluation, @cid, ["c1"], []},
{:add_cell_doctest_report, @cid, "c1", %{status: :running, line: 5}}
])
@@ -2657,11 +2708,11 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1"]},
+ {:queue_cells_evaluation, @cid, ["c1"], []},
{:add_cell_evaluation_response, @cid, "c1", @input, eval_meta()},
- {:queue_cells_evaluation, @cid, ["c2"]}
+ {:queue_cells_evaluation, @cid, ["c2"], []}
])
operation = {:bind_input, @cid, "c2", "i1"}
@@ -2685,9 +2736,9 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
{:insert_cell, @cid, "s1", 2, :code, "c3", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1"]),
- {:queue_cells_evaluation, @cid, ["c2", "c3"]}
+ {:queue_cells_evaluation, @cid, ["c2", "c3"], []}
])
operation = {:reflect_main_evaluation_failure, @cid}
@@ -2716,9 +2767,9 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s2", 0, :code, "c2", %{}},
{:insert_cell, @cid, "s2", 1, :code, "c3", %{}},
{:set_section_parent, @cid, "s2", "s1"},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2"]),
- {:queue_cells_evaluation, @cid, ["c3"]}
+ {:queue_cells_evaluation, @cid, ["c3"], []}
])
operation = {:reflect_main_evaluation_failure, @cid}
@@ -2753,9 +2804,9 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 2, "s3"},
{:insert_cell, @cid, "s3", 0, :code, "c4", %{}},
{:set_section_parent, @cid, "s2", "s1"},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2"]),
- {:queue_cells_evaluation, @cid, ["c3", "c4"]}
+ {:queue_cells_evaluation, @cid, ["c3", "c4"], []}
])
operation = {:reflect_evaluation_failure, @cid, "s2"}
@@ -2793,7 +2844,7 @@ defmodule Livebook.Session.DataTest do
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1"])
])
@@ -2809,9 +2860,9 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
{:insert_section, @cid, 1, "s2"},
{:insert_cell, @cid, "s2", 0, :code, "c3", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1"]),
- {:queue_cells_evaluation, @cid, ["c2", "c3"]}
+ {:queue_cells_evaluation, @cid, ["c2", "c3"], []}
])
operation = {:cancel_cell_evaluation, @cid, "c2"}
@@ -2839,9 +2890,9 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1", "c2"]}
+ {:queue_cells_evaluation, @cid, ["c1", "c2"], []}
])
operation = {:cancel_cell_evaluation, @cid, "c1"}
@@ -2861,9 +2912,9 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 2, "s3"},
{:insert_cell, @cid, "s3", 0, :code, "c4", %{}},
{:set_section_parent, @cid, "s2", "s1"},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1"]),
- {:queue_cells_evaluation, @cid, ["c2", "c3", "c4"]}
+ {:queue_cells_evaluation, @cid, ["c2", "c3", "c4"], []}
])
operation = {:cancel_cell_evaluation, @cid, "c2"}
@@ -2894,9 +2945,9 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1", "c2"]}
+ {:queue_cells_evaluation, @cid, ["c1", "c2"], []}
])
operation = {:cancel_cell_evaluation, @cid, "c2"}
@@ -2921,9 +2972,9 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
{:insert_cell, @cid, "s1", 2, :code, "c3", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1", "c2", "c3"]}
+ {:queue_cells_evaluation, @cid, ["c1", "c2", "c3"], []}
])
operation = {:cancel_cell_evaluation, @cid, "c2"}
@@ -2957,7 +3008,7 @@ defmodule Livebook.Session.DataTest do
data =
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
{:set_smart_cell_definitions, @cid, @smart_cell_definitions},
{:insert_cell, @cid, "s1", 0, :smart, "c1", %{kind: "text"}}
])
@@ -2976,7 +3027,7 @@ defmodule Livebook.Session.DataTest do
data =
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
{:set_smart_cell_definitions, @cid, @smart_cell_definitions},
{:insert_cell, @cid, "s1", 0, :smart, "c1", %{kind: "text"}}
])
@@ -3003,7 +3054,7 @@ defmodule Livebook.Session.DataTest do
data =
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
{:set_smart_cell_definitions, @cid, @smart_cell_definitions},
{:insert_cell, @cid, "s1", 0, :smart, "c1", %{kind: "text"}},
{:smart_cell_started, @cid, "c1", delta1, nil, %{}, nil}
@@ -3031,7 +3082,7 @@ defmodule Livebook.Session.DataTest do
data =
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
{:set_smart_cell_definitions, @cid, @smart_cell_definitions},
{:insert_cell, @cid, "s1", 0, :smart, "c1", %{kind: "text"}},
@@ -3049,7 +3100,7 @@ defmodule Livebook.Session.DataTest do
data =
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
{:set_smart_cell_definitions, @cid, @smart_cell_definitions},
{:insert_cell, @cid, "s1", 0, :smart, "c1", %{kind: "text"}},
@@ -3071,7 +3122,7 @@ defmodule Livebook.Session.DataTest do
data =
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
{:set_smart_cell_definitions, @cid, @smart_cell_definitions},
{:insert_cell, @cid, "s1", 0, :smart, "c1", %{kind: "text"}}
])
@@ -3088,7 +3139,7 @@ defmodule Livebook.Session.DataTest do
data =
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
{:set_smart_cell_definitions, @cid, @smart_cell_definitions},
{:insert_cell, @cid, "s1", 0, :smart, "c1", %{kind: "text"}}
])
@@ -3102,7 +3153,7 @@ defmodule Livebook.Session.DataTest do
data =
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
{:set_smart_cell_definitions, @cid, @smart_cell_definitions},
{:insert_cell, @cid, "s1", 0, :smart, "c1", %{kind: "text"}},
{:smart_cell_down, @cid, "c1"}
@@ -3126,9 +3177,9 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 1, "s2"},
{:insert_cell, @cid, "s2", 0, :code, "c3", %{}},
{:set_section_parent, @cid, "s2", "s1"},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1"]),
- {:queue_cells_evaluation, @cid, ["c2", "c3"]}
+ {:queue_cells_evaluation, @cid, ["c2", "c3"], []}
])
operation = {:erase_outputs, @cid}
@@ -3157,7 +3208,7 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :markdown, "c2", %{}},
{:insert_cell, @cid, "s1", 2, :code, "c3", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c3"])
])
@@ -3185,7 +3236,7 @@ defmodule Livebook.Session.DataTest do
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1"]),
{:add_cell_doctest_report, @cid, "c1", %{status: :running, line: 5}}
])
@@ -3551,7 +3602,7 @@ defmodule Livebook.Session.DataTest do
{:client_join, @cid, User.new()},
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 1, :smart, "c1", %{kind: "text"}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
{:set_smart_cell_definitions, @cid, @smart_cell_definitions},
{:smart_cell_started, @cid, "c1", Delta.new(), nil, %{},
%{language: "text", placement: :bottom, source: ""}}
@@ -3691,7 +3742,7 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2"], uses: %{"c2" => ["c1"]}),
evaluate_cells_operations(["c1"], versions: %{"c1" => 1})
])
@@ -3722,9 +3773,9 @@ defmodule Livebook.Session.DataTest do
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1"]},
+ {:queue_cells_evaluation, @cid, ["c1"], []},
{:add_cell_evaluation_response, @cid, "c1", @input, eval_meta()}
])
@@ -3743,9 +3794,9 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
{:insert_cell, @cid, "s1", 2, :code, "c3", %{}},
{:insert_cell, @cid, "s1", 3, :code, "c4", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1"]},
+ {:queue_cells_evaluation, @cid, ["c1"], []},
{:add_cell_evaluation_response, @cid, "c1", @input, eval_meta()},
evaluate_cells_operations(["c2", "c3", "c4"],
bind_inputs: %{"c3" => ["i1"]},
@@ -3767,13 +3818,149 @@ defmodule Livebook.Session.DataTest do
end
describe "apply_operation/2 given :set_runtime" do
+ test "returns an error if the runtime is connecting" do
+ data =
+ data_after_operations!([
+ {:connect_runtime, @cid}
+ ])
+
+ operation = {:set_runtime, @cid, Livebook.Runtime.NoopRuntime.new()}
+
+ assert :error = Data.apply_operation(data, operation)
+ end
+
test "updates data with the given runtime" do
data = Data.new()
- runtime = connected_noop_runtime()
+ runtime = Livebook.Runtime.NoopRuntime.new()
operation = {:set_runtime, @cid, runtime}
- assert {:ok, %{runtime: ^runtime}, []} = Data.apply_operation(data, operation)
+ assert {:ok, %{runtime: ^runtime, runtime_status: :disconnected}, []} =
+ Data.apply_operation(data, operation)
+ end
+
+ test "disconnects the current runtime if connected" do
+ data =
+ data_after_operations!([
+ {:insert_section, @cid, 0, "s1"},
+ {:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
+ connect_noop_runtime_operations(),
+ evaluate_cells_operations(["setup"]),
+ {:queue_cells_evaluation, @cid, ["c1"], []}
+ ])
+
+ prev_runtime = data.runtime
+
+ runtime = Livebook.Runtime.Embedded.new()
+ operation = {:set_runtime, @cid, runtime}
+
+ assert {:ok,
+ %{
+ runtime: ^runtime,
+ runtime_status: :disconnected,
+ cell_infos: %{
+ "c1" => %{eval: %{validity: :aborted, status: :ready}}
+ }
+ }, [{:disconnect_runtime, ^prev_runtime}]} = Data.apply_operation(data, operation)
+ end
+ end
+
+ describe "apply_operation/2 given :connect_runtime" do
+ test "returns an error if the current runtime is not disconnected" do
+ data =
+ data_after_operations!([
+ {:connect_runtime, @cid}
+ ])
+
+ operation = {:connect_runtime, @cid}
+
+ assert :error = Data.apply_operation(data, operation)
+ end
+
+ test "updates runtime status to connecting and returns connect action" do
+ data = Data.new()
+
+ operation = {:connect_runtime, @cid}
+
+ assert {:ok, %{runtime_status: :connecting}, [:connect_runtime]} =
+ Data.apply_operation(data, operation)
+ end
+ end
+
+ describe "apply_operation/2 given :runtime_connected" do
+ test "returns an error if the runtime is not connecting" do
+ data = Data.new()
+
+ runtime = Livebook.Runtime.NoopRuntime.new()
+ operation = {:runtime_connected, @cid, runtime}
+
+ assert :error = Data.apply_operation(data, operation)
+ end
+
+ test "updates data with the given runtime" do
+ data =
+ data_after_operations!([
+ {:connect_runtime, @cid}
+ ])
+
+ runtime = Livebook.Runtime.Embedded.new()
+ operation = {:runtime_connected, @cid, runtime}
+
+ assert {:ok, %{runtime: ^runtime, runtime_status: :connected}, _actions} =
+ Data.apply_operation(data, operation)
+ end
+
+ test "starts evaluation if there are queued cells" do
+ data =
+ data_after_operations!([
+ {:insert_section, @cid, 0, "s1"},
+ {:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
+ {:queue_cells_evaluation, @cid, ["setup"], []}
+ ])
+
+ runtime = Livebook.Runtime.Embedded.new()
+ operation = {:runtime_connected, @cid, runtime}
+
+ assert {:ok,
+ %{
+ cell_infos: %{
+ "setup" => %{eval: %{status: :evaluating}}
+ },
+ section_infos: %{
+ "setup-section" => %{evaluating_cell_id: "setup"}
+ }
+ } = new_data,
+ [{:start_evaluation, %{id: "setup"}, %{id: "setup-section"}, []}]} =
+ Data.apply_operation(data, operation)
+
+ assert new_data.section_infos["setup-section"].evaluation_queue == MapSet.new([])
+ end
+ end
+
+ describe "apply_operation/2 given :disconnect_runtime" do
+ test "returns an error if the runtime is not connected" do
+ data =
+ data_after_operations!([
+ {:connect_runtime, @cid}
+ ])
+
+ operation = {:disconnect_runtime, @cid}
+
+ assert :error = Data.apply_operation(data, operation)
+ end
+
+ test "returns disconnect runtime action" do
+ data =
+ data_after_operations!([
+ connect_noop_runtime_operations()
+ ])
+
+ runtime = data.runtime
+
+ operation = {:disconnect_runtime, @cid}
+
+ assert {:ok, %{runtime_status: :disconnected}, [{:disconnect_runtime, ^runtime}]} =
+ Data.apply_operation(data, operation)
end
test "clears all statuses and the per-section queues" do
@@ -3785,16 +3972,16 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 1, "s2"},
{:insert_cell, @cid, "s2", 0, :code, "c3", %{}},
{:insert_cell, @cid, "s2", 1, :code, "c4", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1", "c2", "c3", "c4"]}
+ {:queue_cells_evaluation, @cid, ["c1", "c2", "c3", "c4"], []}
])
- runtime = connected_noop_runtime()
- operation = {:set_runtime, @cid, runtime}
+ operation = {:runtime_down, @cid}
assert {:ok,
%{
+ runtime_status: :disconnected,
cell_infos: %{
"c1" => %{eval: %{validity: :aborted, status: :ready}},
"c2" => %{eval: %{validity: :fresh, status: :ready}},
@@ -3805,52 +3992,58 @@ defmodule Livebook.Session.DataTest do
"s1" => %{evaluating_cell_id: nil},
"s2" => %{evaluating_cell_id: nil}
}
- } = new_data, []} = Data.apply_operation(data, operation)
+ } = new_data, _actions} = Data.apply_operation(data, operation)
assert new_data.section_infos["s1"].evaluation_queue == MapSet.new([])
assert new_data.section_infos["s2"].evaluation_queue == MapSet.new([])
end
- test "starts evaluation if there was no runtime before and there is now" do
- data =
- data_after_operations!([
- {:insert_section, @cid, 0, "s1"},
- {:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
- {:queue_cells_evaluation, @cid, ["setup"]}
- ])
-
- runtime = connected_noop_runtime()
- operation = {:set_runtime, @cid, runtime}
-
- assert {:ok,
- %{
- cell_infos: %{
- "setup" => %{eval: %{status: :evaluating}}
- },
- section_infos: %{
- "setup-section" => %{evaluating_cell_id: "setup"}
- }
- } = new_data,
- [{:start_evaluation, %{id: "setup"}, %{id: "setup-section"}}]} =
- Data.apply_operation(data, operation)
-
- assert new_data.section_infos["setup-section"].evaluation_queue == MapSet.new([])
- end
-
test "clears runtime-related state" do
data =
data_after_operations!([
+ connect_noop_runtime_operations(),
{:set_smart_cell_definitions, @cid, @smart_cell_definitions},
{:set_runtime_connected_nodes, @cid, [:node@host]}
])
- runtime = connected_noop_runtime()
- operation = {:set_runtime, @cid, runtime}
+ operation = {:runtime_down, @cid}
assert {:ok,
%{
+ runtime_status: :disconnected,
smart_cell_definitions: [],
runtime_connected_nodes: []
+ }, _actions} = Data.apply_operation(data, operation)
+ end
+ end
+
+ describe "apply_operation/2 given :runtime_down" do
+ test "returns an error if the runtime is disconnected" do
+ data = Data.new()
+
+ operation = {:runtime_down, @cid}
+
+ assert :error = Data.apply_operation(data, operation)
+ end
+
+ test "sets runtime status to disconnected and clear evaluation" do
+ data =
+ data_after_operations!([
+ {:insert_section, @cid, 0, "s1"},
+ {:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
+ connect_noop_runtime_operations(),
+ evaluate_cells_operations(["setup"]),
+ {:queue_cells_evaluation, @cid, ["c1"], []}
+ ])
+
+ operation = {:runtime_down, @cid}
+
+ assert {:ok,
+ %{
+ runtime_status: :disconnected,
+ cell_infos: %{
+ "c1" => %{eval: %{validity: :aborted, status: :ready}}
+ }
}, []} = Data.apply_operation(data, operation)
end
end
@@ -3859,7 +4052,7 @@ defmodule Livebook.Session.DataTest do
test "sets the definitions and starts dead cells with matching kinds" do
data =
data_after_operations!([
- {:set_runtime, @cid, connected_noop_runtime()}
+ connect_noop_runtime_operations()
])
transient_state = %{state: "anything"}
@@ -3876,7 +4069,7 @@ defmodule Livebook.Session.DataTest do
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :smart, "c1", %{kind: "text"}},
- {:set_runtime, @cid, connected_noop_runtime()}
+ connect_noop_runtime_operations()
])
operation = {:set_smart_cell_definitions, @cid, @smart_cell_definitions}
@@ -4179,7 +4372,7 @@ defmodule Livebook.Session.DataTest do
test "updates app status" do
data =
data_after_operations!(Data.new(mode: :app), [
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"])
])
@@ -4192,7 +4385,7 @@ defmodule Livebook.Session.DataTest do
test "does not return terminate action if there are clients" do
data =
data_after_operations!(Data.new(mode: :app), [
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
{:client_join, @cid, User.new()}
])
@@ -4211,9 +4404,9 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1", "c2"]}
+ {:queue_cells_evaluation, @cid, ["c1", "c2"], []}
])
operation = {:add_cell_evaluation_response, @cid, "c1", @eval_resp, eval_meta()}
@@ -4228,9 +4421,9 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1", "c2"]}
+ {:queue_cells_evaluation, @cid, ["c1", "c2"], []}
])
operation =
@@ -4246,9 +4439,9 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1"]),
- {:queue_cells_evaluation, @cid, ["c2"]}
+ {:queue_cells_evaluation, @cid, ["c2"], []}
])
operation = {:add_cell_evaluation_response, @cid, "c2", @eval_resp, eval_meta()}
@@ -4263,9 +4456,9 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1"]}
+ {:queue_cells_evaluation, @cid, ["c1"], []}
])
operation = {:reflect_main_evaluation_failure, @cid}
@@ -4274,33 +4467,62 @@ defmodule Livebook.Session.DataTest do
Data.apply_operation(data, operation)
end
- test "returns recover action when fully executed and then aborted" do
+ test "when fully executed and then aborted, recovers by evaluating from scratch" do
data =
data_after_operations!(Data.new(mode: :app), [
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2"])
])
operation = {:reflect_main_evaluation_failure, @cid}
- assert {:ok, %{app_data: %{status: %{execution: :error}}},
- [:app_report_status, :app_recover]} = Data.apply_operation(data, operation)
+ assert {:ok,
+ %{
+ cell_infos: %{
+ "setup" => %{eval: %{status: :queued}},
+ "c1" => %{eval: %{status: :queued}},
+ "c2" => %{eval: %{status: :queued}}
+ },
+ app_data: %{status: %{execution: :executing}}
+ },
+ [:app_report_status, {:disconnect_runtime, _}, :connect_runtime]} =
+ Data.apply_operation(data, operation)
end
- test "changes status to :error when a non-connected runtime is set" do
+ test "changes status back to :executed after recovery" do
data =
data_after_operations!(Data.new(mode: :app), [
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
- {:queue_cells_evaluation, @cid, ["setup"]}
+ connect_noop_runtime_operations(),
+ evaluate_cells_operations(["setup", "c1", "c2"]),
+ {:reflect_main_evaluation_failure, @cid},
+ {:runtime_connected, @cid, Livebook.Runtime.NoopRuntime.new()},
+ {:add_cell_evaluation_response, @cid, "setup", @eval_resp, eval_meta()},
+ {:add_cell_evaluation_response, @cid, "c1", @eval_resp, eval_meta()}
])
- operation = {:set_runtime, @cid, Livebook.Runtime.NoopRuntime.new()}
+ operation = {:add_cell_evaluation_response, @cid, "c2", @eval_resp, eval_meta()}
+
+ assert {:ok, %{app_data: %{status: %{execution: :executed}}}, [:app_report_status]} =
+ Data.apply_operation(data, operation)
+ end
+
+ test "changes status to :error when the runtime goes down" do
+ data =
+ data_after_operations!(Data.new(mode: :app), [
+ {:insert_section, @cid, 0, "s1"},
+ {:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
+ {:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
+ connect_noop_runtime_operations(),
+ {:queue_cells_evaluation, @cid, ["setup"], []}
+ ])
+
+ operation = {:runtime_down, @cid}
assert {:ok, %{app_data: %{status: %{execution: :error}}}, [:app_report_status]} =
Data.apply_operation(data, operation)
@@ -4312,9 +4534,9 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1"]),
- {:queue_cells_evaluation, @cid, ["c2"]}
+ {:queue_cells_evaluation, @cid, ["c2"], []}
])
operation =
@@ -4330,9 +4552,9 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1"]),
- {:queue_cells_evaluation, @cid, ["c2"]}
+ {:queue_cells_evaluation, @cid, ["c2"], []}
])
operation =
@@ -4342,29 +4564,10 @@ defmodule Livebook.Session.DataTest do
Data.apply_operation(data, operation)
end
- test "changes status back to :executed after recovery" do
- data =
- data_after_operations!(Data.new(mode: :app), [
- {:insert_section, @cid, 0, "s1"},
- {:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
- {:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
- evaluate_cells_operations(["setup", "c1", "c2"]),
- {:reflect_main_evaluation_failure, @cid},
- evaluate_cells_operations(["setup", "c1"]),
- {:queue_cells_evaluation, @cid, ["c2"]}
- ])
-
- operation = {:add_cell_evaluation_response, @cid, "c2", @eval_resp, eval_meta()}
-
- assert {:ok, %{app_data: %{status: %{execution: :executed}}}, [:app_report_status]} =
- Data.apply_operation(data, operation)
- end
-
test "when the app is shutting down and the last client leaves, returns terminate action" do
data =
data_after_operations!(Data.new(mode: :app), [
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
{:client_join, @cid, User.new()},
{:app_shutdown, @cid}
@@ -4434,9 +4637,9 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
{:insert_cell, @cid, "s1", 2, :code, "c3", %{}},
{:insert_cell, @cid, "s1", 4, :code, "c4", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1"]},
+ {:queue_cells_evaluation, @cid, ["c1"], []},
{:add_cell_evaluation_response, @cid, "c1", @input, eval_meta()},
evaluate_cells_operations(["c2", "c3", "c4"], %{
bind_inputs: %{"c2" => ["i1"], "c4" => ["i1"]}
@@ -4456,7 +4659,7 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
{:insert_cell, @cid, "s1", 2, :code, "c3", %{}},
{:insert_cell, @cid, "s1", 3, :code, "c4", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2", "c3", "c4"],
uses: %{"c4" => ["c2"]}
),
@@ -4474,7 +4677,7 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c3", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c3"]),
# Insert a fresh cell between cell 1 and cell 3
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}}
@@ -4489,7 +4692,7 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2"],
uses: %{"c2" => ["c1"]}
),
@@ -4508,7 +4711,7 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
{:insert_cell, @cid, "s1", 2, :code, "c3", %{}},
{:insert_cell, @cid, "s1", 3, :code, "c4", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2", "c3", "c4"],
uses: %{"c4" => ["c2"]}
)
@@ -4524,12 +4727,29 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
{:insert_cell, @cid, "s1", 2, :code, "c3", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
- {:queue_cells_evaluation, @cid, ["c1", "c2"]}
+ connect_noop_runtime_operations(),
+ {:queue_cells_evaluation, @cid, ["c1", "c2"], []}
])
assert Data.cell_ids_for_full_evaluation(data, []) |> Enum.sort() == ["c3"]
end
+
+ test "includes all cells if the setup cell is evaluated and outdated" do
+ data =
+ data_after_operations!([
+ {:insert_section, @cid, 0, "s1"},
+ {:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
+ {:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
+ connect_noop_runtime_operations(),
+ evaluate_cells_operations(["setup", "c1", "c2"]),
+ # Modify the setup cell
+ {:client_join, @cid, User.new()},
+ {:apply_cell_delta, @cid, "setup", :primary, Delta.new() |> Delta.insert("cats"), nil,
+ 0}
+ ])
+
+ assert Data.cell_ids_for_full_evaluation(data, []) |> Enum.sort() == ["c1", "c2", "setup"]
+ end
end
describe "cell_ids_for_reevaluation/2" do
@@ -4537,7 +4757,7 @@ defmodule Livebook.Session.DataTest do
data =
data_after_operations!([
{:insert_section, @cid, 0, "s1"},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"])
])
@@ -4550,7 +4770,7 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2"])
])
@@ -4563,7 +4783,7 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2"],
uses: %{"c2" => ["c1"]}
),
@@ -4580,7 +4800,7 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2"]),
# Insert a new cell between the two evaluated cells
{:insert_cell, @cid, "s1", 1, :code, "c3", %{}}
@@ -4600,7 +4820,7 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 2, "s3"},
{:insert_cell, @cid, "s3", 0, :code, "c4", %{}},
{:set_section_parent, @cid, "s2", "s1"},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup", "c1", "c2", "c4"])
])
@@ -4637,9 +4857,9 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
{:insert_cell, @cid, "s1", 2, :code, "c3", %{}},
{:insert_cell, @cid, "s1", 4, :code, "c4", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1", "c2"]},
+ {:queue_cells_evaluation, @cid, ["c1", "c2"], []},
{:add_cell_evaluation_response, @cid, "c1", input1, eval_meta()},
{:add_cell_evaluation_response, @cid, "c2", input2, eval_meta()},
evaluate_cells_operations(["c3", "c4"], %{
@@ -4658,9 +4878,9 @@ defmodule Livebook.Session.DataTest do
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
{:insert_cell, @cid, "s1", 2, :code, "c3", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1"]},
+ {:queue_cells_evaluation, @cid, ["c1"], []},
{:add_cell_evaluation_response, @cid, "c1", @input, eval_meta()},
evaluate_cells_operations(["c2", "c3"], %{
bind_inputs: %{"c2" => ["i1"], "c3" => ["i1"]}
@@ -4679,9 +4899,9 @@ defmodule Livebook.Session.DataTest do
{:insert_section, @cid, 0, "s1"},
{:insert_cell, @cid, "s1", 0, :code, "c1", %{}},
{:insert_cell, @cid, "s1", 1, :code, "c2", %{}},
- {:set_runtime, @cid, connected_noop_runtime()},
+ connect_noop_runtime_operations(),
evaluate_cells_operations(["setup"]),
- {:queue_cells_evaluation, @cid, ["c1"]},
+ {:queue_cells_evaluation, @cid, ["c1"], []},
{:add_cell_evaluation_response, @cid, "c1", @input, eval_meta()},
evaluate_cells_operations(["c2"], %{bind_inputs: %{"c2" => ["i1"]}}),
{:set_input_value, @cid, "i1", "new value"},
@@ -4698,7 +4918,7 @@ defmodule Livebook.Session.DataTest do
bind_inputs = opts[:bind_inputs] || %{}
[
- {:queue_cells_evaluation, @cid, cell_ids},
+ {:queue_cells_evaluation, @cid, cell_ids, []},
for cell_id <- cell_ids do
# For convenience we make each cell evaluation define an identifier
# corresponding to the cell id, this way it is easy to make any
@@ -4719,8 +4939,13 @@ defmodule Livebook.Session.DataTest do
]
end
- defp connected_noop_runtime() do
- {:ok, runtime} = Livebook.Runtime.NoopRuntime.new() |> Livebook.Runtime.connect()
- runtime
+ defp connect_noop_runtime_operations() do
+ runtime = Livebook.Runtime.NoopRuntime.new()
+
+ [
+ {:set_runtime, @cid, runtime},
+ {:connect_runtime, @cid},
+ {:runtime_connected, @cid, runtime}
+ ]
end
end
diff --git a/test/livebook/session_test.exs b/test/livebook/session_test.exs
index c97316c8a..aa1508f82 100644
--- a/test/livebook/session_test.exs
+++ b/test/livebook/session_test.exs
@@ -3,6 +3,7 @@ defmodule Livebook.SessionTest do
import Livebook.HubHelpers
import Livebook.AppHelpers
+ import Livebook.SessionHelpers
import Livebook.TestHelpers
alias Livebook.{Session, Text, Runtime, Utils, Notebook, FileSystem, Apps, App}
@@ -217,9 +218,6 @@ defmodule Livebook.SessionTest do
test "applies source change to the setup cell to include the given dependencies" do
session = start_session()
- runtime = connected_noop_runtime()
- Session.set_runtime(session.pid, runtime)
-
Session.subscribe(session.id)
Session.add_dependencies(session.pid, [%{dep: {:jason, "~> 1.3.0"}, config: []}])
@@ -248,9 +246,6 @@ defmodule Livebook.SessionTest do
notebook = Notebook.new() |> Notebook.update_cell("setup", &%{&1 | source: "[,]"})
session = start_session(notebook: notebook)
- runtime = connected_noop_runtime()
- Session.set_runtime(session.pid, runtime)
-
Session.subscribe(session.id)
Session.add_dependencies(session.pid, [%{dep: {:jason, "~> 1.3.0"}, config: []}])
@@ -269,7 +264,7 @@ defmodule Livebook.SessionTest do
Session.queue_cell_evaluation(session.pid, cell_id)
- assert_receive {:operation, {:queue_cells_evaluation, _client_id, [^cell_id]}}
+ assert_receive {:operation, {:queue_cells_evaluation, _client_id, [^cell_id], []}}
assert_receive {:operation,
{:add_cell_evaluation_response, _, ^cell_id, _,
@@ -392,10 +387,14 @@ defmodule Livebook.SessionTest do
Session.subscribe(session.id)
- runtime = connected_noop_runtime()
+ runtime = Livebook.Runtime.NoopRuntime.new()
Session.set_runtime(session.pid, runtime)
+ Session.connect_runtime(session.pid)
+
assert_receive {:operation, {:set_runtime, _client_id, ^runtime}}
+ assert_receive {:operation, {:connect_runtime, _client_id}}
+ assert_receive {:operation, {:runtime_connected, _client_id, _runtime}}
end
end
@@ -405,16 +404,13 @@ defmodule Livebook.SessionTest do
Session.subscribe(session.id)
- runtime = connected_noop_runtime()
- Session.set_runtime(session.pid, runtime)
- assert_receive {:operation, {:set_runtime, _client_id, _}}
+ set_noop_runtime(session.pid)
# Calling twice can happen in a race, make sure it doesn't crash
Session.disconnect_runtime(session.pid)
Session.disconnect_runtime([session.pid])
- assert_receive {:operation, {:set_runtime, _client_id, runtime}}
- refute Runtime.connected?(runtime)
+ assert_receive {:operation, {:disconnect_runtime, _client_id}}
end
end
@@ -570,8 +566,9 @@ defmodule Livebook.SessionTest do
File.write!(source_path, "content")
{:ok, old_file_ref} = Session.register_file(session.pid, source_path, "key")
- runtime = connected_noop_runtime(self())
- Session.set_runtime(session.pid, runtime)
+ Session.subscribe(session.id)
+ set_noop_runtime(session.pid, self())
+ connect_and_await_runtime(session.pid)
send(session.pid, {:runtime_file_path_request, self(), old_file_ref})
assert_receive {:runtime_file_path_reply, {:ok, old_path}}
@@ -604,8 +601,9 @@ defmodule Livebook.SessionTest do
{:ok, file_ref} =
Session.register_file(session.pid, source_path, "key", linked_client_id: client_id)
- runtime = connected_noop_runtime(self())
- Session.set_runtime(session.pid, runtime)
+ Session.subscribe(session.id)
+ set_noop_runtime(session.pid, self())
+ connect_and_await_runtime(session.pid)
send(session.pid, {:runtime_file_path_request, self(), file_ref})
assert_receive {:runtime_file_path_reply, {:ok, path}}
@@ -643,8 +641,9 @@ defmodule Livebook.SessionTest do
client_name: "data.txt"
})
- runtime = connected_noop_runtime(self())
- Session.set_runtime(session.pid, runtime)
+ Session.subscribe(session.id)
+ set_noop_runtime(session.pid, self())
+ connect_and_await_runtime(session.pid)
send(session.pid, {:runtime_file_path_request, self(), file_ref})
assert_receive {:runtime_file_path_reply, {:ok, path}}
@@ -800,7 +799,7 @@ defmodule Livebook.SessionTest do
# For most tests we use the lightweight embedded runtime,
# so that they are cheap to run. Here go several integration
- # tests that actually start a Elixir standalone runtime (default in production)
+ # tests that actually start a Standalone runtime (default in production)
# to verify session integrates well with it properly.
test "starts a standalone runtime upon first evaluation if there was none set explicitly" do
@@ -819,20 +818,19 @@ defmodule Livebook.SessionTest do
test "if the runtime node goes down, notifies the subscribers" do
session = start_session()
- {:ok, runtime} = Runtime.ElixirStandalone.new() |> Runtime.connect()
Session.subscribe(session.id)
# Wait for the runtime to be set
- Session.set_runtime(session.pid, runtime)
- assert_receive {:operation, {:set_runtime, _, ^runtime}}
+ Session.set_runtime(session.pid, Runtime.Standalone.new())
+ Session.connect_runtime(session.pid)
+ assert_receive {:operation, {:runtime_connected, _, runtime}}
# Terminate the other node, the session should detect that
Node.spawn(runtime.node, System, :halt, [])
- assert_receive {:operation, {:set_runtime, _, runtime}}
- refute Runtime.connected?(runtime)
- assert_receive {:error, "runtime node terminated unexpectedly - no connection"}
+ assert_receive {:operation, {:runtime_down, _}}
+ assert_receive {:error, "runtime terminated unexpectedly - no connection"}
end
test "on user change sends an update operation subscribers" do
@@ -934,8 +932,7 @@ defmodule Livebook.SessionTest do
notebook = %{Notebook.new() | sections: [%{Notebook.Section.new() | cells: [smart_cell]}]}
session = start_session(notebook: notebook)
- runtime = connected_noop_runtime()
- Session.set_runtime(session.pid, runtime)
+ set_noop_runtime(session.pid)
send(
session.pid,
@@ -962,8 +959,9 @@ defmodule Livebook.SessionTest do
notebook = %{Notebook.new() | sections: [%{Notebook.Section.new() | cells: [smart_cell]}]}
session = start_session(notebook: notebook)
- runtime = connected_noop_runtime()
- Session.set_runtime(session.pid, runtime)
+ Session.subscribe(session.id)
+ set_noop_runtime(session.pid)
+ connect_and_await_runtime(session.pid)
send(
session.pid,
@@ -1000,8 +998,9 @@ defmodule Livebook.SessionTest do
notebook = %{Notebook.new() | sections: [%{Notebook.Section.new() | cells: [smart_cell]}]}
session = start_session(notebook: notebook)
- runtime = connected_noop_runtime()
- Session.set_runtime(session.pid, runtime)
+ Session.subscribe(session.id)
+ set_noop_runtime(session.pid)
+ connect_and_await_runtime(session.pid)
send(
session.pid,
@@ -1039,8 +1038,9 @@ defmodule Livebook.SessionTest do
notebook = %{Notebook.new() | sections: [%{Notebook.Section.new() | cells: [smart_cell]}]}
session = start_session(notebook: notebook)
- runtime = connected_noop_runtime()
- Session.set_runtime(session.pid, runtime)
+ Session.subscribe(session.id)
+ set_noop_runtime(session.pid)
+ connect_and_await_runtime(session.pid)
send(
session.pid,
@@ -1048,8 +1048,6 @@ defmodule Livebook.SessionTest do
[%{kind: "text", name: "Text", requirement_presets: []}]}
)
- Session.subscribe(session.id)
-
editor = %{language: nil, placement: :bottom, source: "", intellisense_node: nil}
send(
@@ -1087,8 +1085,9 @@ defmodule Livebook.SessionTest do
notebook = %{Notebook.new() | sections: [%{Notebook.Section.new() | cells: [smart_cell]}]}
session = start_session(notebook: notebook)
- runtime = connected_noop_runtime()
- Session.set_runtime(session.pid, runtime)
+ Session.subscribe(session.id)
+ set_noop_runtime(session.pid)
+ connect_and_await_runtime(session.pid)
send(
session.pid,
@@ -1096,8 +1095,6 @@ defmodule Livebook.SessionTest do
[%{kind: "text", name: "Text", requirement_presets: []}]}
)
- Session.subscribe(session.id)
-
send(
session.pid,
{:runtime_smart_cell_started, smart_cell.id,
@@ -1145,8 +1142,10 @@ defmodule Livebook.SessionTest do
data =
data_after_operations!(data, [
- {:set_runtime, self(), connected_noop_runtime()},
- {:queue_cells_evaluation, self(), ["c1"]},
+ {:set_runtime, self(), Livebook.Runtime.NoopRuntime.new()},
+ {:connect_runtime, self()},
+ {:runtime_connected, self(), Livebook.Runtime.NoopRuntime.new()},
+ {:queue_cells_evaluation, self(), ["c1"], []},
{:add_cell_evaluation_response, self(), "setup", {:ok, nil}, @eval_meta},
{:add_cell_evaluation_response, self(), "c1", {:ok, nil}, @eval_meta}
])
@@ -1174,8 +1173,10 @@ defmodule Livebook.SessionTest do
data =
data_after_operations!(data, [
- {:set_runtime, self(), connected_noop_runtime()},
- {:queue_cells_evaluation, self(), ["c1"]},
+ {:set_runtime, self(), Livebook.Runtime.NoopRuntime.new()},
+ {:connect_runtime, self()},
+ {:runtime_connected, self(), Livebook.Runtime.NoopRuntime.new()},
+ {:queue_cells_evaluation, self(), ["c1"], []},
{:add_cell_evaluation_response, self(), "setup", {:ok, nil}, @eval_meta},
{:add_cell_evaluation_response, self(), "c1", {:ok, nil}, @eval_meta}
])
@@ -1205,8 +1206,10 @@ defmodule Livebook.SessionTest do
data =
data_after_operations!(data, [
- {:set_runtime, self(), connected_noop_runtime()},
- {:queue_cells_evaluation, self(), ["c1"]},
+ {:set_runtime, self(), Livebook.Runtime.NoopRuntime.new()},
+ {:connect_runtime, self()},
+ {:runtime_connected, self(), Livebook.Runtime.NoopRuntime.new()},
+ {:queue_cells_evaluation, self(), ["c1"], []},
{:add_cell_evaluation_response, self(), "setup", {:ok, nil}, @eval_meta},
{:add_cell_evaluation_response, self(), "c1", {:ok, nil}, @eval_meta}
])
@@ -1261,8 +1264,8 @@ defmodule Livebook.SessionTest do
{_section_id, cell_id} = insert_section_and_cell(session.pid)
- runtime = connected_noop_runtime()
- Session.set_runtime(session.pid, runtime)
+ set_noop_runtime(session.pid)
+ connect_and_await_runtime(session.pid)
archive_path = Path.expand("../support/assets.tar.gz", __DIR__)
hash = "test-" <> Utils.random_id()
@@ -1285,13 +1288,15 @@ defmodule Livebook.SessionTest do
test "restores transient state when restarting runtimes" do
session = start_session()
- runtime = connected_noop_runtime(self())
- Session.set_runtime(session.pid, runtime)
+ Session.subscribe(session.id)
+ set_noop_runtime(session.pid, self())
+ connect_and_await_runtime(session.pid)
+
transient_state = %{state: "anything"}
send(session.pid, {:runtime_transient_state, transient_state})
- runtime = connected_noop_runtime(self())
- Session.set_runtime(session.pid, runtime)
+ set_noop_runtime(session.pid, self())
+ connect_and_await_runtime(session.pid)
assert_receive {:runtime_trace, :restore_transient_state, [^transient_state]}
end
@@ -1419,9 +1424,6 @@ defmodule Livebook.SessionTest do
Process.exit(Process.whereis(test), :shutdown)
- assert_receive {:app_updated,
- %{pid: ^app_pid, sessions: [%{app_status: %{execution: :error}}]}}
-
assert_receive {:app_updated,
%{pid: ^app_pid, sessions: [%{app_status: %{execution: :executing}}]}}
@@ -1596,8 +1598,7 @@ defmodule Livebook.SessionTest do
test "replies with error when file entry does not exist" do
session = start_session()
- runtime = connected_noop_runtime(self())
- Session.set_runtime(session.pid, runtime)
+ set_noop_runtime(session.pid, self())
send(session.pid, {:runtime_file_entry_path_request, self(), "image.jpg"})
assert_receive {:runtime_file_entry_path_reply,
@@ -1623,8 +1624,7 @@ defmodule Livebook.SessionTest do
session = start_session(notebook: notebook)
- runtime = connected_noop_runtime(self())
- Session.set_runtime(session.pid, runtime)
+ set_noop_runtime(session.pid, self())
send(session.pid, {:runtime_file_entry_path_request, self(), "document.pdf"})
assert_receive {:runtime_file_entry_path_reply, {:error, :forbidden}}
@@ -1640,8 +1640,7 @@ defmodule Livebook.SessionTest do
Session.add_file_entries(session.pid, [%{type: :attachment, name: "image.jpg"}])
- runtime = connected_noop_runtime(self())
- Session.set_runtime(session.pid, runtime)
+ set_noop_runtime(session.pid, self())
send(session.pid, {:runtime_file_entry_path_request, self(), "image.jpg"})
assert_receive {:runtime_file_entry_path_reply, {:error, "no file exists at path " <> _}}
@@ -1659,8 +1658,7 @@ defmodule Livebook.SessionTest do
:ok = FileSystem.File.write(image_file, "")
Session.add_file_entries(session.pid, [%{type: :attachment, name: "image.jpg"}])
- runtime = connected_noop_runtime(self())
- Session.set_runtime(session.pid, runtime)
+ set_noop_runtime(session.pid, self())
send(session.pid, {:runtime_file_entry_path_request, self(), "image.jpg"})
path = image_file.path
@@ -1680,8 +1678,7 @@ defmodule Livebook.SessionTest do
:ok = FileSystem.File.write(image_file, "content")
Session.add_file_entries(session.pid, [%{type: :file, name: "image.jpg", file: image_file}])
- runtime = connected_noop_runtime(self())
- Session.set_runtime(session.pid, runtime)
+ set_noop_runtime(session.pid, self())
send(session.pid, {:runtime_file_entry_path_request, self(), "image.jpg"})
path = image_file.path
@@ -1707,8 +1704,7 @@ defmodule Livebook.SessionTest do
image_file = FileSystem.File.new(s3_fs, "/image.jpg")
Session.add_file_entries(session.pid, [%{type: :file, name: "image.jpg", file: image_file}])
- runtime = connected_noop_runtime(self())
- Session.set_runtime(session.pid, runtime)
+ set_noop_runtime(session.pid, self())
send(session.pid, {:runtime_file_entry_path_request, self(), "image.jpg"})
assert_receive {:runtime_file_entry_path_reply, {:ok, path}}
@@ -1745,8 +1741,7 @@ defmodule Livebook.SessionTest do
Session.add_file_entries(session.pid, [%{type: :url, name: "image.jpg", url: url}])
- runtime = connected_noop_runtime(self())
- Session.set_runtime(session.pid, runtime)
+ set_noop_runtime(session.pid, self())
send(session.pid, {:runtime_file_entry_path_request, self(), "image.jpg"})
assert_receive {:runtime_file_entry_path_reply, {:ok, path}}
@@ -1773,8 +1768,7 @@ defmodule Livebook.SessionTest do
Session.add_file_entries(session.pid, [%{type: :url, name: "image.jpg", url: url}])
- runtime = connected_noop_runtime(self())
- Session.set_runtime(session.pid, runtime)
+ set_noop_runtime(session.pid, self())
send(session.pid, {:runtime_file_entry_path_request, self(), "image.jpg"})
assert_receive {:runtime_file_entry_path_reply, {:ok, path}}
@@ -1800,8 +1794,7 @@ defmodule Livebook.SessionTest do
Session.add_file_entries(session.pid, [%{type: :url, name: "image.jpg", url: url}])
- runtime = connected_noop_runtime(self())
- Session.set_runtime(session.pid, runtime)
+ set_noop_runtime(session.pid, self())
send(session.pid, {:runtime_file_entry_path_request, self(), "image.jpg"})
assert_receive {:runtime_file_entry_path_reply, {:ok, path}}
@@ -1824,8 +1817,7 @@ defmodule Livebook.SessionTest do
Session.add_file_entries(session.pid, [%{type: :url, name: "image.jpg", url: url}])
- runtime = connected_noop_runtime(self())
- Session.set_runtime(session.pid, runtime)
+ set_noop_runtime(session.pid, self())
send(session.pid, {:runtime_file_entry_path_request, self(), "image.jpg"})
assert_receive {:runtime_file_entry_path_reply, {:ok, path}}
@@ -1853,8 +1845,7 @@ defmodule Livebook.SessionTest do
:ok = FileSystem.File.write(image_file, "")
Session.add_file_entries(session.pid, [%{type: :attachment, name: "image2.jpg"}])
- runtime = connected_noop_runtime(self())
- Session.set_runtime(session.pid, runtime)
+ set_noop_runtime(session.pid, self())
send(session.pid, {:runtime_file_entry_path_request, self(), "image.jpg"})
assert_receive {:runtime_file_entry_path_reply, {:ok, path}}
@@ -1877,8 +1868,7 @@ defmodule Livebook.SessionTest do
Session.add_file_entries(session.pid, [%{type: :url, name: "image.jpg", url: url}])
- runtime = connected_noop_runtime(self())
- Session.set_runtime(session.pid, runtime)
+ set_noop_runtime(session.pid, self())
send(session.pid, {:runtime_file_entry_path_request, self(), "image.jpg"})
assert_receive {:runtime_file_entry_path_reply, {:ok, path}}
@@ -1905,8 +1895,7 @@ defmodule Livebook.SessionTest do
test "replies with error when the session does not use teams hub" do
session = start_session()
- runtime = connected_noop_runtime(self())
- Session.set_runtime(session.pid, runtime)
+ set_noop_runtime(session.pid, self())
send(session.pid, {:runtime_user_info_request, self(), "c1"})
assert_receive {:runtime_user_info_reply, {:error, :not_available}}
@@ -1916,8 +1905,7 @@ defmodule Livebook.SessionTest do
notebook = %{Notebook.new() | teams_enabled: true}
session = start_session(notebook: notebook)
- runtime = connected_noop_runtime(self())
- Session.set_runtime(session.pid, runtime)
+ set_noop_runtime(session.pid, self())
send(session.pid, {:runtime_user_info_request, self(), "c1"})
assert_receive {:runtime_user_info_reply, {:error, :not_found}}
@@ -1936,8 +1924,7 @@ defmodule Livebook.SessionTest do
{_, client_id} = Session.register_client(session.pid, self(), user)
- runtime = connected_noop_runtime(self())
- Session.set_runtime(session.pid, runtime)
+ set_noop_runtime(session.pid, self())
send(session.pid, {:runtime_user_info_request, self(), client_id})
assert_receive {:runtime_user_info_reply, {:ok, user_info}}
@@ -1959,8 +1946,7 @@ defmodule Livebook.SessionTest do
{_section_id, cell_id} = insert_section_and_cell(session.pid)
- runtime = connected_noop_runtime()
- Session.set_runtime(session.pid, runtime)
+ set_noop_runtime(session.pid)
user = Livebook.Users.User.new()
Session.register_client(session.pid, self(), user)
@@ -2056,15 +2042,8 @@ defmodule Livebook.SessionTest do
{section_id, cell_id}
end
- defp connected_noop_runtime(trace_to \\ nil) do
- {:ok, runtime} = Livebook.Runtime.NoopRuntime.new(trace_to) |> Livebook.Runtime.connect()
- runtime
- end
-
- defp wait_for_session_update(session_pid) do
- # This call is synchronous, so it gives the session time
- # for handling the previously sent change messages.
- Session.get_data(session_pid)
- :ok
+ defp set_noop_runtime(session_pid, trace_to \\ nil) do
+ runtime = Livebook.Runtime.NoopRuntime.new(trace_to)
+ Session.set_runtime(session_pid, runtime)
end
end
diff --git a/test/livebook_web/controllers/session_controller_test.exs b/test/livebook_web/controllers/session_controller_test.exs
index 084b85abe..9253e7583 100644
--- a/test/livebook_web/controllers/session_controller_test.exs
+++ b/test/livebook_web/controllers/session_controller_test.exs
@@ -438,9 +438,11 @@ defmodule LivebookWeb.SessionControllerTest do
defp start_session_and_request_asset(conn, notebook, hash) do
{:ok, session} = Sessions.create_session(notebook: notebook)
+
# We need runtime in place to actually copy the archive
- {:ok, runtime} = Livebook.Runtime.Embedded.new() |> Livebook.Runtime.connect()
- Session.set_runtime(session.pid, runtime)
+ Session.subscribe(session.id)
+ Session.connect_runtime(session.pid)
+ assert_receive {:operation, {:runtime_connected, _, _}}
conn = get(conn, ~p"/public/sessions/#{session.id}/assets/#{hash}/main.js")
diff --git a/test/livebook_web/live/session_live_test.exs b/test/livebook_web/live/session_live_test.exs
index 942656d10..d88448f0a 100644
--- a/test/livebook_web/live/session_live_test.exs
+++ b/test/livebook_web/live/session_live_test.exs
@@ -134,26 +134,12 @@ defmodule LivebookWeb.SessionLiveTest do
continue_fun.()
end
- test "reevaluting the setup cell", %{conn: conn, session: session} do
- Session.subscribe(session.id)
- evaluate_setup(session.pid)
-
- {:ok, view, _} = live(conn, ~p"/sessions/#{session.id}")
-
- view
- |> element(~s{[data-el-session]})
- |> render_hook("queue_cell_evaluation", %{"cell_id" => "setup"})
-
- assert_receive {:operation, {:set_runtime, _pid, %{} = _runtime}}
- end
-
test "reevaluting the setup cell with dependencies cache disabled",
%{conn: conn, session: session} do
Session.subscribe(session.id)
- # Start the standalone runtime, to encapsulate env var changes
- {:ok, runtime} = Runtime.ElixirStandalone.new() |> Runtime.connect()
- Session.set_runtime(session.pid, runtime)
+ # Use the standalone runtime, to encapsulate env var changes
+ Session.set_runtime(session.pid, Runtime.Standalone.new())
evaluate_setup(session.pid)
@@ -294,8 +280,9 @@ defmodule LivebookWeb.SessionLiveTest do
:ok = FileSystem.File.write(image_file, "content")
Session.add_file_entries(session.pid, [%{type: :attachment, name: "file.bin"}])
- {:ok, runtime} = Livebook.Runtime.NoopRuntime.new() |> Livebook.Runtime.connect()
- Session.set_runtime(session.pid, runtime)
+ Session.subscribe(session.id)
+ Session.set_runtime(session.pid, Livebook.Runtime.NoopRuntime.new(self()))
+ connect_and_await_runtime(session.pid)
{:ok, view, _} = live(conn, ~p"/sessions/#{session.id}")
@@ -340,8 +327,9 @@ defmodule LivebookWeb.SessionLiveTest do
:ok = FileSystem.File.write(image_file, "content")
Session.add_file_entries(session.pid, [%{type: :attachment, name: "image.jpg"}])
- {:ok, runtime} = Livebook.Runtime.NoopRuntime.new() |> Livebook.Runtime.connect()
- Session.set_runtime(session.pid, runtime)
+ Session.subscribe(session.id)
+ Session.set_runtime(session.pid, Livebook.Runtime.NoopRuntime.new(self()))
+ connect_and_await_runtime(session.pid)
{:ok, view, _} = live(conn, ~p"/sessions/#{session.id}")
@@ -370,8 +358,9 @@ defmodule LivebookWeb.SessionLiveTest do
section_id = insert_section(session.pid)
cell_id = insert_text_cell(session.pid, section_id, :code)
- {:ok, runtime} = Livebook.Runtime.NoopRuntime.new() |> Livebook.Runtime.connect()
- Session.set_runtime(session.pid, runtime)
+ Session.subscribe(session.id)
+ Session.set_runtime(session.pid, Livebook.Runtime.NoopRuntime.new(self()))
+ connect_and_await_runtime(session.pid)
{:ok, view, _} = live(conn, ~p"/sessions/#{session.id}")
@@ -887,8 +876,8 @@ defmodule LivebookWeb.SessionLiveTest do
%{conn: conn, session: session} do
insert_section(session.pid)
- {:ok, runtime} = Runtime.Embedded.new() |> Runtime.connect()
- Session.set_runtime(session.pid, runtime)
+ Session.subscribe(session.id)
+ connect_and_await_runtime(session.pid)
{:ok, view, _} = live(conn, ~p"/sessions/#{session.id}")
@@ -907,23 +896,22 @@ defmodule LivebookWeb.SessionLiveTest do
end
describe "runtime settings" do
- test "connecting to elixir standalone updates connect button to reconnect",
+ test "connecting to standalone updates connect button to reconnect",
%{conn: conn, session: session} do
{:ok, view, _} = live(conn, ~p"/sessions/#{session.id}/settings/runtime")
Session.subscribe(session.id)
view
- |> element("button", "Elixir standalone")
+ |> element("#runtime-settings-modal button", "Standalone")
|> render_click()
- [elixir_standalone_view] = live_children(view)
-
- elixir_standalone_view
- |> element("button", "Connect")
+ view
+ |> element("#runtime-settings-modal button", "Connect")
|> render_click()
- assert_receive {:operation, {:set_runtime, _pid, %Runtime.ElixirStandalone{} = runtime}}
+ assert_receive {:operation, {:set_runtime, _pid, %Runtime.Standalone{}}}
+ assert_receive {:operation, {:runtime_connected, _pid, %Runtime.Standalone{} = runtime}}
page = render(view)
assert page =~ Atom.to_string(runtime.node)
@@ -932,13 +920,12 @@ defmodule LivebookWeb.SessionLiveTest do
end
test "disconnecting a connected node", %{conn: conn, session: session} do
- {:ok, runtime} = Livebook.Runtime.NoopRuntime.new(self()) |> Livebook.Runtime.connect()
- Session.set_runtime(session.pid, runtime)
+ Session.subscribe(session.id)
+ Session.set_runtime(session.pid, Livebook.Runtime.NoopRuntime.new(self()))
+ connect_and_await_runtime(session.pid)
{:ok, view, _} = live(conn, ~p"/sessions/#{session.id}")
- Session.subscribe(session.id)
-
assert render(view) =~ "No connected nodes"
# Mimic the runtime reporting a connected node
@@ -956,6 +943,229 @@ defmodule LivebookWeb.SessionLiveTest do
assert_receive {:runtime_trace, :disconnect_node, [^node]}
end
+
+ test "configuring fly runtime", %{conn: conn, session: session} do
+ {:ok, view, _} = live(conn, ~p"/sessions/#{session.id}/settings/runtime")
+
+ Session.subscribe(session.id)
+
+ view
+ |> element("#runtime-settings-modal button", "Fly.io machine")
+ |> render_click()
+
+ Livebook.FlyAPI.stub(fn conn when conn.method == "POST" ->
+ Req.Test.json(conn, %{
+ "data" => nil,
+ "errors" => [
+ %{
+ "extensions" => %{"code" => "UNAUTHORIZED"},
+ "locations" => [%{"column" => 3, "line" => 2}],
+ "message" => "You must be authenticated to view this.",
+ "path" => ["organizations"]
+ }
+ ]
+ })
+ end)
+
+ view
+ |> element(~s{form[phx-change="set_token"]})
+ |> render_change(%{token: "invalid"})
+
+ assert render_async(view) =~ "Error: could not authorize with the given token"
+
+ Livebook.FlyAPI.stub(fn conn when conn.method == "POST" ->
+ Req.Test.json(conn, %{
+ "data" => %{
+ "organizations" => %{
+ "nodes" => [
+ %{
+ "id" => "1",
+ "name" => "Grumpy Cat",
+ "rawSlug" => "grumpy-cat",
+ "slug" => "personal"
+ }
+ ]
+ },
+ "platform" => %{
+ "regions" => [
+ %{"code" => "ams", "name" => "Amsterdam, Netherlands"},
+ %{"code" => "fra", "name" => "Frankfurt, Germany"}
+ ],
+ "requestRegion" => "fra"
+ }
+ }
+ })
+ end)
+
+ view
+ |> element(~s{form[phx-change="set_token"]})
+ |> render_change(%{token: "valid"})
+
+ assert render_async(view) =~ "Grumpy Cat"
+
+ # Selects the closest region by default
+ assert view
+ |> element(~s/select[name="region"] option[value="fra"][selected]/)
+ |> has_element?()
+
+ Livebook.FlyAPI.stub(fn conn
+ when conn.method == "GET" and
+ conn.path_info == ["v1", "apps", "new-app", "volumes"] ->
+ conn
+ |> Plug.Conn.put_status(404)
+ |> Req.Test.json(%{"error" => "App not found"})
+ end)
+
+ # Create a new app
+ view
+ |> element(~s{form[phx-change="set_app_name"]})
+ |> render_change(%{app_name: "new-app"})
+
+ assert render_async(view) =~ ~r/App .*new-app.* does not exist yet/
+
+ Livebook.FlyAPI.stub(fn conn
+ when conn.method == "POST" and conn.path_info == ["v1", "apps"] ->
+ Plug.Conn.send_resp(conn, 201, "")
+ end)
+
+ view
+ |> element(~s/button[phx-click="create_app"]/)
+ |> render_click()
+
+ assert render_async(view) =~ "CPU kind"
+
+ # Create a new volume
+
+ Livebook.FlyAPI.stub(fn conn
+ when conn.method == "POST" and
+ conn.path_info == ["v1", "apps", "new-app", "volumes"] ->
+ Req.Test.json(conn, %{
+ "id" => "vol_1",
+ "name" => "new_volume",
+ "region" => "ams",
+ "size_gb" => 1,
+ "state" => "created"
+ })
+ end)
+
+ view
+ |> element(~s/button[phx-click="new_volume"]/)
+ |> render_click()
+
+ view
+ |> element(~s/form[phx-submit="create_volume"]/)
+ |> render_submit(%{volume: %{name: "new_volume", size_gb: "1"}})
+
+ assert render_async(view) =~ "name: new_volume"
+
+ # The volume is automatically selected
+ assert view
+ |> element(~s/select[name="volume_id"] option[value="vol_1"][selected]/)
+ |> has_element?()
+
+ # Delete the volume
+
+ Livebook.FlyAPI.stub(fn conn
+ when conn.method == "DELETE" and
+ conn.path_info == [
+ "v1",
+ "apps",
+ "new-app",
+ "volumes",
+ "vol_1"
+ ] ->
+ Req.Test.json(conn, %{})
+ end)
+
+ view
+ |> element(~s/button[phx-click="delete_volume"]/)
+ |> render_click()
+
+ view
+ |> element(~s/button[phx-click="confirm_delete_volume"]/)
+ |> render_click()
+
+ refute render_async(view) =~ "name: new_volume"
+
+ assert view
+ |> element(~s/select[name="volume_id"] option[value=""][selected]/)
+ |> has_element?()
+
+ # We do not actually connect the runtime. We test connecting
+ # againast the real API separately
+ end
+
+ test "populates fly runtime config form existing runtime", %{conn: conn, session: session} do
+ runtime =
+ Runtime.Fly.new(%{
+ token: "my-token",
+ app_name: "my-app",
+ region: "ams",
+ cpu_kind: "performance",
+ cpus: 1,
+ memory_gb: 1,
+ gpu_kind: nil,
+ gpus: nil,
+ volume_id: "vol_1",
+ docker_tag: "edge"
+ })
+
+ Session.set_runtime(session.pid, runtime)
+
+ Livebook.FlyAPI.stub(fn
+ conn when conn.method == "POST" ->
+ Req.Test.json(conn, %{
+ "data" => %{
+ "organizations" => %{
+ "nodes" => [
+ %{
+ "id" => "1",
+ "name" => "Grumpy Cat",
+ "rawSlug" => "grumpy-cat",
+ "slug" => "personal"
+ }
+ ]
+ },
+ "platform" => %{
+ "regions" => [
+ %{"code" => "ams", "name" => "Amsterdam, Netherlands"},
+ %{"code" => "fra", "name" => "Frankfurt, Germany"}
+ ],
+ "requestRegion" => "fra"
+ }
+ }
+ })
+
+ conn
+ when conn.method == "GET" and
+ conn.path_info == ["v1", "apps", "my-app", "volumes"] ->
+ Req.Test.json(conn, [
+ %{
+ "id" => "vol_1",
+ "name" => "new_volume",
+ "region" => "ams",
+ "size_gb" => 1,
+ "state" => "created"
+ }
+ ])
+ end)
+
+ {:ok, view, _} = live(conn, ~p"/sessions/#{session.id}/settings/runtime")
+
+ assert render_async(view) =~ "Grumpy Cat"
+
+ assert view
+ |> element(~s/select[name="region"] option[value="ams"][selected]/)
+ |> has_element?()
+
+ assert view
+ |> element(~s/select[name="volume_id"] option[value="vol_1"][selected]/)
+ |> has_element?()
+
+ assert view
+ |> element(~s/select[name="specs[cpu_kind]"] option[value="performance"][selected]/)
+ |> has_element?()
+ end
end
describe "persistence settings" do
@@ -1057,8 +1267,8 @@ defmodule LivebookWeb.SessionLiveTest do
section_id = insert_section(session.pid)
cell_id = insert_text_cell(session.pid, section_id, :code, "Process.sleep(10)")
- {:ok, runtime} = Runtime.Embedded.new() |> Runtime.connect()
- Session.set_runtime(session.pid, runtime)
+ Session.subscribe(session.id)
+ connect_and_await_runtime(session.pid)
{:ok, view, _} = live(conn, ~p"/sessions/#{session.id}")
@@ -1750,7 +1960,7 @@ defmodule LivebookWeb.SessionLiveTest do
end
describe "environment variables" do
- test "outputs persisted env var from ets", %{conn: conn, session: session} do
+ test "outputs persisted env var from settings", %{conn: conn, session: session} do
Session.subscribe(session.id)
{:ok, view, _} = live(conn, ~p"/sessions/#{session.id}")
@@ -1802,9 +2012,8 @@ defmodule LivebookWeb.SessionLiveTest do
@tag :tmp_dir
test "outputs persisted PATH delimited with os PATH env var",
%{conn: conn, session: session, tmp_dir: tmp_dir} do
- # Start the standalone runtime, to encapsulate env var changes
- {:ok, runtime} = Runtime.ElixirStandalone.new() |> Runtime.connect()
- Session.set_runtime(session.pid, runtime)
+ # Use the standalone runtime, to encapsulate env var changes
+ Session.set_runtime(session.pid, Runtime.Standalone.new())
separator =
case :os.type() do
diff --git a/test/livebook_web/plugs/proxy_plug_test.exs b/test/livebook_web/plugs/proxy_plug_test.exs
index 69cbac15b..36da3ec65 100644
--- a/test/livebook_web/plugs/proxy_plug_test.exs
+++ b/test/livebook_web/plugs/proxy_plug_test.exs
@@ -6,7 +6,7 @@ defmodule LivebookWeb.ProxyPlugTest do
require Phoenix.LiveViewTest
import Livebook.AppHelpers
- alias Livebook.{Notebook, Runtime, Session, Sessions}
+ alias Livebook.{Notebook, Session, Sessions}
describe "session" do
test "returns error when session doesn't exist", %{conn: conn} do
@@ -28,9 +28,7 @@ defmodule LivebookWeb.ProxyPlugTest do
test "returns the proxied response defined in notebook", %{conn: conn} do
%{sections: [%{cells: [%{id: cell_id}]}]} = notebook = proxy_notebook()
{:ok, session} = Sessions.create_session(notebook: notebook)
- {:ok, runtime} = Runtime.Embedded.new() |> Runtime.connect()
- Session.set_runtime(session.pid, runtime)
Session.subscribe(session.id)
Session.queue_cell_evaluation(session.pid, cell_id)
diff --git a/test/support/noop_runtime.ex b/test/support/noop_runtime.ex
index 8e06352f5..05a8df02f 100644
--- a/test/support/noop_runtime.ex
+++ b/test/support/noop_runtime.ex
@@ -2,10 +2,10 @@ defmodule Livebook.Runtime.NoopRuntime do
# A runtime that doesn't do any actual evaluation,
# thus not requiring any underlying resources.
- defstruct [:started, :trace_to]
+ defstruct [:trace_to]
def new(trace_to \\ nil) do
- %__MODULE__{started: false, trace_to: trace_to}
+ %__MODULE__{trace_to: trace_to}
end
defimpl Livebook.Runtime do
@@ -13,11 +13,17 @@ defmodule Livebook.Runtime.NoopRuntime do
[{"Type", "Noop"}]
end
- def connect(runtime), do: {:ok, %{runtime | started: true}}
- def connected?(runtime), do: runtime.started
+ def connect(runtime) do
+ caller = self()
+
+ spawn(fn ->
+ send(caller, {:runtime_connect_done, self(), {:ok, runtime}})
+ end)
+ end
+
def take_ownership(_, _), do: make_ref()
- def disconnect(runtime), do: {:ok, %{runtime | started: false}}
- def duplicate(_), do: Livebook.Runtime.NoopRuntime.new()
+ def disconnect(_), do: :ok
+ def duplicate(runtime), do: Livebook.Runtime.NoopRuntime.new(runtime.trace_to)
def evaluate_code(_, _, _, _, _, _ \\ []), do: :ok
def forget_evaluation(_, _), do: :ok
@@ -61,8 +67,6 @@ defmodule Livebook.Runtime.NoopRuntime do
def search_packages(_, _, _), do: make_ref()
- def disable_dependencies_cache(_), do: :ok
-
def put_system_envs(_, _), do: :ok
def delete_system_envs(_, _), do: :ok
diff --git a/test/support/session_helpers.ex b/test/support/session_helpers.ex
index e73a170ea..620e45912 100644
--- a/test/support/session_helpers.ex
+++ b/test/support/session_helpers.ex
@@ -13,6 +13,11 @@ defmodule Livebook.SessionHelpers do
:ok
end
+ def connect_and_await_runtime(session_pid) do
+ Session.connect_runtime(session_pid)
+ assert_receive {:operation, {:runtime_connected, _, _}}
+ end
+
def evaluate_setup(session_pid) do
Session.queue_cell_evaluation(session_pid, "setup")
assert_receive {:operation, {:add_cell_evaluation_response, _, "setup", _, _}}
diff --git a/test/test_helper.exs b/test/test_helper.exs
index 0e26ca154..c8427d13b 100644
--- a/test/test_helper.exs
+++ b/test/test_helper.exs
@@ -1,21 +1,21 @@
-# Start manager on the current node and configure it not to
-# terminate automatically, so there is no race condition
-# when starting/stopping Embedded runtimes in parallel
+# Start manager on the current node and configure it not to terminate
+# automatically, so that we can use it to start runtime servers
+# explicitly
Livebook.Runtime.ErlDist.NodeManager.start(
auto_termination: false,
unload_modules_on_termination: false
)
-# Use the embedded runtime in tests by default, so they are
-# cheaper to run. Other runtimes can be tested by starting
-# and setting them explicitly
+# Use the embedded runtime in tests by default, so they are cheaper
+# to run. Other runtimes can be tested by setting them explicitly
Application.put_env(:livebook, :default_runtime, Livebook.Runtime.Embedded.new())
Application.put_env(:livebook, :default_app_runtime, Livebook.Runtime.Embedded.new())
Application.put_env(:livebook, :runtime_modules, [
- Livebook.Runtime.ElixirStandalone,
+ Livebook.Runtime.Standalone,
Livebook.Runtime.Attached,
- Livebook.Runtime.Embedded
+ Livebook.Runtime.Embedded,
+ Livebook.Runtime.Fly
])
defmodule Livebook.Runtime.Embedded.Packages do
@@ -71,15 +71,9 @@ teams_exclude =
[:teams_integration]
end
-# ELIXIR_ERL_OPTIONS="-epmd_module Elixir.Livebook.EPMD -start_epmd false -erl_epmd_port 0" LIVEBOOK_EPMDLESS=true mix test
-epmd_exclude =
- if Livebook.Config.epmdless?() do
- [:with_epmd, :teams_integration]
- else
- [:without_epmd]
- end
+fly_exclude = if System.get_env("TEST_FLY_API_TOKEN"), do: [], else: [:fly]
ExUnit.start(
assert_receive_timeout: if(windows?, do: 2_500, else: 1_500),
- exclude: erl_docs_exclude ++ windows_exclude ++ teams_exclude ++ epmd_exclude
+ exclude: erl_docs_exclude ++ windows_exclude ++ teams_exclude ++ fly_exclude
)