livebook-dev/livebook
1
1
Fork 0
mirror of https://github.com/livebook-dev/livebook.git synced 2025-10-05 11:15:25 +08:00
Code Issues Projects Releases Packages Wiki Activity

Track evaluation dependencies and cache results (#1517)

Browse source
This commit is contained in:
Jonatan Kłosko 2022-11-09 14:40:44 +01:00 • committed by GitHub
parent 7b1addb7eb
commit 484e47142a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
22 changed files with 2234 additions and 1153 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.github/workflows/test.yaml vendored
View file

@ -6,7 +6,8 @@ on:
- main
env:
otp: "25.0"
elixir: "1.14.0"
# TODO: update to v1.14.2 once it is out
elixir: "main"
jobs:
main:
runs-on: ubuntu-latest

17
lib/livebook/notebook.ex
View file

@ -471,23 +471,6 @@ defmodule Livebook.Notebook do
|> Enum.filter(fn {cell, _} -> MapSet.member?(child_cell_ids, cell.id) end)
end
@doc """
Returns the list with the given parent cells and all of
their child cells.
The cells are not ordered in any secific way.
"""
@spec cell_ids_with_children(t(), list(Cell.id())) :: list(Cell.id())
def cell_ids_with_children(notebook, parent_cell_ids) do
graph = cell_dependency_graph(notebook)
for parent_id <- parent_cell_ids,
leaf_id <- Graph.leaves(graph),
cell_id <- Graph.find_path(graph, leaf_id, parent_id),
uniq: true,
do: cell_id
end
@doc """
Computes cell dependency graph.

66
lib/livebook/runtime.ex
View file

@ -26,12 +26,15 @@ defprotocol Livebook.Runtime do
@typedoc """
A pair identifying evaluation together with its container.
When the evaluation reference is `nil`, the `locator` points to
a container and may be used to represent its default evaluation
context.
"""
@type locator :: {container_ref(), evaluation_ref() | nil}
@type locator :: {container_ref(), evaluation_ref()}
@typedoc """
A sequence of locators representing a multi-stage evaluation.
The evaluation locators should be ordered from most recent to oldest.
"""
@type parent_locators :: list(locator())
@typedoc """
An output emitted during evaluation or as the final result.
@ -64,12 +67,22 @@ defprotocol Livebook.Runtime do
| {:error, message :: String.t(), type :: {:missing_secret, String.t()} | :other}
@typedoc """
Additional information about a complted evaluation.
Additional information about a completed evaluation.
## Identifiers
When possible, the metadata may include a list of identifiers (such
as variables, modules, imports) used during evaluation, and a list
of identifiers defined along with the version (such as a hash digest
of the underlying value). With this information, Livebook can track
dependencies between evaluations and avoids unnecessary reevaluations.
"""
@type evaluation_response_metadata :: %{
evaluation_time_ms: non_neg_integer(),
code_error: code_error(),
memory_usage: runtime_memory()
memory_usage: runtime_memory(),
identifiers_used: list(identifier :: term()) | :unknown,
identifiers_defined: %{(identifier :: term()) => version :: term()}
}
@typedoc """
@ -306,9 +319,8 @@ defprotocol Livebook.Runtime do
be evaluated as well as the evaluation reference to store the
resulting context under.
Additionally, `base_locator` points to a previous evaluation to be
used as the starting point of this evaluation. If not applicable,
the previous evaluation reference may be specified as `nil`.
Additionally, `parent_locators` points to a sequence of previous
evaluations to be used as the starting point of this evaluation.
## Communication
@ -347,8 +359,8 @@ defprotocol Livebook.Runtime do
* `:smart_cell_ref` - a reference of the smart cell which code is
to be evaluated, if applicable
"""
@spec evaluate_code(t(), String.t(), locator(), locator(), keyword()) :: :ok
def evaluate_code(runtime, code, locator, base_locator, opts \\ [])
@spec evaluate_code(t(), String.t(), locator(), parent_locators(), keyword()) :: :ok
def evaluate_code(runtime, code, locator, parent_locators, opts \\ [])
@doc """
Disposes of an evaluation identified by the given locator.
@ -378,11 +390,11 @@ defprotocol Livebook.Runtime do
* `{:runtime_intellisense_response, ref, request, response}`.
The given `base_locator` idenfities an evaluation that may be
used as the context when resolving the request (if relevant).
The given `parent_locators` identifies a sequence of evaluations
that may be used as the context when resolving the request (if relevant).
"""
@spec handle_intellisense(t(), pid(), intellisense_request(), locator()) :: reference()
def handle_intellisense(runtime, send_to, request, base_locator)
@spec handle_intellisense(t(), pid(), intellisense_request(), parent_locators()) :: reference()
def handle_intellisense(runtime, send_to, request, parent_locators)
@doc """
Reads file at the given absolute path within the runtime file system.
@ -401,9 +413,9 @@ defprotocol Livebook.Runtime do
The cell may depend on evaluation context to provide a better user
experience, for instance it may suggest relevant variable names.
Similarly to `evaluate_code/5`, `base_locator` must be specified
pointing to the evaluation to use as the context. When the locator
changes, it can be updated with `set_smart_cell_base_locator/3`.
Similarly to `evaluate_code/5`, `parent_locators` must be specified
pointing to the sequence of evaluations to use as the context. When
the sequence changes, it can be updated with `set_smart_cell_parent_locators/3`.
Once the cell starts, the runtime sends the following message
@ -425,16 +437,22 @@ defprotocol Livebook.Runtime do
state later. Note that for persistence they get serialized and
deserialized as JSON.
"""
@spec start_smart_cell(t(), String.t(), smart_cell_ref(), smart_cell_attrs(), locator()) :: :ok
def start_smart_cell(runtime, kind, ref, attrs, base_locator)
@spec start_smart_cell(
t(),
String.t(),
smart_cell_ref(),
smart_cell_attrs(),
parent_locators()
) :: :ok
def start_smart_cell(runtime, kind, ref, attrs, parent_locators)
@doc """
Updates the locator used by a smart cell as its context.
Updates the parent locator used by a smart cell as its context.
See `start_smart_cell/5` for more details.
"""
@spec set_smart_cell_base_locator(t(), smart_cell_ref(), locator()) :: :ok
def set_smart_cell_base_locator(runtime, ref, base_locator)
@spec set_smart_cell_parent_locators(t(), smart_cell_ref(), parent_locators()) :: :ok
def set_smart_cell_parent_locators(runtime, ref, parent_locators)
@doc """
Stops smart cell identified by the given reference.

16
lib/livebook/runtime/attached.ex
View file

@ -95,8 +95,8 @@ defimpl Livebook.Runtime, for: Livebook.Runtime.Attached do
Livebook.Runtime.Attached.new(runtime.node, runtime.cookie)
end
def evaluate_code(runtime, code, locator, base_locator, opts \\ []) do
RuntimeServer.evaluate_code(runtime.server_pid, code, locator, base_locator, opts)
def evaluate_code(runtime, code, locator, parent_locators, opts \\ []) do
RuntimeServer.evaluate_code(runtime.server_pid, code, locator, parent_locators, opts)
end
def forget_evaluation(runtime, locator) do
@ -107,20 +107,20 @@ defimpl Livebook.Runtime, for: Livebook.Runtime.Attached do
RuntimeServer.drop_container(runtime.server_pid, container_ref)
end
def handle_intellisense(runtime, send_to, request, base_locator) do
RuntimeServer.handle_intellisense(runtime.server_pid, send_to, request, base_locator)
def handle_intellisense(runtime, send_to, request, parent_locators) do
RuntimeServer.handle_intellisense(runtime.server_pid, send_to, request, parent_locators)
end
def read_file(runtime, path) do
RuntimeServer.read_file(runtime.server_pid, path)
end
def start_smart_cell(runtime, kind, ref, attrs, base_locator) do
RuntimeServer.start_smart_cell(runtime.server_pid, kind, ref, attrs, base_locator)
def start_smart_cell(runtime, kind, ref, attrs, parent_locators) do
RuntimeServer.start_smart_cell(runtime.server_pid, kind, ref, attrs, parent_locators)
end
def set_smart_cell_base_locator(runtime, ref, base_locator) do
RuntimeServer.set_smart_cell_base_locator(runtime.server_pid, ref, base_locator)
def set_smart_cell_parent_locators(runtime, ref, parent_locators) do
RuntimeServer.set_smart_cell_parent_locators(runtime.server_pid, ref, parent_locators)
end
def stop_smart_cell(runtime, ref) do

16
lib/livebook/runtime/elixir_standalone.ex
View file

@ -179,8 +179,8 @@ defimpl Livebook.Runtime, for: Livebook.Runtime.ElixirStandalone do
Livebook.Runtime.ElixirStandalone.new()
end
def evaluate_code(runtime, code, locator, base_locator, opts \\ []) do
RuntimeServer.evaluate_code(runtime.server_pid, code, locator, base_locator, opts)
def evaluate_code(runtime, code, locator, parent_locators, opts \\ []) do
RuntimeServer.evaluate_code(runtime.server_pid, code, locator, parent_locators, opts)
end
def forget_evaluation(runtime, locator) do
@ -191,20 +191,20 @@ defimpl Livebook.Runtime, for: Livebook.Runtime.ElixirStandalone do
RuntimeServer.drop_container(runtime.server_pid, container_ref)
end
def handle_intellisense(runtime, send_to, request, base_locator) do
RuntimeServer.handle_intellisense(runtime.server_pid, send_to, request, base_locator)
def handle_intellisense(runtime, send_to, request, parent_locators) do
RuntimeServer.handle_intellisense(runtime.server_pid, send_to, request, parent_locators)
end
def read_file(runtime, path) do
RuntimeServer.read_file(runtime.server_pid, path)
end
def start_smart_cell(runtime, kind, ref, attrs, base_locator) do
RuntimeServer.start_smart_cell(runtime.server_pid, kind, ref, attrs, base_locator)
def start_smart_cell(runtime, kind, ref, attrs, parent_locators) do
RuntimeServer.start_smart_cell(runtime.server_pid, kind, ref, attrs, parent_locators)
end
def set_smart_cell_base_locator(runtime, ref, base_locator) do
RuntimeServer.set_smart_cell_base_locator(runtime.server_pid, ref, base_locator)
def set_smart_cell_parent_locators(runtime, ref, parent_locators) do
RuntimeServer.set_smart_cell_parent_locators(runtime.server_pid, ref, parent_locators)
end
def stop_smart_cell(runtime, ref) do

16
lib/livebook/runtime/embedded.ex
View file

@ -76,8 +76,8 @@ defimpl Livebook.Runtime, for: Livebook.Runtime.Embedded do
Livebook.Runtime.Embedded.new()
end
def evaluate_code(runtime, code, locator, base_locator, opts \\ []) do
RuntimeServer.evaluate_code(runtime.server_pid, code, locator, base_locator, opts)
def evaluate_code(runtime, code, locator, parent_locators, opts \\ []) do
RuntimeServer.evaluate_code(runtime.server_pid, code, locator, parent_locators, opts)
end
def forget_evaluation(runtime, locator) do
@ -88,20 +88,20 @@ defimpl Livebook.Runtime, for: Livebook.Runtime.Embedded do
RuntimeServer.drop_container(runtime.server_pid, container_ref)
end
def handle_intellisense(runtime, send_to, request, base_locator) do
RuntimeServer.handle_intellisense(runtime.server_pid, send_to, request, base_locator)
def handle_intellisense(runtime, send_to, request, parent_locators) do
RuntimeServer.handle_intellisense(runtime.server_pid, send_to, request, parent_locators)
end
def read_file(runtime, path) do
RuntimeServer.read_file(runtime.server_pid, path)
end
def start_smart_cell(runtime, kind, ref, attrs, base_locator) do
RuntimeServer.start_smart_cell(runtime.server_pid, kind, ref, attrs, base_locator)
def start_smart_cell(runtime, kind, ref, attrs, parent_locators) do
RuntimeServer.start_smart_cell(runtime.server_pid, kind, ref, attrs, parent_locators)
end
def set_smart_cell_base_locator(runtime, ref, base_locator) do
RuntimeServer.set_smart_cell_base_locator(runtime.server_pid, ref, base_locator)
def set_smart_cell_parent_locators(runtime, ref, parent_locators) do
RuntimeServer.set_smart_cell_parent_locators(runtime.server_pid, ref, parent_locators)
end
def stop_smart_cell(runtime, ref) do

1
lib/livebook/runtime/erl_dist.ex
View file

@ -23,6 +23,7 @@ defmodule Livebook.Runtime.ErlDist do
[
Livebook.Runtime.Evaluator,
Livebook.Runtime.Evaluator.IOProxy,
Livebook.Runtime.Evaluator.Tracer,
Livebook.Runtime.Evaluator.ObjectTracker,
Livebook.Runtime.Evaluator.DefaultFormatter,
Livebook.Intellisense,

141
lib/livebook/runtime/erl_dist/runtime_server.ex
View file

@ -71,9 +71,15 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
See `Livebook.Runtime.Evaluator` for more details.
"""
@spec evaluate_code(pid(), String.t(), Runtime.locator(), Runtime.locator(), keyword()) :: :ok
def evaluate_code(pid, code, locator, base_locator, opts \\ []) do
GenServer.cast(pid, {:evaluate_code, code, locator, base_locator, opts})
@spec evaluate_code(
pid(),
String.t(),
Runtime.locator(),
Runtime.parent_locators(),
keyword()
) :: :ok
def evaluate_code(pid, code, locator, parent_locators, opts \\ []) do
GenServer.cast(pid, {:evaluate_code, code, locator, parent_locators, opts})
end
@doc """
@ -109,11 +115,11 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
pid(),
pid(),
Runtime.intellisense_request(),
Runtime.locator()
Runtime.Runtime.parent_locators()
) :: reference()
def handle_intellisense(pid, send_to, request, base_locator) do
def handle_intellisense(pid, send_to, request, parent_locators) do
ref = make_ref()
GenServer.cast(pid, {:handle_intellisense, send_to, ref, request, base_locator})
GenServer.cast(pid, {:handle_intellisense, send_to, ref, request, parent_locators})
ref
end
@ -144,18 +150,22 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
String.t(),
Runtime.smart_cell_ref(),
Runtime.smart_cell_attrs(),
Runtime.locator()
Runtime.Runtime.parent_locators()
) :: :ok
def start_smart_cell(pid, kind, ref, attrs, base_locator) do
GenServer.cast(pid, {:start_smart_cell, kind, ref, attrs, base_locator})
def start_smart_cell(pid, kind, ref, attrs, parent_locators) do
GenServer.cast(pid, {:start_smart_cell, kind, ref, attrs, parent_locators})
end
@doc """
Updates the locator with smart cell context.
Updates the parent locator used by a smart cell as its context.
"""
@spec set_smart_cell_base_locator(pid(), Runtime.smart_cell_ref(), Runtime.locator()) :: :ok
def set_smart_cell_base_locator(pid, ref, base_locator) do
GenServer.cast(pid, {:set_smart_cell_base_locator, ref, base_locator})
@spec set_smart_cell_parent_locators(
pid(),
Runtime.smart_cell_ref(),
Runtime.Runtime.parent_locators()
) :: :ok
def set_smart_cell_parent_locators(pid, ref, parent_locators) do
GenServer.cast(pid, {:set_smart_cell_parent_locators, ref, parent_locators})
end
@doc """
@ -332,25 +342,12 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
end
def handle_cast(
{:evaluate_code, code, {container_ref, evaluation_ref} = locator, base_locator, opts},
{:evaluate_code, code, {container_ref, evaluation_ref} = locator, parent_locators, opts},
state
) do
state = ensure_evaluator(state, container_ref)
base_evaluation_ref =
case base_locator do
{^container_ref, evaluation_ref} ->
evaluation_ref
{parent_container_ref, evaluation_ref} ->
Evaluator.initialize_from(
state.evaluators[container_ref],
state.evaluators[parent_container_ref],
evaluation_ref
)
nil
end
parent_evaluation_refs = evaluation_refs_for_container(state, container_ref, parent_locators)
{smart_cell_ref, opts} = Keyword.pop(opts, :smart_cell_ref)
smart_cell_info = smart_cell_ref && state.smart_cells[smart_cell_ref]
@ -374,7 +371,7 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
state.evaluators[container_ref],
code,
evaluation_ref,
base_evaluation_ref,
parent_evaluation_refs,
opts
)
@ -394,15 +391,31 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
{:noreply, state}
end
def handle_cast({:handle_intellisense, send_to, ref, request, base_locator}, state) do
{container_ref, evaluation_ref} = base_locator
evaluator = state.evaluators[container_ref]
def handle_cast({:handle_intellisense, send_to, ref, request, parent_locators}, state) do
{container_ref, parent_evaluation_refs} =
case parent_locators do
[] ->
{nil, []}
[{container_ref, _} | _] ->
parent_evaluation_refs =
parent_locators
# If there is a parent evaluator we ignore it and use whatever
# initial context we currently have in the evaluator. We sync
# initial context only on evaluation, since it may be blocking
|> Enum.take_while(&(elem(&1, 0) == container_ref))
|> Enum.map(&elem(&1, 1))
{container_ref, parent_evaluation_refs}
end
evaluator = container_ref && state.evaluators[container_ref]
intellisense_context =
if evaluator == nil or elem(request, 0) in [:format] do
Evaluator.intellisense_context()
else
Evaluator.intellisense_context(evaluator, evaluation_ref)
Evaluator.intellisense_context(evaluator, parent_evaluation_refs)
end
Task.Supervisor.start_child(state.task_supervisor, fn ->
@ -413,7 +426,7 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
{:noreply, state}
end
def handle_cast({:start_smart_cell, kind, ref, attrs, base_locator}, state) do
def handle_cast({:start_smart_cell, kind, ref, attrs, parent_locators}, state) do
definition = Enum.find(state.smart_cell_definitions, &(&1.kind == kind))
state =
@ -440,7 +453,7 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
pid: pid,
monitor_ref: Process.monitor(pid),
scan_binding: scan_binding,
base_locator: base_locator,
parent_locators: parent_locators,
scan_binding_pending: false,
scan_binding_monitor_ref: nil,
scan_eval_result: scan_eval_result
@ -457,11 +470,11 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
{:noreply, state}
end
def handle_cast({:set_smart_cell_base_locator, ref, base_locator}, state) do
def handle_cast({:set_smart_cell_parent_locators, ref, parent_locators}, state) do
state =
update_in(state.smart_cells[ref], fn
%{base_locator: ^base_locator} = info -> info
info -> scan_binding_async(ref, %{info | base_locator: base_locator}, state)
%{parent_locators: ^parent_locators} = info -> info
info -> scan_binding_async(ref, %{info | parent_locators: parent_locators}, state)
end)
{:noreply, state}
@ -608,12 +621,28 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
send(myself, {:scan_binding_ack, ref})
end
{container_ref, evaluation_ref} = info.base_locator
evaluator = state.evaluators[container_ref]
{container_ref, parent_evaluation_refs} =
case info.parent_locators do
[] ->
{nil, []}
[{container_ref, _} | _] = parent_locators ->
parent_evaluation_refs =
evaluation_refs_for_container(state, container_ref, parent_locators)
{container_ref, parent_evaluation_refs}
end
evaluator = container_ref && state.evaluators[container_ref]
worker_pid =
if evaluator do
Evaluator.peek_context(evaluator, evaluation_ref, &scan_and_ack.(&1.binding, &1.env))
Evaluator.peek_context(
evaluator,
parent_evaluation_refs,
&scan_and_ack.(&1.binding, &1.env)
)
evaluator.pid
else
{:ok, pid} =
@ -631,6 +660,26 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
%{info | scan_binding_pending: false, scan_binding_monitor_ref: monitor_ref}
end
defp evaluation_refs_for_container(state, container_ref, locators) do
case Enum.split_while(locators, &(elem(&1, 0) == container_ref)) do
{locators, []} ->
Enum.map(locators, &elem(&1, 1))
{locators, [{source_container_ref, _} | _] = source_locators} ->
source_evaluation_refs = Enum.map(source_locators, &elem(&1, 1))
evaluator = state.evaluators[container_ref]
source_evaluator = state.evaluators[source_container_ref]
if evaluator && source_evaluator do
# Synchronize initial state in the child evaluator
Evaluator.initialize_from(evaluator, source_evaluator, source_evaluation_refs)
end
Enum.map(locators, &elem(&1, 1))
end
end
defp finish_scan_binding(ref, state) do
update_in(state.smart_cells[ref], fn info ->
Process.demonitor(info.scan_binding_monitor_ref, [:flush])
@ -646,12 +695,12 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
defp scan_binding_after_evaluation(state, locator) do
update_in(state.smart_cells, fn smart_cells ->
Map.new(smart_cells, fn
{ref, %{base_locator: ^locator} = info} ->
Map.new(smart_cells, fn {ref, info} ->
if locator in info.parent_locators do
{ref, scan_binding_async(ref, info, state)}
other ->
other
else
{ref, info}
end
end)
end)
end

432
lib/livebook/runtime/evaluator.ex
View file

@ -56,10 +56,12 @@ defmodule Livebook.Runtime.Evaluator do
{:ok, result :: any()}
| {:error, Exception.kind(), error :: any(), Exception.stacktrace()}
# We store evaluation envs in the process dictionary, so that we
# can build intellisense context without asking the evaluator
@env_key :evaluation_env
@initial_env_key :initial_env
# We store some information in the process dictionary for non-blocking
# access from other evaluators. In particular we store context metadata,
# such as envs, this way we can build intellisense context without
# asking the evaluator. We don't store binding though, because that
# would take too much memory
@evaluator_info_key :evaluator_info
@doc """
Starts an evaluator.
@ -117,10 +119,10 @@ defmodule Livebook.Runtime.Evaluator do
Any exceptions are captured and transformed into an error
result.
The resulting contxt (binding and env) is stored under `ref`.
Any subsequent calls may specify `base_ref` pointing to a
previous evaluation, in which case the corresponding context
is used as the entry point for evaluation.
The resulting context (binding and env) is stored under `ref`. Any
subsequent calls may specify `parent_refs` pointing to a sequence
of previous evaluations, in which case the corresponding context is
used as the entry point for evaluation.
The evaluation result is transformed with the configured
formatter send to the configured client (see `start_link/1`).
@ -134,37 +136,29 @@ defmodule Livebook.Runtime.Evaluator do
finished. The function receives `t:evaluation_result/0`
as an argument
"""
@spec evaluate_code(t(), String.t(), ref(), ref() | nil, keyword()) :: :ok
def evaluate_code(evaluator, code, ref, base_ref \\ nil, opts \\ []) when ref != nil do
cast(evaluator, {:evaluate_code, code, ref, base_ref, opts})
@spec evaluate_code(t(), String.t(), ref(), list(ref()), keyword()) :: :ok
def evaluate_code(evaluator, code, ref, parent_refs, opts \\ []) do
cast(evaluator, {:evaluate_code, code, ref, parent_refs, opts})
end
@doc """
Fetches the evaluation context (binding and env) for the given
evaluation reference.
## Options
* `:cached_id` - id of context that the sender may already have,
if it matches the fetched context, `{:error, :not_modified}`
is returned instead
evaluation sequence.
"""
@spec fetch_evaluation_context(t(), ref(), keyword()) ::
{:ok, context()} | {:error, :not_modified}
def fetch_evaluation_context(evaluator, ref, opts \\ []) do
cached_id = opts[:cached_id]
call(evaluator, {:fetch_evaluation_context, ref, cached_id})
@spec get_evaluation_context(t(), list(ref())) :: context()
def get_evaluation_context(evaluator, parent_refs) do
call(evaluator, {:get_evaluation_context, parent_refs})
end
@doc """
Fetches an evaluation context from `source_evaluator` and configures
it as the initial context for `evaluator`.
Fetches an aggregated evaluation context from `source_evaluator`
and caches it as the initial context for `evaluator`.
The process dictionary is also copied to match `source_evaluator`.
"""
@spec initialize_from(t(), t(), ref()) :: :ok
def initialize_from(evaluator, source_evaluator, source_evaluation_ref) do
call(evaluator, {:initialize_from, source_evaluator, source_evaluation_ref})
def initialize_from(evaluator, source_evaluator, source_parent_refs) do
call(evaluator, {:initialize_from, source_evaluator, source_parent_refs})
end
@doc """
@ -190,15 +184,22 @@ defmodule Livebook.Runtime.Evaluator do
@doc """
Builds intellisense context from the given evaluation.
"""
@spec intellisense_context(t(), ref()) :: Livebook.Intellisense.intellisense_context()
def intellisense_context(evaluator, ref) do
@spec intellisense_context(t(), list(ref())) :: Livebook.Intellisense.intellisense_context()
def intellisense_context(evaluator, parent_refs) do
{:dictionary, dictionary} = Process.info(evaluator.pid, :dictionary)
env =
find_in_dictionary(dictionary, {@env_key, ref}) ||
find_in_dictionary(dictionary, @initial_env_key)
evaluator_info = find_in_dictionary(dictionary, @evaluator_info_key)
%{initial_context: {_id, initial_env}} = evaluator_info
map_binding = fn fun -> map_binding(evaluator, ref, fun) end
env =
List.foldr(parent_refs, initial_env, fn ref, prev_env ->
case evaluator_info.contexts do
%{^ref => {_id, env}} -> merge_env(prev_env, env)
_ -> prev_env
end
end)
map_binding = fn fun -> map_binding(evaluator, parent_refs, fun) end
%{env: env, map_binding: map_binding}
end
@ -211,8 +212,8 @@ defmodule Livebook.Runtime.Evaluator do
end
# Applies the given function to evaluation binding
defp map_binding(evaluator, ref, fun) do
call(evaluator, {:map_binding, ref, fun})
defp map_binding(evaluator, parent_refs, fun) do
call(evaluator, {:map_binding, parent_refs, fun})
end
@doc """
@ -221,9 +222,9 @@ defmodule Livebook.Runtime.Evaluator do
Ths function runs within the evaluator process, so that no data
is copied between processes, unless explicitly sent.
"""
@spec peek_context(t(), ref(), (context() -> any())) :: :ok
def peek_context(evaluator, ref, fun) do
cast(evaluator, {:peek_context, ref, fun})
@spec peek_context(t(), list(ref()), (context() -> any())) :: :ok
def peek_context(evaluator, parent_refs, fun) do
cast(evaluator, {:peek_context, parent_refs, fun})
end
defp cast(evaluator, message) do
@ -271,7 +272,13 @@ defmodule Livebook.Runtime.Evaluator do
evaluator = %{pid: self(), ref: evaluator_ref}
context = initial_context()
Process.put(@initial_env_key, context.env)
Process.put(@evaluator_info_key, %{
initial_context: {context.id, context.env},
contexts: %{}
})
ignored_pdict_keys = Process.get_keys() |> MapSet.new()
state = %{
evaluator_ref: evaluator_ref,
@ -281,7 +288,9 @@ defmodule Livebook.Runtime.Evaluator do
runtime_broadcast_to: runtime_broadcast_to,
object_tracker: object_tracker,
contexts: %{},
initial_context: context
initial_context: context,
initial_context_version: nil,
ignored_pdict_keys: ignored_pdict_keys
}
:proc_lib.init_ack(evaluator)
@ -304,34 +313,53 @@ defmodule Livebook.Runtime.Evaluator do
defp initial_context() do
env = Code.env_for_eval([])
%{binding: [], env: env, id: random_id()}
env = Macro.Env.prepend_tracer(env, Evaluator.Tracer)
%{id: random_id(), binding: [], env: env, pdict: %{}}
end
defp handle_cast({:evaluate_code, code, ref, base_ref, opts}, state) do
defp handle_cast({:evaluate_code, code, ref, parent_refs, opts}, state) do
Evaluator.ObjectTracker.remove_reference_sync(state.object_tracker, {self(), ref})
context = get_context(state, base_ref)
context = get_context(state, parent_refs)
file = Keyword.get(opts, :file, "nofile")
context = put_in(context.env.file, file)
start_time = System.monotonic_time()
Evaluator.IOProxy.configure(state.io_proxy, ref, file)
{result_context, result, code_error} =
case eval(code, context.binding, context.env) do
set_pdict(context, state.ignored_pdict_keys)
start_time = System.monotonic_time()
eval_result = eval(code, context.binding, context.env)
evaluation_time_ms = time_diff_ms(start_time)
{result_context, result, code_error, identifiers_used, identifiers_defined} =
case eval_result do
{:ok, value, binding, env} ->
binding = reorder_binding(binding, context.binding)
result_context = %{binding: binding, env: env, id: random_id()}
tracer_info = Evaluator.IOProxy.get_tracer_info(state.io_proxy)
context_id = random_id()
result_context = %{
id: context_id,
binding: binding,
env: prune_env(env, tracer_info),
pdict: current_pdict(state)
}
{identifiers_used, identifiers_defined} =
identifier_dependencies(result_context, tracer_info, context)
result = {:ok, value}
{result_context, result, nil}
{result_context, result, nil, identifiers_used, identifiers_defined}
{:error, kind, error, stacktrace, code_error} ->
result = {:error, kind, error, stacktrace}
{context, result, code_error}
identifiers_used = :unknown
identifiers_defined = %{}
# Empty context
result_context = initial_context()
{result_context, result, code_error, identifiers_used, identifiers_defined}
end
evaluation_time_ms = get_execution_time_delta(start_time)
state = put_context(state, ref, result_context)
Evaluator.IOProxy.flush(state.io_proxy)
@ -342,7 +370,9 @@ defmodule Livebook.Runtime.Evaluator do
metadata = %{
evaluation_time_ms: evaluation_time_ms,
memory_usage: memory(),
code_error: code_error
code_error: code_error,
identifiers_used: identifiers_used,
identifiers_defined: identifiers_defined
}
send(state.send_to, {:runtime_evaluation_response, ref, output, metadata})
@ -363,71 +393,153 @@ defmodule Livebook.Runtime.Evaluator do
{:noreply, state}
end
defp handle_cast({:peek_context, ref, fun}, state) do
context = get_context(state, ref)
defp handle_cast({:peek_context, parent_refs, fun}, state) do
context = get_context(state, parent_refs)
fun.(context)
{:noreply, state}
end
defp handle_call({:fetch_evaluation_context, ref, cached_id}, _from, state) do
context = get_context(state, ref)
reply =
if context.id == cached_id do
{:error, :not_modified}
else
{:ok, context}
end
{:reply, reply, state}
defp handle_call({:get_evaluation_context, parent_refs}, _from, state) do
context = get_context(state, parent_refs)
{:reply, context, state}
end
defp handle_call({:initialize_from, source_evaluator, source_evaluation_ref}, _from, state) do
defp handle_call({:initialize_from, source_evaluator, source_parent_refs}, _from, state) do
{:dictionary, dictionary} = Process.info(source_evaluator.pid, :dictionary)
evaluator_info = find_in_dictionary(dictionary, @evaluator_info_key)
version =
source_parent_refs
|> Enum.map(fn ref ->
with {id, _env} <- evaluator_info.contexts[ref], do: id
end)
|> :erlang.md5()
state =
case Evaluator.fetch_evaluation_context(
source_evaluator,
source_evaluation_ref,
cached_id: state.initial_context.id
) do
{:ok, context} ->
# If the context changed, mirror the process dictionary again
copy_process_dictionary_from(source_evaluator)
if version == state.initial_context_version do
state
else
context = Evaluator.get_evaluation_context(source_evaluator, source_parent_refs)
Process.put(@initial_env_key, context.env)
put_in(state.initial_context, context)
update_evaluator_info(fn info ->
put_in(info.initial_context, {context.id, context.env})
end)
{:error, :not_modified} ->
state
%{state | initial_context: context, initial_context_version: version}
end
{:reply, :ok, state}
end
defp handle_call({:map_binding, ref, fun}, _from, state) do
context = get_context(state, ref)
defp handle_call({:map_binding, parent_refs, fun}, _from, state) do
context = get_context(state, parent_refs)
result = fun.(context.binding)
{:reply, result, state}
end
defp put_context(state, ref, context) do
Process.put({@env_key, ref}, context.env)
update_evaluator_info(fn info ->
put_in(info.contexts[ref], {context.id, context.env})
end)
put_in(state.contexts[ref], context)
end
defp delete_context(state, ref) do
Process.delete({@env_key, ref})
update_evaluator_info(fn info ->
{_, info} = pop_in(info.contexts[ref])
info
end)
{_, state} = pop_in(state.contexts[ref])
state
end
defp get_context(state, ref) do
Map.get_lazy(state.contexts, ref, fn -> state.initial_context end)
defp update_evaluator_info(fun) do
info = Process.get(@evaluator_info_key)
Process.put(@evaluator_info_key, fun.(info))
end
defp get_context(state, parent_refs) do
List.foldr(parent_refs, state.initial_context, fn ref, prev_context ->
if context = state.contexts[ref] do
merge_context(prev_context, context)
else
prev_context
end
end)
end
defp set_pdict(context, ignored_pdict_keys) do
for key <- Process.get_keys(),
key not in ignored_pdict_keys,
not Map.has_key?(context.pdict, key) do
Process.delete(key)
end
for {key, value} <- context.pdict do
Process.put(key, value)
end
end
defp current_pdict(state) do
for {key, value} <- Process.get(),
key not in state.ignored_pdict_keys,
do: {key, value},
into: %{}
end
defp prune_env(env, tracer_info) do
env
|> Map.replace!(:aliases, Map.to_list(tracer_info.aliases_defined))
|> Map.replace!(:requires, MapSet.to_list(tracer_info.requires_defined))
end
defp merge_context(prev_context, context) do
binding = merge_binding(prev_context.binding, context.binding)
env = merge_env(prev_context.env, context.env)
pdict = context.pdict
%{id: random_id(), binding: binding, env: env, pdict: pdict}
end
defp merge_binding(prev_binding, binding) do
binding_map = Map.new(binding)
kept_binding =
Enum.reject(prev_binding, fn {var, _value} ->
Map.has_key?(binding_map, var)
end)
binding ++ kept_binding
end
defp merge_env(prev_env, env) do
env
|> Map.update!(:versioned_vars, fn versioned_vars ->
Enum.uniq(Map.keys(prev_env.versioned_vars) ++ Map.keys(versioned_vars))
|> Enum.with_index()
|> Map.new()
end)
|> Map.update!(:aliases, &Keyword.merge(prev_env.aliases, &1))
|> Map.update!(:requires, &:ordsets.union(prev_env.requires, &1))
|> Map.replace!(:context_modules, [])
end
@compile {:no_warn_undefined, {Code, :eval_quoted_with_env, 4}}
defp eval(code, binding, env) do
try do
quoted = Code.string_to_quoted!(code, file: env.file)
{value, binding, env} = Code.eval_quoted_with_env(quoted, binding, env)
# TODO: remove the else branch when we require Elixir v1.14.2
{value, binding, env} =
if function_exported?(Code, :eval_quoted_with_env, 4) do
Code.eval_quoted_with_env(quoted, binding, env, prune_binding: true)
else
Code.eval_quoted_with_env(quoted, binding, env)
end
{:ok, value, binding, env}
catch
kind, error ->
@ -449,22 +561,129 @@ defmodule Livebook.Runtime.Evaluator do
defp code_error?(%CompileError{}), do: true
defp code_error?(_error), do: false
defp reorder_binding(binding, prev_binding) do
# We keep the order of existing binding entries and move the new
# ones to the beginning
defp identifier_dependencies(context, tracer_info, prev_context) do
identifiers_used = MapSet.new()
identifiers_defined = %{}
binding_map = Map.new(binding)
# Variables
unchanged_binding =
Enum.filter(prev_binding, fn {key, prev_val} ->
val = binding_map[key]
:erts_debug.same(val, prev_val)
end)
identifiers_used =
for var <- vars_used(context, tracer_info, prev_context),
do: {:variable, var},
into: identifiers_used
unchanged_binding
|> Enum.reduce(binding_map, fn {key, _}, acc -> Map.delete(acc, key) end)
|> Map.to_list()
|> Kernel.++(unchanged_binding)
identifiers_used =
for var <- tracer_info.undefined_vars,
do: {:variable, var},
into: identifiers_used
identifiers_defined =
for var <- vars_defined(context, prev_context),
do: {{:variable, var}, context.id},
into: identifiers_defined
# Modules
identifiers_used =
for module <- tracer_info.modules_used,
do: {:module, module},
into: identifiers_used
identifiers_defined =
for {module, {version, _vars}} <- tracer_info.modules_defined,
do: {{:module, module}, version},
into: identifiers_defined
# Aliases
identifiers_used =
for alias <- tracer_info.aliases_used,
do: {:alias, alias},
into: identifiers_used
identifiers_defined =
for {as, alias} <- tracer_info.aliases_defined,
do: {{:alias, as}, alias},
into: identifiers_defined
# Requires
identifiers_used =
for module <- tracer_info.requires_used,
do: {:require, module},
into: identifiers_used
identifiers_defined =
for module <- tracer_info.requires_defined,
do: {{:require, module}, :ok},
into: identifiers_defined
# Imports
identifiers_used =
if tracer_info.imports_used? or tracer_info.imports_defined? do
# Imports are not always incremental, due to :except, so if
# we define imports, we also implicitly rely on prior imports
MapSet.put(identifiers_used, :imports)
else
identifiers_used
end
identifiers_defined =
if tracer_info.imports_defined? do
version = {:erlang.phash2(context.env.functions), :erlang.phash2(context.env.macros)}
put_in(identifiers_defined[:imports], version)
else
identifiers_defined
end
# Process dictionary
# Every evaluation depends on the pdict
identifiers_used = MapSet.put(identifiers_used, :pdict)
identifiers_defined =
if context.pdict == prev_context.pdict do
identifiers_defined
else
version = :erlang.phash2(context.pdict)
put_in(identifiers_defined[:pdict], version)
end
{MapSet.to_list(identifiers_used), identifiers_defined}
end
defp vars_used(context, tracer_info, prev_context) do
prev_vars =
for {var, _version} <- prev_context.env.versioned_vars,
into: MapSet.new(),
do: var
outer_used_vars =
for {var, _version} <- context.env.versioned_vars,
into: MapSet.new(),
do: var
# Note that :prune_binding removes variables used by modules
# (unless used outside), so we get those from the tracer
module_used_vars =
for {_module, {_version, vars}} <- tracer_info.modules_defined,
var <- vars,
into: MapSet.new(),
do: var
# We take an intersection with previous vars, so we ignore variables
# that we know are newly defined
MapSet.intersection(prev_vars, MapSet.union(outer_used_vars, module_used_vars))
end
defp vars_defined(context, prev_context) do
prev_num_vars = map_size(prev_context.env.versioned_vars)
for {var, version} <- context.env.versioned_vars,
version >= prev_num_vars,
into: MapSet.new(),
do: var
end
# Adapted from https://github.com/elixir-lang/elixir/blob/1c1654c88adfdbef38ff07fc30f6fbd34a542c07/lib/iex/lib/iex/evaluator.ex#L355-L372
@ -506,20 +725,7 @@ defmodule Livebook.Runtime.Evaluator do
:crypto.strong_rand_bytes(20) |> Base.encode32(case: :lower)
end
defp copy_process_dictionary_from(source_evaluator) do
{:dictionary, dictionary} = Process.info(source_evaluator.pid, :dictionary)
for {key, value} <- dictionary, not internal_dictionary_key?(key) do
Process.put(key, value)
end
end
defp internal_dictionary_key?("$" <> _), do: true
defp internal_dictionary_key?({@env_key, _ref}), do: true
defp internal_dictionary_key?(@initial_env_key), do: true
defp internal_dictionary_key?(_), do: false
defp get_execution_time_delta(started_at) do
defp time_diff_ms(started_at) do
System.monotonic_time()
|> Kernel.-(started_at)
|> System.convert_time_unit(:native, :millisecond)

30
lib/livebook/runtime/evaluator/io_proxy.ex
View file

@ -61,11 +61,19 @@ defmodule Livebook.Runtime.Evaluator.IOProxy do
end
@doc """
Returns the accumulated widget pids and clears the accumulator.
Updates tracer info.
"""
@spec flush_widgets(pid()) :: MapSet.t(pid())
def flush_widgets(pid) do
GenServer.call(pid, :flush_widgets)
@spec tracer_updates(pid(), list()) :: :ok
def tracer_updates(pid, updates) do
GenServer.cast(pid, {:tracer_updates, updates})
end
@doc """
Returns the accumulated tracer info.
"""
@spec get_tracer_info(pid()) :: %Evaluator.Tracer{}
def get_tracer_info(pid) do
GenServer.call(pid, :get_tracer_info)
end
@impl true
@ -81,24 +89,34 @@ defmodule Livebook.Runtime.Evaluator.IOProxy do
evaluator: evaluator,
send_to: send_to,
runtime_broadcast_to: runtime_broadcast_to,
object_tracker: object_tracker
object_tracker: object_tracker,
tracer_info: %Evaluator.Tracer{}
}}
end
@impl true
def handle_cast({:configure, ref, file}, state) do
{:noreply, %{state | ref: ref, file: file, token_count: 0}}
{:noreply, %{state | ref: ref, file: file, token_count: 0, tracer_info: %Evaluator.Tracer{}}}
end
def handle_cast(:clear_input_cache, state) do
{:noreply, %{state | input_cache: %{}}}
end
def handle_cast({:tracer_updates, updates}, state) do
state = update_in(state.tracer_info, &Evaluator.Tracer.apply_updates(&1, updates))
{:noreply, state}
end
@impl true
def handle_call(:flush, _from, state) do
{:reply, :ok, flush_buffer(state)}
end
def handle_call(:get_tracer_info, _from, state) do
{:reply, state.tracer_info, state}
end
@impl true
def handle_info({:io_request, from, reply_as, req}, state) do
{reply, state} = io_request(req, state)

132
lib/livebook/runtime/evaluator/tracer.ex Normal file
View file

@ -0,0 +1,132 @@
defmodule Livebook.Runtime.Evaluator.Tracer do
@moduledoc false
# Compilation tracer used by the evaluator.
#
# Events are pre-processed and sent to the group leader, where the
# tracer state is accumulated. After evaluation the evaluator reads
# the accumulated state.
alias Livebook.Runtime.Evaluator
defstruct modules_used: MapSet.new(),
modules_defined: %{},
aliases_used: MapSet.new(),
aliases_defined: %{},
requires_used: MapSet.new(),
requires_defined: MapSet.new(),
imports_used?: false,
imports_defined?: false,
undefined_vars: MapSet.new()
@doc false
def trace(event, env) do
case to_updates(event, env) do
[] ->
:ok
updates ->
io_proxy = Process.group_leader()
Evaluator.IOProxy.tracer_updates(io_proxy, updates)
end
:ok
end
defp to_updates(event, env) do
# Note that import/require/alias/defmodule don't trigger `:alias_reference`
# for the used alias, so we add it explicitly
case event do
{:import, _meta, module, _opts} ->
if(env.module, do: [], else: [:import_defined]) ++
[{:module_used, module}, {:alias_used, module}]
{:imported_function, meta, module, name, _arity} ->
var? = Keyword.has_key?(meta, :if_undefined)
[{:module_used, module}, {:import_used, name, var?}]
{:imported_macro, meta, module, name, _arity} ->
var? = Keyword.has_key?(meta, :if_undefined)
[{:module_used, module}, {:import_used, name, var?}]
{:alias, _meta, alias, as, _opts} ->
if(env.module, do: [], else: [{:alias_defined, as, alias}]) ++
[{:alias_used, alias}]
{:alias_expansion, _meta, as, _alias} ->
[{:alias_used, as}]
{:alias_reference, _meta, alias} ->
[{:alias_used, alias}]
{:require, _meta, module, _opts} ->
if(env.module, do: [], else: [{:require_defined, module}]) ++
[{:module_used, module}, {:alias_used, module}]
{:struct_expansion, _meta, module, _keys} ->
[{:module_used, module}]
{:remote_function, _meta, module, _name, _arity} ->
[{:module_used, module}]
{:remote_macro, _meta, module, _name, _arity} ->
[{:module_used, module}, {:require_used, module}]
{:on_module, bytecode, _ignore} ->
module = env.module
version = :erlang.md5(bytecode)
vars = Map.keys(env.versioned_vars)
[{:module_defined, module, version, vars}, {:alias_used, module}]
_ ->
[]
end
end
@doc """
Applies updates to the tracer state.
"""
@spec apply_updates(%__MODULE__{}, list()) :: %__MODULE__{}
def apply_updates(info, updates) do
Enum.reduce(updates, info, &apply_update(&2, &1))
end
defp apply_update(info, {:module_used, module}) do
update_in(info.modules_used, &MapSet.put(&1, module))
end
defp apply_update(info, {:module_defined, module, version, vars}) do
put_in(info.modules_defined[module], {version, vars})
end
defp apply_update(info, {:alias_used, alias}) do
update_in(info.aliases_used, &MapSet.put(&1, alias))
end
defp apply_update(info, {:alias_defined, as, alias}) do
put_in(info.aliases_defined[as], alias)
end
defp apply_update(info, {:require_used, module}) do
update_in(info.requires_used, &MapSet.put(&1, module))
end
defp apply_update(info, {:require_defined, module}) do
update_in(info.requires_defined, &MapSet.put(&1, module))
end
defp apply_update(info, {:import_used, name, var?}) do
info = put_in(info.imports_used?, true)
if var? do
update_in(info.undefined_vars, &MapSet.put(&1, {name, nil}))
else
info
end
end
defp apply_update(info, :import_defined) do
put_in(info.imports_defined?, true)
end
end

72
lib/livebook/session.ex
View file

@ -29,7 +29,7 @@ defmodule Livebook.Session do
# the evaluation context from the parent section, the last context
# needs to be copied from the main flow evaluator to the branching
# section evaluator. The latter synchronously asks the former for
# that context using `Livebook.Runtime.Evaluator.fetch_evaluation_context/3`.
# that context using `Livebook.Runtime.Evaluator.get_evaluation_context/3`.
# Consequently, in order to evaluate the first cell in a branching
# section, the main flow needs to be free of work, otherwise we wait.
# This assumptions are mirrored in by `Livebook.Session.Data` when
@ -1040,7 +1040,7 @@ defmodule Livebook.Session do
def handle_info({:runtime_evaluation_input, cell_id, reply_to, input_id}, state) do
{reply, state} =
with {:ok, cell, _section} <- Notebook.fetch_cell_and_section(state.data.notebook, cell_id),
{:ok, value} <- Map.fetch(state.data.input_values, input_id) do
{:ok, value} <- Data.fetch_input_value_for_cell(state.data, input_id, cell_id) do
state = handle_operation(state, {:bind_input, @client_id, cell.id, input_id})
{{:ok, value}, state}
else
@ -1512,27 +1512,26 @@ defmodule Livebook.Session do
state
end
defp handle_action(state, {:start_smart_cell, cell, section}) do
defp handle_action(state, {:start_smart_cell, cell, _section}) do
if Runtime.connected?(state.data.runtime) do
base_locator = find_base_locator(state.data, cell, section, existing: true)
Runtime.start_smart_cell(state.data.runtime, cell.kind, cell.id, cell.attrs, base_locator)
parent_locators = parent_locators_for_cell(state.data, cell)
Runtime.start_smart_cell(
state.data.runtime,
cell.kind,
cell.id,
cell.attrs,
parent_locators
)
end
state
end
defp handle_action(state, {:set_smart_cell_base, cell, section, parent}) do
defp handle_action(state, {:set_smart_cell_parents, cell, _section, parents}) do
if Runtime.connected?(state.data.runtime) do
base_locator =
case parent do
nil ->
{container_ref_for_section(section), nil}
{parent_cell, parent_section} ->
{container_ref_for_section(parent_section), parent_cell.id}
end
Runtime.set_smart_cell_base_locator(state.data.runtime, cell.id, base_locator)
parent_locators = evaluation_parents_to_locators(parents)
Runtime.set_smart_cell_parent_locators(state.data.runtime, cell.id, parent_locators)
end
state
@ -1566,8 +1565,8 @@ defmodule Livebook.Session do
opts = [file: file, smart_cell_ref: smart_cell_ref]
locator = {container_ref_for_section(section), cell.id}
base_locator = find_base_locator(state.data, cell, section)
Runtime.evaluate_code(state.data.runtime, cell.source, locator, base_locator, opts)
parent_locators = parent_locators_for_cell(state.data, cell)
Runtime.evaluate_code(state.data.runtime, cell.source, locator, parent_locators, opts)
evaluation_digest = :erlang.md5(cell.source)
handle_operation(state, {:evaluation_started, @client_id, cell.id, evaluation_digest})
@ -1799,34 +1798,21 @@ defmodule Livebook.Session do
end
@doc """
Finds evaluation locator that the given cell depends on.
Returns locators of evaluation parents for the given cell.
By default looks up the direct evaluation parent.
## Options
* `:existing` - considers only cells that have been evaluated
as evaluation parents. Defaults to `false`
Considers only cells that have already been evaluated.
"""
@spec find_base_locator(Data.t(), Cell.t(), Section.t(), keyword()) :: Runtime.locator()
def find_base_locator(data, cell, section, opts \\ []) do
parent_filter =
if opts[:existing] do
fn cell ->
info = data.cell_infos[cell.id]
Cell.evaluable?(cell) and info.eval.validity in [:evaluated, :stale]
end
else
&Cell.evaluable?/1
end
@spec parent_locators_for_cell(Data.t(), Cell.t()) :: Runtime.parent_locators()
def parent_locators_for_cell(data, cell) do
data
|> Data.cell_evaluation_parents(cell)
|> evaluation_parents_to_locators()
end
default = {container_ref_for_section(section), nil}
data.notebook
|> Notebook.parent_cells_with_section(cell.id)
|> Enum.find_value(default, fn {cell, section} ->
parent_filter.(cell) && {container_ref_for_section(section), cell.id}
end)
defp evaluation_parents_to_locators(parents) do
for {cell, section} <- parents do
{container_ref_for_section(section), cell.id}
end
end
defp container_ref_for_section(%{parent_id: nil}), do: @main_container_ref

592
lib/livebook/session/data.ex
View file

@ -57,7 +57,7 @@ defmodule Livebook.Session.Data do
@type section_info :: %{
evaluating_cell_id: Cell.id(),
evaluation_queue: list(Cell.id())
evaluation_queue: MapSet.t(Cell.id())
}
@type cell_info :: markdown_cell_info() | code_cell_info() | smart_cell_info()
@ -96,7 +96,10 @@ defmodule Livebook.Session.Data do
evaluation_number: non_neg_integer(),
outputs_batch_number: non_neg_integer(),
bound_to_input_ids: MapSet.t(input_id()),
bound_input_readings: input_reading()
new_bound_to_input_ids: MapSet.t(input_id()),
identifiers_used: list(identifier :: term()) | :unknown,
identifiers_defined: %{(identifier :: term()) => version :: term()},
data: t()
}
@type cell_bin_entry :: %{
@ -125,22 +128,10 @@ defmodule Livebook.Session.Data do
@type secret :: %{name: String.t(), value: String.t()}
# Snapshot holds information about the cell evaluation dependencies,
# for example what is the previous cell, the number of times the
# cell was evaluated, the list of available inputs, etc. Whenever
# the snapshot changes, it implies a new evaluation context, which
# basically means the cell got stale.
#
# The snapshot comprises of two actual snapshots:
#
# * `deps_snapshot` - everything related to parent cells and
# their evaluations. This is recorded once the cell starts
# evaluating
#
# * `bound_inputs_snapshot` - snapshot of the inputs and their
# values used by cell evaluation. This is recorded once the
# cell finishes its evaluation
#
@type snapshot :: {deps_snapshot :: term(), bound_inputs_snapshot :: term()}
# including parent cells and inputs. Whenever the snapshot changes,
# it implies a new evaluation context, which basically means the cell
# got stale.
@type snapshot :: term()
@type input_reading :: {input_id(), input_value :: term()}
@ -202,7 +193,8 @@ defmodule Livebook.Session.Data do
| {:stop_evaluation, Section.t()}
| {:forget_evaluation, Cell.t(), Section.t()}
| {:start_smart_cell, Cell.t(), Section.t()}
| {:set_smart_cell_base, Cell.t(), Section.t(), parent :: {Cell.t(), Section.t()} | nil}
| {:set_smart_cell_parents, Cell.t(), Section.t(),
parent :: {Cell.t(), Section.t()} | nil}
| {:broadcast_delta, client_id(), Cell.t(), cell_source_tag(), Delta.t()}
@doc """
@ -329,7 +321,7 @@ defmodule Livebook.Session.Data do
|> with_actions()
|> cancel_section_evaluation(section)
|> set_section_parent(section, parent_section)
|> compute_snapshots_and_validity()
|> update_validity_and_evaluation()
|> set_dirty()
|> wrap_ok()
else
@ -345,7 +337,7 @@ defmodule Livebook.Session.Data do
|> cancel_section_evaluation(section)
|> add_action({:stop_evaluation, section})
|> unset_section_parent(section)
|> compute_snapshots_and_validity()
|> update_validity_and_evaluation()
|> set_dirty()
|> wrap_ok()
else
@ -361,7 +353,7 @@ defmodule Livebook.Session.Data do
|> with_actions()
|> insert_cell(section_id, index, cell)
|> maybe_start_smart_cells()
|> compute_snapshots_and_validity()
|> update_validity_and_evaluation()
|> set_dirty()
|> wrap_ok()
end
@ -374,7 +366,7 @@ defmodule Livebook.Session.Data do
data
|> with_actions()
|> delete_section(section, delete_cells)
|> compute_snapshots_and_validity()
|> update_validity_and_evaluation()
|> update_smart_cell_bases(data)
|> set_dirty()
|> wrap_ok()
@ -389,7 +381,7 @@ defmodule Livebook.Session.Data do
data
|> with_actions()
|> delete_cell(cell, section)
|> compute_snapshots_and_validity()
|> update_validity_and_evaluation()
|> update_smart_cell_bases(data)
|> set_dirty()
|> wrap_ok()
@ -404,7 +396,7 @@ defmodule Livebook.Session.Data do
data
|> with_actions()
|> restore_cell(cell_bin_entry)
|> compute_snapshots_and_validity()
|> update_validity_and_evaluation()
|> maybe_start_smart_cells()
|> set_dirty()
|> wrap_ok()
@ -421,7 +413,7 @@ defmodule Livebook.Session.Data do
data
|> with_actions()
|> move_cell(cell, offset)
|> compute_snapshots_and_validity()
|> update_validity_and_evaluation()
|> update_smart_cell_bases(data)
|> set_dirty()
|> wrap_ok()
@ -437,7 +429,7 @@ defmodule Livebook.Session.Data do
data
|> with_actions()
|> move_section(section, offset)
|> compute_snapshots_and_validity()
|> update_validity_and_evaluation()
|> update_smart_cell_bases(data)
|> set_dirty()
|> wrap_ok()
@ -459,12 +451,11 @@ defmodule Livebook.Session.Data do
cells_with_section
|> Enum.reduce(with_actions(data), fn {cell, section}, data_actions ->
data_actions
|> queue_prerequisite_cells_evaluation(cell)
|> queue_prerequisite_cells_evaluation(cell.id)
|> queue_cell_evaluation(cell, section)
end)
|> maybe_connect_runtime(data)
|> maybe_evaluate_queued()
|> compute_snapshots_and_validity()
|> update_validity_and_evaluation()
|> wrap_ok()
else
:error
@ -505,9 +496,7 @@ defmodule Livebook.Session.Data do
|> add_cell_output(cell, output)
|> finish_cell_evaluation(cell, section, metadata)
|> garbage_collect_input_values()
|> compute_snapshots_and_validity()
|> maybe_evaluate_queued()
|> compute_snapshots_and_validity()
|> update_validity_and_evaluation()
|> update_smart_cell_bases(data)
|> mark_dirty_if_persisting_outputs()
|> wrap_ok()
@ -519,8 +508,9 @@ defmodule Livebook.Session.Data do
def apply_operation(data, {:bind_input, _client_id, cell_id, input_id}) do
with {:ok, cell, _section} <- Notebook.fetch_cell_and_section(data.notebook, cell_id),
Cell.evaluable?(cell),
:evaluating <- data.cell_infos[cell.id].eval.status,
true <- Map.has_key?(data.input_values, input_id),
false <- MapSet.member?(data.cell_infos[cell.id].eval.bound_to_input_ids, input_id) do
false <- MapSet.member?(data.cell_infos[cell.id].eval.new_bound_to_input_ids, input_id) do
data
|> with_actions()
|> bind_input(cell, input_id)
@ -715,7 +705,7 @@ defmodule Livebook.Session.Data do
data
|> with_actions()
|> set_cell_attributes(cell, attrs)
|> compute_snapshots_and_validity()
|> update_validity_and_evaluation()
|> set_dirty()
|> wrap_ok()
else
@ -728,7 +718,7 @@ defmodule Livebook.Session.Data do
data
|> with_actions()
|> set_input_value(input_id, value)
|> compute_snapshots_and_validity()
|> update_validity_and_evaluation()
|> wrap_ok()
else
_ -> :error
@ -945,8 +935,7 @@ defmodule Livebook.Session.Data do
# For each path in the dependency graph, find the upmost cell
# which parent changed. From that point downwards all cells
# are invalidated. Then gather invalidated cells from all paths
# and unqueue them.
# are potentially affected, so we unqueue them.
invalidted_cell_ids =
graph_after
@ -997,7 +986,7 @@ defmodule Livebook.Session.Data do
defp queue_cell_evaluation(data_actions, cell, section) do
data_actions
|> update_section_info!(section.id, fn section ->
%{section | evaluation_queue: append_new(section.evaluation_queue, cell.id)}
update_in(section.evaluation_queue, &MapSet.put(&1, cell.id))
end)
|> update_cell_eval_info!(cell.id, fn eval_info ->
update_in(eval_info.status, fn
@ -1010,7 +999,7 @@ defmodule Livebook.Session.Data do
defp unqueue_cell_evaluation(data_actions, cell, section) do
data_actions
|> update_section_info!(section.id, fn section ->
%{section | evaluation_queue: List.delete(section.evaluation_queue, cell.id)}
update_in(section.evaluation_queue, &MapSet.delete(&1, cell.id))
end)
|> update_cell_eval_info!(cell.id, &%{&1 | status: :ready})
end
@ -1047,15 +1036,40 @@ defmodule Livebook.Session.Data do
eval_info
| status: :ready,
evaluation_time_ms: metadata.evaluation_time_ms,
# After finished evaluation, take the snapshot of read inputs
evaluation_snapshot:
{elem(eval_info.evaluation_snapshot, 0),
input_readings_snapshot(eval_info.bound_input_readings)}
identifiers_used: metadata.identifiers_used,
identifiers_defined: metadata.identifiers_defined,
bound_to_input_ids: eval_info.new_bound_to_input_ids
}
end)
|> update_cell_evaluation_snapshot(cell, section)
|> set_section_info!(section.id, evaluating_cell_id: nil)
end
defp update_cell_evaluation_snapshot({data, _} = data_actions, cell, section) do
info = data.cell_infos[cell.id]
eval_data = data.cell_infos[cell.id].eval.data
eval_data = put_in(eval_data.cell_infos[cell.id], info)
graph = Notebook.cell_dependency_graph(eval_data.notebook, cell_filter: &Cell.evaluable?/1)
cell_snapshots =
for {cell_id, %{eval: eval_info}} <- eval_data.cell_infos,
do: {cell_id, eval_info.snapshot},
into: %{}
# We compute evaluation snapshot based on the notebook state prior
# to evaluation, but using the information about the dependencies
# obtained during evaluation (identifiers, inputs)
evaluation_snapshot = cell_snapshot(cell, section, graph, cell_snapshots, eval_data)
data_actions
|> update_cell_eval_info!(
cell.id,
&%{&1 | evaluation_snapshot: evaluation_snapshot, data: nil}
)
end
defp maybe_connect_runtime({data, _} = data_actions, prev_data) do
if not Runtime.connected?(data.runtime) and not any_cell_queued?(prev_data) and
any_cell_queued?(data) do
@ -1066,7 +1080,26 @@ defmodule Livebook.Session.Data do
end
defp any_cell_queued?(data) do
Enum.any?(data.section_infos, fn {_section_id, info} -> info.evaluation_queue != [] end)
Enum.any?(data.section_infos, fn {_section_id, info} ->
not Enum.empty?(info.evaluation_queue)
end)
end
defp queue_prerequisite_cells_evaluation_for_queued({data, _} = data_actions) do
{awaiting_branch_sections, awaiting_regular_sections} =
data.notebook
|> Notebook.all_sections()
|> Enum.filter(&section_awaits_evaluation?(data, &1.id))
|> Enum.split_with(& &1.parent_id)
trailing_queued_cell_ids =
for section <- awaiting_branch_sections ++ Enum.take(awaiting_regular_sections, -1),
cell = last_queued_cell(data, section),
do: cell.id
reduce(data_actions, trailing_queued_cell_ids, fn data_actions, cell_id ->
queue_prerequisite_cells_evaluation(data_actions, cell_id)
end)
end
defp maybe_evaluate_queued({data, _} = data_actions) do
@ -1081,7 +1114,7 @@ defmodule Livebook.Session.Data do
data_actions =
reduce(data_actions, awaiting_branch_sections, fn {data, _} = data_actions, section ->
%{evaluation_queue: [id | _]} = data.section_infos[section.id]
%{id: id} = first_queued_cell(data, section)
{:ok, parent} = Notebook.fetch_section(data.notebook, section.parent_id)
@ -1094,7 +1127,7 @@ defmodule Livebook.Session.Data do
prev_section_queued? =
prev_cell_section != nil and
data.section_infos[prev_cell_section.id].evaluation_queue != []
not Enum.empty?(data.section_infos[prev_cell_section.id].evaluation_queue)
# If evaluating this cell requires interaction with the main flow,
# we keep the cell queued. In case of the Elixir runtimes the
@ -1119,6 +1152,25 @@ defmodule Livebook.Session.Data do
end
end
defp first_queued_cell(data, section) do
find_queued_cell(data, section.cells)
end
defp last_queued_cell(data, section) do
find_queued_cell(data, Enum.reverse(section.cells))
end
defp find_queued_cell(data, cells) do
Enum.find_value(cells, fn cell ->
info = data.cell_infos[cell.id]
case info do
%{eval: %{status: :queued}} -> cell
_ -> nil
end
end)
end
defp main_flow_evaluating?(data) do
data.notebook
|> Notebook.all_sections()
@ -1142,51 +1194,51 @@ defmodule Livebook.Session.Data do
defp section_awaits_evaluation?(data, section_id) do
info = data.section_infos[section_id]
info.evaluating_cell_id == nil and info.evaluation_queue != []
info.evaluating_cell_id == nil and not Enum.empty?(info.evaluation_queue)
end
defp evaluate_next_cell_in_section({data, _} = data_actions, section) do
case data.section_infos[section.id] do
%{evaluating_cell_id: nil, evaluation_queue: [id | ids]} ->
cell = Enum.find(section.cells, &(&1.id == id))
section_info = data.section_infos[section.id]
data_actions
|> set!(notebook: Notebook.update_cell(data.notebook, id, &%{&1 | outputs: []}))
|> update_cell_eval_info!(id, fn eval_info ->
%{
eval_info
| # Note: we intentionally mark the cell as evaluating up front,
# so that another queue operation doesn't cause duplicated
# :start_evaluation action
status: :evaluating,
evaluation_number: eval_info.evaluation_number + 1,
outputs_batch_number: eval_info.outputs_batch_number + 1,
evaluation_digest: nil,
evaluation_snapshot: eval_info.snapshot,
bound_to_input_ids: MapSet.new(),
bound_input_readings: [],
# This is a rough estimate, the exact time is measured in the
# evaluator itself
evaluation_start: DateTime.utc_now()
}
end)
|> set_section_info!(section.id, evaluating_cell_id: id, evaluation_queue: ids)
|> add_action({:start_evaluation, cell, section})
if section_info.evaluating_cell_id == nil and not Enum.empty?(section_info.evaluation_queue) do
cell = first_queued_cell(data, section)
_ ->
data_actions
data_actions
|> set!(notebook: Notebook.update_cell(data.notebook, cell.id, &%{&1 | outputs: []}))
|> update_cell_eval_info!(cell.id, fn eval_info ->
%{
eval_info
| # Note: we intentionally mark the cell as evaluating up front,
# so that another queue operation doesn't cause duplicated
# :start_evaluation action
status: :evaluating,
evaluation_number: eval_info.evaluation_number + 1,
outputs_batch_number: eval_info.outputs_batch_number + 1,
evaluation_digest: nil,
new_bound_to_input_ids: MapSet.new(),
# Keep the notebook state before evaluation
data: data,
# This is a rough estimate, the exact time is measured in the
# evaluator itself
evaluation_start: DateTime.utc_now()
}
end)
|> set_section_info!(section.id,
evaluating_cell_id: cell.id,
evaluation_queue: MapSet.delete(section_info.evaluation_queue, cell.id)
)
|> add_action({:start_evaluation, cell, section})
else
data_actions
end
end
defp bind_input({data, _} = data_actions, cell, input_id) do
defp bind_input(data_actions, cell, input_id) do
data_actions
|> update_cell_eval_info!(cell.id, fn eval_info ->
%{
eval_info
| bound_to_input_ids: MapSet.put(eval_info.bound_to_input_ids, input_id),
bound_input_readings: [
{input_id, data.input_values[input_id]} | eval_info.bound_input_readings
]
| new_bound_to_input_ids: MapSet.put(eval_info.new_bound_to_input_ids, input_id)
}
end)
end
@ -1210,7 +1262,7 @@ defmodule Livebook.Session.Data do
evaluable_cells = Enum.filter(section.cells, &Cell.evaluable?/1)
data_actions
|> set_section_info!(section.id, evaluating_cell_id: nil, evaluation_queue: [])
|> set_section_info!(section.id, evaluating_cell_id: nil, evaluation_queue: MapSet.new())
|> reduce(
evaluable_cells,
&update_cell_eval_info!(&1, &2.id, fn eval_info ->
@ -1230,14 +1282,13 @@ defmodule Livebook.Session.Data do
)
end
defp queue_prerequisite_cells_evaluation({data, _} = data_actions, cell) do
defp queue_prerequisite_cells_evaluation({data, _} = data_actions, cell_id) do
prerequisites_queue =
data.notebook
|> Notebook.parent_cells_with_section(cell.id)
|> Enum.filter(fn {cell, _} -> Cell.evaluable?(cell) end)
|> Enum.take_while(fn {parent_cell, _section} ->
info = data.cell_infos[parent_cell.id]
info.eval.validity != :evaluated and info.eval.status == :ready
|> Notebook.parent_cells_with_section(cell_id)
|> Enum.filter(fn {cell, _section} ->
info = data.cell_infos[cell.id]
Cell.evaluable?(cell) and info.eval.validity != :evaluated and info.eval.status == :ready
end)
|> Enum.reverse()
@ -1263,16 +1314,20 @@ defmodule Livebook.Session.Data do
:queued ->
data_actions
|> unqueue_cell_evaluation(cell, section)
|> unqueue_dependent_cells_evaluation(cell)
|> unqueue_child_cells_evaluation(cell)
_ ->
data_actions
end
end
defp unqueue_dependent_cells_evaluation({data, _} = data_actions, cell) do
dependent = dependent_cells_with_section(data, cell.id)
unqueue_cells_evaluation(data_actions, dependent)
defp unqueue_child_cells_evaluation({data, _} = data_actions, cell) do
evaluation_children =
data.notebook
|> Notebook.child_cells_with_section(cell.id)
|> Enum.filter(fn {cell, _} -> Cell.evaluable?(cell) end)
unqueue_cells_evaluation(data_actions, evaluation_children)
end
defp unqueue_cells_evaluation({data, _} = data_actions, cells_with_section) do
@ -1337,7 +1392,7 @@ defmodule Livebook.Session.Data do
if evaluated? and reevaluate do
data_actions
|> queue_prerequisite_cells_evaluation(cell)
|> queue_prerequisite_cells_evaluation(cell.id)
|> queue_cell_evaluation(cell, section)
|> maybe_evaluate_queued()
else
@ -1610,14 +1665,6 @@ defmodule Livebook.Session.Data do
{data, actions ++ [action]}
end
defp append_new(list, item) do
if item in list do
list
else
list ++ [item]
end
end
defp garbage_collect_input_values({data, _} = data_actions) do
if any_section_evaluating?(data) do
# Wait if evaluation is ongoing as it may render inputs
@ -1638,69 +1685,31 @@ defmodule Livebook.Session.Data do
if Enum.empty?(alive_smart_cell_ids) do
data_actions
else
new_eval_graph = cell_evaluation_graph(data)
prev_eval_graph = cell_evaluation_graph(prev_data)
new_eval_parents = cell_evaluation_parents(data)
prev_eval_parents = cell_evaluation_parents(prev_data)
cell_lookup =
data.notebook
|> Notebook.cells_with_section()
|> Map.new(fn {cell, section} -> {cell.id, {cell, section}} end)
for {cell_id, parent_id} <- new_eval_graph,
for {cell_id, eval_parents} <- new_eval_parents,
MapSet.member?(alive_smart_cell_ids, cell_id),
Map.has_key?(prev_eval_graph, cell_id),
prev_eval_graph[cell_id] != parent_id,
Map.has_key?(prev_eval_parents, cell_id),
prev_eval_parents[cell_id] != eval_parents,
reduce: data_actions do
data_actions ->
{cell, section} = cell_lookup[cell_id]
parent = cell_lookup[parent_id]
add_action(data_actions, {:set_smart_cell_base, cell, section, parent})
parents = Enum.map(eval_parents, &cell_lookup[&1])
add_action(data_actions, {:set_smart_cell_parents, cell, section, parents})
end
end
end
# Builds a graph with evaluation parents, where each parent has
# aleady been evaluated. All fresh/aborted cells are leaves in
# this graph
defp cell_evaluation_graph(data) do
graph = Notebook.cell_dependency_graph(data.notebook, cell_filter: &Cell.evaluable?/1)
graph
|> Livebook.Utils.Graph.leaves()
|> Enum.reduce(%{}, fn cell_id, eval_graph ->
build_eval_graph(data, graph, cell_id, [], eval_graph)
end)
end
defp build_eval_graph(_data, _graph, nil, orphan_ids, eval_graph) do
put_parent(eval_graph, orphan_ids, nil)
end
defp build_eval_graph(data, graph, cell_id, orphan_ids, eval_graph) do
# We are traversing from every leaf up, so we want to compute
# the common path only once
if eval_parent_id = eval_graph[cell_id] do
put_parent(eval_graph, orphan_ids, eval_parent_id)
else
info = data.cell_infos[cell_id]
if info.eval.validity in [:evaluated, :stale] do
eval_graph = put_parent(eval_graph, orphan_ids, cell_id)
build_eval_graph(data, graph, graph[cell_id], [cell_id], eval_graph)
else
build_eval_graph(data, graph, graph[cell_id], [cell_id | orphan_ids], eval_graph)
end
end
end
defp put_parent(eval_graph, cell_ids, parent_id) do
Enum.reduce(cell_ids, eval_graph, &Map.put(&2, &1, parent_id))
end
defp new_section_info() do
%{
evaluating_cell_id: nil,
evaluation_queue: []
evaluation_queue: MapSet.new()
}
end
@ -1745,9 +1754,12 @@ defmodule Livebook.Session.Data do
evaluation_number: 0,
outputs_batch_number: 0,
bound_to_input_ids: MapSet.new(),
bound_input_readings: [],
snapshot: {nil, nil},
evaluation_snapshot: nil
new_bound_to_input_ids: MapSet.new(),
identifiers_used: [],
identifiers_defined: %{},
snapshot: nil,
evaluation_snapshot: nil,
data: nil
}
end
@ -1808,6 +1820,52 @@ defmodule Livebook.Session.Data do
Enum.all?(attrs, fn {key, _} -> Map.has_key?(struct, key) end)
end
@doc """
Builds evaluation parent sequence for every evaluable cell.
This function should be used instead of calling `cell_evaluation_parents/2`
multiple times.
"""
@spec cell_evaluation_parents(t()) :: %{Cell.id() => list(Cell.id())}
def cell_evaluation_parents(data) do
graph = Notebook.cell_dependency_graph(data.notebook, cell_filter: &Cell.evaluable?/1)
graph
|> Graph.reduce_paths({nil, %{}}, fn cell_id, {parent_id, chains} ->
if parent_id do
parent_chain = chains[parent_id]
parent_info = data.cell_infos[parent_id]
chain =
if parent_info.eval.validity in [:evaluated, :stale] do
[parent_id | parent_chain]
else
parent_chain
end
{cell_id, put_in(chains[cell_id], chain)}
else
{cell_id, put_in(chains[cell_id], [])}
end
end)
|> Enum.map(&elem(&1, 1))
|> Enum.reduce(&Map.merge/2)
end
@doc """
Builds evaluation parent sequence for the given cell.
Considers only cells that have already been evaluated.
"""
@spec cell_evaluation_parents(Data.t(), Cell.t()) :: list({Cell.t(), Section.t()})
def cell_evaluation_parents(data, cell) do
for {cell, section} <- Notebook.parent_cells_with_section(data.notebook, cell.id),
info = data.cell_infos[cell.id],
Cell.evaluable?(cell),
info.eval.validity in [:evaluated, :stale],
do: {cell, section}
end
@doc """
Find cells bound to the given input.
"""
@ -1821,20 +1879,16 @@ defmodule Livebook.Session.Data do
end)
end
defp dependent_cells_with_section(data, cell_id) do
data.notebook
|> Notebook.child_cells_with_section(cell_id)
|> Enum.filter(fn {cell, _} -> Cell.evaluable?(cell) end)
end
# Computes cell snapshots and updates validity based on the new values.
defp compute_snapshots_and_validity(data_actions) do
# Computes cell snapshots and updates validity based on the new
# values, then triggers further evaluation if applicable.
defp update_validity_and_evaluation(data_actions) do
data_actions
|> compute_snapshots()
|> update_validity()
# After updating validity there may be new stale cells, so we check
# if any of them is configured for automatic reevaluation
|> maybe_queue_reevaluating_cells()
|> queue_prerequisite_cells_evaluation_for_queued()
|> maybe_evaluate_queued()
end
@ -1845,32 +1899,7 @@ defmodule Livebook.Session.Data do
cell_snapshots =
Enum.reduce(cells_with_section, %{}, fn {cell, section}, cell_snapshots ->
info = data.cell_infos[cell.id]
prev_cell_id = graph[cell.id]
is_branch? = section.parent_id != nil
parent_deps =
prev_cell_id &&
{
prev_cell_id,
cell_snapshots[prev_cell_id],
number_of_evaluations(data.cell_infos[prev_cell_id])
}
deps = {is_branch?, parent_deps}
deps_snapshot = :erlang.phash2(deps)
inputs_snapshot =
if info.eval.status == :evaluating do
# While the cell is evaluating the bound inputs snapshot
# is not stable, so we reuse the previous snapshot
elem(info.eval.snapshot, 1)
else
bound_inputs_snapshot(data, cell)
end
snapshot = {deps_snapshot, inputs_snapshot}
snapshot = cell_snapshot(cell, section, graph, cell_snapshots, data)
put_in(cell_snapshots[cell.id], snapshot)
end)
@ -1882,26 +1911,102 @@ defmodule Livebook.Session.Data do
end)
end
defp number_of_evaluations(%{eval: %{status: :evaluating}} = info) do
info.eval.evaluation_number - 1
defp cell_snapshot(cell, section, graph, cell_snapshots, data) do
info = data.cell_infos[cell.id]
# Note that this is an implication of the Elixir runtime, we want
# to reevaluate as much as possible in a branch, rather than copying
# contexts between processes, because all structural sharing is
# lost when copying
is_branch? = section.parent_id != nil
{parent_ids, identifier_versions} = identifier_deps(cell.id, graph, data)
parent_snapshots = Enum.map(parent_ids, &cell_snapshots[&1])
bound_input_values =
for(
input_id <- info.eval.bound_to_input_ids,
do: {input_id, data.input_values[input_id]}
)
|> Enum.sort()
deps = {is_branch?, parent_snapshots, identifier_versions, bound_input_values}
:erlang.phash2(deps)
end
defp number_of_evaluations(info), do: info.eval.evaluation_number
defp identifier_deps(cell_id, graph, data) do
info = data.cell_infos[cell_id]
defp bound_inputs_snapshot(data, cell) do
%{bound_to_input_ids: bound_to_input_ids} = data.cell_infos[cell.id].eval
{parent_ids, identifier_versions} =
case info.eval.identifiers_used do
:unknown ->
all_identifier_deps(graph[cell_id], graph, data)
for(
input_id <- bound_to_input_ids,
do: {input_id, data.input_values[input_id]}
)
|> input_readings_snapshot()
identifiers_used ->
gather_identifier_deps(graph[cell_id], identifiers_used, graph, data, {[], []})
end
{Enum.sort(parent_ids), Enum.sort(identifier_versions)}
end
defp input_readings_snapshot([]), do: :empty
defp all_identifier_deps(cell_id, graph, data) do
parent_ids = graph |> Graph.find_path(cell_id, nil) |> Enum.drop(1)
defp input_readings_snapshot(name_value_pairs) do
name_value_pairs |> Enum.sort() |> :erlang.phash2()
identifier_versions =
parent_ids
|> List.foldr(%{}, fn cell_id, acc ->
identifiers_defined = data.cell_infos[cell_id].eval.identifiers_defined
Map.merge(acc, identifiers_defined)
end)
|> Map.to_list()
{parent_ids, identifier_versions}
end
defp gather_identifier_deps(nil, _identifiers_used, _graph, _data, acc), do: acc
defp gather_identifier_deps(_cell_id, [], _graph, _data, acc), do: acc
defp gather_identifier_deps(
cell_id,
identifiers_used,
graph,
data,
{parent_ids, identifier_versions}
) do
identifiers_defined = data.cell_infos[cell_id].eval.identifiers_defined
identifiers_used
|> Enum.reduce({[], []}, fn identifier, {versions, rest_identifiers} ->
case identifiers_defined do
%{^identifier => version} ->
{[{identifier, version} | versions], rest_identifiers}
_ ->
{versions, [identifier | rest_identifiers]}
end
end)
|> case do
{[], rest_identifiers} ->
gather_identifier_deps(
graph[cell_id],
rest_identifiers,
graph,
data,
{parent_ids, identifier_versions}
)
{versions, rest_identifiers} ->
gather_identifier_deps(
graph[cell_id],
rest_identifiers,
graph,
data,
{[cell_id | parent_ids], versions ++ identifier_versions}
)
end
end
defp update_validity({data, _} = data_actions) do
@ -1924,7 +2029,7 @@ defmodule Livebook.Session.Data do
end
defp maybe_queue_reevaluating_cells({data, _} = data_actions) do
cells_to_reeavaluete =
cells_to_reevaluate =
data.notebook
|> Notebook.evaluable_cells_with_section()
|> Enum.filter(fn {cell, _section} ->
@ -1935,9 +2040,9 @@ defmodule Livebook.Session.Data do
end)
data_actions
|> reduce(cells_to_reeavaluete, fn data_actions, {cell, section} ->
|> reduce(cells_to_reevaluate, fn data_actions, {cell, section} ->
data_actions
|> queue_prerequisite_cells_evaluation(cell)
|> queue_prerequisite_cells_evaluation(cell.id)
|> queue_cell_evaluation(cell, section)
end)
end
@ -1959,7 +2064,7 @@ defmodule Livebook.Session.Data do
Returns the list of cell ids for full evaluation.
The list includes all outdated cells, cells in `forced_cell_ids`
and all of their child cells.
and all cells with identifier dependency on these.
"""
@spec cell_ids_for_full_evaluation(t(), list(Cell.id())) :: list(Cell.id())
def cell_ids_for_full_evaluation(data, forced_cell_ids) do
@ -1968,16 +2073,70 @@ defmodule Livebook.Session.Data do
evaluable_cell_ids =
for {cell, _} <- evaluable_cells_with_section,
cell_outdated?(data, cell) or cell.id in forced_cell_ids,
info = data.cell_infos[cell.id],
info.eval.status == :ready,
uniq: true,
do: cell.id
do: cell.id,
into: MapSet.new()
cell_ids = Notebook.cell_ids_with_children(data.notebook, evaluable_cell_ids)
cell_identifier_parents = cell_identifier_parents(data)
for {cell, _} <- evaluable_cells_with_section,
cell.id in cell_ids,
do: cell.id
child_ids =
for {cell_id, cell_identifier_parents} <- cell_identifier_parents,
Enum.any?(cell_identifier_parents, &(&1 in evaluable_cell_ids)),
do: cell_id
child_ids
|> Enum.into(evaluable_cell_ids)
|> Enum.to_list()
|> Enum.filter(fn cell_id ->
info = data.cell_infos[cell_id]
info.eval.status == :ready
end)
end
# Builds identifier parent list for every evaluable cell.
#
# This is similar to cell_evaluation_parents, but the dependency is
# based on identifiers used/set by each cell.
defp cell_identifier_parents(data) do
graph = Notebook.cell_dependency_graph(data.notebook, cell_filter: &Cell.evaluable?/1)
graph
|> Graph.reduce_paths(
{nil, %{}, %{}},
fn cell_id, {parent_id, setters, identifier_parents} ->
if parent_id do
cell_info = data.cell_infos[cell_id]
direct_parents =
case cell_info.eval.identifiers_used do
:unknown ->
setters |> Map.values() |> Enum.uniq()
identifiers_used ->
for identifier <- identifiers_used,
parent_id = setters[identifier],
uniq: true,
do: parent_id
end
parents =
for parent_id <- direct_parents,
cell_id <- [parent_id | identifier_parents[parent_id]],
uniq: true,
do: cell_id
setters =
for {identifier, _version} <- cell_info.eval.identifiers_defined,
do: {identifier, cell_id},
into: setters
{cell_id, setters, put_in(identifier_parents[cell_id], parents)}
else
{cell_id, setters, put_in(identifier_parents[cell_id], [])}
end
end
)
|> Enum.map(&elem(&1, 2))
|> Enum.reduce(&Map.merge/2)
end
@doc """
@ -2010,4 +2169,21 @@ defmodule Livebook.Session.Data do
end)
|> elem(0)
end
@doc """
Fetches an input value for the given cell.
If the cell is evaluating, the input value at evaluation start is
returned instead of the current value.
"""
@spec fetch_input_value_for_cell(t(), input_id(), Cell.id()) :: {:ok, term()} | :error
def fetch_input_value_for_cell(data, input_id, cell_id) do
data =
case data.cell_infos[cell_id] do
%{eval: %{status: :evaluating, data: data}} -> data
_ -> data
end
Map.fetch(data.input_values, input_id)
end
end

35
lib/livebook/utils/graph.ex
View file

@ -30,7 +30,7 @@ defmodule Livebook.Utils.Graph do
do: find_path(graph, graph[from_id], to_id, [from_id | path])
@doc """
Finds grpah leave nodes, that is, nodes with
Finds graph leave nodes, that is, nodes with
no children.
"""
@spec leaves(t()) :: list(node_id())
@ -39,4 +39,37 @@ defmodule Livebook.Utils.Graph do
parents = MapSet.new(graph, fn {_, value} -> value end)
MapSet.difference(children, parents) |> MapSet.to_list()
end
@doc """
Reduces each top-down path in the graph.
Returns a list of accumulators, one for each leaf in the graph,
in no specific order.
"""
@spec reduce_paths(t(), acc, (node_id(), acc -> acc)) :: acc when acc: term()
def reduce_paths(graph, acc, fun) do
leaves = Livebook.Utils.Graph.leaves(graph)
cache = do_reduce(graph, leaves, acc, fun, %{})
Enum.map(leaves, &cache[&1])
end
defp do_reduce(_graph, [], _initial_acc, _fun, cache), do: cache
defp do_reduce(graph, [cell_id | cell_ids], initial_acc, fun, cache) do
if parent_id = graph[cell_id] do
case cache do
%{^parent_id => acc} ->
acc = fun.(cell_id, acc)
cache = put_in(cache[cell_id], acc)
do_reduce(graph, cell_ids, initial_acc, fun, cache)
_ ->
do_reduce(graph, [parent_id, cell_id | cell_ids], initial_acc, fun, cache)
end
else
acc = fun.(cell_id, initial_acc)
cache = put_in(cache[cell_id], acc)
do_reduce(graph, cell_ids, initial_acc, fun, cache)
end
end
end

2
lib/livebook/web_socket/server.ex
View file

@ -103,7 +103,7 @@ defmodule Livebook.WebSocket.Server do
# Private
defp reply(%{caller: nil} = state, response) do
Logger.warn("The caller is nil, so we can't reply the message: #{inspect(response)}")
Logger.warning("The caller is nil, so we can't reply the message: #{inspect(response)}")
state
end

6
lib/livebook_web/live/session_live.ex
View file

@ -1221,10 +1221,10 @@ defmodule LivebookWeb.SessionLive do
data = socket.private.data
with {:ok, cell, section} <- Notebook.fetch_cell_and_section(data.notebook, cell_id) do
with {:ok, cell, _section} <- Notebook.fetch_cell_and_section(data.notebook, cell_id) do
if Runtime.connected?(data.runtime) do
base_locator = Session.find_base_locator(data, cell, section, existing: true)
ref = Runtime.handle_intellisense(data.runtime, self(), request, base_locator)
parent_locators = Session.parent_locators_for_cell(data, cell)
ref = Runtime.handle_intellisense(data.runtime, self(), request, parent_locators)
{:reply, %{"ref" => inspect(ref)}, socket}
else

6
lib/livebook_web/live/session_live/attached_live.ex
View file

@ -67,7 +67,11 @@ defmodule LivebookWeb.SessionLive.AttachedLive do
</div>
<div>
<div class="input-label">Cookie</div>
<%= text_input(f, :cookie, value: @data["cookie"], class: "input", placeholder: "mycookie") %>
<%= text_input(f, :cookie,
value: @data["cookie"],
class: "input",
placeholder: "mycookie"
) %>
</div>
</div>
<button class="mt-5 button-base button-blue" type="submit" disabled={not data_valid?(@data)}>

59
test/livebook/runtime/erl_dist/runtime_server_test.exs
View file

@ -39,7 +39,7 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServerTest do
describe "evaluate_code/5" do
test "spawns a new evaluator when necessary", %{pid: pid} do
RuntimeServer.evaluate_code(pid, "1 + 1", {:c1, :e1}, {:c1, nil})
RuntimeServer.evaluate_code(pid, "1 + 1", {:c1, :e1}, [])
assert_receive {:runtime_evaluation_response, :e1, _, %{evaluation_time_ms: _time_ms}}
end
@ -48,8 +48,8 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServerTest do
stderr =
ExUnit.CaptureIO.capture_io(:stderr, fn ->
code = "defmodule Foo do end"
RuntimeServer.evaluate_code(pid, code, {:c1, :e1}, {:c1, nil})
RuntimeServer.evaluate_code(pid, code, {:c1, :e2}, {:c1, nil})
RuntimeServer.evaluate_code(pid, code, {:c1, :e1}, [])
RuntimeServer.evaluate_code(pid, code, {:c1, :e2}, [])
assert_receive {:runtime_evaluation_response, :e1, _, %{evaluation_time_ms: _time_ms}}
assert_receive {:runtime_evaluation_response, :e2, _, %{evaluation_time_ms: _time_ms}}
@ -59,7 +59,7 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServerTest do
end
test "proxies evaluation stderr to evaluation stdout", %{pid: pid} do
RuntimeServer.evaluate_code(pid, ~s{IO.puts(:stderr, "error")}, {:c1, :e1}, {:c1, nil})
RuntimeServer.evaluate_code(pid, ~s{IO.puts(:stderr, "error")}, {:c1, :e1}, [])
assert_receive {:runtime_evaluation_output, :e1, {:stdout, "error\n"}}
end
@ -71,17 +71,17 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServerTest do
Logger.error("hey")
"""
RuntimeServer.evaluate_code(pid, code, {:c1, :e1}, {:c1, nil})
RuntimeServer.evaluate_code(pid, code, {:c1, :e1}, [])
assert_receive {:runtime_evaluation_output, :e1, {:stdout, log_message}}
assert log_message =~ "[error] hey"
end
test "supports cross-container evaluation context references", %{pid: pid} do
RuntimeServer.evaluate_code(pid, "x = 1", {:c1, :e1}, {:c1, nil})
RuntimeServer.evaluate_code(pid, "x = 1", {:c1, :e1}, [])
assert_receive {:runtime_evaluation_response, :e1, _, %{evaluation_time_ms: _time_ms}}
RuntimeServer.evaluate_code(pid, "x", {:c2, :e2}, {:c1, :e1})
RuntimeServer.evaluate_code(pid, "x", {:c2, :e2}, [{:c1, :e1}])
assert_receive {:runtime_evaluation_response, :e2, {:text, "\e[34m1\e[0m"},
%{evaluation_time_ms: _time_ms}}
@ -110,7 +110,7 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServerTest do
pid = spawn(fn -> loop.(loop, %{callers: [], count: 0}) end)
"""
RuntimeServer.evaluate_code(pid, code, {:c1, :e1}, {:c1, nil})
RuntimeServer.evaluate_code(pid, code, {:c1, :e1}, [])
assert_receive {:runtime_evaluation_response, :e1, _, %{evaluation_time_ms: _time_ms}}
await_code = """
@ -124,8 +124,8 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServerTest do
# Note: it's important to first start evaluation in :c2,
# because it needs to copy evaluation context from :c1
RuntimeServer.evaluate_code(pid, await_code, {:c2, :e2}, {:c1, :e1})
RuntimeServer.evaluate_code(pid, await_code, {:c1, :e3}, {:c1, :e1})
RuntimeServer.evaluate_code(pid, await_code, {:c2, :e2}, [{:c1, :e1}])
RuntimeServer.evaluate_code(pid, await_code, {:c1, :e3}, [{:c1, :e1}])
assert_receive {:runtime_evaluation_response, :e2, _, %{evaluation_time_ms: _time_ms}}
assert_receive {:runtime_evaluation_response, :e3, _, %{evaluation_time_ms: _time_ms}}
@ -135,7 +135,7 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServerTest do
describe "handle_intellisense/5 given completion request" do
test "provides basic completion when no evaluation reference is given", %{pid: pid} do
request = {:completion, "System.ver"}
ref = RuntimeServer.handle_intellisense(pid, self(), request, {:c1, nil})
ref = RuntimeServer.handle_intellisense(pid, self(), request, [])
assert_receive {:runtime_intellisense_response, ^ref, ^request,
%{items: [%{label: "version/0"}]}}
@ -147,17 +147,17 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServerTest do
number = 10
"""
RuntimeServer.evaluate_code(pid, code, {:c1, :e1}, {:c1, nil})
RuntimeServer.evaluate_code(pid, code, {:c1, :e1}, [])
assert_receive {:runtime_evaluation_response, :e1, _, %{evaluation_time_ms: _time_ms}}
request = {:completion, "num"}
ref = RuntimeServer.handle_intellisense(pid, self(), request, {:c1, :e1})
ref = RuntimeServer.handle_intellisense(pid, self(), request, [{:c1, :e1}])
assert_receive {:runtime_intellisense_response, ^ref, ^request,
%{items: [%{label: "number"}]}}
request = {:completion, "ANSI.brigh"}
ref = RuntimeServer.handle_intellisense(pid, self(), request, {:c1, :e1})
ref = RuntimeServer.handle_intellisense(pid, self(), request, [{:c1, :e1}])
assert_receive {:runtime_intellisense_response, ^ref, ^request,
%{items: [%{label: "bright/0"}]}}
@ -167,7 +167,7 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServerTest do
describe "handle_intellisense/5 given details request" do
test "responds with identifier details", %{pid: pid} do
request = {:details, "System.version", 10}
ref = RuntimeServer.handle_intellisense(pid, self(), request, {:c1, nil})
ref = RuntimeServer.handle_intellisense(pid, self(), request, [])
assert_receive {:runtime_intellisense_response, ^ref, ^request,
%{range: %{from: 1, to: 15}, contents: [_]}}
@ -177,7 +177,7 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServerTest do
describe "handle_intellisense/5 given format request" do
test "responds with a formatted code", %{pid: pid} do
request = {:format, "System.version"}
ref = RuntimeServer.handle_intellisense(pid, self(), request, {:c1, nil})
ref = RuntimeServer.handle_intellisense(pid, self(), request, [])
assert_receive {:runtime_intellisense_response, ^ref, ^request, %{code: "System.version()"}}
end
@ -199,7 +199,7 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServerTest do
spawn_link(fn -> Process.exit(self(), :kill) end)
"""
RuntimeServer.evaluate_code(pid, code, {:c1, :e1}, {:c1, nil})
RuntimeServer.evaluate_code(pid, code, {:c1, :e1}, [])
assert_receive {:runtime_container_down, :c1, message}
assert message =~ "killed"
@ -246,13 +246,13 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServerTest do
@tag opts: @opts
test "notifies runtime owner when a smart cell is started", %{pid: pid} do
RuntimeServer.start_smart_cell(pid, "dumb", "ref", %{}, {:c1, nil})
RuntimeServer.start_smart_cell(pid, "dumb", "ref", %{}, [])
assert_receive {:runtime_smart_cell_started, "ref", %{js_view: %{}, source: "source"}}
end
@tag opts: @opts
test "notifies runtime owner when a smart cell goes down", %{pid: pid} do
RuntimeServer.start_smart_cell(pid, "dumb", "ref", %{}, {:c1, nil})
RuntimeServer.start_smart_cell(pid, "dumb", "ref", %{}, [])
assert_receive {:runtime_smart_cell_started, "ref", %{js_view: %{pid: pid}}}
Process.exit(pid, :crashed)
assert_receive {:runtime_smart_cell_down, "ref"}
@ -260,31 +260,32 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServerTest do
@tag opts: @opts
test "once started scans binding and sends the result to the cell server", %{pid: pid} do
RuntimeServer.start_smart_cell(pid, "dumb", "ref", %{}, {:c1, nil})
RuntimeServer.start_smart_cell(pid, "dumb", "ref", %{}, [])
assert_receive {:smart_cell_debug, "ref", :handle_info, :scan_binding_ping}
end
@tag opts: @opts
test "scans binding when a new base locator is set", %{pid: pid} do
RuntimeServer.start_smart_cell(pid, "dumb", "ref", %{}, {:c1, nil})
test "scans binding when a new parent locators are set", %{pid: pid} do
RuntimeServer.evaluate_code(pid, "1 + 1", {:c1, :e1}, [])
RuntimeServer.start_smart_cell(pid, "dumb", "ref", %{}, [])
assert_receive {:smart_cell_debug, "ref", :handle_info, :scan_binding_ping}
RuntimeServer.set_smart_cell_base_locator(pid, "ref", {:c2, nil})
RuntimeServer.set_smart_cell_parent_locators(pid, "ref", [{:c1, :e1}])
assert_receive {:smart_cell_debug, "ref", :handle_info, :scan_binding_ping}
end
@tag opts: @opts
test "scans binding when the base locator is evaluated", %{pid: pid} do
RuntimeServer.evaluate_code(pid, "1 + 1", {:c1, :e1}, {:c1, nil})
RuntimeServer.start_smart_cell(pid, "dumb", "ref", %{}, {:c1, :e1})
test "scans binding when one of the parent locators is evaluated", %{pid: pid} do
RuntimeServer.evaluate_code(pid, "1 + 1", {:c1, :e1}, [])
RuntimeServer.start_smart_cell(pid, "dumb", "ref", %{}, [{:c1, :e1}])
assert_receive {:smart_cell_debug, "ref", :handle_info, :scan_binding_ping}
RuntimeServer.evaluate_code(pid, "1 + 1", {:c1, :e1}, {:c1, nil})
RuntimeServer.evaluate_code(pid, "1 + 1", {:c1, :e1}, [])
assert_receive {:smart_cell_debug, "ref", :handle_info, :scan_binding_ping}
end
@tag opts: @opts
test "scans evaluation result when the smart cell is evaluated", %{pid: pid} do
RuntimeServer.start_smart_cell(pid, "dumb", "ref", %{}, {:c1, nil})
RuntimeServer.evaluate_code(pid, "1 + 1", {:c1, :e1}, {:c1, nil}, smart_cell_ref: "ref")
RuntimeServer.start_smart_cell(pid, "dumb", "ref", %{}, [])
RuntimeServer.evaluate_code(pid, "1 + 1", {:c1, :e1}, [], smart_cell_ref: "ref")
assert_receive {:smart_cell_debug, "ref", :handle_info, :scan_eval_result_ping}
end
end

526
test/livebook/runtime/evaluator_test.exs
View file

@ -14,7 +14,13 @@ defmodule Livebook.Runtime.EvaluatorTest do
defmacrop metadata do
quote do
%{evaluation_time_ms: _, memory_usage: %{}, code_error: _}
%{
evaluation_time_ms: _,
memory_usage: %{},
code_error: _,
identifiers_used: _,
identifiers_defined: _
}
end
end
@ -26,7 +32,7 @@ defmodule Livebook.Runtime.EvaluatorTest do
x + y
"""
Evaluator.evaluate_code(evaluator, code, :code_1)
Evaluator.evaluate_code(evaluator, code, :code_1, [])
assert_receive {:runtime_evaluation_response, :code_1, {:ok, 3}, metadata() = metadata}
assert metadata.evaluation_time_ms >= 0
@ -35,12 +41,12 @@ defmodule Livebook.Runtime.EvaluatorTest do
metadata.memory_usage
end
test "given no base_ref does not see previous evaluation context", %{evaluator: evaluator} do
Evaluator.evaluate_code(evaluator, "x = 1", :code_1)
test "given no parent refs does not see previous evaluation context", %{evaluator: evaluator} do
Evaluator.evaluate_code(evaluator, "x = 1", :code_1, [])
assert_receive {:runtime_evaluation_response, :code_1, _, metadata()}
ignore_warnings(fn ->
Evaluator.evaluate_code(evaluator, "x", :code_2)
Evaluator.evaluate_code(evaluator, "x", :code_2, [])
assert_receive {:runtime_evaluation_response, :code_2,
{:error, _kind,
@ -50,23 +56,36 @@ defmodule Livebook.Runtime.EvaluatorTest do
end)
end
test "given base_ref sees previous evaluation context", %{evaluator: evaluator} do
Evaluator.evaluate_code(evaluator, "x = 1", :code_1)
test "given parent refs sees previous evaluation context", %{evaluator: evaluator} do
Evaluator.evaluate_code(evaluator, "x = 1", :code_1, [])
assert_receive {:runtime_evaluation_response, :code_1, _, metadata()}
Evaluator.evaluate_code(evaluator, "x", :code_2, :code_1)
Evaluator.evaluate_code(evaluator, "x", :code_2, [:code_1])
assert_receive {:runtime_evaluation_response, :code_2, {:ok, 1}, metadata()}
end
test "given invalid base_ref just uses default context", %{evaluator: evaluator} do
Evaluator.evaluate_code(evaluator, ":hey", :code_1, :code_nonexistent)
test "given invalid parent ref uses the default context", %{evaluator: evaluator} do
Evaluator.evaluate_code(evaluator, ":hey", :code_1, [:code_nonexistent])
assert_receive {:runtime_evaluation_response, :code_1, {:ok, :hey}, metadata()}
end
test "given parent refs sees previous process dictionary", %{evaluator: evaluator} do
Evaluator.evaluate_code(evaluator, "Process.put(:x, 1)", :code_1, [])
assert_receive {:runtime_evaluation_response, :code_1, _, metadata()}
Evaluator.evaluate_code(evaluator, "Process.put(:x, 2)", :code_2, [])
assert_receive {:runtime_evaluation_response, :code_2, _, metadata()}
Evaluator.evaluate_code(evaluator, "Process.get(:x)", :code_3, [:code_1])
assert_receive {:runtime_evaluation_response, :code_3, {:ok, 1}, metadata()}
Evaluator.evaluate_code(evaluator, "Process.get(:x)", :code_3, [:code_2])
assert_receive {:runtime_evaluation_response, :code_3, {:ok, 2}, metadata()}
end
test "captures standard output and sends it to the caller", %{evaluator: evaluator} do
Evaluator.evaluate_code(evaluator, ~s{IO.puts("hey")}, :code_1)
Evaluator.evaluate_code(evaluator, ~s{IO.puts("hey")}, :code_1, [])
assert_receive {:runtime_evaluation_output, :code_1, {:stdout, "hey\n"}}
end
@ -81,7 +100,7 @@ defmodule Livebook.Runtime.EvaluatorTest do
end
"""
Evaluator.evaluate_code(evaluator, code, :code_1)
Evaluator.evaluate_code(evaluator, code, :code_1, [])
assert_receive {:runtime_evaluation_input, :code_1, reply_to, "input1"}
send(reply_to, {:runtime_evaluation_input_reply, {:ok, :value}})
@ -94,7 +113,7 @@ defmodule Livebook.Runtime.EvaluatorTest do
List.first(%{})
"""
Evaluator.evaluate_code(evaluator, code, :code_1, nil, file: "file.ex")
Evaluator.evaluate_code(evaluator, code, :code_1, [], file: "file.ex")
assert_receive {:runtime_evaluation_response, :code_1,
{:error, :error, :function_clause,
@ -107,7 +126,7 @@ defmodule Livebook.Runtime.EvaluatorTest do
test "returns additional metadata when there is a syntax error", %{evaluator: evaluator} do
code = "1+"
Evaluator.evaluate_code(evaluator, code, :code_1, nil, file: "file.ex")
Evaluator.evaluate_code(evaluator, code, :code_1, [], file: "file.ex")
assert_receive {:runtime_evaluation_response, :code_1,
{:error, :error, %TokenMissingError{}, _stacktrace},
@ -122,7 +141,7 @@ defmodule Livebook.Runtime.EvaluatorTest do
test "returns additional metadata when there is a compilation error", %{evaluator: evaluator} do
code = "x"
Evaluator.evaluate_code(evaluator, code, :code_1, nil, file: "file.ex")
Evaluator.evaluate_code(evaluator, code, :code_1, [], file: "file.ex")
assert_receive {:runtime_evaluation_response, :code_1,
{:error, :error, %CompileError{}, _stacktrace},
@ -139,7 +158,7 @@ defmodule Livebook.Runtime.EvaluatorTest do
Code.eval_string("x")
"""
Evaluator.evaluate_code(evaluator, code, :code_1, nil, file: "file.ex")
Evaluator.evaluate_code(evaluator, code, :code_1, [], file: "file.ex")
expected_stacktrace = [
{:elixir_eval, :__FILE__, 1, [file: ~c"file.ex", line: 1]}
@ -152,29 +171,30 @@ defmodule Livebook.Runtime.EvaluatorTest do
test "in case of an error returns only the relevant part of stacktrace",
%{evaluator: evaluator} do
code = """
defmodule Livebook.EvaluatorTest.Stacktrace.Math do
defmodule Livebook.Runtime.EvaluatorTest.Stacktrace.Math do
def bad_math do
result = 1 / 0
{:ok, result}
end
end
defmodule Livebook.EvaluatorTest.Stacktrace.Cat do
defmodule Livebook.Runtime.EvaluatorTest.Stacktrace.Cat do
def meow do
Livebook.EvaluatorTest.Stacktrace.Math.bad_math()
Livebook.Runtime.EvaluatorTest.Stacktrace.Math.bad_math()
:ok
end
end
Livebook.EvaluatorTest.Stacktrace.Cat.meow()
Livebook.Runtime.EvaluatorTest.Stacktrace.Cat.meow()
"""
ignore_warnings(fn ->
Evaluator.evaluate_code(evaluator, code, :code_1)
Evaluator.evaluate_code(evaluator, code, :code_1, [])
expected_stacktrace = [
{Livebook.EvaluatorTest.Stacktrace.Math, :bad_math, 0, [file: ~c"nofile", line: 3]},
{Livebook.EvaluatorTest.Stacktrace.Cat, :meow, 0, [file: ~c"nofile", line: 10]},
{Livebook.Runtime.EvaluatorTest.Stacktrace.Math, :bad_math, 0,
[file: ~c"nofile", line: 3]},
{Livebook.Runtime.EvaluatorTest.Stacktrace.Cat, :meow, 0, [file: ~c"nofile", line: 10]},
{:elixir_eval, :__FILE__, 1, [file: ~c"nofile", line: 15]}
]
@ -185,7 +205,7 @@ defmodule Livebook.Runtime.EvaluatorTest do
end)
end
test "in case of an error uses own evaluation context as the resulting context",
test "in case of an error uses empty evaluation context as the resulting context",
%{evaluator: evaluator} do
code1 = """
x = 2
@ -199,14 +219,14 @@ defmodule Livebook.Runtime.EvaluatorTest do
x * x
"""
Evaluator.evaluate_code(evaluator, code1, :code_1)
Evaluator.evaluate_code(evaluator, code1, :code_1, [])
assert_receive {:runtime_evaluation_response, :code_1, {:ok, _}, metadata()}
Evaluator.evaluate_code(evaluator, code2, :code_2, :code_1)
Evaluator.evaluate_code(evaluator, code2, :code_2, [:code_1])
assert_receive {:runtime_evaluation_response, :code_2, {:error, _, _, _}, metadata()}
Evaluator.evaluate_code(evaluator, code3, :code_3, :code_2)
Evaluator.evaluate_code(evaluator, code3, :code_3, [:code_2, :code_1])
assert_receive {:runtime_evaluation_response, :code_3, {:ok, 4}, metadata()}
end
@ -216,7 +236,7 @@ defmodule Livebook.Runtime.EvaluatorTest do
"""
opts = [file: "/path/dir/file"]
Evaluator.evaluate_code(evaluator, code, :code_1, nil, opts)
Evaluator.evaluate_code(evaluator, code, :code_1, [], opts)
assert_receive {:runtime_evaluation_response, :code_1, {:ok, "/path/dir"}, metadata()}
end
@ -224,15 +244,15 @@ defmodule Livebook.Runtime.EvaluatorTest do
test "kills widgets that that no evaluation points to", %{evaluator: evaluator} do
# Evaluate the code twice, each time a new widget is spawned.
# The evaluation reference is the same, so the second one overrides
# the first one and the first widget should eventually be kiled.
# the first one and the first widget should eventually be killed.
Evaluator.evaluate_code(evaluator, spawn_widget_code(), :code_1)
Evaluator.evaluate_code(evaluator, spawn_widget_code(), :code_1, [])
assert_receive {:runtime_evaluation_response, :code_1, {:ok, widget_pid1}, metadata()}
ref = Process.monitor(widget_pid1)
Evaluator.evaluate_code(evaluator, spawn_widget_code(), :code_1)
Evaluator.evaluate_code(evaluator, spawn_widget_code(), :code_1, [])
assert_receive {:runtime_evaluation_response, :code_1, {:ok, widget_pid2}, metadata()}
@ -245,23 +265,408 @@ defmodule Livebook.Runtime.EvaluatorTest do
# The widget is spawned from a process that terminates,
# so the widget should terminate immediately as well
Evaluator.evaluate_code(evaluator, spawn_widget_from_terminating_process_code(), :code_1)
Evaluator.evaluate_code(
evaluator,
spawn_widget_from_terminating_process_code(),
:code_1,
[]
)
assert_receive {:runtime_evaluation_response, :code_1, {:ok, widget_pid1}, metadata()}
refute Process.alive?(widget_pid1)
ref = Process.monitor(widget_pid1)
assert_receive {:DOWN, ^ref, :process, ^widget_pid1, _reason}
end
end
describe "evaluate_code/6 identifier tracking" do
defp eval(code, evaluator, eval_idx) do
ref = eval_idx
parent_refs = Enum.to_list((eval_idx - 1)..0//-1)
Evaluator.evaluate_code(evaluator, code, ref, parent_refs)
assert_receive {:runtime_evaluation_response, ^ref, {:ok, _}, metadata}
%{used: metadata.identifiers_used, defined: metadata.identifiers_defined}
end
test "variables", %{evaluator: evaluator} do
identifiers =
"""
x = 1
y = 1
"""
|> eval(evaluator, 0)
assert %{
{:variable, {:x, nil}} => _,
{:variable, {:y, nil}} => _
} = identifiers.defined
identifiers =
"""
x
"""
|> eval(evaluator, 1)
assert {:variable, {:x, nil}} in identifiers.used
assert {:variable, {:y, nil}} not in identifiers.used
end
test "variables with non-default context", %{evaluator: evaluator} do
identifiers =
"""
var!(x, :context) = 1
"""
|> eval(evaluator, 0)
assert %{{:variable, {:x, :context}} => _} = identifiers.defined
identifiers =
"""
var!(x, :context)
"""
|> eval(evaluator, 1)
assert {:variable, {:x, :context}} in identifiers.used
end
test "variables used inside a module", %{evaluator: evaluator} do
identifiers =
"""
x = 1
y = 1
z = 1
"""
|> eval(evaluator, 0)
assert %{
{:variable, {:x, nil}} => _,
{:variable, {:y, nil}} => _,
{:variable, {:z, nil}} => _
} = identifiers.defined
identifiers =
"""
defmodule Livebook.Runtime.EvaluatorTest.Identifiers.UsedVars do
def fun(), do: unquote(x)
end
y
"""
|> eval(evaluator, 1)
assert {:variable, {:x, nil}} in identifiers.used
assert {:variable, {:y, nil}} in identifiers.used
assert {:variable, {:z, nil}} not in identifiers.used
end
test "reports parentheses-less arity-0 import as a used variable", %{evaluator: evaluator} do
identifiers =
"""
self
"""
|> eval(evaluator, 0)
assert {:variable, {:self, nil}} in identifiers.used
assert :imports in identifiers.used
end
test "module definition", %{evaluator: evaluator} do
identifiers =
"""
defmodule Livebook.Runtime.EvaluatorTest.Identifiers.ModuleDefinition do
def fun(), do: 1
end
"""
|> eval(evaluator, 0)
assert {:alias, :"Elixir.Livebook.Runtime.EvaluatorTest.Identifiers.ModuleDefinition"} in identifiers.used
assert %{
{:module, :"Elixir.Livebook.Runtime.EvaluatorTest.Identifiers.ModuleDefinition"} =>
version1
} = identifiers.defined
identifiers =
"""
defmodule Livebook.Runtime.EvaluatorTest.Identifiers.ModuleDefinition do
def fun(), do: 1
end
"""
|> eval(evaluator, 0)
assert %{
{:module, :"Elixir.Livebook.Runtime.EvaluatorTest.Identifiers.ModuleDefinition"} =>
^version1
} = identifiers.defined
identifiers =
"""
defmodule Livebook.Runtime.EvaluatorTest.Identifiers.ModuleDefinition do
def fun(), do: 2
end
"""
|> eval(evaluator, 0)
assert %{
{:module, :"Elixir.Livebook.Runtime.EvaluatorTest.Identifiers.ModuleDefinition"} =>
version2
} = identifiers.defined
assert version2 != version1
end
test "module function call", %{evaluator: evaluator} do
identifiers =
"""
Enum.uniq([1, 2])
"""
|> eval(evaluator, 0)
assert {:module, :"Elixir.Enum"} in identifiers.used
end
test "alias", %{evaluator: evaluator} do
identifiers =
"""
alias Map, as: M
"""
|> eval(evaluator, 0)
assert {:alias, :"Elixir.Map"} in identifiers.used
assert %{{:alias, :"Elixir.M"} => :"Elixir.Map"} = identifiers.defined
identifiers =
"""
M.new()
"""
|> eval(evaluator, 1)
assert {:alias, :"Elixir.M"} in identifiers.used
assert {:module, :"Elixir.Map"} in identifiers.used
end
test "require", %{evaluator: evaluator} do
identifiers =
"""
require Integer
"""
|> eval(evaluator, 0)
assert {:alias, :"Elixir.Integer"} in identifiers.used
assert %{{:require, :"Elixir.Integer"} => :ok} = identifiers.defined
identifiers =
"""
Integer.is_even(2)
"""
|> eval(evaluator, 1)
assert {:alias, :"Elixir.Integer"} in identifiers.used
assert {:require, :"Elixir.Integer"} in identifiers.used
assert {:module, :"Elixir.Integer"} in identifiers.used
end
test "import", %{evaluator: evaluator} do
identifiers =
"""
import Enum
"""
|> eval(evaluator, 0)
assert %{:imports => version1} = identifiers.defined
identifiers =
"""
import Enum
"""
|> eval(evaluator, 0)
assert %{:imports => ^version1} = identifiers.defined
identifiers =
"""
import Integer
"""
|> eval(evaluator, 0)
assert :imports in identifiers.used
assert {:alias, :"Elixir.Integer"} in identifiers.used
assert {:module, :"Elixir.Integer"} in identifiers.used
assert %{
:imports => version2,
{:require, :"Elixir.Integer"} => :ok
} = identifiers.defined
assert version2 != version1
identifiers =
"""
is_even(2)
"""
|> eval(evaluator, 1)
assert :imports in identifiers.used
assert {:module, :"Elixir.Integer"} in identifiers.used
identifiers =
"""
Integer.is_even(2)
"""
|> eval(evaluator, 1)
assert {:require, :"Elixir.Integer"} in identifiers.used
assert {:module, :"Elixir.Integer"} in identifiers.used
end
test "struct", %{evaluator: evaluator} do
identifiers =
"""
%URI{}
"""
|> eval(evaluator, 0)
assert {:alias, :"Elixir.URI"} in identifiers.used
assert {:module, :"Elixir.URI"} in identifiers.used
end
test "process dictionary", %{evaluator: evaluator} do
identifiers =
"""
:ok
"""
|> eval(evaluator, 0)
# Every evaluation should depend on process dictionary
assert :pdict in identifiers.used
identifiers =
"""
Process.put(:x, 1)
"""
|> eval(evaluator, 0)
assert %{pdict: version1} = identifiers.defined
identifiers =
"""
Process.put(:x, 1)
"""
|> eval(evaluator, 0)
assert %{pdict: ^version1} = identifiers.defined
identifiers =
"""
Process.put(:x, 2)
"""
|> eval(evaluator, 0)
assert %{pdict: version2} = identifiers.defined
assert version2 != version1
end
test "context merging", %{evaluator: evaluator} do
"""
x = 1
y = 1
alias Enum, as: E
alias Map, as: M
require Enum
import Integer, only: [is_odd: 1, is_even: 1, to_string: 2, to_charlist: 2]
Process.put(:x, 1)
Process.put(:y, 1)
Process.put(:z, 1)
"""
|> eval(evaluator, 0)
"""
y = 2
alias MapSet, as: M
require Map
import Integer, except: [is_even: 1, to_string: 2]
Process.put(:y, 2)
Process.delete(:z)
"""
|> eval(evaluator, 1)
# Evaluation 0 context
context = Evaluator.get_evaluation_context(evaluator, [0])
assert Enum.sort(context.binding) == [x: 1, y: 1]
assert Enum.sort(context.env.aliases) == [{E, Enum}, {M, Map}]
assert Enum in context.env.requires
assert [_, _ | _] = context.env.functions[Integer]
assert [_, _ | _] = context.env.macros[Integer]
assert context.env.versioned_vars == %{{:x, nil} => 0, {:y, nil} => 1}
assert context.pdict == %{x: 1, y: 1, z: 1}
# Evaluation 1 context
context = Evaluator.get_evaluation_context(evaluator, [1])
assert Enum.sort(context.binding) == [y: 2]
assert Enum.sort(context.env.aliases) == [{M, MapSet}]
assert Map in context.env.requires
assert Enum not in context.env.requires
# Imports are not diffed
assert {Integer, [to_charlist: 2]} in context.env.functions
assert {Integer, [is_odd: 1]} in context.env.macros
assert context.env.versioned_vars == %{{:y, nil} => 0}
# Process dictionary is not diffed
assert context.pdict == %{x: 1, y: 2}
# Merged context
context = Evaluator.get_evaluation_context(evaluator, [1, 0])
assert Enum.sort(context.binding) == [x: 1, y: 2]
assert Enum.sort(context.env.aliases) == [{E, Enum}, {M, MapSet}]
assert Enum in context.env.requires
assert Map in context.env.requires
assert {Integer, [to_charlist: 2]} in context.env.functions
assert {Integer, [is_odd: 1]} in context.env.macros
assert context.env.versioned_vars == %{{:x, nil} => 0, {:y, nil} => 1}
assert context.pdict == %{x: 1, y: 2}
end
end
describe "forget_evaluation/2" do
test "invalidates the given reference", %{evaluator: evaluator} do
Evaluator.evaluate_code(evaluator, "x = 1", :code_1)
Evaluator.evaluate_code(evaluator, "x = 1", :code_1, [])
assert_receive {:runtime_evaluation_response, :code_1, _, metadata()}
Evaluator.forget_evaluation(evaluator, :code_1)
ignore_warnings(fn ->
Evaluator.evaluate_code(evaluator, "x", :code_2, :code_1)
Evaluator.evaluate_code(evaluator, "x", :code_2, [:code_1])
assert_receive {:runtime_evaluation_response, :code_2,
{:error, _kind,
@ -272,7 +677,7 @@ defmodule Livebook.Runtime.EvaluatorTest do
end
test "kills widgets that no evaluation points to", %{evaluator: evaluator} do
Evaluator.evaluate_code(evaluator, spawn_widget_code(), :code_1)
Evaluator.evaluate_code(evaluator, spawn_widget_code(), :code_1, [])
assert_receive {:runtime_evaluation_response, :code_1, {:ok, widget_pid1}, metadata()}
@ -295,44 +700,37 @@ defmodule Livebook.Runtime.EvaluatorTest do
test "copies the given context and sets as the initial one",
%{evaluator: evaluator, parent_evaluator: parent_evaluator} do
Evaluator.evaluate_code(parent_evaluator, "x = 1", :code_1)
Evaluator.evaluate_code(parent_evaluator, "x = 1", :code_1, [])
assert_receive {:runtime_evaluation_response, :code_1, _, metadata()}
Evaluator.initialize_from(evaluator, parent_evaluator, :code_1)
Evaluator.initialize_from(evaluator, parent_evaluator, [:code_1])
Evaluator.evaluate_code(evaluator, "x", :code_2)
assert_receive {:runtime_evaluation_response, :code_2, {:ok, 1}, metadata()}
end
test "mirrors process dictionary of the given evaluator",
%{evaluator: evaluator, parent_evaluator: parent_evaluator} do
Evaluator.evaluate_code(parent_evaluator, "Process.put(:data, 1)", :code_1)
assert_receive {:runtime_evaluation_response, :code_1, _, metadata()}
Evaluator.initialize_from(evaluator, parent_evaluator, :code_1)
Evaluator.evaluate_code(evaluator, "Process.get(:data)", :code_2)
Evaluator.evaluate_code(evaluator, "x", :code_2, [])
assert_receive {:runtime_evaluation_response, :code_2, {:ok, 1}, metadata()}
end
end
describe "binding order" do
test "keeps binding in evaluation order, starting from most recent", %{evaluator: evaluator} do
Evaluator.evaluate_code(evaluator, "b = 1", :code_1)
Evaluator.evaluate_code(evaluator, "a = 1", :code_2, :code_1)
Evaluator.evaluate_code(evaluator, "c = 1", :code_3, :code_2)
Evaluator.evaluate_code(evaluator, "x = 1", :code_4, :code_3)
Evaluator.evaluate_code(evaluator, "b = 1", :code_1, [])
Evaluator.evaluate_code(evaluator, "a = 1", :code_2, [:code_1])
Evaluator.evaluate_code(evaluator, "c = 1", :code_3, [:code_2, :code_1])
Evaluator.evaluate_code(evaluator, "x = 1", :code_4, [:code_3, :code_2, :code_1])
%{binding: binding} =
Evaluator.get_evaluation_context(evaluator, [:code_4, :code_3, :code_2, :code_1])
{:ok, %{binding: binding}} = Evaluator.fetch_evaluation_context(evaluator, :code_4)
assert [:x, :c, :a, :b] == Enum.map(binding, &elem(&1, 0))
end
test "treats rebound names as new", %{evaluator: evaluator} do
Evaluator.evaluate_code(evaluator, "b = 1", :code_1)
Evaluator.evaluate_code(evaluator, "a = 1", :code_2, :code_1)
Evaluator.evaluate_code(evaluator, "b = 2", :code_3, :code_2)
Evaluator.evaluate_code(evaluator, "b = 1", :code_1, [])
Evaluator.evaluate_code(evaluator, "a = 1", :code_2, [:code_1])
Evaluator.evaluate_code(evaluator, "b = 2", :code_3, [:code_2, :code_1])
%{binding: binding} =
Evaluator.get_evaluation_context(evaluator, [:code_3, :code_2, :code_1])
{:ok, %{binding: binding}} = Evaluator.fetch_evaluation_context(evaluator, :code_3)
assert [:b, :a] == Enum.map(binding, &elem(&1, 0))
end
end
@ -359,6 +757,11 @@ defmodule Livebook.Runtime.EvaluatorTest do
ref = make_ref()
send(Process.group_leader(), {:io_request, self(), ref, {:livebook_reference_object, widget_pid, self()}})
receive do
{:io_reply, ^ref, :ok} -> :ok
end
send(Process.group_leader(), {:io_request, self(), ref, {:livebook_monitor_object, widget_pid, widget_pid, :stop}})
receive do
@ -383,6 +786,11 @@ defmodule Livebook.Runtime.EvaluatorTest do
ref = make_ref()
send(Process.group_leader(), {:io_request, self(), ref, {:livebook_reference_object, widget_pid, self()}})
receive do
{:io_reply, ^ref, :ok} -> :ok
end
send(Process.group_leader(), {:io_request, self(), ref, {:livebook_monitor_object, widget_pid, widget_pid, :stop}})
receive do

1164
test/livebook/session/data_test.exs
View file

File diff suppressed because it is too large Load diff

53
test/livebook/session_test.exs
View file

@ -7,6 +7,8 @@ defmodule Livebook.SessionTest do
alias Livebook.Notebook.{Section, Cell}
alias Livebook.Session.Data
@eval_meta %{evaluation_time_ms: 10, identifiers_used: [], identifiers_defined: %{}}
setup do
session = start_session()
%{session: session}
@ -759,10 +761,7 @@ defmodule Livebook.SessionTest do
Session.queue_cell_evaluation(session.pid, smart_cell.id)
send(
session.pid,
{:runtime_evaluation_response, "setup", {:ok, ""}, %{evaluation_time_ms: 10}}
)
send(session.pid, {:runtime_evaluation_response, "setup", {:ok, ""}, @eval_meta})
session_pid = session.pid
assert_receive {:ping, ^session_pid, metadata, %{ref: "ref"}}
@ -781,8 +780,8 @@ defmodule Livebook.SessionTest do
end
end
describe "find_base_locator/3" do
test "given cell in main flow returns previous Code cell" do
describe "parent_locators_for_cell/2" do
test "given cell in main flow returns previous Code cells" do
cell1 = %{Cell.new(:code) | id: "c1"}
cell2 = %{Cell.new(:markdown) | id: "c2"}
section1 = %{Section.new() | id: "s1", cells: [cell1, cell2]}
@ -793,10 +792,19 @@ defmodule Livebook.SessionTest do
notebook = %{Notebook.new() | sections: [section1, section2]}
data = Data.new(notebook)
assert {:main_flow, "c1"} = Session.find_base_locator(data, cell3, section2)
data =
data_after_operations!(data, [
{:set_runtime, self(), connected_noop_runtime()},
{:queue_cells_evaluation, self(), ["c1"]},
{:add_cell_evaluation_response, self(), "setup", {:ok, nil}, @eval_meta},
{:add_cell_evaluation_response, self(), "c1", {:ok, nil}, @eval_meta}
])
assert [{:main_flow, "c1"}, {:main_flow, "setup"}] =
Session.parent_locators_for_cell(data, cell3)
end
test "given cell in branching section returns previous Code cell in that section" do
test "given cell in branching section returns Code cells from both sections" do
section1 = %{Section.new() | id: "s1"}
cell1 = %{Cell.new(:code) | id: "c1"}
@ -813,17 +821,25 @@ defmodule Livebook.SessionTest do
notebook = %{Notebook.new() | sections: [section1, section2]}
data = Data.new(notebook)
assert {"s2", "c1"} = Session.find_base_locator(data, cell3, section2)
data =
data_after_operations!(data, [
{:set_runtime, self(), connected_noop_runtime()},
{:queue_cells_evaluation, self(), ["c1"]},
{:add_cell_evaluation_response, self(), "setup", {:ok, nil}, @eval_meta},
{:add_cell_evaluation_response, self(), "c1", {:ok, nil}, @eval_meta}
])
assert [{"s2", "c1"}, {:main_flow, "setup"}] = Session.parent_locators_for_cell(data, cell3)
end
test "given cell in main flow returns nil if there is no previous cell" do
%{setup_section: %{cells: [setup_cell]} = setup_section} = notebook = Notebook.new()
test "given cell in main flow returns an empty list if there is no previous cell" do
%{setup_section: %{cells: [setup_cell]}} = notebook = Notebook.new()
data = Data.new(notebook)
assert {:main_flow, nil} = Session.find_base_locator(data, setup_cell, setup_section)
assert [] = Session.parent_locators_for_cell(data, setup_cell)
end
test "when :existing is set ignores fresh and aborted cells" do
test "ignores fresh and aborted cells" do
cell1 = %{Cell.new(:code) | id: "c1"}
cell2 = %{Cell.new(:code) | id: "c2"}
section1 = %{Section.new() | id: "s1", cells: [cell1, cell2]}
@ -834,24 +850,25 @@ defmodule Livebook.SessionTest do
notebook = %{Notebook.new() | sections: [section1, section2]}
data = Data.new(notebook)
assert {:main_flow, nil} = Session.find_base_locator(data, cell3, section2, existing: true)
assert [] = Session.parent_locators_for_cell(data, cell3)
data =
data_after_operations!(data, [
{:set_runtime, self(), connected_noop_runtime()},
{:queue_cells_evaluation, self(), ["c1"]},
{:add_cell_evaluation_response, self(), "setup", {:ok, nil}, %{evaluation_time_ms: 10}},
{:add_cell_evaluation_response, self(), "c1", {:ok, nil}, %{evaluation_time_ms: 10}}
{:add_cell_evaluation_response, self(), "setup", {:ok, nil}, @eval_meta},
{:add_cell_evaluation_response, self(), "c1", {:ok, nil}, @eval_meta}
])
assert {:main_flow, "c1"} = Session.find_base_locator(data, cell3, section2, existing: true)
assert [{:main_flow, "c1"}, {:main_flow, "setup"}] =
Session.parent_locators_for_cell(data, cell3)
data =
data_after_operations!(data, [
{:reflect_main_evaluation_failure, self()}
])
assert {:main_flow, nil} = Session.find_base_locator(data, cell3, section2, existing: true)
assert [] = Session.parent_locators_for_cell(data, cell3)
end
end

2
test/support/noop_runtime.ex
View file

@ -28,7 +28,7 @@ defmodule Livebook.Runtime.NoopRuntime do
def read_file(_, _), do: raise("not implemented")
def start_smart_cell(_, _, _, _, _), do: :ok
def set_smart_cell_base_locator(_, _, _), do: :ok
def set_smart_cell_parent_locators(_, _, _), do: :ok
def stop_smart_cell(_, _), do: :ok
def fixed_dependencies?(_), do: false