diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index e6cba7a50..f4c1f42a3 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -6,7 +6,8 @@ on:
- main
env:
otp: "25.0"
- elixir: "1.14.0"
+ # TODO: update to v1.14.2 once it is out
+ elixir: "main"
jobs:
main:
runs-on: ubuntu-latest
diff --git a/lib/livebook/notebook.ex b/lib/livebook/notebook.ex
index b5e35f2a1..d7d8337fc 100644
--- a/lib/livebook/notebook.ex
+++ b/lib/livebook/notebook.ex
@@ -471,23 +471,6 @@ defmodule Livebook.Notebook do
|> Enum.filter(fn {cell, _} -> MapSet.member?(child_cell_ids, cell.id) end)
end
- @doc """
- Returns the list with the given parent cells and all of
- their child cells.
-
- The cells are not ordered in any secific way.
- """
- @spec cell_ids_with_children(t(), list(Cell.id())) :: list(Cell.id())
- def cell_ids_with_children(notebook, parent_cell_ids) do
- graph = cell_dependency_graph(notebook)
-
- for parent_id <- parent_cell_ids,
- leaf_id <- Graph.leaves(graph),
- cell_id <- Graph.find_path(graph, leaf_id, parent_id),
- uniq: true,
- do: cell_id
- end
-
@doc """
Computes cell dependency graph.
diff --git a/lib/livebook/runtime.ex b/lib/livebook/runtime.ex
index c1ac459b3..1ef685b70 100644
--- a/lib/livebook/runtime.ex
+++ b/lib/livebook/runtime.ex
@@ -26,12 +26,15 @@ defprotocol Livebook.Runtime do
@typedoc """
A pair identifying evaluation together with its container.
-
- When the evaluation reference is `nil`, the `locator` points to
- a container and may be used to represent its default evaluation
- context.
"""
- @type locator :: {container_ref(), evaluation_ref() | nil}
+ @type locator :: {container_ref(), evaluation_ref()}
+
+ @typedoc """
+ A sequence of locators representing a multi-stage evaluation.
+
+ The evaluation locators should be ordered from most recent to oldest.
+ """
+ @type parent_locators :: list(locator())
@typedoc """
An output emitted during evaluation or as the final result.
@@ -64,12 +67,22 @@ defprotocol Livebook.Runtime do
| {:error, message :: String.t(), type :: {:missing_secret, String.t()} | :other}
@typedoc """
- Additional information about a complted evaluation.
+ Additional information about a completed evaluation.
+
+ ## Identifiers
+
+ When possible, the metadata may include a list of identifiers (such
+ as variables, modules, imports) used during evaluation, and a list
+ of identifiers defined along with the version (such as a hash digest
+ of the underlying value). With this information, Livebook can track
+ dependencies between evaluations and avoids unnecessary reevaluations.
"""
@type evaluation_response_metadata :: %{
evaluation_time_ms: non_neg_integer(),
code_error: code_error(),
- memory_usage: runtime_memory()
+ memory_usage: runtime_memory(),
+ identifiers_used: list(identifier :: term()) | :unknown,
+ identifiers_defined: %{(identifier :: term()) => version :: term()}
}
@typedoc """
@@ -306,9 +319,8 @@ defprotocol Livebook.Runtime do
be evaluated as well as the evaluation reference to store the
resulting context under.
- Additionally, `base_locator` points to a previous evaluation to be
- used as the starting point of this evaluation. If not applicable,
- the previous evaluation reference may be specified as `nil`.
+ Additionally, `parent_locators` points to a sequence of previous
+ evaluations to be used as the starting point of this evaluation.
## Communication
@@ -347,8 +359,8 @@ defprotocol Livebook.Runtime do
* `:smart_cell_ref` - a reference of the smart cell which code is
to be evaluated, if applicable
"""
- @spec evaluate_code(t(), String.t(), locator(), locator(), keyword()) :: :ok
- def evaluate_code(runtime, code, locator, base_locator, opts \\ [])
+ @spec evaluate_code(t(), String.t(), locator(), parent_locators(), keyword()) :: :ok
+ def evaluate_code(runtime, code, locator, parent_locators, opts \\ [])
@doc """
Disposes of an evaluation identified by the given locator.
@@ -378,11 +390,11 @@ defprotocol Livebook.Runtime do
* `{:runtime_intellisense_response, ref, request, response}`.
- The given `base_locator` idenfities an evaluation that may be
- used as the context when resolving the request (if relevant).
+ The given `parent_locators` identifies a sequence of evaluations
+ that may be used as the context when resolving the request (if relevant).
"""
- @spec handle_intellisense(t(), pid(), intellisense_request(), locator()) :: reference()
- def handle_intellisense(runtime, send_to, request, base_locator)
+ @spec handle_intellisense(t(), pid(), intellisense_request(), parent_locators()) :: reference()
+ def handle_intellisense(runtime, send_to, request, parent_locators)
@doc """
Reads file at the given absolute path within the runtime file system.
@@ -401,9 +413,9 @@ defprotocol Livebook.Runtime do
The cell may depend on evaluation context to provide a better user
experience, for instance it may suggest relevant variable names.
- Similarly to `evaluate_code/5`, `base_locator` must be specified
- pointing to the evaluation to use as the context. When the locator
- changes, it can be updated with `set_smart_cell_base_locator/3`.
+ Similarly to `evaluate_code/5`, `parent_locators` must be specified
+ pointing to the sequence of evaluations to use as the context. When
+ the sequence changes, it can be updated with `set_smart_cell_parent_locators/3`.
Once the cell starts, the runtime sends the following message
@@ -425,16 +437,22 @@ defprotocol Livebook.Runtime do
state later. Note that for persistence they get serialized and
deserialized as JSON.
"""
- @spec start_smart_cell(t(), String.t(), smart_cell_ref(), smart_cell_attrs(), locator()) :: :ok
- def start_smart_cell(runtime, kind, ref, attrs, base_locator)
+ @spec start_smart_cell(
+ t(),
+ String.t(),
+ smart_cell_ref(),
+ smart_cell_attrs(),
+ parent_locators()
+ ) :: :ok
+ def start_smart_cell(runtime, kind, ref, attrs, parent_locators)
@doc """
- Updates the locator used by a smart cell as its context.
+ Updates the parent locator used by a smart cell as its context.
See `start_smart_cell/5` for more details.
"""
- @spec set_smart_cell_base_locator(t(), smart_cell_ref(), locator()) :: :ok
- def set_smart_cell_base_locator(runtime, ref, base_locator)
+ @spec set_smart_cell_parent_locators(t(), smart_cell_ref(), parent_locators()) :: :ok
+ def set_smart_cell_parent_locators(runtime, ref, parent_locators)
@doc """
Stops smart cell identified by the given reference.
diff --git a/lib/livebook/runtime/attached.ex b/lib/livebook/runtime/attached.ex
index a556b1ee5..f6266898f 100644
--- a/lib/livebook/runtime/attached.ex
+++ b/lib/livebook/runtime/attached.ex
@@ -95,8 +95,8 @@ defimpl Livebook.Runtime, for: Livebook.Runtime.Attached do
Livebook.Runtime.Attached.new(runtime.node, runtime.cookie)
end
- def evaluate_code(runtime, code, locator, base_locator, opts \\ []) do
- RuntimeServer.evaluate_code(runtime.server_pid, code, locator, base_locator, opts)
+ def evaluate_code(runtime, code, locator, parent_locators, opts \\ []) do
+ RuntimeServer.evaluate_code(runtime.server_pid, code, locator, parent_locators, opts)
end
def forget_evaluation(runtime, locator) do
@@ -107,20 +107,20 @@ defimpl Livebook.Runtime, for: Livebook.Runtime.Attached do
RuntimeServer.drop_container(runtime.server_pid, container_ref)
end
- def handle_intellisense(runtime, send_to, request, base_locator) do
- RuntimeServer.handle_intellisense(runtime.server_pid, send_to, request, base_locator)
+ def handle_intellisense(runtime, send_to, request, parent_locators) do
+ RuntimeServer.handle_intellisense(runtime.server_pid, send_to, request, parent_locators)
end
def read_file(runtime, path) do
RuntimeServer.read_file(runtime.server_pid, path)
end
- def start_smart_cell(runtime, kind, ref, attrs, base_locator) do
- RuntimeServer.start_smart_cell(runtime.server_pid, kind, ref, attrs, base_locator)
+ def start_smart_cell(runtime, kind, ref, attrs, parent_locators) do
+ RuntimeServer.start_smart_cell(runtime.server_pid, kind, ref, attrs, parent_locators)
end
- def set_smart_cell_base_locator(runtime, ref, base_locator) do
- RuntimeServer.set_smart_cell_base_locator(runtime.server_pid, ref, base_locator)
+ def set_smart_cell_parent_locators(runtime, ref, parent_locators) do
+ RuntimeServer.set_smart_cell_parent_locators(runtime.server_pid, ref, parent_locators)
end
def stop_smart_cell(runtime, ref) do
diff --git a/lib/livebook/runtime/elixir_standalone.ex b/lib/livebook/runtime/elixir_standalone.ex
index 0a5106c55..781a32d81 100644
--- a/lib/livebook/runtime/elixir_standalone.ex
+++ b/lib/livebook/runtime/elixir_standalone.ex
@@ -179,8 +179,8 @@ defimpl Livebook.Runtime, for: Livebook.Runtime.ElixirStandalone do
Livebook.Runtime.ElixirStandalone.new()
end
- def evaluate_code(runtime, code, locator, base_locator, opts \\ []) do
- RuntimeServer.evaluate_code(runtime.server_pid, code, locator, base_locator, opts)
+ def evaluate_code(runtime, code, locator, parent_locators, opts \\ []) do
+ RuntimeServer.evaluate_code(runtime.server_pid, code, locator, parent_locators, opts)
end
def forget_evaluation(runtime, locator) do
@@ -191,20 +191,20 @@ defimpl Livebook.Runtime, for: Livebook.Runtime.ElixirStandalone do
RuntimeServer.drop_container(runtime.server_pid, container_ref)
end
- def handle_intellisense(runtime, send_to, request, base_locator) do
- RuntimeServer.handle_intellisense(runtime.server_pid, send_to, request, base_locator)
+ def handle_intellisense(runtime, send_to, request, parent_locators) do
+ RuntimeServer.handle_intellisense(runtime.server_pid, send_to, request, parent_locators)
end
def read_file(runtime, path) do
RuntimeServer.read_file(runtime.server_pid, path)
end
- def start_smart_cell(runtime, kind, ref, attrs, base_locator) do
- RuntimeServer.start_smart_cell(runtime.server_pid, kind, ref, attrs, base_locator)
+ def start_smart_cell(runtime, kind, ref, attrs, parent_locators) do
+ RuntimeServer.start_smart_cell(runtime.server_pid, kind, ref, attrs, parent_locators)
end
- def set_smart_cell_base_locator(runtime, ref, base_locator) do
- RuntimeServer.set_smart_cell_base_locator(runtime.server_pid, ref, base_locator)
+ def set_smart_cell_parent_locators(runtime, ref, parent_locators) do
+ RuntimeServer.set_smart_cell_parent_locators(runtime.server_pid, ref, parent_locators)
end
def stop_smart_cell(runtime, ref) do
diff --git a/lib/livebook/runtime/embedded.ex b/lib/livebook/runtime/embedded.ex
index ffb21c283..2ed49af1a 100644
--- a/lib/livebook/runtime/embedded.ex
+++ b/lib/livebook/runtime/embedded.ex
@@ -76,8 +76,8 @@ defimpl Livebook.Runtime, for: Livebook.Runtime.Embedded do
Livebook.Runtime.Embedded.new()
end
- def evaluate_code(runtime, code, locator, base_locator, opts \\ []) do
- RuntimeServer.evaluate_code(runtime.server_pid, code, locator, base_locator, opts)
+ def evaluate_code(runtime, code, locator, parent_locators, opts \\ []) do
+ RuntimeServer.evaluate_code(runtime.server_pid, code, locator, parent_locators, opts)
end
def forget_evaluation(runtime, locator) do
@@ -88,20 +88,20 @@ defimpl Livebook.Runtime, for: Livebook.Runtime.Embedded do
RuntimeServer.drop_container(runtime.server_pid, container_ref)
end
- def handle_intellisense(runtime, send_to, request, base_locator) do
- RuntimeServer.handle_intellisense(runtime.server_pid, send_to, request, base_locator)
+ def handle_intellisense(runtime, send_to, request, parent_locators) do
+ RuntimeServer.handle_intellisense(runtime.server_pid, send_to, request, parent_locators)
end
def read_file(runtime, path) do
RuntimeServer.read_file(runtime.server_pid, path)
end
- def start_smart_cell(runtime, kind, ref, attrs, base_locator) do
- RuntimeServer.start_smart_cell(runtime.server_pid, kind, ref, attrs, base_locator)
+ def start_smart_cell(runtime, kind, ref, attrs, parent_locators) do
+ RuntimeServer.start_smart_cell(runtime.server_pid, kind, ref, attrs, parent_locators)
end
- def set_smart_cell_base_locator(runtime, ref, base_locator) do
- RuntimeServer.set_smart_cell_base_locator(runtime.server_pid, ref, base_locator)
+ def set_smart_cell_parent_locators(runtime, ref, parent_locators) do
+ RuntimeServer.set_smart_cell_parent_locators(runtime.server_pid, ref, parent_locators)
end
def stop_smart_cell(runtime, ref) do
diff --git a/lib/livebook/runtime/erl_dist.ex b/lib/livebook/runtime/erl_dist.ex
index 0ce06fb2a..d0f780c94 100644
--- a/lib/livebook/runtime/erl_dist.ex
+++ b/lib/livebook/runtime/erl_dist.ex
@@ -23,6 +23,7 @@ defmodule Livebook.Runtime.ErlDist do
[
Livebook.Runtime.Evaluator,
Livebook.Runtime.Evaluator.IOProxy,
+ Livebook.Runtime.Evaluator.Tracer,
Livebook.Runtime.Evaluator.ObjectTracker,
Livebook.Runtime.Evaluator.DefaultFormatter,
Livebook.Intellisense,
diff --git a/lib/livebook/runtime/erl_dist/runtime_server.ex b/lib/livebook/runtime/erl_dist/runtime_server.ex
index 23607c4a8..b354496f6 100644
--- a/lib/livebook/runtime/erl_dist/runtime_server.ex
+++ b/lib/livebook/runtime/erl_dist/runtime_server.ex
@@ -71,9 +71,15 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
See `Livebook.Runtime.Evaluator` for more details.
"""
- @spec evaluate_code(pid(), String.t(), Runtime.locator(), Runtime.locator(), keyword()) :: :ok
- def evaluate_code(pid, code, locator, base_locator, opts \\ []) do
- GenServer.cast(pid, {:evaluate_code, code, locator, base_locator, opts})
+ @spec evaluate_code(
+ pid(),
+ String.t(),
+ Runtime.locator(),
+ Runtime.parent_locators(),
+ keyword()
+ ) :: :ok
+ def evaluate_code(pid, code, locator, parent_locators, opts \\ []) do
+ GenServer.cast(pid, {:evaluate_code, code, locator, parent_locators, opts})
end
@doc """
@@ -109,11 +115,11 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
pid(),
pid(),
Runtime.intellisense_request(),
- Runtime.locator()
+ Runtime.Runtime.parent_locators()
) :: reference()
- def handle_intellisense(pid, send_to, request, base_locator) do
+ def handle_intellisense(pid, send_to, request, parent_locators) do
ref = make_ref()
- GenServer.cast(pid, {:handle_intellisense, send_to, ref, request, base_locator})
+ GenServer.cast(pid, {:handle_intellisense, send_to, ref, request, parent_locators})
ref
end
@@ -144,18 +150,22 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
String.t(),
Runtime.smart_cell_ref(),
Runtime.smart_cell_attrs(),
- Runtime.locator()
+ Runtime.Runtime.parent_locators()
) :: :ok
- def start_smart_cell(pid, kind, ref, attrs, base_locator) do
- GenServer.cast(pid, {:start_smart_cell, kind, ref, attrs, base_locator})
+ def start_smart_cell(pid, kind, ref, attrs, parent_locators) do
+ GenServer.cast(pid, {:start_smart_cell, kind, ref, attrs, parent_locators})
end
@doc """
- Updates the locator with smart cell context.
+ Updates the parent locator used by a smart cell as its context.
"""
- @spec set_smart_cell_base_locator(pid(), Runtime.smart_cell_ref(), Runtime.locator()) :: :ok
- def set_smart_cell_base_locator(pid, ref, base_locator) do
- GenServer.cast(pid, {:set_smart_cell_base_locator, ref, base_locator})
+ @spec set_smart_cell_parent_locators(
+ pid(),
+ Runtime.smart_cell_ref(),
+ Runtime.Runtime.parent_locators()
+ ) :: :ok
+ def set_smart_cell_parent_locators(pid, ref, parent_locators) do
+ GenServer.cast(pid, {:set_smart_cell_parent_locators, ref, parent_locators})
end
@doc """
@@ -332,25 +342,12 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
end
def handle_cast(
- {:evaluate_code, code, {container_ref, evaluation_ref} = locator, base_locator, opts},
+ {:evaluate_code, code, {container_ref, evaluation_ref} = locator, parent_locators, opts},
state
) do
state = ensure_evaluator(state, container_ref)
- base_evaluation_ref =
- case base_locator do
- {^container_ref, evaluation_ref} ->
- evaluation_ref
-
- {parent_container_ref, evaluation_ref} ->
- Evaluator.initialize_from(
- state.evaluators[container_ref],
- state.evaluators[parent_container_ref],
- evaluation_ref
- )
-
- nil
- end
+ parent_evaluation_refs = evaluation_refs_for_container(state, container_ref, parent_locators)
{smart_cell_ref, opts} = Keyword.pop(opts, :smart_cell_ref)
smart_cell_info = smart_cell_ref && state.smart_cells[smart_cell_ref]
@@ -374,7 +371,7 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
state.evaluators[container_ref],
code,
evaluation_ref,
- base_evaluation_ref,
+ parent_evaluation_refs,
opts
)
@@ -394,15 +391,31 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
{:noreply, state}
end
- def handle_cast({:handle_intellisense, send_to, ref, request, base_locator}, state) do
- {container_ref, evaluation_ref} = base_locator
- evaluator = state.evaluators[container_ref]
+ def handle_cast({:handle_intellisense, send_to, ref, request, parent_locators}, state) do
+ {container_ref, parent_evaluation_refs} =
+ case parent_locators do
+ [] ->
+ {nil, []}
+
+ [{container_ref, _} | _] ->
+ parent_evaluation_refs =
+ parent_locators
+ # If there is a parent evaluator we ignore it and use whatever
+ # initial context we currently have in the evaluator. We sync
+ # initial context only on evaluation, since it may be blocking
+ |> Enum.take_while(&(elem(&1, 0) == container_ref))
+ |> Enum.map(&elem(&1, 1))
+
+ {container_ref, parent_evaluation_refs}
+ end
+
+ evaluator = container_ref && state.evaluators[container_ref]
intellisense_context =
if evaluator == nil or elem(request, 0) in [:format] do
Evaluator.intellisense_context()
else
- Evaluator.intellisense_context(evaluator, evaluation_ref)
+ Evaluator.intellisense_context(evaluator, parent_evaluation_refs)
end
Task.Supervisor.start_child(state.task_supervisor, fn ->
@@ -413,7 +426,7 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
{:noreply, state}
end
- def handle_cast({:start_smart_cell, kind, ref, attrs, base_locator}, state) do
+ def handle_cast({:start_smart_cell, kind, ref, attrs, parent_locators}, state) do
definition = Enum.find(state.smart_cell_definitions, &(&1.kind == kind))
state =
@@ -440,7 +453,7 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
pid: pid,
monitor_ref: Process.monitor(pid),
scan_binding: scan_binding,
- base_locator: base_locator,
+ parent_locators: parent_locators,
scan_binding_pending: false,
scan_binding_monitor_ref: nil,
scan_eval_result: scan_eval_result
@@ -457,11 +470,11 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
{:noreply, state}
end
- def handle_cast({:set_smart_cell_base_locator, ref, base_locator}, state) do
+ def handle_cast({:set_smart_cell_parent_locators, ref, parent_locators}, state) do
state =
update_in(state.smart_cells[ref], fn
- %{base_locator: ^base_locator} = info -> info
- info -> scan_binding_async(ref, %{info | base_locator: base_locator}, state)
+ %{parent_locators: ^parent_locators} = info -> info
+ info -> scan_binding_async(ref, %{info | parent_locators: parent_locators}, state)
end)
{:noreply, state}
@@ -608,12 +621,28 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
send(myself, {:scan_binding_ack, ref})
end
- {container_ref, evaluation_ref} = info.base_locator
- evaluator = state.evaluators[container_ref]
+ {container_ref, parent_evaluation_refs} =
+ case info.parent_locators do
+ [] ->
+ {nil, []}
+
+ [{container_ref, _} | _] = parent_locators ->
+ parent_evaluation_refs =
+ evaluation_refs_for_container(state, container_ref, parent_locators)
+
+ {container_ref, parent_evaluation_refs}
+ end
+
+ evaluator = container_ref && state.evaluators[container_ref]
worker_pid =
if evaluator do
- Evaluator.peek_context(evaluator, evaluation_ref, &scan_and_ack.(&1.binding, &1.env))
+ Evaluator.peek_context(
+ evaluator,
+ parent_evaluation_refs,
+ &scan_and_ack.(&1.binding, &1.env)
+ )
+
evaluator.pid
else
{:ok, pid} =
@@ -631,6 +660,26 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
%{info | scan_binding_pending: false, scan_binding_monitor_ref: monitor_ref}
end
+ defp evaluation_refs_for_container(state, container_ref, locators) do
+ case Enum.split_while(locators, &(elem(&1, 0) == container_ref)) do
+ {locators, []} ->
+ Enum.map(locators, &elem(&1, 1))
+
+ {locators, [{source_container_ref, _} | _] = source_locators} ->
+ source_evaluation_refs = Enum.map(source_locators, &elem(&1, 1))
+
+ evaluator = state.evaluators[container_ref]
+ source_evaluator = state.evaluators[source_container_ref]
+
+ if evaluator && source_evaluator do
+ # Synchronize initial state in the child evaluator
+ Evaluator.initialize_from(evaluator, source_evaluator, source_evaluation_refs)
+ end
+
+ Enum.map(locators, &elem(&1, 1))
+ end
+ end
+
defp finish_scan_binding(ref, state) do
update_in(state.smart_cells[ref], fn info ->
Process.demonitor(info.scan_binding_monitor_ref, [:flush])
@@ -646,12 +695,12 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do
defp scan_binding_after_evaluation(state, locator) do
update_in(state.smart_cells, fn smart_cells ->
- Map.new(smart_cells, fn
- {ref, %{base_locator: ^locator} = info} ->
+ Map.new(smart_cells, fn {ref, info} ->
+ if locator in info.parent_locators do
{ref, scan_binding_async(ref, info, state)}
-
- other ->
- other
+ else
+ {ref, info}
+ end
end)
end)
end
diff --git a/lib/livebook/runtime/evaluator.ex b/lib/livebook/runtime/evaluator.ex
index 5635c9f97..fe0c30fd5 100644
--- a/lib/livebook/runtime/evaluator.ex
+++ b/lib/livebook/runtime/evaluator.ex
@@ -56,10 +56,12 @@ defmodule Livebook.Runtime.Evaluator do
{:ok, result :: any()}
| {:error, Exception.kind(), error :: any(), Exception.stacktrace()}
- # We store evaluation envs in the process dictionary, so that we
- # can build intellisense context without asking the evaluator
- @env_key :evaluation_env
- @initial_env_key :initial_env
+ # We store some information in the process dictionary for non-blocking
+ # access from other evaluators. In particular we store context metadata,
+ # such as envs, this way we can build intellisense context without
+ # asking the evaluator. We don't store binding though, because that
+ # would take too much memory
+ @evaluator_info_key :evaluator_info
@doc """
Starts an evaluator.
@@ -117,10 +119,10 @@ defmodule Livebook.Runtime.Evaluator do
Any exceptions are captured and transformed into an error
result.
- The resulting contxt (binding and env) is stored under `ref`.
- Any subsequent calls may specify `base_ref` pointing to a
- previous evaluation, in which case the corresponding context
- is used as the entry point for evaluation.
+ The resulting context (binding and env) is stored under `ref`. Any
+ subsequent calls may specify `parent_refs` pointing to a sequence
+ of previous evaluations, in which case the corresponding context is
+ used as the entry point for evaluation.
The evaluation result is transformed with the configured
formatter send to the configured client (see `start_link/1`).
@@ -134,37 +136,29 @@ defmodule Livebook.Runtime.Evaluator do
finished. The function receives `t:evaluation_result/0`
as an argument
"""
- @spec evaluate_code(t(), String.t(), ref(), ref() | nil, keyword()) :: :ok
- def evaluate_code(evaluator, code, ref, base_ref \\ nil, opts \\ []) when ref != nil do
- cast(evaluator, {:evaluate_code, code, ref, base_ref, opts})
+ @spec evaluate_code(t(), String.t(), ref(), list(ref()), keyword()) :: :ok
+ def evaluate_code(evaluator, code, ref, parent_refs, opts \\ []) do
+ cast(evaluator, {:evaluate_code, code, ref, parent_refs, opts})
end
@doc """
Fetches the evaluation context (binding and env) for the given
- evaluation reference.
-
- ## Options
-
- * `:cached_id` - id of context that the sender may already have,
- if it matches the fetched context, `{:error, :not_modified}`
- is returned instead
+ evaluation sequence.
"""
- @spec fetch_evaluation_context(t(), ref(), keyword()) ::
- {:ok, context()} | {:error, :not_modified}
- def fetch_evaluation_context(evaluator, ref, opts \\ []) do
- cached_id = opts[:cached_id]
- call(evaluator, {:fetch_evaluation_context, ref, cached_id})
+ @spec get_evaluation_context(t(), list(ref())) :: context()
+ def get_evaluation_context(evaluator, parent_refs) do
+ call(evaluator, {:get_evaluation_context, parent_refs})
end
@doc """
- Fetches an evaluation context from `source_evaluator` and configures
- it as the initial context for `evaluator`.
+ Fetches an aggregated evaluation context from `source_evaluator`
+ and caches it as the initial context for `evaluator`.
The process dictionary is also copied to match `source_evaluator`.
"""
@spec initialize_from(t(), t(), ref()) :: :ok
- def initialize_from(evaluator, source_evaluator, source_evaluation_ref) do
- call(evaluator, {:initialize_from, source_evaluator, source_evaluation_ref})
+ def initialize_from(evaluator, source_evaluator, source_parent_refs) do
+ call(evaluator, {:initialize_from, source_evaluator, source_parent_refs})
end
@doc """
@@ -190,15 +184,22 @@ defmodule Livebook.Runtime.Evaluator do
@doc """
Builds intellisense context from the given evaluation.
"""
- @spec intellisense_context(t(), ref()) :: Livebook.Intellisense.intellisense_context()
- def intellisense_context(evaluator, ref) do
+ @spec intellisense_context(t(), list(ref())) :: Livebook.Intellisense.intellisense_context()
+ def intellisense_context(evaluator, parent_refs) do
{:dictionary, dictionary} = Process.info(evaluator.pid, :dictionary)
- env =
- find_in_dictionary(dictionary, {@env_key, ref}) ||
- find_in_dictionary(dictionary, @initial_env_key)
+ evaluator_info = find_in_dictionary(dictionary, @evaluator_info_key)
+ %{initial_context: {_id, initial_env}} = evaluator_info
- map_binding = fn fun -> map_binding(evaluator, ref, fun) end
+ env =
+ List.foldr(parent_refs, initial_env, fn ref, prev_env ->
+ case evaluator_info.contexts do
+ %{^ref => {_id, env}} -> merge_env(prev_env, env)
+ _ -> prev_env
+ end
+ end)
+
+ map_binding = fn fun -> map_binding(evaluator, parent_refs, fun) end
%{env: env, map_binding: map_binding}
end
@@ -211,8 +212,8 @@ defmodule Livebook.Runtime.Evaluator do
end
# Applies the given function to evaluation binding
- defp map_binding(evaluator, ref, fun) do
- call(evaluator, {:map_binding, ref, fun})
+ defp map_binding(evaluator, parent_refs, fun) do
+ call(evaluator, {:map_binding, parent_refs, fun})
end
@doc """
@@ -221,9 +222,9 @@ defmodule Livebook.Runtime.Evaluator do
Ths function runs within the evaluator process, so that no data
is copied between processes, unless explicitly sent.
"""
- @spec peek_context(t(), ref(), (context() -> any())) :: :ok
- def peek_context(evaluator, ref, fun) do
- cast(evaluator, {:peek_context, ref, fun})
+ @spec peek_context(t(), list(ref()), (context() -> any())) :: :ok
+ def peek_context(evaluator, parent_refs, fun) do
+ cast(evaluator, {:peek_context, parent_refs, fun})
end
defp cast(evaluator, message) do
@@ -271,7 +272,13 @@ defmodule Livebook.Runtime.Evaluator do
evaluator = %{pid: self(), ref: evaluator_ref}
context = initial_context()
- Process.put(@initial_env_key, context.env)
+
+ Process.put(@evaluator_info_key, %{
+ initial_context: {context.id, context.env},
+ contexts: %{}
+ })
+
+ ignored_pdict_keys = Process.get_keys() |> MapSet.new()
state = %{
evaluator_ref: evaluator_ref,
@@ -281,7 +288,9 @@ defmodule Livebook.Runtime.Evaluator do
runtime_broadcast_to: runtime_broadcast_to,
object_tracker: object_tracker,
contexts: %{},
- initial_context: context
+ initial_context: context,
+ initial_context_version: nil,
+ ignored_pdict_keys: ignored_pdict_keys
}
:proc_lib.init_ack(evaluator)
@@ -304,34 +313,53 @@ defmodule Livebook.Runtime.Evaluator do
defp initial_context() do
env = Code.env_for_eval([])
- %{binding: [], env: env, id: random_id()}
+ env = Macro.Env.prepend_tracer(env, Evaluator.Tracer)
+ %{id: random_id(), binding: [], env: env, pdict: %{}}
end
- defp handle_cast({:evaluate_code, code, ref, base_ref, opts}, state) do
+ defp handle_cast({:evaluate_code, code, ref, parent_refs, opts}, state) do
Evaluator.ObjectTracker.remove_reference_sync(state.object_tracker, {self(), ref})
- context = get_context(state, base_ref)
+ context = get_context(state, parent_refs)
file = Keyword.get(opts, :file, "nofile")
context = put_in(context.env.file, file)
- start_time = System.monotonic_time()
Evaluator.IOProxy.configure(state.io_proxy, ref, file)
- {result_context, result, code_error} =
- case eval(code, context.binding, context.env) do
+ set_pdict(context, state.ignored_pdict_keys)
+
+ start_time = System.monotonic_time()
+ eval_result = eval(code, context.binding, context.env)
+ evaluation_time_ms = time_diff_ms(start_time)
+
+ {result_context, result, code_error, identifiers_used, identifiers_defined} =
+ case eval_result do
{:ok, value, binding, env} ->
- binding = reorder_binding(binding, context.binding)
- result_context = %{binding: binding, env: env, id: random_id()}
+ tracer_info = Evaluator.IOProxy.get_tracer_info(state.io_proxy)
+ context_id = random_id()
+
+ result_context = %{
+ id: context_id,
+ binding: binding,
+ env: prune_env(env, tracer_info),
+ pdict: current_pdict(state)
+ }
+
+ {identifiers_used, identifiers_defined} =
+ identifier_dependencies(result_context, tracer_info, context)
+
result = {:ok, value}
- {result_context, result, nil}
+ {result_context, result, nil, identifiers_used, identifiers_defined}
{:error, kind, error, stacktrace, code_error} ->
result = {:error, kind, error, stacktrace}
- {context, result, code_error}
+ identifiers_used = :unknown
+ identifiers_defined = %{}
+ # Empty context
+ result_context = initial_context()
+ {result_context, result, code_error, identifiers_used, identifiers_defined}
end
- evaluation_time_ms = get_execution_time_delta(start_time)
-
state = put_context(state, ref, result_context)
Evaluator.IOProxy.flush(state.io_proxy)
@@ -342,7 +370,9 @@ defmodule Livebook.Runtime.Evaluator do
metadata = %{
evaluation_time_ms: evaluation_time_ms,
memory_usage: memory(),
- code_error: code_error
+ code_error: code_error,
+ identifiers_used: identifiers_used,
+ identifiers_defined: identifiers_defined
}
send(state.send_to, {:runtime_evaluation_response, ref, output, metadata})
@@ -363,71 +393,153 @@ defmodule Livebook.Runtime.Evaluator do
{:noreply, state}
end
- defp handle_cast({:peek_context, ref, fun}, state) do
- context = get_context(state, ref)
+ defp handle_cast({:peek_context, parent_refs, fun}, state) do
+ context = get_context(state, parent_refs)
fun.(context)
{:noreply, state}
end
- defp handle_call({:fetch_evaluation_context, ref, cached_id}, _from, state) do
- context = get_context(state, ref)
-
- reply =
- if context.id == cached_id do
- {:error, :not_modified}
- else
- {:ok, context}
- end
-
- {:reply, reply, state}
+ defp handle_call({:get_evaluation_context, parent_refs}, _from, state) do
+ context = get_context(state, parent_refs)
+ {:reply, context, state}
end
- defp handle_call({:initialize_from, source_evaluator, source_evaluation_ref}, _from, state) do
+ defp handle_call({:initialize_from, source_evaluator, source_parent_refs}, _from, state) do
+ {:dictionary, dictionary} = Process.info(source_evaluator.pid, :dictionary)
+
+ evaluator_info = find_in_dictionary(dictionary, @evaluator_info_key)
+
+ version =
+ source_parent_refs
+ |> Enum.map(fn ref ->
+ with {id, _env} <- evaluator_info.contexts[ref], do: id
+ end)
+ |> :erlang.md5()
+
state =
- case Evaluator.fetch_evaluation_context(
- source_evaluator,
- source_evaluation_ref,
- cached_id: state.initial_context.id
- ) do
- {:ok, context} ->
- # If the context changed, mirror the process dictionary again
- copy_process_dictionary_from(source_evaluator)
+ if version == state.initial_context_version do
+ state
+ else
+ context = Evaluator.get_evaluation_context(source_evaluator, source_parent_refs)
- Process.put(@initial_env_key, context.env)
- put_in(state.initial_context, context)
+ update_evaluator_info(fn info ->
+ put_in(info.initial_context, {context.id, context.env})
+ end)
- {:error, :not_modified} ->
- state
+ %{state | initial_context: context, initial_context_version: version}
end
{:reply, :ok, state}
end
- defp handle_call({:map_binding, ref, fun}, _from, state) do
- context = get_context(state, ref)
+ defp handle_call({:map_binding, parent_refs, fun}, _from, state) do
+ context = get_context(state, parent_refs)
result = fun.(context.binding)
{:reply, result, state}
end
defp put_context(state, ref, context) do
- Process.put({@env_key, ref}, context.env)
+ update_evaluator_info(fn info ->
+ put_in(info.contexts[ref], {context.id, context.env})
+ end)
+
put_in(state.contexts[ref], context)
end
defp delete_context(state, ref) do
- Process.delete({@env_key, ref})
+ update_evaluator_info(fn info ->
+ {_, info} = pop_in(info.contexts[ref])
+ info
+ end)
+
{_, state} = pop_in(state.contexts[ref])
state
end
- defp get_context(state, ref) do
- Map.get_lazy(state.contexts, ref, fn -> state.initial_context end)
+ defp update_evaluator_info(fun) do
+ info = Process.get(@evaluator_info_key)
+ Process.put(@evaluator_info_key, fun.(info))
end
+ defp get_context(state, parent_refs) do
+ List.foldr(parent_refs, state.initial_context, fn ref, prev_context ->
+ if context = state.contexts[ref] do
+ merge_context(prev_context, context)
+ else
+ prev_context
+ end
+ end)
+ end
+
+ defp set_pdict(context, ignored_pdict_keys) do
+ for key <- Process.get_keys(),
+ key not in ignored_pdict_keys,
+ not Map.has_key?(context.pdict, key) do
+ Process.delete(key)
+ end
+
+ for {key, value} <- context.pdict do
+ Process.put(key, value)
+ end
+ end
+
+ defp current_pdict(state) do
+ for {key, value} <- Process.get(),
+ key not in state.ignored_pdict_keys,
+ do: {key, value},
+ into: %{}
+ end
+
+ defp prune_env(env, tracer_info) do
+ env
+ |> Map.replace!(:aliases, Map.to_list(tracer_info.aliases_defined))
+ |> Map.replace!(:requires, MapSet.to_list(tracer_info.requires_defined))
+ end
+
+ defp merge_context(prev_context, context) do
+ binding = merge_binding(prev_context.binding, context.binding)
+ env = merge_env(prev_context.env, context.env)
+ pdict = context.pdict
+ %{id: random_id(), binding: binding, env: env, pdict: pdict}
+ end
+
+ defp merge_binding(prev_binding, binding) do
+ binding_map = Map.new(binding)
+
+ kept_binding =
+ Enum.reject(prev_binding, fn {var, _value} ->
+ Map.has_key?(binding_map, var)
+ end)
+
+ binding ++ kept_binding
+ end
+
+ defp merge_env(prev_env, env) do
+ env
+ |> Map.update!(:versioned_vars, fn versioned_vars ->
+ Enum.uniq(Map.keys(prev_env.versioned_vars) ++ Map.keys(versioned_vars))
+ |> Enum.with_index()
+ |> Map.new()
+ end)
+ |> Map.update!(:aliases, &Keyword.merge(prev_env.aliases, &1))
+ |> Map.update!(:requires, &:ordsets.union(prev_env.requires, &1))
+ |> Map.replace!(:context_modules, [])
+ end
+
+ @compile {:no_warn_undefined, {Code, :eval_quoted_with_env, 4}}
+
defp eval(code, binding, env) do
try do
quoted = Code.string_to_quoted!(code, file: env.file)
- {value, binding, env} = Code.eval_quoted_with_env(quoted, binding, env)
+
+ # TODO: remove the else branch when we require Elixir v1.14.2
+ {value, binding, env} =
+ if function_exported?(Code, :eval_quoted_with_env, 4) do
+ Code.eval_quoted_with_env(quoted, binding, env, prune_binding: true)
+ else
+ Code.eval_quoted_with_env(quoted, binding, env)
+ end
+
{:ok, value, binding, env}
catch
kind, error ->
@@ -449,22 +561,129 @@ defmodule Livebook.Runtime.Evaluator do
defp code_error?(%CompileError{}), do: true
defp code_error?(_error), do: false
- defp reorder_binding(binding, prev_binding) do
- # We keep the order of existing binding entries and move the new
- # ones to the beginning
+ defp identifier_dependencies(context, tracer_info, prev_context) do
+ identifiers_used = MapSet.new()
+ identifiers_defined = %{}
- binding_map = Map.new(binding)
+ # Variables
- unchanged_binding =
- Enum.filter(prev_binding, fn {key, prev_val} ->
- val = binding_map[key]
- :erts_debug.same(val, prev_val)
- end)
+ identifiers_used =
+ for var <- vars_used(context, tracer_info, prev_context),
+ do: {:variable, var},
+ into: identifiers_used
- unchanged_binding
- |> Enum.reduce(binding_map, fn {key, _}, acc -> Map.delete(acc, key) end)
- |> Map.to_list()
- |> Kernel.++(unchanged_binding)
+ identifiers_used =
+ for var <- tracer_info.undefined_vars,
+ do: {:variable, var},
+ into: identifiers_used
+
+ identifiers_defined =
+ for var <- vars_defined(context, prev_context),
+ do: {{:variable, var}, context.id},
+ into: identifiers_defined
+
+ # Modules
+
+ identifiers_used =
+ for module <- tracer_info.modules_used,
+ do: {:module, module},
+ into: identifiers_used
+
+ identifiers_defined =
+ for {module, {version, _vars}} <- tracer_info.modules_defined,
+ do: {{:module, module}, version},
+ into: identifiers_defined
+
+ # Aliases
+
+ identifiers_used =
+ for alias <- tracer_info.aliases_used,
+ do: {:alias, alias},
+ into: identifiers_used
+
+ identifiers_defined =
+ for {as, alias} <- tracer_info.aliases_defined,
+ do: {{:alias, as}, alias},
+ into: identifiers_defined
+
+ # Requires
+
+ identifiers_used =
+ for module <- tracer_info.requires_used,
+ do: {:require, module},
+ into: identifiers_used
+
+ identifiers_defined =
+ for module <- tracer_info.requires_defined,
+ do: {{:require, module}, :ok},
+ into: identifiers_defined
+
+ # Imports
+
+ identifiers_used =
+ if tracer_info.imports_used? or tracer_info.imports_defined? do
+ # Imports are not always incremental, due to :except, so if
+ # we define imports, we also implicitly rely on prior imports
+ MapSet.put(identifiers_used, :imports)
+ else
+ identifiers_used
+ end
+
+ identifiers_defined =
+ if tracer_info.imports_defined? do
+ version = {:erlang.phash2(context.env.functions), :erlang.phash2(context.env.macros)}
+ put_in(identifiers_defined[:imports], version)
+ else
+ identifiers_defined
+ end
+
+ # Process dictionary
+
+ # Every evaluation depends on the pdict
+ identifiers_used = MapSet.put(identifiers_used, :pdict)
+
+ identifiers_defined =
+ if context.pdict == prev_context.pdict do
+ identifiers_defined
+ else
+ version = :erlang.phash2(context.pdict)
+ put_in(identifiers_defined[:pdict], version)
+ end
+
+ {MapSet.to_list(identifiers_used), identifiers_defined}
+ end
+
+ defp vars_used(context, tracer_info, prev_context) do
+ prev_vars =
+ for {var, _version} <- prev_context.env.versioned_vars,
+ into: MapSet.new(),
+ do: var
+
+ outer_used_vars =
+ for {var, _version} <- context.env.versioned_vars,
+ into: MapSet.new(),
+ do: var
+
+ # Note that :prune_binding removes variables used by modules
+ # (unless used outside), so we get those from the tracer
+ module_used_vars =
+ for {_module, {_version, vars}} <- tracer_info.modules_defined,
+ var <- vars,
+ into: MapSet.new(),
+ do: var
+
+ # We take an intersection with previous vars, so we ignore variables
+ # that we know are newly defined
+ MapSet.intersection(prev_vars, MapSet.union(outer_used_vars, module_used_vars))
+ end
+
+ defp vars_defined(context, prev_context) do
+ prev_num_vars = map_size(prev_context.env.versioned_vars)
+
+ for {var, version} <- context.env.versioned_vars,
+ version >= prev_num_vars,
+ into: MapSet.new(),
+ do: var
end
# Adapted from https://github.com/elixir-lang/elixir/blob/1c1654c88adfdbef38ff07fc30f6fbd34a542c07/lib/iex/lib/iex/evaluator.ex#L355-L372
@@ -506,20 +725,7 @@ defmodule Livebook.Runtime.Evaluator do
:crypto.strong_rand_bytes(20) |> Base.encode32(case: :lower)
end
- defp copy_process_dictionary_from(source_evaluator) do
- {:dictionary, dictionary} = Process.info(source_evaluator.pid, :dictionary)
-
- for {key, value} <- dictionary, not internal_dictionary_key?(key) do
- Process.put(key, value)
- end
- end
-
- defp internal_dictionary_key?("$" <> _), do: true
- defp internal_dictionary_key?({@env_key, _ref}), do: true
- defp internal_dictionary_key?(@initial_env_key), do: true
- defp internal_dictionary_key?(_), do: false
-
- defp get_execution_time_delta(started_at) do
+ defp time_diff_ms(started_at) do
System.monotonic_time()
|> Kernel.-(started_at)
|> System.convert_time_unit(:native, :millisecond)
diff --git a/lib/livebook/runtime/evaluator/io_proxy.ex b/lib/livebook/runtime/evaluator/io_proxy.ex
index bdc3e1556..ac549568c 100644
--- a/lib/livebook/runtime/evaluator/io_proxy.ex
+++ b/lib/livebook/runtime/evaluator/io_proxy.ex
@@ -61,11 +61,19 @@ defmodule Livebook.Runtime.Evaluator.IOProxy do
end
@doc """
- Returns the accumulated widget pids and clears the accumulator.
+ Updates tracer info.
"""
- @spec flush_widgets(pid()) :: MapSet.t(pid())
- def flush_widgets(pid) do
- GenServer.call(pid, :flush_widgets)
+ @spec tracer_updates(pid(), list()) :: :ok
+ def tracer_updates(pid, updates) do
+ GenServer.cast(pid, {:tracer_updates, updates})
+ end
+
+ @doc """
+ Returns the accumulated tracer info.
+ """
+ @spec get_tracer_info(pid()) :: %Evaluator.Tracer{}
+ def get_tracer_info(pid) do
+ GenServer.call(pid, :get_tracer_info)
end
@impl true
@@ -81,24 +89,34 @@ defmodule Livebook.Runtime.Evaluator.IOProxy do
evaluator: evaluator,
send_to: send_to,
runtime_broadcast_to: runtime_broadcast_to,
- object_tracker: object_tracker
+ object_tracker: object_tracker,
+ tracer_info: %Evaluator.Tracer{}
}}
end
@impl true
def handle_cast({:configure, ref, file}, state) do
- {:noreply, %{state | ref: ref, file: file, token_count: 0}}
+ {:noreply, %{state | ref: ref, file: file, token_count: 0, tracer_info: %Evaluator.Tracer{}}}
end
def handle_cast(:clear_input_cache, state) do
{:noreply, %{state | input_cache: %{}}}
end
+ def handle_cast({:tracer_updates, updates}, state) do
+ state = update_in(state.tracer_info, &Evaluator.Tracer.apply_updates(&1, updates))
+ {:noreply, state}
+ end
+
@impl true
def handle_call(:flush, _from, state) do
{:reply, :ok, flush_buffer(state)}
end
+ def handle_call(:get_tracer_info, _from, state) do
+ {:reply, state.tracer_info, state}
+ end
+
@impl true
def handle_info({:io_request, from, reply_as, req}, state) do
{reply, state} = io_request(req, state)
diff --git a/lib/livebook/runtime/evaluator/tracer.ex b/lib/livebook/runtime/evaluator/tracer.ex
new file mode 100644
index 000000000..37eacfce5
--- /dev/null
+++ b/lib/livebook/runtime/evaluator/tracer.ex
@@ -0,0 +1,132 @@
+defmodule Livebook.Runtime.Evaluator.Tracer do
+ @moduledoc false
+
+ # Compilation tracer used by the evaluator.
+ #
+ # Events are pre-processed and sent to the group leader, where the
+ # tracer state is accumulated. After evaluation the evaluator reads
+ # the accumulated state.
+
+ alias Livebook.Runtime.Evaluator
+
+ defstruct modules_used: MapSet.new(),
+ modules_defined: %{},
+ aliases_used: MapSet.new(),
+ aliases_defined: %{},
+ requires_used: MapSet.new(),
+ requires_defined: MapSet.new(),
+ imports_used?: false,
+ imports_defined?: false,
+ undefined_vars: MapSet.new()
+
+ @doc false
+ def trace(event, env) do
+ case to_updates(event, env) do
+ [] ->
+ :ok
+
+ updates ->
+ io_proxy = Process.group_leader()
+ Evaluator.IOProxy.tracer_updates(io_proxy, updates)
+ end
+
+ :ok
+ end
+
+ defp to_updates(event, env) do
+ # Note that import/require/alias/defmodule don't trigger `:alias_reference`
+ # for the used alias, so we add it explicitly
+
+ case event do
+ {:import, _meta, module, _opts} ->
+ if(env.module, do: [], else: [:import_defined]) ++
+ [{:module_used, module}, {:alias_used, module}]
+
+ {:imported_function, meta, module, name, _arity} ->
+ var? = Keyword.has_key?(meta, :if_undefined)
+ [{:module_used, module}, {:import_used, name, var?}]
+
+ {:imported_macro, meta, module, name, _arity} ->
+ var? = Keyword.has_key?(meta, :if_undefined)
+ [{:module_used, module}, {:import_used, name, var?}]
+
+ {:alias, _meta, alias, as, _opts} ->
+ if(env.module, do: [], else: [{:alias_defined, as, alias}]) ++
+ [{:alias_used, alias}]
+
+ {:alias_expansion, _meta, as, _alias} ->
+ [{:alias_used, as}]
+
+ {:alias_reference, _meta, alias} ->
+ [{:alias_used, alias}]
+
+ {:require, _meta, module, _opts} ->
+ if(env.module, do: [], else: [{:require_defined, module}]) ++
+ [{:module_used, module}, {:alias_used, module}]
+
+ {:struct_expansion, _meta, module, _keys} ->
+ [{:module_used, module}]
+
+ {:remote_function, _meta, module, _name, _arity} ->
+ [{:module_used, module}]
+
+ {:remote_macro, _meta, module, _name, _arity} ->
+ [{:module_used, module}, {:require_used, module}]
+
+ {:on_module, bytecode, _ignore} ->
+ module = env.module
+ version = :erlang.md5(bytecode)
+ vars = Map.keys(env.versioned_vars)
+ [{:module_defined, module, version, vars}, {:alias_used, module}]
+
+ _ ->
+ []
+ end
+ end
+
+ @doc """
+ Applies updates to the tracer state.
+ """
+ @spec apply_updates(%__MODULE__{}, list()) :: %__MODULE__{}
+ def apply_updates(info, updates) do
+ Enum.reduce(updates, info, &apply_update(&2, &1))
+ end
+
+ defp apply_update(info, {:module_used, module}) do
+ update_in(info.modules_used, &MapSet.put(&1, module))
+ end
+
+ defp apply_update(info, {:module_defined, module, version, vars}) do
+ put_in(info.modules_defined[module], {version, vars})
+ end
+
+ defp apply_update(info, {:alias_used, alias}) do
+ update_in(info.aliases_used, &MapSet.put(&1, alias))
+ end
+
+ defp apply_update(info, {:alias_defined, as, alias}) do
+ put_in(info.aliases_defined[as], alias)
+ end
+
+ defp apply_update(info, {:require_used, module}) do
+ update_in(info.requires_used, &MapSet.put(&1, module))
+ end
+
+ defp apply_update(info, {:require_defined, module}) do
+ update_in(info.requires_defined, &MapSet.put(&1, module))
+ end
+
+ defp apply_update(info, {:import_used, name, var?}) do
+ info = put_in(info.imports_used?, true)
+
+ if var? do
+ update_in(info.undefined_vars, &MapSet.put(&1, {name, nil}))
+ else
+ info
+ end
+ end
+
+ defp apply_update(info, :import_defined) do
+ put_in(info.imports_defined?, true)
+ end
+end
diff --git a/lib/livebook/session.ex b/lib/livebook/session.ex
index 5e2d074c2..cb9d52510 100644
--- a/lib/livebook/session.ex
+++ b/lib/livebook/session.ex
@@ -29,7 +29,7 @@ defmodule Livebook.Session do
# the evaluation context from the parent section, the last context
# needs to be copied from the main flow evaluator to the branching
# section evaluator. The latter synchronously asks the former for
- # that context using `Livebook.Runtime.Evaluator.fetch_evaluation_context/3`.
+ # that context using `Livebook.Runtime.Evaluator.get_evaluation_context/3`.
# Consequently, in order to evaluate the first cell in a branching
# section, the main flow needs to be free of work, otherwise we wait.
# This assumptions are mirrored in by `Livebook.Session.Data` when
@@ -1040,7 +1040,7 @@ defmodule Livebook.Session do
def handle_info({:runtime_evaluation_input, cell_id, reply_to, input_id}, state) do
{reply, state} =
with {:ok, cell, _section} <- Notebook.fetch_cell_and_section(state.data.notebook, cell_id),
- {:ok, value} <- Map.fetch(state.data.input_values, input_id) do
+ {:ok, value} <- Data.fetch_input_value_for_cell(state.data, input_id, cell_id) do
state = handle_operation(state, {:bind_input, @client_id, cell.id, input_id})
{{:ok, value}, state}
else
@@ -1512,27 +1512,26 @@ defmodule Livebook.Session do
state
end
- defp handle_action(state, {:start_smart_cell, cell, section}) do
+ defp handle_action(state, {:start_smart_cell, cell, _section}) do
if Runtime.connected?(state.data.runtime) do
- base_locator = find_base_locator(state.data, cell, section, existing: true)
- Runtime.start_smart_cell(state.data.runtime, cell.kind, cell.id, cell.attrs, base_locator)
+ parent_locators = parent_locators_for_cell(state.data, cell)
+
+ Runtime.start_smart_cell(
+ state.data.runtime,
+ cell.kind,
+ cell.id,
+ cell.attrs,
+ parent_locators
+ )
end
state
end
- defp handle_action(state, {:set_smart_cell_base, cell, section, parent}) do
+ defp handle_action(state, {:set_smart_cell_parents, cell, _section, parents}) do
if Runtime.connected?(state.data.runtime) do
- base_locator =
- case parent do
- nil ->
- {container_ref_for_section(section), nil}
-
- {parent_cell, parent_section} ->
- {container_ref_for_section(parent_section), parent_cell.id}
- end
-
- Runtime.set_smart_cell_base_locator(state.data.runtime, cell.id, base_locator)
+ parent_locators = evaluation_parents_to_locators(parents)
+ Runtime.set_smart_cell_parent_locators(state.data.runtime, cell.id, parent_locators)
end
state
@@ -1566,8 +1565,8 @@ defmodule Livebook.Session do
opts = [file: file, smart_cell_ref: smart_cell_ref]
locator = {container_ref_for_section(section), cell.id}
- base_locator = find_base_locator(state.data, cell, section)
- Runtime.evaluate_code(state.data.runtime, cell.source, locator, base_locator, opts)
+ parent_locators = parent_locators_for_cell(state.data, cell)
+ Runtime.evaluate_code(state.data.runtime, cell.source, locator, parent_locators, opts)
evaluation_digest = :erlang.md5(cell.source)
handle_operation(state, {:evaluation_started, @client_id, cell.id, evaluation_digest})
@@ -1799,34 +1798,21 @@ defmodule Livebook.Session do
end
@doc """
- Finds evaluation locator that the given cell depends on.
+ Returns locators of evaluation parents for the given cell.
- By default looks up the direct evaluation parent.
-
- ## Options
-
- * `:existing` - considers only cells that have been evaluated
- as evaluation parents. Defaults to `false`
+ Considers only cells that have already been evaluated.
"""
- @spec find_base_locator(Data.t(), Cell.t(), Section.t(), keyword()) :: Runtime.locator()
- def find_base_locator(data, cell, section, opts \\ []) do
- parent_filter =
- if opts[:existing] do
- fn cell ->
- info = data.cell_infos[cell.id]
- Cell.evaluable?(cell) and info.eval.validity in [:evaluated, :stale]
- end
- else
- &Cell.evaluable?/1
- end
+ @spec parent_locators_for_cell(Data.t(), Cell.t()) :: Runtime.parent_locators()
+ def parent_locators_for_cell(data, cell) do
+ data
+ |> Data.cell_evaluation_parents(cell)
+ |> evaluation_parents_to_locators()
+ end
- default = {container_ref_for_section(section), nil}
-
- data.notebook
- |> Notebook.parent_cells_with_section(cell.id)
- |> Enum.find_value(default, fn {cell, section} ->
- parent_filter.(cell) && {container_ref_for_section(section), cell.id}
- end)
+ defp evaluation_parents_to_locators(parents) do
+ for {cell, section} <- parents do
+ {container_ref_for_section(section), cell.id}
+ end
end
defp container_ref_for_section(%{parent_id: nil}), do: @main_container_ref
diff --git a/lib/livebook/session/data.ex b/lib/livebook/session/data.ex
index 17e35c7fb..0b5197d7d 100644
--- a/lib/livebook/session/data.ex
+++ b/lib/livebook/session/data.ex
@@ -57,7 +57,7 @@ defmodule Livebook.Session.Data do
@type section_info :: %{
evaluating_cell_id: Cell.id(),
- evaluation_queue: list(Cell.id())
+ evaluation_queue: MapSet.t(Cell.id())
}
@type cell_info :: markdown_cell_info() | code_cell_info() | smart_cell_info()
@@ -96,7 +96,10 @@ defmodule Livebook.Session.Data do
evaluation_number: non_neg_integer(),
outputs_batch_number: non_neg_integer(),
bound_to_input_ids: MapSet.t(input_id()),
- bound_input_readings: input_reading()
+ new_bound_to_input_ids: MapSet.t(input_id()),
+ identifiers_used: list(identifier :: term()) | :unknown,
+ identifiers_defined: %{(identifier :: term()) => version :: term()},
+ data: t()
}
@type cell_bin_entry :: %{
@@ -125,22 +128,10 @@ defmodule Livebook.Session.Data do
@type secret :: %{name: String.t(), value: String.t()}
# Snapshot holds information about the cell evaluation dependencies,
- # for example what is the previous cell, the number of times the
- # cell was evaluated, the list of available inputs, etc. Whenever
- # the snapshot changes, it implies a new evaluation context, which
- # basically means the cell got stale.
- #
- # The snapshot comprises of two actual snapshots:
- #
- # * `deps_snapshot` - everything related to parent cells and
- # their evaluations. This is recorded once the cell starts
- # evaluating
- #
- # * `bound_inputs_snapshot` - snapshot of the inputs and their
- # values used by cell evaluation. This is recorded once the
- # cell finishes its evaluation
- #
- @type snapshot :: {deps_snapshot :: term(), bound_inputs_snapshot :: term()}
+ # including parent cells and inputs. Whenever the snapshot changes,
+ # it implies a new evaluation context, which basically means the cell
+ # got stale.
+ @type snapshot :: term()
@type input_reading :: {input_id(), input_value :: term()}
@@ -202,7 +193,8 @@ defmodule Livebook.Session.Data do
| {:stop_evaluation, Section.t()}
| {:forget_evaluation, Cell.t(), Section.t()}
| {:start_smart_cell, Cell.t(), Section.t()}
- | {:set_smart_cell_base, Cell.t(), Section.t(), parent :: {Cell.t(), Section.t()} | nil}
+ | {:set_smart_cell_parents, Cell.t(), Section.t(),
+ parent :: {Cell.t(), Section.t()} | nil}
| {:broadcast_delta, client_id(), Cell.t(), cell_source_tag(), Delta.t()}
@doc """
@@ -329,7 +321,7 @@ defmodule Livebook.Session.Data do
|> with_actions()
|> cancel_section_evaluation(section)
|> set_section_parent(section, parent_section)
- |> compute_snapshots_and_validity()
+ |> update_validity_and_evaluation()
|> set_dirty()
|> wrap_ok()
else
@@ -345,7 +337,7 @@ defmodule Livebook.Session.Data do
|> cancel_section_evaluation(section)
|> add_action({:stop_evaluation, section})
|> unset_section_parent(section)
- |> compute_snapshots_and_validity()
+ |> update_validity_and_evaluation()
|> set_dirty()
|> wrap_ok()
else
@@ -361,7 +353,7 @@ defmodule Livebook.Session.Data do
|> with_actions()
|> insert_cell(section_id, index, cell)
|> maybe_start_smart_cells()
- |> compute_snapshots_and_validity()
+ |> update_validity_and_evaluation()
|> set_dirty()
|> wrap_ok()
end
@@ -374,7 +366,7 @@ defmodule Livebook.Session.Data do
data
|> with_actions()
|> delete_section(section, delete_cells)
- |> compute_snapshots_and_validity()
+ |> update_validity_and_evaluation()
|> update_smart_cell_bases(data)
|> set_dirty()
|> wrap_ok()
@@ -389,7 +381,7 @@ defmodule Livebook.Session.Data do
data
|> with_actions()
|> delete_cell(cell, section)
- |> compute_snapshots_and_validity()
+ |> update_validity_and_evaluation()
|> update_smart_cell_bases(data)
|> set_dirty()
|> wrap_ok()
@@ -404,7 +396,7 @@ defmodule Livebook.Session.Data do
data
|> with_actions()
|> restore_cell(cell_bin_entry)
- |> compute_snapshots_and_validity()
+ |> update_validity_and_evaluation()
|> maybe_start_smart_cells()
|> set_dirty()
|> wrap_ok()
@@ -421,7 +413,7 @@ defmodule Livebook.Session.Data do
data
|> with_actions()
|> move_cell(cell, offset)
- |> compute_snapshots_and_validity()
+ |> update_validity_and_evaluation()
|> update_smart_cell_bases(data)
|> set_dirty()
|> wrap_ok()
@@ -437,7 +429,7 @@ defmodule Livebook.Session.Data do
data
|> with_actions()
|> move_section(section, offset)
- |> compute_snapshots_and_validity()
+ |> update_validity_and_evaluation()
|> update_smart_cell_bases(data)
|> set_dirty()
|> wrap_ok()
@@ -459,12 +451,11 @@ defmodule Livebook.Session.Data do
cells_with_section
|> Enum.reduce(with_actions(data), fn {cell, section}, data_actions ->
data_actions
- |> queue_prerequisite_cells_evaluation(cell)
+ |> queue_prerequisite_cells_evaluation(cell.id)
|> queue_cell_evaluation(cell, section)
end)
|> maybe_connect_runtime(data)
- |> maybe_evaluate_queued()
- |> compute_snapshots_and_validity()
+ |> update_validity_and_evaluation()
|> wrap_ok()
else
:error
@@ -505,9 +496,7 @@ defmodule Livebook.Session.Data do
|> add_cell_output(cell, output)
|> finish_cell_evaluation(cell, section, metadata)
|> garbage_collect_input_values()
- |> compute_snapshots_and_validity()
- |> maybe_evaluate_queued()
- |> compute_snapshots_and_validity()
+ |> update_validity_and_evaluation()
|> update_smart_cell_bases(data)
|> mark_dirty_if_persisting_outputs()
|> wrap_ok()
@@ -519,8 +508,9 @@ defmodule Livebook.Session.Data do
def apply_operation(data, {:bind_input, _client_id, cell_id, input_id}) do
with {:ok, cell, _section} <- Notebook.fetch_cell_and_section(data.notebook, cell_id),
Cell.evaluable?(cell),
+ :evaluating <- data.cell_infos[cell.id].eval.status,
true <- Map.has_key?(data.input_values, input_id),
- false <- MapSet.member?(data.cell_infos[cell.id].eval.bound_to_input_ids, input_id) do
+ false <- MapSet.member?(data.cell_infos[cell.id].eval.new_bound_to_input_ids, input_id) do
data
|> with_actions()
|> bind_input(cell, input_id)
@@ -715,7 +705,7 @@ defmodule Livebook.Session.Data do
data
|> with_actions()
|> set_cell_attributes(cell, attrs)
- |> compute_snapshots_and_validity()
+ |> update_validity_and_evaluation()
|> set_dirty()
|> wrap_ok()
else
@@ -728,7 +718,7 @@ defmodule Livebook.Session.Data do
data
|> with_actions()
|> set_input_value(input_id, value)
- |> compute_snapshots_and_validity()
+ |> update_validity_and_evaluation()
|> wrap_ok()
else
_ -> :error
@@ -945,8 +935,7 @@ defmodule Livebook.Session.Data do
# For each path in the dependency graph, find the upmost cell
# which parent changed. From that point downwards all cells
- # are invalidated. Then gather invalidated cells from all paths
- # and unqueue them.
+ # are potentially affected, so we unqueue them.
invalidted_cell_ids =
graph_after
@@ -997,7 +986,7 @@ defmodule Livebook.Session.Data do
defp queue_cell_evaluation(data_actions, cell, section) do
data_actions
|> update_section_info!(section.id, fn section ->
- %{section | evaluation_queue: append_new(section.evaluation_queue, cell.id)}
+ update_in(section.evaluation_queue, &MapSet.put(&1, cell.id))
end)
|> update_cell_eval_info!(cell.id, fn eval_info ->
update_in(eval_info.status, fn
@@ -1010,7 +999,7 @@ defmodule Livebook.Session.Data do
defp unqueue_cell_evaluation(data_actions, cell, section) do
data_actions
|> update_section_info!(section.id, fn section ->
- %{section | evaluation_queue: List.delete(section.evaluation_queue, cell.id)}
+ update_in(section.evaluation_queue, &MapSet.delete(&1, cell.id))
end)
|> update_cell_eval_info!(cell.id, &%{&1 | status: :ready})
end
@@ -1047,15 +1036,40 @@ defmodule Livebook.Session.Data do
eval_info
| status: :ready,
evaluation_time_ms: metadata.evaluation_time_ms,
- # After finished evaluation, take the snapshot of read inputs
- evaluation_snapshot:
- {elem(eval_info.evaluation_snapshot, 0),
- input_readings_snapshot(eval_info.bound_input_readings)}
+ identifiers_used: metadata.identifiers_used,
+ identifiers_defined: metadata.identifiers_defined,
+ bound_to_input_ids: eval_info.new_bound_to_input_ids
}
end)
+ |> update_cell_evaluation_snapshot(cell, section)
|> set_section_info!(section.id, evaluating_cell_id: nil)
end
+ defp update_cell_evaluation_snapshot({data, _} = data_actions, cell, section) do
+ info = data.cell_infos[cell.id]
+
+ eval_data = data.cell_infos[cell.id].eval.data
+ eval_data = put_in(eval_data.cell_infos[cell.id], info)
+
+ graph = Notebook.cell_dependency_graph(eval_data.notebook, cell_filter: &Cell.evaluable?/1)
+
+ cell_snapshots =
+ for {cell_id, %{eval: eval_info}} <- eval_data.cell_infos,
+ do: {cell_id, eval_info.snapshot},
+ into: %{}
+
+ # We compute evaluation snapshot based on the notebook state prior
+ # to evaluation, but using the information about the dependencies
+ # obtained during evaluation (identifiers, inputs)
+ evaluation_snapshot = cell_snapshot(cell, section, graph, cell_snapshots, eval_data)
+
+ data_actions
+ |> update_cell_eval_info!(
+ cell.id,
+ &%{&1 | evaluation_snapshot: evaluation_snapshot, data: nil}
+ )
+ end
+
defp maybe_connect_runtime({data, _} = data_actions, prev_data) do
if not Runtime.connected?(data.runtime) and not any_cell_queued?(prev_data) and
any_cell_queued?(data) do
@@ -1066,7 +1080,26 @@ defmodule Livebook.Session.Data do
end
defp any_cell_queued?(data) do
- Enum.any?(data.section_infos, fn {_section_id, info} -> info.evaluation_queue != [] end)
+ Enum.any?(data.section_infos, fn {_section_id, info} ->
+ not Enum.empty?(info.evaluation_queue)
+ end)
+ end
+
+ defp queue_prerequisite_cells_evaluation_for_queued({data, _} = data_actions) do
+ {awaiting_branch_sections, awaiting_regular_sections} =
+ data.notebook
+ |> Notebook.all_sections()
+ |> Enum.filter(§ion_awaits_evaluation?(data, &1.id))
+ |> Enum.split_with(& &1.parent_id)
+
+ trailing_queued_cell_ids =
+ for section <- awaiting_branch_sections ++ Enum.take(awaiting_regular_sections, -1),
+ cell = last_queued_cell(data, section),
+ do: cell.id
+
+ reduce(data_actions, trailing_queued_cell_ids, fn data_actions, cell_id ->
+ queue_prerequisite_cells_evaluation(data_actions, cell_id)
+ end)
end
defp maybe_evaluate_queued({data, _} = data_actions) do
@@ -1081,7 +1114,7 @@ defmodule Livebook.Session.Data do
data_actions =
reduce(data_actions, awaiting_branch_sections, fn {data, _} = data_actions, section ->
- %{evaluation_queue: [id | _]} = data.section_infos[section.id]
+ %{id: id} = first_queued_cell(data, section)
{:ok, parent} = Notebook.fetch_section(data.notebook, section.parent_id)
@@ -1094,7 +1127,7 @@ defmodule Livebook.Session.Data do
prev_section_queued? =
prev_cell_section != nil and
- data.section_infos[prev_cell_section.id].evaluation_queue != []
+ not Enum.empty?(data.section_infos[prev_cell_section.id].evaluation_queue)
# If evaluating this cell requires interaction with the main flow,
# we keep the cell queued. In case of the Elixir runtimes the
@@ -1119,6 +1152,25 @@ defmodule Livebook.Session.Data do
end
end
+ defp first_queued_cell(data, section) do
+ find_queued_cell(data, section.cells)
+ end
+
+ defp last_queued_cell(data, section) do
+ find_queued_cell(data, Enum.reverse(section.cells))
+ end
+
+ defp find_queued_cell(data, cells) do
+ Enum.find_value(cells, fn cell ->
+ info = data.cell_infos[cell.id]
+
+ case info do
+ %{eval: %{status: :queued}} -> cell
+ _ -> nil
+ end
+ end)
+ end
+
defp main_flow_evaluating?(data) do
data.notebook
|> Notebook.all_sections()
@@ -1142,51 +1194,51 @@ defmodule Livebook.Session.Data do
defp section_awaits_evaluation?(data, section_id) do
info = data.section_infos[section_id]
- info.evaluating_cell_id == nil and info.evaluation_queue != []
+ info.evaluating_cell_id == nil and not Enum.empty?(info.evaluation_queue)
end
defp evaluate_next_cell_in_section({data, _} = data_actions, section) do
- case data.section_infos[section.id] do
- %{evaluating_cell_id: nil, evaluation_queue: [id | ids]} ->
- cell = Enum.find(section.cells, &(&1.id == id))
+ section_info = data.section_infos[section.id]
- data_actions
- |> set!(notebook: Notebook.update_cell(data.notebook, id, &%{&1 | outputs: []}))
- |> update_cell_eval_info!(id, fn eval_info ->
- %{
- eval_info
- | # Note: we intentionally mark the cell as evaluating up front,
- # so that another queue operation doesn't cause duplicated
- # :start_evaluation action
- status: :evaluating,
- evaluation_number: eval_info.evaluation_number + 1,
- outputs_batch_number: eval_info.outputs_batch_number + 1,
- evaluation_digest: nil,
- evaluation_snapshot: eval_info.snapshot,
- bound_to_input_ids: MapSet.new(),
- bound_input_readings: [],
- # This is a rough estimate, the exact time is measured in the
- # evaluator itself
- evaluation_start: DateTime.utc_now()
- }
- end)
- |> set_section_info!(section.id, evaluating_cell_id: id, evaluation_queue: ids)
- |> add_action({:start_evaluation, cell, section})
+ if section_info.evaluating_cell_id == nil and not Enum.empty?(section_info.evaluation_queue) do
+ cell = first_queued_cell(data, section)
- _ ->
- data_actions
+ data_actions
+ |> set!(notebook: Notebook.update_cell(data.notebook, cell.id, &%{&1 | outputs: []}))
+ |> update_cell_eval_info!(cell.id, fn eval_info ->
+ %{
+ eval_info
+ | # Note: we intentionally mark the cell as evaluating up front,
+ # so that another queue operation doesn't cause duplicated
+ # :start_evaluation action
+ status: :evaluating,
+ evaluation_number: eval_info.evaluation_number + 1,
+ outputs_batch_number: eval_info.outputs_batch_number + 1,
+ evaluation_digest: nil,
+ new_bound_to_input_ids: MapSet.new(),
+ # Keep the notebook state before evaluation
+ data: data,
+ # This is a rough estimate, the exact time is measured in the
+ # evaluator itself
+ evaluation_start: DateTime.utc_now()
+ }
+ end)
+ |> set_section_info!(section.id,
+ evaluating_cell_id: cell.id,
+ evaluation_queue: MapSet.delete(section_info.evaluation_queue, cell.id)
+ )
+ |> add_action({:start_evaluation, cell, section})
+ else
+ data_actions
end
end
- defp bind_input({data, _} = data_actions, cell, input_id) do
+ defp bind_input(data_actions, cell, input_id) do
data_actions
|> update_cell_eval_info!(cell.id, fn eval_info ->
%{
eval_info
- | bound_to_input_ids: MapSet.put(eval_info.bound_to_input_ids, input_id),
- bound_input_readings: [
- {input_id, data.input_values[input_id]} | eval_info.bound_input_readings
- ]
+ | new_bound_to_input_ids: MapSet.put(eval_info.new_bound_to_input_ids, input_id)
}
end)
end
@@ -1210,7 +1262,7 @@ defmodule Livebook.Session.Data do
evaluable_cells = Enum.filter(section.cells, &Cell.evaluable?/1)
data_actions
- |> set_section_info!(section.id, evaluating_cell_id: nil, evaluation_queue: [])
+ |> set_section_info!(section.id, evaluating_cell_id: nil, evaluation_queue: MapSet.new())
|> reduce(
evaluable_cells,
&update_cell_eval_info!(&1, &2.id, fn eval_info ->
@@ -1230,14 +1282,13 @@ defmodule Livebook.Session.Data do
)
end
- defp queue_prerequisite_cells_evaluation({data, _} = data_actions, cell) do
+ defp queue_prerequisite_cells_evaluation({data, _} = data_actions, cell_id) do
prerequisites_queue =
data.notebook
- |> Notebook.parent_cells_with_section(cell.id)
- |> Enum.filter(fn {cell, _} -> Cell.evaluable?(cell) end)
- |> Enum.take_while(fn {parent_cell, _section} ->
- info = data.cell_infos[parent_cell.id]
- info.eval.validity != :evaluated and info.eval.status == :ready
+ |> Notebook.parent_cells_with_section(cell_id)
+ |> Enum.filter(fn {cell, _section} ->
+ info = data.cell_infos[cell.id]
+ Cell.evaluable?(cell) and info.eval.validity != :evaluated and info.eval.status == :ready
end)
|> Enum.reverse()
@@ -1263,16 +1314,20 @@ defmodule Livebook.Session.Data do
:queued ->
data_actions
|> unqueue_cell_evaluation(cell, section)
- |> unqueue_dependent_cells_evaluation(cell)
+ |> unqueue_child_cells_evaluation(cell)
_ ->
data_actions
end
end
- defp unqueue_dependent_cells_evaluation({data, _} = data_actions, cell) do
- dependent = dependent_cells_with_section(data, cell.id)
- unqueue_cells_evaluation(data_actions, dependent)
+ defp unqueue_child_cells_evaluation({data, _} = data_actions, cell) do
+ evaluation_children =
+ data.notebook
+ |> Notebook.child_cells_with_section(cell.id)
+ |> Enum.filter(fn {cell, _} -> Cell.evaluable?(cell) end)
+
+ unqueue_cells_evaluation(data_actions, evaluation_children)
end
defp unqueue_cells_evaluation({data, _} = data_actions, cells_with_section) do
@@ -1337,7 +1392,7 @@ defmodule Livebook.Session.Data do
if evaluated? and reevaluate do
data_actions
- |> queue_prerequisite_cells_evaluation(cell)
+ |> queue_prerequisite_cells_evaluation(cell.id)
|> queue_cell_evaluation(cell, section)
|> maybe_evaluate_queued()
else
@@ -1610,14 +1665,6 @@ defmodule Livebook.Session.Data do
{data, actions ++ [action]}
end
- defp append_new(list, item) do
- if item in list do
- list
- else
- list ++ [item]
- end
- end
-
defp garbage_collect_input_values({data, _} = data_actions) do
if any_section_evaluating?(data) do
# Wait if evaluation is ongoing as it may render inputs
@@ -1638,69 +1685,31 @@ defmodule Livebook.Session.Data do
if Enum.empty?(alive_smart_cell_ids) do
data_actions
else
- new_eval_graph = cell_evaluation_graph(data)
- prev_eval_graph = cell_evaluation_graph(prev_data)
+ new_eval_parents = cell_evaluation_parents(data)
+ prev_eval_parents = cell_evaluation_parents(prev_data)
cell_lookup =
data.notebook
|> Notebook.cells_with_section()
|> Map.new(fn {cell, section} -> {cell.id, {cell, section}} end)
- for {cell_id, parent_id} <- new_eval_graph,
+ for {cell_id, eval_parents} <- new_eval_parents,
MapSet.member?(alive_smart_cell_ids, cell_id),
- Map.has_key?(prev_eval_graph, cell_id),
- prev_eval_graph[cell_id] != parent_id,
+ Map.has_key?(prev_eval_parents, cell_id),
+ prev_eval_parents[cell_id] != eval_parents,
reduce: data_actions do
data_actions ->
{cell, section} = cell_lookup[cell_id]
- parent = cell_lookup[parent_id]
- add_action(data_actions, {:set_smart_cell_base, cell, section, parent})
+ parents = Enum.map(eval_parents, &cell_lookup[&1])
+ add_action(data_actions, {:set_smart_cell_parents, cell, section, parents})
end
end
end
- # Builds a graph with evaluation parents, where each parent has
- # aleady been evaluated. All fresh/aborted cells are leaves in
- # this graph
- defp cell_evaluation_graph(data) do
- graph = Notebook.cell_dependency_graph(data.notebook, cell_filter: &Cell.evaluable?/1)
-
- graph
- |> Livebook.Utils.Graph.leaves()
- |> Enum.reduce(%{}, fn cell_id, eval_graph ->
- build_eval_graph(data, graph, cell_id, [], eval_graph)
- end)
- end
-
- defp build_eval_graph(_data, _graph, nil, orphan_ids, eval_graph) do
- put_parent(eval_graph, orphan_ids, nil)
- end
-
- defp build_eval_graph(data, graph, cell_id, orphan_ids, eval_graph) do
- # We are traversing from every leaf up, so we want to compute
- # the common path only once
- if eval_parent_id = eval_graph[cell_id] do
- put_parent(eval_graph, orphan_ids, eval_parent_id)
- else
- info = data.cell_infos[cell_id]
-
- if info.eval.validity in [:evaluated, :stale] do
- eval_graph = put_parent(eval_graph, orphan_ids, cell_id)
- build_eval_graph(data, graph, graph[cell_id], [cell_id], eval_graph)
- else
- build_eval_graph(data, graph, graph[cell_id], [cell_id | orphan_ids], eval_graph)
- end
- end
- end
-
- defp put_parent(eval_graph, cell_ids, parent_id) do
- Enum.reduce(cell_ids, eval_graph, &Map.put(&2, &1, parent_id))
- end
-
defp new_section_info() do
%{
evaluating_cell_id: nil,
- evaluation_queue: []
+ evaluation_queue: MapSet.new()
}
end
@@ -1745,9 +1754,12 @@ defmodule Livebook.Session.Data do
evaluation_number: 0,
outputs_batch_number: 0,
bound_to_input_ids: MapSet.new(),
- bound_input_readings: [],
- snapshot: {nil, nil},
- evaluation_snapshot: nil
+ new_bound_to_input_ids: MapSet.new(),
+ identifiers_used: [],
+ identifiers_defined: %{},
+ snapshot: nil,
+ evaluation_snapshot: nil,
+ data: nil
}
end
@@ -1808,6 +1820,52 @@ defmodule Livebook.Session.Data do
Enum.all?(attrs, fn {key, _} -> Map.has_key?(struct, key) end)
end
+ @doc """
+ Builds evaluation parent sequence for every evaluable cell.
+
+ This function should be used instead of calling `cell_evaluation_parents/2`
+ multiple times.
+ """
+ @spec cell_evaluation_parents(t()) :: %{Cell.id() => list(Cell.id())}
+ def cell_evaluation_parents(data) do
+ graph = Notebook.cell_dependency_graph(data.notebook, cell_filter: &Cell.evaluable?/1)
+
+ graph
+ |> Graph.reduce_paths({nil, %{}}, fn cell_id, {parent_id, chains} ->
+ if parent_id do
+ parent_chain = chains[parent_id]
+ parent_info = data.cell_infos[parent_id]
+
+ chain =
+ if parent_info.eval.validity in [:evaluated, :stale] do
+ [parent_id | parent_chain]
+ else
+ parent_chain
+ end
+
+ {cell_id, put_in(chains[cell_id], chain)}
+ else
+ {cell_id, put_in(chains[cell_id], [])}
+ end
+ end)
+ |> Enum.map(&elem(&1, 1))
+ |> Enum.reduce(&Map.merge/2)
+ end
+
+ @doc """
+ Builds evaluation parent sequence for the given cell.
+
+ Considers only cells that have already been evaluated.
+ """
+ @spec cell_evaluation_parents(Data.t(), Cell.t()) :: list({Cell.t(), Section.t()})
+ def cell_evaluation_parents(data, cell) do
+ for {cell, section} <- Notebook.parent_cells_with_section(data.notebook, cell.id),
+ info = data.cell_infos[cell.id],
+ Cell.evaluable?(cell),
+ info.eval.validity in [:evaluated, :stale],
+ do: {cell, section}
+ end
+
@doc """
Find cells bound to the given input.
"""
@@ -1821,20 +1879,16 @@ defmodule Livebook.Session.Data do
end)
end
- defp dependent_cells_with_section(data, cell_id) do
- data.notebook
- |> Notebook.child_cells_with_section(cell_id)
- |> Enum.filter(fn {cell, _} -> Cell.evaluable?(cell) end)
- end
-
- # Computes cell snapshots and updates validity based on the new values.
- defp compute_snapshots_and_validity(data_actions) do
+ # Computes cell snapshots and updates validity based on the new
+ # values, then triggers further evaluation if applicable.
+ defp update_validity_and_evaluation(data_actions) do
data_actions
|> compute_snapshots()
|> update_validity()
# After updating validity there may be new stale cells, so we check
# if any of them is configured for automatic reevaluation
|> maybe_queue_reevaluating_cells()
+ |> queue_prerequisite_cells_evaluation_for_queued()
|> maybe_evaluate_queued()
end
@@ -1845,32 +1899,7 @@ defmodule Livebook.Session.Data do
cell_snapshots =
Enum.reduce(cells_with_section, %{}, fn {cell, section}, cell_snapshots ->
- info = data.cell_infos[cell.id]
- prev_cell_id = graph[cell.id]
-
- is_branch? = section.parent_id != nil
-
- parent_deps =
- prev_cell_id &&
- {
- prev_cell_id,
- cell_snapshots[prev_cell_id],
- number_of_evaluations(data.cell_infos[prev_cell_id])
- }
-
- deps = {is_branch?, parent_deps}
- deps_snapshot = :erlang.phash2(deps)
-
- inputs_snapshot =
- if info.eval.status == :evaluating do
- # While the cell is evaluating the bound inputs snapshot
- # is not stable, so we reuse the previous snapshot
- elem(info.eval.snapshot, 1)
- else
- bound_inputs_snapshot(data, cell)
- end
-
- snapshot = {deps_snapshot, inputs_snapshot}
+ snapshot = cell_snapshot(cell, section, graph, cell_snapshots, data)
put_in(cell_snapshots[cell.id], snapshot)
end)
@@ -1882,26 +1911,102 @@ defmodule Livebook.Session.Data do
end)
end
- defp number_of_evaluations(%{eval: %{status: :evaluating}} = info) do
- info.eval.evaluation_number - 1
+ defp cell_snapshot(cell, section, graph, cell_snapshots, data) do
+ info = data.cell_infos[cell.id]
+
+ # Note that this is an implication of the Elixir runtime, we want
+ # to reevaluate as much as possible in a branch, rather than copying
+ # contexts between processes, because all structural sharing is
+ # lost when copying
+ is_branch? = section.parent_id != nil
+
+ {parent_ids, identifier_versions} = identifier_deps(cell.id, graph, data)
+
+ parent_snapshots = Enum.map(parent_ids, &cell_snapshots[&1])
+
+ bound_input_values =
+ for(
+ input_id <- info.eval.bound_to_input_ids,
+ do: {input_id, data.input_values[input_id]}
+ )
+ |> Enum.sort()
+
+ deps = {is_branch?, parent_snapshots, identifier_versions, bound_input_values}
+
+ :erlang.phash2(deps)
end
- defp number_of_evaluations(info), do: info.eval.evaluation_number
+ defp identifier_deps(cell_id, graph, data) do
+ info = data.cell_infos[cell_id]
- defp bound_inputs_snapshot(data, cell) do
- %{bound_to_input_ids: bound_to_input_ids} = data.cell_infos[cell.id].eval
+ {parent_ids, identifier_versions} =
+ case info.eval.identifiers_used do
+ :unknown ->
+ all_identifier_deps(graph[cell_id], graph, data)
- for(
- input_id <- bound_to_input_ids,
- do: {input_id, data.input_values[input_id]}
- )
- |> input_readings_snapshot()
+ identifiers_used ->
+ gather_identifier_deps(graph[cell_id], identifiers_used, graph, data, {[], []})
+ end
+
+ {Enum.sort(parent_ids), Enum.sort(identifier_versions)}
end
- defp input_readings_snapshot([]), do: :empty
+ defp all_identifier_deps(cell_id, graph, data) do
+ parent_ids = graph |> Graph.find_path(cell_id, nil) |> Enum.drop(1)
- defp input_readings_snapshot(name_value_pairs) do
- name_value_pairs |> Enum.sort() |> :erlang.phash2()
+ identifier_versions =
+ parent_ids
+ |> List.foldr(%{}, fn cell_id, acc ->
+ identifiers_defined = data.cell_infos[cell_id].eval.identifiers_defined
+ Map.merge(acc, identifiers_defined)
+ end)
+ |> Map.to_list()
+
+ {parent_ids, identifier_versions}
+ end
+
+ defp gather_identifier_deps(nil, _identifiers_used, _graph, _data, acc), do: acc
+
+ defp gather_identifier_deps(_cell_id, [], _graph, _data, acc), do: acc
+
+ defp gather_identifier_deps(
+ cell_id,
+ identifiers_used,
+ graph,
+ data,
+ {parent_ids, identifier_versions}
+ ) do
+ identifiers_defined = data.cell_infos[cell_id].eval.identifiers_defined
+
+ identifiers_used
+ |> Enum.reduce({[], []}, fn identifier, {versions, rest_identifiers} ->
+ case identifiers_defined do
+ %{^identifier => version} ->
+ {[{identifier, version} | versions], rest_identifiers}
+
+ _ ->
+ {versions, [identifier | rest_identifiers]}
+ end
+ end)
+ |> case do
+ {[], rest_identifiers} ->
+ gather_identifier_deps(
+ graph[cell_id],
+ rest_identifiers,
+ graph,
+ data,
+ {parent_ids, identifier_versions}
+ )
+
+ {versions, rest_identifiers} ->
+ gather_identifier_deps(
+ graph[cell_id],
+ rest_identifiers,
+ graph,
+ data,
+ {[cell_id | parent_ids], versions ++ identifier_versions}
+ )
+ end
end
defp update_validity({data, _} = data_actions) do
@@ -1924,7 +2029,7 @@ defmodule Livebook.Session.Data do
end
defp maybe_queue_reevaluating_cells({data, _} = data_actions) do
- cells_to_reeavaluete =
+ cells_to_reevaluate =
data.notebook
|> Notebook.evaluable_cells_with_section()
|> Enum.filter(fn {cell, _section} ->
@@ -1935,9 +2040,9 @@ defmodule Livebook.Session.Data do
end)
data_actions
- |> reduce(cells_to_reeavaluete, fn data_actions, {cell, section} ->
+ |> reduce(cells_to_reevaluate, fn data_actions, {cell, section} ->
data_actions
- |> queue_prerequisite_cells_evaluation(cell)
+ |> queue_prerequisite_cells_evaluation(cell.id)
|> queue_cell_evaluation(cell, section)
end)
end
@@ -1959,7 +2064,7 @@ defmodule Livebook.Session.Data do
Returns the list of cell ids for full evaluation.
The list includes all outdated cells, cells in `forced_cell_ids`
- and all of their child cells.
+ and all cells with identifier dependency on these.
"""
@spec cell_ids_for_full_evaluation(t(), list(Cell.id())) :: list(Cell.id())
def cell_ids_for_full_evaluation(data, forced_cell_ids) do
@@ -1968,16 +2073,70 @@ defmodule Livebook.Session.Data do
evaluable_cell_ids =
for {cell, _} <- evaluable_cells_with_section,
cell_outdated?(data, cell) or cell.id in forced_cell_ids,
- info = data.cell_infos[cell.id],
- info.eval.status == :ready,
- uniq: true,
- do: cell.id
+ do: cell.id,
+ into: MapSet.new()
- cell_ids = Notebook.cell_ids_with_children(data.notebook, evaluable_cell_ids)
+ cell_identifier_parents = cell_identifier_parents(data)
- for {cell, _} <- evaluable_cells_with_section,
- cell.id in cell_ids,
- do: cell.id
+ child_ids =
+ for {cell_id, cell_identifier_parents} <- cell_identifier_parents,
+ Enum.any?(cell_identifier_parents, &(&1 in evaluable_cell_ids)),
+ do: cell_id
+
+ child_ids
+ |> Enum.into(evaluable_cell_ids)
+ |> Enum.to_list()
+ |> Enum.filter(fn cell_id ->
+ info = data.cell_infos[cell_id]
+ info.eval.status == :ready
+ end)
+ end
+
+ # Builds identifier parent list for every evaluable cell.
+ #
+ # This is similar to cell_evaluation_parents, but the dependency is
+ # based on identifiers used/set by each cell.
+ defp cell_identifier_parents(data) do
+ graph = Notebook.cell_dependency_graph(data.notebook, cell_filter: &Cell.evaluable?/1)
+
+ graph
+ |> Graph.reduce_paths(
+ {nil, %{}, %{}},
+ fn cell_id, {parent_id, setters, identifier_parents} ->
+ if parent_id do
+ cell_info = data.cell_infos[cell_id]
+
+ direct_parents =
+ case cell_info.eval.identifiers_used do
+ :unknown ->
+ setters |> Map.values() |> Enum.uniq()
+
+ identifiers_used ->
+ for identifier <- identifiers_used,
+ parent_id = setters[identifier],
+ uniq: true,
+ do: parent_id
+ end
+
+ parents =
+ for parent_id <- direct_parents,
+ cell_id <- [parent_id | identifier_parents[parent_id]],
+ uniq: true,
+ do: cell_id
+
+ setters =
+ for {identifier, _version} <- cell_info.eval.identifiers_defined,
+ do: {identifier, cell_id},
+ into: setters
+
+ {cell_id, setters, put_in(identifier_parents[cell_id], parents)}
+ else
+ {cell_id, setters, put_in(identifier_parents[cell_id], [])}
+ end
+ end
+ )
+ |> Enum.map(&elem(&1, 2))
+ |> Enum.reduce(&Map.merge/2)
end
@doc """
@@ -2010,4 +2169,21 @@ defmodule Livebook.Session.Data do
end)
|> elem(0)
end
+
+ @doc """
+ Fetches an input value for the given cell.
+
+ If the cell is evaluating, the input value at evaluation start is
+ returned instead of the current value.
+ """
+ @spec fetch_input_value_for_cell(t(), input_id(), Cell.id()) :: {:ok, term()} | :error
+ def fetch_input_value_for_cell(data, input_id, cell_id) do
+ data =
+ case data.cell_infos[cell_id] do
+ %{eval: %{status: :evaluating, data: data}} -> data
+ _ -> data
+ end
+
+ Map.fetch(data.input_values, input_id)
+ end
end
diff --git a/lib/livebook/utils/graph.ex b/lib/livebook/utils/graph.ex
index 1e9b184b2..fc45f8152 100644
--- a/lib/livebook/utils/graph.ex
+++ b/lib/livebook/utils/graph.ex
@@ -30,7 +30,7 @@ defmodule Livebook.Utils.Graph do
do: find_path(graph, graph[from_id], to_id, [from_id | path])
@doc """
- Finds grpah leave nodes, that is, nodes with
+ Finds graph leave nodes, that is, nodes with
no children.
"""
@spec leaves(t()) :: list(node_id())
@@ -39,4 +39,37 @@ defmodule Livebook.Utils.Graph do
parents = MapSet.new(graph, fn {_, value} -> value end)
MapSet.difference(children, parents) |> MapSet.to_list()
end
+
+ @doc """
+ Reduces each top-down path in the graph.
+
+ Returns a list of accumulators, one for each leaf in the graph,
+ in no specific order.
+ """
+ @spec reduce_paths(t(), acc, (node_id(), acc -> acc)) :: acc when acc: term()
+ def reduce_paths(graph, acc, fun) do
+ leaves = Livebook.Utils.Graph.leaves(graph)
+ cache = do_reduce(graph, leaves, acc, fun, %{})
+ Enum.map(leaves, &cache[&1])
+ end
+
+ defp do_reduce(_graph, [], _initial_acc, _fun, cache), do: cache
+
+ defp do_reduce(graph, [cell_id | cell_ids], initial_acc, fun, cache) do
+ if parent_id = graph[cell_id] do
+ case cache do
+ %{^parent_id => acc} ->
+ acc = fun.(cell_id, acc)
+ cache = put_in(cache[cell_id], acc)
+ do_reduce(graph, cell_ids, initial_acc, fun, cache)
+
+ _ ->
+ do_reduce(graph, [parent_id, cell_id | cell_ids], initial_acc, fun, cache)
+ end
+ else
+ acc = fun.(cell_id, initial_acc)
+ cache = put_in(cache[cell_id], acc)
+ do_reduce(graph, cell_ids, initial_acc, fun, cache)
+ end
+ end
end
diff --git a/lib/livebook/web_socket/server.ex b/lib/livebook/web_socket/server.ex
index 6f9fdb3b7..dc132564a 100644
--- a/lib/livebook/web_socket/server.ex
+++ b/lib/livebook/web_socket/server.ex
@@ -103,7 +103,7 @@ defmodule Livebook.WebSocket.Server do
# Private
defp reply(%{caller: nil} = state, response) do
- Logger.warn("The caller is nil, so we can't reply the message: #{inspect(response)}")
+ Logger.warning("The caller is nil, so we can't reply the message: #{inspect(response)}")
state
end
diff --git a/lib/livebook_web/live/session_live.ex b/lib/livebook_web/live/session_live.ex
index 826f6e905..3cf8c12f4 100644
--- a/lib/livebook_web/live/session_live.ex
+++ b/lib/livebook_web/live/session_live.ex
@@ -1221,10 +1221,10 @@ defmodule LivebookWeb.SessionLive do
data = socket.private.data
- with {:ok, cell, section} <- Notebook.fetch_cell_and_section(data.notebook, cell_id) do
+ with {:ok, cell, _section} <- Notebook.fetch_cell_and_section(data.notebook, cell_id) do
if Runtime.connected?(data.runtime) do
- base_locator = Session.find_base_locator(data, cell, section, existing: true)
- ref = Runtime.handle_intellisense(data.runtime, self(), request, base_locator)
+ parent_locators = Session.parent_locators_for_cell(data, cell)
+ ref = Runtime.handle_intellisense(data.runtime, self(), request, parent_locators)
{:reply, %{"ref" => inspect(ref)}, socket}
else
diff --git a/lib/livebook_web/live/session_live/attached_live.ex b/lib/livebook_web/live/session_live/attached_live.ex
index ebf69be1d..17fd8dcaa 100644
--- a/lib/livebook_web/live/session_live/attached_live.ex
+++ b/lib/livebook_web/live/session_live/attached_live.ex
@@ -67,7 +67,11 @@ defmodule LivebookWeb.SessionLive.AttachedLive do
Cookie
- <%= text_input(f, :cookie, value: @data["cookie"], class: "input", placeholder: "mycookie") %>
+ <%= text_input(f, :cookie,
+ value: @data["cookie"],
+ class: "input",
+ placeholder: "mycookie"
+ ) %>