diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e6cba7a50..f4c1f42a3 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -6,7 +6,8 @@ on: - main env: otp: "25.0" - elixir: "1.14.0" + # TODO: update to v1.14.2 once it is out + elixir: "main" jobs: main: runs-on: ubuntu-latest diff --git a/lib/livebook/notebook.ex b/lib/livebook/notebook.ex index b5e35f2a1..d7d8337fc 100644 --- a/lib/livebook/notebook.ex +++ b/lib/livebook/notebook.ex @@ -471,23 +471,6 @@ defmodule Livebook.Notebook do |> Enum.filter(fn {cell, _} -> MapSet.member?(child_cell_ids, cell.id) end) end - @doc """ - Returns the list with the given parent cells and all of - their child cells. - - The cells are not ordered in any secific way. - """ - @spec cell_ids_with_children(t(), list(Cell.id())) :: list(Cell.id()) - def cell_ids_with_children(notebook, parent_cell_ids) do - graph = cell_dependency_graph(notebook) - - for parent_id <- parent_cell_ids, - leaf_id <- Graph.leaves(graph), - cell_id <- Graph.find_path(graph, leaf_id, parent_id), - uniq: true, - do: cell_id - end - @doc """ Computes cell dependency graph. diff --git a/lib/livebook/runtime.ex b/lib/livebook/runtime.ex index c1ac459b3..1ef685b70 100644 --- a/lib/livebook/runtime.ex +++ b/lib/livebook/runtime.ex @@ -26,12 +26,15 @@ defprotocol Livebook.Runtime do @typedoc """ A pair identifying evaluation together with its container. - - When the evaluation reference is `nil`, the `locator` points to - a container and may be used to represent its default evaluation - context. """ - @type locator :: {container_ref(), evaluation_ref() | nil} + @type locator :: {container_ref(), evaluation_ref()} + + @typedoc """ + A sequence of locators representing a multi-stage evaluation. + + The evaluation locators should be ordered from most recent to oldest. + """ + @type parent_locators :: list(locator()) @typedoc """ An output emitted during evaluation or as the final result. @@ -64,12 +67,22 @@ defprotocol Livebook.Runtime do | {:error, message :: String.t(), type :: {:missing_secret, String.t()} | :other} @typedoc """ - Additional information about a complted evaluation. + Additional information about a completed evaluation. + + ## Identifiers + + When possible, the metadata may include a list of identifiers (such + as variables, modules, imports) used during evaluation, and a list + of identifiers defined along with the version (such as a hash digest + of the underlying value). With this information, Livebook can track + dependencies between evaluations and avoids unnecessary reevaluations. """ @type evaluation_response_metadata :: %{ evaluation_time_ms: non_neg_integer(), code_error: code_error(), - memory_usage: runtime_memory() + memory_usage: runtime_memory(), + identifiers_used: list(identifier :: term()) | :unknown, + identifiers_defined: %{(identifier :: term()) => version :: term()} } @typedoc """ @@ -306,9 +319,8 @@ defprotocol Livebook.Runtime do be evaluated as well as the evaluation reference to store the resulting context under. - Additionally, `base_locator` points to a previous evaluation to be - used as the starting point of this evaluation. If not applicable, - the previous evaluation reference may be specified as `nil`. + Additionally, `parent_locators` points to a sequence of previous + evaluations to be used as the starting point of this evaluation. ## Communication @@ -347,8 +359,8 @@ defprotocol Livebook.Runtime do * `:smart_cell_ref` - a reference of the smart cell which code is to be evaluated, if applicable """ - @spec evaluate_code(t(), String.t(), locator(), locator(), keyword()) :: :ok - def evaluate_code(runtime, code, locator, base_locator, opts \\ []) + @spec evaluate_code(t(), String.t(), locator(), parent_locators(), keyword()) :: :ok + def evaluate_code(runtime, code, locator, parent_locators, opts \\ []) @doc """ Disposes of an evaluation identified by the given locator. @@ -378,11 +390,11 @@ defprotocol Livebook.Runtime do * `{:runtime_intellisense_response, ref, request, response}`. - The given `base_locator` idenfities an evaluation that may be - used as the context when resolving the request (if relevant). + The given `parent_locators` identifies a sequence of evaluations + that may be used as the context when resolving the request (if relevant). """ - @spec handle_intellisense(t(), pid(), intellisense_request(), locator()) :: reference() - def handle_intellisense(runtime, send_to, request, base_locator) + @spec handle_intellisense(t(), pid(), intellisense_request(), parent_locators()) :: reference() + def handle_intellisense(runtime, send_to, request, parent_locators) @doc """ Reads file at the given absolute path within the runtime file system. @@ -401,9 +413,9 @@ defprotocol Livebook.Runtime do The cell may depend on evaluation context to provide a better user experience, for instance it may suggest relevant variable names. - Similarly to `evaluate_code/5`, `base_locator` must be specified - pointing to the evaluation to use as the context. When the locator - changes, it can be updated with `set_smart_cell_base_locator/3`. + Similarly to `evaluate_code/5`, `parent_locators` must be specified + pointing to the sequence of evaluations to use as the context. When + the sequence changes, it can be updated with `set_smart_cell_parent_locators/3`. Once the cell starts, the runtime sends the following message @@ -425,16 +437,22 @@ defprotocol Livebook.Runtime do state later. Note that for persistence they get serialized and deserialized as JSON. """ - @spec start_smart_cell(t(), String.t(), smart_cell_ref(), smart_cell_attrs(), locator()) :: :ok - def start_smart_cell(runtime, kind, ref, attrs, base_locator) + @spec start_smart_cell( + t(), + String.t(), + smart_cell_ref(), + smart_cell_attrs(), + parent_locators() + ) :: :ok + def start_smart_cell(runtime, kind, ref, attrs, parent_locators) @doc """ - Updates the locator used by a smart cell as its context. + Updates the parent locator used by a smart cell as its context. See `start_smart_cell/5` for more details. """ - @spec set_smart_cell_base_locator(t(), smart_cell_ref(), locator()) :: :ok - def set_smart_cell_base_locator(runtime, ref, base_locator) + @spec set_smart_cell_parent_locators(t(), smart_cell_ref(), parent_locators()) :: :ok + def set_smart_cell_parent_locators(runtime, ref, parent_locators) @doc """ Stops smart cell identified by the given reference. diff --git a/lib/livebook/runtime/attached.ex b/lib/livebook/runtime/attached.ex index a556b1ee5..f6266898f 100644 --- a/lib/livebook/runtime/attached.ex +++ b/lib/livebook/runtime/attached.ex @@ -95,8 +95,8 @@ defimpl Livebook.Runtime, for: Livebook.Runtime.Attached do Livebook.Runtime.Attached.new(runtime.node, runtime.cookie) end - def evaluate_code(runtime, code, locator, base_locator, opts \\ []) do - RuntimeServer.evaluate_code(runtime.server_pid, code, locator, base_locator, opts) + def evaluate_code(runtime, code, locator, parent_locators, opts \\ []) do + RuntimeServer.evaluate_code(runtime.server_pid, code, locator, parent_locators, opts) end def forget_evaluation(runtime, locator) do @@ -107,20 +107,20 @@ defimpl Livebook.Runtime, for: Livebook.Runtime.Attached do RuntimeServer.drop_container(runtime.server_pid, container_ref) end - def handle_intellisense(runtime, send_to, request, base_locator) do - RuntimeServer.handle_intellisense(runtime.server_pid, send_to, request, base_locator) + def handle_intellisense(runtime, send_to, request, parent_locators) do + RuntimeServer.handle_intellisense(runtime.server_pid, send_to, request, parent_locators) end def read_file(runtime, path) do RuntimeServer.read_file(runtime.server_pid, path) end - def start_smart_cell(runtime, kind, ref, attrs, base_locator) do - RuntimeServer.start_smart_cell(runtime.server_pid, kind, ref, attrs, base_locator) + def start_smart_cell(runtime, kind, ref, attrs, parent_locators) do + RuntimeServer.start_smart_cell(runtime.server_pid, kind, ref, attrs, parent_locators) end - def set_smart_cell_base_locator(runtime, ref, base_locator) do - RuntimeServer.set_smart_cell_base_locator(runtime.server_pid, ref, base_locator) + def set_smart_cell_parent_locators(runtime, ref, parent_locators) do + RuntimeServer.set_smart_cell_parent_locators(runtime.server_pid, ref, parent_locators) end def stop_smart_cell(runtime, ref) do diff --git a/lib/livebook/runtime/elixir_standalone.ex b/lib/livebook/runtime/elixir_standalone.ex index 0a5106c55..781a32d81 100644 --- a/lib/livebook/runtime/elixir_standalone.ex +++ b/lib/livebook/runtime/elixir_standalone.ex @@ -179,8 +179,8 @@ defimpl Livebook.Runtime, for: Livebook.Runtime.ElixirStandalone do Livebook.Runtime.ElixirStandalone.new() end - def evaluate_code(runtime, code, locator, base_locator, opts \\ []) do - RuntimeServer.evaluate_code(runtime.server_pid, code, locator, base_locator, opts) + def evaluate_code(runtime, code, locator, parent_locators, opts \\ []) do + RuntimeServer.evaluate_code(runtime.server_pid, code, locator, parent_locators, opts) end def forget_evaluation(runtime, locator) do @@ -191,20 +191,20 @@ defimpl Livebook.Runtime, for: Livebook.Runtime.ElixirStandalone do RuntimeServer.drop_container(runtime.server_pid, container_ref) end - def handle_intellisense(runtime, send_to, request, base_locator) do - RuntimeServer.handle_intellisense(runtime.server_pid, send_to, request, base_locator) + def handle_intellisense(runtime, send_to, request, parent_locators) do + RuntimeServer.handle_intellisense(runtime.server_pid, send_to, request, parent_locators) end def read_file(runtime, path) do RuntimeServer.read_file(runtime.server_pid, path) end - def start_smart_cell(runtime, kind, ref, attrs, base_locator) do - RuntimeServer.start_smart_cell(runtime.server_pid, kind, ref, attrs, base_locator) + def start_smart_cell(runtime, kind, ref, attrs, parent_locators) do + RuntimeServer.start_smart_cell(runtime.server_pid, kind, ref, attrs, parent_locators) end - def set_smart_cell_base_locator(runtime, ref, base_locator) do - RuntimeServer.set_smart_cell_base_locator(runtime.server_pid, ref, base_locator) + def set_smart_cell_parent_locators(runtime, ref, parent_locators) do + RuntimeServer.set_smart_cell_parent_locators(runtime.server_pid, ref, parent_locators) end def stop_smart_cell(runtime, ref) do diff --git a/lib/livebook/runtime/embedded.ex b/lib/livebook/runtime/embedded.ex index ffb21c283..2ed49af1a 100644 --- a/lib/livebook/runtime/embedded.ex +++ b/lib/livebook/runtime/embedded.ex @@ -76,8 +76,8 @@ defimpl Livebook.Runtime, for: Livebook.Runtime.Embedded do Livebook.Runtime.Embedded.new() end - def evaluate_code(runtime, code, locator, base_locator, opts \\ []) do - RuntimeServer.evaluate_code(runtime.server_pid, code, locator, base_locator, opts) + def evaluate_code(runtime, code, locator, parent_locators, opts \\ []) do + RuntimeServer.evaluate_code(runtime.server_pid, code, locator, parent_locators, opts) end def forget_evaluation(runtime, locator) do @@ -88,20 +88,20 @@ defimpl Livebook.Runtime, for: Livebook.Runtime.Embedded do RuntimeServer.drop_container(runtime.server_pid, container_ref) end - def handle_intellisense(runtime, send_to, request, base_locator) do - RuntimeServer.handle_intellisense(runtime.server_pid, send_to, request, base_locator) + def handle_intellisense(runtime, send_to, request, parent_locators) do + RuntimeServer.handle_intellisense(runtime.server_pid, send_to, request, parent_locators) end def read_file(runtime, path) do RuntimeServer.read_file(runtime.server_pid, path) end - def start_smart_cell(runtime, kind, ref, attrs, base_locator) do - RuntimeServer.start_smart_cell(runtime.server_pid, kind, ref, attrs, base_locator) + def start_smart_cell(runtime, kind, ref, attrs, parent_locators) do + RuntimeServer.start_smart_cell(runtime.server_pid, kind, ref, attrs, parent_locators) end - def set_smart_cell_base_locator(runtime, ref, base_locator) do - RuntimeServer.set_smart_cell_base_locator(runtime.server_pid, ref, base_locator) + def set_smart_cell_parent_locators(runtime, ref, parent_locators) do + RuntimeServer.set_smart_cell_parent_locators(runtime.server_pid, ref, parent_locators) end def stop_smart_cell(runtime, ref) do diff --git a/lib/livebook/runtime/erl_dist.ex b/lib/livebook/runtime/erl_dist.ex index 0ce06fb2a..d0f780c94 100644 --- a/lib/livebook/runtime/erl_dist.ex +++ b/lib/livebook/runtime/erl_dist.ex @@ -23,6 +23,7 @@ defmodule Livebook.Runtime.ErlDist do [ Livebook.Runtime.Evaluator, Livebook.Runtime.Evaluator.IOProxy, + Livebook.Runtime.Evaluator.Tracer, Livebook.Runtime.Evaluator.ObjectTracker, Livebook.Runtime.Evaluator.DefaultFormatter, Livebook.Intellisense, diff --git a/lib/livebook/runtime/erl_dist/runtime_server.ex b/lib/livebook/runtime/erl_dist/runtime_server.ex index 23607c4a8..b354496f6 100644 --- a/lib/livebook/runtime/erl_dist/runtime_server.ex +++ b/lib/livebook/runtime/erl_dist/runtime_server.ex @@ -71,9 +71,15 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do See `Livebook.Runtime.Evaluator` for more details. """ - @spec evaluate_code(pid(), String.t(), Runtime.locator(), Runtime.locator(), keyword()) :: :ok - def evaluate_code(pid, code, locator, base_locator, opts \\ []) do - GenServer.cast(pid, {:evaluate_code, code, locator, base_locator, opts}) + @spec evaluate_code( + pid(), + String.t(), + Runtime.locator(), + Runtime.parent_locators(), + keyword() + ) :: :ok + def evaluate_code(pid, code, locator, parent_locators, opts \\ []) do + GenServer.cast(pid, {:evaluate_code, code, locator, parent_locators, opts}) end @doc """ @@ -109,11 +115,11 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do pid(), pid(), Runtime.intellisense_request(), - Runtime.locator() + Runtime.Runtime.parent_locators() ) :: reference() - def handle_intellisense(pid, send_to, request, base_locator) do + def handle_intellisense(pid, send_to, request, parent_locators) do ref = make_ref() - GenServer.cast(pid, {:handle_intellisense, send_to, ref, request, base_locator}) + GenServer.cast(pid, {:handle_intellisense, send_to, ref, request, parent_locators}) ref end @@ -144,18 +150,22 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do String.t(), Runtime.smart_cell_ref(), Runtime.smart_cell_attrs(), - Runtime.locator() + Runtime.Runtime.parent_locators() ) :: :ok - def start_smart_cell(pid, kind, ref, attrs, base_locator) do - GenServer.cast(pid, {:start_smart_cell, kind, ref, attrs, base_locator}) + def start_smart_cell(pid, kind, ref, attrs, parent_locators) do + GenServer.cast(pid, {:start_smart_cell, kind, ref, attrs, parent_locators}) end @doc """ - Updates the locator with smart cell context. + Updates the parent locator used by a smart cell as its context. """ - @spec set_smart_cell_base_locator(pid(), Runtime.smart_cell_ref(), Runtime.locator()) :: :ok - def set_smart_cell_base_locator(pid, ref, base_locator) do - GenServer.cast(pid, {:set_smart_cell_base_locator, ref, base_locator}) + @spec set_smart_cell_parent_locators( + pid(), + Runtime.smart_cell_ref(), + Runtime.Runtime.parent_locators() + ) :: :ok + def set_smart_cell_parent_locators(pid, ref, parent_locators) do + GenServer.cast(pid, {:set_smart_cell_parent_locators, ref, parent_locators}) end @doc """ @@ -332,25 +342,12 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do end def handle_cast( - {:evaluate_code, code, {container_ref, evaluation_ref} = locator, base_locator, opts}, + {:evaluate_code, code, {container_ref, evaluation_ref} = locator, parent_locators, opts}, state ) do state = ensure_evaluator(state, container_ref) - base_evaluation_ref = - case base_locator do - {^container_ref, evaluation_ref} -> - evaluation_ref - - {parent_container_ref, evaluation_ref} -> - Evaluator.initialize_from( - state.evaluators[container_ref], - state.evaluators[parent_container_ref], - evaluation_ref - ) - - nil - end + parent_evaluation_refs = evaluation_refs_for_container(state, container_ref, parent_locators) {smart_cell_ref, opts} = Keyword.pop(opts, :smart_cell_ref) smart_cell_info = smart_cell_ref && state.smart_cells[smart_cell_ref] @@ -374,7 +371,7 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do state.evaluators[container_ref], code, evaluation_ref, - base_evaluation_ref, + parent_evaluation_refs, opts ) @@ -394,15 +391,31 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do {:noreply, state} end - def handle_cast({:handle_intellisense, send_to, ref, request, base_locator}, state) do - {container_ref, evaluation_ref} = base_locator - evaluator = state.evaluators[container_ref] + def handle_cast({:handle_intellisense, send_to, ref, request, parent_locators}, state) do + {container_ref, parent_evaluation_refs} = + case parent_locators do + [] -> + {nil, []} + + [{container_ref, _} | _] -> + parent_evaluation_refs = + parent_locators + # If there is a parent evaluator we ignore it and use whatever + # initial context we currently have in the evaluator. We sync + # initial context only on evaluation, since it may be blocking + |> Enum.take_while(&(elem(&1, 0) == container_ref)) + |> Enum.map(&elem(&1, 1)) + + {container_ref, parent_evaluation_refs} + end + + evaluator = container_ref && state.evaluators[container_ref] intellisense_context = if evaluator == nil or elem(request, 0) in [:format] do Evaluator.intellisense_context() else - Evaluator.intellisense_context(evaluator, evaluation_ref) + Evaluator.intellisense_context(evaluator, parent_evaluation_refs) end Task.Supervisor.start_child(state.task_supervisor, fn -> @@ -413,7 +426,7 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do {:noreply, state} end - def handle_cast({:start_smart_cell, kind, ref, attrs, base_locator}, state) do + def handle_cast({:start_smart_cell, kind, ref, attrs, parent_locators}, state) do definition = Enum.find(state.smart_cell_definitions, &(&1.kind == kind)) state = @@ -440,7 +453,7 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do pid: pid, monitor_ref: Process.monitor(pid), scan_binding: scan_binding, - base_locator: base_locator, + parent_locators: parent_locators, scan_binding_pending: false, scan_binding_monitor_ref: nil, scan_eval_result: scan_eval_result @@ -457,11 +470,11 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do {:noreply, state} end - def handle_cast({:set_smart_cell_base_locator, ref, base_locator}, state) do + def handle_cast({:set_smart_cell_parent_locators, ref, parent_locators}, state) do state = update_in(state.smart_cells[ref], fn - %{base_locator: ^base_locator} = info -> info - info -> scan_binding_async(ref, %{info | base_locator: base_locator}, state) + %{parent_locators: ^parent_locators} = info -> info + info -> scan_binding_async(ref, %{info | parent_locators: parent_locators}, state) end) {:noreply, state} @@ -608,12 +621,28 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do send(myself, {:scan_binding_ack, ref}) end - {container_ref, evaluation_ref} = info.base_locator - evaluator = state.evaluators[container_ref] + {container_ref, parent_evaluation_refs} = + case info.parent_locators do + [] -> + {nil, []} + + [{container_ref, _} | _] = parent_locators -> + parent_evaluation_refs = + evaluation_refs_for_container(state, container_ref, parent_locators) + + {container_ref, parent_evaluation_refs} + end + + evaluator = container_ref && state.evaluators[container_ref] worker_pid = if evaluator do - Evaluator.peek_context(evaluator, evaluation_ref, &scan_and_ack.(&1.binding, &1.env)) + Evaluator.peek_context( + evaluator, + parent_evaluation_refs, + &scan_and_ack.(&1.binding, &1.env) + ) + evaluator.pid else {:ok, pid} = @@ -631,6 +660,26 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do %{info | scan_binding_pending: false, scan_binding_monitor_ref: monitor_ref} end + defp evaluation_refs_for_container(state, container_ref, locators) do + case Enum.split_while(locators, &(elem(&1, 0) == container_ref)) do + {locators, []} -> + Enum.map(locators, &elem(&1, 1)) + + {locators, [{source_container_ref, _} | _] = source_locators} -> + source_evaluation_refs = Enum.map(source_locators, &elem(&1, 1)) + + evaluator = state.evaluators[container_ref] + source_evaluator = state.evaluators[source_container_ref] + + if evaluator && source_evaluator do + # Synchronize initial state in the child evaluator + Evaluator.initialize_from(evaluator, source_evaluator, source_evaluation_refs) + end + + Enum.map(locators, &elem(&1, 1)) + end + end + defp finish_scan_binding(ref, state) do update_in(state.smart_cells[ref], fn info -> Process.demonitor(info.scan_binding_monitor_ref, [:flush]) @@ -646,12 +695,12 @@ defmodule Livebook.Runtime.ErlDist.RuntimeServer do defp scan_binding_after_evaluation(state, locator) do update_in(state.smart_cells, fn smart_cells -> - Map.new(smart_cells, fn - {ref, %{base_locator: ^locator} = info} -> + Map.new(smart_cells, fn {ref, info} -> + if locator in info.parent_locators do {ref, scan_binding_async(ref, info, state)} - - other -> - other + else + {ref, info} + end end) end) end diff --git a/lib/livebook/runtime/evaluator.ex b/lib/livebook/runtime/evaluator.ex index 5635c9f97..fe0c30fd5 100644 --- a/lib/livebook/runtime/evaluator.ex +++ b/lib/livebook/runtime/evaluator.ex @@ -56,10 +56,12 @@ defmodule Livebook.Runtime.Evaluator do {:ok, result :: any()} | {:error, Exception.kind(), error :: any(), Exception.stacktrace()} - # We store evaluation envs in the process dictionary, so that we - # can build intellisense context without asking the evaluator - @env_key :evaluation_env - @initial_env_key :initial_env + # We store some information in the process dictionary for non-blocking + # access from other evaluators. In particular we store context metadata, + # such as envs, this way we can build intellisense context without + # asking the evaluator. We don't store binding though, because that + # would take too much memory + @evaluator_info_key :evaluator_info @doc """ Starts an evaluator. @@ -117,10 +119,10 @@ defmodule Livebook.Runtime.Evaluator do Any exceptions are captured and transformed into an error result. - The resulting contxt (binding and env) is stored under `ref`. - Any subsequent calls may specify `base_ref` pointing to a - previous evaluation, in which case the corresponding context - is used as the entry point for evaluation. + The resulting context (binding and env) is stored under `ref`. Any + subsequent calls may specify `parent_refs` pointing to a sequence + of previous evaluations, in which case the corresponding context is + used as the entry point for evaluation. The evaluation result is transformed with the configured formatter send to the configured client (see `start_link/1`). @@ -134,37 +136,29 @@ defmodule Livebook.Runtime.Evaluator do finished. The function receives `t:evaluation_result/0` as an argument """ - @spec evaluate_code(t(), String.t(), ref(), ref() | nil, keyword()) :: :ok - def evaluate_code(evaluator, code, ref, base_ref \\ nil, opts \\ []) when ref != nil do - cast(evaluator, {:evaluate_code, code, ref, base_ref, opts}) + @spec evaluate_code(t(), String.t(), ref(), list(ref()), keyword()) :: :ok + def evaluate_code(evaluator, code, ref, parent_refs, opts \\ []) do + cast(evaluator, {:evaluate_code, code, ref, parent_refs, opts}) end @doc """ Fetches the evaluation context (binding and env) for the given - evaluation reference. - - ## Options - - * `:cached_id` - id of context that the sender may already have, - if it matches the fetched context, `{:error, :not_modified}` - is returned instead + evaluation sequence. """ - @spec fetch_evaluation_context(t(), ref(), keyword()) :: - {:ok, context()} | {:error, :not_modified} - def fetch_evaluation_context(evaluator, ref, opts \\ []) do - cached_id = opts[:cached_id] - call(evaluator, {:fetch_evaluation_context, ref, cached_id}) + @spec get_evaluation_context(t(), list(ref())) :: context() + def get_evaluation_context(evaluator, parent_refs) do + call(evaluator, {:get_evaluation_context, parent_refs}) end @doc """ - Fetches an evaluation context from `source_evaluator` and configures - it as the initial context for `evaluator`. + Fetches an aggregated evaluation context from `source_evaluator` + and caches it as the initial context for `evaluator`. The process dictionary is also copied to match `source_evaluator`. """ @spec initialize_from(t(), t(), ref()) :: :ok - def initialize_from(evaluator, source_evaluator, source_evaluation_ref) do - call(evaluator, {:initialize_from, source_evaluator, source_evaluation_ref}) + def initialize_from(evaluator, source_evaluator, source_parent_refs) do + call(evaluator, {:initialize_from, source_evaluator, source_parent_refs}) end @doc """ @@ -190,15 +184,22 @@ defmodule Livebook.Runtime.Evaluator do @doc """ Builds intellisense context from the given evaluation. """ - @spec intellisense_context(t(), ref()) :: Livebook.Intellisense.intellisense_context() - def intellisense_context(evaluator, ref) do + @spec intellisense_context(t(), list(ref())) :: Livebook.Intellisense.intellisense_context() + def intellisense_context(evaluator, parent_refs) do {:dictionary, dictionary} = Process.info(evaluator.pid, :dictionary) - env = - find_in_dictionary(dictionary, {@env_key, ref}) || - find_in_dictionary(dictionary, @initial_env_key) + evaluator_info = find_in_dictionary(dictionary, @evaluator_info_key) + %{initial_context: {_id, initial_env}} = evaluator_info - map_binding = fn fun -> map_binding(evaluator, ref, fun) end + env = + List.foldr(parent_refs, initial_env, fn ref, prev_env -> + case evaluator_info.contexts do + %{^ref => {_id, env}} -> merge_env(prev_env, env) + _ -> prev_env + end + end) + + map_binding = fn fun -> map_binding(evaluator, parent_refs, fun) end %{env: env, map_binding: map_binding} end @@ -211,8 +212,8 @@ defmodule Livebook.Runtime.Evaluator do end # Applies the given function to evaluation binding - defp map_binding(evaluator, ref, fun) do - call(evaluator, {:map_binding, ref, fun}) + defp map_binding(evaluator, parent_refs, fun) do + call(evaluator, {:map_binding, parent_refs, fun}) end @doc """ @@ -221,9 +222,9 @@ defmodule Livebook.Runtime.Evaluator do Ths function runs within the evaluator process, so that no data is copied between processes, unless explicitly sent. """ - @spec peek_context(t(), ref(), (context() -> any())) :: :ok - def peek_context(evaluator, ref, fun) do - cast(evaluator, {:peek_context, ref, fun}) + @spec peek_context(t(), list(ref()), (context() -> any())) :: :ok + def peek_context(evaluator, parent_refs, fun) do + cast(evaluator, {:peek_context, parent_refs, fun}) end defp cast(evaluator, message) do @@ -271,7 +272,13 @@ defmodule Livebook.Runtime.Evaluator do evaluator = %{pid: self(), ref: evaluator_ref} context = initial_context() - Process.put(@initial_env_key, context.env) + + Process.put(@evaluator_info_key, %{ + initial_context: {context.id, context.env}, + contexts: %{} + }) + + ignored_pdict_keys = Process.get_keys() |> MapSet.new() state = %{ evaluator_ref: evaluator_ref, @@ -281,7 +288,9 @@ defmodule Livebook.Runtime.Evaluator do runtime_broadcast_to: runtime_broadcast_to, object_tracker: object_tracker, contexts: %{}, - initial_context: context + initial_context: context, + initial_context_version: nil, + ignored_pdict_keys: ignored_pdict_keys } :proc_lib.init_ack(evaluator) @@ -304,34 +313,53 @@ defmodule Livebook.Runtime.Evaluator do defp initial_context() do env = Code.env_for_eval([]) - %{binding: [], env: env, id: random_id()} + env = Macro.Env.prepend_tracer(env, Evaluator.Tracer) + %{id: random_id(), binding: [], env: env, pdict: %{}} end - defp handle_cast({:evaluate_code, code, ref, base_ref, opts}, state) do + defp handle_cast({:evaluate_code, code, ref, parent_refs, opts}, state) do Evaluator.ObjectTracker.remove_reference_sync(state.object_tracker, {self(), ref}) - context = get_context(state, base_ref) + context = get_context(state, parent_refs) file = Keyword.get(opts, :file, "nofile") context = put_in(context.env.file, file) - start_time = System.monotonic_time() Evaluator.IOProxy.configure(state.io_proxy, ref, file) - {result_context, result, code_error} = - case eval(code, context.binding, context.env) do + set_pdict(context, state.ignored_pdict_keys) + + start_time = System.monotonic_time() + eval_result = eval(code, context.binding, context.env) + evaluation_time_ms = time_diff_ms(start_time) + + {result_context, result, code_error, identifiers_used, identifiers_defined} = + case eval_result do {:ok, value, binding, env} -> - binding = reorder_binding(binding, context.binding) - result_context = %{binding: binding, env: env, id: random_id()} + tracer_info = Evaluator.IOProxy.get_tracer_info(state.io_proxy) + context_id = random_id() + + result_context = %{ + id: context_id, + binding: binding, + env: prune_env(env, tracer_info), + pdict: current_pdict(state) + } + + {identifiers_used, identifiers_defined} = + identifier_dependencies(result_context, tracer_info, context) + result = {:ok, value} - {result_context, result, nil} + {result_context, result, nil, identifiers_used, identifiers_defined} {:error, kind, error, stacktrace, code_error} -> result = {:error, kind, error, stacktrace} - {context, result, code_error} + identifiers_used = :unknown + identifiers_defined = %{} + # Empty context + result_context = initial_context() + {result_context, result, code_error, identifiers_used, identifiers_defined} end - evaluation_time_ms = get_execution_time_delta(start_time) - state = put_context(state, ref, result_context) Evaluator.IOProxy.flush(state.io_proxy) @@ -342,7 +370,9 @@ defmodule Livebook.Runtime.Evaluator do metadata = %{ evaluation_time_ms: evaluation_time_ms, memory_usage: memory(), - code_error: code_error + code_error: code_error, + identifiers_used: identifiers_used, + identifiers_defined: identifiers_defined } send(state.send_to, {:runtime_evaluation_response, ref, output, metadata}) @@ -363,71 +393,153 @@ defmodule Livebook.Runtime.Evaluator do {:noreply, state} end - defp handle_cast({:peek_context, ref, fun}, state) do - context = get_context(state, ref) + defp handle_cast({:peek_context, parent_refs, fun}, state) do + context = get_context(state, parent_refs) fun.(context) {:noreply, state} end - defp handle_call({:fetch_evaluation_context, ref, cached_id}, _from, state) do - context = get_context(state, ref) - - reply = - if context.id == cached_id do - {:error, :not_modified} - else - {:ok, context} - end - - {:reply, reply, state} + defp handle_call({:get_evaluation_context, parent_refs}, _from, state) do + context = get_context(state, parent_refs) + {:reply, context, state} end - defp handle_call({:initialize_from, source_evaluator, source_evaluation_ref}, _from, state) do + defp handle_call({:initialize_from, source_evaluator, source_parent_refs}, _from, state) do + {:dictionary, dictionary} = Process.info(source_evaluator.pid, :dictionary) + + evaluator_info = find_in_dictionary(dictionary, @evaluator_info_key) + + version = + source_parent_refs + |> Enum.map(fn ref -> + with {id, _env} <- evaluator_info.contexts[ref], do: id + end) + |> :erlang.md5() + state = - case Evaluator.fetch_evaluation_context( - source_evaluator, - source_evaluation_ref, - cached_id: state.initial_context.id - ) do - {:ok, context} -> - # If the context changed, mirror the process dictionary again - copy_process_dictionary_from(source_evaluator) + if version == state.initial_context_version do + state + else + context = Evaluator.get_evaluation_context(source_evaluator, source_parent_refs) - Process.put(@initial_env_key, context.env) - put_in(state.initial_context, context) + update_evaluator_info(fn info -> + put_in(info.initial_context, {context.id, context.env}) + end) - {:error, :not_modified} -> - state + %{state | initial_context: context, initial_context_version: version} end {:reply, :ok, state} end - defp handle_call({:map_binding, ref, fun}, _from, state) do - context = get_context(state, ref) + defp handle_call({:map_binding, parent_refs, fun}, _from, state) do + context = get_context(state, parent_refs) result = fun.(context.binding) {:reply, result, state} end defp put_context(state, ref, context) do - Process.put({@env_key, ref}, context.env) + update_evaluator_info(fn info -> + put_in(info.contexts[ref], {context.id, context.env}) + end) + put_in(state.contexts[ref], context) end defp delete_context(state, ref) do - Process.delete({@env_key, ref}) + update_evaluator_info(fn info -> + {_, info} = pop_in(info.contexts[ref]) + info + end) + {_, state} = pop_in(state.contexts[ref]) state end - defp get_context(state, ref) do - Map.get_lazy(state.contexts, ref, fn -> state.initial_context end) + defp update_evaluator_info(fun) do + info = Process.get(@evaluator_info_key) + Process.put(@evaluator_info_key, fun.(info)) end + defp get_context(state, parent_refs) do + List.foldr(parent_refs, state.initial_context, fn ref, prev_context -> + if context = state.contexts[ref] do + merge_context(prev_context, context) + else + prev_context + end + end) + end + + defp set_pdict(context, ignored_pdict_keys) do + for key <- Process.get_keys(), + key not in ignored_pdict_keys, + not Map.has_key?(context.pdict, key) do + Process.delete(key) + end + + for {key, value} <- context.pdict do + Process.put(key, value) + end + end + + defp current_pdict(state) do + for {key, value} <- Process.get(), + key not in state.ignored_pdict_keys, + do: {key, value}, + into: %{} + end + + defp prune_env(env, tracer_info) do + env + |> Map.replace!(:aliases, Map.to_list(tracer_info.aliases_defined)) + |> Map.replace!(:requires, MapSet.to_list(tracer_info.requires_defined)) + end + + defp merge_context(prev_context, context) do + binding = merge_binding(prev_context.binding, context.binding) + env = merge_env(prev_context.env, context.env) + pdict = context.pdict + %{id: random_id(), binding: binding, env: env, pdict: pdict} + end + + defp merge_binding(prev_binding, binding) do + binding_map = Map.new(binding) + + kept_binding = + Enum.reject(prev_binding, fn {var, _value} -> + Map.has_key?(binding_map, var) + end) + + binding ++ kept_binding + end + + defp merge_env(prev_env, env) do + env + |> Map.update!(:versioned_vars, fn versioned_vars -> + Enum.uniq(Map.keys(prev_env.versioned_vars) ++ Map.keys(versioned_vars)) + |> Enum.with_index() + |> Map.new() + end) + |> Map.update!(:aliases, &Keyword.merge(prev_env.aliases, &1)) + |> Map.update!(:requires, &:ordsets.union(prev_env.requires, &1)) + |> Map.replace!(:context_modules, []) + end + + @compile {:no_warn_undefined, {Code, :eval_quoted_with_env, 4}} + defp eval(code, binding, env) do try do quoted = Code.string_to_quoted!(code, file: env.file) - {value, binding, env} = Code.eval_quoted_with_env(quoted, binding, env) + + # TODO: remove the else branch when we require Elixir v1.14.2 + {value, binding, env} = + if function_exported?(Code, :eval_quoted_with_env, 4) do + Code.eval_quoted_with_env(quoted, binding, env, prune_binding: true) + else + Code.eval_quoted_with_env(quoted, binding, env) + end + {:ok, value, binding, env} catch kind, error -> @@ -449,22 +561,129 @@ defmodule Livebook.Runtime.Evaluator do defp code_error?(%CompileError{}), do: true defp code_error?(_error), do: false - defp reorder_binding(binding, prev_binding) do - # We keep the order of existing binding entries and move the new - # ones to the beginning + defp identifier_dependencies(context, tracer_info, prev_context) do + identifiers_used = MapSet.new() + identifiers_defined = %{} - binding_map = Map.new(binding) + # Variables - unchanged_binding = - Enum.filter(prev_binding, fn {key, prev_val} -> - val = binding_map[key] - :erts_debug.same(val, prev_val) - end) + identifiers_used = + for var <- vars_used(context, tracer_info, prev_context), + do: {:variable, var}, + into: identifiers_used - unchanged_binding - |> Enum.reduce(binding_map, fn {key, _}, acc -> Map.delete(acc, key) end) - |> Map.to_list() - |> Kernel.++(unchanged_binding) + identifiers_used = + for var <- tracer_info.undefined_vars, + do: {:variable, var}, + into: identifiers_used + + identifiers_defined = + for var <- vars_defined(context, prev_context), + do: {{:variable, var}, context.id}, + into: identifiers_defined + + # Modules + + identifiers_used = + for module <- tracer_info.modules_used, + do: {:module, module}, + into: identifiers_used + + identifiers_defined = + for {module, {version, _vars}} <- tracer_info.modules_defined, + do: {{:module, module}, version}, + into: identifiers_defined + + # Aliases + + identifiers_used = + for alias <- tracer_info.aliases_used, + do: {:alias, alias}, + into: identifiers_used + + identifiers_defined = + for {as, alias} <- tracer_info.aliases_defined, + do: {{:alias, as}, alias}, + into: identifiers_defined + + # Requires + + identifiers_used = + for module <- tracer_info.requires_used, + do: {:require, module}, + into: identifiers_used + + identifiers_defined = + for module <- tracer_info.requires_defined, + do: {{:require, module}, :ok}, + into: identifiers_defined + + # Imports + + identifiers_used = + if tracer_info.imports_used? or tracer_info.imports_defined? do + # Imports are not always incremental, due to :except, so if + # we define imports, we also implicitly rely on prior imports + MapSet.put(identifiers_used, :imports) + else + identifiers_used + end + + identifiers_defined = + if tracer_info.imports_defined? do + version = {:erlang.phash2(context.env.functions), :erlang.phash2(context.env.macros)} + put_in(identifiers_defined[:imports], version) + else + identifiers_defined + end + + # Process dictionary + + # Every evaluation depends on the pdict + identifiers_used = MapSet.put(identifiers_used, :pdict) + + identifiers_defined = + if context.pdict == prev_context.pdict do + identifiers_defined + else + version = :erlang.phash2(context.pdict) + put_in(identifiers_defined[:pdict], version) + end + + {MapSet.to_list(identifiers_used), identifiers_defined} + end + + defp vars_used(context, tracer_info, prev_context) do + prev_vars = + for {var, _version} <- prev_context.env.versioned_vars, + into: MapSet.new(), + do: var + + outer_used_vars = + for {var, _version} <- context.env.versioned_vars, + into: MapSet.new(), + do: var + + # Note that :prune_binding removes variables used by modules + # (unless used outside), so we get those from the tracer + module_used_vars = + for {_module, {_version, vars}} <- tracer_info.modules_defined, + var <- vars, + into: MapSet.new(), + do: var + + # We take an intersection with previous vars, so we ignore variables + # that we know are newly defined + MapSet.intersection(prev_vars, MapSet.union(outer_used_vars, module_used_vars)) + end + + defp vars_defined(context, prev_context) do + prev_num_vars = map_size(prev_context.env.versioned_vars) + + for {var, version} <- context.env.versioned_vars, + version >= prev_num_vars, + into: MapSet.new(), + do: var end # Adapted from https://github.com/elixir-lang/elixir/blob/1c1654c88adfdbef38ff07fc30f6fbd34a542c07/lib/iex/lib/iex/evaluator.ex#L355-L372 @@ -506,20 +725,7 @@ defmodule Livebook.Runtime.Evaluator do :crypto.strong_rand_bytes(20) |> Base.encode32(case: :lower) end - defp copy_process_dictionary_from(source_evaluator) do - {:dictionary, dictionary} = Process.info(source_evaluator.pid, :dictionary) - - for {key, value} <- dictionary, not internal_dictionary_key?(key) do - Process.put(key, value) - end - end - - defp internal_dictionary_key?("$" <> _), do: true - defp internal_dictionary_key?({@env_key, _ref}), do: true - defp internal_dictionary_key?(@initial_env_key), do: true - defp internal_dictionary_key?(_), do: false - - defp get_execution_time_delta(started_at) do + defp time_diff_ms(started_at) do System.monotonic_time() |> Kernel.-(started_at) |> System.convert_time_unit(:native, :millisecond) diff --git a/lib/livebook/runtime/evaluator/io_proxy.ex b/lib/livebook/runtime/evaluator/io_proxy.ex index bdc3e1556..ac549568c 100644 --- a/lib/livebook/runtime/evaluator/io_proxy.ex +++ b/lib/livebook/runtime/evaluator/io_proxy.ex @@ -61,11 +61,19 @@ defmodule Livebook.Runtime.Evaluator.IOProxy do end @doc """ - Returns the accumulated widget pids and clears the accumulator. + Updates tracer info. """ - @spec flush_widgets(pid()) :: MapSet.t(pid()) - def flush_widgets(pid) do - GenServer.call(pid, :flush_widgets) + @spec tracer_updates(pid(), list()) :: :ok + def tracer_updates(pid, updates) do + GenServer.cast(pid, {:tracer_updates, updates}) + end + + @doc """ + Returns the accumulated tracer info. + """ + @spec get_tracer_info(pid()) :: %Evaluator.Tracer{} + def get_tracer_info(pid) do + GenServer.call(pid, :get_tracer_info) end @impl true @@ -81,24 +89,34 @@ defmodule Livebook.Runtime.Evaluator.IOProxy do evaluator: evaluator, send_to: send_to, runtime_broadcast_to: runtime_broadcast_to, - object_tracker: object_tracker + object_tracker: object_tracker, + tracer_info: %Evaluator.Tracer{} }} end @impl true def handle_cast({:configure, ref, file}, state) do - {:noreply, %{state | ref: ref, file: file, token_count: 0}} + {:noreply, %{state | ref: ref, file: file, token_count: 0, tracer_info: %Evaluator.Tracer{}}} end def handle_cast(:clear_input_cache, state) do {:noreply, %{state | input_cache: %{}}} end + def handle_cast({:tracer_updates, updates}, state) do + state = update_in(state.tracer_info, &Evaluator.Tracer.apply_updates(&1, updates)) + {:noreply, state} + end + @impl true def handle_call(:flush, _from, state) do {:reply, :ok, flush_buffer(state)} end + def handle_call(:get_tracer_info, _from, state) do + {:reply, state.tracer_info, state} + end + @impl true def handle_info({:io_request, from, reply_as, req}, state) do {reply, state} = io_request(req, state) diff --git a/lib/livebook/runtime/evaluator/tracer.ex b/lib/livebook/runtime/evaluator/tracer.ex new file mode 100644 index 000000000..37eacfce5 --- /dev/null +++ b/lib/livebook/runtime/evaluator/tracer.ex @@ -0,0 +1,132 @@ +defmodule Livebook.Runtime.Evaluator.Tracer do + @moduledoc false + + # Compilation tracer used by the evaluator. + # + # Events are pre-processed and sent to the group leader, where the + # tracer state is accumulated. After evaluation the evaluator reads + # the accumulated state. + + alias Livebook.Runtime.Evaluator + + defstruct modules_used: MapSet.new(), + modules_defined: %{}, + aliases_used: MapSet.new(), + aliases_defined: %{}, + requires_used: MapSet.new(), + requires_defined: MapSet.new(), + imports_used?: false, + imports_defined?: false, + undefined_vars: MapSet.new() + + @doc false + def trace(event, env) do + case to_updates(event, env) do + [] -> + :ok + + updates -> + io_proxy = Process.group_leader() + Evaluator.IOProxy.tracer_updates(io_proxy, updates) + end + + :ok + end + + defp to_updates(event, env) do + # Note that import/require/alias/defmodule don't trigger `:alias_reference` + # for the used alias, so we add it explicitly + + case event do + {:import, _meta, module, _opts} -> + if(env.module, do: [], else: [:import_defined]) ++ + [{:module_used, module}, {:alias_used, module}] + + {:imported_function, meta, module, name, _arity} -> + var? = Keyword.has_key?(meta, :if_undefined) + [{:module_used, module}, {:import_used, name, var?}] + + {:imported_macro, meta, module, name, _arity} -> + var? = Keyword.has_key?(meta, :if_undefined) + [{:module_used, module}, {:import_used, name, var?}] + + {:alias, _meta, alias, as, _opts} -> + if(env.module, do: [], else: [{:alias_defined, as, alias}]) ++ + [{:alias_used, alias}] + + {:alias_expansion, _meta, as, _alias} -> + [{:alias_used, as}] + + {:alias_reference, _meta, alias} -> + [{:alias_used, alias}] + + {:require, _meta, module, _opts} -> + if(env.module, do: [], else: [{:require_defined, module}]) ++ + [{:module_used, module}, {:alias_used, module}] + + {:struct_expansion, _meta, module, _keys} -> + [{:module_used, module}] + + {:remote_function, _meta, module, _name, _arity} -> + [{:module_used, module}] + + {:remote_macro, _meta, module, _name, _arity} -> + [{:module_used, module}, {:require_used, module}] + + {:on_module, bytecode, _ignore} -> + module = env.module + version = :erlang.md5(bytecode) + vars = Map.keys(env.versioned_vars) + [{:module_defined, module, version, vars}, {:alias_used, module}] + + _ -> + [] + end + end + + @doc """ + Applies updates to the tracer state. + """ + @spec apply_updates(%__MODULE__{}, list()) :: %__MODULE__{} + def apply_updates(info, updates) do + Enum.reduce(updates, info, &apply_update(&2, &1)) + end + + defp apply_update(info, {:module_used, module}) do + update_in(info.modules_used, &MapSet.put(&1, module)) + end + + defp apply_update(info, {:module_defined, module, version, vars}) do + put_in(info.modules_defined[module], {version, vars}) + end + + defp apply_update(info, {:alias_used, alias}) do + update_in(info.aliases_used, &MapSet.put(&1, alias)) + end + + defp apply_update(info, {:alias_defined, as, alias}) do + put_in(info.aliases_defined[as], alias) + end + + defp apply_update(info, {:require_used, module}) do + update_in(info.requires_used, &MapSet.put(&1, module)) + end + + defp apply_update(info, {:require_defined, module}) do + update_in(info.requires_defined, &MapSet.put(&1, module)) + end + + defp apply_update(info, {:import_used, name, var?}) do + info = put_in(info.imports_used?, true) + + if var? do + update_in(info.undefined_vars, &MapSet.put(&1, {name, nil})) + else + info + end + end + + defp apply_update(info, :import_defined) do + put_in(info.imports_defined?, true) + end +end diff --git a/lib/livebook/session.ex b/lib/livebook/session.ex index 5e2d074c2..cb9d52510 100644 --- a/lib/livebook/session.ex +++ b/lib/livebook/session.ex @@ -29,7 +29,7 @@ defmodule Livebook.Session do # the evaluation context from the parent section, the last context # needs to be copied from the main flow evaluator to the branching # section evaluator. The latter synchronously asks the former for - # that context using `Livebook.Runtime.Evaluator.fetch_evaluation_context/3`. + # that context using `Livebook.Runtime.Evaluator.get_evaluation_context/3`. # Consequently, in order to evaluate the first cell in a branching # section, the main flow needs to be free of work, otherwise we wait. # This assumptions are mirrored in by `Livebook.Session.Data` when @@ -1040,7 +1040,7 @@ defmodule Livebook.Session do def handle_info({:runtime_evaluation_input, cell_id, reply_to, input_id}, state) do {reply, state} = with {:ok, cell, _section} <- Notebook.fetch_cell_and_section(state.data.notebook, cell_id), - {:ok, value} <- Map.fetch(state.data.input_values, input_id) do + {:ok, value} <- Data.fetch_input_value_for_cell(state.data, input_id, cell_id) do state = handle_operation(state, {:bind_input, @client_id, cell.id, input_id}) {{:ok, value}, state} else @@ -1512,27 +1512,26 @@ defmodule Livebook.Session do state end - defp handle_action(state, {:start_smart_cell, cell, section}) do + defp handle_action(state, {:start_smart_cell, cell, _section}) do if Runtime.connected?(state.data.runtime) do - base_locator = find_base_locator(state.data, cell, section, existing: true) - Runtime.start_smart_cell(state.data.runtime, cell.kind, cell.id, cell.attrs, base_locator) + parent_locators = parent_locators_for_cell(state.data, cell) + + Runtime.start_smart_cell( + state.data.runtime, + cell.kind, + cell.id, + cell.attrs, + parent_locators + ) end state end - defp handle_action(state, {:set_smart_cell_base, cell, section, parent}) do + defp handle_action(state, {:set_smart_cell_parents, cell, _section, parents}) do if Runtime.connected?(state.data.runtime) do - base_locator = - case parent do - nil -> - {container_ref_for_section(section), nil} - - {parent_cell, parent_section} -> - {container_ref_for_section(parent_section), parent_cell.id} - end - - Runtime.set_smart_cell_base_locator(state.data.runtime, cell.id, base_locator) + parent_locators = evaluation_parents_to_locators(parents) + Runtime.set_smart_cell_parent_locators(state.data.runtime, cell.id, parent_locators) end state @@ -1566,8 +1565,8 @@ defmodule Livebook.Session do opts = [file: file, smart_cell_ref: smart_cell_ref] locator = {container_ref_for_section(section), cell.id} - base_locator = find_base_locator(state.data, cell, section) - Runtime.evaluate_code(state.data.runtime, cell.source, locator, base_locator, opts) + parent_locators = parent_locators_for_cell(state.data, cell) + Runtime.evaluate_code(state.data.runtime, cell.source, locator, parent_locators, opts) evaluation_digest = :erlang.md5(cell.source) handle_operation(state, {:evaluation_started, @client_id, cell.id, evaluation_digest}) @@ -1799,34 +1798,21 @@ defmodule Livebook.Session do end @doc """ - Finds evaluation locator that the given cell depends on. + Returns locators of evaluation parents for the given cell. - By default looks up the direct evaluation parent. - - ## Options - - * `:existing` - considers only cells that have been evaluated - as evaluation parents. Defaults to `false` + Considers only cells that have already been evaluated. """ - @spec find_base_locator(Data.t(), Cell.t(), Section.t(), keyword()) :: Runtime.locator() - def find_base_locator(data, cell, section, opts \\ []) do - parent_filter = - if opts[:existing] do - fn cell -> - info = data.cell_infos[cell.id] - Cell.evaluable?(cell) and info.eval.validity in [:evaluated, :stale] - end - else - &Cell.evaluable?/1 - end + @spec parent_locators_for_cell(Data.t(), Cell.t()) :: Runtime.parent_locators() + def parent_locators_for_cell(data, cell) do + data + |> Data.cell_evaluation_parents(cell) + |> evaluation_parents_to_locators() + end - default = {container_ref_for_section(section), nil} - - data.notebook - |> Notebook.parent_cells_with_section(cell.id) - |> Enum.find_value(default, fn {cell, section} -> - parent_filter.(cell) && {container_ref_for_section(section), cell.id} - end) + defp evaluation_parents_to_locators(parents) do + for {cell, section} <- parents do + {container_ref_for_section(section), cell.id} + end end defp container_ref_for_section(%{parent_id: nil}), do: @main_container_ref diff --git a/lib/livebook/session/data.ex b/lib/livebook/session/data.ex index 17e35c7fb..0b5197d7d 100644 --- a/lib/livebook/session/data.ex +++ b/lib/livebook/session/data.ex @@ -57,7 +57,7 @@ defmodule Livebook.Session.Data do @type section_info :: %{ evaluating_cell_id: Cell.id(), - evaluation_queue: list(Cell.id()) + evaluation_queue: MapSet.t(Cell.id()) } @type cell_info :: markdown_cell_info() | code_cell_info() | smart_cell_info() @@ -96,7 +96,10 @@ defmodule Livebook.Session.Data do evaluation_number: non_neg_integer(), outputs_batch_number: non_neg_integer(), bound_to_input_ids: MapSet.t(input_id()), - bound_input_readings: input_reading() + new_bound_to_input_ids: MapSet.t(input_id()), + identifiers_used: list(identifier :: term()) | :unknown, + identifiers_defined: %{(identifier :: term()) => version :: term()}, + data: t() } @type cell_bin_entry :: %{ @@ -125,22 +128,10 @@ defmodule Livebook.Session.Data do @type secret :: %{name: String.t(), value: String.t()} # Snapshot holds information about the cell evaluation dependencies, - # for example what is the previous cell, the number of times the - # cell was evaluated, the list of available inputs, etc. Whenever - # the snapshot changes, it implies a new evaluation context, which - # basically means the cell got stale. - # - # The snapshot comprises of two actual snapshots: - # - # * `deps_snapshot` - everything related to parent cells and - # their evaluations. This is recorded once the cell starts - # evaluating - # - # * `bound_inputs_snapshot` - snapshot of the inputs and their - # values used by cell evaluation. This is recorded once the - # cell finishes its evaluation - # - @type snapshot :: {deps_snapshot :: term(), bound_inputs_snapshot :: term()} + # including parent cells and inputs. Whenever the snapshot changes, + # it implies a new evaluation context, which basically means the cell + # got stale. + @type snapshot :: term() @type input_reading :: {input_id(), input_value :: term()} @@ -202,7 +193,8 @@ defmodule Livebook.Session.Data do | {:stop_evaluation, Section.t()} | {:forget_evaluation, Cell.t(), Section.t()} | {:start_smart_cell, Cell.t(), Section.t()} - | {:set_smart_cell_base, Cell.t(), Section.t(), parent :: {Cell.t(), Section.t()} | nil} + | {:set_smart_cell_parents, Cell.t(), Section.t(), + parent :: {Cell.t(), Section.t()} | nil} | {:broadcast_delta, client_id(), Cell.t(), cell_source_tag(), Delta.t()} @doc """ @@ -329,7 +321,7 @@ defmodule Livebook.Session.Data do |> with_actions() |> cancel_section_evaluation(section) |> set_section_parent(section, parent_section) - |> compute_snapshots_and_validity() + |> update_validity_and_evaluation() |> set_dirty() |> wrap_ok() else @@ -345,7 +337,7 @@ defmodule Livebook.Session.Data do |> cancel_section_evaluation(section) |> add_action({:stop_evaluation, section}) |> unset_section_parent(section) - |> compute_snapshots_and_validity() + |> update_validity_and_evaluation() |> set_dirty() |> wrap_ok() else @@ -361,7 +353,7 @@ defmodule Livebook.Session.Data do |> with_actions() |> insert_cell(section_id, index, cell) |> maybe_start_smart_cells() - |> compute_snapshots_and_validity() + |> update_validity_and_evaluation() |> set_dirty() |> wrap_ok() end @@ -374,7 +366,7 @@ defmodule Livebook.Session.Data do data |> with_actions() |> delete_section(section, delete_cells) - |> compute_snapshots_and_validity() + |> update_validity_and_evaluation() |> update_smart_cell_bases(data) |> set_dirty() |> wrap_ok() @@ -389,7 +381,7 @@ defmodule Livebook.Session.Data do data |> with_actions() |> delete_cell(cell, section) - |> compute_snapshots_and_validity() + |> update_validity_and_evaluation() |> update_smart_cell_bases(data) |> set_dirty() |> wrap_ok() @@ -404,7 +396,7 @@ defmodule Livebook.Session.Data do data |> with_actions() |> restore_cell(cell_bin_entry) - |> compute_snapshots_and_validity() + |> update_validity_and_evaluation() |> maybe_start_smart_cells() |> set_dirty() |> wrap_ok() @@ -421,7 +413,7 @@ defmodule Livebook.Session.Data do data |> with_actions() |> move_cell(cell, offset) - |> compute_snapshots_and_validity() + |> update_validity_and_evaluation() |> update_smart_cell_bases(data) |> set_dirty() |> wrap_ok() @@ -437,7 +429,7 @@ defmodule Livebook.Session.Data do data |> with_actions() |> move_section(section, offset) - |> compute_snapshots_and_validity() + |> update_validity_and_evaluation() |> update_smart_cell_bases(data) |> set_dirty() |> wrap_ok() @@ -459,12 +451,11 @@ defmodule Livebook.Session.Data do cells_with_section |> Enum.reduce(with_actions(data), fn {cell, section}, data_actions -> data_actions - |> queue_prerequisite_cells_evaluation(cell) + |> queue_prerequisite_cells_evaluation(cell.id) |> queue_cell_evaluation(cell, section) end) |> maybe_connect_runtime(data) - |> maybe_evaluate_queued() - |> compute_snapshots_and_validity() + |> update_validity_and_evaluation() |> wrap_ok() else :error @@ -505,9 +496,7 @@ defmodule Livebook.Session.Data do |> add_cell_output(cell, output) |> finish_cell_evaluation(cell, section, metadata) |> garbage_collect_input_values() - |> compute_snapshots_and_validity() - |> maybe_evaluate_queued() - |> compute_snapshots_and_validity() + |> update_validity_and_evaluation() |> update_smart_cell_bases(data) |> mark_dirty_if_persisting_outputs() |> wrap_ok() @@ -519,8 +508,9 @@ defmodule Livebook.Session.Data do def apply_operation(data, {:bind_input, _client_id, cell_id, input_id}) do with {:ok, cell, _section} <- Notebook.fetch_cell_and_section(data.notebook, cell_id), Cell.evaluable?(cell), + :evaluating <- data.cell_infos[cell.id].eval.status, true <- Map.has_key?(data.input_values, input_id), - false <- MapSet.member?(data.cell_infos[cell.id].eval.bound_to_input_ids, input_id) do + false <- MapSet.member?(data.cell_infos[cell.id].eval.new_bound_to_input_ids, input_id) do data |> with_actions() |> bind_input(cell, input_id) @@ -715,7 +705,7 @@ defmodule Livebook.Session.Data do data |> with_actions() |> set_cell_attributes(cell, attrs) - |> compute_snapshots_and_validity() + |> update_validity_and_evaluation() |> set_dirty() |> wrap_ok() else @@ -728,7 +718,7 @@ defmodule Livebook.Session.Data do data |> with_actions() |> set_input_value(input_id, value) - |> compute_snapshots_and_validity() + |> update_validity_and_evaluation() |> wrap_ok() else _ -> :error @@ -945,8 +935,7 @@ defmodule Livebook.Session.Data do # For each path in the dependency graph, find the upmost cell # which parent changed. From that point downwards all cells - # are invalidated. Then gather invalidated cells from all paths - # and unqueue them. + # are potentially affected, so we unqueue them. invalidted_cell_ids = graph_after @@ -997,7 +986,7 @@ defmodule Livebook.Session.Data do defp queue_cell_evaluation(data_actions, cell, section) do data_actions |> update_section_info!(section.id, fn section -> - %{section | evaluation_queue: append_new(section.evaluation_queue, cell.id)} + update_in(section.evaluation_queue, &MapSet.put(&1, cell.id)) end) |> update_cell_eval_info!(cell.id, fn eval_info -> update_in(eval_info.status, fn @@ -1010,7 +999,7 @@ defmodule Livebook.Session.Data do defp unqueue_cell_evaluation(data_actions, cell, section) do data_actions |> update_section_info!(section.id, fn section -> - %{section | evaluation_queue: List.delete(section.evaluation_queue, cell.id)} + update_in(section.evaluation_queue, &MapSet.delete(&1, cell.id)) end) |> update_cell_eval_info!(cell.id, &%{&1 | status: :ready}) end @@ -1047,15 +1036,40 @@ defmodule Livebook.Session.Data do eval_info | status: :ready, evaluation_time_ms: metadata.evaluation_time_ms, - # After finished evaluation, take the snapshot of read inputs - evaluation_snapshot: - {elem(eval_info.evaluation_snapshot, 0), - input_readings_snapshot(eval_info.bound_input_readings)} + identifiers_used: metadata.identifiers_used, + identifiers_defined: metadata.identifiers_defined, + bound_to_input_ids: eval_info.new_bound_to_input_ids } end) + |> update_cell_evaluation_snapshot(cell, section) |> set_section_info!(section.id, evaluating_cell_id: nil) end + defp update_cell_evaluation_snapshot({data, _} = data_actions, cell, section) do + info = data.cell_infos[cell.id] + + eval_data = data.cell_infos[cell.id].eval.data + eval_data = put_in(eval_data.cell_infos[cell.id], info) + + graph = Notebook.cell_dependency_graph(eval_data.notebook, cell_filter: &Cell.evaluable?/1) + + cell_snapshots = + for {cell_id, %{eval: eval_info}} <- eval_data.cell_infos, + do: {cell_id, eval_info.snapshot}, + into: %{} + + # We compute evaluation snapshot based on the notebook state prior + # to evaluation, but using the information about the dependencies + # obtained during evaluation (identifiers, inputs) + evaluation_snapshot = cell_snapshot(cell, section, graph, cell_snapshots, eval_data) + + data_actions + |> update_cell_eval_info!( + cell.id, + &%{&1 | evaluation_snapshot: evaluation_snapshot, data: nil} + ) + end + defp maybe_connect_runtime({data, _} = data_actions, prev_data) do if not Runtime.connected?(data.runtime) and not any_cell_queued?(prev_data) and any_cell_queued?(data) do @@ -1066,7 +1080,26 @@ defmodule Livebook.Session.Data do end defp any_cell_queued?(data) do - Enum.any?(data.section_infos, fn {_section_id, info} -> info.evaluation_queue != [] end) + Enum.any?(data.section_infos, fn {_section_id, info} -> + not Enum.empty?(info.evaluation_queue) + end) + end + + defp queue_prerequisite_cells_evaluation_for_queued({data, _} = data_actions) do + {awaiting_branch_sections, awaiting_regular_sections} = + data.notebook + |> Notebook.all_sections() + |> Enum.filter(§ion_awaits_evaluation?(data, &1.id)) + |> Enum.split_with(& &1.parent_id) + + trailing_queued_cell_ids = + for section <- awaiting_branch_sections ++ Enum.take(awaiting_regular_sections, -1), + cell = last_queued_cell(data, section), + do: cell.id + + reduce(data_actions, trailing_queued_cell_ids, fn data_actions, cell_id -> + queue_prerequisite_cells_evaluation(data_actions, cell_id) + end) end defp maybe_evaluate_queued({data, _} = data_actions) do @@ -1081,7 +1114,7 @@ defmodule Livebook.Session.Data do data_actions = reduce(data_actions, awaiting_branch_sections, fn {data, _} = data_actions, section -> - %{evaluation_queue: [id | _]} = data.section_infos[section.id] + %{id: id} = first_queued_cell(data, section) {:ok, parent} = Notebook.fetch_section(data.notebook, section.parent_id) @@ -1094,7 +1127,7 @@ defmodule Livebook.Session.Data do prev_section_queued? = prev_cell_section != nil and - data.section_infos[prev_cell_section.id].evaluation_queue != [] + not Enum.empty?(data.section_infos[prev_cell_section.id].evaluation_queue) # If evaluating this cell requires interaction with the main flow, # we keep the cell queued. In case of the Elixir runtimes the @@ -1119,6 +1152,25 @@ defmodule Livebook.Session.Data do end end + defp first_queued_cell(data, section) do + find_queued_cell(data, section.cells) + end + + defp last_queued_cell(data, section) do + find_queued_cell(data, Enum.reverse(section.cells)) + end + + defp find_queued_cell(data, cells) do + Enum.find_value(cells, fn cell -> + info = data.cell_infos[cell.id] + + case info do + %{eval: %{status: :queued}} -> cell + _ -> nil + end + end) + end + defp main_flow_evaluating?(data) do data.notebook |> Notebook.all_sections() @@ -1142,51 +1194,51 @@ defmodule Livebook.Session.Data do defp section_awaits_evaluation?(data, section_id) do info = data.section_infos[section_id] - info.evaluating_cell_id == nil and info.evaluation_queue != [] + info.evaluating_cell_id == nil and not Enum.empty?(info.evaluation_queue) end defp evaluate_next_cell_in_section({data, _} = data_actions, section) do - case data.section_infos[section.id] do - %{evaluating_cell_id: nil, evaluation_queue: [id | ids]} -> - cell = Enum.find(section.cells, &(&1.id == id)) + section_info = data.section_infos[section.id] - data_actions - |> set!(notebook: Notebook.update_cell(data.notebook, id, &%{&1 | outputs: []})) - |> update_cell_eval_info!(id, fn eval_info -> - %{ - eval_info - | # Note: we intentionally mark the cell as evaluating up front, - # so that another queue operation doesn't cause duplicated - # :start_evaluation action - status: :evaluating, - evaluation_number: eval_info.evaluation_number + 1, - outputs_batch_number: eval_info.outputs_batch_number + 1, - evaluation_digest: nil, - evaluation_snapshot: eval_info.snapshot, - bound_to_input_ids: MapSet.new(), - bound_input_readings: [], - # This is a rough estimate, the exact time is measured in the - # evaluator itself - evaluation_start: DateTime.utc_now() - } - end) - |> set_section_info!(section.id, evaluating_cell_id: id, evaluation_queue: ids) - |> add_action({:start_evaluation, cell, section}) + if section_info.evaluating_cell_id == nil and not Enum.empty?(section_info.evaluation_queue) do + cell = first_queued_cell(data, section) - _ -> - data_actions + data_actions + |> set!(notebook: Notebook.update_cell(data.notebook, cell.id, &%{&1 | outputs: []})) + |> update_cell_eval_info!(cell.id, fn eval_info -> + %{ + eval_info + | # Note: we intentionally mark the cell as evaluating up front, + # so that another queue operation doesn't cause duplicated + # :start_evaluation action + status: :evaluating, + evaluation_number: eval_info.evaluation_number + 1, + outputs_batch_number: eval_info.outputs_batch_number + 1, + evaluation_digest: nil, + new_bound_to_input_ids: MapSet.new(), + # Keep the notebook state before evaluation + data: data, + # This is a rough estimate, the exact time is measured in the + # evaluator itself + evaluation_start: DateTime.utc_now() + } + end) + |> set_section_info!(section.id, + evaluating_cell_id: cell.id, + evaluation_queue: MapSet.delete(section_info.evaluation_queue, cell.id) + ) + |> add_action({:start_evaluation, cell, section}) + else + data_actions end end - defp bind_input({data, _} = data_actions, cell, input_id) do + defp bind_input(data_actions, cell, input_id) do data_actions |> update_cell_eval_info!(cell.id, fn eval_info -> %{ eval_info - | bound_to_input_ids: MapSet.put(eval_info.bound_to_input_ids, input_id), - bound_input_readings: [ - {input_id, data.input_values[input_id]} | eval_info.bound_input_readings - ] + | new_bound_to_input_ids: MapSet.put(eval_info.new_bound_to_input_ids, input_id) } end) end @@ -1210,7 +1262,7 @@ defmodule Livebook.Session.Data do evaluable_cells = Enum.filter(section.cells, &Cell.evaluable?/1) data_actions - |> set_section_info!(section.id, evaluating_cell_id: nil, evaluation_queue: []) + |> set_section_info!(section.id, evaluating_cell_id: nil, evaluation_queue: MapSet.new()) |> reduce( evaluable_cells, &update_cell_eval_info!(&1, &2.id, fn eval_info -> @@ -1230,14 +1282,13 @@ defmodule Livebook.Session.Data do ) end - defp queue_prerequisite_cells_evaluation({data, _} = data_actions, cell) do + defp queue_prerequisite_cells_evaluation({data, _} = data_actions, cell_id) do prerequisites_queue = data.notebook - |> Notebook.parent_cells_with_section(cell.id) - |> Enum.filter(fn {cell, _} -> Cell.evaluable?(cell) end) - |> Enum.take_while(fn {parent_cell, _section} -> - info = data.cell_infos[parent_cell.id] - info.eval.validity != :evaluated and info.eval.status == :ready + |> Notebook.parent_cells_with_section(cell_id) + |> Enum.filter(fn {cell, _section} -> + info = data.cell_infos[cell.id] + Cell.evaluable?(cell) and info.eval.validity != :evaluated and info.eval.status == :ready end) |> Enum.reverse() @@ -1263,16 +1314,20 @@ defmodule Livebook.Session.Data do :queued -> data_actions |> unqueue_cell_evaluation(cell, section) - |> unqueue_dependent_cells_evaluation(cell) + |> unqueue_child_cells_evaluation(cell) _ -> data_actions end end - defp unqueue_dependent_cells_evaluation({data, _} = data_actions, cell) do - dependent = dependent_cells_with_section(data, cell.id) - unqueue_cells_evaluation(data_actions, dependent) + defp unqueue_child_cells_evaluation({data, _} = data_actions, cell) do + evaluation_children = + data.notebook + |> Notebook.child_cells_with_section(cell.id) + |> Enum.filter(fn {cell, _} -> Cell.evaluable?(cell) end) + + unqueue_cells_evaluation(data_actions, evaluation_children) end defp unqueue_cells_evaluation({data, _} = data_actions, cells_with_section) do @@ -1337,7 +1392,7 @@ defmodule Livebook.Session.Data do if evaluated? and reevaluate do data_actions - |> queue_prerequisite_cells_evaluation(cell) + |> queue_prerequisite_cells_evaluation(cell.id) |> queue_cell_evaluation(cell, section) |> maybe_evaluate_queued() else @@ -1610,14 +1665,6 @@ defmodule Livebook.Session.Data do {data, actions ++ [action]} end - defp append_new(list, item) do - if item in list do - list - else - list ++ [item] - end - end - defp garbage_collect_input_values({data, _} = data_actions) do if any_section_evaluating?(data) do # Wait if evaluation is ongoing as it may render inputs @@ -1638,69 +1685,31 @@ defmodule Livebook.Session.Data do if Enum.empty?(alive_smart_cell_ids) do data_actions else - new_eval_graph = cell_evaluation_graph(data) - prev_eval_graph = cell_evaluation_graph(prev_data) + new_eval_parents = cell_evaluation_parents(data) + prev_eval_parents = cell_evaluation_parents(prev_data) cell_lookup = data.notebook |> Notebook.cells_with_section() |> Map.new(fn {cell, section} -> {cell.id, {cell, section}} end) - for {cell_id, parent_id} <- new_eval_graph, + for {cell_id, eval_parents} <- new_eval_parents, MapSet.member?(alive_smart_cell_ids, cell_id), - Map.has_key?(prev_eval_graph, cell_id), - prev_eval_graph[cell_id] != parent_id, + Map.has_key?(prev_eval_parents, cell_id), + prev_eval_parents[cell_id] != eval_parents, reduce: data_actions do data_actions -> {cell, section} = cell_lookup[cell_id] - parent = cell_lookup[parent_id] - add_action(data_actions, {:set_smart_cell_base, cell, section, parent}) + parents = Enum.map(eval_parents, &cell_lookup[&1]) + add_action(data_actions, {:set_smart_cell_parents, cell, section, parents}) end end end - # Builds a graph with evaluation parents, where each parent has - # aleady been evaluated. All fresh/aborted cells are leaves in - # this graph - defp cell_evaluation_graph(data) do - graph = Notebook.cell_dependency_graph(data.notebook, cell_filter: &Cell.evaluable?/1) - - graph - |> Livebook.Utils.Graph.leaves() - |> Enum.reduce(%{}, fn cell_id, eval_graph -> - build_eval_graph(data, graph, cell_id, [], eval_graph) - end) - end - - defp build_eval_graph(_data, _graph, nil, orphan_ids, eval_graph) do - put_parent(eval_graph, orphan_ids, nil) - end - - defp build_eval_graph(data, graph, cell_id, orphan_ids, eval_graph) do - # We are traversing from every leaf up, so we want to compute - # the common path only once - if eval_parent_id = eval_graph[cell_id] do - put_parent(eval_graph, orphan_ids, eval_parent_id) - else - info = data.cell_infos[cell_id] - - if info.eval.validity in [:evaluated, :stale] do - eval_graph = put_parent(eval_graph, orphan_ids, cell_id) - build_eval_graph(data, graph, graph[cell_id], [cell_id], eval_graph) - else - build_eval_graph(data, graph, graph[cell_id], [cell_id | orphan_ids], eval_graph) - end - end - end - - defp put_parent(eval_graph, cell_ids, parent_id) do - Enum.reduce(cell_ids, eval_graph, &Map.put(&2, &1, parent_id)) - end - defp new_section_info() do %{ evaluating_cell_id: nil, - evaluation_queue: [] + evaluation_queue: MapSet.new() } end @@ -1745,9 +1754,12 @@ defmodule Livebook.Session.Data do evaluation_number: 0, outputs_batch_number: 0, bound_to_input_ids: MapSet.new(), - bound_input_readings: [], - snapshot: {nil, nil}, - evaluation_snapshot: nil + new_bound_to_input_ids: MapSet.new(), + identifiers_used: [], + identifiers_defined: %{}, + snapshot: nil, + evaluation_snapshot: nil, + data: nil } end @@ -1808,6 +1820,52 @@ defmodule Livebook.Session.Data do Enum.all?(attrs, fn {key, _} -> Map.has_key?(struct, key) end) end + @doc """ + Builds evaluation parent sequence for every evaluable cell. + + This function should be used instead of calling `cell_evaluation_parents/2` + multiple times. + """ + @spec cell_evaluation_parents(t()) :: %{Cell.id() => list(Cell.id())} + def cell_evaluation_parents(data) do + graph = Notebook.cell_dependency_graph(data.notebook, cell_filter: &Cell.evaluable?/1) + + graph + |> Graph.reduce_paths({nil, %{}}, fn cell_id, {parent_id, chains} -> + if parent_id do + parent_chain = chains[parent_id] + parent_info = data.cell_infos[parent_id] + + chain = + if parent_info.eval.validity in [:evaluated, :stale] do + [parent_id | parent_chain] + else + parent_chain + end + + {cell_id, put_in(chains[cell_id], chain)} + else + {cell_id, put_in(chains[cell_id], [])} + end + end) + |> Enum.map(&elem(&1, 1)) + |> Enum.reduce(&Map.merge/2) + end + + @doc """ + Builds evaluation parent sequence for the given cell. + + Considers only cells that have already been evaluated. + """ + @spec cell_evaluation_parents(Data.t(), Cell.t()) :: list({Cell.t(), Section.t()}) + def cell_evaluation_parents(data, cell) do + for {cell, section} <- Notebook.parent_cells_with_section(data.notebook, cell.id), + info = data.cell_infos[cell.id], + Cell.evaluable?(cell), + info.eval.validity in [:evaluated, :stale], + do: {cell, section} + end + @doc """ Find cells bound to the given input. """ @@ -1821,20 +1879,16 @@ defmodule Livebook.Session.Data do end) end - defp dependent_cells_with_section(data, cell_id) do - data.notebook - |> Notebook.child_cells_with_section(cell_id) - |> Enum.filter(fn {cell, _} -> Cell.evaluable?(cell) end) - end - - # Computes cell snapshots and updates validity based on the new values. - defp compute_snapshots_and_validity(data_actions) do + # Computes cell snapshots and updates validity based on the new + # values, then triggers further evaluation if applicable. + defp update_validity_and_evaluation(data_actions) do data_actions |> compute_snapshots() |> update_validity() # After updating validity there may be new stale cells, so we check # if any of them is configured for automatic reevaluation |> maybe_queue_reevaluating_cells() + |> queue_prerequisite_cells_evaluation_for_queued() |> maybe_evaluate_queued() end @@ -1845,32 +1899,7 @@ defmodule Livebook.Session.Data do cell_snapshots = Enum.reduce(cells_with_section, %{}, fn {cell, section}, cell_snapshots -> - info = data.cell_infos[cell.id] - prev_cell_id = graph[cell.id] - - is_branch? = section.parent_id != nil - - parent_deps = - prev_cell_id && - { - prev_cell_id, - cell_snapshots[prev_cell_id], - number_of_evaluations(data.cell_infos[prev_cell_id]) - } - - deps = {is_branch?, parent_deps} - deps_snapshot = :erlang.phash2(deps) - - inputs_snapshot = - if info.eval.status == :evaluating do - # While the cell is evaluating the bound inputs snapshot - # is not stable, so we reuse the previous snapshot - elem(info.eval.snapshot, 1) - else - bound_inputs_snapshot(data, cell) - end - - snapshot = {deps_snapshot, inputs_snapshot} + snapshot = cell_snapshot(cell, section, graph, cell_snapshots, data) put_in(cell_snapshots[cell.id], snapshot) end) @@ -1882,26 +1911,102 @@ defmodule Livebook.Session.Data do end) end - defp number_of_evaluations(%{eval: %{status: :evaluating}} = info) do - info.eval.evaluation_number - 1 + defp cell_snapshot(cell, section, graph, cell_snapshots, data) do + info = data.cell_infos[cell.id] + + # Note that this is an implication of the Elixir runtime, we want + # to reevaluate as much as possible in a branch, rather than copying + # contexts between processes, because all structural sharing is + # lost when copying + is_branch? = section.parent_id != nil + + {parent_ids, identifier_versions} = identifier_deps(cell.id, graph, data) + + parent_snapshots = Enum.map(parent_ids, &cell_snapshots[&1]) + + bound_input_values = + for( + input_id <- info.eval.bound_to_input_ids, + do: {input_id, data.input_values[input_id]} + ) + |> Enum.sort() + + deps = {is_branch?, parent_snapshots, identifier_versions, bound_input_values} + + :erlang.phash2(deps) end - defp number_of_evaluations(info), do: info.eval.evaluation_number + defp identifier_deps(cell_id, graph, data) do + info = data.cell_infos[cell_id] - defp bound_inputs_snapshot(data, cell) do - %{bound_to_input_ids: bound_to_input_ids} = data.cell_infos[cell.id].eval + {parent_ids, identifier_versions} = + case info.eval.identifiers_used do + :unknown -> + all_identifier_deps(graph[cell_id], graph, data) - for( - input_id <- bound_to_input_ids, - do: {input_id, data.input_values[input_id]} - ) - |> input_readings_snapshot() + identifiers_used -> + gather_identifier_deps(graph[cell_id], identifiers_used, graph, data, {[], []}) + end + + {Enum.sort(parent_ids), Enum.sort(identifier_versions)} end - defp input_readings_snapshot([]), do: :empty + defp all_identifier_deps(cell_id, graph, data) do + parent_ids = graph |> Graph.find_path(cell_id, nil) |> Enum.drop(1) - defp input_readings_snapshot(name_value_pairs) do - name_value_pairs |> Enum.sort() |> :erlang.phash2() + identifier_versions = + parent_ids + |> List.foldr(%{}, fn cell_id, acc -> + identifiers_defined = data.cell_infos[cell_id].eval.identifiers_defined + Map.merge(acc, identifiers_defined) + end) + |> Map.to_list() + + {parent_ids, identifier_versions} + end + + defp gather_identifier_deps(nil, _identifiers_used, _graph, _data, acc), do: acc + + defp gather_identifier_deps(_cell_id, [], _graph, _data, acc), do: acc + + defp gather_identifier_deps( + cell_id, + identifiers_used, + graph, + data, + {parent_ids, identifier_versions} + ) do + identifiers_defined = data.cell_infos[cell_id].eval.identifiers_defined + + identifiers_used + |> Enum.reduce({[], []}, fn identifier, {versions, rest_identifiers} -> + case identifiers_defined do + %{^identifier => version} -> + {[{identifier, version} | versions], rest_identifiers} + + _ -> + {versions, [identifier | rest_identifiers]} + end + end) + |> case do + {[], rest_identifiers} -> + gather_identifier_deps( + graph[cell_id], + rest_identifiers, + graph, + data, + {parent_ids, identifier_versions} + ) + + {versions, rest_identifiers} -> + gather_identifier_deps( + graph[cell_id], + rest_identifiers, + graph, + data, + {[cell_id | parent_ids], versions ++ identifier_versions} + ) + end end defp update_validity({data, _} = data_actions) do @@ -1924,7 +2029,7 @@ defmodule Livebook.Session.Data do end defp maybe_queue_reevaluating_cells({data, _} = data_actions) do - cells_to_reeavaluete = + cells_to_reevaluate = data.notebook |> Notebook.evaluable_cells_with_section() |> Enum.filter(fn {cell, _section} -> @@ -1935,9 +2040,9 @@ defmodule Livebook.Session.Data do end) data_actions - |> reduce(cells_to_reeavaluete, fn data_actions, {cell, section} -> + |> reduce(cells_to_reevaluate, fn data_actions, {cell, section} -> data_actions - |> queue_prerequisite_cells_evaluation(cell) + |> queue_prerequisite_cells_evaluation(cell.id) |> queue_cell_evaluation(cell, section) end) end @@ -1959,7 +2064,7 @@ defmodule Livebook.Session.Data do Returns the list of cell ids for full evaluation. The list includes all outdated cells, cells in `forced_cell_ids` - and all of their child cells. + and all cells with identifier dependency on these. """ @spec cell_ids_for_full_evaluation(t(), list(Cell.id())) :: list(Cell.id()) def cell_ids_for_full_evaluation(data, forced_cell_ids) do @@ -1968,16 +2073,70 @@ defmodule Livebook.Session.Data do evaluable_cell_ids = for {cell, _} <- evaluable_cells_with_section, cell_outdated?(data, cell) or cell.id in forced_cell_ids, - info = data.cell_infos[cell.id], - info.eval.status == :ready, - uniq: true, - do: cell.id + do: cell.id, + into: MapSet.new() - cell_ids = Notebook.cell_ids_with_children(data.notebook, evaluable_cell_ids) + cell_identifier_parents = cell_identifier_parents(data) - for {cell, _} <- evaluable_cells_with_section, - cell.id in cell_ids, - do: cell.id + child_ids = + for {cell_id, cell_identifier_parents} <- cell_identifier_parents, + Enum.any?(cell_identifier_parents, &(&1 in evaluable_cell_ids)), + do: cell_id + + child_ids + |> Enum.into(evaluable_cell_ids) + |> Enum.to_list() + |> Enum.filter(fn cell_id -> + info = data.cell_infos[cell_id] + info.eval.status == :ready + end) + end + + # Builds identifier parent list for every evaluable cell. + # + # This is similar to cell_evaluation_parents, but the dependency is + # based on identifiers used/set by each cell. + defp cell_identifier_parents(data) do + graph = Notebook.cell_dependency_graph(data.notebook, cell_filter: &Cell.evaluable?/1) + + graph + |> Graph.reduce_paths( + {nil, %{}, %{}}, + fn cell_id, {parent_id, setters, identifier_parents} -> + if parent_id do + cell_info = data.cell_infos[cell_id] + + direct_parents = + case cell_info.eval.identifiers_used do + :unknown -> + setters |> Map.values() |> Enum.uniq() + + identifiers_used -> + for identifier <- identifiers_used, + parent_id = setters[identifier], + uniq: true, + do: parent_id + end + + parents = + for parent_id <- direct_parents, + cell_id <- [parent_id | identifier_parents[parent_id]], + uniq: true, + do: cell_id + + setters = + for {identifier, _version} <- cell_info.eval.identifiers_defined, + do: {identifier, cell_id}, + into: setters + + {cell_id, setters, put_in(identifier_parents[cell_id], parents)} + else + {cell_id, setters, put_in(identifier_parents[cell_id], [])} + end + end + ) + |> Enum.map(&elem(&1, 2)) + |> Enum.reduce(&Map.merge/2) end @doc """ @@ -2010,4 +2169,21 @@ defmodule Livebook.Session.Data do end) |> elem(0) end + + @doc """ + Fetches an input value for the given cell. + + If the cell is evaluating, the input value at evaluation start is + returned instead of the current value. + """ + @spec fetch_input_value_for_cell(t(), input_id(), Cell.id()) :: {:ok, term()} | :error + def fetch_input_value_for_cell(data, input_id, cell_id) do + data = + case data.cell_infos[cell_id] do + %{eval: %{status: :evaluating, data: data}} -> data + _ -> data + end + + Map.fetch(data.input_values, input_id) + end end diff --git a/lib/livebook/utils/graph.ex b/lib/livebook/utils/graph.ex index 1e9b184b2..fc45f8152 100644 --- a/lib/livebook/utils/graph.ex +++ b/lib/livebook/utils/graph.ex @@ -30,7 +30,7 @@ defmodule Livebook.Utils.Graph do do: find_path(graph, graph[from_id], to_id, [from_id | path]) @doc """ - Finds grpah leave nodes, that is, nodes with + Finds graph leave nodes, that is, nodes with no children. """ @spec leaves(t()) :: list(node_id()) @@ -39,4 +39,37 @@ defmodule Livebook.Utils.Graph do parents = MapSet.new(graph, fn {_, value} -> value end) MapSet.difference(children, parents) |> MapSet.to_list() end + + @doc """ + Reduces each top-down path in the graph. + + Returns a list of accumulators, one for each leaf in the graph, + in no specific order. + """ + @spec reduce_paths(t(), acc, (node_id(), acc -> acc)) :: acc when acc: term() + def reduce_paths(graph, acc, fun) do + leaves = Livebook.Utils.Graph.leaves(graph) + cache = do_reduce(graph, leaves, acc, fun, %{}) + Enum.map(leaves, &cache[&1]) + end + + defp do_reduce(_graph, [], _initial_acc, _fun, cache), do: cache + + defp do_reduce(graph, [cell_id | cell_ids], initial_acc, fun, cache) do + if parent_id = graph[cell_id] do + case cache do + %{^parent_id => acc} -> + acc = fun.(cell_id, acc) + cache = put_in(cache[cell_id], acc) + do_reduce(graph, cell_ids, initial_acc, fun, cache) + + _ -> + do_reduce(graph, [parent_id, cell_id | cell_ids], initial_acc, fun, cache) + end + else + acc = fun.(cell_id, initial_acc) + cache = put_in(cache[cell_id], acc) + do_reduce(graph, cell_ids, initial_acc, fun, cache) + end + end end diff --git a/lib/livebook/web_socket/server.ex b/lib/livebook/web_socket/server.ex index 6f9fdb3b7..dc132564a 100644 --- a/lib/livebook/web_socket/server.ex +++ b/lib/livebook/web_socket/server.ex @@ -103,7 +103,7 @@ defmodule Livebook.WebSocket.Server do # Private defp reply(%{caller: nil} = state, response) do - Logger.warn("The caller is nil, so we can't reply the message: #{inspect(response)}") + Logger.warning("The caller is nil, so we can't reply the message: #{inspect(response)}") state end diff --git a/lib/livebook_web/live/session_live.ex b/lib/livebook_web/live/session_live.ex index 826f6e905..3cf8c12f4 100644 --- a/lib/livebook_web/live/session_live.ex +++ b/lib/livebook_web/live/session_live.ex @@ -1221,10 +1221,10 @@ defmodule LivebookWeb.SessionLive do data = socket.private.data - with {:ok, cell, section} <- Notebook.fetch_cell_and_section(data.notebook, cell_id) do + with {:ok, cell, _section} <- Notebook.fetch_cell_and_section(data.notebook, cell_id) do if Runtime.connected?(data.runtime) do - base_locator = Session.find_base_locator(data, cell, section, existing: true) - ref = Runtime.handle_intellisense(data.runtime, self(), request, base_locator) + parent_locators = Session.parent_locators_for_cell(data, cell) + ref = Runtime.handle_intellisense(data.runtime, self(), request, parent_locators) {:reply, %{"ref" => inspect(ref)}, socket} else diff --git a/lib/livebook_web/live/session_live/attached_live.ex b/lib/livebook_web/live/session_live/attached_live.ex index ebf69be1d..17fd8dcaa 100644 --- a/lib/livebook_web/live/session_live/attached_live.ex +++ b/lib/livebook_web/live/session_live/attached_live.ex @@ -67,7 +67,11 @@ defmodule LivebookWeb.SessionLive.AttachedLive do
Cookie
- <%= text_input(f, :cookie, value: @data["cookie"], class: "input", placeholder: "mycookie") %> + <%= text_input(f, :cookie, + value: @data["cookie"], + class: "input", + placeholder: "mycookie" + ) %>