From 663ec3283edca2b5ca9613b82e2138465206e241 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Fri, 26 Feb 2021 20:53:29 +0100 Subject: [PATCH] Support starting runtime in Mix context (#61) * Prototype standalone mode with mix * Move runtime initialization into separate LiveViews * Make standalone node initialization async * Refactor async node initialization * Automatically scroll to the bottom of output * Refactor streaming output * Move MessageEmitter under Utils * Add path selector to the mix runtime picker * Update runtime descriptions * Add successful or error message at the end of output * Run formatter * Rename Standalone to ElixirStandalone for consistency * Show only directories when looking for a mix project * Update docs * Extract shared standalone logic * Make the remote primary process monitor Manager instead of session * Further refactoring and docs * Add tests for collectable Callback * Add missing macro doc * Apply review suggestions * Decouple sending asynchronous notifications from the runtime initialization * Apply suggestions --- assets/js/app.js | 2 + assets/js/scroll_on_update/index.js | 19 ++ lib/live_book/runtime/attached.ex | 2 +- lib/live_book/runtime/elixir_standalone.ex | 93 +++++++++ lib/live_book/runtime/erl_dist.ex | 2 +- lib/live_book/runtime/mix_standalone.ex | 129 ++++++++++++ lib/live_book/runtime/standalone.ex | 145 -------------- lib/live_book/runtime/standalone_init.ex | 149 ++++++++++++++ lib/live_book/session.ex | 2 +- lib/live_book/utils.ex | 11 ++ lib/live_book/utils/emitter.ex | 59 ++++++ lib/live_book_web/live/home_live.ex | 1 + .../live/path_select_component.ex | 13 +- lib/live_book_web/live/session_live.ex | 3 +- .../live/session_live/attached_live.ex | 66 +++++++ .../session_live/elixir_standalone_live.ex | 38 ++++ .../live/session_live/mix_standalone_live.ex | 88 +++++++++ .../session_live/persistence_component.ex | 3 +- .../live/session_live/runtime_component.ex | 186 +++++++----------- ...ne_test.exs => elixir_standalone_test.exs} | 24 +-- test/live_book/session_test.exs | 6 +- test/live_book/utils/emitter_test.exs | 45 +++++ .../live/path_select_component_test.exs | 5 +- 23 files changed, 804 insertions(+), 287 deletions(-) create mode 100644 assets/js/scroll_on_update/index.js create mode 100644 lib/live_book/runtime/elixir_standalone.ex create mode 100644 lib/live_book/runtime/mix_standalone.ex delete mode 100644 lib/live_book/runtime/standalone.ex create mode 100644 lib/live_book/runtime/standalone_init.ex create mode 100644 lib/live_book/utils/emitter.ex create mode 100644 lib/live_book_web/live/session_live/attached_live.ex create mode 100644 lib/live_book_web/live/session_live/elixir_standalone_live.ex create mode 100644 lib/live_book_web/live/session_live/mix_standalone_live.ex rename test/live_book/runtime/{standalone_test.exs => elixir_standalone_test.exs} (64%) create mode 100644 test/live_book/utils/emitter_test.exs diff --git a/assets/js/app.js b/assets/js/app.js index 74228c8a8..3e6aa7f3a 100644 --- a/assets/js/app.js +++ b/assets/js/app.js @@ -8,12 +8,14 @@ import ContentEditable from "./content_editable"; import Cell from "./cell"; import Session from "./session"; import FocusOnUpdate from "./focus_on_update"; +import ScrollOnUpdate from "./scroll_on_update"; const Hooks = { ContentEditable, Cell, Session, FocusOnUpdate, + ScrollOnUpdate, }; const csrfToken = document diff --git a/assets/js/scroll_on_update/index.js b/assets/js/scroll_on_update/index.js new file mode 100644 index 000000000..a5554813f --- /dev/null +++ b/assets/js/scroll_on_update/index.js @@ -0,0 +1,19 @@ +/** + * A hook used to scroll to the bottom of an element + * whenever it receives LV update. + */ +const ScrollOnUpdate = { + mounted() { + this.__scroll(); + }, + + updated() { + this.__scroll(); + }, + + __scroll() { + this.el.scrollTop = this.el.scrollHeight; + }, +}; + +export default ScrollOnUpdate; diff --git a/lib/live_book/runtime/attached.ex b/lib/live_book/runtime/attached.ex index 6dfe99097..6228f502f 100644 --- a/lib/live_book/runtime/attached.ex +++ b/lib/live_book/runtime/attached.ex @@ -7,7 +7,7 @@ defmodule LiveBook.Runtime.Attached do # LiveBook doesn't manage its lifetime in any way # and only loads/unloads the necessary elements. # The node can be an oridinary Elixir runtime, - # a mix project shell, a running release or anything else. + # a Mix project shell, a running release or anything else. defstruct [:node] diff --git a/lib/live_book/runtime/elixir_standalone.ex b/lib/live_book/runtime/elixir_standalone.ex new file mode 100644 index 000000000..147b06dfc --- /dev/null +++ b/lib/live_book/runtime/elixir_standalone.ex @@ -0,0 +1,93 @@ +defmodule LiveBook.Runtime.ElixirStandalone do + defstruct [:node, :primary_pid] + + # A runtime backed by a standalone Elixir node managed by LiveBook. + # + # LiveBook is responsible for starting and terminating the node. + # Most importantly we have to make sure the started node doesn't + # stay in the system when the session or the entire LiveBook terminates. + + import LiveBook.Runtime.StandaloneInit + + alias LiveBook.Utils + + @type t :: %__MODULE__{ + node: node(), + primary_pid: pid() + } + + @doc """ + Starts a new Elixir node (i.e. a system process) and initializes + it with LiveBook-specific modules and processes. + + If no process calls `Runtime.connect/1` for a period of time, + the node automatically terminates. Whoever connects, becomes the owner + and as soon as it terminates, the node terminates as well. + The node may also be terminated manually by using `Runtime.disconnect/1`. + + Note: to start the node it is required that `elixir` is a recognised + executable within the system. + """ + @spec init() :: {:ok, t()} | {:error, String.t()} + def init() do + parent_node = node() + child_node = random_node_name() + parent_process_name = random_process_name() + + Utils.temporarily_register(self(), parent_process_name, fn -> + with {:ok, elixir_path} <- find_elixir_executable(), + eval <- child_node_ast(parent_node, parent_process_name) |> Macro.to_string(), + port <- start_elixir_node(elixir_path, child_node, eval), + {:ok, primary_pid} <- parent_init_sequence(child_node, port) do + runtime = %__MODULE__{ + node: child_node, + primary_pid: primary_pid + } + + {:ok, runtime} + else + {:error, error} -> + {:error, error} + end + end) + end + + defp start_elixir_node(elixir_path, node_name, eval) do + # Here we create a port to start the system process in a non-blocking way. + Port.open({:spawn_executable, elixir_path}, [ + :nouse_stdio, + args: elixir_flags(node_name) ++ ["--eval", eval] + ]) + end +end + +defimpl LiveBook.Runtime, for: LiveBook.Runtime.ElixirStandalone do + alias LiveBook.Runtime.ErlDist + + def connect(runtime) do + ErlDist.Manager.set_owner(runtime.node, self()) + Process.monitor({ErlDist.Manager, runtime.node}) + end + + def disconnect(runtime) do + ErlDist.Manager.stop(runtime.node) + end + + def evaluate_code(runtime, code, container_ref, evaluation_ref, prev_evaluation_ref \\ :initial) do + ErlDist.Manager.evaluate_code( + runtime.node, + code, + container_ref, + evaluation_ref, + prev_evaluation_ref + ) + end + + def forget_evaluation(runtime, container_ref, evaluation_ref) do + ErlDist.Manager.forget_evaluation(runtime.node, container_ref, evaluation_ref) + end + + def drop_container(runtime, container_ref) do + ErlDist.Manager.drop_container(runtime.node, container_ref) + end +end diff --git a/lib/live_book/runtime/erl_dist.ex b/lib/live_book/runtime/erl_dist.ex index ae9a319c0..e65eef76e 100644 --- a/lib/live_book/runtime/erl_dist.ex +++ b/lib/live_book/runtime/erl_dist.ex @@ -8,7 +8,7 @@ defmodule LiveBook.Runtime.ErlDist do # code evaluation may take place in a separate Elixir runtime, # which also makes it easy to terminate the whole # evaluation environment without stopping LiveBook. - # This is what both `Runtime.Standalone` and `Runtime.Attached` do + # This is what both `Runtime.ElixirStandalone` and `Runtime.Attached` do # and this module containes the shared functionality they need. # # To work with a separate node, we have to inject the necessary diff --git a/lib/live_book/runtime/mix_standalone.ex b/lib/live_book/runtime/mix_standalone.ex new file mode 100644 index 000000000..4b0be8061 --- /dev/null +++ b/lib/live_book/runtime/mix_standalone.ex @@ -0,0 +1,129 @@ +defmodule LiveBook.Runtime.MixStandalone do + defstruct [:node, :primary_pid, :project_path] + + # A runtime backed by a standalone Elixir node managed by LiveBook. + # + # This runtime is similar to `LiveBook.Runtime.ElixirStandalone`, + # but the node is started in the context of a Mix project. + + import LiveBook.Runtime.StandaloneInit + + alias LiveBook.Utils + alias LiveBook.Utils.Emitter + + @type t :: %__MODULE__{ + node: node(), + primary_pid: pid(), + project_path: String.t() + } + + @doc """ + Starts a new Elixir node (i.e. a system process) and initializes + it with LiveBook-specific modules and processes. + + The node is started together with a Mix environment appropriate + for the given `project_path`. The setup may involve + long-running steps (like fetching dependencies, compiling the project), + so the initialization is asynchronous. This function spawns and links + a process responsible for initialization, which then uses `emitter` + to emit the following notifications: + + * `{:output, string}` - arbitrary output/info sent as the initialization proceeds + * `{:ok, runtime}` - a final message indicating successful initialization + * `{:error, message}` - a final message indicating failure + + If no process calls `Runtime.connect/1` for a period of time, + the node automatically terminates. Whoever connects, becomes the owner + and as soon as it terminates, the node terminates as well. + The node may also be terminated manually by using `Runtime.disconnect/1`. + + Note: to start the node it is required that both `elixir` and `mix` are + recognised executables within the system. + """ + @spec init_async(String.t(), Emitter.t()) :: :ok + def init_async(project_path, emitter) do + output_emitter = Emitter.mapper(emitter, fn output -> {:output, output} end) + + spawn_link(fn -> + parent_node = node() + child_node = random_node_name() + parent_process_name = random_process_name() + + Utils.temporarily_register(self(), parent_process_name, fn -> + with {:ok, elixir_path} <- find_elixir_executable(), + :ok <- run_mix_task("deps.get", project_path, output_emitter), + :ok <- run_mix_task("compile", project_path, output_emitter), + eval <- child_node_ast(parent_node, parent_process_name) |> Macro.to_string(), + port <- start_elixir_mix_node(elixir_path, child_node, eval, project_path), + {:ok, primary_pid} <- parent_init_sequence(child_node, port, output_emitter) do + runtime = %__MODULE__{ + node: child_node, + primary_pid: primary_pid, + project_path: project_path + } + + Emitter.emit(emitter, {:ok, runtime}) + else + {:error, error} -> + Emitter.emit(emitter, {:error, error}) + end + end) + end) + + :ok + end + + defp run_mix_task(task, project_path, output_emitter) do + Emitter.emit(output_emitter, "Running mix #{task}...\n") + + case System.cmd("mix", [task], + cd: project_path, + stderr_to_stdout: true, + into: output_emitter + ) do + {_callback, 0} -> :ok + {_callback, _status} -> {:error, "running mix #{task} failed, see output for more details"} + end + end + + defp start_elixir_mix_node(elixir_path, node_name, eval, project_path) do + # Here we create a port to start the system process in a non-blocking way. + Port.open({:spawn_executable, elixir_path}, [ + :binary, + :stderr_to_stdout, + args: elixir_flags(node_name) ++ ["-S", "mix", "run", "--eval", eval], + cd: project_path + ]) + end +end + +defimpl LiveBook.Runtime, for: LiveBook.Runtime.MixStandalone do + alias LiveBook.Runtime.ErlDist + + def connect(runtime) do + ErlDist.Manager.set_owner(runtime.node, self()) + Process.monitor({ErlDist.Manager, runtime.node}) + end + + def disconnect(runtime) do + ErlDist.Manager.stop(runtime.node) + end + + def evaluate_code(runtime, code, container_ref, evaluation_ref, prev_evaluation_ref \\ :initial) do + ErlDist.Manager.evaluate_code( + runtime.node, + code, + container_ref, + evaluation_ref, + prev_evaluation_ref + ) + end + + def forget_evaluation(runtime, container_ref, evaluation_ref) do + ErlDist.Manager.forget_evaluation(runtime.node, container_ref, evaluation_ref) + end + + def drop_container(runtime, container_ref) do + ErlDist.Manager.drop_container(runtime.node, container_ref) + end +end diff --git a/lib/live_book/runtime/standalone.ex b/lib/live_book/runtime/standalone.ex deleted file mode 100644 index faaa56b6a..000000000 --- a/lib/live_book/runtime/standalone.ex +++ /dev/null @@ -1,145 +0,0 @@ -defmodule LiveBook.Runtime.Standalone do - defstruct [:node, :primary_pid, :init_ref] - - # A runtime backed by a standalone Elixir node managed by LiveBook. - # - # LiveBook is responsible for starting and terminating the node. - # Most importantly we have to make sure the started node doesn't - # stay in the system when the session or the entire LiveBook terminates. - - alias LiveBook.Utils - - @type t :: %__MODULE__{ - node: node(), - primary_pid: pid(), - init_ref: reference() - } - - @doc """ - Starts a new Elixir node (i.e. a system process) and initializes - it with LiveBook-specific modules and processes. - - The new node monitors the given owner process and terminates - as soon as it terminates. It may also be terminated manually - by using `Runtime.disconnect/1`. - - Note: to start the node it is required that `elixir` is a recognised - executable within the system. - """ - @spec init(pid()) :: {:ok, t()} | {:error, :no_elixir_executable | :timeout} - def init(owner_pid) do - case System.find_executable("elixir") do - nil -> - {:error, :no_elixir_executable} - - elixir_path -> - id = Utils.random_short_id() - - node = Utils.node_from_name("live_book_runtime_#{id}") - - # The new Elixir node receives a code to evaluate - # and we have to pass the current pid there, but since pid - # is not a type we can include directly in the code, - # we temporarily register the current process under a name. - waiter = :"live_book_waiter_#{id}" - Process.register(self(), waiter) - - eval = child_node_eval(waiter, node()) |> Macro.to_string() - - # Here we create a port to start the system process in a non-blocking way. - Port.open({:spawn_executable, elixir_path}, [ - # Don't use stdio, so that the caller does not receive - # unexpected messages if the process produces some output. - :nouse_stdio, - args: [ - if(LiveBook.Config.shortnames?(), do: "--sname", else: "--name"), - to_string(node), - "--eval", - eval, - # Minimize shedulers busy wait threshold, - # so that they go to sleep immediately after evaluation. - # Enable ANSI escape codes as we handle them with HTML. - "--erl", - "+sbwt none +sbwtdcpu none +sbwtdio none -elixir ansi_enabled true" - ] - ]) - - receive do - {:node_started, init_ref, ^node, primary_pid} -> - # Unregister the temporary name as it's no longer needed. - Process.unregister(waiter) - # Having the other process pid we can send the owner pid as a message. - send(primary_pid, {:node_acknowledged, init_ref, owner_pid}) - - # There should be no problem initializing the new node - :ok = LiveBook.Runtime.ErlDist.initialize(node) - - {:ok, %__MODULE__{node: node, primary_pid: primary_pid, init_ref: init_ref}} - after - 10_000 -> - {:error, :timeout} - end - end - end - - defp child_node_eval(waiter, parent_node) do - # This is the code that's gonna be evaluated in the newly - # spawned Elixir runtime. This is the primary process - # and as soon as it finishes, the runtime terminates. - quote do - # Initiate communication with the waiting process on the parent node. - init_ref = make_ref() - send({unquote(waiter), unquote(parent_node)}, {:node_started, init_ref, node(), self()}) - - receive do - {:node_acknowledged, ^init_ref, owner_pid} -> - owner_ref = Process.monitor(owner_pid) - - # Wait until either the owner process terminates - # or we receives an explicit stop request. - receive do - {:DOWN, ^owner_ref, :process, _object, _reason} -> - :ok - - {:stop, ^init_ref} -> - :ok - end - after - 10_000 -> :timeout - end - end - end -end - -defimpl LiveBook.Runtime, for: LiveBook.Runtime.Standalone do - alias LiveBook.Runtime.ErlDist - - def connect(runtime) do - ErlDist.Manager.set_owner(runtime.node, self()) - Process.monitor({ErlDist.Manager, runtime.node}) - end - - def disconnect(runtime) do - ErlDist.Manager.stop(runtime.node) - # Instruct the other node to terminate - send(runtime.primary_pid, {:stop, runtime.init_ref}) - end - - def evaluate_code(runtime, code, container_ref, evaluation_ref, prev_evaluation_ref \\ :initial) do - ErlDist.Manager.evaluate_code( - runtime.node, - code, - container_ref, - evaluation_ref, - prev_evaluation_ref - ) - end - - def forget_evaluation(runtime, container_ref, evaluation_ref) do - ErlDist.Manager.forget_evaluation(runtime.node, container_ref, evaluation_ref) - end - - def drop_container(runtime, container_ref) do - ErlDist.Manager.drop_container(runtime.node, container_ref) - end -end diff --git a/lib/live_book/runtime/standalone_init.ex b/lib/live_book/runtime/standalone_init.ex new file mode 100644 index 000000000..b8aff084d --- /dev/null +++ b/lib/live_book/runtime/standalone_init.ex @@ -0,0 +1,149 @@ +defmodule LiveBook.Runtime.StandaloneInit do + @moduledoc false + + # Generic functionality related to starting and setting up + # a new Elixir system process. It's used by both ElixirStandalone + # and MixStandalone runtimes. + + alias LiveBook.Utils + alias LiveBook.Utils.Emitter + + @doc """ + Returns a random name for a dynamically spawned node. + """ + @spec random_node_name() :: atom() + def random_node_name() do + Utils.node_from_name("live_book_runtime_#{Utils.random_short_id()}") + end + + @doc """ + Returns random name to register a process under. + + We have to pass parent process pid to the new Elixir node. + The node receives code to evaluate as string, so we cannot + directly embed the pid there, but we can temporarily register + the process under a random name and pass this name to the child node. + """ + @spec random_process_name() :: atom() + def random_process_name() do + :"live_book_parent_process_name_#{Utils.random_short_id()}" + end + + @doc """ + Tries locating Elixir executable in PATH. + """ + @spec find_elixir_executable() :: {:ok, String.t()} | {:error, String.t()} + def find_elixir_executable() do + case System.find_executable("elixir") do + nil -> {:error, "no Elixir executable found in PATH"} + path -> {:ok, path} + end + end + + @doc """ + A list of common flags used for spawned Elixir runtimes. + """ + @spec elixir_flags(node()) :: list() + def elixir_flags(node_name) do + [ + if(LiveBook.Config.shortnames?(), do: "--sname", else: "--name"), + to_string(node_name), + "--erl", + # Minimize shedulers busy wait threshold, + # so that they go to sleep immediately after evaluation. + # Enable ANSI escape codes as we handle them with HTML. + "+sbwt none +sbwtdcpu none +sbwtdio none -elixir ansi_enabled true" + ] + end + + # --- + # + # Once the new node is spawned we need to establish a connection, + # initialize it and make sure it correctly reacts to the parent node terminating. + # + # The procedure goes as follows: + # + # 1. The child sends {:node_initialized, ref} message to the parent + # to communicate it's ready for initialization. + # + # 2. The parent initializes the child node - loads necessary modules + # and starts the Manager process. + # + # 3. The parent sends {:node_initialized, ref} message back to the child, + # to communicate successful initialization. + # + # 4. The child starts monitoring the Manager process and freezes + # until the Manager process terminates. The Manager process + # serves as the leading remote process and represents the node from now on. + # + # The nodes either successfully go through this flow or return an error, + # either if the other node dies or is not responding for too long. + # + # --- + + @doc """ + Performs the parent side of the initialization contract. + + Should be called by the initializing process on the parent node. + """ + @spec parent_init_sequence(node(), port(), Emitter.t() | nil) :: + {:ok, pid()} | {:error, String.t()} + def parent_init_sequence(child_node, port, emitter \\ nil) do + port_ref = Port.monitor(port) + + loop = fn loop -> + receive do + {:node_started, init_ref, ^child_node, primary_pid} -> + Port.demonitor(port_ref) + + # We've just created the node, so it is surely not in use + :ok = LiveBook.Runtime.ErlDist.initialize(child_node) + + send(primary_pid, {:node_initialized, init_ref}) + + {:ok, primary_pid} + + {^port, {:data, output}} -> + # Pass all the outputs through the given emitter. + emitter && Emitter.emit(emitter, output) + loop.(loop) + + {:DOWN, ^port_ref, :port, _object, _reason} -> + {:error, "Elixir process terminated unexpectedly"} + after + 10_000 -> + {:error, "connection timed out"} + end + end + + loop.(loop) + end + + @doc """ + Performs the child side of the initialization contract. + + This function returns AST that should be evaluated in primary + process on the newly spawned child node. + """ + def child_node_ast(parent_node, parent_process_name) do + # This is the primary process, so as soon as it finishes, the runtime terminates. + quote do + # Initiate communication with the parent process (on the parent node). + init_ref = make_ref() + parent_process = {unquote(parent_process_name), unquote(parent_node)} + send(parent_process, {:node_started, init_ref, node(), self()}) + + receive do + {:node_initialized, ^init_ref} -> + manager_ref = Process.monitor(LiveBook.Runtime.ErlDist.Manager) + + # Wait until the Manager process terminates. + receive do + {:DOWN, ^manager_ref, :process, _object, _reason} -> :ok + end + after + 10_000 -> :timeout + end + end + end +end diff --git a/lib/live_book/session.ex b/lib/live_book/session.ex index c36998c4c..072571e81 100644 --- a/lib/live_book/session.ex +++ b/lib/live_book/session.ex @@ -528,7 +528,7 @@ defmodule LiveBook.Session do # Checks if a runtime already set, and if that's not the case # starts a new standlone one. defp ensure_runtime(%{data: %{runtime: nil}} = state) do - with {:ok, runtime} <- Runtime.Standalone.init(self()) do + with {:ok, runtime} <- Runtime.ElixirStandalone.init() do runtime_monitor_ref = Runtime.connect(runtime) {:ok, diff --git a/lib/live_book/utils.ex b/lib/live_book/utils.ex index 5817a46a2..2e3e92635 100644 --- a/lib/live_book/utils.ex +++ b/lib/live_book/utils.ex @@ -32,4 +32,15 @@ defmodule LiveBook.Utils do :"#{name}@#{host}" end end + + @doc """ + Registers the given process under `name` for the time of `fun` evaluation. + """ + @spec temporarily_register(pid(), atom(), (... -> any())) :: any() + def temporarily_register(pid, name, fun) do + Process.register(pid, name) + fun.() + after + Process.unregister(name) + end end diff --git a/lib/live_book/utils/emitter.ex b/lib/live_book/utils/emitter.ex new file mode 100644 index 000000000..6019b0f6a --- /dev/null +++ b/lib/live_book/utils/emitter.ex @@ -0,0 +1,59 @@ +defmodule LiveBook.Utils.Emitter do + @moduledoc false + + # A wrapper struct for sending messages to the specified process. + + defstruct [:target_pid, :ref, :mapper] + + @type t :: %__MODULE__{ + target_pid: pid(), + ref: reference(), + mapper: mapper() + } + + @type mapper :: (term() -> term()) + + @doc """ + Builds a new structure where `target_pid` represents + the process that will receive all emitted items. + """ + @spec new(pid()) :: t() + def new(target_pid) do + %__MODULE__{target_pid: target_pid, ref: make_ref(), mapper: &Function.identity/1} + end + + @doc """ + Sends {:emitter, ref, item} message to the `target_pid`. + + Note that item may be transformed with emitter's `mapper` + if there is one, see `Emitter.mapper/2`. + """ + @spec emit(t(), term()) :: :ok + def emit(emitter, item) do + message = {:emitter, emitter.ref, emitter.mapper.(item)} + send(emitter.target_pid, message) + :ok + end + + @doc """ + Returns a new emitter that maps all emitted items with `mapper`. + """ + @spec mapper(t(), mapper()) :: t() + def mapper(emitter, mapper) do + mapper = fn x -> mapper.(emitter.mapper.(x)) end + %{emitter | mapper: mapper} + end +end + +defimpl Collectable, for: LiveBook.Utils.Emitter do + alias LiveBook.Utils.Emitter + + def into(emitter) do + collector_fun = fn + :ok, {:cont, item} -> Emitter.emit(emitter, item) + :ok, _ -> :ok + end + + {:ok, collector_fun} + end +end diff --git a/lib/live_book_web/live/home_live.ex b/lib/live_book_web/live/home_live.ex index 2d6b14ed0..b555b2a76 100644 --- a/lib/live_book_web/live/home_live.ex +++ b/lib/live_book_web/live/home_live.ex @@ -31,6 +31,7 @@ defmodule LiveBookWeb.HomeLive do <%= live_component @socket, LiveBookWeb.PathSelectComponent, id: "path_select", path: @path, + extnames: [LiveMarkdown.extension()], running_paths: paths(@session_summaries), target: nil %>
diff --git a/lib/live_book_web/live/path_select_component.ex b/lib/live_book_web/live/path_select_component.ex index 19b4d96d7..1e63ba169 100644 --- a/lib/live_book_web/live/path_select_component.ex +++ b/lib/live_book_web/live/path_select_component.ex @@ -6,11 +6,10 @@ defmodule LiveBookWeb.PathSelectComponent do # * `path` - the currently entered path # * `running_paths` - the list of notebook paths that are already linked to running sessions # * `target` - id of the component to send update events to or nil to send to the parent LV + # * `extnames` - a list of file extensions that should be shown # # The target receives `set_path` events with `%{"path" => path}` payload. - alias LiveBook.LiveMarkdown - @impl true def render(assigns) do ~L""" @@ -26,7 +25,7 @@ defmodule LiveBookWeb.PathSelectComponent do
- <%= for file <- list_matching_files(@path, @running_paths) do %> + <%= for file <- list_matching_files(@path, @extnames, @running_paths) do %> <%= render_file(file, @target) %> <% end %>
@@ -59,7 +58,7 @@ defmodule LiveBookWeb.PathSelectComponent do """ end - defp list_matching_files(path, running_paths) do + defp list_matching_files(path, extnames, running_paths) do # Note: to provide an intuitive behavior when typing the path # we enter a new directory when it has a trailing slash, # so given "/foo/bar" we list files in "foo" and given "/foo/bar/ @@ -93,7 +92,7 @@ defmodule LiveBookWeb.PathSelectComponent do end) |> Enum.filter(fn file -> not hidden?(file.name) and String.starts_with?(file.name, basename) and - (file.is_dir or notebook_file?(file.name)) + (file.is_dir or valid_extension?(file.name, extnames)) end) |> Enum.sort_by(fn file -> {!file.is_dir, file.name} end) @@ -118,8 +117,8 @@ defmodule LiveBookWeb.PathSelectComponent do String.starts_with?(filename, ".") end - defp notebook_file?(filename) do - String.ends_with?(filename, LiveMarkdown.extension()) + defp valid_extension?(filename, extnames) do + Path.extname(filename) in extnames end defp split_path(path) do diff --git a/lib/live_book_web/live/session_live.ex b/lib/live_book_web/live/session_live.ex index 9203ac1c9..61e992672 100644 --- a/lib/live_book_web/live/session_live.ex +++ b/lib/live_book_web/live/session_live.ex @@ -525,6 +525,7 @@ defmodule LiveBookWeb.SessionLive do end defp runtime_description(nil), do: "No runtime" - defp runtime_description(%Runtime.Standalone{}), do: "Standalone runtime" + defp runtime_description(%Runtime.ElixirStandalone{}), do: "Elixir standalone runtime" + defp runtime_description(%Runtime.MixStandalone{}), do: "Mix standalone runtime" defp runtime_description(%Runtime.Attached{}), do: "Attached runtime" end diff --git a/lib/live_book_web/live/session_live/attached_live.ex b/lib/live_book_web/live/session_live/attached_live.ex new file mode 100644 index 000000000..99f7b298d --- /dev/null +++ b/lib/live_book_web/live/session_live/attached_live.ex @@ -0,0 +1,66 @@ +defmodule LiveBookWeb.SessionLive.AttachedLive do + use LiveBookWeb, :live_view + + alias LiveBook.{Session, Runtime, Utils} + + @impl true + def mount(_params, %{"session_id" => session_id}, socket) do + {:ok, assign(socket, session_id: session_id, error_message: nil)} + end + + @impl true + def render(assigns) do + ~L""" +
+ <%= if @error_message do %> +
+ <%= @error_message %> +
+ <% end %> +

+ Connect the session to an already running node + and evaluate code in the context of that node. + Thanks to this approach you can work with + an arbitrary Elixir runtime. + Make sure to give the node a name, for example: +

+
+ <%= if LiveBook.Config.shortnames? do %> + 
iex --sname test
+ <% else %> + 
iex --name test@127.0.0.1
+ <% end %> +
+

+ Then enter the name of the node below: +

+ <%= f = form_for :node, "#", phx_submit: "init" %> + <%= text_input f, :name, class: "input-base shadow", + placeholder: if(LiveBook.Config.shortnames?, do: "test", else: "test@127.0.0.1") %> + + <%= submit "Connect", class: "mt-3 button-base button-sm" %> + +
+ """ + end + + @impl true + def handle_event("init", %{"node" => %{"name" => name}}, socket) do + node = Utils.node_from_name(name) + + case Runtime.Attached.init(node) do + {:ok, runtime} -> + Session.connect_runtime(socket.assigns.session_id, runtime) + {:noreply, assign(socket, error_message: nil)} + + {:error, error} -> + message = runtime_error_to_message(error) + {:noreply, assign(socket, error_message: message)} + end + end + + defp runtime_error_to_message(:unreachable), do: "Node unreachable" + + defp runtime_error_to_message(:already_in_use), + do: "Another session is already connected to this node" +end diff --git a/lib/live_book_web/live/session_live/elixir_standalone_live.ex b/lib/live_book_web/live/session_live/elixir_standalone_live.ex new file mode 100644 index 000000000..f1cc94905 --- /dev/null +++ b/lib/live_book_web/live/session_live/elixir_standalone_live.ex @@ -0,0 +1,38 @@ +defmodule LiveBookWeb.SessionLive.ElixirStandaloneLive do + use LiveBookWeb, :live_view + + alias LiveBook.{Session, Runtime} + + @impl true + def mount(_params, %{"session_id" => session_id}, socket) do + {:ok, assign(socket, session_id: session_id, output: nil)} + end + + @impl true + def render(assigns) do + ~L""" +
+

+ Start a new local node to handle code evaluation. + This is the default runtime and is started automatically + as soon as you evaluate the first cell. +

+ + <%= if @output do %> +
+ 
<%= @output %>
+
+ <% end %> +
+ """ + end + + @impl true + def handle_event("init", _params, socket) do + {:ok, runtime} = Runtime.ElixirStandalone.init() + Session.connect_runtime(socket.assigns.session_id, runtime) + {:noreply, socket} + end +end diff --git a/lib/live_book_web/live/session_live/mix_standalone_live.ex b/lib/live_book_web/live/session_live/mix_standalone_live.ex new file mode 100644 index 000000000..4e13e8fe5 --- /dev/null +++ b/lib/live_book_web/live/session_live/mix_standalone_live.ex @@ -0,0 +1,88 @@ +defmodule LiveBookWeb.SessionLive.MixStandaloneLive do + use LiveBookWeb, :live_view + + alias LiveBook.{Session, Runtime, Utils} + + @type status :: :initial | :initializing | :finished + + @impl true + def mount(_params, %{"session_id" => session_id}, socket) do + {:ok, + assign(socket, + session_id: session_id, + status: :initial, + path: default_path(), + outputs: [], + emitter: nil + ), temporary_assigns: [outputs: []]} + end + + @impl true + def render(assigns) do + ~L""" +
+

+ Start a new local node in the context of a Mix project. + This way all your code and dependencies will be available + within the notebook. +

+ <%= if @status == :initial do %> + <%= live_component @socket, LiveBookWeb.PathSelectComponent, + id: "path_select", + path: @path, + extnames: [], + running_paths: [], + target: nil %> + <%= content_tag :button, "Connect", class: "button-base button-sm", phx_click: "init", disabled: not mix_project_root?(@path) %> + <% end %> + <%= if @status != :initial do %> +
+ 
<%= for {output, i} <- @outputs do %><%= ansi_string_to_html(output) %><% end %>
+
+ <% end %> +
+ """ + end + + @impl true + def handle_event("set_path", %{"path" => path}, socket) do + {:noreply, assign(socket, path: path)} + end + + def handle_event("init", _params, socket) do + emitter = Utils.Emitter.new(self()) + Runtime.MixStandalone.init_async(socket.assigns.path, emitter) + {:noreply, assign(socket, status: :initializing, emitter: emitter)} + end + + @impl true + def handle_info({:emitter, ref, message}, %{assigns: %{emitter: %{ref: ref}}} = socket) do + case message do + {:output, output} -> + {:noreply, add_output(socket, output)} + + {:ok, runtime} -> + Session.connect_runtime(socket.assigns.session_id, runtime) + {:noreply, socket |> assign(status: :finished) |> add_output("Connected successfully")} + + {:error, error} -> + {:noreply, socket |> assign(status: :finished) |> add_output("Error: #{error}")} + end + end + + def handle_info(_, socket), do: {:noreply, socket} + + defp add_output(socket, output) do + assign(socket, outputs: socket.assigns.outputs ++ [{output, Utils.random_id()}]) + end + + defp default_path(), do: File.cwd!() <> "/" + + defp mix_project_root?(path) do + File.dir?(path) and File.exists?(Path.join(path, "mix.exs")) + end +end diff --git a/lib/live_book_web/live/session_live/persistence_component.ex b/lib/live_book_web/live/session_live/persistence_component.ex index b7fdc0bd5..c6e722d8d 100644 --- a/lib/live_book_web/live/session_live/persistence_component.ex +++ b/lib/live_book_web/live/session_live/persistence_component.ex @@ -12,7 +12,7 @@ defmodule LiveBookWeb.SessionLive.PersistenceComponent do @impl true def render(assigns) do ~L""" -
+

Configure file

@@ -39,6 +39,7 @@ defmodule LiveBookWeb.SessionLive.PersistenceComponent do <%= live_component @socket, LiveBookWeb.PathSelectComponent, id: "path_select", path: @path, + extnames: [LiveMarkdown.extension()], running_paths: paths(@session_summaries), target: @myself %>
diff --git a/lib/live_book_web/live/session_live/runtime_component.ex b/lib/live_book_web/live/session_live/runtime_component.ex index 1f8733040..2797ceed3 100644 --- a/lib/live_book_web/live/session_live/runtime_component.ex +++ b/lib/live_book_web/live/session_live/runtime_component.ex @@ -1,144 +1,106 @@ defmodule LiveBookWeb.SessionLive.RuntimeComponent do use LiveBookWeb, :live_component - alias LiveBook.{Session, Runtime, Utils} + alias LiveBook.{Session, Runtime} @impl true def mount(socket) do - {:ok, assign(socket, error_message: nil)} + {:ok, assign(socket, type: "elixir_standalone")} end @impl true def render(assigns) do ~L""" -
- <%= if @error_message do %> -
- <%= @error_message %> -
- <% end %> +

Runtime

-

- The code is evaluated in a separate Elixir runtime (node), - which you can configure yourself here. -

-
- <%= if @runtime do %> - - - - - - - - - - - - - - - -
TypeNode name
<%= runtime_type_label(@runtime) %><%= @runtime.node %> - -
- <% else %> -

- No connected node -

- <% end %> -
-
"> -

- Standalone -

-
-

- You can start a new local node to handle code evaluation. - This happens automatically as soon as you evaluate the first cell. -

- -
-

- Attached -

-
-

- You can connect the session to an already running node - and evaluate code in the context of that node. - This is especially handy when developing mix projects. - Make sure to give the node a name: -

-
- <%= if LiveBook.Config.shortnames? do %> - 
iex --sname test -S mix
+
+

+ The code is evaluated in a separate Elixir runtime (node), + which you can configure yourself here. +

+
+ <%= if @runtime do %> + + + + + + + + + + + + + + + +
TypeNode name
<%= runtime_type_label(@runtime) %><%= @runtime.node %> + +
<% else %> - 
iex --name test@127.0.0.1 -S mix
+

+ No connected node +

<% end %> +
+
+
+ + +
-

- Then enter the name of the node below: -

- <%= f = form_for :node, "#", phx_target: @myself, phx_submit: "init_attached" %> - <%= text_input f, :name, class: "input-base text-sm shadow", - placeholder: if(LiveBook.Config.shortnames?, do: "test", else: "test@127.0.0.1") %> - - <%= submit "Connect", class: "mt-3 button-base text-sm" %> -
+ +
+ <%= if @type == "elixir_standalone" do %> + <%= live_render @socket, LiveBookWeb.SessionLive.ElixirStandaloneLive, + id: :elixir_standalone_runtime, + session: %{"session_id" => @session_id} %> + <% end %> + <%= if @type == "mix_standalone" do %> + <%= live_render @socket, LiveBookWeb.SessionLive.MixStandaloneLive, + id: :mix_standalone_runtime, + session: %{"session_id" => @session_id} %> + <% end %> + <%= if @type == "attached" do %> + <%= live_render @socket, LiveBookWeb.SessionLive.AttachedLive, + id: :attached_runtime, + session: %{"session_id" => @session_id} %> + <% end %>
""" end - defp runtime_type_label(%Runtime.Standalone{}), do: "Standalone" + defp runtime_type_label(%Runtime.ElixirStandalone{}), do: "Elixir standalone" + defp runtime_type_label(%Runtime.MixStandalone{}), do: "Mix standalone" defp runtime_type_label(%Runtime.Attached{}), do: "Attached" @impl true + def handle_event("set_runtime_type", %{"type" => type}, socket) do + {:noreply, assign(socket, type: type)} + end + def handle_event("disconnect", _params, socket) do Session.disconnect_runtime(socket.assigns.session_id) {:noreply, socket} end - - def handle_event("init_standalone", _params, socket) do - session_pid = Session.get_pid(socket.assigns.session_id) - handle_runtime_init_result(socket, Runtime.Standalone.init(session_pid)) - end - - def handle_event("init_attached", %{"node" => %{"name" => name}}, socket) do - node = Utils.node_from_name(name) - handle_runtime_init_result(socket, Runtime.Attached.init(node)) - end - - defp handle_runtime_init_result(socket, {:ok, runtime}) do - Session.connect_runtime(socket.assigns.session_id, runtime) - {:noreply, assign(socket, error_message: nil)} - end - - defp handle_runtime_init_result(socket, {:error, error}) do - message = runtime_error_to_message(error) - {:noreply, assign(socket, error_message: message)} - end - - defp runtime_error_to_message(:unreachable), do: "Node unreachable" - defp runtime_error_to_message(:no_elixir_executable), do: "No Elixir executable found in PATH" - defp runtime_error_to_message(:timeout), do: "Connection timed out" - - defp runtime_error_to_message(:already_in_use), - do: "Another session is already connected to this node" - - defp runtime_error_to_message(_), do: "Something went wrong" end diff --git a/test/live_book/runtime/standalone_test.exs b/test/live_book/runtime/elixir_standalone_test.exs similarity index 64% rename from test/live_book/runtime/standalone_test.exs rename to test/live_book/runtime/elixir_standalone_test.exs index 71feb3bd8..920d38f73 100644 --- a/test/live_book/runtime/standalone_test.exs +++ b/test/live_book/runtime/elixir_standalone_test.exs @@ -1,32 +1,27 @@ -defmodule LiveBook.Runtime.StandaloneTest do +defmodule LiveBook.Runtime.ElixirStandaloneTest do use ExUnit.Case, async: true alias LiveBook.Runtime describe "init/1" do - test "starts a new Elixir runtime in distribution mode and ties its lifetime to the given owner process" do - owner = - spawn(fn -> - receive do - :stop -> :ok - end - end) - - assert {:ok, %{node: node}} = Runtime.Standalone.init(owner) + test "starts a new Elixir runtime in distribution mode and ties its lifetime to the Manager process" do + assert {:ok, %{node: node} = runtime} = Runtime.ElixirStandalone.init() + Runtime.connect(runtime) # Make sure the node is running. Node.monitor(node, true) assert :pong = Node.ping(node) # Tell the owner process to stop. - send(owner, :stop) + LiveBook.Runtime.ErlDist.Manager.stop(node) - # Once the owner process terminates, the node should terminate as well. + # Once Manager terminates, the node should terminate as well. assert_receive {:nodedown, ^node} end test "loads necessary modules and starts manager process" do - assert {:ok, %{node: node}} = Runtime.Standalone.init(self()) + assert {:ok, %{node: node} = runtime} = Runtime.ElixirStandalone.init() + Runtime.connect(runtime) assert evaluator_module_loaded?(node) assert manager_started?(node) @@ -34,7 +29,8 @@ defmodule LiveBook.Runtime.StandaloneTest do end test "Runtime.disconnect/1 makes the node terminate" do - assert {:ok, %{node: node} = runtime} = Runtime.Standalone.init(self()) + assert {:ok, %{node: node} = runtime} = Runtime.ElixirStandalone.init() + Runtime.connect(runtime) # Make sure the node is running. Node.monitor(node, true) diff --git a/test/live_book/session_test.exs b/test/live_book/session_test.exs index 616d5763b..1bc1b62cd 100644 --- a/test/live_book/session_test.exs +++ b/test/live_book/session_test.exs @@ -240,15 +240,15 @@ defmodule LiveBook.SessionTest do test "if the runtime node goes down, notifies the subscribers" do session_id = Utils.random_id() {:ok, _} = Session.start_link(id: session_id) - {:ok, runtime} = Runtime.Standalone.init(self()) + {:ok, runtime} = Runtime.ElixirStandalone.init() Phoenix.PubSub.subscribe(LiveBook.PubSub, "sessions:#{session_id}") - # Wait for the runtime to best set + # Wait for the runtime to be set Session.connect_runtime(session_id, runtime) assert_receive {:operation, {:set_runtime, ^runtime}} - # Terminate the other node, the session should detect that. + # Terminate the other node, the session should detect that Node.spawn(runtime.node, System, :halt, []) assert_receive {:operation, {:set_runtime, nil}} diff --git a/test/live_book/utils/emitter_test.exs b/test/live_book/utils/emitter_test.exs new file mode 100644 index 000000000..e6ee47cf6 --- /dev/null +++ b/test/live_book/utils/emitter_test.exs @@ -0,0 +1,45 @@ +defmodule LiveBook.Utils.EmitterTest do + use ExUnit.Case, async: true + + alias LiveBook.Utils.Emitter + + describe "emit/2" do + test "sends the item as a message to the specified process" do + emitter = Emitter.new(self()) + ref = emitter.ref + Emitter.emit(emitter, :hey) + + assert_receive {:emitter, ^ref, :hey} + end + end + + describe "map/2" do + test "returns a modified emitter that transforms items before they are sent" do + emitter = Emitter.new(self()) + ref = emitter.ref + string_emitter = Emitter.mapper(emitter, &to_string/1) + Emitter.emit(string_emitter, :hey) + + assert_receive {:emitter, ^ref, "hey"} + end + + test "supports chaining" do + emitter = Emitter.new(self()) + ref = emitter.ref + string_emitter = Emitter.mapper(emitter, &to_string/1) + duplicate_emitter = Emitter.mapper(string_emitter, fn x -> {x, x} end) + Emitter.emit(duplicate_emitter, :hey) + + assert_receive {:emitter, ^ref, {"hey", "hey"}} + end + end + + test "implements Collectable so that it emits every item" do + emitter = Emitter.new(self()) + ref = emitter.ref + for x <- ["a", "b"], into: emitter, do: x + + assert_receive {:emitter, ^ref, "a"} + assert_receive {:emitter, ^ref, "b"} + end +end diff --git a/test/live_book_web/live/path_select_component_test.exs b/test/live_book_web/live/path_select_component_test.exs index c6abf2a4f..0378de74e 100644 --- a/test/live_book_web/live/path_select_component_test.exs +++ b/test/live_book_web/live/path_select_component_test.exs @@ -39,7 +39,10 @@ defmodule LiveBookWeb.PathSelectComponentTest do end defp attrs(attrs) do - Keyword.merge([id: 1, path: "/", running_paths: [], target: nil], attrs) + Keyword.merge( + [id: 1, path: "/", extnames: [".livemd"], running_paths: [], target: nil], + attrs + ) end defp notebooks_path() do