Remove Mix standalone runtime, closes #1345 (#1370)

Co-authored-by: Jonatan Kłosko <jonatanklosko@gmail.com>
This commit is contained in:
José Valim 2022-08-29 14:20:24 +02:00 committed by GitHub
parent 648edf23e7
commit 608979471d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 35 additions and 628 deletions

View file

@ -163,9 +163,8 @@ The following environment variables configure Livebook:
* LIVEBOOK_DEFAULT_RUNTIME - sets the runtime type that is used by default
when none is started explicitly for the given notebook. Must be either
"standalone" (Elixir standalone), "mix[:PATH][:FLAGS]" (Mix standalone),
"attached:NODE:COOKIE" (Attached node) or "embedded" (Embedded).
Defaults to "standalone".
"standalone" (Elixir standalone), "attached:NODE:COOKIE" (Attached node)
or "embedded" (Embedded). Defaults to "standalone".
* LIVEBOOK_FORCE_SSL_HOST - sets a host to redirect to if the request is not over HTTP.
Note it does not apply when accessing Livebook via localhost. Defaults to nil.

View file

@ -131,7 +131,6 @@ defmodule Livebook do
:runtime_modules,
[
Livebook.Runtime.ElixirStandalone,
Livebook.Runtime.MixStandalone,
Livebook.Runtime.Attached
]

View file

@ -322,70 +322,17 @@ defmodule Livebook.Config do
"embedded" ->
Livebook.Runtime.Embedded.new()
"mix" ->
case mix_path(File.cwd!()) do
{:ok, path} ->
Livebook.Runtime.MixStandalone.new(path)
:error ->
abort!(
"the current directory is not a Mix project, make sure to specify the path explicitly with mix:path"
)
end
"mix:" <> config ->
{path, flags} = parse_mix_config!(config)
case mix_path(path) do
{:ok, path} ->
if Livebook.Utils.valid_cli_flags?(flags) do
Livebook.Runtime.MixStandalone.new(path, flags)
else
abort!(~s{"#{flags}" is not a valid flag sequence})
end
:error ->
abort!(~s{"#{path}" does not point to a Mix project})
end
"attached:" <> config ->
{node, cookie} = parse_connection_config!(config)
Livebook.Runtime.Attached.new(node, cookie)
other ->
abort!(
~s{expected #{context} to be either "standalone", "mix[:path]" or "embedded", got: #{inspect(other)}}
~s{expected #{context} to be either "standalone", "attached:node:cookie" or "embedded", got: #{inspect(other)}}
)
end
end
defp parse_mix_config!(config) do
case String.split(config, ":", parts: 2) do
# Ignore Windows drive letter
[<<letter>>, rest] when letter in ?a..?z or letter in ?A..?Z ->
[path | rest] = String.split(rest, ":", parts: 2)
[<<letter, ":", path::binary>> | rest]
other ->
other
end
|> case do
[path] -> {path, ""}
[path, flags] -> {path, flags}
end
end
defp mix_path(path) do
path = Path.expand(path)
mixfile = Path.join(path, "mix.exs")
if File.exists?(mixfile) do
{:ok, path}
else
:error
end
end
defp parse_connection_config!(config) do
{node, cookie} = split_at_last_occurrence(config, ":")

View file

@ -52,9 +52,31 @@ By default, a new Elixir node is started (similarly to starting `iex`). You
can click reconnect whenever you want to discard the current node and start
a new one.
You can also choose to run inside a *Mix* project (as you would with `iex -S mix`),
manually *attach* to an existing distributed node, or run your Elixir notebook
*embedded* within the Livebook source itself.
You can also manually *attach* to an existing distributed node.
## Mix projects
Sometimes you may want to run a notebook within the context of an existing
Mix project. This is possible from Elixir v1.14 with the help of `Mix.install/2`.
As an example, imagine you have created a notebook inside your current project,
at `notebooks/example.livemd`. In order to run within the root Mix project, using
the same configuration and dependencies versions, your `Mix.install/2` should look
like this:
<!-- livebook:{"force_markdown":true} -->
```elixir
my_app_root = Path.join(__DIR__, "..")
Mix.install(
[
{:my_app, path: my_app_root, env: :dev}
],
config_path: Path.join(my_app_root, "config/config.exs"),
lockfile: Path.join(my_app_root, "mix.lock")
)
```
## More on branches #1
@ -76,7 +98,7 @@ parent = self()
Process.put(:info, "deal carefully with process dictionaries")
```
<!-- livebook:{"branch_parent_index":3} -->
<!-- livebook:{"branch_parent_index":4} -->
## More on branches #2

View file

@ -8,9 +8,8 @@ defmodule Livebook.Runtime.ErlDist do
# code evaluation may take place in a separate Elixir runtime,
# which also makes it easy to terminate the whole
# evaluation environment without stopping Livebook.
# This is what `Runtime.ElixirStandalone`, `Runtime.MixStandalone`
# and `Runtime.Attached` do, so this module contains the shared
# functionality they need.
# This is what `Runtime.ElixirStandalone` and `Runtime.Attached` do,
# so this module contains the shared functionality they need.
#
# To work with a separate node, we have to inject the necessary
# Livebook modules there and also start the relevant processes

View file

@ -1,219 +0,0 @@
defmodule Livebook.Runtime.MixStandalone do
defstruct [:node, :server_pid, :project_path, :flags]
# A runtime backed by a standalone Elixir node managed by Livebook.
#
# This runtime is similar to `Livebook.Runtime.ElixirStandalone`,
# but the node is started in the context of a Mix project.
import Livebook.Runtime.StandaloneInit
alias Livebook.Utils
alias Livebook.Utils.Emitter
@type t :: %__MODULE__{
project_path: String.t(),
flags: String.t(),
node: node() | nil,
server_pid: pid() | nil
}
@doc """
Returns a new runtime instance.
"""
@spec new(String.t(), String.t()) :: t()
def new(project_path, flags \\ "") do
%__MODULE__{project_path: project_path, flags: flags}
end
@doc """
Starts a new Elixir node (a system process) and initializes it with
Livebook-specific modules and processes.
The node is started together with a Mix environment at the given
`project_path`. The setup may involve long-running steps (like
fetching dependencies, compiling the project), so the initialization
is asynchronous. This function spawns and links a process responsible
for initialization, which then uses `emitter` to emit the following
notifications:
* `{:output, string}` - arbitrary output/info sent as the initialization proceeds
* `{:ok, runtime}` - a final message indicating successful initialization
* `{:error, message}` - a final message indicating failure
Note: to start the node it is required that both `elixir` and `mix`
are recognised executables within the system.
"""
@spec connect_async(t(), Emitter.t()) :: :ok
def connect_async(runtime, emitter) do
%{project_path: project_path} = runtime
flags = OptionParser.split(runtime.flags)
output_emitter = Emitter.mapper(emitter, fn output -> {:output, output} end)
spawn_link(fn ->
parent_node = node()
child_node = child_node_name(parent_node)
Utils.temporarily_register(self(), child_node, fn ->
argv = [parent_node]
with {:ok, elixir_path} <- find_elixir_executable(),
:ok <- run_mix_task("deps.get", project_path, output_emitter),
:ok <- run_mix_task("compile", project_path, output_emitter),
eval = child_node_eval_string(),
port =
start_elixir_mix_node(elixir_path, child_node, flags, eval, argv, project_path),
{:ok, server_pid} <- parent_init_sequence(child_node, port, emitter: output_emitter) do
runtime = %{runtime | node: child_node, server_pid: server_pid}
Emitter.emit(emitter, {:ok, runtime})
else
{:error, error} ->
Emitter.emit(emitter, {:error, error})
end
end)
end)
:ok
end
@doc """
A synchronous version of of `connect_async/2`.
"""
@spec connect(t()) :: {:ok, t()} | {:error, String.t()}
def connect(runtime) do
%{ref: ref} = emitter = Livebook.Utils.Emitter.new(self())
connect_async(runtime, emitter)
await_connect(ref, [])
end
defp await_connect(ref, outputs) do
receive do
{:emitter, ^ref, message} -> message
end
|> case do
{:ok, runtime} ->
{:ok, runtime}
{:error, error} ->
message = IO.iodata_to_binary([error, ". Output:\n\n", Enum.reverse(outputs)])
{:error, message}
{:output, output} ->
await_connect(ref, [output | outputs])
end
end
defp run_mix_task(task, project_path, output_emitter) do
Emitter.emit(output_emitter, "Running mix #{task}...\n")
case System.cmd("mix", [task],
cd: project_path,
stderr_to_stdout: true,
into: output_emitter
) do
{_callback, 0} -> :ok
{_callback, _status} -> {:error, "running mix #{task} failed"}
end
end
defp start_elixir_mix_node(elixir_path, node_name, flags, eval, argv, project_path) do
# Here we create a port to start the system process in a non-blocking way.
Port.open({:spawn_executable, elixir_path}, [
:binary,
# We don't communicate with the system process via stdio,
# contrarily, we want any non-captured output to go directly
# to the terminal
:nouse_stdio,
:hide,
cd: project_path,
args:
elixir_flags(node_name) ++
["-S", "mix", "run", "--eval", eval | flags] ++
["--" | Enum.map(argv, &to_string/1)]
])
end
end
defimpl Livebook.Runtime, for: Livebook.Runtime.MixStandalone do
alias Livebook.Runtime.ErlDist.RuntimeServer
def describe(runtime) do
[
{"Type", "Mix standalone"},
{"Project", runtime.project_path},
runtime.flags != "" && {"Flags", runtime.flags},
connected?(runtime) && {"Node name", Atom.to_string(runtime.node)}
]
|> Enum.filter(&is_tuple/1)
end
def connect(runtime) do
Livebook.Runtime.MixStandalone.connect(runtime)
end
def connected?(runtime) do
runtime.server_pid != nil
end
def take_ownership(runtime, opts \\ []) do
RuntimeServer.attach(runtime.server_pid, self(), opts)
Process.monitor(runtime.server_pid)
end
def disconnect(runtime) do
:ok = RuntimeServer.stop(runtime.server_pid)
{:ok, %{runtime | node: nil, server_pid: nil}}
end
def duplicate(runtime) do
Livebook.Runtime.MixStandalone.new(runtime.project_path, runtime.flags)
end
def evaluate_code(runtime, code, locator, base_locator, opts \\ []) do
RuntimeServer.evaluate_code(runtime.server_pid, code, locator, base_locator, opts)
end
def forget_evaluation(runtime, locator) do
RuntimeServer.forget_evaluation(runtime.server_pid, locator)
end
def drop_container(runtime, container_ref) do
RuntimeServer.drop_container(runtime.server_pid, container_ref)
end
def handle_intellisense(runtime, send_to, request, base_locator) do
RuntimeServer.handle_intellisense(runtime.server_pid, send_to, request, base_locator)
end
def read_file(runtime, path) do
RuntimeServer.read_file(runtime.server_pid, path)
end
def start_smart_cell(runtime, kind, ref, attrs, base_locator) do
RuntimeServer.start_smart_cell(runtime.server_pid, kind, ref, attrs, base_locator)
end
def set_smart_cell_base_locator(runtime, ref, base_locator) do
RuntimeServer.set_smart_cell_base_locator(runtime.server_pid, ref, base_locator)
end
def stop_smart_cell(runtime, ref) do
RuntimeServer.stop_smart_cell(runtime.server_pid, ref)
end
def fixed_dependencies?(_runtime), do: true
def add_dependencies(_runtime, _code, _dependencies) do
raise "not supported"
end
def search_packages(_runtime, _send_to, _search) do
raise "not supported"
end
def put_system_envs(runtime, secrets) do
RuntimeServer.put_system_envs(runtime.server_pid, secrets)
end
end

View file

@ -2,10 +2,8 @@ defmodule Livebook.Runtime.StandaloneInit do
@moduledoc false
# Generic functionality related to starting and setting up
# a new Elixir system process. It's used by both ElixirStandalone
# and MixStandalone runtimes.
# a new Elixir system process. It's used by ElixirStandalone.
alias Livebook.Utils.Emitter
alias Livebook.Runtime.NodePool
@doc """
@ -83,16 +81,12 @@ defmodule Livebook.Runtime.StandaloneInit do
## Options
* `:emitter` - an emitter through which all child outpt is passed
* `:init_opts` - see `Livebook.Runtime.ErlDist.initialize/2`
"""
@spec parent_init_sequence(node(), port(), keyword()) :: {:ok, pid()} | {:error, String.t()}
def parent_init_sequence(child_node, port, opts \\ []) do
port_ref = Port.monitor(port)
emitter = opts[:emitter]
loop = fn loop ->
receive do
{:node_started, init_ref, ^child_node, primary_pid} ->
@ -104,17 +98,14 @@ defmodule Livebook.Runtime.StandaloneInit do
{:ok, server_pid}
{^port, {:data, output}} ->
# Pass all the outputs through the given emitter.
emitter && Emitter.emit(emitter, output)
{^port, {:data, _output}} ->
loop.(loop)
{:DOWN, ^port_ref, :port, _object, _reason} ->
{:error, "Elixir terminated unexpectedly, please check the terminal for errors"}
after
# Use a longer timeout to account for longer child node startup,
# as may happen when starting with Mix.
40_000 ->
# Use a longer timeout to account for longer child node startup.
30_000 ->
{:error, "connection timed out"}
end
end

View file

@ -1,59 +0,0 @@
defmodule Livebook.Utils.Emitter do
@moduledoc false
# A wrapper struct for sending messages to the specified process.
defstruct [:target_pid, :ref, :mapper]
@type t :: %__MODULE__{
target_pid: pid(),
ref: reference(),
mapper: mapper()
}
@type mapper :: (term() -> term())
@doc """
Builds a new structure where `target_pid` represents
the process that will receive all emitted items.
"""
@spec new(pid()) :: t()
def new(target_pid) do
%__MODULE__{target_pid: target_pid, ref: make_ref(), mapper: &Function.identity/1}
end
@doc """
Sends {:emitter, ref, item} message to the `target_pid`.
Note that item may be transformed with emitter's `mapper`
if there is one, see `Emitter.mapper/2`.
"""
@spec emit(t(), term()) :: :ok
def emit(emitter, item) do
message = {:emitter, emitter.ref, emitter.mapper.(item)}
send(emitter.target_pid, message)
:ok
end
@doc """
Returns a new emitter that maps all emitted items with `mapper`.
"""
@spec mapper(t(), mapper()) :: t()
def mapper(emitter, mapper) do
mapper = fn x -> mapper.(emitter.mapper.(x)) end
%{emitter | mapper: mapper}
end
end
defimpl Collectable, for: Livebook.Utils.Emitter do
alias Livebook.Utils.Emitter
def into(emitter) do
collector_fun = fn
:ok, {:cont, item} -> Emitter.emit(emitter, item)
:ok, _ -> :ok
end
{:ok, collector_fun}
end
end

View file

@ -45,7 +45,6 @@ defmodule LivebookCLI.Server do
explicitly for the given notebook, defaults to standalone
Supported options:
* standalone - Elixir standalone
* mix[:PATH][:FLAGS] - Mix standalone
* attached:NODE:COOKIE - Attached
* embedded - Embedded
--home The home path for the Livebook instance

View file

@ -1,169 +0,0 @@
defmodule LivebookWeb.SessionLive.MixStandaloneLive do
use LivebookWeb, :live_view
alias Livebook.{Session, Runtime, Utils, FileSystem}
@type status :: :initial | :initializing | :finished
@impl true
def mount(_params, %{"session" => session, "current_runtime" => current_runtime}, socket) do
unless Livebook.Config.runtime_enabled?(Livebook.Runtime.MixStandalone) do
raise "runtime module not allowed"
end
if connected?(socket) do
Session.subscribe(session.id)
end
{:ok,
assign(socket,
session: session,
status: :initial,
current_runtime: current_runtime,
data: initial_data(current_runtime),
outputs: [],
outputs_version: 0,
emitter: nil
), temporary_assigns: [outputs: []]}
end
@impl true
def render(assigns) do
~H"""
<div class="flex-col space-y-5">
<p class="text-gray-700">
Start a new local node in the context of a Mix project.
This way all your code and dependencies will be available
within the notebook.
</p>
<p class="text-gray-700">
<span class="font-semibold">Warning:</span> Notebooks that use <code>Mix.install/1</code>
do not work
inside a Mix project because the dependencies of the project
itself have been installed instead.
</p>
<%= if @status != :initializing do %>
<div class="h-full h-52">
<.live_component
module={LivebookWeb.FileSelectComponent}
id="mix-project-dir"
file={@data.file}
extnames={[]}
running_files={[]}
submit_event={if(data_valid?(@data), do: :init, else: nil)}
file_system_select_disabled={true}
/>
</div>
<form phx-change="validate" phx-submit="init">
<div>
<div class="input-label"><code>mix run</code> command-line flags</div>
<input
class="input"
type="text"
name="flags"
value={@data.flags}
spellcheck="false"
autocomplete="off"
/>
</div>
<button class="mt-5 button-base button-blue" type="submit" disabled={not data_valid?(@data)}>
<%= if(matching_runtime?(@current_runtime, @data), do: "Reconnect", else: "Connect") %>
</button>
</form>
<% end %>
<%= if @status != :initial do %>
<div class="markdown">
<pre><code
class="max-h-40 overflow-y-auto tiny-scrollbar"
id={"mix-standalone-init-output-#{@outputs_version}"}
phx-update="append"
phx-hook="ScrollOnUpdate"
><%= for {output, i} <- @outputs do %><span id={
"mix-standalone-init-output-#{@outputs_version}-#{i}"
}><%= ansi_string_to_html(output) %></span><% end %></code></pre>
</div>
<% end %>
</div>
"""
end
@impl true
def handle_event("init", _params, socket) do
handle_init(socket)
end
def handle_event("validate", %{"flags" => flags}, socket) do
{:noreply, update(socket, :data, &%{&1 | flags: flags})}
end
@impl true
def handle_info({:set_file, file, _info}, socket) do
{:noreply, update(socket, :data, &%{&1 | file: file})}
end
def handle_info(:init, socket) do
handle_init(socket)
end
def handle_info({:emitter, ref, message}, %{assigns: %{emitter: %{ref: ref}}} = socket) do
case message do
{:output, output} ->
{:noreply, add_output(socket, output)}
{:ok, runtime} ->
Session.set_runtime(socket.assigns.session.pid, runtime)
{:noreply, socket |> assign(status: :finished) |> add_output("Connected successfully")}
{:error, error} ->
{:noreply, socket |> assign(status: :finished) |> add_output("Error: #{error}")}
end
end
def handle_info({:operation, {:set_runtime, _pid, runtime}}, socket) do
{:noreply, assign(socket, current_runtime: runtime)}
end
def handle_info(_, socket), do: {:noreply, socket}
defp handle_init(socket) do
emitter = Utils.Emitter.new(self())
runtime = Runtime.MixStandalone.new(socket.assigns.data.file.path, socket.assigns.data.flags)
Runtime.MixStandalone.connect_async(runtime, emitter)
{:noreply,
socket
|> assign(status: :initializing, emitter: emitter, outputs: [])
|> update(:outputs_version, &(&1 + 1))}
end
defp add_output(socket, output) do
assign(socket, outputs: socket.assigns.outputs ++ [{output, Utils.random_id()}])
end
defp initial_data(%Runtime.MixStandalone{project_path: project_path, flags: flags}) do
file =
project_path
|> FileSystem.Utils.ensure_dir_path()
|> FileSystem.File.local()
%{file: file, flags: flags}
end
defp initial_data(_runtime) do
%{file: Livebook.Config.local_filesystem_home(), flags: ""}
end
defp matching_runtime?(%Runtime.MixStandalone{} = runtime, data) do
Path.expand(runtime.project_path) == Path.expand(data.file.path)
end
defp matching_runtime?(_runtime, _path), do: false
defp data_valid?(data) do
mix_project_root?(data.file.path) and Livebook.Utils.valid_cli_flags?(data.flags)
end
defp mix_project_root?(path) do
File.dir?(path) and File.exists?(Path.join(path, "mix.exs"))
end
end

View file

@ -40,16 +40,6 @@ defmodule LivebookWeb.SessionLive.RuntimeComponent do
Elixir standalone
</.choice_button>
<% end %>
<%= if Livebook.Config. runtime_enabled?(Livebook.Runtime.MixStandalone) do %>
<.choice_button
active={@type == "mix_standalone"}
phx-click="set_runtime_type"
phx-value-type="mix_standalone"
phx-target={@myself}
>
Mix standalone
</.choice_button>
<% end %>
<%= if Livebook.Config. runtime_enabled?(Livebook.Runtime.Attached) do %>
<.choice_button
active={@type == "attached"}
@ -83,12 +73,10 @@ defmodule LivebookWeb.SessionLive.RuntimeComponent do
end
defp runtime_type(%Runtime.ElixirStandalone{}), do: "elixir_standalone"
defp runtime_type(%Runtime.MixStandalone{}), do: "mix_standalone"
defp runtime_type(%Runtime.Attached{}), do: "attached"
defp runtime_type(%Runtime.Embedded{}), do: "embedded"
defp live_view_for_type("elixir_standalone"), do: LivebookWeb.SessionLive.ElixirStandaloneLive
defp live_view_for_type("mix_standalone"), do: LivebookWeb.SessionLive.MixStandaloneLive
defp live_view_for_type("attached"), do: LivebookWeb.SessionLive.AttachedLive
defp live_view_for_type("embedded"), do: LivebookWeb.SessionLive.EmbeddedLive

View file

@ -1,44 +0,0 @@
defmodule Livebook.Runtime.MixStandaloneTest do
use ExUnit.Case, async: true
alias Livebook.Runtime
test "integration" do
# Start node initialization
project_path = Path.expand("../../support/project", __DIR__)
emitter = Livebook.Utils.Emitter.new(self())
runtime = Runtime.MixStandalone.new(project_path)
Runtime.MixStandalone.connect_async(runtime, emitter)
ref = emitter.ref
# Wait for the Mix setup to finish and for node initialization
assert_receive {:emitter, ^ref, {:output, "Running mix deps.get...\n"}}, 15_000
assert_receive {:emitter, ^ref, {:ok, runtime}}, 15_000
Runtime.take_ownership(runtime)
%{node: node} = runtime
# Make sure the node is running.
Node.monitor(node, true)
assert :pong = Node.ping(node)
# Ensure the initialization works
assert evaluator_module_loaded?(node)
assert manager_started?(node)
# Ensure modules from the Mix project are available
assert :rpc.call(node, Project, :hello, []) == "hello"
# Stopping the runtime should also terminate the node
Runtime.disconnect(runtime)
assert_receive {:nodedown, ^node}
end
defp evaluator_module_loaded?(node) do
:rpc.call(node, :code, :is_loaded, [Livebook.Runtime.Evaluator]) != false
end
defp manager_started?(node) do
:rpc.call(node, Process, :whereis, [Livebook.Runtime.ErlDist.NodeManager]) != nil
end
end

View file

@ -1,45 +0,0 @@
defmodule Livebook.Utils.EmitterTest do
use ExUnit.Case, async: true
alias Livebook.Utils.Emitter
describe "emit/2" do
test "sends the item as a message to the specified process" do
emitter = Emitter.new(self())
ref = emitter.ref
Emitter.emit(emitter, :hey)
assert_receive {:emitter, ^ref, :hey}
end
end
describe "map/2" do
test "returns a modified emitter that transforms items before they are sent" do
emitter = Emitter.new(self())
ref = emitter.ref
string_emitter = Emitter.mapper(emitter, &to_string/1)
Emitter.emit(string_emitter, :hey)
assert_receive {:emitter, ^ref, "hey"}
end
test "supports chaining" do
emitter = Emitter.new(self())
ref = emitter.ref
string_emitter = Emitter.mapper(emitter, &to_string/1)
duplicate_emitter = Emitter.mapper(string_emitter, fn x -> {x, x} end)
Emitter.emit(duplicate_emitter, :hey)
assert_receive {:emitter, ^ref, {"hey", "hey"}}
end
end
test "implements Collectable so that it emits every item" do
emitter = Emitter.new(self())
ref = emitter.ref
for x <- ["a", "b"], into: emitter, do: x
assert_receive {:emitter, ^ref, "a"}
assert_receive {:emitter, ^ref, "b"}
end
end

View file

@ -13,7 +13,6 @@ Application.put_env(:livebook, :default_runtime, Livebook.Runtime.Embedded.new()
Application.put_env(:livebook, :runtime_modules, [
Livebook.Runtime.ElixirStandalone,
Livebook.Runtime.MixStandalone,
Livebook.Runtime.Attached,
Livebook.Runtime.Embedded
])