From c4a96bc99c9b319a24fe553e9dd4c66351bca222 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jonatan=20K=C5=82osko?= <jonatanklosko@gmail.com>
Date: Wed, 9 Jun 2021 16:24:02 +0200
Subject: [PATCH]  Add support for Mix runtime as the default one (#334)

* Add support for Mix runtime as a default

* Support default runtime options in the CLI

* Set cell status to queued while runtime is being started

* Clean up tests
---
 README.md                              |   4 +-
 config/config.exs                      |   7 +-
 config/runtime.exs                     |   2 +-
 config/test.exs                        |   2 +-
 lib/livebook/config.ex                 |  63 ++++++++++---
 lib/livebook/runtime/mix_standalone.ex |  15 +++
 lib/livebook/session.ex                |  69 ++++++--------
 lib/livebook/session/data.ex           |  34 ++++++-
 lib/livebook_cli/server.ex             |  30 ++++--
 test/livebook/evaluator_test.exs       |   6 +-
 test/livebook/session/data_test.exs    | 124 ++++++++++++++++++++++---
 test/support/noop_runtime.ex           |  19 ++++
 12 files changed, 289 insertions(+), 86 deletions(-)
 create mode 100644 test/support/noop_runtime.ex

diff --git a/README.md b/README.md
index b1bb6d46c..96a0c5c3e 100644
--- a/README.md
+++ b/README.md
@@ -98,8 +98,8 @@ The following environment variables configure Livebook:
 
   * LIVEBOOK_DEFAULT_RUNTIME - sets the runtime type that is used
     by default when none is started explicitly for the given notebook.
-    Must be either "standalone" (Elixir standalone) or "embedded" (Embedded).
-    Defaults to "standalone".
+    Must be either "standalone" (Elixir standalone), "mix[:path]" (Mix standalone)
+    or "embedded" (Embedded). Defaults to "standalone".
 
   * LIVEBOOK_IP - sets the ip address to start the web application on. Must be a valid IPv4 or IPv6 address.
 
diff --git a/config/config.exs b/config/config.exs
index eccd2a137..a2085c534 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -20,10 +20,11 @@ config :livebook, :authentication_mode, :token
 # Sets the default runtime to ElixirStandalone.
 # This is the desired default most of the time,
 # but in some specific use cases you may want
-# to configure that to the Embedded runtime instead.
+# to configure that to the Embedded or Mix runtime instead.
 # Also make sure the configured runtime has
-# a synchronous `init/0` method.
-config :livebook, :default_runtime, Livebook.Runtime.ElixirStandalone
+# a synchronous `init` function that takes the
+# configured arguments.
+config :livebook, :default_runtime, {Livebook.Runtime.ElixirStandalone, []}
 
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
diff --git a/config/runtime.exs b/config/runtime.exs
index 6ceacbaf2..e2017c3c5 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -29,4 +29,4 @@ config :livebook,
 config :livebook,
        :default_runtime,
        Livebook.Config.default_runtime!("LIVEBOOK_DEFAULT_RUNTIME") ||
-         Livebook.Runtime.ElixirStandalone
+         {Livebook.Runtime.ElixirStandalone, []}
diff --git a/config/test.exs b/config/test.exs
index e57360cc8..af5bac5cd 100644
--- a/config/test.exs
+++ b/config/test.exs
@@ -15,7 +15,7 @@ config :livebook, :authentication_mode, :disabled
 # Use the embedded runtime in tests by default, so they
 # are cheaper to run. Other runtimes can be tested by starting
 # and setting them explicitly
-config :livebook, :default_runtime, Livebook.Runtime.Embedded
+config :livebook, :default_runtime, {Livebook.Runtime.Embedded, []}
 
 # Use longnames when running tests in CI, so that no host resolution is required,
 # see https://github.com/elixir-nx/livebook/pull/173#issuecomment-819468549
diff --git a/lib/livebook/config.ex b/lib/livebook/config.ex
index 55d3335c3..9e458e6be 100644
--- a/lib/livebook/config.ex
+++ b/lib/livebook/config.ex
@@ -16,9 +16,10 @@ defmodule Livebook.Config do
   end
 
   @doc """
-  Returns the runtime module to be used by default.
+  Returns the runtime module and `init` args used to start
+  the default runtime.
   """
-  @spec default_runtime() :: Livebook.Runtime.t()
+  @spec default_runtime() :: {Livebook.Runtime.t(), list()}
   def default_runtime() do
     Application.fetch_env!(:livebook, :default_runtime)
   end
@@ -141,18 +142,56 @@ defmodule Livebook.Config do
   """
   def default_runtime!(env) do
     if runtime = System.get_env(env) do
-      case runtime do
-        "standalone" ->
-          Livebook.Runtime.ElixirStandalone
+      default_runtime!(env, runtime)
+    end
+  end
 
-        "embedded" ->
-          Livebook.Runtime.Embedded
+  @doc """
+  Parses and validates default runtime within context.
+  """
+  def default_runtime!(context, runtime) do
+    case runtime do
+      "standalone" ->
+        {Livebook.Runtime.ElixirStandalone, []}
 
-        other ->
-          abort!(
-            ~s{expected #{env} to be either "standalone" or "embedded", got: #{inspect(other)}}
-          )
-      end
+      "embedded" ->
+        {Livebook.Runtime.Embedded, []}
+
+      "mix" ->
+        case mix_path(File.cwd!()) do
+          {:ok, path} ->
+            {Livebook.Runtime.MixStandalone, [path]}
+
+          :error ->
+            abort!(
+              "the current directory is not a Mix project, make sure to specify the path explicitly with mix:path"
+            )
+        end
+
+      "mix:" <> path ->
+        case mix_path(path) do
+          {:ok, path} ->
+            {Livebook.Runtime.MixStandalone, [path]}
+
+          :error ->
+            abort!(~s{"#{path}" does not point to a Mix project})
+        end
+
+      other ->
+        abort!(
+          ~s{expected #{context} to be either "standalone", "mix[:path]" or "embedded", got: #{inspect(other)}}
+        )
+    end
+  end
+
+  defp mix_path(path) do
+    path = Path.expand(path)
+    mixfile = Path.join(path, "mix.exs")
+
+    if File.exists?(mixfile) do
+      {:ok, path}
+    else
+      :error
     end
   end
 
diff --git a/lib/livebook/runtime/mix_standalone.ex b/lib/livebook/runtime/mix_standalone.ex
index a01ac9765..72d2a3e78 100644
--- a/lib/livebook/runtime/mix_standalone.ex
+++ b/lib/livebook/runtime/mix_standalone.ex
@@ -74,6 +74,21 @@ defmodule Livebook.Runtime.MixStandalone do
     :ok
   end
 
+  @doc """
+  A synchronous version of of `init_async/2`.
+  """
+  @spec init(String.t()) :: {:ok, t()} | {:error, String.t()}
+  def init(project_path) do
+    %{ref: ref} = emitter = Livebook.Utils.Emitter.new(self())
+
+    init_async(project_path, emitter)
+
+    receive do
+      {:emitter, ^ref, {:ok, runtime}} -> {:ok, runtime}
+      {:emitter, ^ref, {:error, error}} -> {:error, error}
+    end
+  end
+
   defp run_mix_task(task, project_path, output_emitter) do
     Emitter.emit(output_emitter, "Running mix #{task}...\n")
 
diff --git a/lib/livebook/session.ex b/lib/livebook/session.ex
index 83e8257f7..7827ab65e 100644
--- a/lib/livebook/session.ex
+++ b/lib/livebook/session.ex
@@ -379,15 +379,8 @@ defmodule Livebook.Session do
   end
 
   def handle_cast({:queue_cell_evaluation, client_pid, cell_id}, state) do
-    case ensure_runtime(state) do
-      {:ok, state} ->
-        operation = {:queue_cell_evaluation, client_pid, cell_id}
-        {:noreply, handle_operation(state, operation)}
-
-      {:error, error} ->
-        broadcast_error(state.session_id, "failed to setup runtime - #{error}")
-        {:noreply, state}
-    end
+    operation = {:queue_cell_evaluation, client_pid, cell_id}
+    {:noreply, handle_operation(state, operation)}
   end
 
   def handle_cast({:cancel_cell_evaluation, client_pid, cell_id}, state) do
@@ -667,8 +660,34 @@ defmodule Livebook.Session do
     Enum.reduce(actions, state, &handle_action(&2, &1))
   end
 
-  defp handle_action(state, {:start_evaluation, cell, section}) do
-    start_evaluation(state, cell, section)
+  defp handle_action(state, :start_runtime) do
+    {runtime_module, args} = Livebook.Config.default_runtime()
+
+    case apply(runtime_module, :init, args) do
+      {:ok, runtime} ->
+        runtime_monitor_ref = Runtime.connect(runtime)
+
+        %{state | runtime_monitor_ref: runtime_monitor_ref}
+        |> handle_operation({:set_runtime, self(), runtime})
+
+      {:error, error} ->
+        broadcast_error(state.session_id, "failed to setup runtime - #{error}")
+        handle_operation(state, {:set_runtime, self(), nil})
+    end
+  end
+
+  defp handle_action(state, {:start_evaluation, cell, _section}) do
+    prev_ref =
+      state.data.notebook
+      |> Notebook.parent_cells_with_section(cell.id)
+      |> Enum.find_value(fn {cell, _} -> is_struct(cell, Cell.Elixir) && cell.id end)
+
+    file = (state.data.path || "") <> "#cell"
+    opts = [file: file]
+
+    Runtime.evaluate_code(state.data.runtime, cell.source, :main, cell.id, prev_ref, opts)
+
+    state
   end
 
   defp handle_action(state, {:stop_evaluation, _section}) do
@@ -705,34 +724,6 @@ defmodule Livebook.Session do
     Phoenix.PubSub.broadcast(Livebook.PubSub, "sessions:#{session_id}", message)
   end
 
-  defp start_evaluation(state, cell, _section) do
-    prev_ref =
-      state.data.notebook
-      |> Notebook.parent_cells_with_section(cell.id)
-      |> Enum.find_value(fn {cell, _} -> is_struct(cell, Cell.Elixir) && cell.id end)
-
-    file = (state.data.path || "") <> "#cell"
-    opts = [file: file]
-
-    Runtime.evaluate_code(state.data.runtime, cell.source, :main, cell.id, prev_ref, opts)
-
-    state
-  end
-
-  # Checks if a runtime already set, and if that's not the case
-  # starts a new standalone one.
-  defp ensure_runtime(%{data: %{runtime: nil}} = state) do
-    with {:ok, runtime} <- Livebook.Config.default_runtime().init() do
-      runtime_monitor_ref = Runtime.connect(runtime)
-
-      {:ok,
-       %{state | runtime_monitor_ref: runtime_monitor_ref}
-       |> handle_operation({:set_runtime, self(), runtime})}
-    end
-  end
-
-  defp ensure_runtime(state), do: {:ok, state}
-
   defp maybe_save_notebook(state) do
     if state.data.path != nil and state.data.dirty do
       content = LiveMarkdown.Export.notebook_to_markdown(state.data.notebook)
diff --git a/lib/livebook/session/data.ex b/lib/livebook/session/data.ex
index 255670b1f..be29db00b 100644
--- a/lib/livebook/session/data.ex
+++ b/lib/livebook/session/data.ex
@@ -101,7 +101,8 @@ defmodule Livebook.Session.Data do
           | {:mark_as_not_dirty, pid()}
 
   @type action ::
-          {:start_evaluation, Cell.t(), Section.t()}
+          :start_runtime
+          | {:start_evaluation, Cell.t(), Section.t()}
           | {:stop_evaluation, Section.t()}
           | {:forget_evaluation, Cell.t(), Section.t()}
           | {:broadcast_delta, pid(), Cell.t(), Delta.t()}
@@ -258,6 +259,7 @@ defmodule Livebook.Session.Data do
       |> with_actions()
       |> queue_prerequisite_cells_evaluation(cell)
       |> queue_cell_evaluation(cell, section)
+      |> maybe_start_runtime(data)
       |> maybe_evaluate_queued()
       |> wrap_ok()
     else
@@ -431,8 +433,7 @@ defmodule Livebook.Session.Data do
   def apply_operation(data, {:set_runtime, _client_pid, runtime}) do
     data
     |> with_actions()
-    |> set!(runtime: runtime)
-    |> clear_evaluation()
+    |> set_runtime(data, runtime)
     |> wrap_ok()
   end
 
@@ -616,14 +617,27 @@ defmodule Livebook.Session.Data do
     |> reduce(invalidated_cells, &set_cell_info!(&1, &2.id, validity_status: :stale))
   end
 
+  defp maybe_start_runtime({data, _} = data_actions, prev_data) do
+    if data.runtime == nil and not any_cell_queued?(prev_data) and any_cell_queued?(data) do
+      add_action(data_actions, :start_runtime)
+    else
+      data_actions
+    end
+  end
+
+  defp any_cell_queued?(data) do
+    Enum.any?(data.section_infos, fn {_section_id, info} -> info.evaluation_queue != [] end)
+  end
+
   defp maybe_evaluate_queued({data, _} = data_actions) do
     ongoing_evaluation? =
       Enum.any?(data.notebook.sections, fn section ->
         data.section_infos[section.id].evaluating_cell_id != nil
       end)
 
-    if ongoing_evaluation? do
-      # A section is evaluating, so we don't start any new evaluation
+    if ongoing_evaluation? or data.runtime == nil do
+      # Don't tigger evaluation if there is one already,
+      # or if we simply don't have a runtime started yet
       data_actions
     else
       Enum.find_value(data.notebook.sections, data_actions, fn section ->
@@ -801,6 +815,16 @@ defmodule Livebook.Session.Data do
     |> set!(notebook: Notebook.update_cell(data.notebook, cell.id, &Map.merge(&1, attrs)))
   end
 
+  defp set_runtime(data_actions, prev_data, runtime) do
+    {data, _} = data_actions = set!(data_actions, runtime: runtime)
+
+    if prev_data.runtime == nil and data.runtime != nil do
+      maybe_evaluate_queued(data_actions)
+    else
+      clear_evaluation(data_actions)
+    end
+  end
+
   defp purge_deltas(cell_info) do
     # Given client at revision X and upstream revision Y,
     # we need Y - X last deltas that the client is not aware of,
diff --git a/lib/livebook_cli/server.ex b/lib/livebook_cli/server.ex
index 12ed6fded..9c37da286 100644
--- a/lib/livebook_cli/server.ex
+++ b/lib/livebook_cli/server.ex
@@ -19,15 +19,21 @@ defmodule LivebookCLI.Server do
 
     Available options:
 
-      --cookie      Sets a cookie for the app distributed node
-      --ip          The ip address to start the web application on, defaults to 127.0.0.1
-                    Must be a valid IPv4 or IPv6 address
-      --name        Set a name for the app distributed node
-      --no-token    Disable token authentication, enabled by default
-                    If LIVEBOOK_PASSWORD is set, it takes precedence over token auth
-      -p, --port    The port to start the web application on, defaults to 8080
-      --root-path   The root path to use for file selection
-      --sname       Set a short name for the app distributed node
+      --cookie             Sets a cookie for the app distributed node
+      --default-runtime    Sets the runtime type that is used by default when none is started
+                           explicitly for the given notebook, defaults to standalone
+                           Supported options:
+                             * standalone - Elixir standalone
+                             * mix[:path] - Mix standalone
+                             * embedded - Embedded
+      --ip                 The ip address to start the web application on, defaults to 127.0.0.1
+                           Must be a valid IPv4 or IPv6 address
+      --name               Set a name for the app distributed node
+      --no-token           Disable token authentication, enabled by default
+                           If LIVEBOOK_PASSWORD is set, it takes precedence over token auth
+      -p, --port           The port to start the web application on, defaults to 8080
+      --root-path          The root path to use for file selection
+      --sname              Set a short name for the app distributed node
 
     The --help option can be given to print this notice.
 
@@ -75,6 +81,7 @@ defmodule LivebookCLI.Server do
 
   @switches [
     cookie: :string,
+    default_runtime: :string,
     ip: :string,
     name: :string,
     port: :integer,
@@ -138,5 +145,10 @@ defmodule LivebookCLI.Server do
     opts_to_config(opts, [{:livebook, :cookie, cookie} | config])
   end
 
+  defp opts_to_config([{:default_runtime, default_runtime} | opts], config) do
+    default_runtime = Livebook.Config.default_runtime!("--default-runtime", default_runtime)
+    opts_to_config(opts, [{:livebook, :default_runtime, default_runtime} | config])
+  end
+
   defp opts_to_config([_opt | opts], config), do: opts_to_config(opts, config)
 end
diff --git a/test/livebook/evaluator_test.exs b/test/livebook/evaluator_test.exs
index 1823dcf92..1233bf369 100644
--- a/test/livebook/evaluator_test.exs
+++ b/test/livebook/evaluator_test.exs
@@ -168,7 +168,7 @@ defmodule Livebook.EvaluatorTest do
   describe "request_completion_items/5" do
     test "sends completion response to the given process", %{evaluator: evaluator} do
       Evaluator.request_completion_items(evaluator, self(), :comp_ref, "System.ver")
-      assert_receive {:completion_response, :comp_ref, [%{label: "version/0"}]}
+      assert_receive {:completion_response, :comp_ref, [%{label: "version/0"}]}, 1_000
     end
 
     test "given evaluation reference uses its bindings and env", %{evaluator: evaluator} do
@@ -181,11 +181,11 @@ defmodule Livebook.EvaluatorTest do
       assert_receive {:evaluation_response, :code_1, _}
 
       Evaluator.request_completion_items(evaluator, self(), :comp_ref, "num", :code_1)
-      assert_receive {:completion_response, :comp_ref, [%{label: "number"}]}
+      assert_receive {:completion_response, :comp_ref, [%{label: "number"}]}, 1_000
 
       Evaluator.request_completion_items(evaluator, self(), :comp_ref, "ANSI.brigh", :code_1)
 
-      assert_receive {:completion_response, :comp_ref, [%{label: "bright/0"}]}
+      assert_receive {:completion_response, :comp_ref, [%{label: "bright/0"}]}, 1_000
     end
   end
 
diff --git a/test/livebook/session/data_test.exs b/test/livebook/session/data_test.exs
index b6f2a8139..5ac0405b1 100644
--- a/test/livebook/session/data_test.exs
+++ b/test/livebook/session/data_test.exs
@@ -2,10 +2,12 @@ defmodule Livebook.Session.DataTest do
   use ExUnit.Case, async: true
 
   alias Livebook.Session.Data
-  alias Livebook.{Delta, Notebook, Runtime}
+  alias Livebook.{Delta, Notebook}
   alias Livebook.Notebook.Cell
   alias Livebook.Users.User
 
+  alias Livebook.Runtime.NoopRuntime
+
   describe "new/1" do
     test "called with no arguments defaults to a blank notebook" do
       empty_map = %{}
@@ -127,6 +129,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_section, self(), 0, "s1"},
           {:insert_cell, self(), "s1", 0, :elixir, "c1"},
           {:insert_cell, self(), "s1", 1, :elixir, "c2"},
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:queue_cell_evaluation, self(), "c2"}
         ])
@@ -166,6 +169,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_section, self(), 0, "s1"},
           {:insert_cell, self(), "s1", 0, :elixir, "c1"},
           {:insert_cell, self(), "s1", 1, :elixir, "c2"},
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:queue_cell_evaluation, self(), "c2"}
         ])
@@ -185,6 +189,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_cell, self(), "s1", 0, :elixir, "c1"},
           {:insert_cell, self(), "s1", 1, :elixir, "c2"},
           # Evaluate both cells
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:add_cell_evaluation_response, self(), "c1", {:ok, [1, 2, 3]}},
           {:queue_cell_evaluation, self(), "c2"},
@@ -241,6 +246,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_cell, self(), "s1", 2, :elixir, "c3"},
           {:insert_cell, self(), "s1", 3, :elixir, "c4"},
           # Evaluate cells
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:add_cell_evaluation_response, self(), "c1", {:ok, nil}},
           {:queue_cell_evaluation, self(), "c2"},
@@ -281,6 +287,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_cell, self(), "s1", 2, :elixir, "c3"},
           {:insert_cell, self(), "s1", 3, :elixir, "c4"},
           # Evaluate cells
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:add_cell_evaluation_response, self(), "c1", {:ok, nil}},
           {:queue_cell_evaluation, self(), "c2"},
@@ -345,6 +352,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_section, self(), 1, "s2"},
           {:insert_cell, self(), "s2", 0, :elixir, "c3"},
           # Evaluate cells
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:add_cell_evaluation_response, self(), "c1", {:ok, nil}},
           {:queue_cell_evaluation, self(), "c2"},
@@ -369,6 +377,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_cell, self(), "s1", 0, :elixir, "c1"},
           {:insert_cell, self(), "s1", 1, :markdown, "c2"},
           # Evaluate the Elixir cell
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:add_cell_evaluation_response, self(), "c1", {:ok, nil}}
         ])
@@ -391,6 +400,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_cell, self(), "s1", 0, :elixir, "c1"},
           {:insert_cell, self(), "s1", 1, :elixir, "c2"},
           # Evaluate the Elixir cell
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:queue_cell_evaluation, self(), "c2"}
         ])
@@ -414,6 +424,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_cell, self(), "s1", 1, :markdown, "c2"},
           {:insert_cell, self(), "s1", 2, :elixir, "c3"},
           # Evaluate cells
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:add_cell_evaluation_response, self(), "c1", {:ok, nil}},
           {:queue_cell_evaluation, self(), "c3"},
@@ -461,6 +472,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_cell, self(), "s2", 0, :elixir, "c3"},
           {:insert_cell, self(), "s2", 1, :elixir, "c4"},
           # Evaluate cells
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:add_cell_evaluation_response, self(), "c1", {:ok, nil}},
           {:queue_cell_evaluation, self(), "c2"},
@@ -505,6 +517,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_cell, self(), "s2", 0, :elixir, "c3"},
           {:insert_cell, self(), "s2", 1, :elixir, "c4"},
           # Evaluate cells
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:add_cell_evaluation_response, self(), "c1", {:ok, nil}},
           {:queue_cell_evaluation, self(), "c2"},
@@ -549,6 +562,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_cell, self(), "s2", 1, :elixir, "c2"},
           {:insert_cell, self(), "s3", 0, :elixir, "c3"},
           # Evaluate cells
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:add_cell_evaluation_response, self(), "c1", {:ok, nil}},
           {:queue_cell_evaluation, self(), "c2"},
@@ -574,6 +588,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_cell, self(), "s1", 0, :elixir, "c1"},
           {:insert_cell, self(), "s2", 0, :markdown, "c2"},
           # Evaluate the Elixir cell
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:add_cell_evaluation_response, self(), "c1", {:ok, nil}}
         ])
@@ -597,6 +612,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_cell, self(), "s1", 0, :elixir, "c1"},
           {:insert_cell, self(), "s2", 0, :elixir, "c2"},
           # Evaluate the Elixir cell
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:queue_cell_evaluation, self(), "c2"}
         ])
@@ -624,6 +640,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_cell, self(), "s3", 0, :elixir, "c3"},
           {:insert_cell, self(), "s4", 0, :markdown, "c4"},
           # Evaluate cells
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:add_cell_evaluation_response, self(), "c1", {:ok, nil}},
           {:queue_cell_evaluation, self(), "c3"},
@@ -665,6 +682,7 @@ defmodule Livebook.Session.DataTest do
         data_after_operations!([
           {:insert_section, self(), 0, "s1"},
           {:insert_cell, self(), "s1", 0, :elixir, "c1"},
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"}
         ])
 
@@ -672,7 +690,7 @@ defmodule Livebook.Session.DataTest do
       assert :error = Data.apply_operation(data, operation)
     end
 
-    test "marks the cell as evaluating if the corresponding section is idle" do
+    test "returns start runtime action if there is no runtime and this is the first evaluation" do
       data =
         data_after_operations!([
           {:insert_section, self(), 0, "s1"},
@@ -681,6 +699,46 @@ defmodule Livebook.Session.DataTest do
 
       operation = {:queue_cell_evaluation, self(), "c1"}
 
+      assert {:ok,
+              %{
+                cell_infos: %{"c1" => %{evaluation_status: :queued}},
+                section_infos: %{"s1" => %{evaluating_cell_id: nil, evaluation_queue: ["c1"]}}
+              }, [:start_runtime]} = Data.apply_operation(data, operation)
+    end
+
+    test "only queues the cell if runtime start has already been requested" do
+      data =
+        data_after_operations!([
+          {:insert_section, self(), 0, "s1"},
+          {:insert_cell, self(), "s1", 0, :elixir, "c1"},
+          {:insert_cell, self(), "s1", 1, :elixir, "c2"},
+          {:queue_cell_evaluation, self(), "c1"}
+        ])
+
+      operation = {:queue_cell_evaluation, self(), "c2"}
+
+      assert {:ok,
+              %{
+                cell_infos: %{
+                  "c1" => %{evaluation_status: :queued},
+                  "c2" => %{evaluation_status: :queued}
+                },
+                section_infos: %{
+                  "s1" => %{evaluating_cell_id: nil, evaluation_queue: ["c1", "c2"]}
+                }
+              }, []} = Data.apply_operation(data, operation)
+    end
+
+    test "marks the cell as evaluating if the corresponding section is idle" do
+      data =
+        data_after_operations!([
+          {:insert_section, self(), 0, "s1"},
+          {:insert_cell, self(), "s1", 0, :elixir, "c1"},
+          {:set_runtime, self(), NoopRuntime.new()}
+        ])
+
+      operation = {:queue_cell_evaluation, self(), "c1"}
+
       assert {:ok,
               %{
                 cell_infos: %{"c1" => %{evaluation_status: :evaluating}},
@@ -692,7 +750,8 @@ defmodule Livebook.Session.DataTest do
       data =
         data_after_operations!([
           {:insert_section, self(), 0, "s1"},
-          {:insert_cell, self(), "s1", 0, :elixir, "c1"}
+          {:insert_cell, self(), "s1", 0, :elixir, "c1"},
+          {:set_runtime, self(), NoopRuntime.new()}
         ])
 
       operation = {:queue_cell_evaluation, self(), "c1"}
@@ -707,6 +766,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_section, self(), 0, "s1"},
           {:insert_cell, self(), "s1", 0, :elixir, "c1"},
           {:insert_cell, self(), "s1", 1, :elixir, "c2"},
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"}
         ])
 
@@ -726,6 +786,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_cell, self(), "s1", 0, :elixir, "c1"},
           {:insert_section, self(), 1, "s2"},
           {:insert_cell, self(), "s2", 0, :elixir, "c2"},
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"}
         ])
 
@@ -751,6 +812,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_cell, self(), "s2", 0, :elixir, "c3"},
           {:insert_cell, self(), "s2", 1, :elixir, "c4"},
           # Evaluate first 2 cells
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:add_cell_evaluation_response, self(), "c1", {:ok, [1, 2, 3]}},
           {:queue_cell_evaluation, self(), "c2"},
@@ -795,6 +857,8 @@ defmodule Livebook.Session.DataTest do
         data_after_operations!([
           {:insert_section, self(), 0, "s1"},
           {:insert_cell, self(), "s1", 0, :elixir, "c1"},
+          {:set_runtime, self(), NoopRuntime.new()},
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"}
         ])
 
@@ -817,6 +881,7 @@ defmodule Livebook.Session.DataTest do
         data_after_operations!([
           {:insert_section, self(), 0, "s1"},
           {:insert_cell, self(), "s1", 0, :elixir, "c1"},
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:add_cell_evaluation_output, self(), "c1", "Hola"}
         ])
@@ -840,6 +905,7 @@ defmodule Livebook.Session.DataTest do
         data_after_operations!([
           {:insert_section, self(), 0, "s1"},
           {:insert_cell, self(), "s1", 0, :elixir, "c1"},
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:add_cell_evaluation_output, self(), "c1", "Hola"}
         ])
@@ -863,6 +929,7 @@ defmodule Livebook.Session.DataTest do
         data_after_operations!([
           {:insert_section, self(), 0, "s1"},
           {:insert_cell, self(), "s1", 0, :elixir, "c1"},
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:add_cell_evaluation_response, self(), "c1", {:ok, [1, 2, 3]}}
         ])
@@ -888,6 +955,7 @@ defmodule Livebook.Session.DataTest do
         data_after_operations!([
           {:insert_section, self(), 0, "s1"},
           {:insert_cell, self(), "s1", 0, :elixir, "c1"},
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"}
         ])
 
@@ -911,6 +979,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_section, self(), 0, "s1"},
           {:insert_cell, self(), "s1", 0, :elixir, "c1"},
           {:insert_cell, self(), "s1", 1, :elixir, "c2"},
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"}
         ])
 
@@ -929,6 +998,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_section, self(), 0, "s1"},
           {:insert_cell, self(), "s1", 0, :elixir, "c1"},
           {:insert_cell, self(), "s1", 1, :elixir, "c2"},
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"}
         ])
 
@@ -955,6 +1025,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_section, self(), 0, "s1"},
           {:insert_cell, self(), "s1", 0, :elixir, "c1"},
           {:insert_cell, self(), "s1", 1, :elixir, "c2"},
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:queue_cell_evaluation, self(), "c2"}
         ])
@@ -977,6 +1048,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_cell, self(), "s1", 0, :elixir, "c1"},
           {:insert_section, self(), 1, "s2"},
           {:insert_cell, self(), "s2", 0, :elixir, "c2"},
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:queue_cell_evaluation, self(), "c2"}
         ])
@@ -995,12 +1067,13 @@ defmodule Livebook.Session.DataTest do
                Data.apply_operation(data, operation)
     end
 
-    test "if parent cells are not executed, marks them for evaluation first" do
+    test "if parent cells are not evaluated, marks them for evaluation first" do
       data =
         data_after_operations!([
           {:insert_section, self(), 0, "s1"},
           {:insert_cell, self(), "s1", 0, :elixir, "c1"},
-          {:insert_cell, self(), "s1", 1, :elixir, "c2"}
+          {:insert_cell, self(), "s1", 1, :elixir, "c2"},
+          {:set_runtime, self(), NoopRuntime.new()}
         ])
 
       operation = {:queue_cell_evaluation, self(), "c2"}
@@ -1026,6 +1099,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_section, self(), 1, "s2"},
           {:insert_cell, self(), "s2", 0, :elixir, "c3"},
           # Evaluate all cells
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:add_cell_evaluation_response, self(), "c1", {:ok, [1, 2, 3]}},
           {:queue_cell_evaluation, self(), "c2"},
@@ -1056,6 +1130,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_cell, self(), "s1", 0, :elixir, "c1"},
           {:insert_cell, self(), "s1", 1, :elixir, "c2"},
           {:insert_cell, self(), "s1", 2, :elixir, "c3"},
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:queue_cell_evaluation, self(), "c2"},
           {:queue_cell_evaluation, self(), "c3"},
@@ -1090,6 +1165,7 @@ defmodule Livebook.Session.DataTest do
         data_after_operations!([
           {:insert_section, self(), 0, "s1"},
           {:insert_cell, self(), "s1", 0, :elixir, "c1"},
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:add_cell_evaluation_response, self(), "c1", {:ok, [1, 2, 3]}}
         ])
@@ -1106,6 +1182,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_cell, self(), "s1", 1, :elixir, "c2"},
           {:insert_section, self(), 1, "s2"},
           {:insert_cell, self(), "s2", 0, :elixir, "c3"},
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:add_cell_evaluation_response, self(), "c1", {:ok, [1, 2, 3]}},
           {:queue_cell_evaluation, self(), "c2"},
@@ -1134,6 +1211,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_section, self(), 0, "s1"},
           {:insert_cell, self(), "s1", 0, :elixir, "c1"},
           {:insert_cell, self(), "s1", 1, :elixir, "c2"},
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:queue_cell_evaluation, self(), "c2"}
         ])
@@ -1150,6 +1228,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_section, self(), 0, "s1"},
           {:insert_cell, self(), "s1", 0, :elixir, "c1"},
           {:insert_cell, self(), "s1", 1, :elixir, "c2"},
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:queue_cell_evaluation, self(), "c2"}
         ])
@@ -1170,6 +1249,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_cell, self(), "s1", 0, :elixir, "c1"},
           {:insert_cell, self(), "s1", 1, :elixir, "c2"},
           {:insert_cell, self(), "s1", 2, :elixir, "c3"},
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:queue_cell_evaluation, self(), "c2"},
           {:queue_cell_evaluation, self(), "c3"}
@@ -1649,6 +1729,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_section, self(), 0, "s1"},
           {:insert_cell, self(), "s1", 0, :input, "c1"},
           {:insert_cell, self(), "s1", 1, :elixir, "c2"},
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c2"},
           {:add_cell_evaluation_response, self(), "c2", {:ok, [1, 2, 3]}}
         ])
@@ -1667,9 +1748,7 @@ defmodule Livebook.Session.DataTest do
     test "updates data with the given runtime" do
       data = Data.new()
 
-      {:ok, runtime} = Runtime.Embedded.init()
-      Runtime.connect(runtime)
-
+      runtime = NoopRuntime.new()
       operation = {:set_runtime, self(), runtime}
 
       assert {:ok, %{runtime: ^runtime}, []} = Data.apply_operation(data, operation)
@@ -1682,6 +1761,7 @@ defmodule Livebook.Session.DataTest do
           {:insert_section, self(), 0, "s1"},
           {:insert_cell, self(), "s1", 0, :elixir, "c1"},
           {:insert_cell, self(), "s1", 1, :elixir, "c2"},
+          {:set_runtime, self(), NoopRuntime.new()},
           {:queue_cell_evaluation, self(), "c1"},
           {:queue_cell_evaluation, self(), "c2"},
           # Second section with evaluating and queued cells
@@ -1692,9 +1772,7 @@ defmodule Livebook.Session.DataTest do
           {:queue_cell_evaluation, self(), "c4"}
         ])
 
-      {:ok, runtime} = Runtime.Embedded.init()
-      Runtime.connect(runtime)
-
+      runtime = NoopRuntime.new()
       operation = {:set_runtime, self(), runtime}
 
       assert {:ok,
@@ -1711,6 +1789,30 @@ defmodule Livebook.Session.DataTest do
                 }
               }, []} = Data.apply_operation(data, operation)
     end
+
+    test "starts evaluation if there was no runtime before and there is now" do
+      data =
+        data_after_operations!([
+          {:insert_section, self(), 0, "s1"},
+          {:insert_cell, self(), "s1", 0, :elixir, "c1"},
+          {:queue_cell_evaluation, self(), "c1"}
+        ])
+
+      runtime = NoopRuntime.new()
+      operation = {:set_runtime, self(), runtime}
+
+      assert {:ok,
+              %{
+                cell_infos: %{
+                  "c1" => %{evaluation_status: :evaluating}
+                },
+                section_infos: %{
+                  "s1" => %{evaluating_cell_id: "c1", evaluation_queue: []}
+                }
+              },
+              [{:start_evaluation, %{id: "c1"}, %{id: "s1"}}]} =
+               Data.apply_operation(data, operation)
+    end
   end
 
   describe "apply_operation/2 given :set_path" do
diff --git a/test/support/noop_runtime.ex b/test/support/noop_runtime.ex
new file mode 100644
index 000000000..43bb2b68e
--- /dev/null
+++ b/test/support/noop_runtime.ex
@@ -0,0 +1,19 @@
+defmodule Livebook.Runtime.NoopRuntime do
+  @moduledoc false
+
+  # A runtime that doesn't do any actual evaluation,
+  # thus not requiring any underlying resources.
+
+  defstruct []
+
+  def new(), do: %__MODULE__{}
+
+  defimpl Livebook.Runtime do
+    def connect(_), do: :ok
+    def disconnect(_), do: :ok
+    def evaluate_code(_, _, _, _, _, _ \\ []), do: :ok
+    def forget_evaluation(_, _, _), do: :ok
+    def drop_container(_, _), do: :ok
+    def request_completion_items(_, _, _, _, _, _), do: :ok
+  end
+end