Add erlang-module support

Single modules are now allowed to be defined in an Erlang-cell.

In this case the entire code-block is interpreted as an
erlang-module and if there are no errors the module is compiled
and loaded.

If the cell containing the module is deleted subsequent code
invocations will fail.

Stale indicator is not working yet due to missing notion of
functions, this will be fixed in a next version.

Error handling - basics are working
 - Still not happy with it, but it is usable

The code is now pre-analyzed using the epp-module. This also
has the added advantage that erlang modules can become a bit
more expressive with defines and includes.

TODO: Examples need to be added to the example livebook.

lib/livebook/intellisense.ex

@@ -303,7 +303,11 @@ defmodule Livebook.Intellisense do
            end
        }
 
-  defp format_completion_item(%{kind: :module_attribute, name: name, documentation: documentation}),
+  defp format_completion_item(%{
+         kind: :module_attribute,
+         name: name,
+         documentation: documentation
+       }),
        do: %{
          label: Atom.to_string(name),
          kind: :variable,

lib/livebook/notebook_manager.ex

@@ -258,7 +258,11 @@ defmodule Livebook.NotebookManager do
     end
   end
 
-  defp load_file(%{file_system_id: file_system_id, file_system_type: file_system_type, path: path}) do
+  defp load_file(%{
+         file_system_id: file_system_id,
+         file_system_type: file_system_type,
+         path: path
+       }) do
     %FileSystem.File{
       file_system_id: file_system_id,
       file_system_module: Livebook.FileSystems.type_to_module(file_system_type),

lib/livebook/runtime/evaluator.ex

@@ -318,7 +318,8 @@ defmodule Livebook.Runtime.Evaluator do
       contexts: %{},
       initial_context: context,
       initial_context_version: nil,
-      ignored_pdict_keys: ignored_pdict_keys
+      ignored_pdict_keys: ignored_pdict_keys,
+      tmp_dir: tmp_dir
     }
 
     :proc_lib.init_ack(evaluator)
@@ -434,7 +435,10 @@ defmodule Livebook.Runtime.Evaluator do
     end
 
     start_time = System.monotonic_time()
-    {eval_result, code_markers} = eval(language, code, context.binding, context.env)
+
+    {eval_result, code_markers} =
+      eval(language, code, context.binding, context.env, state.tmp_dir)
+
     evaluation_time_ms = time_diff_ms(start_time)
 
     %{tracer_info: tracer_info} = Evaluator.IOProxy.after_evaluation(state.io_proxy)
@@ -636,7 +640,7 @@ defmodule Livebook.Runtime.Evaluator do
     |> Map.update!(:context_modules, &(&1 ++ prev_env.context_modules))
   end
 
-  defp eval(:elixir, code, binding, env) do
+  defp eval(:elixir, code, binding, env, _tmp_dir) do
     {{result, extra_diagnostics}, diagnostics} =
       Code.with_diagnostics([log: true], fn ->
         try do
@@ -700,20 +704,87 @@ defmodule Livebook.Runtime.Evaluator do
     {result, code_markers}
   end
 
-  defp eval(:erlang, code, binding, env) do
+  # Erlang code is either statements as currently supported, or modules.
+  # In case we want to support modules - it makes sense to allow users to use
+  # includes, defines and thus we use the epp-module first - try to find out
+  #
+  # if in the tokens from erl_scan we find at least 1 module-token we assume
+  # that the user is defining a module, if not the previous code is called.
+
+  defp eval(:erlang, code, binding, env, tmp_dir) do
+    case :erl_scan.string(String.to_charlist(code), {1, 1}, [:text]) do
+      {:ok, [{:-, _}, {:atom, _, :module} | _], _} ->
+        eval_erlang_module(code, binding, env, tmp_dir)
+
+      {:ok, tokens, _} ->
+        eval_erlang_statements(code, tokens, binding, env)
+
+      {:error, {location, module, description}, _end_loc} ->
+        process_erlang_error(env, code, location, module, description)
+    end
+  end
+
+  # Explain to user: without tmp_dir to write files, they cannot compile erlang-modules
+  defp eval_erlang_module(_code, _binding, _env, nil) do
+    {{:error, :error, "writing Erlang modules requires a writeable file system", []}, []}
+  end
+
+  defp eval_erlang_module(code, binding, env, tmp_dir) do
+    # Consider using in-memory file, once :ram file supports IO device API.
+    # See https://github.com/erlang/otp/issues/7239
+    filename = Path.join(tmp_dir, "epp.tmp")
+    File.mkdir_p!(tmp_dir)
+    File.write!(filename, code)
+
+    try do
+      {:ok, forms} = :epp.parse_file(filename, source_name: String.to_charlist(env.file))
+
+      case :compile.forms(forms) do
+        {:ok, module, binary} ->
+          file =
+            if ebin_path = ebin_path() do
+              Path.join(ebin_path, "#{module}.beam")
+            else
+              "#{module}.beam"
+            end
+
+          {:module, module} =
+            :code.load_binary(module, String.to_charlist(file), binary)
+
+          # Registration of module
+          env = %{env | module: module, versioned_vars: %{}}
+          Evaluator.Tracer.trace({:on_module, binary, %{}}, env)
+
+          {{:ok, {:ok, module}, binding, env}, []}
+
+        # TODO: deal with errors and reports as diagnostics
+        :error ->
+          {{:error, :error, "compile forms error", []}, []}
+      end
+    catch
+      kind, error ->
+        stacktrace = prune_stacktrace(:erl_eval, __STACKTRACE__)
+        {{:error, kind, error, stacktrace}, []}
+    after
+      # Clean up after ourselves.
+      _ = File.rm(filename)
+    end
+  end
+
+  defp eval_erlang_statements(code, tokens, binding, env) do
     try do
       erl_binding =
         Enum.reduce(binding, %{}, fn {name, value}, erl_binding ->
           :erl_eval.add_binding(elixir_to_erlang_var(name), value, erl_binding)
         end)
 
-      with {:ok, tokens, _} <- :erl_scan.string(String.to_charlist(code), {1, 1}, [:text]),
+      with {:ok, parsed} <- :erl_parse.parse_exprs(tokens),
-           {:ok, parsed} <- :erl_parse.parse_exprs(tokens),
            {:value, result, new_erl_binding} <- :erl_eval.exprs(parsed, erl_binding) do
         # Simple heuristic to detect the used variables. We look at
         # the tokens and assume all var tokens are used variables.
         # This will not handle shadowing of variables in fun definitions
         # and will only work well enough for expressions, not for modules.
+
         used_vars =
           for {:var, _anno, name} <- tokens,
               do: {erlang_to_elixir_var(name), nil},
@@ -743,10 +814,6 @@ defmodule Livebook.Runtime.Evaluator do
 
         {{:ok, result, binding, env}, []}
       else
-        # Tokenizer error
-        {:error, {location, module, description}, _end_loc} ->
-          process_erlang_error(env, code, location, module, description)
-
         # Parser error
         {:error, {location, module, description}} ->
           process_erlang_error(env, code, location, module, description)
@@ -885,11 +952,11 @@ defmodule Livebook.Runtime.Evaluator do
           do: {:module, module},
           into: identifiers_used
 
+    # Note: `module_info` works for both Erlang and Elixir modules, as opposed to `__info__`
     identifiers_defined =
-      for {module, _line_vars} <- tracer_info.modules_defined,
+      for {module, _line_vars} <- tracer_info.modules_defined, into: identifiers_defined do
-          version = module.__info__(:md5),
+        {{:module, module}, module.module_info(:md5)}
-          do: {{:module, module}, version},
+      end
-          into: identifiers_defined
 
     # Aliases
 

lib/livebook_web/plugs/static_plug.ex

@@ -135,7 +135,7 @@ defmodule LivebookWeb.StaticPlug do
 
   defp encoding_with_file(conn, file_provider, segments, gzip?) do
     cond do
-      file = gzip? and accept_encoding?(conn, "gzip") && file_provider.get_file(segments, :gzip) ->
+      file = (gzip? and accept_encoding?(conn, "gzip")) && file_provider.get_file(segments, :gzip) ->
         {"gzip", file}
 
       file = file_provider.get_file(segments, nil) ->

test/livebook/runtime/evaluator_test.exs

@@ -2,18 +2,18 @@ defmodule Livebook.Runtime.EvaluatorTest do
   use ExUnit.Case, async: true
 
   import Livebook.TestHelpers
-
   alias Livebook.Runtime.Evaluator
 
+  @moduletag :tmp_dir
+
   setup ctx do
     ebin_path =
       if ctx[:with_ebin_path] do
-        hash = ctx.test |> to_string() |> :erlang.md5() |> Base.encode32(padding: false)
+        ebin_path = Path.join(ctx.tmp_dir, "ebin")
-        path = ["tmp", inspect(ctx.module), hash, "ebin"] |> Path.join() |> Path.expand()
+        File.rm_rf!(ebin_path)
-        File.rm_rf!(path)
+        File.mkdir_p!(ebin_path)
-        File.mkdir_p!(path)
+        Code.append_path(ebin_path)
-        Code.append_path(path)
+        ebin_path
-        path
       end
 
     {:ok, object_tracker} = start_supervised(Evaluator.ObjectTracker)
@@ -23,6 +23,7 @@ defmodule Livebook.Runtime.EvaluatorTest do
       send_to: self(),
       object_tracker: object_tracker,
       client_tracker: client_tracker,
+      tmp_dir: ctx.tmp_dir || nil,
       ebin_path: ebin_path
     ]
 
@@ -1388,6 +1389,61 @@ defmodule Livebook.Runtime.EvaluatorTest do
       assert_receive {:runtime_evaluation_response, :code_1, terminal_text("6"), metadata()}
     end
 
+    @tag :with_ebin_path
+    test "evaluate erlang-module code", %{evaluator: evaluator} do
+      code = """
+      -module(tryme).
+
+      -export([go/0, macros/0]).
+
+      go() -> {ok,went}.
+      macros() -> {?MODULE, ?FILE}.
+      """
+
+      Evaluator.evaluate_code(evaluator, :erlang, code, :code_1, [])
+
+      assert_receive {:runtime_evaluation_response, :code_1, terminal_text(_), metadata()}
+
+      Evaluator.evaluate_code(evaluator, :erlang, "tryme:macros().", :code_2, [:code_1])
+      assert_receive {:runtime_evaluation_response, :code_2, terminal_text(output), metadata()}
+      assert output == "{tryme,\"nofile\"}"
+    end
+
+    @tag tmp_dir: false
+    test "evaluate erlang-module code without filesystem", %{evaluator: evaluator} do
+      code = """
+      -module(tryme).
+
+      -export([go/0]).
+
+      go() -> {ok,went}.
+      """
+
+      Evaluator.evaluate_code(evaluator, :erlang, code, :code_1, [])
+      assert_receive {:runtime_evaluation_response, :code_1, error(message), metadata()}
+      assert message =~ "writing Erlang modules requires a writeable file system"
+    end
+
+    @tag :with_ebin_path
+    test "evaluate erlang-module error", %{
+      evaluator: evaluator
+    } do
+      code = """
+      -module(tryme).
+
+      -export([go/0]).
+
+      go() ->{ok,went}.
+      go() ->{ok,went}.
+      """
+
+      Evaluator.evaluate_code(evaluator, :erlang, code, :code_1, [])
+
+      assert_receive {:runtime_evaluation_response, :code_1, error(message), metadata()}
+
+      assert message =~ "compile forms error"
+    end
+
     test "mixed erlang/elixir bindings", %{evaluator: evaluator} do
       Evaluator.evaluate_code(evaluator, :elixir, "x = 1", :code_1, [])
       Evaluator.evaluate_code(evaluator, :erlang, "Y = X.", :code_2, [:code_1])

test/livebook_web/live/session_live_test.exs

@@ -2020,7 +2020,9 @@ defmodule LivebookWeb.SessionLiveTest do
 
       view
       |> element(~s{form[phx-submit="save"]})
-      |> render_submit(%{secret: %{name: secret.name, value: secret.value, hub_id: secret.hub_id}})
+      |> render_submit(%{
+        secret: %{name: secret.name, value: secret.value, hub_id: secret.hub_id}
+      })
 
       assert_session_secret(view, session.pid, secret)
     end
@@ -2078,7 +2080,9 @@ defmodule LivebookWeb.SessionLiveTest do
 
       view
       |> element(~s{form[phx-submit="save"]})
-      |> render_submit(%{secret: %{name: secret.name, value: secret.value, hub_id: secret.hub_id}})
+      |> render_submit(%{
+        secret: %{name: secret.name, value: secret.value, hub_id: secret.hub_id}
+      })
 
       assert_session_secret(view, session.pid, secret)
       refute secret in Livebook.Hubs.get_secrets(hub)