From 50ac90a79d27bbb72f5c0d3817a440fc0f148151 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20N=2EC=2E=20van=20=C2=B4t=20Hooft?= Date: Wed, 19 Jul 2023 14:00:47 -0300 Subject: [PATCH] Add erlang-module support Single modules are now allowed to be defined in an Erlang-cell. In this case the entire code-block is interpreted as an erlang-module and if there are no errors the module is compiled and loaded. If the cell containing the module is deleted subsequent code invocations will fail. Stale indicator is not working yet due to missing notion of functions, this will be fixed in a next version. Error handling - basics are working - Still not happy with it, but it is usable The code is now pre-analyzed using the epp-module. This also has the added advantage that erlang modules can become a bit more expressive with defines and includes. TODO: Examples need to be added to the example livebook. --- lib/livebook/intellisense.ex | 6 +- lib/livebook/notebook_manager.ex | 6 +- lib/livebook/runtime/evaluator.ex | 95 +++++++++++++++++--- lib/livebook_web/plugs/static_plug.ex | 2 +- test/livebook/runtime/evaluator_test.exs | 70 +++++++++++++-- test/livebook_web/live/session_live_test.exs | 8 +- 6 files changed, 161 insertions(+), 26 deletions(-) diff --git a/lib/livebook/intellisense.ex b/lib/livebook/intellisense.ex index 5ecf4abf9..0ba2215c7 100644 --- a/lib/livebook/intellisense.ex +++ b/lib/livebook/intellisense.ex @@ -303,7 +303,11 @@ defmodule Livebook.Intellisense do end } - defp format_completion_item(%{kind: :module_attribute, name: name, documentation: documentation}), + defp format_completion_item(%{ + kind: :module_attribute, + name: name, + documentation: documentation + }), do: %{ label: Atom.to_string(name), kind: :variable, diff --git a/lib/livebook/notebook_manager.ex b/lib/livebook/notebook_manager.ex index a1ac4b2ec..40e741fd4 100644 --- a/lib/livebook/notebook_manager.ex +++ b/lib/livebook/notebook_manager.ex @@ -258,7 +258,11 @@ defmodule Livebook.NotebookManager do end end - defp load_file(%{file_system_id: file_system_id, file_system_type: file_system_type, path: path}) do + defp load_file(%{ + file_system_id: file_system_id, + file_system_type: file_system_type, + path: path + }) do %FileSystem.File{ file_system_id: file_system_id, file_system_module: Livebook.FileSystems.type_to_module(file_system_type), diff --git a/lib/livebook/runtime/evaluator.ex b/lib/livebook/runtime/evaluator.ex index e1b952e4d..ae506af53 100644 --- a/lib/livebook/runtime/evaluator.ex +++ b/lib/livebook/runtime/evaluator.ex @@ -318,7 +318,8 @@ defmodule Livebook.Runtime.Evaluator do contexts: %{}, initial_context: context, initial_context_version: nil, - ignored_pdict_keys: ignored_pdict_keys + ignored_pdict_keys: ignored_pdict_keys, + tmp_dir: tmp_dir } :proc_lib.init_ack(evaluator) @@ -434,7 +435,10 @@ defmodule Livebook.Runtime.Evaluator do end start_time = System.monotonic_time() - {eval_result, code_markers} = eval(language, code, context.binding, context.env) + + {eval_result, code_markers} = + eval(language, code, context.binding, context.env, state.tmp_dir) + evaluation_time_ms = time_diff_ms(start_time) %{tracer_info: tracer_info} = Evaluator.IOProxy.after_evaluation(state.io_proxy) @@ -636,7 +640,7 @@ defmodule Livebook.Runtime.Evaluator do |> Map.update!(:context_modules, &(&1 ++ prev_env.context_modules)) end - defp eval(:elixir, code, binding, env) do + defp eval(:elixir, code, binding, env, _tmp_dir) do {{result, extra_diagnostics}, diagnostics} = Code.with_diagnostics([log: true], fn -> try do @@ -700,20 +704,87 @@ defmodule Livebook.Runtime.Evaluator do {result, code_markers} end - defp eval(:erlang, code, binding, env) do + # Erlang code is either statements as currently supported, or modules. + # In case we want to support modules - it makes sense to allow users to use + # includes, defines and thus we use the epp-module first - try to find out + # + # if in the tokens from erl_scan we find at least 1 module-token we assume + # that the user is defining a module, if not the previous code is called. + + defp eval(:erlang, code, binding, env, tmp_dir) do + case :erl_scan.string(String.to_charlist(code), {1, 1}, [:text]) do + {:ok, [{:-, _}, {:atom, _, :module} | _], _} -> + eval_erlang_module(code, binding, env, tmp_dir) + + {:ok, tokens, _} -> + eval_erlang_statements(code, tokens, binding, env) + + {:error, {location, module, description}, _end_loc} -> + process_erlang_error(env, code, location, module, description) + end + end + + # Explain to user: without tmp_dir to write files, they cannot compile erlang-modules + defp eval_erlang_module(_code, _binding, _env, nil) do + {{:error, :error, "writing Erlang modules requires a writeable file system", []}, []} + end + + defp eval_erlang_module(code, binding, env, tmp_dir) do + # Consider using in-memory file, once :ram file supports IO device API. + # See https://github.com/erlang/otp/issues/7239 + filename = Path.join(tmp_dir, "epp.tmp") + File.mkdir_p!(tmp_dir) + File.write!(filename, code) + + try do + {:ok, forms} = :epp.parse_file(filename, source_name: String.to_charlist(env.file)) + + case :compile.forms(forms) do + {:ok, module, binary} -> + file = + if ebin_path = ebin_path() do + Path.join(ebin_path, "#{module}.beam") + else + "#{module}.beam" + end + + {:module, module} = + :code.load_binary(module, String.to_charlist(file), binary) + + # Registration of module + env = %{env | module: module, versioned_vars: %{}} + Evaluator.Tracer.trace({:on_module, binary, %{}}, env) + + {{:ok, {:ok, module}, binding, env}, []} + + # TODO: deal with errors and reports as diagnostics + :error -> + {{:error, :error, "compile forms error", []}, []} + end + catch + kind, error -> + stacktrace = prune_stacktrace(:erl_eval, __STACKTRACE__) + {{:error, kind, error, stacktrace}, []} + after + # Clean up after ourselves. + _ = File.rm(filename) + end + end + + defp eval_erlang_statements(code, tokens, binding, env) do try do erl_binding = Enum.reduce(binding, %{}, fn {name, value}, erl_binding -> :erl_eval.add_binding(elixir_to_erlang_var(name), value, erl_binding) end) - with {:ok, tokens, _} <- :erl_scan.string(String.to_charlist(code), {1, 1}, [:text]), - {:ok, parsed} <- :erl_parse.parse_exprs(tokens), + with {:ok, parsed} <- :erl_parse.parse_exprs(tokens), {:value, result, new_erl_binding} <- :erl_eval.exprs(parsed, erl_binding) do # Simple heuristic to detect the used variables. We look at # the tokens and assume all var tokens are used variables. # This will not handle shadowing of variables in fun definitions # and will only work well enough for expressions, not for modules. + used_vars = for {:var, _anno, name} <- tokens, do: {erlang_to_elixir_var(name), nil}, @@ -743,10 +814,6 @@ defmodule Livebook.Runtime.Evaluator do {{:ok, result, binding, env}, []} else - # Tokenizer error - {:error, {location, module, description}, _end_loc} -> - process_erlang_error(env, code, location, module, description) - # Parser error {:error, {location, module, description}} -> process_erlang_error(env, code, location, module, description) @@ -885,11 +952,11 @@ defmodule Livebook.Runtime.Evaluator do do: {:module, module}, into: identifiers_used + # Note: `module_info` works for both Erlang and Elixir modules, as opposed to `__info__` identifiers_defined = - for {module, _line_vars} <- tracer_info.modules_defined, - version = module.__info__(:md5), - do: {{:module, module}, version}, - into: identifiers_defined + for {module, _line_vars} <- tracer_info.modules_defined, into: identifiers_defined do + {{:module, module}, module.module_info(:md5)} + end # Aliases diff --git a/lib/livebook_web/plugs/static_plug.ex b/lib/livebook_web/plugs/static_plug.ex index 14528b5a8..3620ebb12 100644 --- a/lib/livebook_web/plugs/static_plug.ex +++ b/lib/livebook_web/plugs/static_plug.ex @@ -135,7 +135,7 @@ defmodule LivebookWeb.StaticPlug do defp encoding_with_file(conn, file_provider, segments, gzip?) do cond do - file = gzip? and accept_encoding?(conn, "gzip") && file_provider.get_file(segments, :gzip) -> + file = (gzip? and accept_encoding?(conn, "gzip")) && file_provider.get_file(segments, :gzip) -> {"gzip", file} file = file_provider.get_file(segments, nil) -> diff --git a/test/livebook/runtime/evaluator_test.exs b/test/livebook/runtime/evaluator_test.exs index 9e0a4f63e..354bcb90b 100644 --- a/test/livebook/runtime/evaluator_test.exs +++ b/test/livebook/runtime/evaluator_test.exs @@ -2,18 +2,18 @@ defmodule Livebook.Runtime.EvaluatorTest do use ExUnit.Case, async: true import Livebook.TestHelpers - alias Livebook.Runtime.Evaluator + @moduletag :tmp_dir + setup ctx do ebin_path = if ctx[:with_ebin_path] do - hash = ctx.test |> to_string() |> :erlang.md5() |> Base.encode32(padding: false) - path = ["tmp", inspect(ctx.module), hash, "ebin"] |> Path.join() |> Path.expand() - File.rm_rf!(path) - File.mkdir_p!(path) - Code.append_path(path) - path + ebin_path = Path.join(ctx.tmp_dir, "ebin") + File.rm_rf!(ebin_path) + File.mkdir_p!(ebin_path) + Code.append_path(ebin_path) + ebin_path end {:ok, object_tracker} = start_supervised(Evaluator.ObjectTracker) @@ -23,6 +23,7 @@ defmodule Livebook.Runtime.EvaluatorTest do send_to: self(), object_tracker: object_tracker, client_tracker: client_tracker, + tmp_dir: ctx.tmp_dir || nil, ebin_path: ebin_path ] @@ -1388,6 +1389,61 @@ defmodule Livebook.Runtime.EvaluatorTest do assert_receive {:runtime_evaluation_response, :code_1, terminal_text("6"), metadata()} end + @tag :with_ebin_path + test "evaluate erlang-module code", %{evaluator: evaluator} do + code = """ + -module(tryme). + + -export([go/0, macros/0]). + + go() -> {ok,went}. + macros() -> {?MODULE, ?FILE}. + """ + + Evaluator.evaluate_code(evaluator, :erlang, code, :code_1, []) + + assert_receive {:runtime_evaluation_response, :code_1, terminal_text(_), metadata()} + + Evaluator.evaluate_code(evaluator, :erlang, "tryme:macros().", :code_2, [:code_1]) + assert_receive {:runtime_evaluation_response, :code_2, terminal_text(output), metadata()} + assert output == "{tryme,\"nofile\"}" + end + + @tag tmp_dir: false + test "evaluate erlang-module code without filesystem", %{evaluator: evaluator} do + code = """ + -module(tryme). + + -export([go/0]). + + go() -> {ok,went}. + """ + + Evaluator.evaluate_code(evaluator, :erlang, code, :code_1, []) + assert_receive {:runtime_evaluation_response, :code_1, error(message), metadata()} + assert message =~ "writing Erlang modules requires a writeable file system" + end + + @tag :with_ebin_path + test "evaluate erlang-module error", %{ + evaluator: evaluator + } do + code = """ + -module(tryme). + + -export([go/0]). + + go() ->{ok,went}. + go() ->{ok,went}. + """ + + Evaluator.evaluate_code(evaluator, :erlang, code, :code_1, []) + + assert_receive {:runtime_evaluation_response, :code_1, error(message), metadata()} + + assert message =~ "compile forms error" + end + test "mixed erlang/elixir bindings", %{evaluator: evaluator} do Evaluator.evaluate_code(evaluator, :elixir, "x = 1", :code_1, []) Evaluator.evaluate_code(evaluator, :erlang, "Y = X.", :code_2, [:code_1]) diff --git a/test/livebook_web/live/session_live_test.exs b/test/livebook_web/live/session_live_test.exs index 7dd623fda..92c4c4b41 100644 --- a/test/livebook_web/live/session_live_test.exs +++ b/test/livebook_web/live/session_live_test.exs @@ -2020,7 +2020,9 @@ defmodule LivebookWeb.SessionLiveTest do view |> element(~s{form[phx-submit="save"]}) - |> render_submit(%{secret: %{name: secret.name, value: secret.value, hub_id: secret.hub_id}}) + |> render_submit(%{ + secret: %{name: secret.name, value: secret.value, hub_id: secret.hub_id} + }) assert_session_secret(view, session.pid, secret) end @@ -2078,7 +2080,9 @@ defmodule LivebookWeb.SessionLiveTest do view |> element(~s{form[phx-submit="save"]}) - |> render_submit(%{secret: %{name: secret.name, value: secret.value, hub_id: secret.hub_id}}) + |> render_submit(%{ + secret: %{name: secret.name, value: secret.value, hub_id: secret.hub_id} + }) assert_session_secret(view, session.pid, secret) refute secret in Livebook.Hubs.get_secrets(hub)