diff --git a/lib/livebook/intellisense.ex b/lib/livebook/intellisense.ex index 5ecf4abf9..0ba2215c7 100644 --- a/lib/livebook/intellisense.ex +++ b/lib/livebook/intellisense.ex @@ -303,7 +303,11 @@ defmodule Livebook.Intellisense do end } - defp format_completion_item(%{kind: :module_attribute, name: name, documentation: documentation}), + defp format_completion_item(%{ + kind: :module_attribute, + name: name, + documentation: documentation + }), do: %{ label: Atom.to_string(name), kind: :variable, diff --git a/lib/livebook/notebook_manager.ex b/lib/livebook/notebook_manager.ex index a1ac4b2ec..40e741fd4 100644 --- a/lib/livebook/notebook_manager.ex +++ b/lib/livebook/notebook_manager.ex @@ -258,7 +258,11 @@ defmodule Livebook.NotebookManager do end end - defp load_file(%{file_system_id: file_system_id, file_system_type: file_system_type, path: path}) do + defp load_file(%{ + file_system_id: file_system_id, + file_system_type: file_system_type, + path: path + }) do %FileSystem.File{ file_system_id: file_system_id, file_system_module: Livebook.FileSystems.type_to_module(file_system_type), diff --git a/lib/livebook/runtime/evaluator.ex b/lib/livebook/runtime/evaluator.ex index e1b952e4d..ae506af53 100644 --- a/lib/livebook/runtime/evaluator.ex +++ b/lib/livebook/runtime/evaluator.ex @@ -318,7 +318,8 @@ defmodule Livebook.Runtime.Evaluator do contexts: %{}, initial_context: context, initial_context_version: nil, - ignored_pdict_keys: ignored_pdict_keys + ignored_pdict_keys: ignored_pdict_keys, + tmp_dir: tmp_dir } :proc_lib.init_ack(evaluator) @@ -434,7 +435,10 @@ defmodule Livebook.Runtime.Evaluator do end start_time = System.monotonic_time() - {eval_result, code_markers} = eval(language, code, context.binding, context.env) + + {eval_result, code_markers} = + eval(language, code, context.binding, context.env, state.tmp_dir) + evaluation_time_ms = time_diff_ms(start_time) %{tracer_info: tracer_info} = Evaluator.IOProxy.after_evaluation(state.io_proxy) @@ -636,7 +640,7 @@ defmodule Livebook.Runtime.Evaluator do |> Map.update!(:context_modules, &(&1 ++ prev_env.context_modules)) end - defp eval(:elixir, code, binding, env) do + defp eval(:elixir, code, binding, env, _tmp_dir) do {{result, extra_diagnostics}, diagnostics} = Code.with_diagnostics([log: true], fn -> try do @@ -700,20 +704,87 @@ defmodule Livebook.Runtime.Evaluator do {result, code_markers} end - defp eval(:erlang, code, binding, env) do + # Erlang code is either statements as currently supported, or modules. + # In case we want to support modules - it makes sense to allow users to use + # includes, defines and thus we use the epp-module first - try to find out + # + # if in the tokens from erl_scan we find at least 1 module-token we assume + # that the user is defining a module, if not the previous code is called. + + defp eval(:erlang, code, binding, env, tmp_dir) do + case :erl_scan.string(String.to_charlist(code), {1, 1}, [:text]) do + {:ok, [{:-, _}, {:atom, _, :module} | _], _} -> + eval_erlang_module(code, binding, env, tmp_dir) + + {:ok, tokens, _} -> + eval_erlang_statements(code, tokens, binding, env) + + {:error, {location, module, description}, _end_loc} -> + process_erlang_error(env, code, location, module, description) + end + end + + # Explain to user: without tmp_dir to write files, they cannot compile erlang-modules + defp eval_erlang_module(_code, _binding, _env, nil) do + {{:error, :error, "writing Erlang modules requires a writeable file system", []}, []} + end + + defp eval_erlang_module(code, binding, env, tmp_dir) do + # Consider using in-memory file, once :ram file supports IO device API. + # See https://github.com/erlang/otp/issues/7239 + filename = Path.join(tmp_dir, "epp.tmp") + File.mkdir_p!(tmp_dir) + File.write!(filename, code) + + try do + {:ok, forms} = :epp.parse_file(filename, source_name: String.to_charlist(env.file)) + + case :compile.forms(forms) do + {:ok, module, binary} -> + file = + if ebin_path = ebin_path() do + Path.join(ebin_path, "#{module}.beam") + else + "#{module}.beam" + end + + {:module, module} = + :code.load_binary(module, String.to_charlist(file), binary) + + # Registration of module + env = %{env | module: module, versioned_vars: %{}} + Evaluator.Tracer.trace({:on_module, binary, %{}}, env) + + {{:ok, {:ok, module}, binding, env}, []} + + # TODO: deal with errors and reports as diagnostics + :error -> + {{:error, :error, "compile forms error", []}, []} + end + catch + kind, error -> + stacktrace = prune_stacktrace(:erl_eval, __STACKTRACE__) + {{:error, kind, error, stacktrace}, []} + after + # Clean up after ourselves. + _ = File.rm(filename) + end + end + + defp eval_erlang_statements(code, tokens, binding, env) do try do erl_binding = Enum.reduce(binding, %{}, fn {name, value}, erl_binding -> :erl_eval.add_binding(elixir_to_erlang_var(name), value, erl_binding) end) - with {:ok, tokens, _} <- :erl_scan.string(String.to_charlist(code), {1, 1}, [:text]), - {:ok, parsed} <- :erl_parse.parse_exprs(tokens), + with {:ok, parsed} <- :erl_parse.parse_exprs(tokens), {:value, result, new_erl_binding} <- :erl_eval.exprs(parsed, erl_binding) do # Simple heuristic to detect the used variables. We look at # the tokens and assume all var tokens are used variables. # This will not handle shadowing of variables in fun definitions # and will only work well enough for expressions, not for modules. + used_vars = for {:var, _anno, name} <- tokens, do: {erlang_to_elixir_var(name), nil}, @@ -743,10 +814,6 @@ defmodule Livebook.Runtime.Evaluator do {{:ok, result, binding, env}, []} else - # Tokenizer error - {:error, {location, module, description}, _end_loc} -> - process_erlang_error(env, code, location, module, description) - # Parser error {:error, {location, module, description}} -> process_erlang_error(env, code, location, module, description) @@ -885,11 +952,11 @@ defmodule Livebook.Runtime.Evaluator do do: {:module, module}, into: identifiers_used + # Note: `module_info` works for both Erlang and Elixir modules, as opposed to `__info__` identifiers_defined = - for {module, _line_vars} <- tracer_info.modules_defined, - version = module.__info__(:md5), - do: {{:module, module}, version}, - into: identifiers_defined + for {module, _line_vars} <- tracer_info.modules_defined, into: identifiers_defined do + {{:module, module}, module.module_info(:md5)} + end # Aliases diff --git a/lib/livebook_web/plugs/static_plug.ex b/lib/livebook_web/plugs/static_plug.ex index 14528b5a8..3620ebb12 100644 --- a/lib/livebook_web/plugs/static_plug.ex +++ b/lib/livebook_web/plugs/static_plug.ex @@ -135,7 +135,7 @@ defmodule LivebookWeb.StaticPlug do defp encoding_with_file(conn, file_provider, segments, gzip?) do cond do - file = gzip? and accept_encoding?(conn, "gzip") && file_provider.get_file(segments, :gzip) -> + file = (gzip? and accept_encoding?(conn, "gzip")) && file_provider.get_file(segments, :gzip) -> {"gzip", file} file = file_provider.get_file(segments, nil) -> diff --git a/test/livebook/runtime/evaluator_test.exs b/test/livebook/runtime/evaluator_test.exs index 9e0a4f63e..354bcb90b 100644 --- a/test/livebook/runtime/evaluator_test.exs +++ b/test/livebook/runtime/evaluator_test.exs @@ -2,18 +2,18 @@ defmodule Livebook.Runtime.EvaluatorTest do use ExUnit.Case, async: true import Livebook.TestHelpers - alias Livebook.Runtime.Evaluator + @moduletag :tmp_dir + setup ctx do ebin_path = if ctx[:with_ebin_path] do - hash = ctx.test |> to_string() |> :erlang.md5() |> Base.encode32(padding: false) - path = ["tmp", inspect(ctx.module), hash, "ebin"] |> Path.join() |> Path.expand() - File.rm_rf!(path) - File.mkdir_p!(path) - Code.append_path(path) - path + ebin_path = Path.join(ctx.tmp_dir, "ebin") + File.rm_rf!(ebin_path) + File.mkdir_p!(ebin_path) + Code.append_path(ebin_path) + ebin_path end {:ok, object_tracker} = start_supervised(Evaluator.ObjectTracker) @@ -23,6 +23,7 @@ defmodule Livebook.Runtime.EvaluatorTest do send_to: self(), object_tracker: object_tracker, client_tracker: client_tracker, + tmp_dir: ctx.tmp_dir || nil, ebin_path: ebin_path ] @@ -1388,6 +1389,61 @@ defmodule Livebook.Runtime.EvaluatorTest do assert_receive {:runtime_evaluation_response, :code_1, terminal_text("6"), metadata()} end + @tag :with_ebin_path + test "evaluate erlang-module code", %{evaluator: evaluator} do + code = """ + -module(tryme). + + -export([go/0, macros/0]). + + go() -> {ok,went}. + macros() -> {?MODULE, ?FILE}. + """ + + Evaluator.evaluate_code(evaluator, :erlang, code, :code_1, []) + + assert_receive {:runtime_evaluation_response, :code_1, terminal_text(_), metadata()} + + Evaluator.evaluate_code(evaluator, :erlang, "tryme:macros().", :code_2, [:code_1]) + assert_receive {:runtime_evaluation_response, :code_2, terminal_text(output), metadata()} + assert output == "{tryme,\"nofile\"}" + end + + @tag tmp_dir: false + test "evaluate erlang-module code without filesystem", %{evaluator: evaluator} do + code = """ + -module(tryme). + + -export([go/0]). + + go() -> {ok,went}. + """ + + Evaluator.evaluate_code(evaluator, :erlang, code, :code_1, []) + assert_receive {:runtime_evaluation_response, :code_1, error(message), metadata()} + assert message =~ "writing Erlang modules requires a writeable file system" + end + + @tag :with_ebin_path + test "evaluate erlang-module error", %{ + evaluator: evaluator + } do + code = """ + -module(tryme). + + -export([go/0]). + + go() ->{ok,went}. + go() ->{ok,went}. + """ + + Evaluator.evaluate_code(evaluator, :erlang, code, :code_1, []) + + assert_receive {:runtime_evaluation_response, :code_1, error(message), metadata()} + + assert message =~ "compile forms error" + end + test "mixed erlang/elixir bindings", %{evaluator: evaluator} do Evaluator.evaluate_code(evaluator, :elixir, "x = 1", :code_1, []) Evaluator.evaluate_code(evaluator, :erlang, "Y = X.", :code_2, [:code_1]) diff --git a/test/livebook_web/live/session_live_test.exs b/test/livebook_web/live/session_live_test.exs index 7dd623fda..92c4c4b41 100644 --- a/test/livebook_web/live/session_live_test.exs +++ b/test/livebook_web/live/session_live_test.exs @@ -2020,7 +2020,9 @@ defmodule LivebookWeb.SessionLiveTest do view |> element(~s{form[phx-submit="save"]}) - |> render_submit(%{secret: %{name: secret.name, value: secret.value, hub_id: secret.hub_id}}) + |> render_submit(%{ + secret: %{name: secret.name, value: secret.value, hub_id: secret.hub_id} + }) assert_session_secret(view, session.pid, secret) end @@ -2078,7 +2080,9 @@ defmodule LivebookWeb.SessionLiveTest do view |> element(~s{form[phx-submit="save"]}) - |> render_submit(%{secret: %{name: secret.name, value: secret.value, hub_id: secret.hub_id}}) + |> render_submit(%{ + secret: %{name: secret.name, value: secret.value, hub_id: secret.hub_id} + }) assert_session_secret(view, session.pid, secret) refute secret in Livebook.Hubs.get_secrets(hub)