diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index e8be85bdf..88af6329c 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -6,8 +6,8 @@ on: tags: - "v*.*.*" env: - otp: "25.0" - elixir: "1.14.2" + otp: "26.0" + elixir: "1.15.0-rc.1" jobs: assets: outputs: @@ -66,9 +66,9 @@ jobs: # we pass an empty ref and and the default is used ref: ${{ needs.assets.outputs.sha }} - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: @@ -84,7 +84,7 @@ jobs: type=semver,pattern={{version}} type=edge,branch=main - name: Build and push - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v4 with: context: . platforms: linux/amd64,linux/arm64 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 14860a82d..4f1f793a7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,8 +5,8 @@ on: branches: - main env: - otp: "25.0.4" - elixir: "1.14.2" + otp: "26.0" + elixir: "1.15.0-rc.1" jobs: main: runs-on: ubuntu-latest diff --git a/.github/workflows/uffizzi-build.yml b/.github/workflows/uffizzi-build.yml index 9ea6468eb..46c24ce52 100644 --- a/.github/workflows/uffizzi-build.yml +++ b/.github/workflows/uffizzi-build.yml @@ -4,8 +4,8 @@ on: types: [opened, synchronize, reopened, closed] env: - otp: "25.0" - elixir: "1.14.2" + otp: "26.0" + elixir: "1.15.0-rc.1" jobs: build-application: diff --git a/assets/js/hooks/cell.js b/assets/js/hooks/cell.js index 9c559fb9f..dfdb7fd66 100644 --- a/assets/js/hooks/cell.js +++ b/assets/js/hooks/cell.js @@ -238,8 +238,8 @@ const Cell = { this.handleEvent( `evaluation_finished:${this.props.cellId}`, - ({ code_error }) => { - liveEditor.setCodeErrorMarker(code_error); + ({ code_markers }) => { + liveEditor.setCodeMarkers(code_markers); } ); diff --git a/assets/js/hooks/cell_editor/live_editor.js b/assets/js/hooks/cell_editor/live_editor.js index 54f6b5db5..964a9bc93 100644 --- a/assets/js/hooks/cell_editor/live_editor.js +++ b/assets/js/hooks/cell_editor/live_editor.js @@ -210,33 +210,34 @@ class LiveEditor { } /** - * Adds an underline marker for the given syntax error. + * Sets underline markers for warnings and errors. * - * To clear an existing marker `null` error is also supported. + * Passing an empty list clears all markers. */ - setCodeErrorMarker(error) { + setCodeMarkers(codeMarkers) { this._ensureMounted(); - const owner = "livebook.error.syntax"; + const owner = "livebook.code-marker"; - if (error) { - const line = this.editor.getModel().getLineContent(error.line); + const editorMarkers = codeMarkers.map((codeMarker) => { + const line = this.editor.getModel().getLineContent(codeMarker.line); const [, leadingWhitespace, trailingWhitespace] = line.match(/^(\s*).*?(\s*)$/); - monaco.editor.setModelMarkers(this.editor.getModel(), owner, [ - { - startLineNumber: error.line, - startColumn: leadingWhitespace.length + 1, - endLineNumber: error.line, - endColumn: line.length + 1 - trailingWhitespace.length, - message: error.description, - severity: monaco.MarkerSeverity.Error, - }, - ]); - } else { - monaco.editor.setModelMarkers(this.editor.getModel(), owner, []); - } + return { + startLineNumber: codeMarker.line, + startColumn: leadingWhitespace.length + 1, + endLineNumber: codeMarker.line, + endColumn: line.length + 1 - trailingWhitespace.length, + message: codeMarker.description, + severity: { + error: monaco.MarkerSeverity.Error, + warning: monaco.MarkerSeverity.Warning, + }[codeMarker.severity], + }; + }); + + monaco.editor.setModelMarkers(this.editor.getModel(), owner, editorMarkers); } _mountEditor() { @@ -506,7 +507,7 @@ class LiveEditor { return this._asyncIntellisenseRequest("format", { code: content }) .then((response) => { - this.setCodeErrorMarker(response.code_error); + this.setCodeMarkers(response.code_markers); if (response.code) { /** diff --git a/lib/livebook/intellisense.ex b/lib/livebook/intellisense.ex index b91da05ce..ba643703b 100644 --- a/lib/livebook/intellisense.ex +++ b/lib/livebook/intellisense.ex @@ -65,11 +65,11 @@ defmodule Livebook.Intellisense do |> Code.format_string!() |> IO.iodata_to_binary() - %{code: formatted, code_error: nil} + %{code: formatted, code_markers: []} rescue error -> - code_error = %{line: error.line, description: error.description} - %{code: nil, code_error: code_error} + code_marker = %{line: error.line, description: error.description, severity: :error} + %{code: nil, code_markers: [code_marker]} end end @@ -148,6 +148,15 @@ defmodule Livebook.Intellisense do insert_text: Atom.to_string(name) } + defp format_completion_item(%{kind: :in_map_field, name: name}), + do: %{ + label: Atom.to_string(name), + kind: :field, + detail: "field", + documentation: nil, + insert_text: "#{name}: " + } + defp format_completion_item(%{ kind: :in_struct_field, struct: struct, @@ -413,6 +422,8 @@ defmodule Livebook.Intellisense do defp format_details_item(%{kind: :map_field, name: name}), do: code(name) + defp format_details_item(%{kind: :in_map_field, name: name}), do: code(name) + defp format_details_item(%{kind: :in_struct_field, name: name, default: default}) do join_with_divider([ code(name), diff --git a/lib/livebook/intellisense/identifier_matcher.ex b/lib/livebook/intellisense/identifier_matcher.ex index affacdf69..891b6ef29 100644 --- a/lib/livebook/intellisense/identifier_matcher.ex +++ b/lib/livebook/intellisense/identifier_matcher.ex @@ -28,6 +28,10 @@ defmodule Livebook.Intellisense.IdentifierMatcher do kind: :map_field, name: name() } + | %{ + kind: :in_map_field, + name: name() + } | %{ kind: :in_struct_field, module: module(), @@ -94,6 +98,9 @@ defmodule Livebook.Intellisense.IdentifierMatcher do {:utf32, 0} ] + @alias_only_atoms ~w(alias import require)a + @alias_only_charlists ~w(alias import require)c + @doc """ Returns a list of identifiers matching the given `hint` together with relevant information. @@ -161,7 +168,11 @@ defmodule Livebook.Intellisense.IdentifierMatcher do match_erlang_module(List.to_string(unquoted_atom), ctx) {:dot, path, hint} -> - match_dot(path, List.to_string(hint), ctx) + if alias = dot_alias_only(path, hint, ctx.fragment, ctx) do + match_alias(List.to_string(alias), ctx, false) + else + match_dot(path, List.to_string(hint), ctx) + end {:dot_arity, path, hint} -> match_dot(path, List.to_string(hint), %{ctx | matcher: @exact_matcher}) @@ -170,17 +181,18 @@ defmodule Livebook.Intellisense.IdentifierMatcher do match_default(ctx) :expr -> - match_default(ctx) + match_container_context(ctx.fragment, :expr, "", ctx) || match_default(ctx) {:local_or_var, local_or_var} -> hint = List.to_string(local_or_var) - - match_container_context(ctx.fragment, hint) || - match_in_struct_fields_or_local_or_var(hint, ctx) + match_container_context(ctx.fragment, :expr, hint, ctx) || match_local_or_var(hint, ctx) {:local_arity, local} -> match_local(List.to_string(local), %{ctx | matcher: @exact_matcher}) + {:local_call, local} when local in @alias_only_charlists -> + match_alias("", ctx, false) + {:local_call, local} -> case ctx.type do :completion -> match_default(ctx) @@ -188,7 +200,7 @@ defmodule Livebook.Intellisense.IdentifierMatcher do end {:operator, operator} when operator in ~w(:: -)c -> - match_container_context(ctx.fragment, "") || + match_container_context(ctx.fragment, :operator, "", ctx) || match_local_or_var(List.to_string(operator), ctx) {:operator, operator} -> @@ -197,6 +209,9 @@ defmodule Livebook.Intellisense.IdentifierMatcher do {:operator_arity, operator} -> match_local(List.to_string(operator), %{ctx | matcher: @exact_matcher}) + {:operator_call, operator} when operator in ~w(|)c -> + match_container_context(ctx.fragment, :expr, "", ctx) || match_default(ctx) + {:operator_call, _operator} -> match_default(ctx) @@ -280,7 +295,7 @@ defmodule Livebook.Intellisense.IdentifierMatcher do end defp match_default(ctx) do - match_in_struct_fields_or_local_or_var("", ctx) + match_local_or_var("", ctx) end defp match_alias(hint, ctx, nested?) do @@ -294,6 +309,16 @@ defmodule Livebook.Intellisense.IdentifierMatcher do end end + defp dot_alias_only(path, hint, code, ctx) do + with {:alias, alias} <- path, + [] <- hint, + :alias_only <- container_context(code, ctx) do + alias ++ [?.] + else + _ -> nil + end + end + defp match_struct(hint, ctx) do for %{kind: :module, module: module} = item <- match_alias(hint, ctx, true), has_struct?(module), @@ -313,21 +338,14 @@ defmodule Livebook.Intellisense.IdentifierMatcher do match_module_function(mod, hint, ctx) ++ match_module_type(mod, hint, ctx) end - defp match_in_struct_fields_or_local_or_var(hint, ctx) do - case expand_struct_fields(ctx) do - {:ok, struct, fields} -> - for {field, default} <- fields, - name = Atom.to_string(field), - ctx.matcher.(name, hint), - do: %{kind: :in_struct_field, struct: struct, name: field, default: default} + defp match_container_context(code, context, hint, ctx) do + case container_context(code, ctx) do + {:map, map, pairs} when context == :expr -> + container_context_map_fields(pairs, map, hint, ctx) - _ -> - match_local_or_var(hint, ctx) - end - end + {:struct, alias, pairs} when context == :expr -> + container_context_struct_fields(pairs, alias, hint, ctx) - defp match_container_context(code, hint) do - case container_context(code) do :bitstring_modifier -> existing = code |> String.split("::") |> List.last() |> String.split("-") @@ -341,12 +359,36 @@ defmodule Livebook.Intellisense.IdentifierMatcher do end end - defp container_context(code) do + defp container_context(code, ctx) do case Code.Fragment.container_cursor_to_quoted(code) do {:ok, quoted} -> case Macro.path(quoted, &match?({:__cursor__, _, []}, &1)) do - [cursor, {:"::", _, [_, cursor]}, {:<<>>, _, [_ | _]} | _] -> :bitstring_modifier - _ -> nil + [cursor, {:%{}, _, pairs}, {:%, _, [{:__aliases__, _, aliases}, _map]} | _] -> + container_context_struct(cursor, pairs, aliases, ctx) + + [ + cursor, + pairs, + {:|, _, _}, + {:%{}, _, _}, + {:%, _, [{:__aliases__, _, aliases}, _map]} | _ + ] -> + container_context_struct(cursor, pairs, aliases, ctx) + + [cursor, pairs, {:|, _, [{variable, _, nil} | _]}, {:%{}, _, _} | _] -> + container_context_map(cursor, pairs, variable, ctx) + + [cursor, {special_form, _, [cursor]} | _] when special_form in @alias_only_atoms -> + :alias_only + + [cursor | tail] -> + case remove_operators(tail, cursor) do + [{:"::", _, [_, _]}, {:<<>>, _, [_ | _]} | _] -> :bitstring_modifier + _ -> nil + end + + _ -> + nil end {:error, _} -> @@ -354,41 +396,59 @@ defmodule Livebook.Intellisense.IdentifierMatcher do end end - defp expand_struct_fields(ctx) do - with {:ok, quoted} <- Code.Fragment.container_cursor_to_quoted(ctx.fragment), - {aliases, pairs} <- find_struct_fields(quoted) do - mod_name = Enum.join(aliases, ".") - mod = expand_alias(mod_name, ctx) + defp remove_operators([{op, _, [_, previous]} = head | tail], previous) when op in [:-], + do: remove_operators(tail, head) - fields = - if has_struct?(mod) do - # Remove the keys that have already been filled, and internal keys - Map.from_struct(mod.__struct__) - |> Map.drop(Keyword.keys(pairs)) - |> Map.reject(fn {key, _} -> - key - |> Atom.to_string() - |> String.starts_with?("_") - end) - else - %{} - end + defp remove_operators(tail, _previous), + do: tail - {:ok, mod, fields} + defp container_context_struct(cursor, pairs, aliases, ctx) do + with {pairs, [^cursor]} <- Enum.split(pairs, -1), + alias = expand_alias(aliases, ctx), + true <- Keyword.keyword?(pairs) and has_struct?(alias) do + {:struct, alias, pairs} + else + _ -> nil end end - defp find_struct_fields(ast) do - ast - |> Macro.prewalker() - |> Enum.find_value(fn node -> - with {:%, _, [{:__aliases__, _, aliases}, {:%{}, _, pairs}]} <- node, - {pairs, [{:__cursor__, _, []}]} <- Enum.split(pairs, -1), - true <- Keyword.keyword?(pairs) do - {aliases, pairs} - else - _ -> nil - end + defp container_context_map(cursor, pairs, variable, ctx) do + with {pairs, [^cursor]} <- Enum.split(pairs, -1), + {:ok, map} when is_map(map) <- value_from_binding([variable], ctx), + true <- Keyword.keyword?(pairs) do + {:map, map, pairs} + else + _ -> nil + end + end + + defp container_context_map_fields(pairs, map, hint, ctx) do + map = filter_out_fields(map, pairs) + + for {key, _value} <- map, + name = Atom.to_string(key), + ctx.matcher.(name, hint), + do: %{kind: :in_map_field, name: key} + end + + defp container_context_struct_fields(pairs, mod, hint, ctx) do + map = Map.from_struct(mod.__struct__) + map = filter_out_fields(map, pairs) + + for {field, default} <- map, + name = Atom.to_string(field), + ctx.matcher.(name, hint), + do: %{kind: :in_struct_field, struct: mod, name: field, default: default} + end + + defp filter_out_fields(map, pairs) do + # Remove the keys that have already been filled, and internal keys + map + |> Map.drop(Keyword.keys(pairs)) + |> Map.reject(fn {key, _} -> + key + |> Atom.to_string() + |> String.starts_with?("_") end) end @@ -445,9 +505,14 @@ defmodule Livebook.Intellisense.IdentifierMatcher do end # Converts alias string to module atom with regard to the given env - defp expand_alias(alias, ctx) do - [name | rest] = alias |> String.split(".") |> Enum.map(&String.to_atom/1) + defp expand_alias(alias, ctx) when is_binary(alias) do + alias + |> String.split(".") + |> Enum.map(&String.to_atom/1) + |> expand_alias(ctx) + end + defp expand_alias([name | rest], ctx) do case Macro.Env.fetch_alias(ctx.intellisense_context.env, name) do {:ok, name} when rest == [] -> name {:ok, name} -> Module.concat([name | rest]) diff --git a/lib/livebook/live_markdown/export.ex b/lib/livebook/live_markdown/export.ex index ff2fdcb27..cf3dcd5bb 100644 --- a/lib/livebook/live_markdown/export.ex +++ b/lib/livebook/live_markdown/export.ex @@ -250,7 +250,7 @@ defmodule Livebook.LiveMarkdown.Export do defp render_output(_output, _ctx), do: :ignored defp encode_js_data(data) when is_binary(data), do: {:ok, data} - defp encode_js_data(data), do: Jason.encode(data) + defp encode_js_data(data), do: data |> ensure_order() |> Jason.encode() defp get_code_cell_code(%{source: source, disable_formatting: true}), do: source @@ -258,7 +258,7 @@ defmodule Livebook.LiveMarkdown.Export do defp get_code_cell_code(%{source: source}), do: format_code(source) defp render_metadata(metadata) do - metadata_json = Jason.encode!(metadata) + metadata_json = metadata |> ensure_order() |> Jason.encode!() [""] end @@ -333,7 +333,7 @@ defmodule Livebook.LiveMarkdown.Export do with {:ok, hub} <- Livebook.Hubs.fetch_hub(notebook.hub_id), {:ok, stamp} <- Livebook.Hubs.notebook_stamp(hub, notebook_source, metadata) do offset = IO.iodata_length(notebook_source) - json = Jason.encode!(%{"offset" => offset, "stamp" => stamp}) + json = %{"offset" => offset, "stamp" => stamp} |> ensure_order() |> Jason.encode!() ["\n", "", "\n"] else _ -> [] @@ -344,4 +344,14 @@ defmodule Livebook.LiveMarkdown.Export do keys = [:hub_secret_names] put_unless_default(%{}, Map.take(notebook, keys), Map.take(Notebook.new(), keys)) end + + defp ensure_order(map) do + map + |> Enum.sort() + |> Enum.map(fn + {key, %{} = value} -> {key, ensure_order(value)} + pair -> pair + end) + |> Jason.OrderedObject.new() + end end diff --git a/lib/livebook/notebook/learn/intro_to_livebook.livemd b/lib/livebook/notebook/learn/intro_to_livebook.livemd index bf2cb4bde..42992a266 100644 --- a/lib/livebook/notebook/learn/intro_to_livebook.livemd +++ b/lib/livebook/notebook/learn/intro_to_livebook.livemd @@ -230,14 +230,12 @@ setup cell to invoke `Mix.install/2` with the following arguments: ```elixir -my_app_root = Path.join(__DIR__, "..") - Mix.install( [ - {:my_app, path: my_app_root, env: :dev} + {:my_app, path: Path.join(__DIR__, ".."), env: :dev} ], - config_path: Path.join(my_app_root, "config/config.exs"), - lockfile: Path.join(my_app_root, "mix.lock") + config_path: :my_app, + lockfile: :my_app ) ``` diff --git a/lib/livebook/runtime.ex b/lib/livebook/runtime.ex index 4ec5ea3dd..7b283455b 100644 --- a/lib/livebook/runtime.ex +++ b/lib/livebook/runtime.ex @@ -86,7 +86,7 @@ defprotocol Livebook.Runtime do @type evaluation_response_metadata :: %{ errored: boolean(), evaluation_time_ms: non_neg_integer(), - code_error: code_error(), + code_markers: list(code_marker()), memory_usage: runtime_memory(), identifiers_used: list(identifier :: term()) | :unknown, identifiers_defined: %{(identifier :: term()) => version :: term()} @@ -175,13 +175,17 @@ defprotocol Livebook.Runtime do @type format_response :: %{ code: String.t() | nil, - code_error: code_error() | nil + code_markers: list(code_marker()) } @typedoc """ - A descriptive error pointing to a specific line in the code. + A descriptive error or warning pointing to a specific line in the code. """ - @type code_error :: %{line: pos_integer(), description: String.t()} + @type code_marker :: %{ + line: pos_integer(), + description: String.t(), + severity: :error | :warning + } @typedoc """ A detailed runtime memory usage. diff --git a/lib/livebook/runtime/erl_dist.ex b/lib/livebook/runtime/erl_dist.ex index 96a5de2b4..16a91c867 100644 --- a/lib/livebook/runtime/erl_dist.ex +++ b/lib/livebook/runtime/erl_dist.ex @@ -38,6 +38,7 @@ defmodule Livebook.Runtime.ErlDist do Livebook.Runtime.ErlDist.EvaluatorSupervisor, Livebook.Runtime.ErlDist.IOForwardGL, Livebook.Runtime.ErlDist.LoggerGLBackend, + Livebook.Runtime.ErlDist.LoggerGLHandler, Livebook.Runtime.ErlDist.SmartCellGL ] end diff --git a/lib/livebook/runtime/erl_dist/logger_gl_handler.ex b/lib/livebook/runtime/erl_dist/logger_gl_handler.ex new file mode 100644 index 000000000..c20332e0f --- /dev/null +++ b/lib/livebook/runtime/erl_dist/logger_gl_handler.ex @@ -0,0 +1,27 @@ +defmodule Livebook.Runtime.ErlDist.LoggerGLHandler do + @moduledoc false + + @doc false + def log(%{meta: meta} = event, %{formatter: {formatter_module, formatter_config}}) do + message = apply(formatter_module, :format, [event, formatter_config]) + + if io_proxy?(meta.gl) do + async_io(meta.gl, message) + else + send(Livebook.Runtime.ErlDist.NodeManager, {:orphan_log, message}) + end + end + + defp io_proxy?(pid) do + try do + info = Process.info(pid, [:dictionary]) + info[:dictionary][:"$initial_call"] == {Livebook.Runtime.Evaluator.IOProxy, :init, 1} + rescue + _ -> false + end + end + + def async_io(device, output) when is_pid(device) do + send(device, {:io_request, self(), make_ref(), {:put_chars, :unicode, output}}) + end +end diff --git a/lib/livebook/runtime/erl_dist/node_manager.ex b/lib/livebook/runtime/erl_dist/node_manager.ex index c786f5b29..8289b7e5d 100644 --- a/lib/livebook/runtime/erl_dist/node_manager.ex +++ b/lib/livebook/runtime/erl_dist/node_manager.ex @@ -103,7 +103,14 @@ defmodule Livebook.Runtime.ErlDist.NodeManager do Process.unregister(:standard_error) Process.register(io_forward_gl_pid, :standard_error) - Logger.add_backend(Livebook.Runtime.ErlDist.LoggerGLBackend) + # TODO: remove logger backend once we require Elixir v1.15 + if Code.ensure_loaded?(Logger) and function_exported?(Logger, :add_handlers, 1) do + :logger.add_handler(:livebook_gl_handler, Livebook.Runtime.ErlDist.LoggerGLHandler, %{ + formatter: Logger.Formatter.new() + }) + else + Logger.add_backend(Livebook.Runtime.ErlDist.LoggerGLBackend) + end # Set `ignore_module_conflict` only for the NodeManager lifetime. initial_ignore_module_conflict = Code.compiler_options()[:ignore_module_conflict] @@ -152,7 +159,12 @@ defmodule Livebook.Runtime.ErlDist.NodeManager do Process.unregister(:standard_error) Process.register(state.original_standard_error, :standard_error) - Logger.remove_backend(Livebook.Runtime.ErlDist.LoggerGLBackend) + # TODO: remove logger backend once we require Elixir v1.15 + if Code.ensure_loaded?(Logger) and function_exported?(Logger, :add_handlers, 1) do + :logger.remove_handler(:livebook_gl_handler) + else + Logger.remove_backend(Livebook.Runtime.ErlDist.LoggerGLBackend) + end if state.unload_modules_on_termination do ErlDist.unload_required_modules() diff --git a/lib/livebook/runtime/evaluator.ex b/lib/livebook/runtime/evaluator.ex index 4a35b3583..e87f42c6d 100644 --- a/lib/livebook/runtime/evaluator.ex +++ b/lib/livebook/runtime/evaluator.ex @@ -419,12 +419,12 @@ defmodule Livebook.Runtime.Evaluator do set_pdict(context, state.ignored_pdict_keys) start_time = System.monotonic_time() - eval_result = eval(code, context.binding, context.env) + {eval_result, code_markers} = eval(code, context.binding, context.env) evaluation_time_ms = time_diff_ms(start_time) %{tracer_info: tracer_info} = Evaluator.IOProxy.after_evaluation(state.io_proxy) - {new_context, result, code_error, identifiers_used, identifiers_defined} = + {new_context, result, identifiers_used, identifiers_defined} = case eval_result do {:ok, value, binding, env} -> context_id = random_id() @@ -440,9 +440,9 @@ defmodule Livebook.Runtime.Evaluator do identifier_dependencies(new_context, tracer_info, context) result = {:ok, value} - {new_context, result, nil, identifiers_used, identifiers_defined} + {new_context, result, identifiers_used, identifiers_defined} - {:error, kind, error, stacktrace, code_error} -> + {:error, kind, error, stacktrace} -> for {module, _} <- tracer_info.modules_defined do delete_module(module) end @@ -452,7 +452,7 @@ defmodule Livebook.Runtime.Evaluator do identifiers_defined = %{} # Empty context new_context = initial_context() - {new_context, result, code_error, identifiers_used, identifiers_defined} + {new_context, result, identifiers_used, identifiers_defined} end if ebin_path() do @@ -474,7 +474,7 @@ defmodule Livebook.Runtime.Evaluator do ), evaluation_time_ms: evaluation_time_ms, memory_usage: memory(), - code_error: code_error, + code_markers: code_markers, identifiers_used: identifiers_used, identifiers_defined: identifiers_defined } @@ -603,32 +603,91 @@ defmodule Livebook.Runtime.Evaluator do |> Map.update!(:context_modules, &(&1 ++ prev_env.context_modules)) end - @compile {:no_warn_undefined, {Code, :eval_quoted_with_env, 4}} - defp eval(code, binding, env) do - try do - quoted = Code.string_to_quoted!(code, file: env.file) - {value, binding, env} = Code.eval_quoted_with_env(quoted, binding, env, prune_binding: true) - {:ok, value, binding, env} - catch - kind, error -> - stacktrace = prune_stacktrace(__STACKTRACE__) + {{result, extra_diagnostics}, diagnostics} = + with_diagnostics([log: true], fn -> + try do + quoted = Code.string_to_quoted!(code, file: env.file) - code_error = - if code_error?(error) and (error.file == env.file and error.file != "nofile") do - %{line: error.line, description: error.description} - else - nil + try do + {value, binding, env} = + Code.eval_quoted_with_env(quoted, binding, env, prune_binding: true) + + {:ok, value, binding, env} + catch + kind, error -> + stacktrace = prune_stacktrace(__STACKTRACE__) + {:error, kind, error, stacktrace} end + catch + kind, error -> + {:error, kind, error, []} + end + |> case do + {:ok, value, binding, env} -> + {{:ok, value, binding, env}, []} - {:error, kind, error, stacktrace, code_error} + {:error, kind, error, stacktrace} -> + # Mimic a diagnostic for relevant errors where it's not + # the case by default + extra_diagnostics = + if extra_diagnostic?(error) do + [ + %{ + file: error.file, + severity: :error, + message: error.description, + position: error.line, + stacktrace: stacktrace + } + ] + else + [] + end + + {{:error, kind, error, stacktrace}, extra_diagnostics} + end + end) + + code_markers = + for diagnostic <- diagnostics ++ extra_diagnostics, + # Ignore diagnostics from other evaluations, such as inner Code.eval_string/3 + diagnostic.file == env.file and diagnostic.file != "nofile" do + %{ + line: + case diagnostic.position do + {line, _column} -> line + line -> line + end, + description: diagnostic.message, + severity: diagnostic.severity + } + end + + {result, code_markers} + end + + # TODO: remove once we require Elixir v1.15 + if Code.ensure_loaded?(Code) and function_exported?(Code, :with_diagnostics, 2) do + defp with_diagnostics(opts, fun) do + Code.with_diagnostics(opts, fun) + end + else + defp with_diagnostics(_opts, fun) do + {fun.(), []} end end - defp code_error?(%SyntaxError{}), do: true - defp code_error?(%TokenMissingError{}), do: true - defp code_error?(%CompileError{}), do: true - defp code_error?(_error), do: false + defp extra_diagnostic?(%SyntaxError{}), do: true + defp extra_diagnostic?(%TokenMissingError{}), do: true + + defp extra_diagnostic?(%CompileError{ + description: "cannot compile file (errors have been logged)" + }), + do: false + + defp extra_diagnostic?(%CompileError{}), do: true + defp extra_diagnostic?(_error), do: false defp identifier_dependencies(context, tracer_info, prev_context) do identifiers_used = MapSet.new() diff --git a/lib/livebook_web/live/session_live.ex b/lib/livebook_web/live/session_live.ex index 0a3c36aae..3df9340bd 100644 --- a/lib/livebook_web/live/session_live.ex +++ b/lib/livebook_web/live/session_live.ex @@ -1806,7 +1806,7 @@ defmodule LivebookWeb.SessionLive do ) do socket |> prune_outputs() - |> push_event("evaluation_finished:#{cell_id}", %{code_error: metadata.code_error}) + |> push_event("evaluation_finished:#{cell_id}", %{code_markers: metadata.code_markers}) end defp after_operation( diff --git a/main.zip b/main.zip new file mode 100644 index 000000000..94e49551f Binary files /dev/null and b/main.zip differ diff --git a/test/livebook/intellisense_test.exs b/test/livebook/intellisense_test.exs index 21fa641fb..f91a3a711 100644 --- a/test/livebook/intellisense_test.exs +++ b/test/livebook/intellisense_test.exs @@ -21,16 +21,19 @@ defmodule Livebook.IntellisenseTest do describe "format_code/1" do test "formats valid code" do - assert %{code: "1 + 1", code_error: nil} = Intellisense.format_code("1+1") + assert %{code: "1 + 1", code_markers: []} = Intellisense.format_code("1+1") end test "returns a syntax error when invalid code is given" do assert %{ code: nil, - code_error: %{ - line: 1, - description: "syntax error: expression is incomplete" - } + code_markers: [ + %{ + line: 1, + description: "syntax error: expression is incomplete", + severity: :error + } + ] } = Intellisense.format_code("1+") end end @@ -1061,10 +1064,7 @@ defmodule Livebook.IntellisenseTest do end test "completion for struct keys inside struct removes filled keys" do - context = - eval do - struct = %Livebook.IntellisenseTest.MyStruct{} - end + context = eval(do: nil) assert [] = Intellisense.get_completion_items( @@ -1081,6 +1081,73 @@ defmodule Livebook.IntellisenseTest do refute Enum.find(completions, &match?(%{label: "__exception__"}, &1)) end + test "completion for struct keys in update syntax" do + context = eval(do: nil) + + assert [ + %{ + label: "my_val", + kind: :field, + detail: "Livebook.IntellisenseTest.MyStruct struct field", + documentation: _my_val_doc, + insert_text: "my_val: " + } + ] = + Intellisense.get_completion_items( + "%Livebook.IntellisenseTest.MyStruct{struct | ", + context + ) + + assert [ + %{ + label: "my_val", + kind: :field, + detail: "Livebook.IntellisenseTest.MyStruct struct field", + documentation: _my_val_doc, + insert_text: "my_val: " + } + ] = + Intellisense.get_completion_items( + "%Livebook.IntellisenseTest.MyStruct{struct | my_v", + context + ) + + assert [] = + Intellisense.get_completion_items( + "%Livebook.IntellisenseTest.MyStruct{struct | my_val: 123, ", + context + ) + end + + test "completion for map keys in update syntax" do + context = + eval do + map = %{foo: 1} + end + + assert [ + %{ + label: "foo", + kind: :field, + detail: "field", + documentation: nil, + insert_text: "foo: " + } + ] = Intellisense.get_completion_items("%{map | ", context) + + assert [ + %{ + label: "foo", + kind: :field, + detail: "field", + documentation: nil, + insert_text: "foo: " + } + ] = Intellisense.get_completion_items("%{map | fo", context) + + assert [] = Intellisense.get_completion_items("%{map | foo: 2, ", context) + end + test "ignore invalid Elixir module literals" do context = eval(do: nil) @@ -1190,6 +1257,31 @@ defmodule Livebook.IntellisenseTest do label: "integer" } in Intellisense.get_completion_items("< Enum.any?(&(&1.kind == :function)) + end end describe "get_details/3" do diff --git a/test/livebook/runtime/evaluator_test.exs b/test/livebook/runtime/evaluator_test.exs index 297729abe..e09fafb37 100644 --- a/test/livebook/runtime/evaluator_test.exs +++ b/test/livebook/runtime/evaluator_test.exs @@ -29,7 +29,7 @@ defmodule Livebook.Runtime.EvaluatorTest do %{ evaluation_time_ms: _, memory_usage: %{}, - code_error: _, + code_markers: _, identifiers_used: _, identifiers_defined: _ } @@ -61,10 +61,7 @@ defmodule Livebook.Runtime.EvaluatorTest do Evaluator.evaluate_code(evaluator, "x", :code_2, []) assert_receive {:runtime_evaluation_response, :code_2, - {:error, _kind, - %CompileError{ - description: "undefined function x/0 (there is no such import)" - }, _stacktrace}, metadata()} + {:error, _kind, %CompileError{}, _stacktrace}, metadata()} end) end @@ -163,12 +160,15 @@ defmodule Livebook.Runtime.EvaluatorTest do Evaluator.evaluate_code(evaluator, code, :code_1, [], file: "file.ex") assert_receive {:runtime_evaluation_response, :code_1, - {:error, :error, %TokenMissingError{}, _stacktrace}, + {:error, :error, %TokenMissingError{}, []}, %{ - code_error: %{ - line: 1, - description: "syntax error: expression is incomplete" - } + code_markers: [ + %{ + line: 1, + description: "syntax error: expression is incomplete", + severity: :error + } + ] }} end @@ -180,10 +180,13 @@ defmodule Livebook.Runtime.EvaluatorTest do assert_receive {:runtime_evaluation_response, :code_1, {:error, :error, %CompileError{}, _stacktrace}, %{ - code_error: %{ - line: 1, - description: "undefined function x/0 (there is no such import)" - } + code_markers: [ + %{ + line: 1, + description: ~s/undefined variable "x"/, + severity: :error + } + ] }} end @@ -195,11 +198,13 @@ defmodule Livebook.Runtime.EvaluatorTest do Evaluator.evaluate_code(evaluator, code, :code_1, [], file: "file.ex") expected_stacktrace = [ + {:elixir_expand, :expand, 3, [file: ~c"src/elixir_expand.erl", line: 383]}, {:elixir_eval, :__FILE__, 1, [file: ~c"file.ex", line: 1]} ] assert_receive {:runtime_evaluation_response, :code_1, - {:error, :error, %CompileError{}, ^expected_stacktrace}, %{code_error: nil}} + {:error, :error, %CompileError{}, ^expected_stacktrace}, + %{code_markers: []}} end test "in case of an error returns only the relevant part of stacktrace", @@ -331,11 +336,14 @@ defmodule Livebook.Runtime.EvaluatorTest do assert_receive {:runtime_evaluation_response, :code_2, {:error, :error, %CompileError{}, []}, %{ - code_error: %{ - line: 1, - description: - "module Livebook.Runtime.EvaluatorTest.Redefinition is already defined" - } + code_markers: [ + %{ + line: 1, + description: + "module Livebook.Runtime.EvaluatorTest.Redefinition is already defined", + severity: :error + } + ] }} end @@ -633,12 +641,17 @@ defmodule Livebook.Runtime.EvaluatorTest do end test "reports parentheses-less arity-0 import as a used variable", %{evaluator: evaluator} do + # TODO: remove all logic around undefined unused vars once we require Elixir v1.15 + Code.put_compiler_option(:on_undefined_variable, :warn) + identifiers = """ self """ |> eval(evaluator, 0) + Code.put_compiler_option(:on_undefined_variable, :raise) + assert {:variable, {:self, nil}} in identifiers.used assert :imports in identifiers.used end @@ -961,10 +974,7 @@ defmodule Livebook.Runtime.EvaluatorTest do Evaluator.evaluate_code(evaluator, "x", :code_2, [:code_1]) assert_receive {:runtime_evaluation_response, :code_2, - {:error, _kind, - %CompileError{ - description: "undefined function x/0 (there is no such import)" - }, _stacktrace}, metadata()} + {:error, _kind, %CompileError{}, _stacktrace}, metadata()} end) end diff --git a/test/livebook/session_test.exs b/test/livebook/session_test.exs index 68e3d003b..7d94a71b3 100644 --- a/test/livebook/session_test.exs +++ b/test/livebook/session_test.exs @@ -718,6 +718,8 @@ defmodule Livebook.SessionTest do describe "start_link/1" do @tag :tmp_dir test "fails if the given path is already in use", %{tmp_dir: tmp_dir} do + Process.flag(:trap_exit, true) + tmp_dir = FileSystem.File.local(tmp_dir <> "/") file = FileSystem.File.resolve(tmp_dir, "notebook.livemd") start_session(file: file)