Don't run doctests for generated functions (#1966)

.github/workflows/deploy.yml

@@ -7,7 +7,7 @@ on:
       - "v*.*.*"
 env:
   otp: "25.3.2"
-  elixir: "1.15.0-rc.1"
+  elixir: "1.15.0-rc.2"
 jobs:
   assets:
     outputs:

.github/workflows/test.yml

@@ -6,7 +6,7 @@ on:
       - main
 env:
   otp: "25.3.2"
-  elixir: "1.15.0-rc.1"
+  elixir: "1.15.0-rc.2"
 jobs:
   main:
     runs-on: ubuntu-latest

.github/workflows/uffizzi-build.yml

@@ -5,7 +5,7 @@ on:
 
 env:
   otp: "25.3.2"
-  elixir: "1.15.0-rc.1"
+  elixir: "1.15.0-rc.2"
 
 jobs:
   build-application:

Dockerfile

@@ -1,5 +1,4 @@
-# TODO: update the image once available on hexpm/ builds
-ARG BASE_IMAGE=jonatanklosko/elixir:1.15.0-rc.1-erlang-25.3.2-debian-bullseye-20230522-slim
+ARG BASE_IMAGE=hexpm/elixir:1.15.0-rc.2-erlang-25.3.2-debian-bullseye-20230522-slim
 
 # Stage 1
 # Builds the Livebook release

docker/build_and_push.sh

@@ -5,7 +5,7 @@
 set -ex
 cd "$(dirname "$0")/.."
 
-elixir="1.15.0-rc.1"
+elixir="1.15.0-rc.2"
 erlang="25.3.2"
 ubuntu="focal-20230126"
 

lib/livebook/intellisense.ex

@@ -534,12 +534,14 @@ defmodule Livebook.Intellisense do
         name -> name
       end
 
+    # TODO: remove the first check on Elixir v1.15.0
     is_otp? =
-      case :code.which(app || module) do
-        :preloaded -> true
-        [_ | _] = path -> List.starts_with?(path, :code.lib_dir())
-        _ -> false
-      end
+      app == :erts or
+        case :code.which(app || module) do
+          :preloaded -> true
+          [_ | _] = path -> List.starts_with?(path, :code.lib_dir())
+          _ -> false
+        end
 
     cond do
       is_otp? ->

lib/livebook/intellisense/docs.ex

@@ -163,21 +163,4 @@ defmodule Livebook.Intellisense.Docs do
   # loads elixir.beam, so we explicitly list it.
   defp ensure_loaded?(Elixir), do: false
   defp ensure_loaded?(module), do: Code.ensure_loaded?(module)
-
-  @doc """
-  Checks if the module has any documentation.
-  """
-  @spec any_docs?(module()) :: boolean()
-  def any_docs?(module) do
-    case Code.fetch_docs(module) do
-      {:docs_v1, _, _, _, %{}, _, _} ->
-        true
-
-      {:docs_v1, _, _, _, _, _, docs} ->
-        Enum.any?(docs, &match?({_, _, _, %{}, _}, &1))
-
-      _ ->
-        false
-    end
-  end
 end

lib/livebook/runtime/evaluator.ex

@@ -460,9 +460,7 @@ defmodule Livebook.Runtime.Evaluator do
       end
 
     if ebin_path() do
-      new_context.env.context_modules
-      |> Enum.filter(&Livebook.Intellisense.Docs.any_docs?/1)
-      |> Livebook.Runtime.Evaluator.Doctests.run(code)
+      Livebook.Runtime.Evaluator.Doctests.run(new_context.env.context_modules, code)
     end
 
     state = put_context(state, ref, new_context)

lib/livebook/runtime/evaluator/doctests.ex

@@ -9,17 +9,45 @@ defmodule Livebook.Runtime.Evaluator.Doctests do
   Runs doctests in the given modules.
   """
   @spec run(list(module()), String.t()) :: :ok
-  def run(modules, code)
-
-  def run([], _code), do: :ok
-
   def run(modules, code) do
-    case define_test_module(modules) do
-      {:ok, test_module} ->
-        if test_module.tests != [] do
-          lines = String.split(code, ["\r\n", "\n"])
+    doctests_specs =
+      for module <- modules, doctests_spec = doctests_spec(module), do: doctests_spec
 
-          test_module.tests
+    do_run(doctests_specs, code)
+  end
+
+  defp doctests_spec(module) do
+    case Code.fetch_docs(module) do
+      {:docs_v1, _, _, _, doc_content, _, member_docs} ->
+        funs =
+          for {{:function, name, arity}, annotation, _signatures, _doc, _meta} <- member_docs,
+              do: %{name: name, arity: arity, generated: :erl_anno.generated(annotation)}
+
+        {generated_funs, regular_funs} = Enum.split_with(funs, & &1.generated)
+
+        if regular_funs != [] or is_map(doc_content) do
+          except = Enum.map(generated_funs, &{&1.name, &1.arity})
+          %{module: module, except: except}
+        end
+
+      _ ->
+        nil
+    end
+  end
+
+  defp do_run([], _code), do: :ok
+
+  defp do_run(doctests_specs, code) do
+    case define_test_module(doctests_specs) do
+      {:ok, test_module} ->
+        lines = String.split(code, ["\r\n", "\n"])
+
+        # Ignore test cases that don't actually point to a doctest
+        # in the source code
+        tests = Enum.filter(test_module.tests, &doctest_at_line?(lines, &1.tags.doctest_line))
+
+        if tests != [] do
+          tests
           |> Enum.sort_by(& &1.tags.doctest_line)
           |> Enum.each(fn test ->
             report_doctest_running(test)
@@ -38,6 +66,18 @@ defmodule Livebook.Runtime.Evaluator.Doctests do
     :ok
   end
 
+  defp doctest_at_line?(lines, line_number) do
+    if line = Enum.at(lines, line_number - 1) do
+      case String.trim_leading(line) do
+        "iex>" <> _ -> true
+        "iex(" <> _ -> true
+        _ -> false
+      end
+    else
+      false
+    end
+  end
+
   defp report_doctest_running(test) do
     send_doctest_report(%{
       line: test.tags.doctest_line,
@@ -122,9 +162,10 @@ defmodule Livebook.Runtime.Evaluator.Doctests do
     end
   end
 
-  defp define_test_module(modules) do
+  defp define_test_module(doctests_specs) do
     id =
-      modules
+      doctests_specs
+      |> Enum.map(& &1.module)
       |> Enum.sort()
       |> Enum.map_join("-", fn module ->
         module
@@ -140,8 +181,8 @@ defmodule Livebook.Runtime.Evaluator.Doctests do
       defmodule name do
         use ExUnit.Case, register: false
 
-        for module <- modules do
-          doctest module
+        for doctests_spec <- doctests_specs do
+          doctest doctests_spec.module, except: doctests_spec.except
         end
       end
 

mix.exs

@@ -5,7 +5,7 @@ defmodule Livebook.MixProject do
   @version "0.9.2"
   @description "Automate code & data workflows with interactive notebooks"
 
-  @app_elixir_version "1.15.0-rc.1"
+  @app_elixir_version "1.15.0-rc.2"
   @app_rebar3_version "3.22.0"
 
   def project do

test/livebook/intellisense/docs_test.exs

@@ -1,13 +0,0 @@
-defmodule Livebook.Intellisense.DocsTest do
-  use ExUnit.Case, async: true
-
-  alias Livebook.Intellisense.Docs
-
-  test "any_docs?/1" do
-    refute Docs.any_docs?(Livebook.TestModules.Docs.Without)
-    refute Docs.any_docs?(Livebook.TestModules.Docs.ModuleHidden)
-    refute Docs.any_docs?(Livebook.TestModules.Docs.FunctionHidden)
-    assert Docs.any_docs?(Livebook.TestModules.Docs.Module)
-    assert Docs.any_docs?(Livebook.TestModules.Docs.Function)
-  end
-end

test/livebook/runtime/evaluator_test.exs

@@ -154,7 +154,7 @@ defmodule Livebook.Runtime.EvaluatorTest do
       assert_receive {:runtime_evaluation_response, :code_1, {:error, message, :other},
                       metadata()}
 
-      assert clean_message(message) == """
+      assert """
              ** (FunctionClauseError) no function clause matching in List.first/2
 
                  The following arguments were given to List.first/2:
@@ -170,9 +170,7 @@ defmodule Livebook.Runtime.EvaluatorTest do
                      def first([], default)
                      def first([head | _], _default)
 
-                 (elixir 1.15.0-rc.1) lib/list.ex:293: List.first/2
-                 file.ex:1: (file)
-             """
+             """ <> _ = clean_message(message)
     end
 
     test "returns additional metadata when there is a syntax error", %{evaluator: evaluator} do
@@ -630,6 +628,69 @@ defmodule Livebook.Runtime.EvaluatorTest do
                         status: :failed
                       }}
     end
+
+    test "does not run generated doctests", %{evaluator: evaluator} do
+      code = ~S'''
+      defmodule Livebook.Runtime.EvaluatorTest.DoctestsGeneratedBase do
+        defmacro __using__(_) do
+          quote do
+            @doc """
+
+                iex> 1
+                2
+
+                iex> 2
+                2
+
+            """
+            def foo, do: :ok
+          end
+        end
+      end
+
+      defmodule Livebook.Runtime.EvaluatorTest.DoctestsGenerated do
+        use Livebook.Runtime.EvaluatorTest.DoctestsGeneratedBase
+      end
+      '''
+
+      Evaluator.evaluate_code(evaluator, :elixir, code, :code_1, [])
+
+      assert_receive {:runtime_evaluation_response, :code_1, _, metadata()}
+      refute_received {:runtime_doctest_report, :code_1, %{}}
+
+      # Here the generated doctest line matches another iex> prompt
+      # in the module, but we expect the :erl_anno check to filter
+      # it out
+
+      code = ~S'''
+      defmodule Livebook.Runtime.EvaluatorTest.DoctestsGeneratedBase do
+        defmacro __using__(_) do
+          quote do
+            @doc """
+
+                iex> 1
+                2
+
+            """
+            def foo, do: :ok
+          end
+        end
+      end
+
+      defmodule Livebook.Runtime.EvaluatorTest.DoctestsGenerated do
+        use Livebook.Runtime.EvaluatorTest.DoctestsGeneratedBase
+        @string """
+            iex> 1
+            2
+        """
+      end
+      '''
+
+      Evaluator.evaluate_code(evaluator, :elixir, code, :code_1, [])
+
+      assert_receive {:runtime_evaluation_response, :code_1, _, metadata()}
+      refute_received {:runtime_doctest_report, :code_1, %{}}
+    end
   end
 
   describe "evaluate_code/6 identifier tracking" do

test/support/test_modules/docs.ex

@@ -1,20 +0,0 @@
-defmodule Livebook.TestModules.Docs.Without do
-end
-
-defmodule Livebook.TestModules.Docs.ModuleHidden do
-  @moduledoc false
-end
-
-defmodule Livebook.TestModules.Docs.Module do
-  @moduledoc "Hello."
-end
-
-defmodule Livebook.TestModules.Docs.FunctionHidden do
-  @doc false
-  def hello(), do: "hello"
-end
-
-defmodule Livebook.TestModules.Docs.Function do
-  @doc "Hello."
-  def hello(), do: "hello"
-end