diff --git a/lib/livebook/runtime/evaluator.ex b/lib/livebook/runtime/evaluator.ex
index d56939493..7043eb5b7 100644
--- a/lib/livebook/runtime/evaluator.ex
+++ b/lib/livebook/runtime/evaluator.ex
@@ -829,12 +829,10 @@ defmodule Livebook.Runtime.Evaluator do
   defp extra_diagnostic?(%SyntaxError{}), do: true
   defp extra_diagnostic?(%TokenMissingError{}), do: true
 
-  defp extra_diagnostic?(%CompileError{
-         description: "cannot compile file (errors have been logged)"
-       }),
-       do: false
+  defp extra_diagnostic?(%CompileError{description: description}) do
+    not String.contains?(description, "(errors have been logged)")
+  end
 
-  defp extra_diagnostic?(%CompileError{}), do: true
   defp extra_diagnostic?(_error), do: false
 
   defp identifier_dependencies(context, tracer_info, prev_context) do
diff --git a/test/livebook/runtime/evaluator_test.exs b/test/livebook/runtime/evaluator_test.exs
index 1b7171d5d..081ac7130 100644
--- a/test/livebook/runtime/evaluator_test.exs
+++ b/test/livebook/runtime/evaluator_test.exs
@@ -217,6 +217,32 @@ defmodule Livebook.Runtime.EvaluatorTest do
                       }}
     end
 
+    test "returns additional metadata when there is a module compilation error", %{
+      evaluator: evaluator
+    } do
+      code = """
+      defmodule Livebook.Runtime.EvaluatorTest.Invalid do
+        x
+      end
+      """
+
+      Evaluator.evaluate_code(evaluator, :elixir, code, :code_1, [], file: "file.ex")
+
+      assert_receive {:runtime_evaluation_response, :code_1,
+                      {:error,
+                       "\e[31m** (CompileError) file.ex: cannot compile module Livebook.Runtime.EvaluatorTest.Invalid " <>
+                         "(errors have been logged)\e[0m" <> _, :other},
+                      %{
+                        code_markers: [
+                          %{
+                            line: 2,
+                            description: ~s/undefined variable "x"/,
+                            severity: :error
+                          }
+                        ]
+                      }}
+    end
+
     test "ignores code errors when they happen in the actual evaluation", %{evaluator: evaluator} do
       code = """
       Code.eval_string("x")