Ensure doctests failures are properly tested

Author: José Valim

lib/ex_unit/examples/one_of_each.exs

@@ -1,29 +1,5 @@
 ExUnit.start [seed: 0]
 
-defmodule DocTest do
-  @moduledoc """
-
-      iex> 1 + * 1
-      1
-
-      iex> 1 + 1
-      3
-
-      iex> :oops
-      #HashDict<[]>
-
-      iex> Hello.world
-      :world
-
-      iex> raise "oops"
-      ** (WhatIsThis) oops
-
-      iex> raise "oops"
-      ** (RuntimeError) hello
-
-  """
-end
-
 defmodule TestOneOfEach do
   @moduledoc """
   This module contains one of each type of failing test.
@@ -146,8 +122,6 @@ defmodule TestOneOfEach do
     end
   end
 
-  doctest DocTest
-
   defp blows_up do
     ignite(0) + 1
   end

lib/ex_unit/lib/ex_unit/assertions.ex

@@ -1,14 +1,16 @@
-defexception ExUnit.AssertionError,
-    left:    :ex_unit_no_meaningful_value,
-    right:   :ex_unit_no_meaningful_value,
-    message: :ex_unit_no_meaningful_value,
-    expr:    :ex_unit_no_meaningful_value do
+defmodule ExUnit.AssertionError do
+  @no_value :ex_unit_no_meaningful_value
+
+  defexception left:    @no_value,
+               right:   @no_value,
+               message: @no_value,
+               expr:    @no_value
 
   @doc """
   Indicates no meaningful value for a field.
   """
   def no_value do
-    :ex_unit_no_meaningful_value
+    @no_value
   end
 end
 

lib/ex_unit/lib/ex_unit/case.ex

@@ -136,7 +136,7 @@ defmodule ExUnit.Case do
   via `ExUnit.configure/1`:
 
       # Exclude all external tests from running
-      ExUnit.configure exclude: [external: true]
+      ExUnit.configure(exclude: [external: true])
 
   From now on, ExUnit will not run any test that has the `external` flag
   set to true. This behaviour can be reversed with the `:include` option
@@ -150,7 +150,7 @@ defmodule ExUnit.Case do
   a particular tag by default, regardless of its value, and include only
   a certain subset:
 
-      ExUnit.configure exclude: :os, include: [os: :unix]
+      ExUnit.configure(exclude: :os, include: [os: :unix])
 
   Keep in mind that all tests are included by default, so unless they are
   excluded first, the `include` option has no effect.

lib/ex_unit/lib/ex_unit/doc_test.ex

@@ -69,7 +69,7 @@ defmodule ExUnit.DocTest do
   This is useful in two use cases:
 
   * Being able to refer to specific numbered scenarios
-  * Copy-pasting examples from an actual iex sessions
+  * Copy-pasting examples from an actual iex session
 
   We also allow you to select or skip some functions when calling
   `doctest`. See the documentation for more info.
@@ -107,7 +107,7 @@ defmodule ExUnit.DocTest do
   side effects. For example, if a doctest prints to standard output, doctest
   will not try to capture the output.
 
-  Similarly, doctest does not run in any kind of sandbox. So any module
+  Similarly, doctests do not run in any kind of sandbox. So any module
   defined in a code example is going to linger throughout the whole test
   suite run.
   """
@@ -235,11 +235,11 @@ defmodule ExUnit.DocTest do
         unquote_splicing(tests)
       rescue
         e in [ExUnit.AssertionError] ->
-          reraise e, [], stack
+          reraise e, stack
 
         error ->
           reraise ExUnit.AssertionError,
-            [message: "Doctest failed: got #{inspect(elem(error, 0))} with message #{Exception.message(error)}",
+            [message: "Doctest failed: got #{inspect(error.__struct__)} with message #{Exception.message(error)}",
              expr: unquote(whole_expr)],
             stack
       end
@@ -257,8 +257,8 @@ defmodule ExUnit.DocTest do
         actual ->
           reraise ExUnit.AssertionError,
             [message: "Doctest failed",
-              expr: "#{unquote(String.strip(expr))} === #{unquote(String.strip(expected))}",
-              left: actual],
+             expr: "#{unquote(String.strip(expr))} === #{unquote(String.strip(expected))}",
+             left: actual],
             unquote(stack)
       end
     end
@@ -274,9 +274,9 @@ defmodule ExUnit.DocTest do
         ^expected -> :ok
         actual ->
           reraise ExUnit.AssertionError,
-            [ message: "Doctest failed",
-              expr: "inspect(#{unquote(String.strip(expr))}) === #{unquote(String.strip(expected))}",
-              left: actual],
+            [message: "Doctest failed",
+             expr: "inspect(#{unquote(String.strip(expr))}) === #{unquote(String.strip(expected))}",
+             left: actual],
             unquote(stack)
       end
     end
@@ -296,7 +296,7 @@ defmodule ExUnit.DocTest do
         error ->
           unless error.__struct__ == unquote(exception) and
                  Exception.message(error) == unquote(message) do
-            got = inspect(elem(error, 0)) <> " with message " <> inspect(Exception.message(error))
+            got = inspect(error.__struct__) <> " with message " <> inspect(Exception.message(error))
             reraise ExUnit.AssertionError,
               [message: "Doctest failed: expected exception #{spec} but got #{got}",
                expr: expr],
@@ -328,7 +328,7 @@ defmodule ExUnit.DocTest do
         quote do
           reraise ExUnit.AssertionError,
             [message: "Doctest did not compile, got: #{unquote(message)}",
-             expr: unquote(String.strip(expr))]
+             expr: unquote(String.strip(expr))],
             unquote(stack)
         end
     end

lib/ex_unit/test/ex_unit/callbacks_test.exs

@@ -55,7 +55,7 @@ defmodule ExUnit.CallbacksTest do
 
   test "doesn't choke on setup errors" do
     defmodule SetupTest do
-      use ExUnit.Case, async: false
+      use ExUnit.Case
 
       setup _ do
         :ok = error
@@ -127,8 +127,6 @@ defmodule ExUnit.CallbacksTest do
       defp error, do: :error
     end
 
-    ExUnit.configure(formatters: [ExUnit.CLIFormatter])
-
     assert capture_io(fn -> ExUnit.run end) =~
            "** (MatchError) no match of right hand side value: :error"
   end

lib/ex_unit/test/ex_unit/doc_test_test.exs

@@ -47,7 +47,7 @@ defmodule ExUnit.DocTestTest.GoodModule do
   def inspect2_test, do: :ok
 end
 
-defmodule ExUnit.DocTestTest.ExceptionModule do
+defmodule ExUnit.DocTestTest.MultipleExceptions do
   @doc """
   iex> 1 + ""
   ** (ArithmeticError) bad argument in arithmetic expression
@@ -102,14 +102,27 @@ defmodule ExUnit.DocTestTest.NoImport do
 end
 
 defmodule ExUnit.DocTestTest.Invalid do
-  @doc """
-  iex> _a = 1
-  1
+  @moduledoc """
+
+      iex> 1 + * 1
+      1
+
+      iex> 1 + hd(List.flatten([1]))
+      3
+
+      iex> :oops
+      #HashDict<[]>
+
+      iex> Hello.world
+      :world
+
+      iex> raise "oops"
+      ** (WhatIsThis) oops
+
+      iex> raise "oops"
+      ** (RuntimeError) hello
 
-  iex> _a + 1
-  2
   """
-  def no_leak, do: :ok
 end
 
 defmodule ExUnit.DocTestTest.IndentationHeredocs do
@@ -163,7 +176,7 @@ defmodule ExUnit.DocTestTest.Incomplete do
 end
 
 defmodule ExUnit.DocTestTest do
-  use ExUnit.Case, async: true
+  use ExUnit.Case
 
   # This is intentional. The doctests in DocTest's docs
   # fail for demonstration purposes.
@@ -175,20 +188,79 @@ defmodule ExUnit.DocTestTest do
   doctest ExUnit.DocTestTest.NoImport
   doctest ExUnit.DocTestTest.IndentationHeredocs
 
-  test "multiple exceptions in one test case is not supported" do
-    assert_raise ExUnit.DocTest.Error, ~r"multiple exceptions in one doctest case are not supported", fn ->
-      defmodule NeverCompiled do
-        import ExUnit.DocTest
-        doctest ExUnit.DocTestTest.ExceptionModule
-      end
+  import ExUnit.CaptureIO
+
+  test "doctest failures" do
+    defmodule ActuallyCompiled do
+      use ExUnit.Case
+      doctest ExUnit.DocTestTest.Invalid
     end
+
+    ExUnit.configure(seed: 0)
+    output = capture_io(fn -> ExUnit.run end)
+
+    assert output =~ """
+      1) test moduledoc at ExUnit.DocTestTest.Invalid (1) (ExUnit.DocTestTest.ActuallyCompiled)
+         test/ex_unit/doc_test_test.exs:196
+         Doctest did not compile, got: (SyntaxError) test/ex_unit/doc_test_test.exs:104: syntax error before: '*'
+         code: 1 + * 1
+         stacktrace:
+           test/ex_unit/doc_test_test.exs:104: ExUnit.DocTestTest.Invalid (module)
+    """
+
+    assert output =~ """
+      2) test moduledoc at ExUnit.DocTestTest.Invalid (2) (ExUnit.DocTestTest.ActuallyCompiled)
+         test/ex_unit/doc_test_test.exs:196
+         Doctest failed
+         code: 1 + hd(List.flatten([1])) === 3
+         lhs:  2
+         stacktrace:
+           test/ex_unit/doc_test_test.exs:104: ExUnit.DocTestTest.Invalid (module)
+    """
+
+    assert output =~ """
+      3) test moduledoc at ExUnit.DocTestTest.Invalid (3) (ExUnit.DocTestTest.ActuallyCompiled)
+         test/ex_unit/doc_test_test.exs:196
+         Doctest failed
+         code: inspect(:oops) === "#HashDict<[]>"
+         lhs:  ":oops"
+         stacktrace:
+           test/ex_unit/doc_test_test.exs:104: ExUnit.DocTestTest.Invalid (module)
+    """
+
+    assert output =~ """
+      4) test moduledoc at ExUnit.DocTestTest.Invalid (4) (ExUnit.DocTestTest.ActuallyCompiled)
+         test/ex_unit/doc_test_test.exs:196
+         Doctest failed: got UndefinedFunctionError with message undefined function: Hello.world/0
+         code:  Hello.world
+         stacktrace:
+           test/ex_unit/doc_test_test.exs:104: ExUnit.DocTestTest.Invalid (module)
+    """
+
+    assert output =~ """
+      5) test moduledoc at ExUnit.DocTestTest.Invalid (5) (ExUnit.DocTestTest.ActuallyCompiled)
+         test/ex_unit/doc_test_test.exs:196
+         Doctest failed: expected exception WhatIsThis with message "oops" but got RuntimeError with message "oops"
+         code: raise "oops"
+         stacktrace:
+           test/ex_unit/doc_test_test.exs:104: ExUnit.DocTestTest.Invalid (module)
+    """
+
+    assert output =~ """
+      6) test moduledoc at ExUnit.DocTestTest.Invalid (6) (ExUnit.DocTestTest.ActuallyCompiled)
+         test/ex_unit/doc_test_test.exs:196
+         Doctest failed: expected exception RuntimeError with message "hello" but got RuntimeError with message "oops"
+         code: raise "oops"
+         stacktrace:
+           test/ex_unit/doc_test_test.exs:104: ExUnit.DocTestTest.Invalid (module)
+    """
   end
 
-  test "variables in heredocs do not leak" do
-    assert_raise ArgumentError, fn ->
+  test "multiple exceptions in one test case is not supported" do
+    assert_raise ExUnit.DocTest.Error, ~r"multiple exceptions in one doctest case are not supported", fn ->
       defmodule NeverCompiled do
         import ExUnit.DocTest
-        doctest ExUnit.DocTestTest.Invalid
+        doctest ExUnit.DocTestTest.MultipleExceptions
       end
     end
   end
@@ -221,6 +293,5 @@ defmodule ExUnit.DocTestTest do
         doctest ExUnit.DocTestTest.Incomplete
       end
     end
-
   end
 end