Make strict assertion failures and single operand evaluation canonical

Readme.md

@@ -255,3 +255,13 @@ While logging at the end, we can record the time consuming for this function.
 Thanks to the `tiv` tool:
 https://github.com/stefanhaustein/TerminalImageViewer
 See [header dependencies and optional modules](docs/dependencies.md) for the lightweight consumer contract, runtime dependencies and migration notes.
+
+### Strict check failures
+
+`CHECK` and its comparison variants keep their warning-only behavior by default.
+Define `ZEROERR_CHECK_AS_ERROR` for the zeroerr library and its consumers to make
+these checks count as errors and make assertion warnings fail the runner. With
+CMake, `target_compile_definitions(zeroerr PUBLIC ZEROERR_CHECK_AS_ERROR)` applies
+the policy consistently. Failed `REQUIRE`/`ASSERT` calls from context-free helpers
+also fail the enclosing test or subcase. Comparisons evaluate each operand once,
+including when formatting diagnostics for temporary operands.

include/zeroerr/assert.h

@@ -92,7 +92,12 @@ ZEROERR_CLANG_SUPPRESS_WARNING_POP
         print.isQuoted = false;                                                                  \
         zeroerr::AssertionData assertion_data(__FILE__, __LINE__, #lhs " " #op " " #rhs, info);  \
         try {                                                                                    \
-            assertion_data.setResult({(lhs)op(rhs), print(lhs, #op, rhs)});                      \
+            ZEROERR_SUPPRESS_COMPARE                                                           \
+            [&](const auto& zeroerr_lhs, const auto& zeroerr_rhs) {                            \
+                assertion_data.setResult(                                                      \
+                    {zeroerr_lhs op zeroerr_rhs, print(zeroerr_lhs, #op, zeroerr_rhs)});        \
+            }((lhs), (rhs));                                                                  \
+            ZEROERR_SUPPRESS_COMPARE_POP                                                       \
         } catch (const std::exception& e) {                                                      \
             assertion_data.setException(e);                                                      \
         }                                                                                        \
@@ -145,9 +150,16 @@ ZEROERR_CLANG_SUPPRESS_WARNING_POP
 // clang-format off
 ZEROERR_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wgnu-zero-variadic-macro-arguments")
 
-#define ZEROERR_CHECK(cond, ...)          ZEROERR_EXPAND(ZEROERR_ASSERT_EXP(cond, ZEROERR_WARN, no_throw, false, __VA_ARGS__))
-#define ZEROERR_CHECK_NOT(cond, ...)      ZEROERR_EXPAND(ZEROERR_ASSERT_EXP(cond, ZEROERR_WARN, no_throw, true, __VA_ARGS__))
-#define ZEROERR_CHECK_THROWS(cond, ...)   ZEROERR_EXPAND(ZEROERR_ASSERT_EXP(cond, ZEROERR_WARN, throws, false, __VA_ARGS__))
+// Keep warning-only checks compatible by default. Consumers can opt into runner
+// failure accounting, including context-free helpers, before including zeroerr.
+#ifdef ZEROERR_CHECK_AS_ERROR
+#define ZEROERR_CHECK_LEVEL ZEROERR_ERROR
+#else
+#define ZEROERR_CHECK_LEVEL ZEROERR_WARN
+#endif
+#define ZEROERR_CHECK(cond, ...)          ZEROERR_EXPAND(ZEROERR_ASSERT_EXP(cond, ZEROERR_CHECK_LEVEL, no_throw, false, __VA_ARGS__))
+#define ZEROERR_CHECK_NOT(cond, ...)      ZEROERR_EXPAND(ZEROERR_ASSERT_EXP(cond, ZEROERR_CHECK_LEVEL, no_throw, true, __VA_ARGS__))
+#define ZEROERR_CHECK_THROWS(cond, ...)   ZEROERR_EXPAND(ZEROERR_ASSERT_EXP(cond, ZEROERR_CHECK_LEVEL, throws, false, __VA_ARGS__))
 #define ZEROERR_REQUIRE(cond, ...)        ZEROERR_EXPAND(ZEROERR_ASSERT_EXP(cond, ZEROERR_ERROR, no_throw, false, __VA_ARGS__))
 #define ZEROERR_REQUIRE_NOT(cond, ...)    ZEROERR_EXPAND(ZEROERR_ASSERT_EXP(cond, ZEROERR_ERROR, no_throw, true, __VA_ARGS__))
 #define ZEROERR_REQUIRE_THROWS(cond, ...) ZEROERR_EXPAND(ZEROERR_ASSERT_EXP(cond, ZEROERR_ERROR, throws, false, __VA_ARGS__))
@@ -165,12 +177,12 @@ ZEROERR_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wgnu-zero-variadic-macro-arguments")
 #define ASSERT_NOT(...)     ZEROERR_SUPPRESS_VARIADIC_MACRO ZEROERR_EXPAND(ZEROERR_ASSERT_NOT(__VA_ARGS__)) ZEROERR_SUPPRESS_VARIADIC_MACRO_POP
 #define ASSERT_THROWS(...)  ZEROERR_SUPPRESS_VARIADIC_MACRO ZEROERR_EXPAND(ZEROERR_ASSERT_THROWS(__VA_ARGS__)) ZEROERR_SUPPRESS_VARIADIC_MACRO_POP
 
-#define ZEROERR_CHECK_EQ(lhs, rhs, ...) ZEROERR_ASSERT_CMP(lhs, ==, rhs, ZEROERR_WARN, no_throw, false, __VA_ARGS__)
-#define ZEROERR_CHECK_NE(lhs, rhs, ...) ZEROERR_ASSERT_CMP(lhs, !=, rhs, ZEROERR_WARN, no_throw, false, __VA_ARGS__)
-#define ZEROERR_CHECK_LT(lhs, rhs, ...) ZEROERR_ASSERT_CMP(lhs, <, rhs, ZEROERR_WARN, no_throw, false, __VA_ARGS__)
-#define ZEROERR_CHECK_LE(lhs, rhs, ...) ZEROERR_ASSERT_CMP(lhs, <=, rhs, ZEROERR_WARN, no_throw, false, __VA_ARGS__)
-#define ZEROERR_CHECK_GT(lhs, rhs, ...) ZEROERR_ASSERT_CMP(lhs, >, rhs, ZEROERR_WARN, no_throw, false, __VA_ARGS__)
-#define ZEROERR_CHECK_GE(lhs, rhs, ...) ZEROERR_ASSERT_CMP(lhs, >=, rhs, ZEROERR_WARN, no_throw, false, __VA_ARGS__)
+#define ZEROERR_CHECK_EQ(lhs, rhs, ...) ZEROERR_ASSERT_CMP(lhs, ==, rhs, ZEROERR_CHECK_LEVEL, no_throw, false, __VA_ARGS__)
+#define ZEROERR_CHECK_NE(lhs, rhs, ...) ZEROERR_ASSERT_CMP(lhs, !=, rhs, ZEROERR_CHECK_LEVEL, no_throw, false, __VA_ARGS__)
+#define ZEROERR_CHECK_LT(lhs, rhs, ...) ZEROERR_ASSERT_CMP(lhs, <, rhs, ZEROERR_CHECK_LEVEL, no_throw, false, __VA_ARGS__)
+#define ZEROERR_CHECK_LE(lhs, rhs, ...) ZEROERR_ASSERT_CMP(lhs, <=, rhs, ZEROERR_CHECK_LEVEL, no_throw, false, __VA_ARGS__)
+#define ZEROERR_CHECK_GT(lhs, rhs, ...) ZEROERR_ASSERT_CMP(lhs, >, rhs, ZEROERR_CHECK_LEVEL, no_throw, false, __VA_ARGS__)
+#define ZEROERR_CHECK_GE(lhs, rhs, ...) ZEROERR_ASSERT_CMP(lhs, >=, rhs, ZEROERR_CHECK_LEVEL, no_throw, false, __VA_ARGS__)
 
 #define ZEROERR_REQUIRE_EQ(lhs, rhs, ...) ZEROERR_ASSERT_CMP(lhs, ==, rhs, ZEROERR_ERROR, no_throw, false, __VA_ARGS__)
 #define ZEROERR_REQUIRE_NE(lhs, rhs, ...) ZEROERR_ASSERT_CMP(lhs, !=, rhs, ZEROERR_ERROR, no_throw, false, __VA_ARGS__)
@@ -314,4 +326,4 @@ struct context_helper<T, false> {
 
 }  // namespace zeroerr
 
-ZEROERR_SUPPRESS_COMMON_WARNINGS_POP
+ZEROERR_SUPPRESS_COMMON_WARNINGS_POP

src/unittest.cpp

@@ -94,6 +94,8 @@ void SubCase::operator<<(std::function<void(TestContext*)> op) {
     try {
         op(&local);
     } catch (const AssertionData&) {
+        // A helper may throw without having received this subcase's context.
+        if (local.failed_as == 0) local.failed_as = 1;
     } catch (const FuzzFinishedException&) {
     } catch (const std::exception& e) {
         std::cerr << e.what() << std::endl;
@@ -363,6 +365,8 @@ int UnitTest::run() {
             try {
                 tc.func(&context);  // run the test case
             } catch (const AssertionData&) {
+                // Context-free helper assertions must not disappear at the catch.
+                if (context.failed_as == 0) context.failed_as = 1;
             } catch (const FuzzFinishedException&) {
             } catch (const std::exception& e) {
                 std::cerr << e.what() << std::endl;
@@ -383,7 +387,11 @@ int UnitTest::run() {
     }
     reporter->testEnd(sum);
     delete reporter;
+#ifdef ZEROERR_CHECK_AS_ERROR
+    return (sum.failed > 0 || sum.failed_as > 0 || sum.warning_as > 0) ? 1 : 0;
+#else
     return (sum.failed > 0 || sum.failed_as > 0) ? 1 : 0;
+#endif
 }
 
 // sorted by file names and line numbers

test/CMakeLists.txt

@@ -108,3 +108,4 @@ foreach(order IN ITEMS zeroerr_first stl_first)
 endforeach()
 
 add_subdirectory(native_fixture_identity)
+add_subdirectory(assertion_contract)

test/assertion_contract/CMakeLists.txt

@@ -0,0 +1,22 @@
+cmake_minimum_required(VERSION 3.21)
+project(AssertionContract LANGUAGES C CXX)
+include(CTest)
+find_package(Python3 REQUIRED COMPONENTS Interpreter)
+if(NOT TARGET zeroerr)
+    add_subdirectory("${CMAKE_CURRENT_LIST_DIR}/../.." zeroerr EXCLUDE_FROM_ALL)
+    target_compile_definitions(zeroerr PRIVATE ZEROERR_NO_MAIN)
+    set(_contract_own_main ON)
+endif()
+foreach(mode IN ITEMS legacy strict)
+    add_executable(assertion_contract_${mode} probe.cpp)
+    target_link_libraries(assertion_contract_${mode} PRIVATE zeroerr)
+    if(_contract_own_main)
+        target_compile_definitions(assertion_contract_${mode} PRIVATE ZEROERR_ASSERTION_CONTRACT_MAIN)
+    endif()
+    if(mode STREQUAL "strict")
+        target_compile_definitions(assertion_contract_${mode} PRIVATE ZEROERR_CHECK_AS_ERROR)
+    endif()
+endforeach()
+add_test(NAME assertion_contract
+    COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/verify.py"
+            "$<TARGET_FILE:assertion_contract_strict>" "$<TARGET_FILE:assertion_contract_legacy>")

test/assertion_contract/probe.cpp

@@ -0,0 +1,42 @@
+#include <zeroerr/assert.h>
+#include <zeroerr/unittest.h>
+#include <stdexcept>
+#include <string>
+
+namespace {
+void helper() { CHECK(false); }
+void invariant() { ASSERT(false); }
+void requirement() { REQUIRE(false); }
+}  // namespace
+
+TEST_CASE("assertion.singleEvaluation") {
+    int left = 0, right = 0;
+    CHECK_EQ(++left, ++right);
+    REQUIRE_EQ(left, 1);
+    REQUIRE_EQ(right, 1);
+    CHECK_EQ(std::string("value"), std::string("value"));
+    CHECK_NE(left, right + 1);
+    CHECK_LT(left, right + 1);
+    CHECK_LE(left, right);
+    CHECK_GT(left + 1, right);
+    CHECK_GE(left, right);
+}
+TEST_CASE("assertion.check") { CHECK(false); }
+TEST_CASE("assertion.comparison") { CHECK_EQ(1, 2); }
+TEST_CASE("assertion.checkNot") { CHECK_NOT(true); }
+TEST_CASE("assertion.checkThrows") { CHECK_THROWS(false); }
+TEST_CASE("assertion.require") { REQUIRE(false); }
+TEST_CASE("assertion.assert") { ASSERT(false); }
+TEST_CASE("assertion.helper") { helper(); }
+TEST_CASE("assertion.helperRequire") { requirement(); }
+TEST_CASE("assertion.helperAssert") { invariant(); }
+TEST_CASE("assertion.subcaseHelper") {
+    SUB_CASE("helper assertion") { helper(); };
+}
+TEST_CASE("assertion.expectedInvariant") { CHECK_THROWS((invariant(), false)); }
+
+#ifdef ZEROERR_ASSERTION_CONTRACT_MAIN
+int main(int argc, char** argv) {
+    return zeroerr::UnitTest().parseArgs(argc, const_cast<const char**>(argv)).run();
+}
+#endif

test/assertion_contract/verify.py

@@ -0,0 +1,21 @@
+"""Check actual runner exit codes in both compatible and strict check modes."""
+import subprocess
+import sys
+
+for runner, strict in ((sys.argv[1], True), (sys.argv[2], False)):
+    for name in ("singleEvaluation", "expectedInvariant", "check", "comparison",
+                 "checkNot", "checkThrows", "require", "assert", "helper",
+                 "helperRequire", "helperAssert", "subcaseHelper"):
+        result = subprocess.run(
+            [runner, "--testcase=^assertion." + name + "$"],
+            capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=30,
+        )
+        succeeds = name in ("singleEvaluation", "expectedInvariant") or (
+            not strict and name in ("check", "comparison", "checkNot", "checkThrows", "helper", "subcaseHelper")
+        )
+        if (result.returncode == 0) != succeeds:
+            raise SystemExit(f"{runner}: {name}: unexpected exit {result.returncode}\n"
+                             + result.stdout + result.stderr)
+        if not succeeds and "Assertion Failed" not in result.stdout + result.stderr:
+            raise SystemExit(f"{name}: missing assertion diagnostic")
+print("single evaluation and strict/compatible assertion exit contracts passed")