testrunner: actually bail out on failed assert (danmar#7436)

firewave · web-flow · commit c78d1f2a9940 · 2025-04-07T13:14:46.000+02:00
diff --git a/test/fixture.cpp b/test/fixture.cpp
@@ -169,6 +169,7 @@ void TestFixture::assert_(const char * const filename, const unsigned int linenr
         errmsg << getLocationStr(filename, linenr) << ": Assertion failed." << std::endl << "_____" << std::endl;
         if (!msg.empty())
             errmsg << "Hint:" << std::endl << msg << std::endl;
+        throw AssertFailedError();
     }
 }
 
@@ -183,6 +184,7 @@ void TestFixture::assertFailure(const char* const filename, const unsigned int l
     if (!msg.empty())
         errmsg << "Hint:" << std::endl << msg << std::endl;
     errmsg << "_____" << std::endl;
+    throw AssertFailedError();
 }
 
 void TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const std::string &expected, const std::string &actual, const std::string &msg) const
@@ -289,25 +291,28 @@ void TestFixture::assertThrow(const char * const filename, const unsigned int li
     ++fails_counter;
     errmsg << getLocationStr(filename, linenr) << ": Assertion succeeded. "
            << "The expected exception was thrown" << std::endl << "_____" << std::endl;
+    throw AssertFailedError();
 }
 
 void TestFixture::assertThrowFail(const char * const filename, const unsigned int linenr) const
 {
     ++fails_counter;
     errmsg << getLocationStr(filename, linenr) << ": Assertion failed. "
            << "The expected exception was not thrown"  << std::endl << "_____" << std::endl;
+    throw AssertFailedError();
 }
 
-void TestFixture::assertNoThrowFail(const char * const filename, const unsigned int linenr) const
+void TestFixture::assertNoThrowFail(const char * const filename, const unsigned int linenr, bool bailout) const
 {
-    ++fails_counter;
-
     std::string ex_msg;
 
     try {
         // cppcheck-suppress rethrowNoCurrentException
         throw;
     }
+    catch (const AssertFailedError&) {
+        return;
+    }
     catch (const InternalError& e) {
         ex_msg = e.errorMessage;
     }
@@ -318,8 +323,11 @@ void TestFixture::assertNoThrowFail(const char * const filename, const unsigned
         ex_msg = "unknown exception";
     }
 
+    ++fails_counter;
     errmsg << getLocationStr(filename, linenr) << ": Assertion failed. "
            << "Unexpected exception was thrown: " << ex_msg << std::endl << "_____" << std::endl;
+    if (bailout)
+        throw AssertFailedError();
 }
 
 void TestFixture::printHelp()
diff --git a/test/fixture.h b/test/fixture.h
@@ -22,13 +22,15 @@
 
 #include "check.h"
 #include "color.h"
+#include "config.h"
 #include "errorlogger.h"
 #include "platform.h"
 #include "settings.h"
 #include "standards.h"
 
 #include <cstddef>
 #include <cstdint>
+#include <exception>
 #include <list>
 #include <memory>
 #include <sstream>
@@ -69,6 +71,8 @@ class TestFixture : public ErrorLogger {
     virtual void teardownTestInternal() {}
     std::string getLocationStr(const char * filename, unsigned int linenr) const;
 
+    class AssertFailedError : public std::exception {};
+
     void assert_(const char * filename, unsigned int linenr, bool condition, const std::string& msg = "") const;
 
     template<typename T>
@@ -113,9 +117,9 @@ class TestFixture : public ErrorLogger {
                           const char current[], const std::string &actual) const;
     void todoAssertEquals(const char * filename, unsigned int linenr, long long wanted,
                           long long current, long long actual) const;
-    void assertThrow(const char * filename, unsigned int linenr) const;
-    void assertThrowFail(const char * filename, unsigned int linenr) const;
-    void assertNoThrowFail(const char * filename, unsigned int linenr) const;
+    NORETURN void assertThrow(const char * filename, unsigned int linenr) const;
+    NORETURN void assertThrowFail(const char * filename, unsigned int linenr) const;
+    void assertNoThrowFail(const char * filename, unsigned int linenr, bool bailout) const;
     static std::string deleteLineNumber(const std::string &message);
 
     void setVerbose(bool v) {
@@ -279,7 +283,7 @@ class TestFixture : public ErrorLogger {
 private:
     //Helper function to be called when an assertEquals assertion fails.
     //Writes the appropriate failure message to errmsg and increments fails_counter
-    void assertFailure(const char* filename, unsigned int linenr, const std::string& expected, const std::string& actual, const std::string& msg) const;
+    NORETURN void assertFailure(const char* filename, unsigned int linenr, const std::string& expected, const std::string& actual, const std::string& msg) const;
 
     std::ostringstream mOutput;
     std::ostringstream mErrout;
@@ -309,14 +313,13 @@ class TestInstance {
     std::unique_ptr<TestFixture> impl;
 };
 
-#define TEST_CASE( NAME ) do { if (prepareTest(#NAME)) { setVerbose(false); try { NAME(); teardownTest(); } catch (...) { assertNoThrowFail(__FILE__, __LINE__); } } } while (false)
+#define TEST_CASE( NAME ) do { if (prepareTest(#NAME)) { setVerbose(false); try { NAME(); teardownTest(); } catch (const AssertFailedError&) {} catch (...) { assertNoThrowFail(__FILE__, __LINE__, false); } } } while (false)
 
-// TODO: the asserts do not actually assert i.e. do stop executing the test
 #define ASSERT( CONDITION ) assert_(__FILE__, __LINE__, (CONDITION))
 #define ASSERT_LOC( CONDITION, FILE_, LINE_ ) assert_(FILE_, LINE_, (CONDITION))
 #define ASSERT_LOC_MSG( CONDITION, MSG, FILE_, LINE_ ) assert_(FILE_, LINE_, (CONDITION), MSG)
 // *INDENT-OFF*
-#define ASSERT_EQUALS( EXPECTED, ACTUAL ) do { try { assertEquals(__FILE__, __LINE__, (EXPECTED), (ACTUAL)); } catch (...) { assertNoThrowFail(__FILE__, __LINE__); } } while (false)
+#define ASSERT_EQUALS( EXPECTED, ACTUAL ) do { try { assertEquals(__FILE__, __LINE__, (EXPECTED), (ACTUAL)); } catch(const AssertFailedError&) { throw; } catch (...) { assertNoThrowFail(__FILE__, __LINE__, true); } } while (false)
 // *INDENT-ON*
 #define ASSERT_EQUALS_WITHOUT_LINENUMBERS( EXPECTED, ACTUAL ) assertEqualsWithoutLineNumbers(__FILE__, __LINE__, EXPECTED, ACTUAL)
 #define ASSERT_EQUALS_DOUBLE( EXPECTED, ACTUAL, TOLERANCE ) assertEqualsDouble(__FILE__, __LINE__, EXPECTED, ACTUAL, TOLERANCE)
@@ -328,15 +331,17 @@ class TestInstance {
 #define ASSERT_EQUALS_ENUM_LOC_MSG( EXPECTED, ACTUAL, MSG, FILE_, LINE_ ) assertEqualsEnum(FILE_, LINE_, (EXPECTED), (ACTUAL), MSG)
 #define TODO_ASSERT_EQUALS_ENUM( WANTED, CURRENT, ACTUAL ) todoAssertEqualsEnum(__FILE__, __LINE__, WANTED, CURRENT, ACTUAL)
 #define ASSERT_THROW_EQUALS( CMD, EXCEPTION, EXPECTED ) do { try { (void)(CMD); assertThrowFail(__FILE__, __LINE__); } catch (const EXCEPTION&e) { assertEquals(__FILE__, __LINE__, EXPECTED, e.errorMessage); } catch (...) { assertThrowFail(__FILE__, __LINE__); } } while (false)
-#define ASSERT_THROW_EQUALS_2( CMD, EXCEPTION, EXPECTED ) do { try { (void)(CMD); assertThrowFail(__FILE__, __LINE__); } catch (const EXCEPTION&e) { assertEquals(__FILE__, __LINE__, EXPECTED, e.what()); } catch (...) { assertThrowFail(__FILE__, __LINE__); } } while (false)
-#define ASSERT_THROW_INTERNAL( CMD, TYPE ) do { try { (void)(CMD); assertThrowFail(__FILE__, __LINE__); } catch (const InternalError& e) { assertEqualsEnum(__FILE__, __LINE__, InternalError::TYPE, e.type); } catch (...) { assertThrowFail(__FILE__, __LINE__); } } while (false)
-#define ASSERT_THROW_INTERNAL_EQUALS( CMD, TYPE, EXPECTED ) do { try { (void)(CMD); assertThrowFail(__FILE__, __LINE__); } catch (const InternalError& e) { assertEqualsEnum(__FILE__, __LINE__, InternalError::TYPE, e.type); assertEquals(__FILE__, __LINE__, EXPECTED, e.errorMessage); } catch (...) { assertThrowFail(__FILE__, __LINE__); } } while (false)
-#define ASSERT_NO_THROW( CMD ) do { try { (void)(CMD); } catch (...) { assertNoThrowFail(__FILE__, __LINE__); } } while (false)
-#define TODO_ASSERT_THROW( CMD, EXCEPTION ) do { try { (void)(CMD); } catch (const EXCEPTION&) {} catch (...) { assertThrow(__FILE__, __LINE__); } } while (false)
+#define ASSERT_THROW_EQUALS_2( CMD, EXCEPTION, EXPECTED ) do { try { (void)(CMD); assertThrowFail(__FILE__, __LINE__); } catch (const AssertFailedError&) { throw; } catch (const EXCEPTION&e) { assertEquals(__FILE__, __LINE__, EXPECTED, e.what()); } catch (...) { assertThrowFail(__FILE__, __LINE__); } } while (false)
+#define ASSERT_THROW_INTERNAL( CMD, TYPE ) do { try { (void)(CMD); assertThrowFail(__FILE__, __LINE__); } catch (const AssertFailedError&) { throw; } catch (const InternalError& e) { assertEqualsEnum(__FILE__, __LINE__, InternalError::TYPE, e.type); } catch (...) { assertThrowFail(__FILE__, __LINE__); } } while (false)
+#define ASSERT_THROW_INTERNAL_EQUALS( CMD, TYPE, EXPECTED ) do { try { (void)(CMD); assertThrowFail(__FILE__, __LINE__); } catch (const AssertFailedError&) { throw; } catch (const InternalError& e) { assertEqualsEnum(__FILE__, __LINE__, InternalError::TYPE, e.type); assertEquals(__FILE__, __LINE__, EXPECTED, e.errorMessage); } catch (...) { assertThrowFail(__FILE__, __LINE__); } } while (false)
+#define ASSERT_NO_THROW( CMD ) do { try { (void)(CMD); } catch (const AssertFailedError&) { throw; } catch (...) { assertNoThrowFail(__FILE__, __LINE__, true); } } while (false)
+#define TODO_ASSERT_THROW( CMD, EXCEPTION ) do { try { (void)(CMD); } catch (const AssertFailedError&) { throw; } catch (const EXCEPTION&) {} catch (...) { assertThrow(__FILE__, __LINE__); } } while (false)
 #define TODO_ASSERT( CONDITION ) do { const bool condition=(CONDITION); todoAssertEquals(__FILE__, __LINE__, true, false, condition); } while (false)
 #define TODO_ASSERT_EQUALS( WANTED, CURRENT, ACTUAL ) todoAssertEquals(__FILE__, __LINE__, WANTED, CURRENT, ACTUAL)
 
+// *INDENT-OFF*
 #define REGISTER_TEST( CLASSNAME ) namespace { class CLASSNAME ## Instance : public TestInstance { public: CLASSNAME ## Instance() : TestInstance(#CLASSNAME) {} TestFixture* create() override { impl.reset(new CLASSNAME); return impl.get(); } }; CLASSNAME ## Instance instance_ ## CLASSNAME; }
+// *INDENT-ON*
 
 #define PLATFORM( P, T ) do { std::string errstr; assertEquals(__FILE__, __LINE__, true, P.set(Platform::toString(T), errstr, {exename}), errstr); } while (false)
 

Original file line number	Diff line number	Diff line change
`@@ -169,6 +169,7 @@ void TestFixture::assert_(const char * const filename, const unsigned int linenr`
`169`	`169`	`errmsg << getLocationStr(filename, linenr) << ": Assertion failed." << std::endl << "_____" << std::endl;`
`170`	`170`	`if (!msg.empty())`
`171`	`171`	`errmsg << "Hint:" << std::endl << msg << std::endl;`
	`172`	`+ throw AssertFailedError();`
`172`	`173`	`}`
`173`	`174`	`}`
`174`	`175`
`@@ -183,6 +184,7 @@ void TestFixture::assertFailure(const char* const filename, const unsigned int l`
`183`	`184`	`if (!msg.empty())`
`184`	`185`	`errmsg << "Hint:" << std::endl << msg << std::endl;`
`185`	`186`	`errmsg << "_____" << std::endl;`
	`187`	`+ throw AssertFailedError();`
`186`	`188`	`}`
`187`	`189`
`188`	`190`	`void TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const std::string &expected, const std::string &actual, const std::string &msg) const`
`@@ -289,25 +291,28 @@ void TestFixture::assertThrow(const char * const filename, const unsigned int li`
`289`	`291`	`++fails_counter;`
`290`	`292`	`errmsg << getLocationStr(filename, linenr) << ": Assertion succeeded. "`
`291`	`293`	`<< "The expected exception was thrown" << std::endl << "_____" << std::endl;`
	`294`	`+ throw AssertFailedError();`
`292`	`295`	`}`
`293`	`296`
`294`	`297`	`void TestFixture::assertThrowFail(const char * const filename, const unsigned int linenr) const`
`295`	`298`	`{`
`296`	`299`	`++fails_counter;`
`297`	`300`	`errmsg << getLocationStr(filename, linenr) << ": Assertion failed. "`
`298`	`301`	`<< "The expected exception was not thrown" << std::endl << "_____" << std::endl;`
	`302`	`+ throw AssertFailedError();`
`299`	`303`	`}`
`300`	`304`
`301`		`-void TestFixture::assertNoThrowFail(const char * const filename, const unsigned int linenr) const`
	`305`	`+void TestFixture::assertNoThrowFail(const char * const filename, const unsigned int linenr, bool bailout) const`
`302`	`306`	`{`
`303`		`- ++fails_counter;`
`304`		`-`
`305`	`307`	`std::string ex_msg;`
`306`	`308`
`307`	`309`	`try {`
`308`	`310`	`// cppcheck-suppress rethrowNoCurrentException`
`309`	`311`	`throw;`
`310`	`312`	`}`
	`313`	`+ catch (const AssertFailedError&) {`
	`314`	`+ return;`
	`315`	`+ }`
`311`	`316`	`catch (const InternalError& e) {`
`312`	`317`	`ex_msg = e.errorMessage;`
`313`	`318`	`}`
`@@ -318,8 +323,11 @@ void TestFixture::assertNoThrowFail(const char * const filename, const unsigned`
`318`	`323`	`ex_msg = "unknown exception";`
`319`	`324`	`}`
`320`	`325`
	`326`	`+ ++fails_counter;`
`321`	`327`	`errmsg << getLocationStr(filename, linenr) << ": Assertion failed. "`
`322`	`328`	`<< "Unexpected exception was thrown: " << ex_msg << std::endl << "_____" << std::endl;`
	`329`	`+ if (bailout)`
	`330`	`+ throw AssertFailedError();`
`323`	`331`	`}`
`324`	`332`
`325`	`333`	`void TestFixture::printHelp()`