diff --git a/junit5/junit5-14-4-jvm-test/pom.xml b/junit5/junit5-14-4-jvm-test/pom.xml new file mode 100644 index 00000000..db40859a --- /dev/null +++ b/junit5/junit5-14-4-jvm-test/pom.xml @@ -0,0 +1,86 @@ + + + + + + org.quickperf + quick-perf-junit5-parent + 1.1.1-SNAPSHOT + + 4.0.0 + + + quick-perf-junit5-14-4-jvm-test + + + 1.8 + 1.8 + 1.8 + 5.14.4 + 1.14.4 + true + true + true + true + true + + + + + + + org.junit + junit-bom + ${junit5.jupiter.version} + pom + import + + + + + + + org.quickperf + quick-perf-junit5 + 1.1.1-SNAPSHOT + test + + + org.quickperf + quick-perf-junit5-test-util + 1.1.1-SNAPSHOT + tests + test-jar + test + + + org.junit.jupiter + junit-jupiter-params + ${junit5.jupiter.version} + test + + + org.junit.platform + junit-platform-suite-api + ${junit5.platform.version} + test + + + org.junit.platform + junit-platform-suite-engine + ${junit5.platform.version} + test + + + + diff --git a/junit5/junit5-14-4-jvm-test/src/test/java/org/quickperf/junit5/parameterizedclass/ParameterizedClassInNewJvmJUnit5Test.java b/junit5/junit5-14-4-jvm-test/src/test/java/org/quickperf/junit5/parameterizedclass/ParameterizedClassInNewJvmJUnit5Test.java new file mode 100644 index 00000000..2c912818 --- /dev/null +++ b/junit5/junit5-14-4-jvm-test/src/test/java/org/quickperf/junit5/parameterizedclass/ParameterizedClassInNewJvmJUnit5Test.java @@ -0,0 +1,348 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * Copyright 2019-2022 the original author or authors. + */ +package org.quickperf.junit5.parameterizedclass; + +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.RepeatedTest; +import org.junit.jupiter.api.RepetitionInfo; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.Parameter; +import org.junit.jupiter.params.ParameterizedClass; +import org.junit.jupiter.params.provider.ValueSource; +import org.quickperf.junit5.JUnit5Tests; +import org.quickperf.junit5.JUnit5Tests.JUnit5TestsResult; +import org.quickperf.junit5.JvmExecutionReport; +import org.quickperf.junit5.QuickPerfTest; +import org.quickperf.jvm.allocation.AllocationUnit; +import org.quickperf.jvm.annotations.ExpectMaxHeapAllocation; +import org.quickperf.jvm.annotations.HeapSize; + +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.quickperf.junit5.JvmExecutionReport.assertThatEachTestWasExecutedInItsOwnNewJvm; +import static org.quickperf.junit5.JvmExecutionReport.reportExecutionOf; + +/** + * QuickPerf executes each test of each invocation of a @ParameterizedClass having an annotation that + * needs a specific JVM (@HeapSize, @ExpectMaxHeapAllocation...) in its own JVM. This JVM + * has to execute this test once, with the arguments of its class invocation, and QuickPerf has to report the measures + * and the failures of this execution for this test. + *

+ * Some test methods of the nested test classes fail on purpose with a {@link JvmExecutionReport}, to tell where they + * were executed. Others fail if their JVM executes them more than once. QuickPerf fails a test that JUnit does not + * execute in the specific JVM. The argument of the invocation #n of each parameterized class is n. + */ +public class ParameterizedClassInNewJvmJUnit5Test { + + private static final Pattern CLASS_INVOCATION_INDEX = Pattern.compile("\\[class-template-invocation:#(\\d+)]"); + + @QuickPerfTest + @ParameterizedClass + @ValueSource(ints = {1, 2, 3}) + static class ParameterizedClassFailingIfExecutedAgainInTheSameJvm { + + private static final AtomicInteger EXECUTIONS_IN_THIS_JVM = new AtomicInteger(); + + private final int value; + + public ParameterizedClassFailingIfExecutedAgainInTheSameJvm(int value) { + this.value = value; + } + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @Test + public void test_method() { + assertThat(EXECUTIONS_IN_THIS_JVM.incrementAndGet()).as("Executions in this JVM, value %s", value).isOne(); + } + + } + + @Test public void + should_execute_the_test_of_each_invocation_of_a_parameterized_class_once_in_its_jvm() { + + // GIVEN + Class testClass = ParameterizedClassFailingIfExecutedAgainInTheSameJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as(jUnit5TestsResult.getErrorReport()) + .isZero(); + + assertThat(jUnit5TestsResult.getNumberOfSucceededTests()).isEqualTo(3); + + } + + @QuickPerfTest + @ParameterizedClass + @ValueSource(ints = {1, 2, 3}) + static class ParameterizedClassInASpecificJvm { + + private final int value; + + public ParameterizedClassInASpecificJvm(int value) { + this.value = value; + } + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @Test + public void test_method() { + throw reportExecutionOf("test_method() with value " + value); + } + + } + + @Test public void + should_execute_the_test_of_each_invocation_of_a_parameterized_class_in_its_own_jvm_with_its_argument() { + + // GIVEN + Class testClass = ParameterizedClassInASpecificJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + String errorReport = jUnit5TestsResult.getErrorReport(); + + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as("The test of each class invocation should fail with the report of its execution.%s", errorReport) + .isEqualTo(3); + + List jvmExecutionReports = jUnit5TestsResult.getJvmExecutionReports(); + assertThat(jvmExecutionReports).extracting("executedTest") + .containsExactlyInAnyOrder("test_method() with value 1" + , "test_method() with value 2" + , "test_method() with value 3"); + assertThatEachTestWasExecutedInItsOwnNewJvm(jvmExecutionReports); + assertThatEachTestExecutedItsOwnInvocation(jvmExecutionReports); + + } + + @QuickPerfTest + @ParameterizedClass + @ValueSource(ints = {1, 2}) + static class ParameterizedClassHavingAnInjectedFieldAndTwoTestMethodsInASpecificJvm { + + @Parameter + int value; + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @Test + public void first_test_method() { + throw reportExecutionOf("first_test_method() with value " + value); + } + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @Test + public void second_test_method() { + throw reportExecutionOf("second_test_method() with value " + value); + } + + } + + @Test public void + should_execute_each_test_of_each_invocation_of_a_parameterized_class_in_its_own_jvm() { + + // GIVEN + Class testClass = ParameterizedClassHavingAnInjectedFieldAndTwoTestMethodsInASpecificJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + String errorReport = jUnit5TestsResult.getErrorReport(); + + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as("Each test of each class invocation should fail with the report of its execution.%s", errorReport) + .isEqualTo(4); + + List jvmExecutionReports = jUnit5TestsResult.getJvmExecutionReports(); + assertThat(jvmExecutionReports).extracting("executedTest") + .containsExactlyInAnyOrder("first_test_method() with value 1" + , "first_test_method() with value 2" + , "second_test_method() with value 1" + , "second_test_method() with value 2"); + assertThatEachTestWasExecutedInItsOwnNewJvm(jvmExecutionReports); + assertThatEachTestExecutedItsOwnInvocation(jvmExecutionReports); + + } + + @QuickPerfTest + @ParameterizedClass + @ValueSource(ints = {1, 2}) + static class ParameterizedClassHavingARepeatedTestInASpecificJvm { + + private final int value; + + public ParameterizedClassHavingARepeatedTestInASpecificJvm(int value) { + this.value = value; + } + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @RepeatedTest(2) + public void repeated_test(RepetitionInfo repetitionInfo) { + throw reportExecutionOf("repetition " + repetitionInfo.getCurrentRepetition() + + " of " + repetitionInfo.getTotalRepetitions() + " with value " + value); + } + + } + + @Test public void + should_execute_each_repetition_of_each_invocation_of_a_parameterized_class_in_its_own_jvm() { + + // GIVEN + Class testClass = ParameterizedClassHavingARepeatedTestInASpecificJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + String errorReport = jUnit5TestsResult.getErrorReport(); + + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as("Each repetition of each class invocation should fail with the report of its execution.%s" + , errorReport) + .isEqualTo(4); + + List jvmExecutionReports = jUnit5TestsResult.getJvmExecutionReports(); + assertThat(jvmExecutionReports).extracting("executedTest") + .containsExactlyInAnyOrder("repetition 1 of 2 with value 1" + , "repetition 2 of 2 with value 1" + , "repetition 1 of 2 with value 2" + , "repetition 2 of 2 with value 2"); + assertThatEachTestWasExecutedInItsOwnNewJvm(jvmExecutionReports); + assertThatEachTestExecutedItsOwnInvocation(jvmExecutionReports); + + } + + @QuickPerfTest + @ParameterizedClass + @ValueSource(ints = {1, 2}) + static class ParameterizedClassHavingANestedTestClassInASpecificJvm { + + private final int value; + + public ParameterizedClassHavingANestedTestClassInASpecificJvm(int value) { + this.value = value; + } + + @Nested + class NestedTestClass { + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @Test + public void test_method() { + throw reportExecutionOf("test_method() with value " + value); + } + + } + + } + + @Test public void + should_execute_the_test_of_a_nested_class_of_each_invocation_of_a_parameterized_class_in_its_own_jvm() { + + // GIVEN + Class testClass = ParameterizedClassHavingANestedTestClassInASpecificJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + String errorReport = jUnit5TestsResult.getErrorReport(); + + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as("The test of each class invocation should fail with the report of its execution.%s", errorReport) + .isEqualTo(2); + + List jvmExecutionReports = jUnit5TestsResult.getJvmExecutionReports(); + assertThat(jvmExecutionReports).extracting("executedTest") + .containsExactlyInAnyOrder("test_method() with value 1" + , "test_method() with value 2"); + assertThatEachTestWasExecutedInItsOwnNewJvm(jvmExecutionReports); + assertThatEachTestExecutedItsOwnInvocation(jvmExecutionReports); + + } + + @QuickPerfTest + @ParameterizedClass(name = "allocating {0} MB") + @ValueSource(ints = {1, 2, 3}) + static class ParameterizedClassAllocatingAccordingToItsArgument { + + private final int allocationInMegaBytes; + + public ParameterizedClassAllocatingAccordingToItsArgument(int allocationInMegaBytes) { + this.allocationInMegaBytes = allocationInMegaBytes; + } + + @ExpectMaxHeapAllocation(value = 2.5, unit = AllocationUnit.MEGA_BYTE) + @Test + public void test_method() { + byte[] allocatedBytes = new byte[allocationInMegaBytes * 1024 * 1024]; + } + + } + + @Test public void + should_report_the_heap_allocation_of_the_test_of_each_invocation_of_a_parameterized_class_for_this_test() { + + // GIVEN + Class testClass = ParameterizedClassAllocatingAccordingToItsArgument.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + String errorReport = jUnit5TestsResult.getErrorReport(); + + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as("Only the test of the class invocation allocating 3 MB should fail.%s", errorReport) + .isOne(); + + assertThat(jUnit5TestsResult.getNumberOfSucceededTests()).isEqualTo(2); + + assertThat(errorReport).contains("allocating 3 MB") + .contains("Expected heap allocation (test method thread) to be less than 2.50 Mega bytes but is 3.") + .doesNotContain("allocating 1 MB") + .doesNotContain("allocating 2 MB"); + + } + + // The test JVM reports the execution of a new JVM for the test that this new JVM executed: the test of the class + // invocation #n reports its display name with the value n + private static void assertThatEachTestExecutedItsOwnInvocation(List jvmExecutionReports) { + for (JvmExecutionReport jvmExecutionReport : jvmExecutionReports) { + String testUniqueId = jvmExecutionReport.getTestUniqueId(); + Matcher classInvocationIndex = CLASS_INVOCATION_INDEX.matcher(testUniqueId); + assertThat(classInvocationIndex.find()).as("Class invocation index in %s", testUniqueId).isTrue(); + String expectedExecutedTest = jvmExecutionReport.getTestDisplayName() + + " with value " + classInvocationIndex.group(1); + assertThat(jvmExecutionReport.getExecutedTest()) + .as("Test executed in the new JVM of %s", testUniqueId) + .isEqualTo(expectedExecutedTest); + } + } + +} diff --git a/junit5/junit5-14-4-jvm-test/src/test/java/org/quickperf/junit5/suite/SuiteInNewJvmJUnit5Test.java b/junit5/junit5-14-4-jvm-test/src/test/java/org/quickperf/junit5/suite/SuiteInNewJvmJUnit5Test.java new file mode 100644 index 00000000..c55f70b4 --- /dev/null +++ b/junit5/junit5-14-4-jvm-test/src/test/java/org/quickperf/junit5/suite/SuiteInNewJvmJUnit5Test.java @@ -0,0 +1,126 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * Copyright 2019-2022 the original author or authors. + */ +package org.quickperf.junit5.suite; + +import org.junit.jupiter.api.RepeatedTest; +import org.junit.jupiter.api.RepetitionInfo; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.platform.suite.api.BeforeSuite; +import org.junit.platform.suite.api.SelectClasses; +import org.junit.platform.suite.api.Suite; +import org.quickperf.junit5.JUnit5Tests; +import org.quickperf.junit5.JUnit5Tests.JUnit5TestsResult; +import org.quickperf.junit5.JvmExecutionReport; +import org.quickperf.junit5.QuickPerfTest; +import org.quickperf.jvm.allocation.AllocationUnit; +import org.quickperf.jvm.annotations.HeapSize; + +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.quickperf.junit5.JvmExecutionReport.assertThatEachTestWasExecutedInItsOwnNewJvm; +import static org.quickperf.junit5.JvmExecutionReport.reportExecutionOf; + +/** + * The JUnit unique id of a test executed through a @Suite starts with segments of the suite engine. The + * specific JVM executes the test with the Jupiter engine only, without the @BeforeSuite and + * @AfterSuite methods, which were already executed in the test JVM. + *

+ * The test methods of the nested test classes fail on purpose with a {@link JvmExecutionReport}, to tell where they + * were executed. + */ +public class SuiteInNewJvmJUnit5Test { + + private static final AtomicInteger BEFORE_SUITE_EXECUTIONS_IN_THIS_JVM = new AtomicInteger(); + + @QuickPerfTest + static class TestsInASpecificJvm { + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @Test + public void test_method(TestInfo testInfo) { + throw reportExecutionOf("@BeforeSuite executions: " + BEFORE_SUITE_EXECUTIONS_IN_THIS_JVM.get()); + } + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @RepeatedTest(2) + public void repeated_test(RepetitionInfo repetitionInfo) { + throw reportExecutionOf("@BeforeSuite executions: " + BEFORE_SUITE_EXECUTIONS_IN_THIS_JVM.get()); + } + + } + + @Suite + @SelectClasses(TestsInASpecificJvm.class) + static class SuiteOfTestsInASpecificJvm { + + @BeforeSuite + public static void before_suite() { + BEFORE_SUITE_EXECUTIONS_IN_THIS_JVM.incrementAndGet(); + } + + } + + @Suite + @SelectClasses(SuiteOfTestsInASpecificJvm.class) + static class SuiteOfSuite { + } + + @Test public void + should_execute_each_test_of_a_suite_in_its_own_jvm_without_the_before_suite_methods() { + + // GIVEN + Class suiteClass = SuiteOfTestsInASpecificJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(suiteClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + assertThatEachTestWasExecutedInItsOwnNewJvmWithoutTheBeforeSuiteMethods(jUnit5TestsResult); + + } + + @Test public void + should_execute_each_test_of_a_nested_suite_in_its_own_jvm_without_the_before_suite_methods() { + + // GIVEN + Class suiteClass = SuiteOfSuite.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(suiteClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + assertThatEachTestWasExecutedInItsOwnNewJvmWithoutTheBeforeSuiteMethods(jUnit5TestsResult); + + } + + private static void assertThatEachTestWasExecutedInItsOwnNewJvmWithoutTheBeforeSuiteMethods(JUnit5TestsResult jUnit5TestsResult) { + String errorReport = jUnit5TestsResult.getErrorReport(); + + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as("Each test should fail with the report of its execution.%s", errorReport) + .isEqualTo(3); + + List jvmExecutionReports = jUnit5TestsResult.getJvmExecutionReports(); + assertThat(jvmExecutionReports).extracting("executedTest") + .containsExactly("@BeforeSuite executions: 0" + , "@BeforeSuite executions: 0" + , "@BeforeSuite executions: 0"); + assertThatEachTestWasExecutedInItsOwnNewJvm(jvmExecutionReports); + } + +} diff --git a/junit5/junit5-extension/src/main/java/org/quickperf/junit5/JUnit5TestRunner.java b/junit5/junit5-extension/src/main/java/org/quickperf/junit5/JUnit5TestRunner.java index 87c69ccb..8a7aa9f3 100644 --- a/junit5/junit5-extension/src/main/java/org/quickperf/junit5/JUnit5TestRunner.java +++ b/junit5/junit5-extension/src/main/java/org/quickperf/junit5/JUnit5TestRunner.java @@ -12,51 +12,257 @@ */ package org.quickperf.junit5; -import org.junit.platform.engine.discovery.MethodSelector; +import org.junit.platform.engine.TestExecutionResult; import org.junit.platform.launcher.Launcher; import org.junit.platform.launcher.LauncherDiscoveryRequest; +import org.junit.platform.launcher.TestExecutionListener; +import org.junit.platform.launcher.TestIdentifier; import org.junit.platform.launcher.core.LauncherFactory; -import org.junit.platform.launcher.listeners.SummaryGeneratingListener; -import org.junit.platform.launcher.listeners.TestExecutionSummary; +import org.opentest4j.TestAbortedException; +import org.quickperf.SystemProperties; +import org.quickperf.issue.JvmError; import org.quickperf.issue.TestIssue; import org.quickperf.testlauncher.TestRunnerFromMain; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectOutputStream; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.Collections; +import java.util.IdentityHashMap; import java.util.List; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicInteger; -import static java.util.stream.Collectors.toList; -import static org.junit.platform.engine.discovery.DiscoverySelectors.selectMethod; +import static org.junit.platform.engine.TestExecutionResult.Status.FAILED; +import static org.junit.platform.engine.discovery.DiscoverySelectors.selectUniqueId; import static org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder.request; class JUnit5TestRunner implements TestRunnerFromMain.FrameworkTestRunner { + private static final String JUPITER_ENGINE_SEGMENT = "[engine:junit-jupiter]"; + + // Jupiter also aborts a test throwing this JUnit 4 exception, when JUnit 4 is on the classpath + private static final String JUNIT4_ASSUMPTION_VIOLATED_EXCEPTION = "org.junit.internal.AssumptionViolatedException"; + @Override public TestIssue executeTestMethod(Class testClass, String methodName) { - TestExecutionSummary testResult = runTestMethod(testClass, methodName); + String workingFolderPath = SystemProperties.WORKING_FOLDER.evaluate(); + TestToExecuteInNewJvm testToExecute = TestToExecuteInNewJvm.findIn(workingFolderPath); + if (testToExecute == null) { + String message = "QuickPerf did not find the JUnit unique id of " + testClass.getName() + "." + methodName + + " in the working folder " + workingFolderPath + "."; + return TestIssue.buildFrom(JvmError.buildFrom(message)); + } - List failures = testResult.getFailures(); + String uniqueId = testToExecute.getUniqueId(); + TestExecutionRecorder testExecutionRecorder = new TestExecutionRecorder(); + try { + runTest(uniqueId, testExecutionRecorder); + } catch (Throwable unrecoverableError) { + // JUnit rethrows the errors that it does not recover from, an OutOfMemoryError for example. Reporting them + // lets this JVM end with System.exit, even if the test left non-daemon threads. + return TestIssue.buildFrom(buildUnrecoverableError(uniqueId, unrecoverableError)); + } - List jUnit5failuresAsThrowables = convertToThrowables(failures); + if (testExecutionRecorder.startedTestCount.get() != 1) { + if (testExecutionRecorder.onlyContainersAborted()) { + // An assumption failing in a @BeforeAll method or in an arguments source, for example + Throwable containerAbort = testExecutionRecorder.containerAborts.get(0); + return TestIssue.buildFrom(new TestAbortedBeforeTestMethodInNewJvm(toSerializableAbort(containerAbort))); + } + JvmError jvmError = buildNotOneStartedTestError(uniqueId, testExecutionRecorder); + return TestIssue.buildFrom(jvmError); + } - return TestIssue.buildInNewJvmFrom(jUnit5failuresAsThrowables); - } + List failures = testExecutionRecorder.failures; + if (!failures.isEmpty()) { + return TestIssue.buildInNewJvmFrom(failures); + } - private List convertToThrowables(List failures) { - return failures.stream() - .map(TestExecutionSummary.Failure::getException) - .collect(toList()); + List testAborts = testExecutionRecorder.testAborts; + if (!testAborts.isEmpty()) { + return buildTestAbortIssue(testAborts.get(0)); + } + + return TestIssue.NONE; } - private static TestExecutionSummary runTestMethod(Class testClass, String methodName) { - MethodSelector testMethodSelector = selectMethod(testClass, methodName); - LauncherDiscoveryRequest request = request().selectors(testMethodSelector) + private static void runTest(String uniqueId, TestExecutionListener listener) { + LauncherDiscoveryRequest request = request().selectors(selectUniqueId(removeSuiteSegments(uniqueId))) .build(); - SummaryGeneratingListener listener = new SummaryGeneratingListener(); - Launcher launcher = LauncherFactory.create(); launcher.execute(request, listener); + } + + // The unique id of a test executed through a suite (@Suite) starts with segments of the suite engine. The new JVM + // executes the test with the Jupiter engine only: the @BeforeSuite and @AfterSuite methods are not executed again. + private static String removeSuiteSegments(String uniqueId) { + int jupiterEngineSegmentIndex = uniqueId.lastIndexOf(JUPITER_ENGINE_SEGMENT); + return jupiterEngineSegmentIndex == -1 ? uniqueId : uniqueId.substring(jupiterEngineSegmentIndex); + } + + private static TestIssue buildTestAbortIssue(Throwable testAbort) { + TestMethodInvocationInNewJvm testMethodInvocation = TestMethodInvocationInNewJvm.INSTANCE; + if (!testMethodInvocation.isTestMethodInvoked()) { + // An assumption failing in a @BeforeEach method, for example + return TestIssue.buildFrom(new TestAbortedBeforeTestMethodInNewJvm(toSerializableAbort(testAbort))); + } + + Throwable testMethodThrowable = testMethodInvocation.getTestMethodThrowable(); + if (testMethodThrowable != null && !isAbort(testMethodThrowable) && isACauseOf(testAbort, testMethodThrowable)) { + // A retry extension aborted the failed test to execute it again. The test JVM gets the failure, as when it + // executes the test, so that the retry extension handles it there. + return TestIssue.buildInNewJvmFrom(Collections.singletonList(testMethodThrowable)); + } + + // An assumption failing in the test method or in an @AfterEach method, for example. The test JVM evaluates the + // performance records, as when it executes the test. + return TestIssue.buildFrom(toSerializableAbort(testAbort)); + } + + private static boolean isAbort(Throwable throwable) { + if (throwable instanceof TestAbortedException) { + return true; + } + for (Class type = throwable.getClass(); type != null; type = type.getSuperclass()) { + if (type.getName().equals(JUNIT4_ASSUMPTION_VIOLATED_EXCEPTION)) { + return true; + } + } + return false; + } + + private static boolean isACauseOf(Throwable throwable, Throwable possibleCause) { + Set causes = Collections.newSetFromMap(new IdentityHashMap()); + for (Throwable cause = throwable.getCause(); cause != null && causes.add(cause); cause = cause.getCause()) { + if (cause == possibleCause) { + return true; + } + } + return false; + } + + // TestIssueRepository would replace an abort that cannot be serialized with a failure + private static Throwable toSerializableAbort(Throwable abort) { + try (ObjectOutputStream objectOutputStream = new ObjectOutputStream(new ByteArrayOutputStream())) { + objectOutputStream.writeObject(abort); + return abort; + } catch (IOException notSerializable) { + return new TestAbortedException(abort.getMessage()); + } + } + + private static JvmError buildUnrecoverableError(String uniqueId, Throwable unrecoverableError) { + StringWriter stackTrace = new StringWriter(); + unrecoverableError.printStackTrace(new PrintWriter(stackTrace)); + String message = "JUnit stopped in the specific JVM where QuickPerf executes " + uniqueId + ":" + + System.lineSeparator() + stackTrace; + return JvmError.buildFrom(message); + } + + private static JvmError buildNotOneStartedTestError(String uniqueId, TestExecutionRecorder testExecutionRecorder) { + String lineSeparator = System.lineSeparator(); + StringBuilder message = new StringBuilder(); + message.append("JUnit started ").append(testExecutionRecorder.startedTestCount.get()) + .append(" tests instead of 1 in the specific JVM where QuickPerf executes ").append(uniqueId).append('.') + .append(lineSeparator) + .append("QuickPerf executes a test in a specific JVM for annotations such as @HeapSize, @JvmOptions,") + .append(" @MeasureHeapAllocation or @ProfileJvm."); + List unsuccessfulExecutions = testExecutionRecorder.unsuccessfulExecutions; + if (!unsuccessfulExecutions.isEmpty()) { + message.append(lineSeparator).append("Reported by JUnit in the specific JVM:"); + for (String unsuccessfulExecution : unsuccessfulExecutions) { + message.append(lineSeparator).append("- ").append(unsuccessfulExecution); + } + } + message.append(lineSeparator).append("Possible causes:") + .append(lineSeparator) + .append("- A condition (@EnabledIfSystemProperty, @DisabledIf, an ExecutionCondition extension...)") + .append(" disables the test or its class in the specific JVM. The specific JVM does not get the system") + .append(" properties of the test JVM: you can set them with @JvmOptions(\"-Dname=value\")."); + if (isTemplateInvocation(uniqueId)) { + message.append(lineSeparator) + .append("- The arguments source of a parameterized test provides other arguments in the specific") + .append(" JVM. It has to provide the same arguments, in the same order, in every JVM: do not use") + .append(" Set.of(...), a HashSet or random values.") + .append(lineSeparator) + .append("- The invocations of a test template depend on the results of the previous invocations, as") + .append(" with retry extensions such as junit-pioneer @RetryingTest: only the first attempt exists in") + .append(" the specific JVM."); + } + message.append(lineSeparator) + .append("- A failure before the test, for example in a @BeforeAll method."); + return JvmError.buildFrom(message.toString()); + } + + private static boolean isTemplateInvocation(String uniqueId) { + return uniqueId.contains("[test-template-invocation:") || uniqueId.contains("[class-template-invocation:"); + } + + private static class TestExecutionRecorder implements TestExecutionListener { + + private final AtomicInteger startedTestCount = new AtomicInteger(); + + private final AtomicInteger skippedCount = new AtomicInteger(); + + private final List failures = new CopyOnWriteArrayList<>(); + + private final List testAborts = new CopyOnWriteArrayList<>(); + + private final List containerAborts = new CopyOnWriteArrayList<>(); + + private final List unsuccessfulExecutions = new CopyOnWriteArrayList<>(); + + @Override + public void executionSkipped(TestIdentifier testIdentifier, String reason) { + skippedCount.incrementAndGet(); + unsuccessfulExecutions.add(testIdentifier.getDisplayName() + " skipped: " + reason); + } + + @Override + public void executionStarted(TestIdentifier testIdentifier) { + if (testIdentifier.isTest()) { + startedTestCount.incrementAndGet(); + } + } + + @Override + public void executionFinished(TestIdentifier testIdentifier, TestExecutionResult testExecutionResult) { + if (!testExecutionResult.getThrowable().isPresent()) { + return; + } + Throwable throwable = testExecutionResult.getThrowable().get(); + if (testExecutionResult.getStatus() == FAILED) { + failures.add(throwable); + unsuccessfulExecutions.add(testIdentifier.getDisplayName() + " failed: " + rootCauseOf(throwable)); + } else { + if (testIdentifier.isTest()) { + testAborts.add(throwable); + } else { + containerAborts.add(throwable); + } + unsuccessfulExecutions.add(testIdentifier.getDisplayName() + " aborted: " + throwable); + } + } + + private boolean onlyContainersAborted() { + return startedTestCount.get() == 0 && skippedCount.get() == 0 && failures.isEmpty() + && !containerAborts.isEmpty(); + } + + private static Throwable rootCauseOf(Throwable throwable) { + Set causes = Collections.newSetFromMap(new IdentityHashMap()); + Throwable rootCause = throwable; + while (rootCause.getCause() != null && causes.add(rootCause)) { + rootCause = rootCause.getCause(); + } + return rootCause; + } - return listener.getSummary(); } } diff --git a/junit5/junit5-extension/src/main/java/org/quickperf/junit5/QuickPerfTestExtension.java b/junit5/junit5-extension/src/main/java/org/quickperf/junit5/QuickPerfTestExtension.java index 424830e7..df4f3711 100644 --- a/junit5/junit5-extension/src/main/java/org/quickperf/junit5/QuickPerfTestExtension.java +++ b/junit5/junit5-extension/src/main/java/org/quickperf/junit5/QuickPerfTestExtension.java @@ -21,6 +21,8 @@ import org.quickperf.config.library.QuickPerfConfigs; import org.quickperf.config.library.QuickPerfConfigsLoader; import org.quickperf.config.library.SetOfAnnotationConfigs; +import org.quickperf.issue.JvmError; +import org.quickperf.issue.JvmIssue; import org.quickperf.issue.TestIssue; import org.quickperf.issue.JvmOrTestIssue; import org.quickperf.issue.PerfIssuesEvaluator; @@ -33,6 +35,7 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.Collection; +import java.util.Collections; public class QuickPerfTestExtension implements BeforeEachCallback, InvocationInterceptor { @@ -102,9 +105,7 @@ public void interceptAfterEachMethod(Invocation invocation, ReflectiveInvo public void interceptTestTemplateMethod(Invocation invocation, ReflectiveInvocationContext invocationContext, ExtensionContext extensionContext) throws Throwable { // Be careful that this method will be called by each invocation of the test template defines by a single test template method. // Normal lifecycle will apply. - // There is no allocation offset with template method TestExecutionContext testExecutionContext = getTestExecutionContext(extensionContext); - testExecutionContext.setRunnerAllocationOffset(0); if (testExecutionContext.isQuickPerfDisabled()) { invocation.proceed(); @@ -116,7 +117,8 @@ public void interceptTestTemplateMethod(Invocation invocation, ReflectiveI return; } - JvmOrTestIssue jvmOrTestIssue = executeTestMethodAndRecordPerformance(invocation, invocationContext, testExecutionContext); + TestToExecuteInNewJvm testToExecuteInNewJvm = TestToExecuteInNewJvm.buildFrom(extensionContext); + JvmOrTestIssue jvmOrTestIssue = executeTestMethodAndRecordPerformance(invocation, invocationContext, testExecutionContext, testToExecuteInNewJvm); processJvmOrTestIssue(jvmOrTestIssue, testExecutionContext); } @@ -135,7 +137,8 @@ public void interceptTestMethod( Invocation invocation return; } - JvmOrTestIssue jvmOrTestIssue = executeTestMethodAndRecordPerformance(invocation, invocationContext, testExecutionContext); + TestToExecuteInNewJvm testToExecuteInNewJvm = TestToExecuteInNewJvm.buildFrom(extensionContext); + JvmOrTestIssue jvmOrTestIssue = executeTestMethodAndRecordPerformance(invocation, invocationContext, testExecutionContext, testToExecuteInNewJvm); processJvmOrTestIssue(jvmOrTestIssue, testExecutionContext); } @@ -168,6 +171,17 @@ public void interceptDynamicTest(Invocation invocation, ExtensionContext e } private void processJvmOrTestIssue(JvmOrTestIssue jvmOrTestIssue, TestExecutionContext testExecutionContext) throws Throwable { + Throwable testIssueThrowable = jvmOrTestIssue.getTestIssue().asThrowable(); + if (testIssueThrowable instanceof TestAbortedBeforeTestMethodInNewJvm) { + // The new JVM did not invoke the test method: there is no performance record to evaluate + testExecutionContext.cleanResources(); + TestIssue testAbort = TestIssue.buildFrom(testIssueThrowable.getCause()); + quickPerfReporter.report(JvmOrTestIssue.buildFrom(testAbort) + , Collections.emptyList() + , testExecutionContext); + return; + } + SetOfAnnotationConfigs testAnnotationConfigs = quickPerfConfigs.getTestAnnotationConfigs(); Collection groupOfPerfIssuesToFormat @@ -181,17 +195,25 @@ private void processJvmOrTestIssue(JvmOrTestIssue jvmOrTestIssue, TestExecutionC , testExecutionContext); } - private void executeTestMethodInNewJvmAndRecordPerformance(Invocation invocation, ReflectiveInvocationContext invocationContext, TestExecutionContext testExecutionContext) throws IllegalAccessException, InvocationTargetException { + private void executeTestMethodInNewJvmAndRecordPerformance(Invocation invocation, ReflectiveInvocationContext invocationContext, TestExecutionContext testExecutionContext) throws Throwable { Object[] args = invocationContext.getArguments().toArray(); Object target = invocationContext.getTarget().orElse(null); Method method = makeAccessible(invocationContext.getExecutable()); invocation.skip();//skip the invocation as we directly invoke the test method + TestMethodInvocationInNewJvm testMethodInvocation = TestMethodInvocationInNewJvm.INSTANCE; + testMethodInvocation.recordInvocation(); + performanceRecording.start(testExecutionContext); try { //directly invoke the method to lower the interaction between JUnit, other extensions and QuickPerf. method.invoke(target, args); + } catch (InvocationTargetException invocationTargetException) { + // As JUnit, throw what the test method threw: JUnit and the other extensions handle it as usual + Throwable testMethodThrowable = invocationTargetException.getTargetException(); + testMethodInvocation.recordThrowable(testMethodThrowable); + throw testMethodThrowable; } finally { performanceRecording.stop(testExecutionContext); } @@ -205,11 +227,11 @@ private Method makeAccessible(Method executable) { return executable; } - private JvmOrTestIssue executeTestMethodAndRecordPerformance(Invocation invocation, ReflectiveInvocationContext invocationContext, TestExecutionContext testExecutionContext) { + private JvmOrTestIssue executeTestMethodAndRecordPerformance(Invocation invocation, ReflectiveInvocationContext invocationContext, TestExecutionContext testExecutionContext, TestToExecuteInNewJvm testToExecuteInNewJvm) { if (testExecutionContext.testExecutionUsesTwoJVMs()) { Method testMethod = invocationContext.getExecutable(); Class testClass = invocationContext.getTargetClass(); - JvmOrTestIssue jvmOrTestIssue = executeTestMethodInNewJvm(testClass, testMethod, testExecutionContext); + JvmOrTestIssue jvmOrTestIssue = executeTestMethodInNewJvm(testClass, testMethod, testExecutionContext, testToExecuteInNewJvm); tryToSkipInvocation(invocation); // because the test method is invoked directly inside the 'newJvmTestLauncher' return jvmOrTestIssue; } @@ -226,12 +248,20 @@ private void tryToSkipInvocation(Invocation invocation) { } } - private JvmOrTestIssue executeTestMethodInNewJvm(Class testClass, Method testMethod, TestExecutionContext testExecutionContext) { + private JvmOrTestIssue executeTestMethodInNewJvm(Class testClass, Method testMethod, TestExecutionContext testExecutionContext, TestToExecuteInNewJvm testToExecuteInNewJvm) { + testToExecuteInNewJvm.saveIn(testExecutionContext.getWorkingFolder()); NewJvmTestLauncher newJvmTestLauncher = NewJvmTestLauncher.INSTANCE; - return newJvmTestLauncher.executeTestMethodInNewJvm(testClass - , testMethod - , testExecutionContext - , QuickPerfJunit5Core.class); + JvmOrTestIssue jvmOrTestIssue = newJvmTestLauncher.executeTestMethodInNewJvm(testClass + , testMethod + , testExecutionContext + , QuickPerfJunit5Core.class); + Throwable testIssueThrowable = jvmOrTestIssue.getTestIssue().asThrowable(); + if (testIssueThrowable instanceof JvmError) { + // The new JVM did not execute the test once: there is no performance record to evaluate + JvmIssue jvmIssue = JvmIssue.buildFrom(testIssueThrowable.getMessage()); + return JvmOrTestIssue.buildFrom(jvmIssue); + } + return jvmOrTestIssue; } private TestIssue executeTestMethodAndRecordPerformanceInSameJvm(Invocation invocation, TestExecutionContext testExecutionContext) { diff --git a/junit5/junit5-extension/src/main/java/org/quickperf/junit5/TestAbortedBeforeTestMethodInNewJvm.java b/junit5/junit5-extension/src/main/java/org/quickperf/junit5/TestAbortedBeforeTestMethodInNewJvm.java new file mode 100644 index 00000000..7d54d637 --- /dev/null +++ b/junit5/junit5-extension/src/main/java/org/quickperf/junit5/TestAbortedBeforeTestMethodInNewJvm.java @@ -0,0 +1,30 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * Copyright 2019-2022 the original author or authors. + */ +package org.quickperf.junit5; + +import org.opentest4j.TestAbortedException; + +/** + * Sent by the new JVM when JUnit aborted the test before QuickPerf invoked the test method, for example because an + * assumption failed in a @BeforeEach or @BeforeAll method. Its cause is the abort. The test + * JVM reports this abort without evaluating performance records, because no performance was recorded. + */ +class TestAbortedBeforeTestMethodInNewJvm extends TestAbortedException { + + private static final long serialVersionUID = 1L; + + TestAbortedBeforeTestMethodInNewJvm(Throwable abort) { + super(abort.getMessage(), abort); + } + +} diff --git a/junit5/junit5-extension/src/main/java/org/quickperf/junit5/TestMethodInvocationInNewJvm.java b/junit5/junit5-extension/src/main/java/org/quickperf/junit5/TestMethodInvocationInNewJvm.java new file mode 100644 index 00000000..e68e8249 --- /dev/null +++ b/junit5/junit5-extension/src/main/java/org/quickperf/junit5/TestMethodInvocationInNewJvm.java @@ -0,0 +1,46 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * Copyright 2019-2022 the original author or authors. + */ +package org.quickperf.junit5; + +/** + * The new JVM executes one test. There, the QuickPerf extension records whether it invoked the test method and what + * the test method threw, and {@link JUnit5TestRunner} reads it once JUnit has executed the test. + */ +class TestMethodInvocationInNewJvm { + + static final TestMethodInvocationInNewJvm INSTANCE = new TestMethodInvocationInNewJvm(); + + // Written by the thread executing the test method, read by the thread that started JUnit + private volatile boolean testMethodInvoked; + + private volatile Throwable testMethodThrowable; + + private TestMethodInvocationInNewJvm() { } + + void recordInvocation() { + testMethodInvoked = true; + } + + void recordThrowable(Throwable throwable) { + testMethodThrowable = throwable; + } + + boolean isTestMethodInvoked() { + return testMethodInvoked; + } + + Throwable getTestMethodThrowable() { + return testMethodThrowable; + } + +} diff --git a/junit5/junit5-extension/src/main/java/org/quickperf/junit5/TestToExecuteInNewJvm.java b/junit5/junit5-extension/src/main/java/org/quickperf/junit5/TestToExecuteInNewJvm.java new file mode 100644 index 00000000..624366ae --- /dev/null +++ b/junit5/junit5-extension/src/main/java/org/quickperf/junit5/TestToExecuteInNewJvm.java @@ -0,0 +1,60 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * Copyright 2019-2022 the original author or authors. + */ +package org.quickperf.junit5; + +import org.junit.jupiter.api.extension.ExtensionContext; +import org.quickperf.WorkingFolder; +import org.quickperf.repository.ObjectFileRepository; + +import java.io.File; +import java.io.Serializable; + +/** + * The test JVM saves the JUnit unique id of the test in the working folder, and the new JVM executes the test having + * this unique id. The unique id of a test template invocation contains its index, for example + * [test-template-invocation:#2]: so the arguments source of a parameterized test has to provide the same + * arguments, in the same order, in every JVM. + */ +class TestToExecuteInNewJvm implements Serializable { + + private static final long serialVersionUID = 1L; + + private static final String FILE_NAME = "junit5-test-to-execute.ser"; + + private final String uniqueId; + + private TestToExecuteInNewJvm(String uniqueId) { + this.uniqueId = uniqueId; + } + + static TestToExecuteInNewJvm buildFrom(ExtensionContext extensionContext) { + return new TestToExecuteInNewJvm(extensionContext.getUniqueId()); + } + + void saveIn(WorkingFolder workingFolder) { + ObjectFileRepository.INSTANCE.save(workingFolder, FILE_NAME, this); + } + + static TestToExecuteInNewJvm findIn(String workingFolderPath) { + File file = new File(workingFolderPath, FILE_NAME); + if (!file.exists()) { + return null; + } + return (TestToExecuteInNewJvm) ObjectFileRepository.INSTANCE.find(workingFolderPath, FILE_NAME); + } + + String getUniqueId() { + return uniqueId; + } + +} diff --git a/junit5/junit5-jvm-test/pom.xml b/junit5/junit5-jvm-test/pom.xml index 361153f5..f5b17ce2 100644 --- a/junit5/junit5-jvm-test/pom.xml +++ b/junit5/junit5-jvm-test/pom.xml @@ -52,6 +52,12 @@ 1.1.1-SNAPSHOT test + + org.junit.jupiter + junit-jupiter-params + ${junit5.jupiter.version} + test + \ No newline at end of file diff --git a/junit5/junit5-jvm-test/src/test/java/org/quickperf/junit5/jvm/parameter/TestMethodWithParametersInNewJvmJUnit5Test.java b/junit5/junit5-jvm-test/src/test/java/org/quickperf/junit5/jvm/parameter/TestMethodWithParametersInNewJvmJUnit5Test.java new file mode 100644 index 00000000..18f504ec --- /dev/null +++ b/junit5/junit5-jvm-test/src/test/java/org/quickperf/junit5/jvm/parameter/TestMethodWithParametersInNewJvmJUnit5Test.java @@ -0,0 +1,330 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * Copyright 2019-2022 the original author or authors. + */ +package org.quickperf.junit5.jvm.parameter; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestReporter; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ParameterContext; +import org.junit.jupiter.api.extension.ParameterResolver; +import org.junit.jupiter.api.io.TempDir; +import org.quickperf.junit5.JUnit5Tests; +import org.quickperf.junit5.JUnit5Tests.JUnit5TestsResult; +import org.quickperf.junit5.JvmExecutionReport; +import org.quickperf.junit5.QuickPerfTest; +import org.quickperf.jvm.allocation.AllocationUnit; +import org.quickperf.jvm.annotations.ExpectMaxHeapAllocation; +import org.quickperf.jvm.annotations.HeapSize; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.quickperf.junit5.JvmExecutionReport.assertThatEachTestWasExecutedInItsOwnNewJvm; +import static org.quickperf.junit5.JvmExecutionReport.assertThatEachTestWasTheTestExecutedInItsNewJvm; +import static org.quickperf.junit5.JvmExecutionReport.reportExecutionOf; + +/** + * QuickPerf executes a test method having an annotation that needs a specific JVM (@HeapSize, + * @ExpectMaxHeapAllocation...) in a new JVM. There, JUnit and its extensions resolve the parameters of the + * test method (TestInfo, @TempDir, a Spring bean, a Mockito mock...). + *

+ * Some test methods of the nested test classes fail on purpose with a {@link JvmExecutionReport}, to tell where they + * were executed. Others fail if their JVM executes them more than once. QuickPerf fails a test that JUnit does not + * execute in the specific JVM. + */ +public class TestMethodWithParametersInNewJvmJUnit5Test { + + @QuickPerfTest + static class TestMethodHavingATestInfoParameterInASpecificJvm { + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @Test + public void test_method(TestInfo testInfo) { + throw reportExecutionOf(testInfo.getDisplayName()); + } + + } + + @Test public void + should_execute_a_test_method_having_a_test_info_parameter_in_a_new_jvm() { + + // GIVEN + Class testClass = TestMethodHavingATestInfoParameterInASpecificJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + List jvmExecutionReports = jUnit5TestsResult.getJvmExecutionReports(); + assertThat(jvmExecutionReports).as(jUnit5TestsResult.getErrorReport()) + .extracting("executedTest") + .containsExactly("test_method(TestInfo)"); + assertThatEachTestWasExecutedInItsOwnNewJvm(jvmExecutionReports); + + } + + @QuickPerfTest + static class TestMethodHavingATestReporterParameterInASpecificJvm { + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @Test + public void test_method(TestReporter testReporter) { + testReporter.publishEntry("key", "value"); + throw reportExecutionOf("entry published"); + } + + } + + @Test public void + should_execute_a_test_method_having_a_test_reporter_parameter_in_a_new_jvm() { + + // GIVEN + Class testClass = TestMethodHavingATestReporterParameterInASpecificJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + List jvmExecutionReports = jUnit5TestsResult.getJvmExecutionReports(); + assertThat(jvmExecutionReports).as(jUnit5TestsResult.getErrorReport()) + .extracting("executedTest") + .containsExactly("entry published"); + assertThatEachTestWasExecutedInItsOwnNewJvm(jvmExecutionReports); + + } + + @QuickPerfTest + static class TestMethodHavingATempDirParameterInASpecificJvm { + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @Test + public void test_method(@TempDir Path temporaryDirectory) throws IOException { + Path file = temporaryDirectory.resolve("file.txt"); + Files.write(file, Collections.singletonList("content"), StandardCharsets.UTF_8); + List lines = Files.readAllLines(file, StandardCharsets.UTF_8); + throw reportExecutionOf("read from the temporary directory: " + lines); + } + + } + + @Test public void + should_execute_a_test_method_having_a_temp_dir_parameter_in_a_new_jvm() { + + // GIVEN + Class testClass = TestMethodHavingATempDirParameterInASpecificJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + List jvmExecutionReports = jUnit5TestsResult.getJvmExecutionReports(); + assertThat(jvmExecutionReports).as(jUnit5TestsResult.getErrorReport()) + .extracting("executedTest") + .containsExactly("read from the temporary directory: [content]"); + assertThatEachTestWasExecutedInItsOwnNewJvm(jvmExecutionReports); + + } + + // As the extensions of Spring or Mockito, which resolve beans or mocks + static class StringParameterResolver implements ParameterResolver { + + @Override + public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) { + return parameterContext.getParameter().getType() == String.class; + } + + @Override + public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) { + return "resolved by an extension"; + } + + } + + @QuickPerfTest + @ExtendWith(StringParameterResolver.class) + static class TestMethodHavingAParameterResolvedByAnExtensionInASpecificJvm { + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @Test + public void test_method(String parameter) { + throw reportExecutionOf("parameter " + parameter); + } + + } + + @Test public void + should_execute_a_test_method_having_a_parameter_resolved_by_an_extension_in_a_new_jvm() { + + // GIVEN + Class testClass = TestMethodHavingAParameterResolvedByAnExtensionInASpecificJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + List jvmExecutionReports = jUnit5TestsResult.getJvmExecutionReports(); + assertThat(jvmExecutionReports).as(jUnit5TestsResult.getErrorReport()) + .extracting("executedTest") + .containsExactly("parameter resolved by an extension"); + assertThatEachTestWasExecutedInItsOwnNewJvm(jvmExecutionReports); + + } + + @QuickPerfTest + static class TestMethodsHavingParametersInASpecificJvm { + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @Test + public void test_method_having_several_parameters(TestInfo testInfo + , TestReporter testReporter + , @TempDir Path temporaryDirectory) { + throw reportExecutionOf(testInfo.getDisplayName()); + } + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @Test + public void test_method_having_a_parameter(TestInfo testInfo) { + throw reportExecutionOf(testInfo.getDisplayName()); + } + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @Test + public void test_method_without_parameter() { + throw reportExecutionOf("test_method_without_parameter()"); + } + + } + + @Test public void + should_execute_each_test_method_of_a_test_class_in_its_own_jvm() { + + // GIVEN + Class testClass = TestMethodsHavingParametersInASpecificJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + String errorReport = jUnit5TestsResult.getErrorReport(); + + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as("Each test method should fail with the report of its execution.%s", errorReport) + .isEqualTo(3); + + List jvmExecutionReports = jUnit5TestsResult.getJvmExecutionReports(); + assertThat(jvmExecutionReports).extracting("executedTest") + .containsExactlyInAnyOrder("test_method_having_several_parameters(TestInfo, TestReporter, Path)" + , "test_method_having_a_parameter(TestInfo)" + , "test_method_without_parameter()"); + assertThatEachTestWasExecutedInItsOwnNewJvm(jvmExecutionReports); + assertThatEachTestWasTheTestExecutedInItsNewJvm(jvmExecutionReports); + + } + + @QuickPerfTest + static class TestMethodsHavingParametersAndFailingIfExecutedAgainInTheSameJvm { + + private static final AtomicInteger EXECUTIONS_IN_THIS_JVM = new AtomicInteger(); + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @Test + public void test_method_having_a_test_info_parameter(TestInfo testInfo) { + assertThat(EXECUTIONS_IN_THIS_JVM.incrementAndGet()).as("Executions in this JVM").isOne(); + assertThat(testInfo.getDisplayName()).isEqualTo("test_method_having_a_test_info_parameter(TestInfo)"); + } + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @Test + public void test_method_having_a_temp_dir_parameter(@TempDir Path temporaryDirectory) throws IOException { + assertThat(EXECUTIONS_IN_THIS_JVM.incrementAndGet()).as("Executions in this JVM").isOne(); + Path file = temporaryDirectory.resolve("file.txt"); + Files.write(file, Collections.singletonList("content"), StandardCharsets.UTF_8); + assertThat(Files.readAllLines(file, StandardCharsets.UTF_8)).containsExactly("content"); + } + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @Test + public void test_method_having_several_parameters(TestInfo testInfo + , TestReporter testReporter + , @TempDir Path temporaryDirectory) { + assertThat(EXECUTIONS_IN_THIS_JVM.incrementAndGet()).as("Executions in this JVM").isOne(); + assertThat(temporaryDirectory).isDirectory(); + } + + } + + @Test public void + should_execute_each_test_method_having_parameters_once_in_its_jvm() { + + // GIVEN + Class testClass = TestMethodsHavingParametersAndFailingIfExecutedAgainInTheSameJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as(jUnit5TestsResult.getErrorReport()) + .isZero(); + + assertThat(jUnit5TestsResult.getNumberOfSucceededTests()).isEqualTo(3); + + } + + @QuickPerfTest + static class TestMethodHavingATestInfoParameterAndAllocatingThreeMegaBytes { + + @ExpectMaxHeapAllocation(value = 1, unit = AllocationUnit.MEGA_BYTE) + @Test + public void test_method(TestInfo testInfo) { + byte[] allocatedBytes = new byte[3 * 1024 * 1024]; + } + + } + + @Test public void + should_measure_the_heap_allocation_of_a_test_method_having_a_parameter() { + + // GIVEN + Class testClass = TestMethodHavingATestInfoParameterAndAllocatingThreeMegaBytes.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + String errorReport = jUnit5TestsResult.getErrorReport(); + + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as(errorReport) + .isOne(); + + assertThat(errorReport).contains("Expected heap allocation (test method thread) to be less than 1.00 Mega bytes but is 3."); + + } + +} diff --git a/junit5/junit5-jvm-test/src/test/java/org/quickperf/junit5/jvm/testtemplate/TestExecutionChecksInNewJvmJUnit5Test.java b/junit5/junit5-jvm-test/src/test/java/org/quickperf/junit5/jvm/testtemplate/TestExecutionChecksInNewJvmJUnit5Test.java new file mode 100644 index 00000000..e42d50f0 --- /dev/null +++ b/junit5/junit5-jvm-test/src/test/java/org/quickperf/junit5/jvm/testtemplate/TestExecutionChecksInNewJvmJUnit5Test.java @@ -0,0 +1,610 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * Copyright 2019-2022 the original author or authors. + */ +package org.quickperf.junit5.jvm.testtemplate; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.condition.DisabledIfSystemProperty; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.Extension; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.TestExecutionExceptionHandler; +import org.junit.jupiter.api.extension.TestTemplateInvocationContext; +import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.opentest4j.AssertionFailedError; +import org.opentest4j.TestAbortedException; +import org.quickperf.SystemProperties; +import org.quickperf.junit5.JUnit5Tests; +import org.quickperf.junit5.JUnit5Tests.JUnit5TestsResult; +import org.quickperf.junit5.QuickPerfTest; +import org.quickperf.jvm.allocation.AllocationUnit; +import org.quickperf.jvm.annotations.ExpectMaxHeapAllocation; +import org.quickperf.jvm.annotations.HeapSize; + +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assumptions.assumeFalse; + +/** + * QuickPerf executes a test having an annotation that needs a specific JVM (@HeapSize, + * @ExpectMaxHeapAllocation...) in a specific JVM, which has to execute this test once. These tests check + * the result of tests that the specific JVM does not execute as the test JVM would. + */ +public class TestExecutionChecksInNewJvmJUnit5Test { + + private static boolean executingInTheSpecificJvm() { + return SystemProperties.TEST_CODE_EXECUTING_IN_NEW_JVM.evaluate(); + } + + @QuickPerfTest + static class TestDisabledInTheSpecificJvm { + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @DisabledIfSystemProperty(named = "quickPerfToExecInASpecificJvm", matches = "true") + @Test + public void test_disabled_in_the_specific_jvm() { + } + + } + + @Test public void + should_fail_if_a_condition_disables_the_test_in_the_specific_jvm() { + + // GIVEN + Class testClass = TestDisabledInTheSpecificJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + String errorReport = jUnit5TestsResult.getErrorReport(); + + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as("The test should fail because the specific JVM did not execute it.%s", errorReport) + .isOne(); + + assertThat(errorReport).contains("JUnit started 0 tests instead of 1 in the specific JVM") + .contains("[method:test_disabled_in_the_specific_jvm()]") + .contains("test_disabled_in_the_specific_jvm() skipped: ") + .contains("quickPerfToExecInASpecificJvm") + .contains("@JvmOptions(\"-Dname=value\")"); + + } + + @QuickPerfTest + static class ParameterizedTestWithFewerArgumentsInTheSpecificJvm { + + static IntStream values() { + return executingInTheSpecificJvm() ? IntStream.of(1, 2) : IntStream.of(1, 2, 3); + } + + @ExpectMaxHeapAllocation(value = 1, unit = AllocationUnit.MEGA_BYTE) + @ParameterizedTest + @MethodSource("values") + public void parameterized_test(int value) { + } + + } + + @Test public void + should_fail_the_test_template_invocation_that_does_not_exist_in_the_specific_jvm() { + + // GIVEN + Class testClass = ParameterizedTestWithFewerArgumentsInTheSpecificJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + String errorReport = jUnit5TestsResult.getErrorReport(); + + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as("Only the third invocation should fail, because it does not exist in the specific JVM.%s" + , errorReport) + .isOne(); + + assertThat(errorReport).contains("JUnit started 0 tests instead of 1 in the specific JVM") + .contains("[test-template-invocation:#3]") + .contains("provides other arguments in the specific JVM") + .doesNotContain("UnexpectedQuickPerfException"); + + } + + @QuickPerfTest + static class BeforeAllMethodFailingInTheSpecificJvm { + + @BeforeAll + public static void before_all() { + if (executingInTheSpecificJvm()) { + throw new IllegalStateException("@BeforeAll failure in the specific JVM"); + } + } + + @ExpectMaxHeapAllocation(value = 1, unit = AllocationUnit.MEGA_BYTE) + @Test + public void test_method() { + } + + } + + @Test public void + should_report_the_failure_of_a_before_all_method_in_the_specific_jvm() { + + // GIVEN + Class testClass = BeforeAllMethodFailingInTheSpecificJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + String errorReport = jUnit5TestsResult.getErrorReport(); + + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as(errorReport) + .isOne(); + + assertThat(errorReport).contains("JUnit started 0 tests instead of 1 in the specific JVM") + .contains("java.lang.IllegalStateException: @BeforeAll failure in the specific JVM") + .doesNotContain("UnexpectedQuickPerfException"); + + } + + @QuickPerfTest + static class AssumptionFailingInABeforeEachMethodInTheSpecificJvm { + + @BeforeEach + public void before_each() { + assumeFalse(executingInTheSpecificJvm(), "Assumption failing in the specific JVM"); + } + + @ExpectMaxHeapAllocation(value = 1, unit = AllocationUnit.MEGA_BYTE) + @Test + public void test_method() { + } + + } + + @Test public void + should_report_a_test_aborted_in_the_specific_jvm_as_aborted() { + + // GIVEN + Class testClass = AssumptionFailingInABeforeEachMethodInTheSpecificJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + String errorReport = jUnit5TestsResult.getErrorReport(); + + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as(errorReport) + .isZero(); + + assertThat(jUnit5TestsResult.getNumberOfAbortedTests()) + .as("The test should be aborted, as in the specific JVM.") + .isOne(); + + } + + // As an AssertJ assumption, which throws a TestAbortedException having an AssertionFailedError cause + private static void assumeNotExecutingInTheSpecificJvm(String description) { + if (executingInTheSpecificJvm()) { + String assertionMessage = "[" + description + "] Expecting value to be false but was true"; + AssertionFailedError assertionFailedError = new AssertionFailedError(assertionMessage, false, true); + throw new TestAbortedException("assumption was not met due to: " + assertionMessage, assertionFailedError); + } + } + + @QuickPerfTest + static class AssumptionWithACauseFailingInABeforeEachMethodInTheSpecificJvm { + + @BeforeEach + public void before_each() { + assumeNotExecutingInTheSpecificJvm("Database available"); + } + + @ExpectMaxHeapAllocation(value = 1, unit = AllocationUnit.MEGA_BYTE) + @Test + public void test_method() { + } + + } + + @Test public void + should_report_a_test_aborted_with_a_cause_in_the_specific_jvm_as_aborted() { + + // GIVEN + Class testClass = AssumptionWithACauseFailingInABeforeEachMethodInTheSpecificJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as(jUnit5TestsResult.getErrorReport()) + .isZero(); + + assertThat(jUnit5TestsResult.getNumberOfAbortedTests()) + .as("The test should be aborted, as in the specific JVM.") + .isOne(); + + assertThat(jUnit5TestsResult.getAbortReport()) + .contains("TestAbortedException: assumption was not met due to: [Database available]") + .contains("Caused by: org.opentest4j.AssertionFailedError: [Database available]"); + + } + + @QuickPerfTest + static class AssumptionFailingInABeforeAllMethodInTheSpecificJvm { + + @BeforeAll + public static void before_all() { + assumeFalse(executingInTheSpecificJvm(), "Assumption failing in a @BeforeAll method of the specific JVM"); + } + + @ExpectMaxHeapAllocation(value = 1, unit = AllocationUnit.MEGA_BYTE) + @Test + public void test_method() { + } + + } + + @Test public void + should_report_a_test_aborted_by_a_before_all_method_of_the_specific_jvm_as_aborted() { + + // GIVEN + Class testClass = AssumptionFailingInABeforeAllMethodInTheSpecificJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as(jUnit5TestsResult.getErrorReport()) + .isZero(); + + assertThat(jUnit5TestsResult.getNumberOfAbortedTests()) + .as("The test should be aborted, as its class in the specific JVM.") + .isOne(); + + assertThat(jUnit5TestsResult.getAbortReport()) + .contains("Assumption failing in a @BeforeAll method of the specific JVM"); + + } + + @QuickPerfTest + static class AssumptionFailingInAnArgumentsSourceInTheSpecificJvm { + + static Stream values() { + assumeFalse(executingInTheSpecificJvm(), "Assumption failing in an arguments source of the specific JVM"); + return Stream.of(1, 2); + } + + @ExpectMaxHeapAllocation(value = 1, unit = AllocationUnit.MEGA_BYTE) + @ParameterizedTest + @MethodSource("values") + public void parameterized_test(int value) { + } + + } + + @Test public void + should_report_the_invocations_aborted_by_an_arguments_source_of_the_specific_jvm_as_aborted() { + + // GIVEN + Class testClass = AssumptionFailingInAnArgumentsSourceInTheSpecificJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as(jUnit5TestsResult.getErrorReport()) + .isZero(); + + assertThat(jUnit5TestsResult.getNumberOfAbortedTests()) + .as("Each invocation should be aborted, as its parameterized test in the specific JVM.") + .isEqualTo(2); + + assertThat(jUnit5TestsResult.getAbortReport()) + .contains("Assumption failing in an arguments source of the specific JVM"); + + } + + @QuickPerfTest + static class AssumptionFailingInAnAfterEachMethodInTheSpecificJvm { + + @AfterEach + public void after_each() { + assumeFalse(executingInTheSpecificJvm(), "Assumption failing in an @AfterEach method of the specific JVM"); + } + + @ExpectMaxHeapAllocation(value = 1, unit = AllocationUnit.MEGA_BYTE) + @Test + public void test_method_allocating_3_mega_bytes() { + byte[] allocatedBytes = new byte[3 * 1024 * 1024]; + } + + } + + @Test public void + should_report_the_performance_issue_of_a_test_aborted_after_its_test_method_in_the_specific_jvm() { + + // GIVEN + Class testClass = AssumptionFailingInAnAfterEachMethodInTheSpecificJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + String errorReport = jUnit5TestsResult.getErrorReport(); + + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as("The performance issue should fail the test, as when the test does not need a specific JVM.%s" + , errorReport) + .isOne(); + + assertThat(errorReport).contains("Expected heap allocation (test method thread) to be less than 1.00 Mega bytes") + .contains("Assumption failing in an @AfterEach method of the specific JVM"); + + } + + @QuickPerfTest + static class AssumptionWithACauseFailingInTheTestMethodInTheSpecificJvm { + + @ExpectMaxHeapAllocation(value = 1, unit = AllocationUnit.MEGA_BYTE) + @Test + public void test_method() { + assumeNotExecutingInTheSpecificJvm("Server available"); + } + + } + + @Test public void + should_report_a_test_aborted_by_its_test_method_in_the_specific_jvm_as_aborted() { + + // GIVEN + Class testClass = AssumptionWithACauseFailingInTheTestMethodInTheSpecificJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as(jUnit5TestsResult.getErrorReport()) + .isZero(); + + assertThat(jUnit5TestsResult.getNumberOfAbortedTests()) + .as("The test should be aborted, as in the specific JVM.") + .isOne(); + + assertThat(jUnit5TestsResult.getAbortReport()) + .contains("TestAbortedException: assumption was not met due to: [Server available]"); + + } + + // Like junit-pioneer @RetryingTest(3): provides the attempt n + 1 only if the attempt n failed, and aborts a failed + // attempt if another attempt remains + static class RetryingTestTemplate implements TestTemplateInvocationContextProvider { + + private static final int MAX_ATTEMPTS = 3; + + @Override + public boolean supportsTestTemplate(ExtensionContext extensionContext) { + return true; + } + + @Override + public Stream provideTestTemplateInvocationContexts(ExtensionContext extensionContext) { + Iterator attempts = new Attempts(); + return StreamSupport.stream(Spliterators.spliteratorUnknownSize(attempts, Spliterator.ORDERED), false); + } + + private static class Attempts implements Iterator { + + private int attemptCount; + + private int failedAttemptCount; + + @Override + public boolean hasNext() { + return attemptCount == failedAttemptCount && attemptCount < MAX_ATTEMPTS; + } + + @Override + public TestTemplateInvocationContext next() { + int attempt = ++attemptCount; + return new TestTemplateInvocationContext() { + @Override + public List getAdditionalExtensions() { + TestExecutionExceptionHandler retryIfAnAttemptRemains = (extensionContext, throwable) -> { + failedAttemptCount++; + if (attempt < MAX_ATTEMPTS) { + throw new TestAbortedException("Attempt " + attempt + " failed, retrying", throwable); + } + throw throwable; + }; + return Collections.singletonList(retryIfAnAttemptRemains); + } + }; + } + + } + + } + + @QuickPerfTest + static class AlwaysFailingRetryingTestInASpecificJvm { + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @TestTemplate + @ExtendWith(RetryingTestTemplate.class) + public void always_failing_test() { + throw new AssertionError("Failure of the test method"); + } + + } + + @Test public void + should_fail_an_always_failing_test_template_retrying_its_failed_invocations() { + + // GIVEN + Class testClass = AlwaysFailingRetryingTestInASpecificJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + String errorReport = jUnit5TestsResult.getErrorReport(); + + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as("The last attempt should fail, and no attempt should pass.%s", errorReport) + .isOne(); + + assertThat(jUnit5TestsResult.getNumberOfAbortedTests()) + .as("The first two attempts should be aborted to be retried") + .isEqualTo(2); + + // Only the first attempt exists in a specific JVM, where the previous attempts were not executed + assertThat(errorReport).contains("JUnit started 0 tests instead of 1 in the specific JVM") + .contains("[test-template-invocation:#3]") + .contains("retry extensions such as junit-pioneer @RetryingTest"); + + } + + @QuickPerfTest + static class OverloadedTestMethodsInASpecificJvm { + + private static final AtomicInteger EXECUTIONS_IN_THIS_JVM = new AtomicInteger(); + + // The test methods fail to report which method the specific JVM executed + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @Test + public void overloaded_test_method() { + throw new AssertionError("Execution " + EXECUTIONS_IN_THIS_JVM.incrementAndGet() + + " in this JVM: overloaded_test_method()"); + } + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @ParameterizedTest + @ValueSource(ints = {1, 2}) + public void overloaded_test_method(int value) { + throw new AssertionError("Execution " + EXECUTIONS_IN_THIS_JVM.incrementAndGet() + + " in this JVM: overloaded_test_method(" + value + ")"); + } + + } + + @Test public void + should_execute_the_overloaded_test_method_of_each_test_once_in_its_own_jvm() { + + // GIVEN + Class testClass = OverloadedTestMethodsInASpecificJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + String errorReport = jUnit5TestsResult.getErrorReport(); + + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as(errorReport) + .isEqualTo(3); + + assertThat(errorReport).contains("Execution 1 in this JVM: overloaded_test_method()") + .contains("Execution 1 in this JVM: overloaded_test_method(1)") + .contains("Execution 1 in this JVM: overloaded_test_method(2)"); + + } + + @QuickPerfTest + static class OutOfMemoryErrorInTheTestMethodInTheSpecificJvm { + + private static final long TWENTY_SECONDS = 20_000L; + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @Test + public void test_method_throwing_an_out_of_memory_error_and_leaving_a_non_daemon_thread() { + // This thread keeps the specific JVM alive if the JVM does not end with System.exit + Thread nonDaemonThread = new Thread(() -> sleep(TWENTY_SECONDS)); + nonDaemonThread.setDaemon(false); + nonDaemonThread.start(); + throw new OutOfMemoryError("Out of memory error in the specific JVM"); + } + + private static void sleep(long durationInMs) { + try { + Thread.sleep(durationInMs); + } catch (InterruptedException interruptedException) { + Thread.currentThread().interrupt(); + } + } + + } + + @Test public void + should_report_an_out_of_memory_error_of_the_test_method_without_waiting_for_the_threads_of_the_specific_jvm() { + + // GIVEN + Class testClass = OutOfMemoryErrorInTheTestMethodInTheSpecificJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + long start = System.nanoTime(); + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + long durationInSeconds = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - start); + + // THEN + String errorReport = jUnit5TestsResult.getErrorReport(); + + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as(errorReport) + .isOne(); + + assertThat(errorReport).contains("java.lang.OutOfMemoryError: Out of memory error in the specific JVM"); + + assertThat(durationInSeconds) + .as("The specific JVM should end without waiting for the non-daemon thread of the test") + .isLessThan(15); + + } + +} diff --git a/junit5/junit5-jvm-test/src/test/java/org/quickperf/junit5/jvm/testtemplate/TestTemplateInNewJvmJUnit5Test.java b/junit5/junit5-jvm-test/src/test/java/org/quickperf/junit5/jvm/testtemplate/TestTemplateInNewJvmJUnit5Test.java new file mode 100644 index 00000000..418f4c5e --- /dev/null +++ b/junit5/junit5-jvm-test/src/test/java/org/quickperf/junit5/jvm/testtemplate/TestTemplateInNewJvmJUnit5Test.java @@ -0,0 +1,393 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * Copyright 2019-2022 the original author or authors. + */ +package org.quickperf.junit5.jvm.testtemplate; + +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.RepeatedTest; +import org.junit.jupiter.api.RepetitionInfo; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.quickperf.junit5.JUnit5Tests; +import org.quickperf.junit5.JUnit5Tests.JUnit5TestsResult; +import org.quickperf.junit5.JvmExecutionReport; +import org.quickperf.junit5.QuickPerfTest; +import org.quickperf.jvm.allocation.AllocationUnit; +import org.quickperf.jvm.annotations.ExpectMaxHeapAllocation; +import org.quickperf.jvm.annotations.HeapSize; + +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.params.provider.Arguments.arguments; +import static org.quickperf.junit5.JvmExecutionReport.assertThatEachTestWasExecutedInItsOwnNewJvm; +import static org.quickperf.junit5.JvmExecutionReport.assertThatEachTestWasTheTestExecutedInItsNewJvm; +import static org.quickperf.junit5.JvmExecutionReport.reportExecutionOf; + +/** + * QuickPerf executes each invocation of a test template (@RepeatedTest, @ParameterizedTest...) + * having an annotation that needs a specific JVM (@HeapSize, @ExpectMaxHeapAllocation...) in + * its own JVM. This JVM has to execute this invocation once, with its arguments, and QuickPerf has to report the + * measures and the failures of this execution for this invocation. + *

+ * Some test methods of the nested test classes fail on purpose with a {@link JvmExecutionReport}, to tell where they + * were executed. Others fail if their JVM executes them more than once. QuickPerf fails a test that JUnit does not + * execute in the specific JVM. + */ +public class TestTemplateInNewJvmJUnit5Test { + + @QuickPerfTest + static class RepeatedTestFailingIfExecutedAgainInTheSameJvm { + + private static final AtomicInteger EXECUTIONS_IN_THIS_JVM = new AtomicInteger(); + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @RepeatedTest(3) + public void repeated_test() { + assertThat(EXECUTIONS_IN_THIS_JVM.incrementAndGet()).as("Executions in this JVM").isOne(); + } + + } + + @Test public void + should_execute_each_repetition_once_in_its_jvm() { + + // GIVEN + Class testClass = RepeatedTestFailingIfExecutedAgainInTheSameJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as(jUnit5TestsResult.getErrorReport()) + .isZero(); + + assertThat(jUnit5TestsResult.getNumberOfSucceededTests()).isEqualTo(3); + + } + + @QuickPerfTest + static class RepeatedTestHavingARepetitionInfoParameterInASpecificJvm { + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @RepeatedTest(3) + public void repeated_test(RepetitionInfo repetitionInfo) { + throw reportExecutionOf("repetition " + repetitionInfo.getCurrentRepetition() + + " of " + repetitionInfo.getTotalRepetitions()); + } + + } + + @Test public void + should_execute_each_repetition_of_a_repeated_test_having_a_repetition_info_parameter_in_its_own_jvm() { + + // GIVEN + Class testClass = RepeatedTestHavingARepetitionInfoParameterInASpecificJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + String errorReport = jUnit5TestsResult.getErrorReport(); + + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as("Each repetition should fail with the report of its execution.%s", errorReport) + .isEqualTo(3); + + List jvmExecutionReports = jUnit5TestsResult.getJvmExecutionReports(); + assertThat(jvmExecutionReports).extracting("executedTest") + .containsExactlyInAnyOrder("repetition 1 of 3", "repetition 2 of 3", "repetition 3 of 3"); + assertThatEachTestWasExecutedInItsOwnNewJvm(jvmExecutionReports); + assertThatEachTestWasTheTestExecutedInItsNewJvm(jvmExecutionReports); + + } + + @QuickPerfTest + static class RepeatedTestAllocatingMoreAtEachExecutionInTheSameJvm { + + private static final AtomicInteger EXECUTIONS_IN_THIS_JVM = new AtomicInteger(); + + // The first execution in a JVM allocates 1 MB, the second one 2 MB... + @ExpectMaxHeapAllocation(value = 1.5, unit = AllocationUnit.MEGA_BYTE) + @RepeatedTest(3) + public void repeated_test() { + int executionNumber = EXECUTIONS_IN_THIS_JVM.incrementAndGet(); + byte[] allocatedBytes = new byte[executionNumber * 1024 * 1024]; + } + + } + + @Test public void + should_measure_the_heap_allocation_of_each_repetition_in_its_own_jvm() { + + // GIVEN + Class testClass = RepeatedTestAllocatingMoreAtEachExecutionInTheSameJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + String errorReport = jUnit5TestsResult.getErrorReport(); + + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as("Each repetition should be executed alone in its JVM, and allocate 1 MB.%s", errorReport) + .isZero(); + + assertThat(jUnit5TestsResult.getNumberOfSucceededTests()).isEqualTo(3); + + } + + @QuickPerfTest + static class ParameterizedTestFailingIfExecutedAgainInTheSameJvm { + + private static final AtomicInteger EXECUTIONS_IN_THIS_JVM = new AtomicInteger(); + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @ParameterizedTest + @ValueSource(ints = {1, 2, 3}) + public void parameterized_test(int value) { + assertThat(EXECUTIONS_IN_THIS_JVM.incrementAndGet()).as("Executions in this JVM").isOne(); + } + + } + + @Test public void + should_execute_each_invocation_of_a_parameterized_test_once_in_its_jvm() { + + // GIVEN + Class testClass = ParameterizedTestFailingIfExecutedAgainInTheSameJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as(jUnit5TestsResult.getErrorReport()) + .isZero(); + + assertThat(jUnit5TestsResult.getNumberOfSucceededTests()).isEqualTo(3); + + } + + @QuickPerfTest + static class ParameterizedTestInASpecificJvm { + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @ParameterizedTest(name = "value {0}") + @ValueSource(ints = {1, 2, 3}) + public void parameterized_test(int value) { + throw reportExecutionOf("value " + value); + } + + } + + @Test public void + should_execute_each_invocation_of_a_parameterized_test_in_its_own_jvm_with_its_argument() { + + // GIVEN + Class testClass = ParameterizedTestInASpecificJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + String errorReport = jUnit5TestsResult.getErrorReport(); + + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as("Each invocation should fail with the report of its execution.%s", errorReport) + .isEqualTo(3); + + List jvmExecutionReports = jUnit5TestsResult.getJvmExecutionReports(); + assertThat(jvmExecutionReports).extracting("executedTest") + .containsExactlyInAnyOrder("value 1", "value 2", "value 3"); + assertThatEachTestWasExecutedInItsOwnNewJvm(jvmExecutionReports); + assertThatEachTestWasTheTestExecutedInItsNewJvm(jvmExecutionReports); + + } + + @QuickPerfTest + static class ParameterizedTestAllocatingAccordingToItsArgument { + + @ExpectMaxHeapAllocation(value = 2.5, unit = AllocationUnit.MEGA_BYTE) + @ParameterizedTest(name = "allocating {0} MB") + @ValueSource(ints = {1, 2, 3}) + public void parameterized_test(int allocationInMegaBytes) { + byte[] allocatedBytes = new byte[allocationInMegaBytes * 1024 * 1024]; + } + + } + + @Test public void + should_report_the_heap_allocation_of_each_invocation_of_a_parameterized_test_for_this_invocation() { + + // GIVEN + Class testClass = ParameterizedTestAllocatingAccordingToItsArgument.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + String errorReport = jUnit5TestsResult.getErrorReport(); + + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as("Only the invocation allocating 3 MB should fail.%s", errorReport) + .isOne(); + + assertThat(jUnit5TestsResult.getNumberOfSucceededTests()).isEqualTo(2); + + assertThat(errorReport).contains("allocating 3 MB") + .contains("Expected heap allocation (test method thread) to be less than 2.50 Mega bytes but is 3.") + .doesNotContain("allocating 1 MB") + .doesNotContain("allocating 2 MB"); + + } + + @QuickPerfTest + static class ParameterizedTestHavingSeveralArgumentsAndATestInfoParameterInASpecificJvm { + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @ParameterizedTest(name = "{0} x {1}") + @CsvSource({"apple, 1", "banana, 2"}) + public void parameterized_test(String fruit, int quantity, TestInfo testInfo) { + String executedTest = fruit + " x " + quantity; + assertThat(testInfo.getDisplayName()).isEqualTo(executedTest); + throw reportExecutionOf(executedTest); + } + + } + + @Test public void + should_execute_each_invocation_of_a_parameterized_test_having_several_arguments_and_a_test_info_parameter_in_its_own_jvm() { + + // GIVEN + Class testClass = ParameterizedTestHavingSeveralArgumentsAndATestInfoParameterInASpecificJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + String errorReport = jUnit5TestsResult.getErrorReport(); + + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as("Each invocation should fail with the report of its execution.%s", errorReport) + .isEqualTo(2); + + List jvmExecutionReports = jUnit5TestsResult.getJvmExecutionReports(); + assertThat(jvmExecutionReports).extracting("executedTest") + .containsExactlyInAnyOrder("apple x 1", "banana x 2"); + assertThatEachTestWasExecutedInItsOwnNewJvm(jvmExecutionReports); + assertThatEachTestWasTheTestExecutedInItsNewJvm(jvmExecutionReports); + + } + + @QuickPerfTest + static class ParameterizedTestHavingAnArrayParameterInASpecificJvm { + + static Stream fruits() { + return Stream.of(arguments((Object) new String[] {"apple", "banana"}) + , arguments((Object) new String[] {"cherry"})); + } + + // The JUnit unique id of the test contains the array type, encoded: parameterized_test(%5BLjava.lang.String;) + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @ParameterizedTest(name = "{0}") + @MethodSource("fruits") + public void parameterized_test(String[] fruits) { + throw reportExecutionOf(Arrays.toString(fruits)); + } + + } + + @Test public void + should_execute_each_invocation_of_a_parameterized_test_having_an_array_parameter_in_its_own_jvm() { + + // GIVEN + Class testClass = ParameterizedTestHavingAnArrayParameterInASpecificJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + String errorReport = jUnit5TestsResult.getErrorReport(); + + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as("Each invocation should fail with the report of its execution.%s", errorReport) + .isEqualTo(2); + + List jvmExecutionReports = jUnit5TestsResult.getJvmExecutionReports(); + assertThat(jvmExecutionReports).extracting("executedTest") + .containsExactlyInAnyOrder("[apple, banana]", "[cherry]"); + assertThatEachTestWasExecutedInItsOwnNewJvm(jvmExecutionReports); + assertThatEachTestWasTheTestExecutedInItsNewJvm(jvmExecutionReports); + + } + + @QuickPerfTest + static class NestedTestClassHavingAParameterizedTestInASpecificJvm { + + @Nested + class NestedTestClass { + + @HeapSize(value = 50, unit = AllocationUnit.MEGA_BYTE) + @ParameterizedTest(name = "nested value {0}") + @ValueSource(ints = {1, 2}) + public void parameterized_test(int value) { + throw reportExecutionOf("nested value " + value); + } + + } + + } + + @Test public void + should_execute_each_invocation_of_a_parameterized_test_of_a_nested_test_class_in_its_own_jvm() { + + // GIVEN + Class testClass = NestedTestClassHavingAParameterizedTestInASpecificJvm.class; + JUnit5Tests jUnit5Tests = JUnit5Tests.createInstance(testClass); + + // WHEN + JUnit5TestsResult jUnit5TestsResult = jUnit5Tests.run(); + + // THEN + String errorReport = jUnit5TestsResult.getErrorReport(); + + assertThat(jUnit5TestsResult.getNumberOfFailures()) + .as("Each invocation should fail with the report of its execution.%s", errorReport) + .isEqualTo(2); + + List jvmExecutionReports = jUnit5TestsResult.getJvmExecutionReports(); + assertThat(jvmExecutionReports).extracting("executedTest") + .containsExactlyInAnyOrder("nested value 1", "nested value 2"); + assertThatEachTestWasExecutedInItsOwnNewJvm(jvmExecutionReports); + assertThatEachTestWasTheTestExecutedInItsNewJvm(jvmExecutionReports); + + } + +} diff --git a/junit5/junit5-test-util/src/test/java/org/quickperf/junit5/JUnit5Tests.java b/junit5/junit5-test-util/src/test/java/org/quickperf/junit5/JUnit5Tests.java index 1b07e9d7..577fab9d 100644 --- a/junit5/junit5-test-util/src/test/java/org/quickperf/junit5/JUnit5Tests.java +++ b/junit5/junit5-test-util/src/test/java/org/quickperf/junit5/JUnit5Tests.java @@ -12,8 +12,11 @@ */ package org.quickperf.junit5; +import org.junit.platform.engine.TestExecutionResult; import org.junit.platform.launcher.Launcher; import org.junit.platform.launcher.LauncherDiscoveryRequest; +import org.junit.platform.launcher.TestExecutionListener; +import org.junit.platform.launcher.TestIdentifier; import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder; import org.junit.platform.launcher.core.LauncherFactory; import org.junit.platform.launcher.listeners.SummaryGeneratingListener; @@ -21,8 +24,11 @@ import java.io.PrintWriter; import java.io.StringWriter; +import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import static org.junit.platform.engine.TestExecutionResult.Status.ABORTED; import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass; public class JUnit5Tests { @@ -33,37 +39,59 @@ public class JUnit5Tests { private final SummaryGeneratingListener listener; + private final TestAbortListener testAbortListener; + private JUnit5Tests(Launcher launcher , LauncherDiscoveryRequest request - , SummaryGeneratingListener listener) { + , SummaryGeneratingListener listener + , TestAbortListener testAbortListener) { this.launcher = launcher; this.request = request; this.listener = listener; + this.testAbortListener = testAbortListener; } public static JUnit5Tests createInstance(Class testClass) { SummaryGeneratingListener listener = new SummaryGeneratingListener(); + TestAbortListener testAbortListener = new TestAbortListener(); LauncherDiscoveryRequest request = LauncherDiscoveryRequestBuilder.request() .selectors(selectClass(testClass)) .build(); Launcher launcher = LauncherFactory.create(); - launcher.registerTestExecutionListeners(listener); - return new JUnit5Tests(launcher, request, listener); + launcher.registerTestExecutionListeners(listener, testAbortListener); + return new JUnit5Tests(launcher, request, listener, testAbortListener); } public JUnit5TestsResult run() { launcher.execute(request); TestExecutionSummary testExecutionSummary = listener.getSummary(); - return new JUnit5TestsResult(testExecutionSummary); + return new JUnit5TestsResult(testExecutionSummary, testAbortListener.testAborts); + } + + private static class TestAbortListener implements TestExecutionListener { + + private final List testAborts = new CopyOnWriteArrayList<>(); + + @Override + public void executionFinished(TestIdentifier testIdentifier, TestExecutionResult testExecutionResult) { + if (testIdentifier.isTest() && testExecutionResult.getStatus() == ABORTED + && testExecutionResult.getThrowable().isPresent()) { + testAborts.add(testExecutionResult.getThrowable().get()); + } + } + } public static class JUnit5TestsResult { private final TestExecutionSummary testExecutionSummary; - public JUnit5TestsResult(TestExecutionSummary testExecutionSummary) { + private final List testAborts; + + public JUnit5TestsResult(TestExecutionSummary testExecutionSummary, List testAborts) { this.testExecutionSummary = testExecutionSummary; + this.testAborts = testAborts; } public int getNumberOfFailures() { @@ -71,6 +99,26 @@ public int getNumberOfFailures() { return failures.size(); } + public long getNumberOfAbortedTests() { + return testExecutionSummary.getTestsAbortedCount(); + } + + public long getNumberOfSucceededTests() { + return testExecutionSummary.getTestsSucceededCount(); + } + + public List getJvmExecutionReports() { + List jvmExecutionReports = new ArrayList<>(); + for (TestExecutionSummary.Failure failure : testExecutionSummary.getFailures()) { + JvmExecutionReport jvmExecutionReport = + JvmExecutionReport.buildFrom(failure.getTestIdentifier(), failure.getException()); + if (jvmExecutionReport != null) { + jvmExecutionReports.add(jvmExecutionReport); + } + } + return jvmExecutionReports; + } + public String getErrorReport() { StringWriter stringWriter = new StringWriter(); PrintWriter printWriter = new PrintWriter(stringWriter); @@ -78,6 +126,15 @@ public String getErrorReport() { return stringWriter.getBuffer().toString(); } + public String getAbortReport() { + StringWriter stringWriter = new StringWriter(); + PrintWriter printWriter = new PrintWriter(stringWriter); + for (Throwable testAbort : testAborts) { + testAbort.printStackTrace(printWriter); + } + return stringWriter.getBuffer().toString(); + } + } } diff --git a/junit5/junit5-test-util/src/test/java/org/quickperf/junit5/JvmExecutionReport.java b/junit5/junit5-test-util/src/test/java/org/quickperf/junit5/JvmExecutionReport.java new file mode 100644 index 00000000..61006cc9 --- /dev/null +++ b/junit5/junit5-test-util/src/test/java/org/quickperf/junit5/JvmExecutionReport.java @@ -0,0 +1,116 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * Copyright 2019-2022 the original author or authors. + */ +package org.quickperf.junit5; + +import org.junit.platform.launcher.TestIdentifier; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * A test method executed with {@link JUnit5Tests} can fail with {@link #reportExecutionOf(String)} to report in which + * JVM it was executed, and what it executed. The test executing it gets the reports with + * {@link JUnit5Tests.JUnit5TestsResult#getJvmExecutionReports()}. + *

+ * QuickPerf reports the first failure of a test executed in a specific JVM: a report does not tell whether this JVM + * executed other tests after it. + */ +public class JvmExecutionReport { + + // Unlike a process id, which the operating system can give to a JVM started after the end of another one + private static final String THIS_JVM_ID = UUID.randomUUID().toString(); + + private static final Pattern MESSAGE_PATTERN = Pattern.compile("Executed in the JVM (\\S+): (.*)", Pattern.DOTALL); + + private final String testUniqueId; + + private final String testDisplayName; + + private final String jvmId; + + private final String executedTest; + + private JvmExecutionReport(String testUniqueId, String testDisplayName, String jvmId, String executedTest) { + this.testUniqueId = testUniqueId; + this.testDisplayName = testDisplayName; + this.jvmId = jvmId; + this.executedTest = executedTest; + } + + /** + * @param executedTest what the test method executed, for example its arguments + */ + public static AssertionError reportExecutionOf(String executedTest) { + return new AssertionError("Executed in the JVM " + THIS_JVM_ID + ": " + executedTest); + } + + static JvmExecutionReport buildFrom(TestIdentifier testIdentifier, Throwable testFailure) { + String message = testFailure.getMessage(); + Matcher matcher = MESSAGE_PATTERN.matcher(message == null ? "" : message); + if (!matcher.matches()) { + return null; + } + return new JvmExecutionReport(testIdentifier.getUniqueId() + , testIdentifier.getDisplayName() + , matcher.group(1) + , matcher.group(2)); + } + + public static void assertThatEachTestWasExecutedInItsOwnNewJvm(List reports) { + List jvmIds = new ArrayList<>(); + for (JvmExecutionReport report : reports) { + jvmIds.add(report.jvmId); + } + assertThat(jvmIds).as("JVMs executing the tests") + .doesNotHaveDuplicates() + .doesNotContain(THIS_JVM_ID); + } + + // The test JVM reports the execution of a new JVM for the test that this new JVM executed + public static void assertThatEachTestWasTheTestExecutedInItsNewJvm(List reports) { + for (JvmExecutionReport report : reports) { + assertThat(report.executedTest) + .as("Test executed in the new JVM of %s", report.testUniqueId) + .isEqualTo(report.testDisplayName); + } + } + + /** + * The JUnit unique id, in the test JVM, of the test that failed with this report + */ + public String getTestUniqueId() { + return testUniqueId; + } + + /** + * The JUnit display name, in the test JVM, of the test that failed with this report + */ + public String getTestDisplayName() { + return testDisplayName; + } + + public String getExecutedTest() { + return executedTest; + } + + @Override + public String toString() { + return testUniqueId + " -> executed in the JVM " + jvmId + ": " + executedTest; + } + +} diff --git a/junit5/pom.xml b/junit5/pom.xml index f16286a8..375e0a0a 100644 --- a/junit5/pom.xml +++ b/junit5/pom.xml @@ -35,6 +35,7 @@ junit5-test-util junit5-jvm-test junit5-sql-test + junit5-14-4-jvm-test \ No newline at end of file