From cfe6d467081a631b65fe9ede5850e2177004bdd4 Mon Sep 17 00:00:00 2001 From: Terje Sandstrom Date: Tue, 30 Jun 2026 22:50:55 +0200 Subject: [PATCH 1/6] Document RepeatAttribute threshold features for NUnit 5 (issue #5220) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add RequiredPassPercentage (int, 1-100) and StopWhenOverallResultDetermined (bool) to the repeat attribute docs, with version notes, updated properties table, two new examples, and expanded Notes section. The new snippet regions are guarded with #if NUNIT_REPEAT_THRESHOLD until the matching NUnit package version is published — remove the guard and add the define constant to the csproj at that point. Co-Authored-By: Claude Sonnet 4.6 --- .../nunit/writing-tests/attributes/repeat.md | 25 ++++++++++++- .../Attributes/RepeatAttributeExample.cs | 37 +++++++++++++++++-- 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/docs/articles/nunit/writing-tests/attributes/repeat.md b/docs/articles/nunit/writing-tests/attributes/repeat.md index 4a7343b95..1e6e890f8 100644 --- a/docs/articles/nunit/writing-tests/attributes/repeat.md +++ b/docs/articles/nunit/writing-tests/attributes/repeat.md @@ -23,10 +23,15 @@ RepeatAttribute(int count, bool stopOnFailure) | Property | Type | Description | Default | |----------|------|-------------|---------| | `StopOnFailure` | `bool` | Whether to stop repeating when a test fails. When `false`, all repetitions run regardless of failures. | `true` | +| `RequiredPassPercentage` | `int` | The minimum percentage of runs (1–100) that must pass for the test to succeed. When set below 100, `StopOnFailure` is ignored and all runs always execute. | `100` | +| `StopWhenOverallResultDetermined` | `bool` | When `true` and `RequiredPassPercentage` is below 100, stops iterating as soon as the final outcome is determined — either because the pass threshold is already guaranteed or can no longer be reached. | `false` | > [!NOTE] > The `StopOnFailure` property was added in **NUnit 4.3.0**. +> [!NOTE] +> The `RequiredPassPercentage` and `StopWhenOverallResultDetermined` properties were added in **NUnit 5.0**. + ## Applies To | Test Methods | Test Fixtures (Classes) | Assembly | @@ -43,12 +48,28 @@ RepeatAttribute(int count, bool stopOnFailure) [!code-csharp[RepeatWithFaultAttributeExample](~/snippets/Snippets.NUnit/Attributes/RepeatAttributeExample.cs#RepeatWithFaultAttributeExample)] +### Pass Threshold + +When testing non-deterministic systems (such as LLM-based tests or flaky integrations), you can allow a percentage of runs to fail. The test passes as long as at least `RequiredPassPercentage` percent of the runs succeed. + +[!code-csharp[RepeatWithPassThresholdExample](~/snippets/Snippets.NUnit/Attributes/RepeatAttributeExample.cs#RepeatWithPassThresholdExample)] + +### Early Stop When Outcome Is Determined + +When `StopWhenOverallResultDetermined` is `true`, NUnit stops iterating as soon as it knows whether the threshold will be met or missed, saving unnecessary test runs. + +[!code-csharp[RepeatWithStopWhenDeterminedExample](~/snippets/Snippets.NUnit/Attributes/RepeatAttributeExample.cs#RepeatWithStopWhenDeterminedExample)] + ## Notes 1. When `StopOnFailure` is `true` (default), the test stops at the first failure. 2. When `StopOnFailure` is `false`, all repetitions run and all failures are collected. -3. If `RepeatAttribute` is used on a parameterized method, each individual test case is repeated. -4. It is not currently possible to use `RepeatAttribute` on a `TestFixture` or any higher level suite. Only test methods may be repeated. +3. When `RequiredPassPercentage` is set below 100, `StopOnFailure` is automatically ignored — all runs always execute so the full pass count can be evaluated. +4. When `RequiredPassPercentage` is below 100 and `StopWhenOverallResultDetermined` is `true`, NUnit exits the loop as soon as the result is certain: + - **Early success**: the number of passes already guarantees the threshold regardless of the remaining runs. + - **Early failure**: even if all remaining runs pass, the threshold can no longer be reached. +5. If `RepeatAttribute` is used on a parameterized method, each individual test case is repeated independently. +6. It is not currently possible to use `RepeatAttribute` on a `TestFixture` or any higher level suite. Only test methods may be repeated. ## See Also diff --git a/docs/snippets/Snippets.NUnit/Attributes/RepeatAttributeExample.cs b/docs/snippets/Snippets.NUnit/Attributes/RepeatAttributeExample.cs index 9ca0532b8..46c5f1b7b 100644 --- a/docs/snippets/Snippets.NUnit/Attributes/RepeatAttributeExample.cs +++ b/docs/snippets/Snippets.NUnit/Attributes/RepeatAttributeExample.cs @@ -1,4 +1,4 @@ -using NUnit.Framework; +using NUnit.Framework; namespace Snippets.NUnit.Attributes; @@ -29,7 +29,7 @@ public void TestMethod2() #region RepeatWithFaultAttributeExample private int count2; - + [Test,Explicit] // Marking the test as Explicit to avoid failing our doc build. You can skip this. [Repeat(5, StopOnFailure = false)] public void TestMethod3() @@ -38,4 +38,35 @@ public void TestMethod3() Assert.That(count2, Is.Not.EqualTo(3)); // Intentional failure on 3rd iteration } #endregion -} \ No newline at end of file + + // TODO: Remove the #if guard and update the package reference to the NUnit version that + // ships RequiredPassPercentage and StopWhenOverallResultDetermined. +#if NUNIT_REPEAT_THRESHOLD + #region RepeatWithPassThresholdExample + // The test passes if at least 80% of the 10 runs succeed. + // Useful for non-deterministic systems where some variation is acceptable. + [Test] + [Repeat(10, RequiredPassPercentage = 80)] + public void NonDeterministicTest() + { + bool result = CallFlakyService(); + Assert.That(result, Is.True); + } + + private bool CallFlakyService() => true; // placeholder + #endregion + + #region RepeatWithStopWhenDeterminedExample + // Stops as soon as NUnit can guarantee pass or failure: + // - Early success: enough passes accumulated (threshold already guaranteed). + // - Early failure: too many failures (threshold no longer achievable). + [Test] + [Repeat(10, RequiredPassPercentage = 80, StopWhenOverallResultDetermined = true)] + public void NonDeterministicTestWithEarlyStop() + { + bool result = CallFlakyService(); + Assert.That(result, Is.True); + } + #endregion +#endif +} From 4662c193556a15fb60967b6e81cb3250c5a520ae Mon Sep 17 00:00:00 2001 From: Terje Sandstrom Date: Fri, 3 Jul 2026 23:05:27 +0200 Subject: [PATCH 2/6] Updated nunit and adapter --- docs/articles/nunit/writing-tests/attributes/repeat.md | 2 +- docs/snippets/Snippets.NUnit/Snippets.NUnit.csproj | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/articles/nunit/writing-tests/attributes/repeat.md b/docs/articles/nunit/writing-tests/attributes/repeat.md index 1e6e890f8..fb400bcc5 100644 --- a/docs/articles/nunit/writing-tests/attributes/repeat.md +++ b/docs/articles/nunit/writing-tests/attributes/repeat.md @@ -64,7 +64,7 @@ When `StopWhenOverallResultDetermined` is `true`, NUnit stops iterating as soon 1. When `StopOnFailure` is `true` (default), the test stops at the first failure. 2. When `StopOnFailure` is `false`, all repetitions run and all failures are collected. -3. When `RequiredPassPercentage` is set below 100, `StopOnFailure` is automatically ignored — all runs always execute so the full pass count can be evaluated. +3. When `RequiredPassPercentage` is set below 100, `StopOnFailure` must not be explicitly set to `true` — combining the two is a configuration error that produces `ResultState.Error` without running the test. Setting `StopOnFailure=false` explicitly alongside a threshold is valid. When `StopOnFailure` is left at its default (not explicitly set), it is silently treated as `false` so all runs execute. 4. When `RequiredPassPercentage` is below 100 and `StopWhenOverallResultDetermined` is `true`, NUnit exits the loop as soon as the result is certain: - **Early success**: the number of passes already guarantees the threshold regardless of the remaining runs. - **Early failure**: even if all remaining runs pass, the threshold can no longer be reached. diff --git a/docs/snippets/Snippets.NUnit/Snippets.NUnit.csproj b/docs/snippets/Snippets.NUnit/Snippets.NUnit.csproj index 1630375f0..950455782 100644 --- a/docs/snippets/Snippets.NUnit/Snippets.NUnit.csproj +++ b/docs/snippets/Snippets.NUnit/Snippets.NUnit.csproj @@ -10,8 +10,8 @@ - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive From 64428ecbe27a4fa4ca6ce38d04254a82bc5051c4 Mon Sep 17 00:00:00 2001 From: Terje Sandstrom Date: Fri, 3 Jul 2026 23:10:45 +0200 Subject: [PATCH 3/6] fixing markdown in repeat.md --- docs/articles/nunit/writing-tests/attributes/repeat.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/articles/nunit/writing-tests/attributes/repeat.md b/docs/articles/nunit/writing-tests/attributes/repeat.md index fb400bcc5..431a3f29d 100644 --- a/docs/articles/nunit/writing-tests/attributes/repeat.md +++ b/docs/articles/nunit/writing-tests/attributes/repeat.md @@ -26,9 +26,10 @@ RepeatAttribute(int count, bool stopOnFailure) | `RequiredPassPercentage` | `int` | The minimum percentage of runs (1–100) that must pass for the test to succeed. When set below 100, `StopOnFailure` is ignored and all runs always execute. | `100` | | `StopWhenOverallResultDetermined` | `bool` | When `true` and `RequiredPassPercentage` is below 100, stops iterating as soon as the final outcome is determined — either because the pass threshold is already guaranteed or can no longer be reached. | `false` | +----- > [!NOTE] > The `StopOnFailure` property was added in **NUnit 4.3.0**. - +----- > [!NOTE] > The `RequiredPassPercentage` and `StopWhenOverallResultDetermined` properties were added in **NUnit 5.0**. @@ -73,4 +74,4 @@ When `StopWhenOverallResultDetermined` is `true`, NUnit stops iterating as soon ## See Also -* [Retry Attribute](xref:attribute-retry) +- [Retry Attribute](xref:attribute-retry) From 379a47c1b81a6f1ca02913554bff6a43c24e9c2f Mon Sep 17 00:00:00 2001 From: Terje Sandstrom Date: Fri, 3 Jul 2026 23:25:58 +0200 Subject: [PATCH 4/6] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../Snippets.NUnit/Attributes/RepeatAttributeExample.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/snippets/Snippets.NUnit/Attributes/RepeatAttributeExample.cs b/docs/snippets/Snippets.NUnit/Attributes/RepeatAttributeExample.cs index 46c5f1b7b..e72cbc86f 100644 --- a/docs/snippets/Snippets.NUnit/Attributes/RepeatAttributeExample.cs +++ b/docs/snippets/Snippets.NUnit/Attributes/RepeatAttributeExample.cs @@ -39,8 +39,6 @@ public void TestMethod3() } #endregion - // TODO: Remove the #if guard and update the package reference to the NUnit version that - // ships RequiredPassPercentage and StopWhenOverallResultDetermined. #if NUNIT_REPEAT_THRESHOLD #region RepeatWithPassThresholdExample // The test passes if at least 80% of the 10 runs succeed. From 9311a9348e1c5ea0394e9162a896053aea30c5db Mon Sep 17 00:00:00 2001 From: Terje Sandstrom Date: Fri, 3 Jul 2026 23:26:54 +0200 Subject: [PATCH 5/6] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- docs/articles/nunit/writing-tests/attributes/repeat.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/articles/nunit/writing-tests/attributes/repeat.md b/docs/articles/nunit/writing-tests/attributes/repeat.md index 431a3f29d..117f9f387 100644 --- a/docs/articles/nunit/writing-tests/attributes/repeat.md +++ b/docs/articles/nunit/writing-tests/attributes/repeat.md @@ -23,7 +23,7 @@ RepeatAttribute(int count, bool stopOnFailure) | Property | Type | Description | Default | |----------|------|-------------|---------| | `StopOnFailure` | `bool` | Whether to stop repeating when a test fails. When `false`, all repetitions run regardless of failures. | `true` | -| `RequiredPassPercentage` | `int` | The minimum percentage of runs (1–100) that must pass for the test to succeed. When set below 100, `StopOnFailure` is ignored and all runs always execute. | `100` | +| `RequiredPassPercentage` | `int` | The minimum percentage of runs (1–100) that must pass for the test to succeed. When set below 100, `StopOnFailure` is treated as `false` (and must not be explicitly set to `true`); iterations run up to the repeat count unless `StopWhenOverallResultDetermined` is enabled. | `100` | | `StopWhenOverallResultDetermined` | `bool` | When `true` and `RequiredPassPercentage` is below 100, stops iterating as soon as the final outcome is determined — either because the pass threshold is already guaranteed or can no longer be reached. | `false` | ----- From 9011fa1b089b83ab9f84a55faa90b7762425b77f Mon Sep 17 00:00:00 2001 From: Terje Sandstrom Date: Fri, 3 Jul 2026 23:28:16 +0200 Subject: [PATCH 6/6] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- docs/articles/nunit/writing-tests/attributes/repeat.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/articles/nunit/writing-tests/attributes/repeat.md b/docs/articles/nunit/writing-tests/attributes/repeat.md index 117f9f387..660e9d74a 100644 --- a/docs/articles/nunit/writing-tests/attributes/repeat.md +++ b/docs/articles/nunit/writing-tests/attributes/repeat.md @@ -65,7 +65,7 @@ When `StopWhenOverallResultDetermined` is `true`, NUnit stops iterating as soon 1. When `StopOnFailure` is `true` (default), the test stops at the first failure. 2. When `StopOnFailure` is `false`, all repetitions run and all failures are collected. -3. When `RequiredPassPercentage` is set below 100, `StopOnFailure` must not be explicitly set to `true` — combining the two is a configuration error that produces `ResultState.Error` without running the test. Setting `StopOnFailure=false` explicitly alongside a threshold is valid. When `StopOnFailure` is left at its default (not explicitly set), it is silently treated as `false` so all runs execute. +3. When `RequiredPassPercentage` is set below 100, `StopOnFailure` must not be explicitly set to `true` — combining the two is a configuration error that produces `ResultState.Error` without running the test. Setting `StopOnFailure=false` explicitly alongside a threshold is valid. When `StopOnFailure` is left at its default (not explicitly set), it is silently treated as `false` so repetitions continue after failures (unless `StopWhenOverallResultDetermined` is enabled). 4. When `RequiredPassPercentage` is below 100 and `StopWhenOverallResultDetermined` is `true`, NUnit exits the loop as soon as the result is certain: - **Early success**: the number of passes already guarantees the threshold regardless of the remaining runs. - **Early failure**: even if all remaining runs pass, the threshold can no longer be reached.