From 1ea3dac76e6e61b665400a509508614a38477f0d Mon Sep 17 00:00:00 2001
From: Mike Kistler <mikekistler@microsoft.com>
Date: Mon, 26 Jan 2026 10:23:49 -0600
Subject: [PATCH 1/2] Add allowClientError to Scenario interface for expected
 client failures

---
 src/index.ts                                   |  3 ++-
 src/runner/client.ts                           | 14 ++++++++++----
 src/scenarios/client/auth/resource-mismatch.ts |  1 +
 src/scenarios/client/auth/scope-handling.ts    |  1 +
 src/types.ts                                   |  5 +++++
 5 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/src/index.ts b/src/index.ts
index 46ed8a5..039b042 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -207,7 +207,8 @@ program
       const { overallFailure } = printClientResults(
         result.checks,
         verbose,
-        result.clientOutput
+        result.clientOutput,
+        result.allowClientError
       );
 
       if (options.expectedFailures) {
diff --git a/src/runner/client.ts b/src/runner/client.ts
index 27a8b6e..8eae558 100644
--- a/src/runner/client.ts
+++ b/src/runner/client.ts
@@ -97,6 +97,7 @@ export async function runConformanceTest(
   checks: ConformanceCheck[];
   clientOutput: ClientExecutionResult;
   resultDir?: string;
+  allowClientError?: boolean;
 }> {
   let resultDir: string | undefined;
 
@@ -164,7 +165,8 @@ export async function runConformanceTest(
     return {
       checks,
       clientOutput,
-      resultDir
+      resultDir,
+      allowClientError: scenario.allowClientError
     };
   } finally {
     await scenario.stop();
@@ -174,7 +176,8 @@ export async function runConformanceTest(
 export function printClientResults(
   checks: ConformanceCheck[],
   verbose: boolean = false,
-  clientOutput?: ClientExecutionResult
+  clientOutput?: ClientExecutionResult,
+  allowClientError: boolean = false
 ): {
   passed: number;
   failed: number;
@@ -195,7 +198,10 @@ export function printClientResults(
     ? clientOutput.exitCode !== 0
     : false;
   const overallFailure =
-    failed > 0 || warnings > 0 || clientTimedOut || clientExitedWithError;
+    failed > 0 ||
+    warnings > 0 ||
+    clientTimedOut ||
+    (clientOutput?.exitCode !== 0 && !allowClientError);
 
   if (verbose) {
     // Verbose mode: JSON goes to stdout for piping to jq/jless
@@ -215,7 +221,7 @@ export function printClientResults(
     console.error(`\n⚠️  CLIENT TIMED OUT - Test incomplete`);
   }
 
-  if (clientExitedWithError && !clientTimedOut) {
+  if (clientExitedWithError && !clientTimedOut && !allowClientError) {
     console.error(
       `\n⚠️  CLIENT EXITED WITH ERROR (code ${clientOutput?.exitCode}) - Test may be incomplete`
     );
diff --git a/src/scenarios/client/auth/resource-mismatch.ts b/src/scenarios/client/auth/resource-mismatch.ts
index 4783bc8..b38968f 100644
--- a/src/scenarios/client/auth/resource-mismatch.ts
+++ b/src/scenarios/client/auth/resource-mismatch.ts
@@ -29,6 +29,7 @@ export class ResourceMismatchScenario implements Scenario {
   name = 'auth/resource-mismatch';
   description =
     'Tests that client rejects when PRM resource does not match server URL';
+  allowClientError = true;
 
   private authServer = new ServerLifecycle();
   private server = new ServerLifecycle();
diff --git a/src/scenarios/client/auth/scope-handling.ts b/src/scenarios/client/auth/scope-handling.ts
index d94760b..b9865bd 100644
--- a/src/scenarios/client/auth/scope-handling.ts
+++ b/src/scenarios/client/auth/scope-handling.ts
@@ -479,6 +479,7 @@ export class ScopeRetryLimitScenario implements Scenario {
   name = 'auth/scope-retry-limit';
   description =
     'Tests that client implements retry limits to prevent infinite authorization loops on repeated 403 responses';
+  allowClientError = true;
   private authServer = new ServerLifecycle();
   private server = new ServerLifecycle();
   private checks: ConformanceCheck[] = [];
diff --git a/src/types.ts b/src/types.ts
index d5192b7..5dd6421 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -36,6 +36,11 @@ export interface ScenarioUrls {
 export interface Scenario {
   name: string;
   description: string;
+  /**
+   * If true, a non-zero client exit code is expected and will not cause the test to fail.
+   * Use this for scenarios where the client is expected to error (e.g., rejecting invalid auth).
+   */
+  allowClientError?: boolean;
   start(): Promise<ScenarioUrls>;
   stop(): Promise<void>;
   getChecks(): ConformanceCheck[];

From f27cd37c3f1fbccdfc0f6b00f7623be30ca8c8e4 Mon Sep 17 00:00:00 2001
From: Mike Kistler <mikekistler@microsoft.com>
Date: Mon, 26 Jan 2026 11:35:25 -0600
Subject: [PATCH 2/2] Apply suggestion from @pcarleton

Co-authored-by: Paul Carleton <paulcarletonjr@gmail.com>
---
 src/runner/client.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/runner/client.ts b/src/runner/client.ts
index 8eae558..4525416 100644
--- a/src/runner/client.ts
+++ b/src/runner/client.ts
@@ -201,7 +201,7 @@ export function printClientResults(
     failed > 0 ||
     warnings > 0 ||
     clientTimedOut ||
-    (clientOutput?.exitCode !== 0 && !allowClientError);
+    (clientExitedWithError && !allowClientError);
 
   if (verbose) {
     // Verbose mode: JSON goes to stdout for piping to jq/jless