From edeb7750e1258d228bae46dbc47baa4aa5d3e842 Mon Sep 17 00:00:00 2001 From: Alex Shamrai Date: Mon, 26 Oct 2026 18:12:21 +0200 Subject: [PATCH] Add optional threadId to attempt history entries Attempt history entries can record the thread or worker that executed them, using the same contract as the Test threadId. The Test threadId now explicitly describes the final attempt, and producers that run test cases in more than one process or on more than one host are asked to qualify threadId values. No omission rule and no serial-execution guarantee are added. --- CHANGELOG.md | 12 ++ examples/comprehensive.json | 1 + schema/ctrf.schema.json | 6 +- spec/ctrf.md | 25 ++- tests/normative/type-validation.test.json | 190 ++++++++++++++++++++++ tests/normative/valid-documents.test.json | 46 ++++++ 6 files changed, 275 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0fb2d1..1711c39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Optional `threadId` on attempt history entries, using the same contract as Test + `threadId`, so previous attempts can record the thread or worker that executed + them (#74). + +### Changed + +- Clarified that Test `threadId` identifies the thread or worker that executed the + final attempt, and recommended qualifying `threadId` values when test cases run + across processes or hosts (#74). + ## [0.2.0] - 2026-10-10 ### Added diff --git a/examples/comprehensive.json b/examples/comprehensive.json index aec49ce..c009e9a 100644 --- a/examples/comprehensive.json +++ b/examples/comprehensive.json @@ -104,6 +104,7 @@ "duration": 5100, "start": 1609459203500, "stop": 1609459208600, + "threadId": "worker-1", "message": "Payment gateway timeout", "trace": "Error: Payment gateway timeout\n at processPayment (payment.js:23:10)", "attachments": [ diff --git a/schema/ctrf.schema.json b/schema/ctrf.schema.json index 0135d5e..8228832 100644 --- a/schema/ctrf.schema.json +++ b/schema/ctrf.schema.json @@ -337,6 +337,10 @@ "description": "Attempt end time (milliseconds since Unix epoch)", "type": "integer" }, + "threadId": { + "description": "Thread or worker that executed this attempt", + "type": "string" + }, "attachments": { "description": "Artifacts from this attempt", "type": "array", @@ -397,7 +401,7 @@ } }, "threadId": { - "description": "Thread or worker identifier", + "description": "Thread or worker that executed the final attempt represented by the test object", "type": "string" }, "browser": { diff --git a/spec/ctrf.md b/spec/ctrf.md index 25e1399..8c4772f 100644 --- a/spec/ctrf.md +++ b/spec/ctrf.md @@ -1147,11 +1147,12 @@ If present, it MUST be an array of strings. ### 9.25. `threadId` **Description:** -Identifies the thread or worker on which the test executed. +Identifies the thread or worker that executed the final attempt represented by this test object. Earlier attempts can record their own `threadId` in `retryAttempts` (Section 11). **Requirements:** `threadId` is OPTIONAL. -If present, it MUST be a string. +If present, it MUST be a string. +When test cases run in more than one process or on more than one host, producers SHOULD qualify the value with the process or host to avoid collisions, for example, `"host-a/worker-1"`. ### 9.26. `browser` @@ -1549,6 +1550,16 @@ The timestamp indicating when the attempt ended, in milliseconds since the Unix If present, it MUST be an integer. If both `start` and `stop` are present, `stop` SHOULD be greater than or equal to `start`. +### 11.12.1. `threadId` + +**Description:** +Identifies the thread or worker that executed this attempt. + +**Requirements:** +`threadId` is OPTIONAL. +If present, it MUST be a string. +When test cases run in more than one process or on more than one host, producers SHOULD qualify the value with the process or host to avoid collisions, for example, `"host-a/worker-1"`. + ### 11.13. `attachments` **Description:** @@ -2427,6 +2438,10 @@ to this specification. "description": "Attempt end time (milliseconds since Unix epoch)", "type": "integer" }, + "threadId": { + "description": "Thread or worker that executed this attempt", + "type": "string" + }, "attachments": { "description": "Artifacts from this attempt", "type": "array", @@ -2487,7 +2502,7 @@ to this specification. } }, "threadId": { - "description": "Thread or worker identifier", + "description": "Thread or worker that executed the final attempt represented by the test object", "type": "string" }, "browser": { @@ -3337,10 +3352,12 @@ It includes: "duration": 1500, "message": "Timeout waiting for response", "stdout": ["Submitting login form"], - "stderr": ["Request timed out"] + "stderr": ["Request timed out"], + "threadId": "worker-1" } ], "stdout": ["Test completed successfully"], + "threadId": "worker-2", "insights": { "passRate": { "current": 0.9, diff --git a/tests/normative/type-validation.test.json b/tests/normative/type-validation.test.json index 0419fad..c645e6e 100644 --- a/tests/normative/type-validation.test.json +++ b/tests/normative/type-validation.test.json @@ -1234,6 +1234,196 @@ ] } } + }, + { + "description": "RetryAttempt with threadId as number instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100, + "retries": 1, + "retryAttempts": [ + { + "attempt": 1, + "status": "failed", + "threadId": 12345 + } + ] + } + ] + } + } + }, + { + "description": "RetryAttempt with threadId as null instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100, + "retries": 1, + "retryAttempts": [ + { + "attempt": 1, + "status": "failed", + "threadId": null + } + ] + } + ] + } + } + }, + { + "description": "RetryAttempt with threadId as boolean instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100, + "retries": 1, + "retryAttempts": [ + { + "attempt": 1, + "status": "failed", + "threadId": true + } + ] + } + ] + } + } + }, + { + "description": "RetryAttempt with threadId as array instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100, + "retries": 1, + "retryAttempts": [ + { + "attempt": 1, + "status": "failed", + "threadId": [] + } + ] + } + ] + } + } + }, + { + "description": "RetryAttempt with threadId as object instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100, + "retries": 1, + "retryAttempts": [ + { + "attempt": 1, + "status": "failed", + "threadId": {} + } + ] + } + ] + } + } } ] } diff --git a/tests/normative/valid-documents.test.json b/tests/normative/valid-documents.test.json index 3d26cdf..5edd46e 100644 --- a/tests/normative/valid-documents.test.json +++ b/tests/normative/valid-documents.test.json @@ -43,6 +43,52 @@ ] } } + }, + { + "description": "Previous attempt threadId differs from the final attempt threadId", + "valid": true, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "flaky": 1, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "checkout accepts a saved card", + "status": "passed", + "duration": 500, + "start": 1609459200500, + "stop": 1609459201000, + "threadId": "fork-2/Test worker", + "flaky": true, + "retries": 1, + "retryAttempts": [ + { + "attempt": 1, + "status": "failed", + "duration": 400, + "start": 1609459200000, + "stop": 1609459200400, + "threadId": "fork-1/Test worker" + } + ] + } + ] + } + } } ] }