From 9e1cd31b584d6e21ed93def33cabad033cb0b808 Mon Sep 17 00:00:00 2001 From: Gautam Bharti <3028386+gautamgb@users.noreply.github.com> Date: Mon, 14 Sep 2026 19:34:55 -0700 Subject: [PATCH] datasets: pin down the agentdataflow break-test fixtures in tests/ #62's review ran sixteen deliberately-broken copies by hand to confirm validate.py's checks actually fire, one error and a non-zero exit each, no false positives on the real files. None of that was committed, so a future change to validate.py could silently stop catching any of them and nothing would say so - the thing CI now runs on every PR would keep passing regardless. tests/fixtures/valid_minimal.json is the smallest trace that satisfies schema.json. Every other fixture in tests/fixtures/ is that same trace with exactly one field changed, named for the one thing it breaks: a span naming itself as its own parent, a dangling parent_span_id, a backwards t_offset_ms, a bearer token or an internal hostname or a credential-shaped key=value assignment or two routable-looking public IPv4 addresses in one string, a non-ISO date_added or span timestamp, a non-hypothetical tier with no evidence, a cve- or url-typed citation in the wrong shape, an evidence item missing supports, attestation: false, a dsgai_mapping id off the schema enum, and a filename that disagrees with its own trace_id. tests/test_validate.py runs validate.py as a subprocess against a fresh copy of schema.json/example.json/validate.py plus one fixture at a time, asserting the exit code and a message substring specific to that fixture - not by importing validate.py's functions, so a refactor of its internals can't quietter this suite while still changing behavior. test_every_fixture_file_is_covered checks the fixture directory and the test's own case list agree, so a fixture added without a matching case (or vice versa) fails loudly instead of testing nothing. Wired into the existing CI workflow as one more step; SHA-pinned actions and path filters are untouched from what was already reviewed. --- .../dataset-agentdataflow-validate.yml | 8 +- .../tests/fixtures/date-added-not-iso.json | 42 ++++++ .../fixtures/dsgai-mapping-off-enum.json | 42 ++++++ .../evidence-citation-bad-cve-form.json | 49 +++++++ .../evidence-citation-bad-url-form.json | 49 +++++++ .../fixtures/evidence-missing-supports.json | 48 +++++++ .../fixtures/filename-trace-id-mismatch.json | 42 ++++++ .../fixtures/provenance-missing-evidence.json | 42 ++++++ .../sanitization-attestation-false.json | 42 ++++++ .../tests/fixtures/secret-bearer-token.json | 42 ++++++ .../secret-credential-assignment.json | 42 ++++++ .../fixtures/secret-internal-hostname.json | 42 ++++++ .../fixtures/secret-public-ip-cluster.json | 42 ++++++ .../tests/fixtures/span-backwards-offset.json | 42 ++++++ .../tests/fixtures/span-dangling-parent.json | 42 ++++++ .../tests/fixtures/span-self-parent.json | 42 ++++++ .../fixtures/span-timestamp-not-iso.json | 43 ++++++ .../tests/fixtures/valid_minimal.json | 36 +++++ .../tests/test_validate.py | 132 ++++++++++++++++++ 19 files changed, 867 insertions(+), 2 deletions(-) create mode 100644 datasets/agentdataflow_toolexchange_traces/tests/fixtures/date-added-not-iso.json create mode 100644 datasets/agentdataflow_toolexchange_traces/tests/fixtures/dsgai-mapping-off-enum.json create mode 100644 datasets/agentdataflow_toolexchange_traces/tests/fixtures/evidence-citation-bad-cve-form.json create mode 100644 datasets/agentdataflow_toolexchange_traces/tests/fixtures/evidence-citation-bad-url-form.json create mode 100644 datasets/agentdataflow_toolexchange_traces/tests/fixtures/evidence-missing-supports.json create mode 100644 datasets/agentdataflow_toolexchange_traces/tests/fixtures/filename-trace-id-mismatch.json create mode 100644 datasets/agentdataflow_toolexchange_traces/tests/fixtures/provenance-missing-evidence.json create mode 100644 datasets/agentdataflow_toolexchange_traces/tests/fixtures/sanitization-attestation-false.json create mode 100644 datasets/agentdataflow_toolexchange_traces/tests/fixtures/secret-bearer-token.json create mode 100644 datasets/agentdataflow_toolexchange_traces/tests/fixtures/secret-credential-assignment.json create mode 100644 datasets/agentdataflow_toolexchange_traces/tests/fixtures/secret-internal-hostname.json create mode 100644 datasets/agentdataflow_toolexchange_traces/tests/fixtures/secret-public-ip-cluster.json create mode 100644 datasets/agentdataflow_toolexchange_traces/tests/fixtures/span-backwards-offset.json create mode 100644 datasets/agentdataflow_toolexchange_traces/tests/fixtures/span-dangling-parent.json create mode 100644 datasets/agentdataflow_toolexchange_traces/tests/fixtures/span-self-parent.json create mode 100644 datasets/agentdataflow_toolexchange_traces/tests/fixtures/span-timestamp-not-iso.json create mode 100644 datasets/agentdataflow_toolexchange_traces/tests/fixtures/valid_minimal.json create mode 100644 datasets/agentdataflow_toolexchange_traces/tests/test_validate.py diff --git a/.github/workflows/dataset-agentdataflow-validate.yml b/.github/workflows/dataset-agentdataflow-validate.yml index 27bfcf9..88cc4a0 100644 --- a/.github/workflows/dataset-agentdataflow-validate.yml +++ b/.github/workflows/dataset-agentdataflow-validate.yml @@ -32,8 +32,8 @@ jobs: with: python-version: '3.11' - - name: Install jsonschema - run: python -m pip install --quiet jsonschema + - name: Install jsonschema and pytest + run: python -m pip install --quiet jsonschema pytest - name: Validate entries and example against schema.json run: python validate.py @@ -44,3 +44,7 @@ jobs: python build_index.py git diff --exit-code index.csv working-directory: datasets/agentdataflow_toolexchange_traces + + - name: Known-bad fixtures still fail the way they're supposed to + run: python -m pytest tests/ -v + working-directory: datasets/agentdataflow_toolexchange_traces diff --git a/datasets/agentdataflow_toolexchange_traces/tests/fixtures/date-added-not-iso.json b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/date-added-not-iso.json new file mode 100644 index 0000000..0ff05c3 --- /dev/null +++ b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/date-added-not-iso.json @@ -0,0 +1,42 @@ +{ + "trace_id": "DSGAI-TRACE-fixture-date-added-not-iso", + "title": "Minimal valid trace, used as the base every broken fixture mutates", + "description": "Not a real security finding. Exists only so each broken fixture in this directory differs from a passing trace by exactly one thing, and so a future contributor changing the schema has something small to update instead of hand-writing a new minimal trace from scratch.", + "category": "tool_call", + "disposition": "benign", + "dsgai_mapping": [ + "DSGAI06" + ], + "provenance": { + "tier": "hypothetical", + "collection_method": "Constructed by hand as a schema-validity fixture; models nothing." + }, + "sanitization": { + "attestation": true, + "techniques": [ + "not_applicable_fully_synthetic" + ] + }, + "spans": [ + { + "span_id": "s1", + "parent_span_id": null, + "t_offset_ms": 0, + "actor": "agent", + "operation": "tool.call", + "summary": "Placeholder span with no finding." + }, + { + "span_id": "s2", + "parent_span_id": "s1", + "t_offset_ms": 10, + "actor": "tool", + "operation": "tool.result", + "summary": "Placeholder span with no finding." + } + ], + "security_observations": [ + "This fixture demonstrates nothing; it exists to validate cleanly." + ], + "date_added": "15-09-2026" +} \ No newline at end of file diff --git a/datasets/agentdataflow_toolexchange_traces/tests/fixtures/dsgai-mapping-off-enum.json b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/dsgai-mapping-off-enum.json new file mode 100644 index 0000000..98f459a --- /dev/null +++ b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/dsgai-mapping-off-enum.json @@ -0,0 +1,42 @@ +{ + "trace_id": "DSGAI-TRACE-fixture-dsgai-mapping-off-enum", + "title": "Minimal valid trace, used as the base every broken fixture mutates", + "description": "Not a real security finding. Exists only so each broken fixture in this directory differs from a passing trace by exactly one thing, and so a future contributor changing the schema has something small to update instead of hand-writing a new minimal trace from scratch.", + "category": "tool_call", + "disposition": "benign", + "dsgai_mapping": [ + "DSGAI99" + ], + "provenance": { + "tier": "hypothetical", + "collection_method": "Constructed by hand as a schema-validity fixture; models nothing." + }, + "sanitization": { + "attestation": true, + "techniques": [ + "not_applicable_fully_synthetic" + ] + }, + "spans": [ + { + "span_id": "s1", + "parent_span_id": null, + "t_offset_ms": 0, + "actor": "agent", + "operation": "tool.call", + "summary": "Placeholder span with no finding." + }, + { + "span_id": "s2", + "parent_span_id": "s1", + "t_offset_ms": 10, + "actor": "tool", + "operation": "tool.result", + "summary": "Placeholder span with no finding." + } + ], + "security_observations": [ + "This fixture demonstrates nothing; it exists to validate cleanly." + ], + "date_added": "2026-09-15" +} \ No newline at end of file diff --git a/datasets/agentdataflow_toolexchange_traces/tests/fixtures/evidence-citation-bad-cve-form.json b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/evidence-citation-bad-cve-form.json new file mode 100644 index 0000000..2030e01 --- /dev/null +++ b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/evidence-citation-bad-cve-form.json @@ -0,0 +1,49 @@ +{ + "trace_id": "DSGAI-TRACE-fixture-evidence-citation-bad-cve-form", + "title": "Minimal valid trace, used as the base every broken fixture mutates", + "description": "Not a real security finding. Exists only so each broken fixture in this directory differs from a passing trace by exactly one thing, and so a future contributor changing the schema has something small to update instead of hand-writing a new minimal trace from scratch.", + "category": "tool_call", + "disposition": "benign", + "dsgai_mapping": [ + "DSGAI06" + ], + "provenance": { + "tier": "hypothetical", + "collection_method": "Constructed by hand as a schema-validity fixture; models nothing.", + "evidence": [ + { + "type": "cve", + "citation": "CVE-25-1234", + "supports": "A malformed CVE id, for the citation-form check." + } + ] + }, + "sanitization": { + "attestation": true, + "techniques": [ + "not_applicable_fully_synthetic" + ] + }, + "spans": [ + { + "span_id": "s1", + "parent_span_id": null, + "t_offset_ms": 0, + "actor": "agent", + "operation": "tool.call", + "summary": "Placeholder span with no finding." + }, + { + "span_id": "s2", + "parent_span_id": "s1", + "t_offset_ms": 10, + "actor": "tool", + "operation": "tool.result", + "summary": "Placeholder span with no finding." + } + ], + "security_observations": [ + "This fixture demonstrates nothing; it exists to validate cleanly." + ], + "date_added": "2026-09-15" +} \ No newline at end of file diff --git a/datasets/agentdataflow_toolexchange_traces/tests/fixtures/evidence-citation-bad-url-form.json b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/evidence-citation-bad-url-form.json new file mode 100644 index 0000000..fd7e574 --- /dev/null +++ b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/evidence-citation-bad-url-form.json @@ -0,0 +1,49 @@ +{ + "trace_id": "DSGAI-TRACE-fixture-evidence-citation-bad-url-form", + "title": "Minimal valid trace, used as the base every broken fixture mutates", + "description": "Not a real security finding. Exists only so each broken fixture in this directory differs from a passing trace by exactly one thing, and so a future contributor changing the schema has something small to update instead of hand-writing a new minimal trace from scratch.", + "category": "tool_call", + "disposition": "benign", + "dsgai_mapping": [ + "DSGAI06" + ], + "provenance": { + "tier": "hypothetical", + "collection_method": "Constructed by hand as a schema-validity fixture; models nothing.", + "evidence": [ + { + "type": "url", + "citation": "example.com/not-a-scheme", + "supports": "A bare hostname typed as a url citation." + } + ] + }, + "sanitization": { + "attestation": true, + "techniques": [ + "not_applicable_fully_synthetic" + ] + }, + "spans": [ + { + "span_id": "s1", + "parent_span_id": null, + "t_offset_ms": 0, + "actor": "agent", + "operation": "tool.call", + "summary": "Placeholder span with no finding." + }, + { + "span_id": "s2", + "parent_span_id": "s1", + "t_offset_ms": 10, + "actor": "tool", + "operation": "tool.result", + "summary": "Placeholder span with no finding." + } + ], + "security_observations": [ + "This fixture demonstrates nothing; it exists to validate cleanly." + ], + "date_added": "2026-09-15" +} \ No newline at end of file diff --git a/datasets/agentdataflow_toolexchange_traces/tests/fixtures/evidence-missing-supports.json b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/evidence-missing-supports.json new file mode 100644 index 0000000..f69e8ed --- /dev/null +++ b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/evidence-missing-supports.json @@ -0,0 +1,48 @@ +{ + "trace_id": "DSGAI-TRACE-fixture-evidence-missing-supports", + "title": "Minimal valid trace, used as the base every broken fixture mutates", + "description": "Not a real security finding. Exists only so each broken fixture in this directory differs from a passing trace by exactly one thing, and so a future contributor changing the schema has something small to update instead of hand-writing a new minimal trace from scratch.", + "category": "tool_call", + "disposition": "benign", + "dsgai_mapping": [ + "DSGAI06" + ], + "provenance": { + "tier": "hypothetical", + "collection_method": "Constructed by hand as a schema-validity fixture; models nothing.", + "evidence": [ + { + "type": "url", + "citation": "https://example.com/spec" + } + ] + }, + "sanitization": { + "attestation": true, + "techniques": [ + "not_applicable_fully_synthetic" + ] + }, + "spans": [ + { + "span_id": "s1", + "parent_span_id": null, + "t_offset_ms": 0, + "actor": "agent", + "operation": "tool.call", + "summary": "Placeholder span with no finding." + }, + { + "span_id": "s2", + "parent_span_id": "s1", + "t_offset_ms": 10, + "actor": "tool", + "operation": "tool.result", + "summary": "Placeholder span with no finding." + } + ], + "security_observations": [ + "This fixture demonstrates nothing; it exists to validate cleanly." + ], + "date_added": "2026-09-15" +} \ No newline at end of file diff --git a/datasets/agentdataflow_toolexchange_traces/tests/fixtures/filename-trace-id-mismatch.json b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/filename-trace-id-mismatch.json new file mode 100644 index 0000000..832e212 --- /dev/null +++ b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/filename-trace-id-mismatch.json @@ -0,0 +1,42 @@ +{ + "trace_id": "DSGAI-TRACE-fixture-does-not-match-filename", + "title": "Minimal valid trace, used as the base every broken fixture mutates", + "description": "Not a real security finding. Exists only so each broken fixture in this directory differs from a passing trace by exactly one thing, and so a future contributor changing the schema has something small to update instead of hand-writing a new minimal trace from scratch.", + "category": "tool_call", + "disposition": "benign", + "dsgai_mapping": [ + "DSGAI06" + ], + "provenance": { + "tier": "hypothetical", + "collection_method": "Constructed by hand as a schema-validity fixture; models nothing." + }, + "sanitization": { + "attestation": true, + "techniques": [ + "not_applicable_fully_synthetic" + ] + }, + "spans": [ + { + "span_id": "s1", + "parent_span_id": null, + "t_offset_ms": 0, + "actor": "agent", + "operation": "tool.call", + "summary": "Placeholder span with no finding." + }, + { + "span_id": "s2", + "parent_span_id": "s1", + "t_offset_ms": 10, + "actor": "tool", + "operation": "tool.result", + "summary": "Placeholder span with no finding." + } + ], + "security_observations": [ + "This fixture demonstrates nothing; it exists to validate cleanly." + ], + "date_added": "2026-09-15" +} \ No newline at end of file diff --git a/datasets/agentdataflow_toolexchange_traces/tests/fixtures/provenance-missing-evidence.json b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/provenance-missing-evidence.json new file mode 100644 index 0000000..84ec054 --- /dev/null +++ b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/provenance-missing-evidence.json @@ -0,0 +1,42 @@ +{ + "trace_id": "DSGAI-TRACE-fixture-provenance-missing-evidence", + "title": "Minimal valid trace, used as the base every broken fixture mutates", + "description": "Not a real security finding. Exists only so each broken fixture in this directory differs from a passing trace by exactly one thing, and so a future contributor changing the schema has something small to update instead of hand-writing a new minimal trace from scratch.", + "category": "tool_call", + "disposition": "benign", + "dsgai_mapping": [ + "DSGAI06" + ], + "provenance": { + "tier": "observed_lab", + "collection_method": "Constructed by hand as a schema-validity fixture; models nothing." + }, + "sanitization": { + "attestation": true, + "techniques": [ + "not_applicable_fully_synthetic" + ] + }, + "spans": [ + { + "span_id": "s1", + "parent_span_id": null, + "t_offset_ms": 0, + "actor": "agent", + "operation": "tool.call", + "summary": "Placeholder span with no finding." + }, + { + "span_id": "s2", + "parent_span_id": "s1", + "t_offset_ms": 10, + "actor": "tool", + "operation": "tool.result", + "summary": "Placeholder span with no finding." + } + ], + "security_observations": [ + "This fixture demonstrates nothing; it exists to validate cleanly." + ], + "date_added": "2026-09-15" +} \ No newline at end of file diff --git a/datasets/agentdataflow_toolexchange_traces/tests/fixtures/sanitization-attestation-false.json b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/sanitization-attestation-false.json new file mode 100644 index 0000000..21b2fd1 --- /dev/null +++ b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/sanitization-attestation-false.json @@ -0,0 +1,42 @@ +{ + "trace_id": "DSGAI-TRACE-fixture-sanitization-attestation-false", + "title": "Minimal valid trace, used as the base every broken fixture mutates", + "description": "Not a real security finding. Exists only so each broken fixture in this directory differs from a passing trace by exactly one thing, and so a future contributor changing the schema has something small to update instead of hand-writing a new minimal trace from scratch.", + "category": "tool_call", + "disposition": "benign", + "dsgai_mapping": [ + "DSGAI06" + ], + "provenance": { + "tier": "hypothetical", + "collection_method": "Constructed by hand as a schema-validity fixture; models nothing." + }, + "sanitization": { + "attestation": false, + "techniques": [ + "not_applicable_fully_synthetic" + ] + }, + "spans": [ + { + "span_id": "s1", + "parent_span_id": null, + "t_offset_ms": 0, + "actor": "agent", + "operation": "tool.call", + "summary": "Placeholder span with no finding." + }, + { + "span_id": "s2", + "parent_span_id": "s1", + "t_offset_ms": 10, + "actor": "tool", + "operation": "tool.result", + "summary": "Placeholder span with no finding." + } + ], + "security_observations": [ + "This fixture demonstrates nothing; it exists to validate cleanly." + ], + "date_added": "2026-09-15" +} \ No newline at end of file diff --git a/datasets/agentdataflow_toolexchange_traces/tests/fixtures/secret-bearer-token.json b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/secret-bearer-token.json new file mode 100644 index 0000000..4adb4ed --- /dev/null +++ b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/secret-bearer-token.json @@ -0,0 +1,42 @@ +{ + "trace_id": "DSGAI-TRACE-fixture-secret-bearer-token", + "title": "Minimal valid trace, used as the base every broken fixture mutates", + "description": "Not a real security finding. Exists only so each broken fixture in this directory differs from a passing trace by exactly one thing, and so a future contributor changing the schema has something small to update instead of hand-writing a new minimal trace from scratch.", + "category": "tool_call", + "disposition": "benign", + "dsgai_mapping": [ + "DSGAI06" + ], + "provenance": { + "tier": "hypothetical", + "collection_method": "Constructed by hand as a schema-validity fixture; models nothing." + }, + "sanitization": { + "attestation": true, + "techniques": [ + "not_applicable_fully_synthetic" + ] + }, + "spans": [ + { + "span_id": "s1", + "parent_span_id": null, + "t_offset_ms": 0, + "actor": "agent", + "operation": "tool.call", + "summary": "Authenticated with Bearer abcdEFGH12345678901234567890 before calling the tool." + }, + { + "span_id": "s2", + "parent_span_id": "s1", + "t_offset_ms": 10, + "actor": "tool", + "operation": "tool.result", + "summary": "Placeholder span with no finding." + } + ], + "security_observations": [ + "This fixture demonstrates nothing; it exists to validate cleanly." + ], + "date_added": "2026-09-15" +} \ No newline at end of file diff --git a/datasets/agentdataflow_toolexchange_traces/tests/fixtures/secret-credential-assignment.json b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/secret-credential-assignment.json new file mode 100644 index 0000000..93fc2bc --- /dev/null +++ b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/secret-credential-assignment.json @@ -0,0 +1,42 @@ +{ + "trace_id": "DSGAI-TRACE-fixture-secret-credential-assignment", + "title": "Minimal valid trace, used as the base every broken fixture mutates", + "description": "Not a real security finding. Exists only so each broken fixture in this directory differs from a passing trace by exactly one thing, and so a future contributor changing the schema has something small to update instead of hand-writing a new minimal trace from scratch.", + "category": "tool_call", + "disposition": "benign", + "dsgai_mapping": [ + "DSGAI06" + ], + "provenance": { + "tier": "hypothetical", + "collection_method": "Constructed by hand as a schema-validity fixture; models nothing." + }, + "sanitization": { + "attestation": true, + "techniques": [ + "not_applicable_fully_synthetic" + ] + }, + "spans": [ + { + "span_id": "s1", + "parent_span_id": null, + "t_offset_ms": 0, + "actor": "agent", + "operation": "tool.call", + "summary": "Placeholder span with no finding." + }, + { + "span_id": "s2", + "parent_span_id": "s1", + "t_offset_ms": 10, + "actor": "tool", + "operation": "tool.result", + "summary": "Placeholder span with no finding." + } + ], + "security_observations": [ + "Debug note left in by mistake: password=hunter2 was used for the test account." + ], + "date_added": "2026-09-15" +} \ No newline at end of file diff --git a/datasets/agentdataflow_toolexchange_traces/tests/fixtures/secret-internal-hostname.json b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/secret-internal-hostname.json new file mode 100644 index 0000000..b2bb84f --- /dev/null +++ b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/secret-internal-hostname.json @@ -0,0 +1,42 @@ +{ + "trace_id": "DSGAI-TRACE-fixture-secret-internal-hostname", + "title": "Minimal valid trace, used as the base every broken fixture mutates", + "description": "Not a real security finding. Exists only so each broken fixture in this directory differs from a passing trace by exactly one thing, and so a future contributor changing the schema has something small to update instead of hand-writing a new minimal trace from scratch.", + "category": "tool_call", + "disposition": "benign", + "dsgai_mapping": [ + "DSGAI06" + ], + "provenance": { + "tier": "hypothetical", + "collection_method": "Constructed by hand as a schema-validity fixture; models nothing." + }, + "sanitization": { + "attestation": true, + "techniques": [ + "not_applicable_fully_synthetic" + ] + }, + "spans": [ + { + "span_id": "s1", + "parent_span_id": null, + "t_offset_ms": 0, + "actor": "agent", + "operation": "tool.call", + "summary": "Result served from prod-db-07.corp.internal for this call." + }, + { + "span_id": "s2", + "parent_span_id": "s1", + "t_offset_ms": 10, + "actor": "tool", + "operation": "tool.result", + "summary": "Placeholder span with no finding." + } + ], + "security_observations": [ + "This fixture demonstrates nothing; it exists to validate cleanly." + ], + "date_added": "2026-09-15" +} \ No newline at end of file diff --git a/datasets/agentdataflow_toolexchange_traces/tests/fixtures/secret-public-ip-cluster.json b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/secret-public-ip-cluster.json new file mode 100644 index 0000000..36b4266 --- /dev/null +++ b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/secret-public-ip-cluster.json @@ -0,0 +1,42 @@ +{ + "trace_id": "DSGAI-TRACE-fixture-secret-public-ip-cluster", + "title": "Minimal valid trace, used as the base every broken fixture mutates", + "description": "Not a real security finding. Exists only so each broken fixture in this directory differs from a passing trace by exactly one thing, and so a future contributor changing the schema has something small to update instead of hand-writing a new minimal trace from scratch.", + "category": "tool_call", + "disposition": "benign", + "dsgai_mapping": [ + "DSGAI06" + ], + "provenance": { + "tier": "hypothetical", + "collection_method": "Constructed by hand as a schema-validity fixture; models nothing." + }, + "sanitization": { + "attestation": true, + "techniques": [ + "not_applicable_fully_synthetic" + ] + }, + "spans": [ + { + "span_id": "s1", + "parent_span_id": null, + "t_offset_ms": 0, + "actor": "agent", + "operation": "tool.call", + "summary": "Traffic observed from 8.8.8.8 failing over to 1.1.1.1 mid-call." + }, + { + "span_id": "s2", + "parent_span_id": "s1", + "t_offset_ms": 10, + "actor": "tool", + "operation": "tool.result", + "summary": "Placeholder span with no finding." + } + ], + "security_observations": [ + "This fixture demonstrates nothing; it exists to validate cleanly." + ], + "date_added": "2026-09-15" +} \ No newline at end of file diff --git a/datasets/agentdataflow_toolexchange_traces/tests/fixtures/span-backwards-offset.json b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/span-backwards-offset.json new file mode 100644 index 0000000..415594d --- /dev/null +++ b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/span-backwards-offset.json @@ -0,0 +1,42 @@ +{ + "trace_id": "DSGAI-TRACE-fixture-span-backwards-offset", + "title": "Minimal valid trace, used as the base every broken fixture mutates", + "description": "Not a real security finding. Exists only so each broken fixture in this directory differs from a passing trace by exactly one thing, and so a future contributor changing the schema has something small to update instead of hand-writing a new minimal trace from scratch.", + "category": "tool_call", + "disposition": "benign", + "dsgai_mapping": [ + "DSGAI06" + ], + "provenance": { + "tier": "hypothetical", + "collection_method": "Constructed by hand as a schema-validity fixture; models nothing." + }, + "sanitization": { + "attestation": true, + "techniques": [ + "not_applicable_fully_synthetic" + ] + }, + "spans": [ + { + "span_id": "s1", + "parent_span_id": null, + "t_offset_ms": 0, + "actor": "agent", + "operation": "tool.call", + "summary": "Placeholder span with no finding." + }, + { + "span_id": "s2", + "parent_span_id": "s1", + "t_offset_ms": -5, + "actor": "tool", + "operation": "tool.result", + "summary": "Placeholder span with no finding." + } + ], + "security_observations": [ + "This fixture demonstrates nothing; it exists to validate cleanly." + ], + "date_added": "2026-09-15" +} \ No newline at end of file diff --git a/datasets/agentdataflow_toolexchange_traces/tests/fixtures/span-dangling-parent.json b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/span-dangling-parent.json new file mode 100644 index 0000000..322ea1b --- /dev/null +++ b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/span-dangling-parent.json @@ -0,0 +1,42 @@ +{ + "trace_id": "DSGAI-TRACE-fixture-span-dangling-parent", + "title": "Minimal valid trace, used as the base every broken fixture mutates", + "description": "Not a real security finding. Exists only so each broken fixture in this directory differs from a passing trace by exactly one thing, and so a future contributor changing the schema has something small to update instead of hand-writing a new minimal trace from scratch.", + "category": "tool_call", + "disposition": "benign", + "dsgai_mapping": [ + "DSGAI06" + ], + "provenance": { + "tier": "hypothetical", + "collection_method": "Constructed by hand as a schema-validity fixture; models nothing." + }, + "sanitization": { + "attestation": true, + "techniques": [ + "not_applicable_fully_synthetic" + ] + }, + "spans": [ + { + "span_id": "s1", + "parent_span_id": null, + "t_offset_ms": 0, + "actor": "agent", + "operation": "tool.call", + "summary": "Placeholder span with no finding." + }, + { + "span_id": "s2", + "parent_span_id": "s99", + "t_offset_ms": 10, + "actor": "tool", + "operation": "tool.result", + "summary": "Placeholder span with no finding." + } + ], + "security_observations": [ + "This fixture demonstrates nothing; it exists to validate cleanly." + ], + "date_added": "2026-09-15" +} \ No newline at end of file diff --git a/datasets/agentdataflow_toolexchange_traces/tests/fixtures/span-self-parent.json b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/span-self-parent.json new file mode 100644 index 0000000..4db894f --- /dev/null +++ b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/span-self-parent.json @@ -0,0 +1,42 @@ +{ + "trace_id": "DSGAI-TRACE-fixture-span-self-parent", + "title": "Minimal valid trace, used as the base every broken fixture mutates", + "description": "Not a real security finding. Exists only so each broken fixture in this directory differs from a passing trace by exactly one thing, and so a future contributor changing the schema has something small to update instead of hand-writing a new minimal trace from scratch.", + "category": "tool_call", + "disposition": "benign", + "dsgai_mapping": [ + "DSGAI06" + ], + "provenance": { + "tier": "hypothetical", + "collection_method": "Constructed by hand as a schema-validity fixture; models nothing." + }, + "sanitization": { + "attestation": true, + "techniques": [ + "not_applicable_fully_synthetic" + ] + }, + "spans": [ + { + "span_id": "s1", + "parent_span_id": "s1", + "t_offset_ms": 0, + "actor": "agent", + "operation": "tool.call", + "summary": "Placeholder span with no finding." + }, + { + "span_id": "s2", + "parent_span_id": "s1", + "t_offset_ms": 10, + "actor": "tool", + "operation": "tool.result", + "summary": "Placeholder span with no finding." + } + ], + "security_observations": [ + "This fixture demonstrates nothing; it exists to validate cleanly." + ], + "date_added": "2026-09-15" +} \ No newline at end of file diff --git a/datasets/agentdataflow_toolexchange_traces/tests/fixtures/span-timestamp-not-iso.json b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/span-timestamp-not-iso.json new file mode 100644 index 0000000..ca3e023 --- /dev/null +++ b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/span-timestamp-not-iso.json @@ -0,0 +1,43 @@ +{ + "trace_id": "DSGAI-TRACE-fixture-span-timestamp-not-iso", + "title": "Minimal valid trace, used as the base every broken fixture mutates", + "description": "Not a real security finding. Exists only so each broken fixture in this directory differs from a passing trace by exactly one thing, and so a future contributor changing the schema has something small to update instead of hand-writing a new minimal trace from scratch.", + "category": "tool_call", + "disposition": "benign", + "dsgai_mapping": [ + "DSGAI06" + ], + "provenance": { + "tier": "hypothetical", + "collection_method": "Constructed by hand as a schema-validity fixture; models nothing." + }, + "sanitization": { + "attestation": true, + "techniques": [ + "not_applicable_fully_synthetic" + ] + }, + "spans": [ + { + "span_id": "s1", + "parent_span_id": null, + "t_offset_ms": 0, + "actor": "agent", + "operation": "tool.call", + "summary": "Placeholder span with no finding.", + "timestamp": "not-a-datetime" + }, + { + "span_id": "s2", + "parent_span_id": "s1", + "t_offset_ms": 10, + "actor": "tool", + "operation": "tool.result", + "summary": "Placeholder span with no finding." + } + ], + "security_observations": [ + "This fixture demonstrates nothing; it exists to validate cleanly." + ], + "date_added": "2026-09-15" +} \ No newline at end of file diff --git a/datasets/agentdataflow_toolexchange_traces/tests/fixtures/valid_minimal.json b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/valid_minimal.json new file mode 100644 index 0000000..2ec30f5 --- /dev/null +++ b/datasets/agentdataflow_toolexchange_traces/tests/fixtures/valid_minimal.json @@ -0,0 +1,36 @@ +{ + "trace_id": "DSGAI-TRACE-fixture-valid-minimal", + "title": "Minimal valid trace, used as the base every broken fixture mutates", + "description": "Not a real security finding. Exists only so each broken fixture in this directory differs from a passing trace by exactly one thing, and so a future contributor changing the schema has something small to update instead of hand-writing a new minimal trace from scratch.", + "category": "tool_call", + "disposition": "benign", + "dsgai_mapping": ["DSGAI06"], + "provenance": { + "tier": "hypothetical", + "collection_method": "Constructed by hand as a schema-validity fixture; models nothing." + }, + "sanitization": { + "attestation": true, + "techniques": ["not_applicable_fully_synthetic"] + }, + "spans": [ + { + "span_id": "s1", + "parent_span_id": null, + "t_offset_ms": 0, + "actor": "agent", + "operation": "tool.call", + "summary": "Placeholder span with no finding." + }, + { + "span_id": "s2", + "parent_span_id": "s1", + "t_offset_ms": 10, + "actor": "tool", + "operation": "tool.result", + "summary": "Placeholder span with no finding." + } + ], + "security_observations": ["This fixture demonstrates nothing; it exists to validate cleanly."], + "date_added": "2026-09-15" +} diff --git a/datasets/agentdataflow_toolexchange_traces/tests/test_validate.py b/datasets/agentdataflow_toolexchange_traces/tests/test_validate.py new file mode 100644 index 0000000..ebd3b76 --- /dev/null +++ b/datasets/agentdataflow_toolexchange_traces/tests/test_validate.py @@ -0,0 +1,132 @@ +"""End-to-end tests for ../validate.py against the fixtures in fixtures/. + +These run `validate.py` as a subprocess, the same way a contributor or CI +does, rather than importing its functions - so a refactor of validate.py's +internals cannot silently stop testing anything as long as the CLI's +contract (exit code, stderr/stdout message) holds. Each fixture in +fixtures/ is a full trace built from valid_minimal.json with exactly one +change, so a fixture that starts passing again means the check for that +change stopped firing. + +Every fixture below is a case emmanuelgjr asked to keep from rotting during +review of #62: "With CI now running the validator, a small tests/ of known +-bad fixtures would keep them from rotting." +""" +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent +FIXTURES_DIR = Path(__file__).resolve().parent / "fixtures" + +# name -> a substring that must appear in validate.py's output when this +# fixture is the only entry, so a failure for the WRONG reason still fails +# the test. +BROKEN_FIXTURES: dict[str, str] = { + "span-self-parent": "which is not a preceding span", + "span-backwards-offset": "moves t_offset_ms backwards", + "span-dangling-parent": "which is not a preceding span", + "secret-bearer-token": "bearer credential", + "secret-internal-hostname": "internal hostname", + "secret-credential-assignment": "secret-looking assignment", + "secret-public-ip-cluster": "real routable IPv4 addresses", + "date-added-not-iso": "is not an ISO 8601 date", + "span-timestamp-not-iso": "is not an ISO 8601 date-time", + "provenance-missing-evidence": "requires at least one", + "evidence-citation-bad-cve-form": "is not a CVE id", + "evidence-citation-bad-url-form": "is not an absolute http", + "evidence-missing-supports": "'supports' is a required property", + "sanitization-attestation-false": "was expected", + "dsgai-mapping-off-enum": "is not one of", + "filename-trace-id-mismatch": "does not match the filename stem", +} + + +def _run_validate(tmp_path: Path) -> subprocess.CompletedProcess: + # Run the COPY staged into tmp_path, not the real ROOT/validate.py. + # validate.py resolves its own ROOT from __file__, not from cwd, so + # running the original file here would silently validate the real + # (empty) entries/ in the source tree instead of the fixture. + return subprocess.run( + [sys.executable, str(tmp_path / "validate.py")], + cwd=tmp_path, + capture_output=True, + text=True, + ) + + +def _stage(tmp_path: Path) -> Path: + """Set up a dataset directory: real schema/example, a copy of the real + shared taxonomy one level up (validate.py looks for it at + ROOT.parent / "_shared", so the fixture tree needs the same shape + relative to itself), and an empty entries/ for the caller to populate.""" + shutil.copy(ROOT / "schema.json", tmp_path / "schema.json") + shutil.copy(ROOT / "example.json", tmp_path / "example.json") + shutil.copy(ROOT / "validate.py", tmp_path / "validate.py") + + shared_dir = tmp_path.parent / "_shared" + shared_dir.mkdir(exist_ok=True) + taxonomy_dest = shared_dir / "dsgai_taxonomy.json" + if not taxonomy_dest.exists(): + shutil.copy(ROOT.parent / "_shared" / "dsgai_taxonomy.json", taxonomy_dest) + + entries = tmp_path / "entries" + entries.mkdir(exist_ok=True) + return entries + + +@pytest.fixture +def entries_dir(tmp_path): + return _stage(tmp_path) + + +def test_valid_minimal_passes(tmp_path, entries_dir): + entry = json.loads((FIXTURES_DIR / "valid_minimal.json").read_text()) + (entries_dir / f"{entry['trace_id']}.json").write_text(json.dumps(entry)) + + result = _run_validate(tmp_path) + + assert result.returncode == 0, result.stdout + result.stderr + assert "OK" in result.stdout + + +@pytest.mark.parametrize("fixture_name,expected_substring", sorted(BROKEN_FIXTURES.items())) +def test_broken_fixture_fails(tmp_path, entries_dir, fixture_name, expected_substring): + entry = json.loads((FIXTURES_DIR / f"{fixture_name}.json").read_text()) + + if fixture_name == "filename-trace-id-mismatch": + # The one fixture whose whole point is that its filename does NOT + # match trace_id - name the file after the fixture, not the trace_id. + dest = entries_dir / f"{fixture_name}.json" + else: + dest = entries_dir / f"{entry['trace_id']}.json" + dest.write_text(json.dumps(entry)) + + result = _run_validate(tmp_path) + + assert result.returncode == 1, ( + f"{fixture_name} was expected to fail validate.py but exit code was " + f"{result.returncode}\n{result.stdout}{result.stderr}" + ) + assert expected_substring in result.stdout, ( + f"{fixture_name} failed, but not with the expected message " + f"({expected_substring!r} not found)\n{result.stdout}" + ) + + +def test_every_fixture_file_is_covered(): + """Every .json in fixtures/ other than the valid base is exercised by + BROKEN_FIXTURES above - a fixture nobody references is not testing + anything.""" + on_disk = {p.stem for p in FIXTURES_DIR.glob("*.json")} - {"valid_minimal"} + assert on_disk == set(BROKEN_FIXTURES), ( + "fixtures/ and BROKEN_FIXTURES have drifted apart: " + f"on disk but not tested: {on_disk - set(BROKEN_FIXTURES)}; " + f"tested but missing from disk: {set(BROKEN_FIXTURES) - on_disk}" + )