diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index f00ef3d6..e0cc16bf 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -51,6 +51,11 @@ jobs: run: | make test + - name: Verify Generated Test Cases + run: | + make test_update + git diff --exit-code + - name: Parse Coverage Value From Feature Branch run: | echo "current_coverage=$(tail -n 1 .coverage/coverage.txt | awk '{print $3}' | awk 'sub("%", "")')" >> $GITHUB_ENV @@ -115,6 +120,8 @@ jobs: - name: Run integration tests env: CONNECTION_STRING: postgres://dawgs:weneedbetterpasswords@localhost:5432/dawgs?sslmode=disable + DAWGS_INTEGRATION_ALLOW_DESTRUCTIVE: "1" + DAWGS_INTEGRATION_DISPOSABLE_TARGETS: postgresql://localhost:5432/dawgs run: | make test_integration @@ -148,5 +155,7 @@ jobs: - name: Run integration tests env: CONNECTION_STRING: neo4j://neo4j:weneedbetterpasswords@localhost:7687 + DAWGS_INTEGRATION_ALLOW_DESTRUCTIVE: "1" + DAWGS_INTEGRATION_DISPOSABLE_TARGETS: neo4j://localhost:7687/ run: | make test_integration diff --git a/.gitignore b/.gitignore index 9834c914..387c1901 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,7 @@ integration/testdata/local/ # Local benchmark comparison output .bench/ + +# Local performance captures and live-run artifacts +/artifacts/perf/ +/artifacts/live/ diff --git a/AGENTS.md b/AGENTS.md index 1a6c190f..340ffab0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,6 +16,7 @@ These instructions apply to the entire repository. - Always format after code edits. Use `make format` unless a narrower formatting command is clearly sufficient for the touched files. - `make test_all` is the default validation command. It runs unit tests and all integration suites. - Integration suites consume `CONNECTION_STRING`. If `CONNECTION_STRING` is not present in the LLM context, ask the user to add it before running `make test_all`. +- Destructive integration suites also require `DAWGS_INTEGRATION_ALLOW_DESTRUCTIVE=1` and an exact credential-free target in `DAWGS_INTEGRATION_DISPOSABLE_TARGETS`. Never add the live benchmark database to that allowlist. - Run `make test_all` only for the backend selected by the scheme in `CONNECTION_STRING`. Tests for other backends should skip themselves. - Core integration cases in `integration/testdata/cases` and `integration/testdata/templates` must be backend-equivalent. Do not add driver-specific skips or driver-specific expected assertions to these suites. If a backend capability needs dedicated coverage, put it in a clearly driver-scoped test that is skipped unless `CONNECTION_STRING` selects that backend. - `make test` is available for unit tests only. diff --git a/GOLANG_CODING_STANDARD.md b/GOLANG_CODING_STANDARD.md index c7958ca4..e613a4ab 100644 --- a/GOLANG_CODING_STANDARD.md +++ b/GOLANG_CODING_STANDARD.md @@ -10,11 +10,13 @@ type. The goal is to remove per-type receiver-name churn and reduce cognitive load while reading method bodies. ```go +// Start begins serving requests. func (s *Server) Start() error { go s.loop() return nil } +// Validate reports whether the configuration is supported. func (s Config) Validate() error { if s.Firewall.Backend != "nftables" { return fmt.Errorf("unsupported firewall backend %q", s.Firewall.Backend) @@ -243,6 +245,29 @@ case <-s.joiner.StopC: Avoid splitting tightly coupled statements when the second line is the immediate effect of the first. +## Documentation Comments + +Every function and method declaration and every struct or interface definition +must have a semantically relevant Go doc comment, whether it is exported or +unexported. Document every struct field and interface member individually, +including embedded fields and embedded interface elements. Follow Go doc form +by starting each comment with the declared identifier when applicable. + +Comments must explain the declaration's purpose, meaning, behavior, or contract. +Merely restating the identifier without adding useful information does not +satisfy this requirement. + +```go +// RecordStore retrieves and persists records. +type RecordStore interface { + // Find returns the record identified by key. + Find(ctx context.Context, key string) (Record, error) + + // Save persists record and returns any write failure. + Save(ctx context.Context, record Record) error +} +``` + ## Function Ordering Prefer ordering functions in the same file so dependencies appear before the @@ -256,12 +281,22 @@ Write struct type definitions across multiple lines, with one field per line. Align naturally with `gofmt`; do not compress structs onto one line. ```go +// FirewallConfig controls how the firewall backend manages bans. type FirewallConfig struct { - Backend string `toml:"backend"` - Table string `toml:"table"` - BanSet string `toml:"ban_set"` - Family string `toml:"family"` - DryRunSummaryOnly bool `toml:"dry_run_summary_only"` + // Backend selects the firewall implementation. + Backend string `toml:"backend"` + + // Table identifies the firewall table managed by the backend. + Table string `toml:"table"` + + // BanSet identifies the set containing banned addresses. + BanSet string `toml:"ban_set"` + + // Family selects the address family managed by the backend. + Family string `toml:"family"` + + // DryRunSummaryOnly limits dry-run output to a summary. + DryRunSummaryOnly bool `toml:"dry_run_summary_only"` } ``` @@ -320,13 +355,41 @@ defer fin.Close() ``` Use package-level grouped `const` and `var` declarations for related values. +Every package-level (global) `var` and `const` entity must have a semantically +relevant Go doc comment, whether it is exported or unexported. Document each +member of a grouped declaration individually. Comments on function-local `var` +declarations are optional and left to the author's discretion. ```go +// ErrNotFound indicates that the requested record does not exist. +var ErrNotFound = errors.New("not found") + const ( - ErrNotFound = errors.New("not found") + // fileWatchKey formats a file watch key. + fileWatchKey KeyFormat = "file_watch.%s" + + // hostRecordKey formats a host record key. + hostRecordKey KeyFormat = "hosts.%s" - fileWatchKey KeyFormat = "file_watch.%s" - hostRecordKey KeyFormat = "hosts.%s" + // hostRecordKeyPrefix identifies the host record key namespace. hostRecordKeyPrefix KeyFormat = "hosts." ) ``` + +In grouped `var` and `const` declarations, treat each leading comment and the +member definition it documents as one unit. Separate that unit from the next +comment and member definition with exactly one blank line. The final member +definition may instead be followed directly by the closing `)`. + +```go +var ( + // expansionRootFilter identifies the recursive traversal root filter. + expansionRootFilter = pgsql.Identifier("traversal_root_filter") + + // expansionTerminalFilter identifies the recursive traversal terminal filter. + expansionTerminalFilter = pgsql.Identifier("traversal_terminal_filter") + + // expansionPairFilter identifies the recursive traversal pair filter. + expansionPairFilter = pgsql.Identifier("traversal_pair_filter") +) +``` diff --git a/Makefile b/Makefile index 31cad4a9..21acc310 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ THIS_FILE := $(lastword $(MAKEFILE_LIST)) # Go configuration GO_CMD ?= go +GOIMPORTS_CMD ?= goimports CGO_ENABLED ?= 0 BENCH ?= . BENCH_COUNT ?= 10 @@ -33,6 +34,37 @@ METRICS_ENFORCE ?= 0 BENCHMARK_REPORT ?= BENCHMARK_BASELINE ?= BENCHMARK_REGRESSION ?= 0.20 +PERF_BASELINE ?= +PERF_CANDIDATE ?= +PERF_GATE_OUTPUT ?= $(METRICS_DIR)/perf-gate.json +PERF_GATE_SEED ?= 1 +PERF_CONFIDENCE ?= 0.975 +PERF_REGRESSION ?= 0.05 +# Promotion-grade gates must override this with one or more workload names +# whose improvement is required to clear the host A/A-aware materiality floor. +PERF_TARGETS ?= +PERF_MATERIALITY_RATIO ?= 0.95 +PERF_MATERIALITY_ABSOLUTE ?= 100us +PERF_AA_ARTIFACT ?= +PERF_AA_OUTPUT ?= $(METRICS_DIR)/perf-aa-resolution.json +PERF_GATE_AA ?= $(PERF_AA_OUTPUT) +PERF_LEFT ?= +PERF_RIGHT ?= +PERF_CONFIRM_AA ?= +PERF_CONFIRM_OUTPUT ?= $(METRICS_DIR)/perf-confirmation.json +PERF_CASES ?= +PERF_FILTER_CASES ?= +PERF_DIAGNOSTIC_GATE ?= 0 +PERF_BUNDLE_VERIFY_DIR ?= +PERF_BUNDLE_VERIFY_OUTPUT ?= $(METRICS_DIR)/capture-bundle-verification.json +PERF_BUNDLE_REQUIRE_CLEAN ?= 0 +PERF_EXPAND_INTO_ARTIFACT ?= +PERF_EXPAND_INTO_OUTPUT ?= $(METRICS_DIR)/expand-into-study.json +PERF_EXPAND_INTO_PROTOCOL ?= discovery +PERF_TOURNAMENT_ARTIFACT ?= +PERF_TOURNAMENT_OUTPUT ?= $(METRICS_DIR)/reference-tournament.json +PERF_TOURNAMENT_ARMS ?= +PERF_TOURNAMENT_PROTOCOL ?= confirmation FUZZ_REPORT ?= MUTATION_REPORT ?= BACKEND_RESULT_ARGS ?= @@ -56,7 +88,7 @@ QUALITY_INPUTS += -mutation-report $(MUTATION_REPORT) endif QUALITY_INPUTS += -benchmark-regression $(BENCHMARK_REGRESSION) -.PHONY: default all build deps tidy lint format test test_all test_integration test_neo4j test_pg test_update plan_corpus complexity complexity_check crap crap_check quality quality_check quality_backend quality_bench metrics metrics_check generate clean help +.PHONY: default all build deps tidy lint format test test_all test_integration test_neo4j test_pg test_update plan_corpus perf_gate perf_aa perf_confirm perf_bundle_verify perf_expand_into perf_tournament complexity complexity_check crap crap_check quality quality_check quality_backend quality_bench metrics metrics_check generate clean help # Default target default: help @@ -79,11 +111,12 @@ tidy: # Code quality lint: @echo "Running linter..." - @$(GO_CMD) vet ./... + @$(GO_CMD) vet -unreachable=false ./... + @$(GO_CMD) list ./... | grep -v '/cypher/parser$$' | xargs $(GO_CMD) vet -unreachable format: @echo "Formatting code..." - @find ./ -name '*.go' -print0 | xargs -P 12 -0 -I '{}' goimports -w '{}' + @find ./ \( -path './.git' -o -path './.coverage' \) -prune -o -name '*.go' -print0 | xargs -P 12 -0 -I '{}' $(GOIMPORTS_CMD) -w '{}' # Test targets test: $(METRICS_DIR) @@ -96,6 +129,7 @@ test_all: test test_integration test_integration: @echo "Running all integration tests..." + @$(GO_CMD) run ./cmd/integrationguard @$(GO_CMD) test -tags 'manual_integration integration' -race -cover -count=1 -p=1 -parallel=1 $(MAIN_PACKAGES) test_bench: @@ -108,10 +142,12 @@ bench_diff: test_neo4j: @echo "Running Neo4j integration tests..." + @$(GO_CMD) run ./cmd/integrationguard @$(GO_CMD) test -tags integration -race -cover -count=1 -p=1 -parallel=1 $(MAIN_PACKAGES) test_pg: @echo "Running PostgreSQL integration tests..." + @$(GO_CMD) run ./cmd/integrationguard @$(GO_CMD) test -tags manual_integration -race -cover -count=1 -p=1 -parallel=1 $(MAIN_PACKAGES) test_update: @@ -127,6 +163,93 @@ plan_corpus: $(METRICS_DIR) @echo "Capturing Cypher plan corpus..." @$(GO_CMD) run ./cmd/plancorpus +perf_gate: $(METRICS_DIR) + @if [ -z "$(PERF_BASELINE)" ] || [ -z "$(PERF_CANDIDATE)" ]; then \ + echo "PERF_BASELINE and PERF_CANDIDATE are required."; \ + exit 1; \ + fi + @if [ "$(PERF_DIAGNOSTIC_GATE)" != "1" ] && [ -z "$(strip $(PERF_TARGETS))" ]; then \ + echo "PERF_TARGETS is required for a promotion-grade performance gate."; \ + exit 1; \ + fi + @$(GO_CMD) run ./cmd/graphbench \ + -gate-baseline "$(PERF_BASELINE)" \ + -gate-candidate "$(PERF_CANDIDATE)" \ + -gate-output "$(PERF_GATE_OUTPUT)" \ + -gate-aa "$(PERF_GATE_AA)" \ + -seed "$(PERF_GATE_SEED)" \ + -confidence-level "$(PERF_CONFIDENCE)" \ + -regression-threshold "$(PERF_REGRESSION)" \ + -gate-targets "$(PERF_TARGETS)" \ + -materiality-ratio "$(PERF_MATERIALITY_RATIO)" \ + -materiality-absolute "$(PERF_MATERIALITY_ABSOLUTE)" \ + -cases "$(PERF_FILTER_CASES)" \ + -diagnostic-gate="$(PERF_DIAGNOSTIC_GATE)" + +perf_aa: $(METRICS_DIR) + @if [ -z "$(PERF_AA_ARTIFACT)" ]; then \ + echo "PERF_AA_ARTIFACT is required."; \ + exit 1; \ + fi + @$(GO_CMD) run ./cmd/graphbench \ + -aa-artifact "$(PERF_AA_ARTIFACT)" \ + -aa-output "$(PERF_AA_OUTPUT)" \ + -seed "$(PERF_GATE_SEED)" \ + -confidence-level "$(PERF_CONFIDENCE)" + +perf_confirm: $(METRICS_DIR) + @if [ -z "$(PERF_LEFT)" ] || [ -z "$(PERF_RIGHT)" ]; then \ + echo "PERF_LEFT and PERF_RIGHT are required."; \ + exit 1; \ + fi + @$(GO_CMD) run ./cmd/graphbench \ + -confirm-left "$(PERF_LEFT)" \ + -confirm-right "$(PERF_RIGHT)" \ + -confirm-aa "$(PERF_CONFIRM_AA)" \ + -confirm-output "$(PERF_CONFIRM_OUTPUT)" \ + -confirm-cases "$(PERF_CASES)" \ + -seed "$(PERF_GATE_SEED)" \ + -confidence-level "$(PERF_CONFIDENCE)" + +perf_bundle_verify: $(METRICS_DIR) + @if [ -z "$(PERF_BUNDLE_VERIFY_DIR)" ]; then \ + echo "PERF_BUNDLE_VERIFY_DIR is required."; \ + exit 1; \ + fi + @$(GO_CMD) run ./cmd/graphbench \ + -bundle-verify "$(PERF_BUNDLE_VERIFY_DIR)" \ + -bundle-verify-output "$(PERF_BUNDLE_VERIFY_OUTPUT)" \ + -bundle-require-clean="$(PERF_BUNDLE_REQUIRE_CLEAN)" + +perf_expand_into: $(METRICS_DIR) + @if [ -z "$(PERF_EXPAND_INTO_ARTIFACT)" ]; then \ + echo "PERF_EXPAND_INTO_ARTIFACT is required."; \ + exit 1; \ + fi + @$(GO_CMD) run ./cmd/graphbench \ + -expand-into-artifact "$(PERF_EXPAND_INTO_ARTIFACT)" \ + -expand-into-output "$(PERF_EXPAND_INTO_OUTPUT)" \ + -expand-into-protocol "$(PERF_EXPAND_INTO_PROTOCOL)" \ + -seed "$(PERF_GATE_SEED)" \ + -confidence-level "$(PERF_CONFIDENCE)" \ + -materiality-ratio "$(PERF_MATERIALITY_RATIO)" \ + -materiality-absolute "$(PERF_MATERIALITY_ABSOLUTE)" + +perf_tournament: $(METRICS_DIR) + @if [ -z "$(PERF_TOURNAMENT_ARTIFACT)" ] || [ -z "$(PERF_TOURNAMENT_ARMS)" ]; then \ + echo "PERF_TOURNAMENT_ARTIFACT and PERF_TOURNAMENT_ARMS are required."; \ + exit 1; \ + fi + @$(GO_CMD) run ./cmd/graphbench \ + -reference-tournament-artifact "$(PERF_TOURNAMENT_ARTIFACT)" \ + -reference-tournament-output "$(PERF_TOURNAMENT_OUTPUT)" \ + -reference-tournament-arms "$(PERF_TOURNAMENT_ARMS)" \ + -reference-tournament-protocol "$(PERF_TOURNAMENT_PROTOCOL)" \ + -seed "$(PERF_GATE_SEED)" \ + -confidence-level "$(PERF_CONFIDENCE)" \ + -materiality-ratio "$(PERF_MATERIALITY_RATIO)" \ + -materiality-absolute "$(PERF_MATERIALITY_ABSOLUTE)" + # Metric targets $(METRICS_DIR): @mkdir -p $(METRICS_DIR) @@ -179,6 +302,8 @@ quality_backend: test echo "PG_CONNECTION_STRING and NEO4J_CONNECTION_STRING are required."; \ exit 1; \ fi + @CONNECTION_STRING="$(PG_CONNECTION_STRING)" $(GO_CMD) run ./cmd/integrationguard + @CONNECTION_STRING="$(NEO4J_CONNECTION_STRING)" $(GO_CMD) run ./cmd/integrationguard @set +e; \ CONNECTION_STRING="$(PG_CONNECTION_STRING)" $(GO_CMD) test -json -tags 'manual_integration integration' -race -cover -count=1 -p=1 -parallel=1 $(MAIN_PACKAGES) > $(BACKEND_PG_REPORT); \ pg_status=$$?; \ @@ -238,6 +363,12 @@ help: @echo " test_neo4j - Run Neo4j integration tests" @echo " test_pg - Run PostgreSQL integration tests" @echo " plan_corpus - Capture shared corpus query plans for configured backends" + @echo " perf_gate - Compare complete declared GraphBench artifacts" + @echo " perf_aa - Calculate A/A measurement resolution for GraphBench" + @echo " perf_confirm - Build a paired GraphBench confirmation report" + @echo " perf_bundle_verify - Verify a portable GraphBench capture bundle" + @echo " perf_expand_into - Build the fixed-one-hop three-arm study report" + @echo " perf_tournament - Qualify a predeclared three- or five-arm reference tournament" @echo " test_update - Update test cases" @echo " complexity - Report cyclomatic complexity" @echo " crap - Report CRAP scores from unit test coverage" diff --git a/README.md b/README.md index 39fec353..fa1a05cc 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,13 @@ plugins. It exposes a backend abstraction for graph queries, with current backen The query interface is built around openCypher, including a PostgreSQL SQL translator for environments that do not support Cypher natively. +The PostgreSQL driver bounds repeated work with immutable 256-entry Cypher AST and SQL translation caches. Translation +entries are keyed by normalized query text, graph ID, a collision-safe parameter-name/type shape, and the effective +versioned traversal-policy identity; they retain SQL +and parameter-source mappings, never request values or defaults, and fail closed when a required source value is absent. +Cached query text is released by LRU eviction or driver close, and diagnostics expose aggregate counters without query +text. + ## Quick Start Build the repository: @@ -27,9 +34,18 @@ Run integration tests when a backend is available: ```bash export CONNECTION_STRING="postgresql://dawgs:weneedbetterpasswords@localhost:65432/dawgs" +export DAWGS_INTEGRATION_ALLOW_DESTRUCTIVE=1 +export DAWGS_INTEGRATION_DISPOSABLE_TARGETS="postgresql://localhost:65432/dawgs" make test_integration ``` +Integration suites and fixture-loading GraphBench runs delete graph data. The +acknowledgement and credential-free target allowlist above are both required; +an absent or mismatched target is rejected before testing. Existing-graph +GraphBench runs reject mutating cases and do not require destructive +acknowledgement. PostgreSQL sessions remain read-write so temporary traversal +workspaces use the same reset strategy as production. + Use this module from another Go project: ```bash @@ -47,6 +63,18 @@ Run the package benchmark suite with: make test_bench ``` +The direct-write regression benchmark is integration-scoped because it +measures real driver batch APIs. It reloads or clears its fixture outside the +timed region and validates post-state after every iteration: + +```bash +DAWGS_INTEGRATION_ALLOW_DESTRUCTIVE=1 \ +DAWGS_INTEGRATION_DISPOSABLE_TARGETS="postgresql://localhost:65432/dawgs" \ + CONNECTION_STRING="postgresql://dawgs:weneedbetterpasswords@localhost:65432/dawgs" \ + go test -tags manual_integration ./integration -run '^$' \ + -bench BenchmarkMutationSafeDirectWrites -benchtime=1x +``` + Use `cmd/benchdiff` to compare benchmarks between two committed refs without changing the active worktree: ```bash @@ -64,18 +92,134 @@ The harness writes raw outputs and a Markdown report under `.bench/runs/` by def findings, includes the raw `benchstat` output for each benchmark suite, and ends with a table of all captured benchmark numbers. -The integration benchmark runner includes committed `base`, `adcs_fanout`, and `traversal_shapes` datasets by default. +The integration benchmark runner includes committed `base`, +`fixed_suffix_expansion_fanout`, and `traversal_shapes` datasets by default. The traversal shape suite checks expected result counts for chain, fanout, bounded cycle, disconnected, edge-kind-selective, and multi-path shortest-path scenarios before recording timings. `make plan_corpus` captures plan diagnostics for the shared Cypher integration corpus. It accepts either `CONNECTION_STRING` for one backend or `PG_CONNECTION_STRING` and `NEO4J_CONNECTION_STRING` for both backends, then -writes JSONL captures and markdown/JSON summaries under `.coverage/`. +writes JSONL captures and markdown/JSON summaries under `.coverage/`. Captures record the DAWGS source version, which +can be overridden with a command flag when needed. Because it reloads fixtures, it also requires +`DAWGS_INTEGRATION_ALLOW_DESTRUCTIVE=1` and every selected credential-free target in +`DAWGS_INTEGRATION_DISPOSABLE_TARGETS`. `go run ./cmd/graphbench` captures runtime diagnostics for the scale corpus under `benchmark/testdata/scale`. The -current modes are `postgres_sql`, `local_traversal`, and `neo4j`; AGE is reference-design input only and is not a direct -comparison mode yet. The command can emit JSONL records plus Markdown and JSON summaries, and can compare current timings -against a previous JSONL baseline. +implemented execution modes are `postgres_sql` and `neo4j`; `local_traversal` is an explicit, non-gating +`not_implemented` diagnostic placeholder, and AGE is reference-design input only. The command can emit JSONL records +plus Markdown and JSON summaries, and can compare current timings against a previous JSONL baseline. Mutating scale +cases must declare a `write_scenario`; each warm-up and timed iteration runs in a rollback transaction and verifies +matched, affected, and post-state cardinality. +Read timings retain every raw warm sample and are bracketed by untimed exact-row +multiset checks. PostgreSQL datasets are vacuumed and analyzed after loading and +before measured reads. Fixture reloads truncate the active relationship and node +partitions together, and +PostgreSQL captures fail before timing unless the active partitions' physical +node and edge counts exactly match the declared fixture; active child-partition +sizes are retained with each fixture. Node-ID expectations and recorded paths use stable +fixture identities rather than backend-assigned IDs, while preserving duplicate +rows and path order. +For sanitized production-like data, `-existing-graph` uses a versioned +logical-key anchor manifest and bypasses every schema/load/clear/vacuum path. +It rejects mutation cases, verifies before/after cardinalities, redacts anchor +values, and supports atomic checkpoints, resume, progress JSONL, and explicitly +labeled adaptive discovery. See `cmd/graphbench/README.md` for the fixed +confirmation and timeout-class workflows. +The executable gate uses the complete corpus/backend declaration instead of the +intersection of successful records, treats Neo4j only as an exact-result and +informational latency oracle, and supports predeclared materiality thresholds. +`make perf_aa` derives host-fingerprinted p50/p95 measurement resolution from +order-balanced repeated A/A captures. Complete normal/envelope performance +gates require that checksummed per-case evidence and use minimum 5%/100us +floors; stress timing remains diagnostic. Exact case/dataset/category/tag selectors create diagnostic-only +artifacts that the complete gate refuses; configured warmups and matched +arm/block/run metadata support isolated confirmation. The GraphBench CLI accepts +repeated `-aa-artifact` inputs so two immutable append-series arms can be +validated without an external merge. `make perf_confirm` +reports paired absolute and relative p50/p95 changes with optional block/reload +A/A floors. Capture bundles can retain the source patch, untracked sources, +module state, binary, manifest, raw records, and checksums. Opt-in pool +concurrency blocks and PostgreSQL component/full-query +references are documented in `cmd/graphbench/README.md`. Path-observed +singleton captures include exact benchmark-only M0/M1 materializer arms with a +shared search boundary; they do not enable an experimental production executor. +Generated fixed-suffix expansion captures provide selectable exact root-reuse, +late-hydration, factored-suffix forward, suffix-seeded reverse, and +backward-viability forward arms plus versioned fixtures with independent +suffix-density and reverse-fan-in controls. V3 fixtures additionally encode +matching-root multiplicity and independent relationship-distinct cycle and +self-loop controls at the productive boundary. The optimizer reports a typed +expansion-search decision. Repository-native +`EXPANSION-SUFFIX-SEEDED-REVERSE` is an exact qualification-only implementation. +Production selection remains on the stepwise incumbent because query shape and +available metadata do not provide hard suffix-density or reverse-state bounds. +The staged, tool-only `orientation-probe-v2` experiment uses +`F2 = root_rows + maximum_depth * forward_degree_rows` and +`R2 = suffix_rows + boundary_rows + reverse_degree_rows`, selecting reverse +only when every bounded probe is complete and `4 * R2 < 3 * F2`. Its frozen v3 +corpus contains eight selector-training cases and four evaluation holdouts whose +timings remain unopened. Qualification requires matched `shadow`, `incumbent`, +`reverse`, and `guarded` artifacts captured under Repeatable Read with traversal +telemetry. On a clean tree, discovery must emit both its report and freeze +manifest from the exact eight training cases; confirmation must consume those +checksum-bound files and the exact eight-training/four-holdout cohort. Per-case +A/A evidence also binds the PostgreSQL timing environment, including transaction +isolation, and the exact validated fixture. See +[GraphBench](cmd/graphbench/README.md) for the exact capture and report protocol. +No v2 qualification benchmark has passed. The existing +`orientation-probe-v1` report, exact-query production seam, and default +production behavior are unchanged by this staging work. +For the distinct one-fixed-prefix plus selective-terminal-expansion shape, +production uses guarded `EXPANSION-ENDPOINT-SEEDED-REVERSE`: 32 endpoint and +4096 reverse-state caps select either the reverse candidate or an exact +same-statement forward fallback without exposing partial candidate rows. + +PostgreSQL recursive shortest-path execution includes contained S3/S4 +singleton selection, a guarded canonical inline witness canary, and an +all-shortest predecessor-DAG executor, with exact same-statement fallback, +reusable session-local workspace-v2 state, late hydration, event-chain runtime +receipts, and +a parameter-shape-aware translation cache. The implementation and its +qualification boundaries are documented in +[Recursive-descent cost controls](docs/recursive_descent_cost_controls.md). + +New inline SP and ordinary-orientation lowerings remain default-off. Canonical +SP-I1 authorization now uses selector `sp-static-v6` and accepts only the +qualified inbound, typed, single-kind, one-path `min=1`/`max=64` bucket; the +automatic `sp-static-v5-contained` S3/S4 choices are unchanged. The +PostgreSQL driver's `SetTraversalPolicy` API can expose one eligible candidate +to an explicit normalized-query SHA-256 allowlist under a nonzero generation. +Activation requires the exact promotion manifest, including its measured +execution boundary and evidence digests. Manifest schema v2 also requires every +evidence report to repeat the exact candidate, selector, source, binary, +corpus, cap, bucket, and query-cohort identity; a digest-shaped string alone is +not authorization. B1/B2 and `SP-I1-C-D` remain tooling-only. Endpoint-seeded reverse, +inline ASP, and inline canonical SP each have an evidence-free emergency +disable switch. Resetting the policy to its +zero value immediately returns all queries to incumbent cache identities. This +is a reversible canary seam, not evidence that a candidate is qualified for +broad production use. + +The PostgreSQL scale-plan gate runs as part of `make test_all` when +`CONNECTION_STRING` selects PostgreSQL. It executes every required Cypher scale +representative with `EXPLAIN ANALYZE`, enforces declared result or mutation +cardinality, and checks stable mutation-target and anchored edge-index +invariants. Run it directly with: + +```bash +DAWGS_INTEGRATION_ALLOW_DESTRUCTIVE=1 \ +DAWGS_INTEGRATION_DISPOSABLE_TARGETS="postgresql://localhost:65432/dawgs" \ + CONNECTION_STRING="postgresql://dawgs:weneedbetterpasswords@localhost:65432/dawgs" \ + go test -tags manual_integration ./cmd/graphbench \ + -run 'Test(PostgreSQLScalePlanInvariants|ScaleCorpusRequiredRepresentativesDeclareCardinality)' \ + -count=1 +``` + +Runtime and plan captures are intentionally generated under the ignored +`.coverage/` directory. Keep them as reviewed environment-specific artifacts; +use the stable `GFSE-*`, `REC-*`, `TRUST-*`, `PRUNE-*`, `HOP-*`, `SCAN-*`, and +`LOOKUP-*` IDs to compare captures with their semantic fixtures and manifest +entries. `go run ./cmd/retriever` dumps and loads live Dawgs graph databases as manifest-based collections of compressed JSONL fragments. It supports @@ -113,8 +257,13 @@ replace github.com/specterops/dawgs => /path/to/dawgs - [Development workflow](docs/development.md): build, test, integration, metrics, quality, and corpus-capture commands. - [Cypher library](cypher/README.md): parser generation and Cypher package overview. - [PostgreSQL translation](docs/postgresql_translation.md): PostgreSQL translator behavior, optimizer lowerings, indexing notes, and validation expectations. +- [CySQL traversal performance priorities](docs/cysql_traversal_priorities.md): source-grounded roadmap for orientation, SP/ASP, probes, statistics, telemetry, and qualification. +- [Traversal priority implementation status](docs/experiments/traversal_priority_implementation_status_v1.md): implemented candidate identities, fail-closed gates, and current no-promotion disposition. - [Plan corpus capture](cmd/plancorpus/README.md): shared integration corpus plan diagnostics. - [Graph benchmark capture](cmd/graphbench/README.md): runtime diagnostics for scale scenarios. +- [Integration corpus](integration/testdata/README.md): fixture, mutation post-state, and typed-parameter schema. +- [BloodHound regression coverage manifest](regression_coverage_manifest.md): per-query-form layer status and existing primitive links. +- [BloodHound source-parity workflow](docs/regression_source_parity.md): dormant-form activation rules and repeatable BHE/BHCE source audits. - [Cypher syntax support](cypher/Cypher%20Syntax%20Support.md): supported Cypher behavior and semantic notes. ## Repository Map diff --git a/benchmark/testdata/scale/README.md b/benchmark/testdata/scale/README.md index 85c2f788..cc3b8403 100644 --- a/benchmark/testdata/scale/README.md +++ b/benchmark/testdata/scale/README.md @@ -12,17 +12,154 @@ Apache AGE is intentionally not a benchmark mode here; it may appear only in Each JSON file contains a list of scale cases with: -- `source`: the source corpus or workload family. - `dataset`: the fixture dataset to load from `integration/testdata`. - `name` and `category`: stable identifiers used in reports. - `cypher`: the Cypher query under test. -- `parameters`: named parameter values. -- `expected_rows`: the expected result cardinality. +- `params`: named parameter values. A typed temporal parameter uses + `{"$type":"datetime","value":"2026-01-02T03:04:05Z"}`. A deterministic + large string list uses + `{"$type":"string_list","prefix":"missing","count":1000,"include":["target"]}`. +- `node_params`: scalar parameters resolved from fixture node names. +- `node_list_params`: list parameters resolved from fixture node names. +- `generated_node_list_params`: high-cardinality fixture-ID lists made from + optional included names plus a prefix/count sequence, for example + `{"ids":{"prefix":"target","count":2000,"include":["matched-target"]}}`. +- `expected.row_count`: the expected result cardinality for a read case. - `observes`: whether the query observes paths, nodes, relationships, properties, or only IDs internally. - `candidate_modes`: the execution modes that should attempt the case. +- `unsupported_modes`: explicit backend-to-reason declarations for matrix + points retained as correctness oracles but not supported by that backend. - `reference_design`: optional design notes, including AGE observations when useful. +Mutations are rejected as ordinary read cases. A mutation must add a +`write_scenario` with: + +- a selection query and `expected_matched` count; +- an `affected_entity` (`node` or `relationship`) and `expected_affected` + count; +- one or more `post_state` queries with expected row counts or integer scalar + values. + +The runner drains the mutation result and validates those expectations inside +one rollback transaction. Warm-up, every timed iteration, and PostgreSQL +`EXPLAIN ANALYZE` therefore start from the same committed fixture state. + +The `generated_reconciliation`, `generated_trust_pruning`, `generated_hops`, +and `generated_scan_lookups` datasets are constructed by +`testutil.NewReconciliationScaleFixture`, +`testutil.NewTrustPruningScaleFixture`, `testutil.NewHopScaleFixture`, and +`testutil.NewScanLookupScaleFixture`; they are intentionally not large +handwritten OpenGraph JSON files. + +The corpus also executes parameterized `generated_shortest_paths_d*_f*` and +`generated_fixed_suffix_expansion_d*_f*_v*_p*` variants. Cases in +`cases/generated_fixed_suffix_expansion.json` use stable `GFSE-*` identifiers. +The normal pairwise subset covers shortest depth 1/2/4/8/16/32/64, fanout +1/16/128/512/1000, +outbound/inbound/directionless, distance/path/all-shortest output, and +disconnected, diamond, cycle, parallel-edge, and self-loop shapes. The +fixed-suffix expansion subset covers depth 0/1/2/4/8/16, fanout +1/10/100/1000, none/sparse/half/all valid branch suffix density, endpoint/path +output, decoys, and a 4 KiB payload. +Each result records the exact configuration name, deterministic graph checksum, +and node/edge cardinality. + +Version-two shortest fixtures use +`generated_shortest_paths_v2_d_o_r_fo_fi_l_k_t_w_x_p_c_s`. +Names are strict and round-trippable: negative values, partial scans, unknown +suffixes, non-canonical numbers, impossible intermediate levels, and partial +parallel configurations are rejected. The fixture has independent outbound +and physical-inbound paths, so hidden downstream fan-in and its mirrored +fan-out control coexist without changing legacy fixture identities. Every edge +has a stable `logical_key`. Metadata records root and per-level degrees, +physical edges by kind, distinct reachable nodes by level, minimum distance, +path cardinalities, predecessor edges, disconnected state, parallel physical +edges and distinct targets, checksum, and loaded physical cardinality. +The ASP qualification subset includes separate training and frozen holdout +cases for outbound and inbound searches, early and maximum-depth targets, +disconnected pairs, parallel relationship kinds, diamond multiplicity, and +stress enumeration. These shapes distinguish stored-helper `ASP-A1-DAG` from +inline `ASP-I1-U-DAG+MAT-M0` at the same full path-multiset boundary. + +`cases/generated_sp_i1_inbound_v1.json` is a separate canonical-witness cohort +for comparing exact S4 with guarded `SP-I1-C-WE+MAT-M0`. Its four training +cases use generated depths 4 and 16 and cover full-depth, early-target, and +disconnected inbound searches. Its three blind holdouts use fresh depths 8 and +32 and cover full-depth and disconnected searches. Every case uses the same +typed one-kind `shortestPath` query with maximum depth 64 and an exact path-set +observation. The disjoint `sp-i1-inbound-v1-training` and +`sp-i1-inbound-v1-holdout` tags are protocol identities; holdout execution is +authorized only after GraphBench validates the training freeze. Ordinary +default, category, dataset, and generic-tag selection omit these protected +holdouts; only the exact holdout protocol tag or an exact holdout case name +enters the frozen authorization path. Partial selections remain forbidden: an +authorized confirmation executes the exact four-training/three-holdout cohort +on PostgreSQL. Neo4j remains in the declaration for cross-backend semantic +coverage, not as a holdout timing arm in this study. + +`shape.fixture_tier` is one of `normal`, `envelope`, or `stress`. +`shape.qualification_split` is independently one of `training`, `holdout`, or +`diagnostic`; selector thresholds may use training records but must be frozen +before holdout records are opened. Direction, +relationship-kind count, expected state class, and result-cardinality class +are stored alongside it. Stress cases remain exact diagnostics and are not +silently promoted to release p95 evidence. + +Version-two fixed-suffix expansion fixtures use +`generated_fixed_suffix_expansion_v2_d_f_r_x_i_m_z_p`. +Unlike the legacy modulus form, every integer is exact: `r0` represents zero +reachable branch suffixes, `x` varies false boundaries independently, `i` +controls reverse fan-in, `m` controls physical suffix multiplicity, and `z` is +either zero or one. Fixture records include declared root rows, forward +expansion states, suffix rows/boundaries, expected reverse states, output +trails, physical cardinality, and checksum. Semantic relationships carry +deterministic `logical_key` properties so relationship-distinct paths can be +compared across backends whose physical IDs differ. + +Version-three fixed-suffix fixtures extend that exact grammar as +`generated_fixed_suffix_expansion_v3_d_f_r_x_i_m_q_z_c_s_p`. +`q` independently controls how many distinct `ExpansionRoot` nodes match the +root predicate; only the primary root owns the declared fanout and suffixes. +`c1` adds two distinctly keyed `Expand` relationships from the deterministic +productive boundary through a dedicated node and back, while `s1` adds one +distinctly keyed `Expand` self-loop at that boundary. The primary root is the +productive boundary when only `z1` supplies a reachable suffix; otherwise the +first reachable branch boundary is used. Both controls require a productive +boundary, may be enabled independently, and preserve Cypher's +relationship-distinct trail semantics. V3 names reject implicit populations, +invalid booleans, unproductive fan-in/topology controls, and noncanonical +numbers. Metadata derives exact forward/reverse relationship-distinct states +and complete output trails from the generated graph. +The orientation-v2 declaration freezes eight training cases spanning every +encoded dimension and four holdouts at previously unused depths 7, 11, 13, +and 15. `orientation-v2-training` and `orientation-v2-holdout` are disjoint +cohort tags; the legacy v2 declarations retain their original v1 evidence +splits. + +`cases/fixed_suffix_expansion_limits.json` is an optimization-neutral cardinality +holdout suite. It covers 511, 512, 513, and 600 physical suffix rows, productive +endpoint and full-path observations, and exactly 512 physical rows with two +suffix paths per boundary to prove bag multiplicity. These `GFSE-BOUNDARY-*` +cases are not owned by any one optimization design; archived experiment reports +retain their historical case names. +The file-backed `fixed_suffix_expansion_adversarial` fixture adds 17 distinct +root lanes converging on one boundary, a reusable-node cycle, two physical suffix +paths, and noncanonical logical IDs. Its 68-row endpoint bag proves +relationship-trail rejection and multiplicity independently of the generated +limit fixtures. + +The file-backed `expand_into` fixture and `cases/expand_into.json` form the +fixed-one-hop, bound-pair plan study. They cover typed, wildcard, and multi-kind +matches; cross-kind relationship multiplicity; duplicate and missing outer +pairs; self-loops; and both asymmetric degree orientations. The +`source_lower_degree` and `target_lower_degree` cases deliberately reverse which +endpoint has the cheaper typed adjacency so the pair join, lower-degree scan, +and statement-local pair-cache references are compared at the same complete +relationship observation boundary. + Use `cmd/graphbench` to run this corpus and produce JSONL, Markdown, and JSON -summaries. +summaries. Exact case/dataset/category/tag selectors are intended for targeted +diagnosis and mark their outputs diagnostic-only; they never replace a complete +corpus capture. diff --git a/benchmark/testdata/scale/cases/expand_into.json b/benchmark/testdata/scale/cases/expand_into.json new file mode 100644 index 00000000..b24de17a --- /dev/null +++ b/benchmark/testdata/scale/cases/expand_into.json @@ -0,0 +1,147 @@ +{ + "cases": [ + { + "name": "EXPAND-INTO-01-typed-singleton-pair", + "dataset": "expand_into", + "category": "expand_into_one_hop", + "cypher": "UNWIND $start_ids AS start_id MATCH (s), (e) WHERE id(s) = start_id AND id(e) = $end_id MATCH (s)-[r:ExpandIntoKindA]->(e) RETURN r", + "node_list_params": {"start_ids": ["pair-source"]}, + "node_params": {"end_id": "pair-target"}, + "expected": {"row_count": 1}, + "observes": {"paths": false, "nodes": false, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "fixture_tier": "normal", "direction": "outbound", "edge_kinds": ["ExpandIntoKindA"], "relationship_kind_count": 1, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["expand-into", "typed", "singleton-pair"] + }, + { + "name": "EXPAND-INTO-02-wildcard-cross-kind-pair", + "dataset": "expand_into", + "category": "expand_into_one_hop", + "cypher": "UNWIND $start_ids AS start_id MATCH (s), (e) WHERE id(s) = start_id AND id(e) = $end_id MATCH (s)-[r]->(e) RETURN r", + "node_list_params": {"start_ids": ["pair-source"]}, + "node_params": {"end_id": "pair-target"}, + "expected": {"row_count": 2}, + "observes": {"paths": false, "nodes": false, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "fixture_tier": "normal", "direction": "outbound", "relationship_kind_count": 0, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["expand-into", "wildcard", "cross-kind"] + }, + { + "name": "EXPAND-INTO-03-multi-kind-pair", + "dataset": "expand_into", + "category": "expand_into_one_hop", + "cypher": "UNWIND $start_ids AS start_id MATCH (s), (e) WHERE id(s) = start_id AND id(e) = $end_id MATCH (s)-[r:ExpandIntoKindA|ExpandIntoKindB]->(e) RETURN r", + "node_list_params": {"start_ids": ["pair-source"]}, + "node_params": {"end_id": "pair-target"}, + "expected": {"row_count": 2}, + "observes": {"paths": false, "nodes": false, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "fixture_tier": "normal", "direction": "outbound", "edge_kinds": ["ExpandIntoKindA", "ExpandIntoKindB"], "relationship_kind_count": 2, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["expand-into", "multi-kind", "cross-kind"] + }, + { + "name": "EXPAND-INTO-04-duplicate-pair-multiplicity", + "dataset": "expand_into", + "category": "expand_into_one_hop", + "cypher": "UNWIND $start_ids AS start_id MATCH (s), (e) WHERE id(s) = start_id AND id(e) = $end_id MATCH (s)-[r:ExpandIntoKindA|ExpandIntoKindB]->(e) RETURN r", + "node_list_params": {"start_ids": ["pair-source", "pair-missing", "pair-source"]}, + "node_params": {"end_id": "pair-target"}, + "expected": {"row_count": 4}, + "observes": {"paths": false, "nodes": false, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "fixture_tier": "envelope", "direction": "outbound", "edge_kinds": ["ExpandIntoKindA", "ExpandIntoKindB"], "relationship_kind_count": 2, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["expand-into", "pair-cache", "duplicate-outer-rows", "hit-miss", "holdout"] + }, + { + "name": "EXPAND-INTO-05-self-loop", + "dataset": "expand_into", + "category": "expand_into_one_hop", + "cypher": "UNWIND $start_ids AS start_id MATCH (s), (e) WHERE id(s) = start_id AND id(e) = $end_id MATCH (s)-[r:ExpandIntoKindA]->(e) RETURN r", + "node_list_params": {"start_ids": ["pair-loop"]}, + "node_params": {"end_id": "pair-loop"}, + "expected": {"row_count": 1}, + "observes": {"paths": false, "nodes": false, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "fixture_tier": "envelope", "direction": "outbound", "edge_kinds": ["ExpandIntoKindA"], "relationship_kind_count": 1, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["expand-into", "self-loop", "holdout"] + }, + { + "name": "EXPAND-INTO-06-missing-pair", + "dataset": "expand_into", + "category": "expand_into_one_hop", + "cypher": "UNWIND $start_ids AS start_id MATCH (s), (e) WHERE id(s) = start_id AND id(e) = $end_id MATCH (s)-[r]->(e) RETURN r", + "node_list_params": {"start_ids": ["pair-missing"]}, + "node_params": {"end_id": "pair-target"}, + "expected": {"row_count": 0}, + "observes": {"paths": false, "nodes": false, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "fixture_tier": "envelope", "direction": "outbound", "relationship_kind_count": 0, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["expand-into", "missing-pair", "holdout"] + }, + { + "name": "EXPAND-INTO-07-source-lower-degree", + "dataset": "expand_into", + "category": "expand_into_one_hop", + "cypher": "UNWIND $start_ids AS start_id MATCH (s), (e) WHERE id(s) = start_id AND id(e) = $end_id MATCH (s)-[r:ExpandIntoKindA]->(e) RETURN r", + "node_list_params": {"start_ids": ["low-source"]}, + "node_params": {"end_id": "high-target"}, + "expected": {"row_count": 1}, + "observes": {"paths": false, "nodes": false, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "fixture_tier": "envelope", "direction": "outbound", "edge_kinds": ["ExpandIntoKindA"], "relationship_kind_count": 1, "path_materialization_required": false, "expected_state_class": "source_lower_degree"}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["expand-into", "lower-degree", "source-lower-degree", "holdout"] + }, + { + "name": "EXPAND-INTO-08-target-lower-degree", + "dataset": "expand_into", + "category": "expand_into_one_hop", + "cypher": "UNWIND $start_ids AS start_id MATCH (s), (e) WHERE id(s) = start_id AND id(e) = $end_id MATCH (s)-[r:ExpandIntoKindA]->(e) RETURN r", + "node_list_params": {"start_ids": ["high-source"]}, + "node_params": {"end_id": "low-target"}, + "expected": {"row_count": 1}, + "observes": {"paths": false, "nodes": false, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "fixture_tier": "envelope", "direction": "outbound", "edge_kinds": ["ExpandIntoKindA"], "relationship_kind_count": 1, "path_materialization_required": false, "expected_state_class": "target_lower_degree"}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["expand-into", "lower-degree", "target-lower-degree", "holdout"] + }, + { + "name": "EXPAND-INTO-09-directionless-reversed-pair", + "dataset": "expand_into", + "category": "expand_into_one_hop", + "cypher": "UNWIND $start_ids AS start_id MATCH (s), (e) WHERE id(s) = start_id AND id(e) = $end_id MATCH (s)-[r:ExpandIntoKindA|ExpandIntoKindB]-(e) RETURN r", + "node_list_params": {"start_ids": ["pair-target"]}, + "node_params": {"end_id": "pair-source"}, + "expected": {"row_count": 2}, + "observes": {"paths": false, "nodes": false, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "fixture_tier": "envelope", "direction": "directionless", "edge_kinds": ["ExpandIntoKindA", "ExpandIntoKindB"], "relationship_kind_count": 2, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["expand-into", "directionless", "cross-kind", "holdout"] + }, + { + "name": "EXPAND-INTO-10-inbound-pair", + "dataset": "expand_into", + "category": "expand_into_one_hop", + "cypher": "UNWIND $start_ids AS start_id MATCH (s), (e) WHERE id(s) = start_id AND id(e) = $end_id MATCH (s)<-[r:ExpandIntoKindA|ExpandIntoKindB]-(e) RETURN r", + "node_list_params": {"start_ids": ["pair-target"]}, + "node_params": {"end_id": "pair-source"}, + "expected": {"row_count": 2}, + "observes": {"paths": false, "nodes": false, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "fixture_tier": "envelope", "direction": "inbound", "edge_kinds": ["ExpandIntoKindA", "ExpandIntoKindB"], "relationship_kind_count": 2, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["expand-into", "inbound", "cross-kind", "holdout"] + }, + { + "name": "EXPAND-INTO-11-directionless-self-loop", + "dataset": "expand_into", + "category": "expand_into_one_hop", + "cypher": "UNWIND $start_ids AS start_id MATCH (s), (e) WHERE id(s) = start_id AND id(e) = $end_id MATCH (s)-[r:ExpandIntoKindA]-(e) RETURN r", + "node_list_params": {"start_ids": ["pair-loop"]}, + "node_params": {"end_id": "pair-loop"}, + "expected": {"row_count": 1}, + "observes": {"paths": false, "nodes": false, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "fixture_tier": "envelope", "direction": "directionless", "edge_kinds": ["ExpandIntoKindA"], "relationship_kind_count": 1, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["expand-into", "directionless", "self-loop", "holdout"] + } + ] +} diff --git a/benchmark/testdata/scale/cases/fixed_suffix_expansion_limits.json b/benchmark/testdata/scale/cases/fixed_suffix_expansion_limits.json new file mode 100644 index 00000000..c8537773 --- /dev/null +++ b/benchmark/testdata/scale/cases/fixed_suffix_expansion_limits.json @@ -0,0 +1,311 @@ +{ + "cases": [ + { + "name": "GFSE-BOUNDARY-S511-endpoint", + "dataset": "generated_fixed_suffix_expansion_v2_d16_f129_r0_x510_i0_m1_z1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": { + "root_key": "generated-fse-root" + }, + "expected": { + "row_count": 1, + "result_kind": "id_rows", + "id_rows": [ + [ + "fse-head-root-00", + "fse-terminal" + ] + ] + }, + "observes": { + "paths": false, + "nodes": false, + "relationships": false, + "properties": true + }, + "shape": { + "qualification_split": "holdout", + "fixture_tier": "normal", + "root_predicate": "selective_property", + "terminal_predicate": "fixed_suffix", + "edge_kinds": [ + "Expand", + "EnterSuffix", + "ContinueSuffix", + "CompleteSuffix" + ], + "min_depth": 0, + "max_depth": 16, + "path_materialization_required": false + }, + "candidate_modes": [ + "postgres_sql", + "neo4j" + ], + "tags": [ + "generated", + "normal-tier", + "fixed-suffix-expansion-boundary", + "suffix-cardinality-511", + "holdout" + ] + }, + { + "name": "GFSE-BOUNDARY-S512-endpoint", + "dataset": "generated_fixed_suffix_expansion_v2_d16_f129_r0_x511_i0_m1_z1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": { + "root_key": "generated-fse-root" + }, + "expected": { + "row_count": 1, + "result_kind": "id_rows", + "id_rows": [ + [ + "fse-head-root-00", + "fse-terminal" + ] + ] + }, + "observes": { + "paths": false, + "nodes": false, + "relationships": false, + "properties": true + }, + "shape": { + "qualification_split": "holdout", + "fixture_tier": "normal", + "root_predicate": "selective_property", + "terminal_predicate": "fixed_suffix", + "edge_kinds": [ + "Expand", + "EnterSuffix", + "ContinueSuffix", + "CompleteSuffix" + ], + "min_depth": 0, + "max_depth": 16, + "path_materialization_required": false + }, + "candidate_modes": [ + "postgres_sql", + "neo4j" + ], + "tags": [ + "generated", + "normal-tier", + "fixed-suffix-expansion-boundary", + "suffix-cardinality-512", + "holdout" + ] + }, + { + "name": "GFSE-BOUNDARY-S513-endpoint", + "dataset": "generated_fixed_suffix_expansion_v2_d16_f129_r0_x512_i0_m1_z1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": { + "root_key": "generated-fse-root" + }, + "expected": { + "row_count": 1, + "result_kind": "id_rows", + "id_rows": [ + [ + "fse-head-root-00", + "fse-terminal" + ] + ] + }, + "observes": { + "paths": false, + "nodes": false, + "relationships": false, + "properties": true + }, + "shape": { + "qualification_split": "holdout", + "fixture_tier": "normal", + "root_predicate": "selective_property", + "terminal_predicate": "fixed_suffix", + "edge_kinds": [ + "Expand", + "EnterSuffix", + "ContinueSuffix", + "CompleteSuffix" + ], + "min_depth": 0, + "max_depth": 16, + "path_materialization_required": false + }, + "candidate_modes": [ + "postgres_sql", + "neo4j" + ], + "tags": [ + "generated", + "normal-tier", + "fixed-suffix-expansion-boundary", + "suffix-cardinality-513", + "holdout" + ] + }, + { + "name": "GFSE-BOUNDARY-S600-productive-endpoint", + "dataset": "generated_fixed_suffix_expansion_v2_d16_f129_r0_x599_i0_m1_z1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": { + "root_key": "generated-fse-root" + }, + "expected": { + "row_count": 1, + "result_kind": "id_rows", + "id_rows": [ + [ + "fse-head-root-00", + "fse-terminal" + ] + ] + }, + "observes": { + "paths": false, + "nodes": false, + "relationships": false, + "properties": true + }, + "shape": { + "qualification_split": "holdout", + "fixture_tier": "normal", + "root_predicate": "selective_property", + "terminal_predicate": "fixed_suffix", + "edge_kinds": [ + "Expand", + "EnterSuffix", + "ContinueSuffix", + "CompleteSuffix" + ], + "min_depth": 0, + "max_depth": 16, + "path_materialization_required": false + }, + "candidate_modes": [ + "postgres_sql", + "neo4j" + ], + "tags": [ + "generated", + "normal-tier", + "fixed-suffix-expansion-boundary", + "suffix-cardinality-600", + "nonempty-remainder", + "holdout" + ] + }, + { + "name": "GFSE-BOUNDARY-S513-productive-path", + "dataset": "generated_fixed_suffix_expansion_v2_d16_f129_r0_x512_i0_m1_z1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN p", + "params": { + "root_key": "generated-fse-root" + }, + "expected": { + "row_count": 1, + "result_kind": "path_set" + }, + "observes": { + "paths": true, + "nodes": true, + "relationships": true, + "properties": true + }, + "shape": { + "qualification_split": "holdout", + "fixture_tier": "normal", + "root_predicate": "selective_property", + "terminal_predicate": "fixed_suffix", + "edge_kinds": [ + "Expand", + "EnterSuffix", + "ContinueSuffix", + "CompleteSuffix" + ], + "min_depth": 0, + "max_depth": 16, + "path_materialization_required": true + }, + "candidate_modes": [ + "postgres_sql", + "neo4j" + ], + "tags": [ + "generated", + "normal-tier", + "fixed-suffix-expansion-boundary", + "suffix-cardinality-513", + "path", + "holdout" + ] + }, + { + "name": "GFSE-BOUNDARY-S512-physical-bag-multiplicity", + "dataset": "generated_fixed_suffix_expansion_v2_d16_f129_r0_x255_i0_m2_z1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": { + "root_key": "generated-fse-root" + }, + "expected": { + "row_count": 2, + "result_kind": "id_rows", + "id_rows": [ + [ + "fse-head-root-00", + "fse-terminal" + ], + [ + "fse-head-root-01", + "fse-terminal" + ] + ] + }, + "observes": { + "paths": false, + "nodes": false, + "relationships": false, + "properties": true + }, + "shape": { + "qualification_split": "holdout", + "fixture_tier": "normal", + "root_predicate": "selective_property", + "terminal_predicate": "fixed_suffix", + "edge_kinds": [ + "Expand", + "EnterSuffix", + "ContinueSuffix", + "CompleteSuffix" + ], + "min_depth": 0, + "max_depth": 16, + "path_materialization_required": false + }, + "candidate_modes": [ + "postgres_sql", + "neo4j" + ], + "tags": [ + "generated", + "normal-tier", + "fixed-suffix-expansion-boundary", + "suffix-cardinality-512", + "physical-bag-multiplicity", + "holdout" + ] + } + ] +} diff --git a/benchmark/testdata/scale/cases/generated_endpoint_seeded_expansion_v1.json b/benchmark/testdata/scale/cases/generated_endpoint_seeded_expansion_v1.json new file mode 100644 index 00000000..72f6f92e --- /dev/null +++ b/benchmark/testdata/scale/cases/generated_endpoint_seeded_expansion_v1.json @@ -0,0 +1,37 @@ +{ + "cases": [ + { + "name": "GESE-01-guard-admitted", + "dataset": "generated_endpoint_seeded_expansion_v1_d3_e2_q1_w2_o1_x1_m1_c0_p8", + "category": "generated_endpoint_seeded_expansion", + "cypher": "MATCH (c:Computer)-[:HasSession]->(:User)-[:MemberOf*1..64]->(g:Group) WHERE g.objectid ENDS WITH '-512' RETURN count(*)", + "expected": {"row_count": 1, "scalar_int": 2, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "training", "root_predicate": "unbound", "terminal_predicate": "selective_property", "edge_kinds": ["HasSession", "MemberOf"], "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["endpoint-seeded-expansion", "guard-admitted", "scalar"] + }, + { + "name": "GESE-02-endpoint-guard-fallback", + "dataset": "generated_endpoint_seeded_expansion_v1_d2_e33_q0_w33_o0_x0_m1_c0_p0", + "category": "generated_endpoint_seeded_expansion", + "cypher": "MATCH (c:Computer)-[:HasSession]->(:User)-[:MemberOf*1..64]->(g:Group) WHERE g.objectid ENDS WITH '-512' RETURN count(*)", + "expected": {"row_count": 1, "scalar_int": 33, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "holdout", "root_predicate": "unbound", "terminal_predicate": "selective_property", "edge_kinds": ["HasSession", "MemberOf"], "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["endpoint-seeded-expansion", "endpoint-guard-overflow", "fallback", "scalar", "holdout"] + }, + { + "name": "GESE-03-state-guard-fallback", + "dataset": "generated_endpoint_seeded_expansion_v1_d1_e1_q0_w1_o0_x4097_m1_c0_p0", + "category": "generated_endpoint_seeded_expansion", + "cypher": "MATCH (c:Computer)-[:HasSession]->(:User)-[:MemberOf*1..64]->(g:Group) WHERE g.objectid ENDS WITH '-512' RETURN count(*)", + "expected": {"row_count": 1, "scalar_int": 1, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "holdout", "root_predicate": "unbound", "terminal_predicate": "selective_property", "edge_kinds": ["HasSession", "MemberOf"], "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["endpoint-seeded-expansion", "state-guard-overflow", "fallback", "scalar", "holdout"] + } + ] +} diff --git a/benchmark/testdata/scale/cases/generated_fixed_suffix_expansion.json b/benchmark/testdata/scale/cases/generated_fixed_suffix_expansion.json new file mode 100644 index 00000000..ea3615cd --- /dev/null +++ b/benchmark/testdata/scale/cases/generated_fixed_suffix_expansion.json @@ -0,0 +1,340 @@ +{ + "cases": [ + { + "name": "GFSE-V2-D16-F1000-R1-X1-M1-sparse_endpoint_ids", + "dataset": "generated_fixed_suffix_expansion_v2_d16_f1000_r1_x1_i0_m1_z1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 2, "result_kind": "id_rows", "id_rows": [["fse-head-root-00", "fse-terminal"], ["fse-head-branch-0000-depth-16-00", "fse-terminal"]]}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "training", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 16, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v2", "endpoint-ids", "depth-16", "fanout-1000", "reachable-1", "disconnected-1", "discovery"] + }, + { + "name": "GFSE-V2-D16-F1000-R1-X1-M1-sparse_path", + "dataset": "generated_fixed_suffix_expansion_v2_d16_f1000_r1_x1_i0_m1_z1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 2, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 16, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v2", "path", "depth-16", "fanout-1000", "reachable-1", "disconnected-1", "discovery"] + }, + { + "name": "GFSE-V2-D08-F512-R0-X512-zero_reachable", + "dataset": "generated_fixed_suffix_expansion_v2_d8_f512_r0_x512_i0_m1_z0_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..8]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 0, "result_kind": "id_rows", "id_rows": []}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "holdout", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 8, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v2", "endpoint-ids", "zero-result", "reachable-0", "disconnected-512", "adversarial", "holdout"] + }, + { + "name": "GFSE-V2-D08-F016-R1-I1000-high_reverse_fanin", + "dataset": "generated_fixed_suffix_expansion_v2_d8_f16_r1_x0_i1000_m1_z0_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..8]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 1, "result_kind": "id_rows", "id_rows": [["fse-head-branch-0000-depth-08-00", "fse-terminal"]]}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "holdout", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 8, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v2", "endpoint-ids", "reverse-fanin-1000", "adversarial", "holdout"] + }, + { + "name": "GFSE-V3-TRAIN-Q1-C0-S0-root_baseline", + "dataset": "generated_fixed_suffix_expansion_v3_d2_f4_r0_x2_i0_m1_q1_z1_c0_s0_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..2]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 1, "result_kind": "id_rows", "id_rows": [["fse-head-root-00", "fse-terminal"]]}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "training", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 2, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "orientation-v2-training", "endpoint-ids", "root-rows-1", "productive-boundary-controls-none", "training"] + }, + { + "name": "GFSE-V3-TRAIN-Q4-C0-S0-root_multiplicity", + "dataset": "generated_fixed_suffix_expansion_v3_d2_f4_r0_x2_i0_m1_q4_z1_c0_s0_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..2]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 1, "result_kind": "id_rows", "id_rows": [["fse-head-root-00", "fse-terminal"]]}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "training", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 2, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "orientation-v2-training", "endpoint-ids", "root-rows-4", "productive-boundary-controls-none", "training"] + }, + { + "name": "GFSE-V3-TRAIN-Q4-C1-S0-productive_cycle", + "dataset": "generated_fixed_suffix_expansion_v3_d2_f4_r0_x2_i0_m1_q4_z1_c1_s0_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..2]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 2, "result_kind": "id_rows", "id_rows": [["fse-head-root-00", "fse-terminal"], ["fse-head-root-00", "fse-terminal"]]}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "training", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 2, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "orientation-v2-training", "endpoint-ids", "root-rows-4", "productive-boundary-cycle", "relationship-distinct", "training"] + }, + { + "name": "GFSE-V3-TRAIN-Q4-C0-S1-productive_self_loop", + "dataset": "generated_fixed_suffix_expansion_v3_d2_f4_r0_x2_i0_m1_q4_z1_c0_s1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..2]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 2, "result_kind": "id_rows", "id_rows": [["fse-head-root-00", "fse-terminal"], ["fse-head-root-00", "fse-terminal"]]}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "training", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 2, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "orientation-v2-training", "endpoint-ids", "root-rows-4", "productive-boundary-self-loop", "relationship-distinct", "training"] + }, + { + "name": "GFSE-V3-TRAIN-Q4-C1-S1-productive_cycle_self_loop_path", + "dataset": "generated_fixed_suffix_expansion_v3_d2_f4_r0_x2_i0_m1_q4_z1_c1_s1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..2]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 3, "result_kind": "path_set", "path_rows": [{"nodes":["fse-root","fse-head-root-00","fse-middle-root-00","fse-terminal"],"relationship_kinds":["EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["root:enter","root:continue","root:complete"]},{"nodes":["fse-root","fse-productive-boundary-cycle","fse-root","fse-head-root-00","fse-middle-root-00","fse-terminal"],"relationship_kinds":["Expand","Expand","EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["productive-boundary-cycle-enter","productive-boundary-cycle-return","root:enter","root:continue","root:complete"]},{"nodes":["fse-root","fse-root","fse-head-root-00","fse-middle-root-00","fse-terminal"],"relationship_kinds":["Expand","EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["productive-boundary-self-loop","root:enter","root:continue","root:complete"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 2, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "orientation-v2-training", "path", "root-rows-4", "productive-boundary-cycle", "productive-boundary-self-loop", "relationship-distinct", "training"] + }, + { + "name": "GFSE-V3-TRAIN-D03-F006-R1-X0-I4-M2-Q2-endpoint", + "dataset": "generated_fixed_suffix_expansion_v3_d3_f6_r1_x0_i4_m2_q2_z0_c0_s0_p32", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..3]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 2, "result_kind": "id_rows", "id_rows": [["fse-head-branch-0000-depth-03-00", "fse-terminal"], ["fse-head-branch-0000-depth-03-01", "fse-terminal"]]}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "training", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 3, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "orientation-v2-training", "endpoint-ids", "reachable-sparse", "reverse-fanin", "suffix-multiplicity-2", "root-rows-2", "payload", "training"] + }, + { + "name": "GFSE-V3-TRAIN-D05-F008-R4-X3-I0-M1-Q3-path", + "dataset": "generated_fixed_suffix_expansion_v3_d5_f8_r4_x3_i0_m1_q3_z0_c0_s0_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..5]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 4, "result_kind": "path_set", "path_rows": [{"nodes":["fse-root","fse-branch-0000-level-01","fse-branch-0000-level-02","fse-branch-0000-level-03","fse-branch-0000-level-04","fse-branch-0000-level-05","fse-head-branch-0000-depth-05-00","fse-middle-branch-0000-depth-05-00","fse-terminal"],"relationship_kinds":["Expand","Expand","Expand","Expand","Expand","EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["branch-0000-level-01","branch-0000-level-02","branch-0000-level-03","branch-0000-level-04","branch-0000-level-05","branch-0000-depth-05:enter","branch-0000-depth-05:continue","branch-0000-depth-05:complete"]},{"nodes":["fse-root","fse-branch-0001-level-01","fse-branch-0001-level-02","fse-branch-0001-level-03","fse-branch-0001-level-04","fse-branch-0001-level-05","fse-head-branch-0001-depth-05-00","fse-middle-branch-0001-depth-05-00","fse-terminal"],"relationship_kinds":["Expand","Expand","Expand","Expand","Expand","EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["branch-0001-level-01","branch-0001-level-02","branch-0001-level-03","branch-0001-level-04","branch-0001-level-05","branch-0001-depth-05:enter","branch-0001-depth-05:continue","branch-0001-depth-05:complete"]},{"nodes":["fse-root","fse-branch-0002-level-01","fse-branch-0002-level-02","fse-branch-0002-level-03","fse-branch-0002-level-04","fse-branch-0002-level-05","fse-head-branch-0002-depth-05-00","fse-middle-branch-0002-depth-05-00","fse-terminal"],"relationship_kinds":["Expand","Expand","Expand","Expand","Expand","EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["branch-0002-level-01","branch-0002-level-02","branch-0002-level-03","branch-0002-level-04","branch-0002-level-05","branch-0002-depth-05:enter","branch-0002-depth-05:continue","branch-0002-depth-05:complete"]},{"nodes":["fse-root","fse-branch-0003-level-01","fse-branch-0003-level-02","fse-branch-0003-level-03","fse-branch-0003-level-04","fse-branch-0003-level-05","fse-head-branch-0003-depth-05-00","fse-middle-branch-0003-depth-05-00","fse-terminal"],"relationship_kinds":["Expand","Expand","Expand","Expand","Expand","EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["branch-0003-level-01","branch-0003-level-02","branch-0003-level-03","branch-0003-level-04","branch-0003-level-05","branch-0003-depth-05:enter","branch-0003-depth-05:continue","branch-0003-depth-05:complete"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 5, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "orientation-v2-training", "path", "reachable-half", "disconnected-3", "root-rows-3", "training"] + }, + { + "name": "GFSE-V3-TRAIN-D06-F010-R10-X1-I7-M3-Q1-endpoint", + "dataset": "generated_fixed_suffix_expansion_v3_d6_f10_r10_x1_i7_m3_q1_z0_c0_s0_p64", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..6]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 30, "result_kind": "id_rows"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "training", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 6, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "orientation-v2-training", "endpoint-ids", "reachable-all", "disconnected-1", "reverse-fanin", "suffix-multiplicity-3", "payload", "training"] + }, + { + "name": "GFSE-V3-HOLDOUT-D07-F005-R1-X3-I6-M2-Q6-C1-S1-path", + "dataset": "generated_fixed_suffix_expansion_v3_d7_f5_r1_x3_i6_m2_q6_z0_c1_s1_p24", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..7]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 2, "result_kind": "path_set", "path_rows": [{"nodes":["fse-root","fse-branch-0000-level-01","fse-branch-0000-level-02","fse-branch-0000-level-03","fse-branch-0000-level-04","fse-branch-0000-level-05","fse-branch-0000-level-06","fse-branch-0000-level-07","fse-head-branch-0000-depth-07-00","fse-middle-branch-0000-depth-07-00","fse-terminal"],"relationship_kinds":["Expand","Expand","Expand","Expand","Expand","Expand","Expand","EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["branch-0000-level-01","branch-0000-level-02","branch-0000-level-03","branch-0000-level-04","branch-0000-level-05","branch-0000-level-06","branch-0000-level-07","branch-0000-depth-07:enter","branch-0000-depth-07:continue","branch-0000-depth-07:complete"]},{"nodes":["fse-root","fse-branch-0000-level-01","fse-branch-0000-level-02","fse-branch-0000-level-03","fse-branch-0000-level-04","fse-branch-0000-level-05","fse-branch-0000-level-06","fse-branch-0000-level-07","fse-head-branch-0000-depth-07-01","fse-middle-branch-0000-depth-07-01","fse-terminal"],"relationship_kinds":["Expand","Expand","Expand","Expand","Expand","Expand","Expand","EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["branch-0000-level-01","branch-0000-level-02","branch-0000-level-03","branch-0000-level-04","branch-0000-level-05","branch-0000-level-06","branch-0000-level-07","branch-0000-depth-07:enter","branch-0000-depth-07:continue","branch-0000-depth-07:complete"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 7, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "orientation-v2-holdout", "path", "productive-boundary-cycle", "productive-boundary-self-loop", "relationship-distinct", "holdout"] + }, + { + "name": "GFSE-V3-HOLDOUT-D11-F007-R0-X4-I0-M3-Q2-C1-S0-endpoint", + "dataset": "generated_fixed_suffix_expansion_v3_d11_f7_r0_x4_i0_m3_q2_z1_c1_s0_p96", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..11]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 6, "result_kind": "id_rows"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "holdout", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 11, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "orientation-v2-holdout", "endpoint-ids", "zero-depth-suffix", "productive-boundary-cycle", "relationship-distinct", "holdout"] + }, + { + "name": "GFSE-V3-HOLDOUT-D13-F009-R4-X1-I2-M1-Q7-C0-S1-path", + "dataset": "generated_fixed_suffix_expansion_v3_d13_f9_r4_x1_i2_m1_q7_z0_c0_s1_p8", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..13]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 4, "result_kind": "path_set", "path_rows": [{"nodes":["fse-root","fse-branch-0000-level-01","fse-branch-0000-level-02","fse-branch-0000-level-03","fse-branch-0000-level-04","fse-branch-0000-level-05","fse-branch-0000-level-06","fse-branch-0000-level-07","fse-branch-0000-level-08","fse-branch-0000-level-09","fse-branch-0000-level-10","fse-branch-0000-level-11","fse-branch-0000-level-12","fse-branch-0000-level-13","fse-head-branch-0000-depth-13-00","fse-middle-branch-0000-depth-13-00","fse-terminal"],"relationship_kinds":["Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["branch-0000-level-01","branch-0000-level-02","branch-0000-level-03","branch-0000-level-04","branch-0000-level-05","branch-0000-level-06","branch-0000-level-07","branch-0000-level-08","branch-0000-level-09","branch-0000-level-10","branch-0000-level-11","branch-0000-level-12","branch-0000-level-13","branch-0000-depth-13:enter","branch-0000-depth-13:continue","branch-0000-depth-13:complete"]},{"nodes":["fse-root","fse-branch-0001-level-01","fse-branch-0001-level-02","fse-branch-0001-level-03","fse-branch-0001-level-04","fse-branch-0001-level-05","fse-branch-0001-level-06","fse-branch-0001-level-07","fse-branch-0001-level-08","fse-branch-0001-level-09","fse-branch-0001-level-10","fse-branch-0001-level-11","fse-branch-0001-level-12","fse-branch-0001-level-13","fse-head-branch-0001-depth-13-00","fse-middle-branch-0001-depth-13-00","fse-terminal"],"relationship_kinds":["Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["branch-0001-level-01","branch-0001-level-02","branch-0001-level-03","branch-0001-level-04","branch-0001-level-05","branch-0001-level-06","branch-0001-level-07","branch-0001-level-08","branch-0001-level-09","branch-0001-level-10","branch-0001-level-11","branch-0001-level-12","branch-0001-level-13","branch-0001-depth-13:enter","branch-0001-depth-13:continue","branch-0001-depth-13:complete"]},{"nodes":["fse-root","fse-branch-0002-level-01","fse-branch-0002-level-02","fse-branch-0002-level-03","fse-branch-0002-level-04","fse-branch-0002-level-05","fse-branch-0002-level-06","fse-branch-0002-level-07","fse-branch-0002-level-08","fse-branch-0002-level-09","fse-branch-0002-level-10","fse-branch-0002-level-11","fse-branch-0002-level-12","fse-branch-0002-level-13","fse-head-branch-0002-depth-13-00","fse-middle-branch-0002-depth-13-00","fse-terminal"],"relationship_kinds":["Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["branch-0002-level-01","branch-0002-level-02","branch-0002-level-03","branch-0002-level-04","branch-0002-level-05","branch-0002-level-06","branch-0002-level-07","branch-0002-level-08","branch-0002-level-09","branch-0002-level-10","branch-0002-level-11","branch-0002-level-12","branch-0002-level-13","branch-0002-depth-13:enter","branch-0002-depth-13:continue","branch-0002-depth-13:complete"]},{"nodes":["fse-root","fse-branch-0003-level-01","fse-branch-0003-level-02","fse-branch-0003-level-03","fse-branch-0003-level-04","fse-branch-0003-level-05","fse-branch-0003-level-06","fse-branch-0003-level-07","fse-branch-0003-level-08","fse-branch-0003-level-09","fse-branch-0003-level-10","fse-branch-0003-level-11","fse-branch-0003-level-12","fse-branch-0003-level-13","fse-head-branch-0003-depth-13-00","fse-middle-branch-0003-depth-13-00","fse-terminal"],"relationship_kinds":["Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["branch-0003-level-01","branch-0003-level-02","branch-0003-level-03","branch-0003-level-04","branch-0003-level-05","branch-0003-level-06","branch-0003-level-07","branch-0003-level-08","branch-0003-level-09","branch-0003-level-10","branch-0003-level-11","branch-0003-level-12","branch-0003-level-13","branch-0003-depth-13:enter","branch-0003-depth-13:continue","branch-0003-depth-13:complete"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 13, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "orientation-v2-holdout", "path", "productive-boundary-self-loop", "relationship-distinct", "holdout"] + }, + { + "name": "GFSE-V3-HOLDOUT-D15-F012-R6-X6-I9-M2-Q3-Z1-endpoint", + "dataset": "generated_fixed_suffix_expansion_v3_d15_f12_r6_x6_i9_m2_q3_z1_c0_s0_p128", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..15]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 14, "result_kind": "id_rows"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "holdout", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 15, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "orientation-v2-holdout", "endpoint-ids", "zero-depth-suffix", "reverse-fanin", "suffix-multiplicity-2", "payload", "holdout"] + }, + { + "name": "GFSE-D00-F001-none_endpoint_ids", + "dataset": "generated_fixed_suffix_expansion_d0_f1_v1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..0]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 1, "result_kind": "id_rows", "id_rows": [["fse-head", "fse-terminal"]]}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 0, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion", "endpoint-ids", "depth-0", "fanout-1", "density-none"] + }, + { + "name": "GFSE-D00-F001-none_path", + "dataset": "generated_fixed_suffix_expansion_d0_f1_v1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..0]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 0, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion", "path", "depth-0", "fanout-1", "density-none"] + }, + { + "name": "GFSE-D01-F010-sparse_endpoint_ids", + "dataset": "generated_fixed_suffix_expansion_d1_f10_v10_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..1]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 2}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 1, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion", "endpoint-ids", "depth-1", "fanout-10", "density-sparse"] + }, + { + "name": "GFSE-D01-F010-sparse_path", + "dataset": "generated_fixed_suffix_expansion_d1_f10_v10_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..1]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 2, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 1, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion", "path", "depth-1", "fanout-10", "density-sparse"] + }, + { + "name": "GFSE-D02-F100-sparse_endpoint_ids", + "dataset": "generated_fixed_suffix_expansion_d2_f100_v10_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..2]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 11}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 2, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion", "endpoint-ids", "depth-2", "fanout-100", "density-sparse"] + }, + { + "name": "GFSE-D02-F100-sparse_path", + "dataset": "generated_fixed_suffix_expansion_d2_f100_v10_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..2]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 11, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 2, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion", "path", "depth-2", "fanout-100", "density-sparse"] + }, + { + "name": "GFSE-D04-F010-half_payload_endpoint_ids", + "dataset": "generated_fixed_suffix_expansion_d4_f10_v2_p4096", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..4]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 6}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 4, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion", "endpoint-ids", "depth-4", "fanout-10", "density-half", "payload-4k"] + }, + { + "name": "GFSE-D04-F010-half_payload_path", + "dataset": "generated_fixed_suffix_expansion_d4_f10_v2_p4096", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..4]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 6, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 4, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion", "path", "depth-4", "fanout-10", "density-half", "payload-4k"] + }, + { + "name": "GFSE-D08-F001-all_endpoint_ids", + "dataset": "generated_fixed_suffix_expansion_d8_f1_v1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..8]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 2}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 8, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion", "endpoint-ids", "depth-8", "fanout-1", "density-all"] + }, + { + "name": "GFSE-D08-F001-all_path", + "dataset": "generated_fixed_suffix_expansion_d8_f1_v1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..8]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 2, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 8, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion", "path", "depth-8", "fanout-1", "density-all"] + }, + { + "name": "GFSE-D16-F1000-sparse_endpoint_ids", + "dataset": "generated_fixed_suffix_expansion_d16_f1000_v1000_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 2}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 16, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion", "endpoint-ids", "depth-16", "fanout-1000", "density-sparse"] + }, + { + "name": "GFSE-D16-F1000-sparse_path", + "dataset": "generated_fixed_suffix_expansion_d16_f1000_v1000_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 2, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 16, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion", "path", "depth-16", "fanout-1000", "density-sparse"] + } + ] +} diff --git a/benchmark/testdata/scale/cases/generated_shortest_paths.json b/benchmark/testdata/scale/cases/generated_shortest_paths.json new file mode 100644 index 00000000..1c07682f --- /dev/null +++ b/benchmark/testdata/scale/cases/generated_shortest_paths.json @@ -0,0 +1,320 @@ +{ + "cases": [ + { + "name": "GSP-D01-F001_distance", + "dataset": "generated_shortest_paths_d1_f1", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..1]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "scalar_int": 1, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 1, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "distance", "depth-1", "fanout-1"] + }, + { + "name": "GSP-D01-F001_path", + "dataset": "generated_shortest_paths_d1_f1", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..1]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 1, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "path", "depth-1", "fanout-1"] + }, + { + "name": "GSP-D00-F001_path_zero", + "dataset": "generated_shortest_paths_d1_f1", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*0..1]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-start", "end_id": "sp-start"}, + "expected": {"row_count": 1, "result_kind": "path_set", "path_rows": [{"nodes": ["sp-start"], "relationship_kinds": []}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 0, "max_depth": 1, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "path", "zero-depth", "depth-0", "fanout-1"] + }, + { + "name": "GSP-D02-F016_distance", + "dataset": "generated_shortest_paths_d2_f16", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..2]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "scalar_int": 2, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 2, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "distance", "depth-2", "fanout-16"] + }, + { + "name": "GSP-D02-F016_path", + "dataset": "generated_shortest_paths_d2_f16", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..2]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 2, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "path", "depth-2", "fanout-16"] + }, + { + "name": "GSP-D04-F128_distance", + "dataset": "generated_shortest_paths_d4_f128", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "scalar_int": 4, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 4, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "distance", "depth-4", "fanout-128"] + }, + { + "name": "GSP-D04-F128_path", + "dataset": "generated_shortest_paths_d4_f128", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 4, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "path", "depth-4", "fanout-128"] + }, + { + "name": "GSP-D08-F001_distance_inbound", + "dataset": "generated_shortest_paths_d8_f1", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((e)<-[:Traverse*1..8]-(s)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "scalar_int": 8, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 8, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "distance", "inbound", "depth-8", "fanout-1"] + }, + { + "name": "GSP-D08-F001_path_inbound", + "dataset": "generated_shortest_paths_d8_f1", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((e)<-[:Traverse*1..8]-(s)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 8, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "path", "inbound", "depth-8", "fanout-1"] + }, + { + "name": "GSP-D08-F128_path_directionless", + "dataset": "generated_shortest_paths_d8_f128", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..8]-(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 8, "path_materialization_required": true}, + "candidate_modes": ["neo4j"], + "unsupported_modes": {"postgres_sql": "the PostgreSQL translator does not support directionless variable-length expansion"}, + "tags": ["generated", "normal-tier", "path", "directionless", "depth-8", "fanout-128"] + }, + { + "name": "GSP-D16-F016_distance", + "dataset": "generated_shortest_paths_d16_f16", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..16]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "scalar_int": 16, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 16, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "distance", "depth-16", "fanout-16"] + }, + { + "name": "GSP-D16-F016_path", + "dataset": "generated_shortest_paths_d16_f16", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..16]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 16, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "path", "depth-16", "fanout-16"] + }, + { + "name": "GSP-D04-F128_disconnected", + "dataset": "generated_shortest_paths_d4_f128", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-start", "end_id": "sp-disconnected"}, + "expected": {"row_count": 0}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 4, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "disconnected", "depth-4", "fanout-128"] + }, + { + "name": "GSP-D04-F128_path_disconnected", + "dataset": "generated_shortest_paths_d4_f128", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-start", "end_id": "sp-disconnected"}, + "expected": {"row_count": 0, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 4, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "path", "disconnected", "depth-4", "fanout-128"] + }, + { + "name": "GSP-D02-F016_distance_cycle", + "dataset": "generated_shortest_paths_d2_f16", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-start", "end_id": "sp-cycle-b"}, + "expected": {"row_count": 1, "scalar_int": 2, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 4, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "distance", "cycle", "depth-2", "fanout-16"] + }, + { + "name": "GSP-D02-F016_path_cycle", + "dataset": "generated_shortest_paths_d2_f16", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-start", "end_id": "sp-cycle-b"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 4, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "path", "cycle", "depth-2", "fanout-16"] + }, + { + "name": "GSP-D01-F016_distance_parallel", + "dataset": "generated_shortest_paths_d2_f16", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse|TypedTraverse*1..2]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-start", "end_id": "sp-parallel-end"}, + "expected": {"row_count": 1, "scalar_int": 1, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse", "TypedTraverse"], "min_depth": 1, "max_depth": 2, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "distance", "parallel-edges", "depth-1", "fanout-16"] + }, + { + "name": "GSP-D01-F016_path_parallel", + "dataset": "generated_shortest_paths_d2_f16", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse|TypedTraverse*1..2]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-start", "end_id": "sp-parallel-end"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse", "TypedTraverse"], "min_depth": 1, "max_depth": 2, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "path", "parallel-edges", "depth-1", "fanout-16"] + }, + { + "name": "GSP-D02-F016_distance_self_loop", + "dataset": "generated_shortest_paths_d2_f16", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-start", "end_id": "sp-self-loop-exit"}, + "expected": {"row_count": 1, "scalar_int": 2, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 4, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "distance", "self-loop", "depth-2", "fanout-16"] + }, + { + "name": "GSP-D02-F016_path_self_loop", + "dataset": "generated_shortest_paths_d2_f16", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-start", "end_id": "sp-self-loop-exit"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 4, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "path", "self-loop", "depth-2", "fanout-16"] + }, + { + "name": "GSP-D32-F512_distance", + "dataset": "generated_shortest_paths_d32_f512", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..32]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "scalar_int": 32, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 32, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "envelope-tier", "distance", "depth-32", "fanout-512"] + }, + { + "name": "GSP-D32-F512_path", + "dataset": "generated_shortest_paths_d32_f512", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..32]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 32, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "envelope-tier", "path", "depth-32", "fanout-512"] + }, + { + "name": "GSP-D64-F1000_distance", + "dataset": "generated_shortest_paths_d64_f1000", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..64]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "scalar_int": 64, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "envelope-tier", "distance", "depth-64", "fanout-1000"] + }, + { + "name": "GSP-D64-F1000_path", + "dataset": "generated_shortest_paths_d64_f1000", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..64]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 64, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "envelope-tier", "path", "depth-64", "fanout-1000"] + }, + { + "name": "GSP-D64-F1000_disconnected", + "dataset": "generated_shortest_paths_d64_f1000", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..64]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-start", "end_id": "sp-disconnected"}, + "expected": {"row_count": 0}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "envelope-tier", "distance", "disconnected", "depth-64", "fanout-1000"] + }, + { + "name": "GSP-D04-F128_all_shortest_diamond", + "dataset": "generated_shortest_paths_d4_f128", + "category": "generated_all_shortest_paths", + "cypher": "MATCH p = allShortestPaths((s)-[:Traverse|TypedTraverse*1..2]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-start", "end_id": "sp-diamond-end"}, + "expected": {"row_count": 2, "result_kind": "path_set", "path_rows": [ + {"nodes": ["sp-start", "sp-diamond-left", "sp-diamond-end"], "relationship_kinds": ["Traverse", "TypedTraverse"]}, + {"nodes": ["sp-start", "sp-diamond-right", "sp-diamond-end"], "relationship_kinds": ["Traverse", "TypedTraverse"]} + ]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse", "TypedTraverse"], "min_depth": 1, "max_depth": 2, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "all-shortest", "diamond", "ties"] + } + ] +} diff --git a/benchmark/testdata/scale/cases/generated_shortest_paths_v2.json b/benchmark/testdata/scale/cases/generated_shortest_paths_v2.json new file mode 100644 index 00000000..61e81714 --- /dev/null +++ b/benchmark/testdata/scale/cases/generated_shortest_paths_v2.json @@ -0,0 +1,316 @@ +{ + "cases": [ + { + "name": "GSPV2-NORMAL-outbound-distance", + "dataset": "generated_shortest_paths_v2_d3_o2_r1_fo2_fi128_l2_k7_t16_w2_x16_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..3]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-v2-start", "end_id": "sp-v2-end"}, + "expected": {"row_count": 1, "scalar_int": 3, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "training", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "mirrored_fanout", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 3, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "distance", "outbound"] + }, + { + "name": "GSPV2-NORMAL-hidden-fanin-distance", + "dataset": "generated_shortest_paths_v2_d3_o2_r1_fo2_fi128_l2_k7_t16_w2_x16_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..3]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": {"row_count": 1, "scalar_int": 3, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "training", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "hidden_intermediate_fan_in", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 3, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "distance", "inbound", "hidden-fan-in"] + }, + { + "name": "GSPV2-NORMAL-outbound-all-shortest-depth3", + "dataset": "generated_shortest_paths_v2_d3_o2_r1_fo2_fi128_l2_k7_t16_w2_x16_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((s)-[:Traverse*1..3]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-start", "end_id": "sp-v2-end"}, + "expected": {"row_count": 1, "result_kind": "path_set", "path_rows": [{"nodes": ["sp-v2-start", "sp-v2-linear-01", "sp-v2-linear-02", "sp-v2-end"], "relationship_kinds": ["Traverse", "Traverse", "Traverse"], "relationship_keys": ["primary-01", "primary-02", "primary-03"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "two_sided_predecessor_dag", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 3, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "all-shortest", "predecessor-dag", "training"] + }, + { + "name": "GSPV2-TRAINING-early-depth1-all-shortest-max16", + "dataset": "generated_shortest_paths_v2_d8_o4_r2_fo8_fi64_l4_k3_t16_w4_x32_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((s)-[:Traverse*1..16]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-start", "end_id": "sp-v2-linear-01"}, + "expected": {"row_count": 1, "result_kind": "path_set", "path_rows": [{"nodes": ["sp-v2-start", "sp-v2-linear-01"], "relationship_kinds": ["Traverse"], "relationship_keys": ["primary-01"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "predecessor_dag_early_target_max_slack", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 16, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "all-shortest", "early-target", "early-depth-1", "max-16", "training"] + }, + { + "name": "GSPV2-TRAINING-early-depth2-all-shortest-max64", + "dataset": "generated_shortest_paths_v2_d8_o4_r2_fo8_fi64_l4_k3_t16_w4_x32_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((s)-[:Traverse*1..64]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-start", "end_id": "sp-v2-linear-02"}, + "expected": {"row_count": 1, "result_kind": "path_set", "path_rows": [{"nodes": ["sp-v2-start", "sp-v2-linear-01", "sp-v2-linear-02"], "relationship_kinds": ["Traverse", "Traverse"], "relationship_keys": ["primary-01", "primary-02"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "predecessor_dag_early_target_max_slack", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 64, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "all-shortest", "early-target", "early-depth-2", "max-64", "training"] + }, + { + "name": "GSPV2-TRAINING-early-depth3-all-shortest-max16", + "dataset": "generated_shortest_paths_v2_d8_o4_r2_fo8_fi64_l4_k3_t16_w4_x32_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((s)-[:Traverse*1..16]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-start", "end_id": "sp-v2-linear-03"}, + "expected": {"row_count": 1, "result_kind": "path_set", "path_rows": [{"nodes": ["sp-v2-start", "sp-v2-linear-01", "sp-v2-linear-02", "sp-v2-linear-03"], "relationship_kinds": ["Traverse", "Traverse", "Traverse"], "relationship_keys": ["primary-01", "primary-02", "primary-03"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "predecessor_dag_early_target_max_slack", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 16, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "all-shortest", "early-target", "early-depth-3", "max-16", "training"] + }, + { + "name": "GSPV2-TRAINING-inbound-early-depth1-all-shortest-max16", + "dataset": "generated_shortest_paths_v2_d8_o4_r2_fo8_fi64_l4_k3_t16_w4_x32_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((r)<-[:Traverse*1..16]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN p", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-linear-01"}, + "expected": {"row_count": 1, "result_kind": "path_set", "path_rows": [{"nodes": ["sp-v2-inbound-root", "sp-v2-inbound-linear-01"], "relationship_kinds": ["Traverse"], "relationship_keys": ["inbound-primary-08"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "predecessor_dag_early_target_hidden_fanin", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 16, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "all-shortest", "inbound", "early-target", "max-16", "training"] + }, + { + "name": "GSPV2-TRAINING-inbound-early-depth3-all-shortest-max64", + "dataset": "generated_shortest_paths_v2_d8_o4_r2_fo8_fi64_l4_k3_t16_w4_x32_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN p", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-linear-03"}, + "expected": {"row_count": 1, "result_kind": "path_set", "path_rows": [{"nodes": ["sp-v2-inbound-root", "sp-v2-inbound-linear-01", "sp-v2-inbound-linear-02", "sp-v2-inbound-linear-03"], "relationship_kinds": ["Traverse", "Traverse", "Traverse"], "relationship_keys": ["inbound-primary-08", "inbound-primary-07", "inbound-primary-06"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "predecessor_dag_early_target_hidden_fanin", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 64, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "all-shortest", "inbound", "early-target", "max-64", "training"] + }, + { + "name": "GSPV2-TRAINING-cycle-dead-tail-all-shortest-max64", + "dataset": "generated_shortest_paths_v2_d8_o4_r2_fo8_fi64_l4_k3_t16_w4_x32_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((s)-[:Traverse*1..64]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-start", "end_id": "sp-v2-end"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "predecessor_dag_cycle_dead_tail", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 64, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "all-shortest", "cycle-dead-tail", "max-64", "training"] + }, + { + "name": "GSPV2-TRAINING-reconvergent-all-shortest-max16", + "dataset": "generated_shortest_paths_v2_d3_o2_r1_fo2_fi128_l2_k7_t16_w2_x16_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((s)-[:DiamondTraverse*1..16]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-diamond-start", "end_id": "sp-v2-diamond-end"}, + "expected": {"row_count": 2, "result_kind": "path_set", "path_rows": [{"nodes": ["sp-v2-diamond-start", "sp-v2-diamond-000000", "sp-v2-diamond-end"], "relationship_kinds": ["DiamondTraverse", "DiamondTraverse"], "relationship_keys": ["diamond-000000-a", "diamond-000000-b"]}, {"nodes": ["sp-v2-diamond-start", "sp-v2-diamond-000001", "sp-v2-diamond-end"], "relationship_kinds": ["DiamondTraverse", "DiamondTraverse"], "relationship_keys": ["diamond-000001-a", "diamond-000001-b"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["DiamondTraverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "predecessor_dag_reconvergence", "result_cardinality_class": "small_multi", "min_depth": 1, "max_depth": 16, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "all-shortest", "reconvergence", "max-16", "training"] + }, + { + "name": "GSPV2-TRAINING-disconnected-all-shortest-max64", + "dataset": "generated_shortest_paths_v2_d8_o4_r2_fo8_fi64_l4_k3_t16_w4_x32_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((s)-[:Traverse*1..64]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-disconnected-start", "end_id": "sp-v2-disconnected-end"}, + "expected": {"row_count": 0, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "predecessor_dag_disconnected_max_miss", "result_cardinality_class": "empty", "min_depth": 1, "max_depth": 64, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "all-shortest", "disconnected", "max-miss", "max-64", "training"] + }, + { + "name": "GSPV2-NORMAL-hidden-fanin-path", + "dataset": "generated_shortest_paths_v2_d3_o2_r1_fo2_fi128_l2_k7_t16_w2_x16_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..3]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN p", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": {"row_count": 1, "result_kind": "path_set", "path_rows": [{"nodes": ["sp-v2-inbound-root", "sp-v2-inbound-linear-01", "sp-v2-inbound-linear-02", "sp-v2-inbound-end"], "relationship_kinds": ["Traverse", "Traverse", "Traverse"], "relationship_keys": ["inbound-primary-03", "inbound-primary-02", "inbound-primary-01"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "hidden_intermediate_fan_in", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 3, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "path", "inbound", "hidden-fan-in"] + }, + { + "name": "GSPV2-NORMAL-parallel-kind-distance", + "dataset": "generated_shortest_paths_v2_d3_o2_r1_fo2_fi128_l2_k7_t16_w2_x16_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((s)-[:ParallelKind00|ParallelKind01|ParallelKind02|ParallelKind03|ParallelKind04|ParallelKind05|ParallelKind06*1..2]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-v2-parallel-start", "end_id": "sp-v2-parallel-target-000000"}, + "expected": {"row_count": 1, "scalar_int": 1, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "holdout", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["ParallelKind00", "ParallelKind01", "ParallelKind02", "ParallelKind03", "ParallelKind04", "ParallelKind05", "ParallelKind06"], "direction": "outbound", "relationship_kind_count": 7, "fixture_tier": "normal", "expected_state_class": "parallel_kind_high_cardinality", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 2, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "distance", "parallel-kinds", "holdout"] + }, + { + "name": "GSPV2-NORMAL-parallel-kind-path", + "dataset": "generated_shortest_paths_v2_d3_o2_r1_fo2_fi128_l2_k7_t16_w2_x16_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((s)-[:ParallelKind00|ParallelKind01|ParallelKind02|ParallelKind03|ParallelKind04|ParallelKind05|ParallelKind06*1..2]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-parallel-start", "end_id": "sp-v2-parallel-target-000000"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["ParallelKind00", "ParallelKind01", "ParallelKind02", "ParallelKind03", "ParallelKind04", "ParallelKind05", "ParallelKind06"], "direction": "outbound", "relationship_kind_count": 7, "fixture_tier": "normal", "expected_state_class": "parallel_kind_high_cardinality", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 2, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "path", "parallel-kinds", "holdout"] + }, + { + "name": "GSPV2-NORMAL-diamond-all-shortest", + "dataset": "generated_shortest_paths_v2_d3_o2_r1_fo2_fi128_l2_k7_t16_w2_x16_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((s)-[:DiamondTraverse*1..2]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-diamond-start", "end_id": "sp-v2-diamond-end"}, + "expected": {"row_count": 2, "result_kind": "path_set", "path_rows": [{"nodes": ["sp-v2-diamond-start", "sp-v2-diamond-000000", "sp-v2-diamond-end"], "relationship_kinds": ["DiamondTraverse", "DiamondTraverse"], "relationship_keys": ["diamond-000000-a", "diamond-000000-b"]}, {"nodes": ["sp-v2-diamond-start", "sp-v2-diamond-000001", "sp-v2-diamond-end"], "relationship_kinds": ["DiamondTraverse", "DiamondTraverse"], "relationship_keys": ["diamond-000001-a", "diamond-000001-b"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["DiamondTraverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "predecessor_dag", "result_cardinality_class": "small_multi", "min_depth": 1, "max_depth": 2, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "all-shortest", "diamond", "holdout"] + }, + { + "name": "GSPV2-HOLDOUT-depth8-outbound-distance", + "dataset": "generated_shortest_paths_v2_d8_o4_r2_fo8_fi64_l4_k3_t16_w4_x32_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..8]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-v2-start", "end_id": "sp-v2-end"}, + "expected": {"row_count": 1, "scalar_int": 8, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "holdout", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "independent_recursive_depth8", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 8, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "distance", "outbound", "recursive-kernel", "holdout"] + }, + { + "name": "GSPV2-HOLDOUT-depth8-inbound-path", + "dataset": "generated_shortest_paths_v2_d8_o4_r2_fo8_fi64_l4_k3_t16_w4_x32_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..8]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN p", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": {"row_count": 1, "result_kind": "path_set", "path_rows": [{"nodes": ["sp-v2-inbound-root", "sp-v2-inbound-linear-01", "sp-v2-inbound-linear-02", "sp-v2-inbound-linear-03", "sp-v2-inbound-linear-04", "sp-v2-inbound-linear-05", "sp-v2-inbound-linear-06", "sp-v2-inbound-linear-07", "sp-v2-inbound-end"], "relationship_kinds": ["Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse"], "relationship_keys": ["inbound-primary-08", "inbound-primary-07", "inbound-primary-06", "inbound-primary-05", "inbound-primary-04", "inbound-primary-03", "inbound-primary-02", "inbound-primary-01"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "independent_recursive_hidden_fanin_depth8", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 8, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "path", "inbound", "recursive-kernel", "holdout"] + }, + { + "name": "GSPV2-HOLDOUT-depth8-all-shortest", + "dataset": "generated_shortest_paths_v2_d8_o4_r2_fo8_fi64_l4_k3_t16_w4_x32_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((s)-[:Traverse*1..8]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-start", "end_id": "sp-v2-end"}, + "expected": {"row_count": 1, "result_kind": "path_set", "path_rows": [{"nodes": ["sp-v2-start", "sp-v2-linear-01", "sp-v2-linear-02", "sp-v2-linear-03", "sp-v2-linear-04", "sp-v2-linear-05", "sp-v2-linear-06", "sp-v2-linear-07", "sp-v2-end"], "relationship_kinds": ["Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse"], "relationship_keys": ["primary-01", "primary-02", "primary-03", "primary-04", "primary-05", "primary-06", "primary-07", "primary-08"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "independent_recursive_predecessor_dag_depth8", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 8, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "all-shortest", "recursive-kernel", "holdout"] + }, + { + "name": "GSPV2-HOLDOUT-disconnected-depth8", + "dataset": "generated_shortest_paths_v2_d8_o4_r2_fo8_fi64_l4_k3_t16_w4_x32_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..8]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-v2-disconnected-start", "end_id": "sp-v2-disconnected-end"}, + "expected": {"row_count": 0, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "holdout", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "recursive_disconnected_max_miss", "result_cardinality_class": "empty", "min_depth": 1, "max_depth": 8, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "distance", "disconnected", "max-miss", "recursive-kernel", "holdout"] + }, + { + "name": "GSPV2-HOLDOUT-depth8-inbound-all-shortest", + "dataset": "generated_shortest_paths_v2_d8_o4_r2_fo8_fi64_l4_k3_t16_w4_x32_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((r)<-[:Traverse*1..8]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN p", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": {"row_count": 1, "result_kind": "path_set", "path_rows": [{"nodes": ["sp-v2-inbound-root", "sp-v2-inbound-linear-01", "sp-v2-inbound-linear-02", "sp-v2-inbound-linear-03", "sp-v2-inbound-linear-04", "sp-v2-inbound-linear-05", "sp-v2-inbound-linear-06", "sp-v2-inbound-linear-07", "sp-v2-inbound-end"], "relationship_kinds": ["Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse"], "relationship_keys": ["inbound-primary-08", "inbound-primary-07", "inbound-primary-06", "inbound-primary-05", "inbound-primary-04", "inbound-primary-03", "inbound-primary-02", "inbound-primary-01"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "independent_recursive_predecessor_dag_hidden_fanin_depth8", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 8, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "all-shortest", "inbound", "hidden-fan-in", "holdout"] + }, + { + "name": "GSPV2-HOLDOUT-disconnected-all-shortest-depth8", + "dataset": "generated_shortest_paths_v2_d8_o4_r2_fo8_fi64_l4_k3_t16_w4_x32_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((s)-[:Traverse*1..8]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-disconnected-start", "end_id": "sp-v2-disconnected-end"}, + "expected": {"row_count": 0, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "predecessor_dag_disconnected_max_miss", "result_cardinality_class": "empty", "min_depth": 1, "max_depth": 8, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "all-shortest", "disconnected", "max-miss", "holdout"] + }, + { + "name": "GSPV2-HOLDOUT-parallel-kind-all-shortest", + "dataset": "generated_shortest_paths_v2_d3_o2_r1_fo2_fi128_l2_k7_t16_w2_x16_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((s)-[:ParallelKind00|ParallelKind01|ParallelKind02|ParallelKind03|ParallelKind04|ParallelKind05|ParallelKind06*1..2]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-parallel-start", "end_id": "sp-v2-parallel-target-000000"}, + "expected": {"row_count": 7, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["ParallelKind00", "ParallelKind01", "ParallelKind02", "ParallelKind03", "ParallelKind04", "ParallelKind05", "ParallelKind06"], "direction": "outbound", "relationship_kind_count": 7, "fixture_tier": "normal", "expected_state_class": "predecessor_dag_parallel_kind_multiplicity", "result_cardinality_class": "small_multi", "min_depth": 1, "max_depth": 2, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "all-shortest", "parallel-kinds", "holdout"] + }, + { + "name": "GSPV2-DIAGNOSTIC-early-target-all-shortest-max16", + "dataset": "generated_shortest_paths_v2_d16_o16_r1_fo16_fi16384_l2_k30_t1024_w100_x1024_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((s)-[:Traverse*1..16]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-start", "end_id": "sp-v2-linear-01"}, + "expected": {"row_count": 1, "result_kind": "path_set", "path_rows": [{"nodes": ["sp-v2-start", "sp-v2-linear-01"], "relationship_kinds": ["Traverse"], "relationship_keys": ["primary-01"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "diagnostic", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "stress", "expected_state_class": "predecessor_dag_early_target_max_slack", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 16, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "stress-tier", "all-shortest", "early-target", "max-slack", "diagnostic"] + }, + { + "name": "GSPV2-STRESS-hidden-fanin-distance", + "dataset": "generated_shortest_paths_v2_d16_o16_r1_fo16_fi16384_l2_k30_t1024_w100_x1024_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..16]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": {"row_count": 1, "scalar_int": 16, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "diagnostic", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "stress", "expected_state_class": "hidden_intermediate_fan_in", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 16, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "stress-tier", "distance", "inbound", "hidden-fan-in"] + }, + { + "name": "GSPV2-STRESS-outbound-all-shortest-depth16", + "dataset": "generated_shortest_paths_v2_d16_o16_r1_fo16_fi16384_l2_k30_t1024_w100_x1024_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((s)-[:Traverse*1..16]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-start", "end_id": "sp-v2-end"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "diagnostic", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "stress", "expected_state_class": "two_sided_predecessor_dag_hidden_fanout", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 16, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "stress-tier", "all-shortest", "predecessor-dag"] + }, + { + "name": "GSPV2-STRESS-diamond-all-shortest-128", + "dataset": "generated_shortest_paths_v2_d3_o0_r0_fo0_fi0_l0_k0_t0_w128_x0_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((s)-[:DiamondTraverse*1..2]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-diamond-start", "end_id": "sp-v2-diamond-end"}, + "expected": {"row_count": 128, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "diagnostic", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["DiamondTraverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "stress", "expected_state_class": "predecessor_output_multiplicity", "result_cardinality_class": "large_multi", "min_depth": 1, "max_depth": 2, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "stress-tier", "all-shortest", "diamond", "output-multiplicity"] + } + ] +} diff --git a/benchmark/testdata/scale/cases/generated_sp_i1_inbound_v1.json b/benchmark/testdata/scale/cases/generated_sp_i1_inbound_v1.json new file mode 100644 index 00000000..3c901654 --- /dev/null +++ b/benchmark/testdata/scale/cases/generated_sp_i1_inbound_v1.json @@ -0,0 +1,226 @@ +{ + "cases": [ + { + "name": "GSP-I1-V1-TRAIN-D04-FI016-full", + "dataset": "generated_shortest_paths_v2_d4_o0_r4_fo0_fi16_l2_k0_t0_w0_x4_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN p", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": { + "row_count": 1, + "result_kind": "path_set", + "path_rows": [{ + "nodes": ["sp-v2-inbound-root", "sp-v2-inbound-linear-01", "sp-v2-inbound-linear-02", "sp-v2-inbound-linear-03", "sp-v2-inbound-end"], + "relationship_kinds": ["Traverse", "Traverse", "Traverse", "Traverse"], + "relationship_keys": ["inbound-primary-04", "inbound-primary-03", "inbound-primary-02", "inbound-primary-01"] + }] + }, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": { + "qualification_split": "training", + "fallback_expectation": "forbidden", + "root_predicate": "bound_id", + "terminal_predicate": "bound_id", + "edge_kinds": ["Traverse"], + "direction": "inbound", + "relationship_kind_count": 1, + "fixture_tier": "normal", + "expected_state_class": "inbound_predecessor_full_depth_fanin_16", + "result_cardinality_class": "singleton", + "min_depth": 1, + "max_depth": 64, + "path_materialization_required": true + }, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "path", "inbound", "hidden-fan-in", "sp-i1-inbound-v1-training"] + }, + { + "name": "GSP-I1-V1-TRAIN-D16-FI256-early-d04", + "dataset": "generated_shortest_paths_v2_d16_o0_r8_fo0_fi256_l8_k0_t0_w0_x16_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN p", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-linear-04"}, + "expected": { + "row_count": 1, + "result_kind": "path_set", + "path_rows": [{ + "nodes": ["sp-v2-inbound-root", "sp-v2-inbound-linear-01", "sp-v2-inbound-linear-02", "sp-v2-inbound-linear-03", "sp-v2-inbound-linear-04"], + "relationship_kinds": ["Traverse", "Traverse", "Traverse", "Traverse"], + "relationship_keys": ["inbound-primary-16", "inbound-primary-15", "inbound-primary-14", "inbound-primary-13"] + }] + }, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": { + "qualification_split": "training", + "fallback_expectation": "forbidden", + "root_predicate": "bound_id", + "terminal_predicate": "bound_id", + "edge_kinds": ["Traverse"], + "direction": "inbound", + "relationship_kind_count": 1, + "fixture_tier": "normal", + "expected_state_class": "inbound_predecessor_early_target_fanin_256", + "result_cardinality_class": "singleton", + "min_depth": 1, + "max_depth": 64, + "path_materialization_required": true + }, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "path", "inbound", "hidden-fan-in", "early-target", "early-depth-4", "sp-i1-inbound-v1-training"] + }, + { + "name": "GSP-I1-V1-TRAIN-D16-FI256-full", + "dataset": "generated_shortest_paths_v2_d16_o0_r8_fo0_fi256_l8_k0_t0_w0_x16_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN p", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": { + "row_count": 1, + "result_kind": "path_set", + "path_rows": [{ + "nodes": ["sp-v2-inbound-root", "sp-v2-inbound-linear-01", "sp-v2-inbound-linear-02", "sp-v2-inbound-linear-03", "sp-v2-inbound-linear-04", "sp-v2-inbound-linear-05", "sp-v2-inbound-linear-06", "sp-v2-inbound-linear-07", "sp-v2-inbound-linear-08", "sp-v2-inbound-linear-09", "sp-v2-inbound-linear-10", "sp-v2-inbound-linear-11", "sp-v2-inbound-linear-12", "sp-v2-inbound-linear-13", "sp-v2-inbound-linear-14", "sp-v2-inbound-linear-15", "sp-v2-inbound-end"], + "relationship_kinds": ["Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse"], + "relationship_keys": ["inbound-primary-16", "inbound-primary-15", "inbound-primary-14", "inbound-primary-13", "inbound-primary-12", "inbound-primary-11", "inbound-primary-10", "inbound-primary-09", "inbound-primary-08", "inbound-primary-07", "inbound-primary-06", "inbound-primary-05", "inbound-primary-04", "inbound-primary-03", "inbound-primary-02", "inbound-primary-01"] + }] + }, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": { + "qualification_split": "training", + "fallback_expectation": "forbidden", + "root_predicate": "bound_id", + "terminal_predicate": "bound_id", + "edge_kinds": ["Traverse"], + "direction": "inbound", + "relationship_kind_count": 1, + "fixture_tier": "normal", + "expected_state_class": "inbound_predecessor_full_depth_fanin_256", + "result_cardinality_class": "singleton", + "min_depth": 1, + "max_depth": 64, + "path_materialization_required": true + }, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "path", "inbound", "hidden-fan-in", "sp-i1-inbound-v1-training"] + }, + { + "name": "GSP-I1-V1-TRAIN-D16-FI256-disconnected", + "dataset": "generated_shortest_paths_v2_d16_o0_r8_fo0_fi256_l8_k0_t0_w0_x16_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN p", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-disconnected-end"}, + "expected": {"row_count": 0, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": { + "qualification_split": "training", + "fallback_expectation": "forbidden", + "root_predicate": "bound_id", + "terminal_predicate": "bound_id", + "edge_kinds": ["Traverse"], + "direction": "inbound", + "relationship_kind_count": 1, + "fixture_tier": "normal", + "expected_state_class": "inbound_predecessor_disconnected_fanin_256", + "result_cardinality_class": "empty", + "min_depth": 1, + "max_depth": 64, + "path_materialization_required": true + }, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "path", "inbound", "hidden-fan-in", "disconnected", "max-miss", "sp-i1-inbound-v1-training"] + }, + { + "name": "GSP-I1-V1-HOLDOUT-D08-FI031-full", + "dataset": "generated_shortest_paths_v2_d8_o0_r3_fo0_fi31_l3_k0_t0_w0_x7_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN p", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": { + "row_count": 1, + "result_kind": "path_set", + "path_rows": [{ + "nodes": ["sp-v2-inbound-root", "sp-v2-inbound-linear-01", "sp-v2-inbound-linear-02", "sp-v2-inbound-linear-03", "sp-v2-inbound-linear-04", "sp-v2-inbound-linear-05", "sp-v2-inbound-linear-06", "sp-v2-inbound-linear-07", "sp-v2-inbound-end"], + "relationship_kinds": ["Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse"], + "relationship_keys": ["inbound-primary-08", "inbound-primary-07", "inbound-primary-06", "inbound-primary-05", "inbound-primary-04", "inbound-primary-03", "inbound-primary-02", "inbound-primary-01"] + }] + }, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": { + "qualification_split": "holdout", + "fallback_expectation": "forbidden", + "root_predicate": "bound_id", + "terminal_predicate": "bound_id", + "edge_kinds": ["Traverse"], + "direction": "inbound", + "relationship_kind_count": 1, + "fixture_tier": "normal", + "expected_state_class": "inbound_predecessor_full_depth_fanin_31", + "result_cardinality_class": "singleton", + "min_depth": 1, + "max_depth": 64, + "path_materialization_required": true + }, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "path", "inbound", "hidden-fan-in", "holdout", "sp-i1-inbound-v1-holdout"] + }, + { + "name": "GSP-I1-V1-HOLDOUT-D32-FI191-full", + "dataset": "generated_shortest_paths_v2_d32_o0_r11_fo0_fi191_l21_k0_t0_w0_x13_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN p", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": { + "row_count": 1, + "result_kind": "path_set", + "path_rows": [{ + "nodes": ["sp-v2-inbound-root", "sp-v2-inbound-linear-01", "sp-v2-inbound-linear-02", "sp-v2-inbound-linear-03", "sp-v2-inbound-linear-04", "sp-v2-inbound-linear-05", "sp-v2-inbound-linear-06", "sp-v2-inbound-linear-07", "sp-v2-inbound-linear-08", "sp-v2-inbound-linear-09", "sp-v2-inbound-linear-10", "sp-v2-inbound-linear-11", "sp-v2-inbound-linear-12", "sp-v2-inbound-linear-13", "sp-v2-inbound-linear-14", "sp-v2-inbound-linear-15", "sp-v2-inbound-linear-16", "sp-v2-inbound-linear-17", "sp-v2-inbound-linear-18", "sp-v2-inbound-linear-19", "sp-v2-inbound-linear-20", "sp-v2-inbound-linear-21", "sp-v2-inbound-linear-22", "sp-v2-inbound-linear-23", "sp-v2-inbound-linear-24", "sp-v2-inbound-linear-25", "sp-v2-inbound-linear-26", "sp-v2-inbound-linear-27", "sp-v2-inbound-linear-28", "sp-v2-inbound-linear-29", "sp-v2-inbound-linear-30", "sp-v2-inbound-linear-31", "sp-v2-inbound-end"], + "relationship_kinds": ["Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse"], + "relationship_keys": ["inbound-primary-32", "inbound-primary-31", "inbound-primary-30", "inbound-primary-29", "inbound-primary-28", "inbound-primary-27", "inbound-primary-26", "inbound-primary-25", "inbound-primary-24", "inbound-primary-23", "inbound-primary-22", "inbound-primary-21", "inbound-primary-20", "inbound-primary-19", "inbound-primary-18", "inbound-primary-17", "inbound-primary-16", "inbound-primary-15", "inbound-primary-14", "inbound-primary-13", "inbound-primary-12", "inbound-primary-11", "inbound-primary-10", "inbound-primary-09", "inbound-primary-08", "inbound-primary-07", "inbound-primary-06", "inbound-primary-05", "inbound-primary-04", "inbound-primary-03", "inbound-primary-02", "inbound-primary-01"] + }] + }, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": { + "qualification_split": "holdout", + "fallback_expectation": "forbidden", + "root_predicate": "bound_id", + "terminal_predicate": "bound_id", + "edge_kinds": ["Traverse"], + "direction": "inbound", + "relationship_kind_count": 1, + "fixture_tier": "normal", + "expected_state_class": "inbound_predecessor_full_depth_fanin_191", + "result_cardinality_class": "singleton", + "min_depth": 1, + "max_depth": 64, + "path_materialization_required": true + }, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "path", "inbound", "hidden-fan-in", "holdout", "sp-i1-inbound-v1-holdout"] + }, + { + "name": "GSP-I1-V1-HOLDOUT-D32-FI191-disconnected", + "dataset": "generated_shortest_paths_v2_d32_o0_r11_fo0_fi191_l21_k0_t0_w0_x13_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN p", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-disconnected-end"}, + "expected": {"row_count": 0, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": { + "qualification_split": "holdout", + "fallback_expectation": "forbidden", + "root_predicate": "bound_id", + "terminal_predicate": "bound_id", + "edge_kinds": ["Traverse"], + "direction": "inbound", + "relationship_kind_count": 1, + "fixture_tier": "normal", + "expected_state_class": "inbound_predecessor_disconnected_fanin_191", + "result_cardinality_class": "empty", + "min_depth": 1, + "max_depth": 64, + "path_materialization_required": true + }, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "path", "inbound", "hidden-fan-in", "disconnected", "max-miss", "holdout", "sp-i1-inbound-v1-holdout"] + } + ] +} diff --git a/benchmark/testdata/scale/cases/hops.json b/benchmark/testdata/scale/cases/hops.json new file mode 100644 index 00000000..2835c2ac --- /dev/null +++ b/benchmark/testdata/scale/cases/hops.json @@ -0,0 +1,92 @@ +{ + "cases": [ + { + "name": "HOP-01_dense_outbound_bound_anchor", + "dataset": "generated_hops", + "category": "standalone_one_hop", + "cypher": "MATCH (s)-[r:HopKind01]->(e) WHERE id(s) = $anchor RETURN r, e", + "node_params": {"anchor": "hop-out-root"}, + "expected": {"row_count": 128}, + "observes": {"paths": false, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_start_id", "edge_kinds": ["HopKind01"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["HOP-01", "outbound", "dense", "full-direction"] + }, + { + "name": "HOP-02_dense_inbound_bound_anchor", + "dataset": "generated_hops", + "category": "standalone_one_hop", + "cypher": "MATCH (s)-[r:HopKind01]->(e) WHERE id(e) = $anchor RETURN r, s", + "node_params": {"anchor": "hop-in-root"}, + "expected": {"row_count": 128}, + "observes": {"paths": false, "nodes": true, "relationships": true, "properties": true}, + "shape": {"terminal_predicate": "bound_end_id", "edge_kinds": ["HopKind01"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["HOP-02", "inbound", "dense", "full-direction"] + }, + { + "name": "HOP-03_dense_thirty_kind_outbound_anchor", + "dataset": "generated_hops", + "category": "standalone_one_hop", + "cypher": "MATCH (s)-[r:HopKind01|HopKind02|HopKind03|HopKind04|HopKind05|HopKind06|HopKind07|HopKind08|HopKind09|HopKind10|HopKind11|HopKind12|HopKind13|HopKind14|HopKind15|HopKind16|HopKind17|HopKind18|HopKind19|HopKind20|HopKind21|HopKind22|HopKind23|HopKind24|HopKind25|HopKind26|HopKind27|HopKind28|HopKind29|HopKind30]->(e) WHERE id(s) = $anchor RETURN r, e", + "node_params": {"anchor": "hop-kind-root"}, + "expected": {"row_count": 128}, + "observes": {"paths": false, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_start_id", "edge_kinds": ["HopKind01", "HopKind02", "HopKind03", "HopKind04", "HopKind05", "HopKind06", "HopKind07", "HopKind08", "HopKind09", "HopKind10", "HopKind11", "HopKind12", "HopKind13", "HopKind14", "HopKind15", "HopKind16", "HopKind17", "HopKind18", "HopKind19", "HopKind20", "HopKind21", "HopKind22", "HopKind23", "HopKind24", "HopKind25", "HopKind26", "HopKind27", "HopKind28", "HopKind29", "HopKind30"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["HOP-03", "outbound", "dense", "30-kinds"] + }, + { + "name": "HOP-04_dense_opposite_endpoint_kind_disjunction", + "dataset": "generated_hops", + "category": "standalone_one_hop", + "cypher": "MATCH (s)-[r:HopTypedEdge]->(e) WHERE id(s) = $anchor AND (e:HopEndA OR e:HopEndB) RETURN r, e", + "node_params": {"anchor": "hop-out-root"}, + "expected": {"row_count": 128}, + "observes": {"paths": false, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_start_id", "terminal_predicate": "endpoint_kind_disjunction", "edge_kinds": ["HopTypedEdge"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["HOP-04", "dense", "endpoint-kinds", "multi-kind-nodes"] + }, + { + "name": "HOP-05_thousand_endpoint_IDs_with_sparse_matches", + "dataset": "generated_hops", + "category": "standalone_one_hop", + "cypher": "MATCH (s)-[r:HopIDEdge]->(e) WHERE id(s) = $anchor AND id(e) IN $end_ids RETURN r, e", + "node_params": {"anchor": "hop-out-root"}, + "generated_node_list_params": {"end_ids": {"prefix": "hop-id-target", "count": 1000}}, + "expected": {"row_count": 128}, + "observes": {"paths": false, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_start_id", "terminal_predicate": "large_end_id_list", "edge_kinds": ["HopIDEdge"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["HOP-05", "1000-list", "endpoint-ids", "sparse-match"] + }, + { + "name": "HOP-07_nested_branch_selectivity", + "dataset": "generated_hops", + "category": "standalone_one_hop", + "cypher": "MATCH (s)-[r:HopNestedEdge]->(e:HopTemplate) WHERE id(s) = $anchor AND ((e.requiresmanagerapproval = false AND e.schemaversion > 1 AND e.authorizedsignatures = 0 AND e.authenticationenabled = true) OR (e.requiresmanagerapproval = false AND e.schemaversion = 1 AND e.authenticationenabled = true)) RETURN r, e", + "node_params": {"anchor": "hop-out-root"}, + "expected": {"row_count": 64}, + "observes": {"paths": false, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_start_id", "terminal_predicate": "nested_property_disjunction", "edge_kinds": ["HopNestedEdge"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["HOP-07", "nested-or", "branch-local", "selectivity"] + }, + { + "name": "HOP-09_dense_two_sided_ID_sets", + "dataset": "generated_hops", + "category": "standalone_one_hop", + "cypher": "MATCH (s)-[r:HopSetEdge]->(e) WHERE id(s) IN $start_ids AND id(e) IN $end_ids RETURN r, e", + "generated_node_list_params": { + "start_ids": {"prefix": "hop-set-start", "count": 32}, + "end_ids": {"prefix": "hop-set-end", "count": 32} + }, + "expected": {"row_count": 1024}, + "observes": {"paths": false, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "start_id_list", "terminal_predicate": "end_id_list", "edge_kinds": ["HopSetEdge"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["HOP-09", "dense", "two-sided-ids", "32x32"] + } + ] +} diff --git a/benchmark/testdata/scale/cases/reconciliation.json b/benchmark/testdata/scale/cases/reconciliation.json new file mode 100644 index 00000000..c45bafb2 --- /dev/null +++ b/benchmark/testdata/scale/cases/reconciliation.json @@ -0,0 +1,150 @@ +{ + "cases": [ + { + "name": "REC-01_inbound_30_kind_delete", + "dataset": "generated_reconciliation", + "category": "reconciliation_mutation", + "cypher": "MATCH ()-[r:RecKind01|RecKind02|RecKind03|RecKind04|RecKind05|RecKind06|RecKind07|RecKind08|RecKind09|RecKind10|RecKind11|RecKind12|RecKind13|RecKind14|RecKind15|RecKind16|RecKind17|RecKind18|RecKind19|RecKind20|RecKind21|RecKind22|RecKind23|RecKind24|RecKind25|RecKind26|RecKind27|RecKind28|RecKind29|RecKind30]->(e:ADEntity) WHERE e.objectid = $object_id DELETE r", + "params": {"object_id": "rec-in"}, + "expected": {"result_kind": "mutation"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": { + "terminal_predicate": "typed_endpoint_property", + "edge_kinds": ["RecKind01", "RecKind02", "RecKind03", "RecKind04", "RecKind05", "RecKind06", "RecKind07", "RecKind08", "RecKind09", "RecKind10", "RecKind11", "RecKind12", "RecKind13", "RecKind14", "RecKind15", "RecKind16", "RecKind17", "RecKind18", "RecKind19", "RecKind20", "RecKind21", "RecKind22", "RecKind23", "RecKind24", "RecKind25", "RecKind26", "RecKind27", "RecKind28", "RecKind29", "RecKind30"], + "path_materialization_required": false + }, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["REC-01", "mutation", "inbound", "30-kinds"], + "write_scenario": { + "selection_cypher": "MATCH ()-[r:RecKind01|RecKind02|RecKind03|RecKind04|RecKind05|RecKind06|RecKind07|RecKind08|RecKind09|RecKind10|RecKind11|RecKind12|RecKind13|RecKind14|RecKind15|RecKind16|RecKind17|RecKind18|RecKind19|RecKind20|RecKind21|RecKind22|RecKind23|RecKind24|RecKind25|RecKind26|RecKind27|RecKind28|RecKind29|RecKind30]->(e:ADEntity) WHERE e.objectid = $object_id RETURN id(r)", + "params": {"object_id": "rec-in"}, + "affected_entity": "relationship", + "expected_matched": 2, + "expected_affected": 2, + "post_state": [ + {"name": "target relationships deleted", "cypher": "MATCH ()-[r]->(e:ADEntity) WHERE e.objectid = $object_id RETURN count(r)", "params": {"object_id": "rec-in"}, "expected": {"scalar_int": 0}}, + {"name": "wrong endpoint survivor", "cypher": "MATCH ()-[r:RecKind02]->(e:ADEntity) WHERE e.objectid = $object_id RETURN count(r)", "params": {"object_id": "survivor"}, "expected": {"scalar_int": 1}} + ] + } + }, + { + "name": "REC-02_outbound_30_kind_delete", + "dataset": "generated_reconciliation", + "category": "reconciliation_mutation", + "cypher": "MATCH (s:ADEntity)-[r:RecKind01|RecKind02|RecKind03|RecKind04|RecKind05|RecKind06|RecKind07|RecKind08|RecKind09|RecKind10|RecKind11|RecKind12|RecKind13|RecKind14|RecKind15|RecKind16|RecKind17|RecKind18|RecKind19|RecKind20|RecKind21|RecKind22|RecKind23|RecKind24|RecKind25|RecKind26|RecKind27|RecKind28|RecKind29|RecKind30]->() WHERE s.objectid = $object_id DELETE r", + "params": {"object_id": "rec-out"}, + "expected": {"result_kind": "mutation"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": { + "root_predicate": "typed_endpoint_property", + "edge_kinds": ["RecKind01", "RecKind02", "RecKind03", "RecKind04", "RecKind05", "RecKind06", "RecKind07", "RecKind08", "RecKind09", "RecKind10", "RecKind11", "RecKind12", "RecKind13", "RecKind14", "RecKind15", "RecKind16", "RecKind17", "RecKind18", "RecKind19", "RecKind20", "RecKind21", "RecKind22", "RecKind23", "RecKind24", "RecKind25", "RecKind26", "RecKind27", "RecKind28", "RecKind29", "RecKind30"], + "path_materialization_required": false + }, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["REC-02", "mutation", "outbound", "30-kinds"], + "write_scenario": { + "selection_cypher": "MATCH (s:ADEntity)-[r:RecKind01|RecKind02|RecKind03|RecKind04|RecKind05|RecKind06|RecKind07|RecKind08|RecKind09|RecKind10|RecKind11|RecKind12|RecKind13|RecKind14|RecKind15|RecKind16|RecKind17|RecKind18|RecKind19|RecKind20|RecKind21|RecKind22|RecKind23|RecKind24|RecKind25|RecKind26|RecKind27|RecKind28|RecKind29|RecKind30]->() WHERE s.objectid = $object_id RETURN id(r)", + "params": {"object_id": "rec-out"}, + "affected_entity": "relationship", + "expected_matched": 2, + "expected_affected": 2, + "post_state": [ + {"name": "target relationships deleted", "cypher": "MATCH (s:ADEntity)-[r]->() WHERE s.objectid = $object_id RETURN count(r)", "params": {"object_id": "rec-out"}, "expected": {"scalar_int": 0}}, + {"name": "wrong start survivor", "cypher": "MATCH (s:Source)-[r:RecKind02]->() RETURN count(r)", "expected": {"scalar_int": 1}} + ] + } + }, + { + "name": "REC-04_large_high_match_object_id_list_delete", + "dataset": "generated_reconciliation", + "category": "reconciliation_mutation", + "cypher": "MATCH ()-[r:ADReconcile]->(e:ADEntity) WHERE e.objectid IN $object_ids DELETE r", + "params": {"object_ids": {"$type": "string_list", "prefix": "missing-high", "count": 2000, "include": ["rec-list"]}}, + "expected": {"result_kind": "mutation"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"terminal_predicate": "large_property_list", "edge_kinds": ["ADReconcile"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["REC-04", "mutation", "large-list", "high-match"], + "write_scenario": { + "selection_cypher": "MATCH ()-[r:ADReconcile]->(e:ADEntity) WHERE e.objectid IN $object_ids RETURN id(r)", + "params": {"object_ids": {"$type": "string_list", "prefix": "missing-high", "count": 2000, "include": ["rec-list"]}}, + "affected_entity": "relationship", + "expected_matched": 2, + "expected_affected": 2, + "post_state": [ + {"name": "selected list relationships deleted", "cypher": "MATCH ()-[r:ADReconcile]->(e:ADEntity) WHERE e.objectid = $object_id RETURN count(r)", "params": {"object_id": "rec-list"}, "expected": {"scalar_int": 0}}, + {"name": "unrelated relationship survives", "cypher": "MATCH ()-[r:Survivor]->() RETURN count(r)", "expected": {"scalar_int": 1}} + ] + } + }, + { + "name": "REC-04_thousand_item_no_match_delete", + "dataset": "generated_reconciliation", + "category": "reconciliation_mutation", + "cypher": "MATCH ()-[r:ADReconcile]->(e:ADEntity) WHERE e.objectid IN $object_ids DELETE r", + "params": {"object_ids": {"$type": "string_list", "prefix": "missing-only", "count": 1000}}, + "expected": {"result_kind": "mutation"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"terminal_predicate": "large_property_list", "edge_kinds": ["ADReconcile"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["REC-04", "mutation", "1000-list", "no-match"], + "write_scenario": { + "selection_cypher": "MATCH ()-[r:ADReconcile]->(e:ADEntity) WHERE e.objectid IN $object_ids RETURN id(r)", + "params": {"object_ids": {"$type": "string_list", "prefix": "missing-only", "count": 1000}}, + "affected_entity": "relationship", + "expected_matched": 0, + "expected_affected": 0, + "post_state": [ + {"name": "all list relationships survive", "cypher": "MATCH ()-[r:ADReconcile]->() RETURN count(r)", "expected": {"scalar_int": 2}} + ] + } + }, + { + "name": "REC-06_large_endpoint_id_list_delete", + "dataset": "generated_reconciliation", + "category": "reconciliation_mutation", + "cypher": "MATCH ()-[r:DelegatedEnrollmentAgent]->(e:CertTemplate) WHERE id(e) IN $template_ids DELETE r", + "generated_node_list_params": {"template_ids": {"prefix": "scale-template", "count": 2000, "include": ["template"]}}, + "expected": {"result_kind": "mutation"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"terminal_predicate": "large_id_list", "edge_kinds": ["DelegatedEnrollmentAgent"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["REC-06", "mutation", "large-id-list"], + "write_scenario": { + "selection_cypher": "MATCH ()-[r:DelegatedEnrollmentAgent]->(e:CertTemplate) WHERE id(e) IN $template_ids RETURN id(r)", + "generated_node_list_params": {"template_ids": {"prefix": "scale-template", "count": 2000, "include": ["template"]}}, + "affected_entity": "relationship", + "expected_matched": 2, + "expected_affected": 2, + "post_state": [ + {"name": "delegations deleted", "cypher": "MATCH ()-[r:DelegatedEnrollmentAgent]->() RETURN count(r)", "expected": {"scalar_int": 0}}, + {"name": "unrelated relationship survives", "cypher": "MATCH ()-[r:Survivor]->() RETURN count(r)", "expected": {"scalar_int": 1}} + ] + } + }, + { + "name": "REC-08_large_list_high_degree_detach_delete", + "dataset": "generated_reconciliation", + "category": "reconciliation_mutation", + "cypher": "MATCH (n:ADEntity) WHERE n.objectid IN $object_ids DETACH DELETE n", + "params": {"object_ids": {"$type": "string_list", "prefix": "missing-node", "count": 2000, "include": ["delete-target"]}}, + "expected": {"result_kind": "mutation"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "large_property_list", "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["REC-08", "mutation", "large-list", "high-degree", "detach-delete"], + "write_scenario": { + "selection_cypher": "MATCH (n:ADEntity) WHERE n.objectid IN $object_ids RETURN id(n)", + "params": {"object_ids": {"$type": "string_list", "prefix": "missing-node", "count": 2000, "include": ["delete-target"]}}, + "affected_entity": "node", + "expected_matched": 1, + "expected_affected": 1, + "post_state": [ + {"name": "target node deleted", "cypher": "MATCH (n:ADEntity) WHERE n.objectid = $object_id RETURN count(n)", "params": {"object_id": "delete-target"}, "expected": {"scalar_int": 0}}, + {"name": "all incident relationships cascaded", "cypher": "MATCH ()-[r:Incident]->() RETURN count(r)", "expected": {"scalar_int": 0}}, + {"name": "decoy node survives", "cypher": "MATCH (n:ADEntity) WHERE n.objectid = $object_id RETURN count(n)", "params": {"object_id": "survivor"}, "expected": {"scalar_int": 1}} + ] + } + } + ] +} diff --git a/benchmark/testdata/scale/cases/scans_lookups.json b/benchmark/testdata/scale/cases/scans_lookups.json new file mode 100644 index 00000000..b3ae498e --- /dev/null +++ b/benchmark/testdata/scale/cases/scans_lookups.json @@ -0,0 +1,214 @@ +{ + "cases": [ + { + "name": "SCAN-01_dense_base_endpoint_relationship_ID_scan", + "dataset": "generated_scan_lookups", + "category": "relationship_scans", + "cypher": "MATCH (s)-[r:ScanPostProcessed]->(e) WHERE s:ADBase AND e:AZBase RETURN id(r)", + "expected": {"row_count": 128, "result_kind": "id_set"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "start_base_kind", "terminal_predicate": "end_base_kind", "edge_kinds": ["ScanPostProcessed"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["SCAN-01", "dense", "relationship-id", "projection-id-only"] + }, + { + "name": "SCAN-02_dense_non_Meta_relationship_hydration", + "dataset": "generated_scan_lookups", + "category": "relationship_scans", + "cypher": "MATCH (s)-[r:TrackerA|TrackerB]->(e) WHERE NOT (s:Meta OR s:MetaDetail) AND NOT (e:Meta OR e:MetaDetail) RETURN r", + "expected": {"row_count": 256}, + "observes": {"paths": false, "nodes": false, "relationships": true, "properties": true}, + "shape": {"root_predicate": "excluded_start_kinds", "terminal_predicate": "excluded_end_kinds", "edge_kinds": ["TrackerA", "TrackerB"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["SCAN-02", "dense", "full-relationship", "projection-full-hydration"] + }, + { + "name": "SCAN-03_present_lastseen_selectivity", + "dataset": "generated_scan_lookups", + "category": "relationship_scans", + "cypher": "MATCH (s)-[r:MigratedEdge]->(e) WHERE NOT (s:Meta OR s:MetaDetail) AND r.lastseen IS NOT NULL AND NOT (e:Meta OR e:MetaDetail) RETURN id(r)", + "expected": {"row_count": 64, "result_kind": "id_set"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "excluded_start_kinds", "terminal_predicate": "relationship_property_presence", "edge_kinds": ["MigratedEdge"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["SCAN-03", "null-missing", "selective-property"] + }, + { + "name": "SCAN-04_dense_raw_ownership_hydration", + "dataset": "generated_scan_lookups", + "category": "relationship_scans", + "cypher": "MATCH (s:Entity)-[r:OwnsRaw]->() RETURN r", + "expected": {"row_count": 128}, + "observes": {"paths": false, "nodes": false, "relationships": true, "properties": true}, + "shape": {"root_predicate": "start_entity_kind", "edge_kinds": ["OwnsRaw"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["SCAN-04", "dense", "full-relationship"] + }, + { + "name": "SCAN-05_nine_kind_bound_end_inbound_scan", + "dataset": "generated_scan_lookups", + "category": "relationship_scans", + "cypher": "MATCH (s:Entity)-[r:ScanEdge01|ScanEdge02|ScanEdge03|ScanEdge04|ScanEdge05|ScanEdge06|ScanEdge07|ScanEdge08|ScanEdge09]->(e) WHERE id(e) = $target RETURN r, s", + "node_params": {"target": "scan-nine-kind-target"}, + "expected": {"row_count": 128}, + "observes": {"paths": false, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "start_entity_kind", "terminal_predicate": "bound_end_id", "edge_kinds": ["ScanEdge01", "ScanEdge02", "ScanEdge03", "ScanEdge04", "ScanEdge05", "ScanEdge06", "ScanEdge07", "ScanEdge08", "ScanEdge09"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["SCAN-05", "nine-kinds", "dense-inbound", "full-direction"] + }, + { + "name": "SCAN-06_dense_shallow_IDs_and_kind_projection", + "dataset": "generated_scan_lookups", + "category": "relationship_scans", + "cypher": "MATCH (s)-[r:LocalToComputer]->(e:Computer) RETURN id(s), id(r), type(r), id(e)", + "expected": {"row_count": 256, "result_kind": "shallow_ids_kind"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"terminal_predicate": "typed_end", "edge_kinds": ["LocalToComputer"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["SCAN-06", "dense", "shallow-projection", "projection-shallow-ids-kind"] + }, + { + "name": "SCAN-07_dense_directed_ID_pairs", + "dataset": "generated_scan_lookups", + "category": "relationship_scans", + "cypher": "MATCH (s)-[r:MemberOf|MemberOfLocalGroup]->(e) RETURN id(s), id(e)", + "expected": {"row_count": 256, "result_kind": "id_set"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"edge_kinds": ["MemberOf", "MemberOfLocalGroup"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["SCAN-07", "dense", "directed-pairs", "duplicate-endpoints"] + }, + { + "name": "SCAN-08_thousand_victim_IDs_scenario_A", + "dataset": "generated_scan_lookups", + "category": "relationship_scans", + "cypher": "MATCH (s)-[r:GenericAll|GenericWrite|Owns|WriteOwner|WriteDACL|WritePublicInformation]->(e) WHERE (s:Group OR s:User OR s:Computer) AND id(e) IN $victims RETURN id(s)", + "generated_node_list_params": {"victims": {"prefix": "scan-victim", "count": 1000}}, + "expected": {"row_count": 128, "result_kind": "id_set"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "three_start_kinds", "terminal_predicate": "large_end_id_list", "edge_kinds": ["GenericAll", "GenericWrite", "Owns", "WriteOwner", "WriteDACL", "WritePublicInformation"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["SCAN-08", "1000-list", "scenario-a", "dense"] + }, + { + "name": "SCAN-08_thousand_victim_IDs_scenario_B", + "dataset": "generated_scan_lookups", + "category": "relationship_scans", + "cypher": "MATCH (s)-[r:GenericAll|GenericWrite|Owns|WriteOwner|WriteDACL]->(e:Computer) WHERE (s:Group OR s:User OR s:Computer) AND id(e) IN $victims RETURN id(s)", + "generated_node_list_params": {"victims": {"prefix": "scan-victim", "count": 1000}}, + "expected": {"row_count": 64, "result_kind": "id_set"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "three_start_kinds", "terminal_predicate": "typed_large_end_id_list", "edge_kinds": ["GenericAll", "GenericWrite", "Owns", "WriteOwner", "WriteDACL"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["SCAN-08", "1000-list", "scenario-b", "selective-end-kind"] + }, + { + "name": "LOOKUP-02_repeated_exact_objectid_lookup", + "dataset": "generated_scan_lookups", + "category": "lookups", + "cypher": "MATCH (n:Computer) WHERE n.objectid = $objectid RETURN id(n)", + "params": {"objectid": "S-1-5-21-scale"}, + "expected": {"row_count": 128, "result_kind": "id_set"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "exact_objectid", "terminal_predicate": "node_kind", "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["LOOKUP-02", "exact-property", "multiple-hit"] + }, + { + "name": "LOOKUP-04_suffix_kind_and_domain_filter", + "dataset": "generated_scan_lookups", + "category": "lookups", + "cypher": "MATCH (n:Group) WHERE n.objectid ENDS WITH $suffix AND n.domainsid = $domain RETURN id(n)", + "params": {"suffix": "-512", "domain": "S-1-5-21"}, + "expected": {"row_count": 64, "result_kind": "id_set"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "suffix_and_equality", "terminal_predicate": "node_kind", "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["LOOKUP-04", "suffix", "selectivity"] + }, + { + "name": "LOOKUP-05_repeated_case_insensitive_prefix", + "dataset": "generated_scan_lookups", + "category": "lookups", + "cypher": "MATCH (n:Group) WHERE toLower(n.name) STARTS WITH $prefix RETURN id(n)", + "params": {"prefix": "remote desktop users"}, + "expected": {"row_count": 128, "result_kind": "id_set"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "case_insensitive_prefix", "terminal_predicate": "node_kind", "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["LOOKUP-05", "case-insensitive", "repeated"] + }, + { + "name": "LOOKUP-09_thousand_ID_full_node_hydration", + "dataset": "generated_scan_lookups", + "category": "lookups", + "cypher": "MATCH (n) WHERE id(n) IN $ids RETURN n", + "generated_node_list_params": {"ids": {"prefix": "lookup-id-target", "count": 1000}}, + "expected": {"row_count": 1000}, + "observes": {"paths": false, "nodes": true, "relationships": false, "properties": true}, + "shape": {"root_predicate": "large_node_id_list", "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["LOOKUP-09", "1000-list", "dense", "full-node", "projection-full-hydration"] + }, + { + "name": "LOOKUP-11_tenant_adjacency_thousand_property_list", + "dataset": "generated_scan_lookups", + "category": "lookups", + "cypher": "MATCH (s)-[:Contains]->(e:AZRole) WHERE id(s) = $tenant AND e.roletemplateid IN $roles RETURN e", + "node_params": {"tenant": "lookup-tenant"}, + "params": {"roles": {"$type": "string_list", "prefix": "role-template", "count": 1000}}, + "expected": {"row_count": 1000}, + "observes": {"paths": false, "nodes": true, "relationships": false, "properties": true}, + "shape": {"root_predicate": "bound_tenant", "terminal_predicate": "large_endpoint_property_list", "edge_kinds": ["Contains"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["LOOKUP-11", "1000-list", "tenant-adjacency", "full-node"] + }, + { + "name": "LOOKUP-13_dense_suffix_bound_endpoint", + "dataset": "generated_scan_lookups", + "category": "lookups", + "cypher": "MATCH (s)-[:LocalToComputer]->(e) WHERE s.objectid ENDS WITH $suffix AND id(e) = $target RETURN s", + "params": {"suffix": "-555"}, + "node_params": {"target": "lookup-local-target"}, + "expected": {"row_count": 128}, + "observes": {"paths": false, "nodes": true, "relationships": false, "properties": true}, + "shape": {"root_predicate": "start_property_suffix", "terminal_predicate": "bound_end_id", "edge_kinds": ["LocalToComputer"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["LOOKUP-13", "dense-inbound", "suffix", "bound-end"] + }, + { + "name": "LOOKUP-15_all_node_count", + "dataset": "generated_scan_lookups", + "category": "counts", + "cypher": "MATCH (n) RETURN count(n)", + "expected": {"row_count": 1, "scalar_int": 3774, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["LOOKUP-15", "count", "dense-graph"] + }, + { + "name": "LOOKUP-15_all_relationship_count", + "dataset": "generated_scan_lookups", + "category": "counts", + "cypher": "MATCH ()-[r]->() RETURN count(r)", + "expected": {"row_count": 1, "scalar_int": 2408, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["LOOKUP-15", "count", "dense-graph"] + }, + { + "name": "LOOKUP-16_typed_four_property_NTLM_filter", + "dataset": "generated_scan_lookups", + "category": "lookups", + "cypher": "MATCH (n:Computer) WHERE n.domainsid = $domain AND n.isdc = true AND n.ldapavailable = true AND n.ldapsigning = false RETURN id(n)", + "params": {"domain": "S-1-5-21"}, + "expected": {"row_count": 128, "result_kind": "id_set"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "four_property_equalities", "terminal_predicate": "node_kind", "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["LOOKUP-16", "typed", "four-properties", "dense"] + } + ] +} diff --git a/benchmark/testdata/scale/cases/shortest_paths.json b/benchmark/testdata/scale/cases/shortest_paths.json index b9539b36..9902a8c1 100644 --- a/benchmark/testdata/scale/cases/shortest_paths.json +++ b/benchmark/testdata/scale/cases/shortest_paths.json @@ -39,7 +39,13 @@ }, "expected": { "row_count": 1, - "result_kind": "path_set" + "result_kind": "path_set", + "path_rows": [ + { + "nodes": ["n1", "n2", "n3"], + "relationship_kinds": ["EdgeKind1", "EdgeKind2"] + } + ] }, "observes": { "paths": true, @@ -58,4 +64,3 @@ } ] } - diff --git a/benchmark/testdata/scale/cases/traversal.json b/benchmark/testdata/scale/cases/traversal.json index 2bab928d..86cf247c 100644 --- a/benchmark/testdata/scale/cases/traversal.json +++ b/benchmark/testdata/scale/cases/traversal.json @@ -83,16 +83,22 @@ "tags": ["path-materialization"] }, { - "name": "adcs_p1_endpoint_ids", - "dataset": "adcs_fanout", - "category": "bloodhound_search", - "cypher": "MATCH (n:Group) WHERE n.objectid = $objectid MATCH (n)-[:MemberOf*0..]->()-[:Enroll]->(ca:EnterpriseCA)-[:TrustedForNTAuth]->(:NTAuthStore)-[:NTAuthStoreFor]->(d:Domain) RETURN id(ca), id(d)", + "name": "fixed_suffix_expansion_endpoint_ids", + "dataset": "fixed_suffix_expansion_fanout", + "category": "fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", "params": { - "objectid": "S-1-5-21-2643190041-1319121918-239771340-513" + "root_key": "fixed-suffix-fanout-root" }, "expected": { "row_count": 4, - "result_kind": "id_rows" + "result_kind": "id_rows", + "id_rows": [ + ["fse-head", "fse-terminal"], + ["fse-head", "fse-terminal"], + ["fse-head", "fse-terminal"], + ["fse-head", "fse-terminal"] + ] }, "observes": { "paths": false, @@ -103,24 +109,43 @@ "shape": { "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", - "edge_kinds": ["MemberOf", "Enroll", "TrustedForNTAuth", "NTAuthStoreFor"], + "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, + "max_depth": 16, "path_materialization_required": false }, "candidate_modes": ["postgres_sql", "local_traversal", "neo4j"], - "tags": ["bloodhound", "adcs", "id-only", "local-traversal-candidate"] + "tags": ["fixed-suffix-expansion", "fanout", "id-only", "local-traversal-candidate"] }, { - "name": "adcs_p1_path_observed", - "dataset": "adcs_fanout", - "category": "bloodhound_search", - "cypher": "MATCH (n:Group) WHERE n.objectid = $objectid MATCH p = (n)-[:MemberOf*0..]->()-[:Enroll]->(ca:EnterpriseCA)-[:TrustedForNTAuth]->(:NTAuthStore)-[:NTAuthStoreFor]->(d:Domain) RETURN p", + "name": "GFSE-BOUNDARY-cyclic-relationship-distinct-bag", + "dataset": "fixed_suffix_expansion_adversarial", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..3]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "suffix-overflow-adversarial-root"}, + "expected": {"row_count": 68, "result_kind": "id_rows"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "holdout", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 3, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["normal-tier", "fixed-suffix-expansion-boundary", "suffix-overflow", "cycle", "relationship-distinct", "physical-bag-multiplicity", "noncanonical-logical-ids", "holdout"] + }, + { + "name": "fixed_suffix_expansion_path_observed", + "dataset": "fixed_suffix_expansion_fanout", + "category": "fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(:SuffixTerminal) RETURN p", "params": { - "objectid": "S-1-5-21-2643190041-1319121918-239771340-513" + "root_key": "fixed-suffix-fanout-root" }, "expected": { "row_count": 4, - "result_kind": "path_set" + "result_kind": "path_set", + "path_rows": [ + {"nodes": ["fse-root", "fse-head", "fse-middle", "fse-terminal"], "relationship_kinds": ["EnterSuffix", "ContinueSuffix", "CompleteSuffix"]}, + {"nodes": ["fse-root", "fse-expansion-a", "fse-head", "fse-middle", "fse-terminal"], "relationship_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"]}, + {"nodes": ["fse-root", "fse-expansion-b", "fse-head", "fse-middle", "fse-terminal"], "relationship_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"]}, + {"nodes": ["fse-root", "fse-expansion-b", "fse-expansion-c", "fse-head", "fse-middle", "fse-terminal"], "relationship_kinds": ["Expand", "Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"]} + ] }, "observes": { "paths": true, @@ -131,13 +156,13 @@ "shape": { "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", - "edge_kinds": ["MemberOf", "Enroll", "TrustedForNTAuth", "NTAuthStoreFor"], + "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, + "max_depth": 16, "path_materialization_required": true }, "candidate_modes": ["postgres_sql", "neo4j"], - "tags": ["bloodhound", "adcs", "path-materialization"] + "tags": ["fixed-suffix-expansion", "fanout", "path-materialization"] } ] } - diff --git a/benchmark/testdata/scale/cases/trust_pruning.json b/benchmark/testdata/scale/cases/trust_pruning.json new file mode 100644 index 00000000..bcf9cda8 --- /dev/null +++ b/benchmark/testdata/scale/cases/trust_pruning.json @@ -0,0 +1,133 @@ +{ + "cases": [ + { + "name": "TRUST-01_dense_same_forest_relationship_ids", + "dataset": "generated_trust_pruning", + "category": "trust_reconciliation", + "cypher": "MATCH (s:Domain)-[r:SameForestTrust]->(e:Domain) WHERE datetime(r.lastseen) < datetime(s.lastcollected) OR datetime(r.lastseen) < datetime(e.lastcollected) RETURN id(r)", + "expected": {"row_count": 128}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "typed_temporal_disjunction", "edge_kinds": ["SameForestTrust"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["TRUST-01", "dense", "relationship-ids", "temporal-or"] + }, + { + "name": "TRUST-02_dense_cross_forest_relationship_hydration", + "dataset": "generated_trust_pruning", + "category": "trust_reconciliation", + "cypher": "MATCH (s:Domain)-[r:CrossForestTrust]->(e:Domain) WHERE datetime(r.lastseen) < datetime(s.lastcollected) OR datetime(r.lastseen) < datetime(e.lastcollected) RETURN r", + "expected": {"row_count": 128}, + "observes": {"paths": false, "nodes": false, "relationships": true, "properties": true}, + "shape": {"root_predicate": "typed_temporal_disjunction", "edge_kinds": ["CrossForestTrust"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["TRUST-02", "dense", "relationship-hydration", "temporal-or"] + }, + { + "name": "TRUST-03_directional_branch_local_kinds", + "dataset": "generated_trust_pruning", + "category": "trust_reconciliation", + "cypher": "MATCH (s:Domain)-[r]->(e:Domain) WHERE (id(s) = $forward_start AND id(e) = $forward_end AND r:AbuseTGTDelegation) OR (id(s) = $forward_end AND id(e) = $forward_start AND r:SpoofSIDHistory) RETURN id(r)", + "node_params": {"forward_start": "trust-late-a", "forward_end": "trust-late-b"}, + "expected": {"row_count": 2}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "directional_id_disjunction", "edge_kinds": ["AbuseTGTDelegation", "SpoofSIDHistory"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["TRUST-03", "directional", "branch-local-kind", "relationship-ids"] + }, + { + "name": "PRUNE-01_dense_old_relationship_selection", + "dataset": "generated_trust_pruning", + "category": "pruning_selection", + "cypher": "MATCH ()-[r]->() WHERE NOT (r:HasSession OR r:MetaIncludes) AND datetime(r.lastseen) < datetime($threshold) RETURN id(r)", + "params": {"threshold": "2026-01-03T00:00:00Z"}, + "expected": {"row_count": 128}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "kind_negation_and_temporal", "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["PRUNE-01", "dense", "kind-negation", "relationship-ids"] + }, + { + "name": "PRUNE-02_dense_missing_or_old_session_selection", + "dataset": "generated_trust_pruning", + "category": "pruning_selection", + "cypher": "MATCH ()-[r:HasSession]->() WHERE r.lastseen IS NULL OR datetime(r.lastseen) < datetime($threshold) RETURN id(r)", + "params": {"threshold": "2026-01-03T00:00:00Z"}, + "expected": {"row_count": 256}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "missing_or_temporal", "edge_kinds": ["HasSession"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["PRUNE-02", "dense", "missing-property", "relationship-ids"] + }, + { + "name": "PRUNE-03_dense_missing_or_old_node_selection", + "dataset": "generated_trust_pruning", + "category": "pruning_selection", + "cypher": "MATCH (n:PruneCandidate) WHERE NOT n:Domain AND (n.lastseen IS NULL OR datetime(n.lastseen) < datetime($threshold)) RETURN id(n)", + "params": {"threshold": "2026-01-03T00:00:00Z"}, + "expected": {"row_count": 262}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "kind_negation_and_missing_or_temporal", "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["PRUNE-03", "dense", "missing-property", "node-ids"] + }, + { + "name": "PRUNE-04_dense_orphan_sid_selection", + "dataset": "generated_trust_pruning", + "category": "pruning_selection", + "cypher": "MATCH (n) WHERE NOT n:Domain AND n.name IS NULL AND n.objectid STARTS WITH $sid_prefix RETURN id(n)", + "params": {"sid_prefix": "S-1-5"}, + "expected": {"row_count": 130}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "kind_negation_missing_name_prefix", "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["PRUNE-04", "dense", "missing-property", "prefix", "node-ids"] + }, + { + "name": "PRUNE-05_dense_relationship_batch_delete_equivalent", + "dataset": "generated_trust_pruning", + "category": "pruning_mutation", + "cypher": "MATCH ()-[r:PruneBatch]->() WHERE r.remove = $flag DELETE r", + "params": {"flag": true}, + "expected": {"result_kind": "mutation"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "relationship_property", "edge_kinds": ["PruneBatch"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["PRUNE-05", "mutation", "direct-batch-equivalent"], + "write_scenario": { + "selection_cypher": "MATCH ()-[r:PruneBatch]->() WHERE r.remove = $flag RETURN id(r)", + "params": {"flag": true}, + "affected_entity": "relationship", + "expected_matched": 128, + "expected_affected": 128, + "post_state": [ + {"name": "selected relationships deleted", "cypher": "MATCH ()-[r:PruneBatch]->() WHERE r.remove = $flag RETURN count(r)", "params": {"flag": true}, "expected": {"scalar_int": 0}}, + {"name": "survivor relationship remains", "cypher": "MATCH ()-[r:PruneBatchSurvivor]->() RETURN count(r)", "expected": {"scalar_int": 1}} + ] + } + }, + { + "name": "PRUNE-06_high_degree_node_batch_delete_equivalent", + "dataset": "generated_trust_pruning", + "category": "pruning_mutation", + "cypher": "MATCH (n:PruneBatchNode) WHERE n.remove = $flag DETACH DELETE n", + "params": {"flag": true}, + "expected": {"result_kind": "mutation"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "node_property_high_degree", "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["PRUNE-06", "mutation", "high-degree", "cascade", "direct-batch-equivalent"], + "write_scenario": { + "selection_cypher": "MATCH (n:PruneBatchNode) WHERE n.remove = $flag RETURN id(n)", + "params": {"flag": true}, + "affected_entity": "node", + "expected_matched": 65, + "expected_affected": 65, + "post_state": [ + {"name": "selected nodes deleted", "cypher": "MATCH (n:PruneBatchNode) WHERE n.remove = $flag RETURN count(n)", "params": {"flag": true}, "expected": {"scalar_int": 0}}, + {"name": "high degree incident relationships cascaded", "cypher": "MATCH ()-[r:PruneIncident]->() RETURN count(r)", "expected": {"scalar_int": 0}}, + {"name": "unselected batch nodes survive", "cypher": "MATCH (n:PruneBatchNode) RETURN count(n)", "expected": {"scalar_int": 65}} + ] + } + } + ] +} diff --git a/cmd/benchmark/README.md b/cmd/benchmark/README.md index 1cba4e28..8076d916 100644 --- a/cmd/benchmark/README.md +++ b/cmd/benchmark/README.md @@ -5,14 +5,14 @@ Runs query scenarios against a real database and outputs markdown, JSON, or benc ## Usage ```bash -# Default datasets (base, adcs_fanout, and traversal_shapes) +# Default datasets (base, fixed_suffix_expansion_fanout, and traversal_shapes) go run ./cmd/benchmark -connection "postgresql://dawgs:dawgs@localhost:5432/dawgs" # Traversal shape dataset only go run ./cmd/benchmark -connection "..." -dataset traversal_shapes -# ADCS fanout dataset with PostgreSQL EXPLAIN diagnostics -go run ./cmd/benchmark -connection "..." -dataset adcs_fanout -json-output report.json -explain +# Fixed-suffix expansion fanout dataset with PostgreSQL EXPLAIN diagnostics +go run ./cmd/benchmark -connection "..." -dataset fixed_suffix_expansion_fanout -json-output report.json -explain # Local dataset (not committed to repo) go run ./cmd/benchmark -connection "..." -dataset local/phantom @@ -50,9 +50,11 @@ go run ./cmd/benchmark -connection "..." -format benchfmt -output report.bench Use `-format benchfmt` when comparing scenario timings with `benchstat`. Each timed scenario iteration is emitted as a separate `ns/op` sample so two benchmark runs can be compared directly. -The committed default datasets are `base`, `adcs_fanout`, and `traversal_shapes`. `traversal_shapes` covers chain, -fanout, bounded cycle, disconnected, edge-kind-selective, and multi-path shortest-path traversal shapes. Scenarios with -declared expected row counts fail before reporting timings if a query returns the wrong result shape. +The committed default datasets are `base`, `fixed_suffix_expansion_fanout`, and +`traversal_shapes`. `traversal_shapes` covers chain, fanout, bounded cycle, +disconnected, edge-kind-selective, and multi-path shortest-path traversal +shapes. Scenarios with declared expected row counts fail before reporting +timings if a query returns the wrong result shape. ## Example: Neo4j on local/phantom diff --git a/cmd/benchmark/report_test.go b/cmd/benchmark/report_test.go index 92b460bb..867ab45f 100644 --- a/cmd/benchmark/report_test.go +++ b/cmd/benchmark/report_test.go @@ -27,6 +27,7 @@ import ( "github.com/stretchr/testify/require" ) +// TestWriteJSONEmitsBaselineFriendlyReport verifies that JSON retains row diagnostics, timing values, SQL, and every optimizer decision needed for baseline comparisons. func TestWriteJSONEmitsBaselineFriendlyReport(t *testing.T) { var ( distinctRows = int64(2) @@ -105,6 +106,7 @@ func TestWriteJSONEmitsBaselineFriendlyReport(t *testing.T) { } } +// TestWriteMarkdownIncludesDiagnosticColumns verifies that Markdown exposes distinct and duplicate row counts alongside timing and plan-capture status. func TestWriteMarkdownIncludesDiagnosticColumns(t *testing.T) { var ( distinctRows = int64(2) @@ -115,8 +117,8 @@ func TestWriteMarkdownIncludesDiagnosticColumns(t *testing.T) { Date: "2026-05-14", Iterations: 3, Results: []Result{{ - Section: "ADCS Fanout", - Dataset: "adcs_fanout", + Section: "Fixed Suffix Expansion Fanout", + Dataset: "fixed_suffix_expansion_fanout", Label: "combined", RowCount: 2, DistinctRowCount: &distinctRows, @@ -138,22 +140,25 @@ func TestWriteMarkdownIncludesDiagnosticColumns(t *testing.T) { for _, expected := range []string{ "Distinct Rows", "Duplicate Rows", - "| ADCS Fanout / combined | adcs_fanout | 2 | 2 | 0 | 10.0ms | 20.0ms | 30.0ms | captured |", + "| Fixed Suffix Expansion Fanout / combined | fixed_suffix_expansion_fanout | 2 | 2 | 0 | 10.0ms | 20.0ms | 30.0ms | captured |", } { require.Contains(t, text, expected) } } +// TestValidateIterationsRejectsZero verifies that benchmark execution requires at least one measured iteration. func TestValidateIterationsRejectsZero(t *testing.T) { require.Error(t, validateIterations(0)) require.NoError(t, validateIterations(1)) } +// TestWriteReportRejectsUnknownFormat verifies that report dispatch fails instead of silently choosing a serializer for an unsupported format. func TestWriteReportRejectsUnknownFormat(t *testing.T) { err := writeReport(&bytes.Buffer{}, Report{}, "xml") require.ErrorContains(t, err, "unsupported output format") } +// TestWriteJSON verifies that JSON dispatch preserves the selected driver and emits raw duration samples in nanoseconds. func TestWriteJSON(t *testing.T) { report := testReport() var out bytes.Buffer @@ -165,6 +170,7 @@ func TestWriteJSON(t *testing.T) { require.Contains(t, out.String(), `1000000`) } +// TestWriteBenchfmt verifies that benchfmt output carries platform metadata, a stable benchmark name, and one ns/op observation per sample. func TestWriteBenchfmt(t *testing.T) { report := testReport() var out bytes.Buffer @@ -180,6 +186,7 @@ func TestWriteBenchfmt(t *testing.T) { require.Contains(t, output, "\t1\t2000000 ns/op") } +// TestSanitizeBenchNamePart verifies that benchmark labels normalize whitespace and arrows without destroying hierarchy separators, and that empty labels receive a fallback. func TestSanitizeBenchNamePart(t *testing.T) { require.Equal(t, "Shortest_Paths", sanitizeBenchNamePart("Shortest Paths")) require.Equal(t, "n1_-_n3", sanitizeBenchNamePart("n1 -> n3")) @@ -187,6 +194,7 @@ func TestSanitizeBenchNamePart(t *testing.T) { require.Equal(t, "unknown", sanitizeBenchNamePart("")) } +// TestWriteMarkdownOmitsSamples verifies that Markdown reports aggregate timings without leaking the raw nanosecond sample series. func TestWriteMarkdownOmitsSamples(t *testing.T) { report := testReport() var out bytes.Buffer @@ -198,6 +206,7 @@ func TestWriteMarkdownOmitsSamples(t *testing.T) { require.False(t, strings.Contains(output, "1000000")) } +// testReport returns a representative report used by serializer tests. func testReport() Report { return Report{ Driver: "pg", diff --git a/cmd/benchmark/scenarios.go b/cmd/benchmark/scenarios.go index ef819e62..c364701d 100644 --- a/cmd/benchmark/scenarios.go +++ b/cmd/benchmark/scenarios.go @@ -25,35 +25,45 @@ import ( "github.com/specterops/dawgs/opengraph" ) -// Measurement captures the warm-up result shape for a benchmark scenario. +// Measurement pairs a benchmark duration with the number of rows observed. type Measurement struct { - RowCount int64 - DistinctRowCount *int64 + // RowCount records the number of rows produced. + RowCount int64 + // DistinctRowCount records unique rows returned by the benchmark scenario. + DistinctRowCount *int64 + // DuplicateRowCount records repeated rows retained by the benchmark scenario. DuplicateRowCount *int64 } -// Scenario defines a single benchmark query to run against a loaded dataset. +// Scenario defines one query, its parameters, and expected cardinality. type Scenario struct { - Section string // grouping key in the report (e.g. "Match Nodes") - Dataset string - Label string // human-readable row label + // Section groups baseline rows under a Markdown summary section. + Section string // grouping key in the report (e.g. "Match Nodes") + // Dataset identifies the fixture dataset. + Dataset string + // Label provides the benchfmt label for the benchmark scenario. + Label string // human-readable row label + // ExpectedRows sets the row count required for a scenario to succeed. ExpectedRows *int64 - Cypher string - Query func(tx graph.Transaction) (Measurement, error) + // Cypher contains the Cypher statement under test. + Cypher string + // Query executes the scenario in a transaction and returns its duration and observed row count. + Query func(tx graph.Transaction) (Measurement, error) } +// traversalShapesDataset is the fixture key shared by traversal-shape scenario selection and dataset loading. const traversalShapesDataset = "traversal_shapes" // defaultDatasets is the set of datasets committed to the repo. -var defaultDatasets = []string{"base", "adcs_fanout", traversalShapesDataset} +var defaultDatasets = []string{"base", "fixed_suffix_expansion_fanout", traversalShapesDataset} // scenariosForDataset returns all benchmark scenarios for a given dataset and its loaded ID map. func scenariosForDataset(dataset string, idMap opengraph.IDMap) []Scenario { switch dataset { case "base": return baseScenarios(idMap) - case "adcs_fanout": - return adcsFanoutScenarios() + case "fixed_suffix_expansion_fanout": + return fixedSuffixExpansionFanoutScenarios() case traversalShapesDataset: return traversalShapesScenarios(idMap) case "local/phantom": @@ -63,18 +73,22 @@ func scenariosForDataset(dataset string, idMap opengraph.IDMap) []Scenario { } } +// expectRows returns an addressable row expectation so zero expected rows remains distinguishable from an unspecified expectation. func expectRows(rows int64) *int64 { return &rows } +// countNodes measures the transaction-visible node cardinality for dataset sanity benchmarks. func countNodes(tx graph.Transaction) (int64, error) { return tx.Nodes().Count() } +// countEdges measures the transaction-visible relationship cardinality for dataset sanity benchmarks. func countEdges(tx graph.Transaction) (int64, error) { return tx.Relationships().Count() } +// cypherQuery adapts Cypher text into a benchmark callback that drains the result and records returned row count. func cypherQuery(cypher string) func(tx graph.Transaction) (Measurement, error) { return func(tx graph.Transaction) (Measurement, error) { result := tx.Query(cypher, nil) @@ -89,6 +103,7 @@ func cypherQuery(cypher string) func(tx graph.Transaction) (Measurement, error) } } +// countQuery adapts a cardinality callback into a benchmark Measurement while preserving the callback error. func countQuery(query func(tx graph.Transaction) (int64, error)) func(tx graph.Transaction) (Measurement, error) { return func(tx graph.Transaction) (Measurement, error) { rowCount, err := query(tx) @@ -100,6 +115,7 @@ func countQuery(query func(tx graph.Transaction) (int64, error)) func(tx graph.T } } +// cypherScenario builds a row-counting Scenario from its corpus identity and Cypher text. func cypherScenario(section, dataset, label, cypher string) Scenario { return Scenario{ Section: section, @@ -110,6 +126,7 @@ func cypherScenario(section, dataset, label, cypher string) Scenario { } } +// cypherPathScenario builds a Scenario that validates and counts path-valued columns while consuming results. func cypherPathScenario(section, dataset, label, cypher string, pathColumns int) Scenario { return Scenario{ Section: section, @@ -120,11 +137,13 @@ func cypherPathScenario(section, dataset, label, cypher string, pathColumns int) } } +// expectScenarioRows returns scenario with an explicit correctness expectation attached. func expectScenarioRows(scenario Scenario, rows int64) Scenario { scenario.ExpectedRows = expectRows(rows) return scenario } +// cypherPathQuery adapts Cypher text into a benchmark callback that validates path columns and hashes their node/edge identities while draining rows. func cypherPathQuery(cypher string, pathColumns int) func(tx graph.Transaction) (Measurement, error) { return func(tx graph.Transaction) (Measurement, error) { result := tx.Query(cypher, nil) @@ -171,6 +190,7 @@ func cypherPathQuery(cypher string, pathColumns int) func(tx graph.Transaction) } } +// pathRowKey serializes path node and edge IDs into an unambiguous key used to prevent result materialization from being optimized away. func pathRowKey(paths []graph.Path) string { var builder strings.Builder @@ -213,6 +233,7 @@ func pathRowKey(paths []graph.Path) string { // --- Base dataset scenarios (n1 -> n2 -> n3) --- +// baseScenarios defines cardinality, lookup, and one-hop checks for the three-node base fixture. func baseScenarios(idMap opengraph.IDMap) []Scenario { ds := "base" return []Scenario{ @@ -235,46 +256,49 @@ func baseScenarios(idMap opengraph.IDMap) []Scenario { } } -const adcsFanoutObjectID = "S-1-5-21-2643190041-1319121918-239771340-513" +// fixedSuffixFanoutRootKey identifies the fanout fixture root whose generated ID is injected into fixed-suffix scenarios. +const fixedSuffixFanoutRootKey = "fixed-suffix-fanout-root" -func adcsFanoutScenarios() []Scenario { +// fixedSuffixExpansionFanoutScenarios exercises bounded reverse-suffix expansion at increasing depths and with path projection enabled. +func fixedSuffixExpansionFanoutScenarios() []Scenario { var ( - ds = "adcs_fanout" + ds = "fixed_suffix_expansion_fanout" p1 = fmt.Sprintf(` - MATCH (n:Group) WHERE n.objectid = '%s' - MATCH p1 = (n)-[:MemberOf*0..]->()-[:Enroll]->(ca:EnterpriseCA)-[:TrustedForNTAuth]->(:NTAuthStore)-[:NTAuthStoreFor]->(d:Domain) + MATCH (root:ExpansionRoot) WHERE root.root_key = '%s' + MATCH p1 = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN p1 - `, adcsFanoutObjectID) + `, fixedSuffixFanoutRootKey) p2 = fmt.Sprintf(` - MATCH (n:Group) WHERE n.objectid = '%s' - MATCH p2 = (n)-[:MemberOf*0..]->()-[:GenericAll|Enroll|AllExtendedRights]->(ct:CertTemplate)-[:PublishedTo]->(ca:EnterpriseCA)-[:IssuedSignedBy|EnterpriseCAFor*1..]->(:RootCA)-[:RootCAFor]->(d:Domain) - WHERE ct.authenticationenabled = true - AND ct.requiresmanagerapproval = false - AND ct.enrolleesuppliessubject = true - AND (ct.schemaversion = 1 OR ct.authorizedsignatures = 0) + MATCH (root:ExpansionRoot) WHERE root.root_key = '%s' + MATCH p2 = (root)-[:Expand*0..16]->()-[:OptionA|OptionB|OptionC]->(predicate:PredicateNode)-[:JoinSuffix]->(head:SuffixHead)-[:HeadToBridge|HeadToAlternateBridge*1..16]->(:BridgeNode)-[:ReachTerminal]->(terminal:SuffixTerminal) + WHERE predicate.eligible = true + AND predicate.requires_review = false + AND predicate.allows_direct = true + AND (predicate.version = 1 OR predicate.required_approvals = 0) RETURN p2 - `, adcsFanoutObjectID) + `, fixedSuffixFanoutRootKey) combinedMatch = fmt.Sprintf(` - MATCH (n:Group) WHERE n.objectid = '%s' - MATCH p1 = (n)-[:MemberOf*0..]->()-[:Enroll]->(ca:EnterpriseCA)-[:TrustedForNTAuth]->(:NTAuthStore)-[:NTAuthStoreFor]->(d:Domain) - MATCH p2 = (n)-[:MemberOf*0..]->()-[:GenericAll|Enroll|AllExtendedRights]->(ct:CertTemplate)-[:PublishedTo]->(ca)-[:IssuedSignedBy|EnterpriseCAFor*1..]->(:RootCA)-[:RootCAFor]->(d) - WHERE ct.authenticationenabled = true - AND ct.requiresmanagerapproval = false - AND ct.enrolleesuppliessubject = true - AND (ct.schemaversion = 1 OR ct.authorizedsignatures = 0) - `, adcsFanoutObjectID) + MATCH (root:ExpansionRoot) WHERE root.root_key = '%s' + MATCH p1 = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) + MATCH p2 = (root)-[:Expand*0..16]->()-[:OptionA|OptionB|OptionC]->(predicate:PredicateNode)-[:JoinSuffix]->(head)-[:HeadToBridge|HeadToAlternateBridge*1..16]->(:BridgeNode)-[:ReachTerminal]->(terminal) + WHERE predicate.eligible = true + AND predicate.requires_review = false + AND predicate.allows_direct = true + AND (predicate.version = 1 OR predicate.required_approvals = 0) + `, fixedSuffixFanoutRootKey) ) return []Scenario{ - cypherPathScenario("ADCS Fanout", ds, "p1 only", p1, 1), - cypherPathScenario("ADCS Fanout", ds, "p2 only", p2, 1), - cypherPathScenario("ADCS Fanout", ds, "combined", combinedMatch+"RETURN p1,p2", 2), - cypherScenario("ADCS Fanout", ds, "combined endpoints", combinedMatch+"RETURN id(ca), id(d), id(ct)"), + cypherPathScenario("Fixed Suffix Expansion Fanout", ds, "p1 only", p1, 1), + cypherPathScenario("Fixed Suffix Expansion Fanout", ds, "p2 only", p2, 1), + cypherPathScenario("Fixed Suffix Expansion Fanout", ds, "combined", combinedMatch+"RETURN p1,p2", 2), + cypherScenario("Fixed Suffix Expansion Fanout", ds, "combined endpoints", combinedMatch+"RETURN id(head), id(terminal), id(predicate)"), } } // --- Traversal shape scenarios --- +// traversalShapesScenarios covers single-hop, bounded variable-length, shortest-path, and repeated-edge traversal forms over the shared fixture. func traversalShapesScenarios(idMap opengraph.IDMap) []Scenario { ds := traversalShapesDataset return []Scenario{ @@ -333,6 +357,7 @@ func traversalShapesScenarios(idMap opengraph.IDMap) []Scenario { // --- Phantom scenarios (hardcoded node IDs from the dataset) --- +// phantomScenarios preserves legacy benchmark cases that intentionally address the phantom fixture by its stable generated IDs. func phantomScenarios(idMap opengraph.IDMap) []Scenario { var ( ds = "local/phantom" diff --git a/cmd/benchmark/scenarios_test.go b/cmd/benchmark/scenarios_test.go index 0206c020..9789bdf9 100644 --- a/cmd/benchmark/scenarios_test.go +++ b/cmd/benchmark/scenarios_test.go @@ -25,6 +25,7 @@ import ( "github.com/stretchr/testify/require" ) +// TestBaseScenariosDeclareExpectedRows verifies the canonical row-count contract for every query family in the base fixture. func TestBaseScenariosDeclareExpectedRows(t *testing.T) { scenarios := baseScenarios(opengraph.IDMap{ "n1": graph.ID(1), @@ -41,6 +42,7 @@ func TestBaseScenariosDeclareExpectedRows(t *testing.T) { requireExpectedRows(t, scenarios, "Filter By Kind", "NodeKind2", 2) } +// TestTraversalShapesDatasetIsValid verifies that the checked-in traversal fixture parses and retains its expected 45-node, 41-edge topology. func TestTraversalShapesDatasetIsValid(t *testing.T) { file, err := os.Open("../../integration/testdata/traversal_shapes.json") require.NoError(t, err) @@ -52,6 +54,7 @@ func TestTraversalShapesDatasetIsValid(t *testing.T) { require.Len(t, doc.Graph.Edges, 41) } +// TestTraversalShapesScenariosDeclareExpectedRows verifies the expected cardinalities for depth, fanout, cycle, dead-end, kind-filtered, and shortest-path fixture cases. func TestTraversalShapesScenariosDeclareExpectedRows(t *testing.T) { scenarios := traversalShapesScenarios(traversalShapesIDMap()) @@ -71,11 +74,13 @@ func TestTraversalShapesScenariosDeclareExpectedRows(t *testing.T) { requireExpectedRows(t, scenarios, "Shortest Paths", "disconnected", 0) } +// TestDefaultDatasetsIncludeTraversalShapes verifies that ordinary benchmark runs include both traversal-shape and fixed-suffix fanout coverage. func TestDefaultDatasetsIncludeTraversalShapes(t *testing.T) { require.Contains(t, defaultDatasets, traversalShapesDataset) - require.Contains(t, defaultDatasets, "adcs_fanout") + require.Contains(t, defaultDatasets, "fixed_suffix_expansion_fanout") } +// TestValidateScenarioRows verifies that observed cardinality must match the scenario contract and that failures identify the scenario and both counts. func TestValidateScenarioRows(t *testing.T) { scenario := Scenario{ Section: "Traversal", @@ -88,6 +93,7 @@ func TestValidateScenarioRows(t *testing.T) { require.ErrorContains(t, validateScenarioRows(scenario, 1), "Traversal/n1 on base expected 2 rows, got 1") } +// traversalShapesIDMap resolves traversal-shape fixture node keys to database identifiers. func traversalShapesIDMap() opengraph.IDMap { ids := []string{ "c0", "c10", @@ -106,6 +112,7 @@ func traversalShapesIDMap() opengraph.IDMap { return idMap } +// requireExpectedRows locates a scenario by section and label and asserts its declared cardinality. func requireExpectedRows(t *testing.T, scenarios []Scenario, section, label string, expectedRows int64) { t.Helper() diff --git a/cmd/graphbench/README.md b/cmd/graphbench/README.md index ac530326..2cccdad3 100644 --- a/cmd/graphbench/README.md +++ b/cmd/graphbench/README.md @@ -5,12 +5,14 @@ It is meant for runtime gap accounting: query duration, returned row counts, PostgreSQL plan details, Neo4j plan operators, fallback reasons, and comparison summaries. -The current execution modes are: +The implemented execution modes are: - `postgres_sql`: runs DAWGS' PostgreSQL SQL translation against a PostgreSQL database. -- `local_traversal`: records explicit `not_implemented` placeholders until the local traversal executor lands. - `neo4j`: runs the same corpus against Neo4j through the DAWGS Neo4j backend. +`local_traversal` is accepted only to record explicit `not_implemented` diagnostic placeholders. It is excluded from +performance gates and must not be presented as an executor result. + Apache AGE is not an execution mode in this harness yet. AGE behavior can be captured in corpus `reference_design` notes so DAWGS can use it as design input without treating it as a direct benchmark comparison. @@ -20,11 +22,42 @@ without treating it as a direct benchmark comparison. The command loads cases from `benchmark/testdata/scale` by default and imports the fixture datasets from `integration/testdata`. +Corpus parameters support fixture IDs through `node_params` and +`node_list_params`. Tagged datetime values are decoded to `time.Time`, avoiding +lexical string comparisons in temporal cases. Mutating cases require an +explicit `write_scenario`; the runner checks matched and affected counts plus +post-state queries and rolls back warm-up, timed iterations, and PostgreSQL +plan capture. + +Read cases that return node IDs can declare `expected.id_rows` using fixture +node names. GraphBench reverse-maps backend-assigned IDs through the complete +dataset ID map and compares the rows as a multiset, preserving duplicates. + Connection strings can be supplied as flags or environment variables: - PostgreSQL: `-pg-connection`, `PG_CONNECTION_STRING`, `-connection`, or `CONNECTION_STRING`. - Neo4j: `-neo4j-connection`, `NEO4J_CONNECTION_STRING`, `-connection`, or `CONNECTION_STRING`. +Every output record includes the DAWGS source version plus source commit, +dirty-worktree hash (including untracked files), binary hash, sanitized invocation, +Go/OS/CPU/kernel/cgroup data, run UUID, arm/block/order timestamps, pool settings, +and declared memory ceilings. Use +`-dawgs-version` to override the auto-detected DAWGS version. +Portable bundle creation reconstructs that dirty-worktree hash from the exact +bundled binary patch plus sorted untracked path/content bytes and refuses a +run-environment mismatch. Verification repeats the reconstruction and rejects +malformed, duplicated, mismatched, or unchecksummed untracked entries/copies. + +GraphBench clears and reloads fixtures. A non-blocking local lock at +`.coverage/graphbench.lock` prevents overlapping processes; override it with +`-destructive-lock`. Runners on different hosts must use distinct disposable +databases because a filesystem lock cannot coordinate across machines. +Fixture-loading runs also require `DAWGS_INTEGRATION_ALLOW_DESTRUCTIVE=1` and +an exact credential-free target in `DAWGS_INTEGRATION_DISPOSABLE_TARGETS`, for +example `postgresql://localhost:65432/dawgs`. Non-mutating `-existing-graph` +runs do not require this acknowledgement; their PostgreSQL sessions remain +read-write so temporary workspace behavior matches production. + ## Examples Run only PostgreSQL SQL translation: @@ -38,11 +71,11 @@ go run ./cmd/graphbench \ -summary-json .coverage/graphbench-postgres.json ``` -Capture PostgreSQL, local traversal placeholders, and Neo4j in one report: +Capture PostgreSQL and Neo4j in one report: ```bash go run ./cmd/graphbench \ - -modes postgres_sql,local_traversal,neo4j \ + -modes postgres_sql,neo4j \ -pg-connection "$PG_CONNECTION_STRING" \ -neo4j-connection "$NEO4J_CONNECTION_STRING" \ -jsonl-output .coverage/graphbench.jsonl \ @@ -62,6 +95,1096 @@ go run ./cmd/graphbench \ -summary .coverage/graphbench.md ``` +Capture independent rounds with 30-50 warm observations each. The PostgreSQL +runner resets its one-connection pool before every case, records the first +query execution as `cold`, and keeps connection establishment outside that +sample. Use a distinct `-round` value for every independently reloaded run: +Even-numbered rounds reverse the requested backend order to alternate which +backend runs first. + +```bash +go run ./cmd/graphbench \ + -round 1 \ + -iterations 30 \ + -modes postgres_sql,neo4j \ + -pg-connection "$PG_CONNECTION_STRING" \ + -neo4j-connection "$NEO4J_CONNECTION_STRING" \ + -jsonl-output .coverage/graphbench-round-1.jsonl +``` + +Concatenate the JSONL rounds for each version, then run the executable +confidence gate: + +```bash +make perf_gate \ + PERF_BASELINE=.coverage/graphbench-baseline.jsonl \ + PERF_CANDIDATE=.coverage/graphbench-candidate.jsonl \ + PERF_GATE_AA=.coverage/perf-aa-resolution.json +``` + +The versioned gate report includes artifact and A/A-report SHA-256 checksums, +seeded 97.5% +bootstrap intervals over matched round medians, stratified p95 intervals once +each side has at least 150 samples, and candidate-minus-baseline p95 duration +intervals. Normal and envelope timing uses the greater of the matching host +A/A resolution and the 5%/100us minimum floors. Stress timing is descriptive; +stress correctness and the independent resource gate still apply. Matched +timing artifacts must carry complementary, round-balanced arm order, block, +run UUID, and warmup evidence. The version-controlled corpus `candidate_modes` declarations are the +required-key/status manifest: a missing or non-`ok` PostgreSQL record fails +instead of disappearing through intersection-only comparison. Neo4j records +must be present and `ok`, but Neo4j latency is informational and never fails a +CySQL performance gate. Missing/malformed host A/A, tier, pairing, selection, +round, or p95 evidence fails production promotion. Diagnostic comparisons may +omit promotion evidence but cannot emit a passing promotion result. +Prioritized traversal candidates additionally require nonempty, independently +passing training and frozen-holdout cases for every concrete runtime candidate +family; a holdout from ASP, another scheduler, or another observation boundary +cannot qualify an SP candidate. + +Predeclare cases expected to improve with `PERF_TARGETS` (or +`-gate-targets`). A target passes materiality when its median-ratio upper bound +is at most `0.95` or its median-saving lower bound is at least `100us`; both +defaults are configurable. Calculate host-specific A/A resolution from a +baseline artifact with: + +```bash +make perf_aa PERF_AA_ARTIFACT=.coverage/graphbench-aa.jsonl +``` + +The report accepts exactly two explicitly executed A/A arms sharing one run +UUID and SQL/workload identity. It requires complementary balanced order across +at least five independent rounds, ten samples per arm and round, and fingerprints +the host, reports p50/p95 ratio and absolute resolution, and keeps p99 +diagnostic until each arm has at least 10,000 samples. When append-safe capture +keeps the two arm labels in separate files, repeat `-aa-artifact` instead of +concatenating them outside GraphBench: + +```bash +graphbench \ + -aa-artifact .coverage/aa-a.jsonl \ + -aa-artifact .coverage/aa-b.jsonl \ + -aa-output .coverage/aa.json +``` + +### Targeted matched diagnostics + +`-cases` accepts exact, unambiguous case names. `-datasets`, `-categories`, and +`-tags` add exact selectors; values within one selector are alternatives and +different selector dimensions are intersected. Unknown, duplicate, ambiguous, +or empty selections fail before a fixture is changed. Filtered captures are +marked `diagnostic_only`, record both the requested and resolved selection and +the omitted declaration count, and are refused by the ordinary complete gate. +Selection-manifest schema v2 also records the count and digest of any protocol-protected +declarations removed from its runnable universe; those omissions do not by +themselves make an otherwise unfiltered run diagnostic-only. +Use `-diagnostic-gate` only to compare two artifacts with the same resolved +subset checksum. + +Configured `-warmup-iterations` run outside the recorded samples. The cold +diagnostic, exact preflight/postflight observations, and fixture reload/analyze +contract remain separate. A matched arm records `-arm`, `-arm-order`, `-block`, +`-round`, and a shared `-run-uuid`: + +```bash +go build -trimpath -o .coverage/confirm/bin/graphbench ./cmd/graphbench +.coverage/confirm/bin/graphbench \ + -modes postgres_sql \ + -cases 'LOOKUP-05_repeated_case_insensitive_prefix,GSP-D02-F016_distance' \ + -warmup-iterations 20 -iterations 50 -pool-size 1 \ + -arm candidate -arm-order 1 -block 1 -round 1 -run-uuid "$RUN_UUID" \ + -pg-connection "$PG_CONNECTION_STRING" \ + -bundle-dir .coverage/confirm/candidate-round-1 \ + -jsonl-output .coverage/confirm/candidate-round-1.jsonl +``` + +`-bundle-dir` retains the tracked patch, checksummed copies of untracked files, +`go.mod`/`go.sum`, the running executable, the complete sorted corpus +declaration and its independently recomputed identity, raw +JSONL, a sanitized manifest, and bundle checksums. It never records connection +strings or arbitrary environment variables. Add repeatable, stable-named +auxiliary evidence with `-bundle-evidence name=path`, for example +`-bundle-evidence host-aa=.coverage/host-aa.json` and +`-bundle-evidence plan-delta=.coverage/plan-delta.json`. Evidence names use only +lowercase letters, digits, `-`, and `_`; the bundle records each source digest +without retaining its host path. The destination must be new or empty so stale +payloads cannot enter its checksum inventory. + +Verify a portable bundle independently of database access. Verification rejects +missing, additional, symlinked, malformed, or checksum-mismatched payloads and +writes its report outside the bundle being checked: + +```bash +go run ./cmd/graphbench \ + -bundle-verify .coverage/confirm/candidate-round-1 \ + -bundle-verify-output .coverage/candidate-round-1-verification.json \ + -bundle-require-clean + +make perf_bundle_verify \ + PERF_BUNDLE_VERIFY_DIR=.coverage/confirm/candidate-round-1 \ + PERF_BUNDLE_REQUIRE_CLEAN=1 +``` + +Omit `-bundle-require-clean` for a diagnostic capture that deliberately carries +a source patch. Structural or checksum failures always produce a nonzero exit; +the optional clean-source policy additionally rejects any dirty capture. + +Compare matched arms with the matching checksummed host A/A report. Only a +same-executable block/reload A/A comparison may omit this input: + +```bash +make perf_confirm \ + PERF_LEFT=.coverage/confirm/predecessor.jsonl \ + PERF_RIGHT=.coverage/confirm/candidate.jsonl \ + PERF_CONFIRM_AA=.coverage/confirm/block-aa.json \ + PERF_CASES='LOOKUP-05_repeated_case_insensitive_prefix,GSP-D02-F016_distance' +``` + +The report emits paired relative and absolute p50/p95 intervals. It classifies +fresh p95 evidence as confirmed, cleared/non-inferior, inconclusive, or a +fingerprint mismatch using a minimum 5%/0.10 ms noise floor. Comparing two +captures of the same executable produces a `block_reload_aa` report; alternate +arm order across independently reloaded rounds. + +## Concurrency and PostgreSQL references + +Serial behavior remains the default (`-pool-size 1`). An opt-in concurrency +smoke retains a physical pool and records pool wait, transaction setup, +execute/decode/drain time, backend PID, cold/warm session classification, wall +time, and QPS: + +```bash +go run ./cmd/graphbench \ + -modes postgres_sql \ + -pg-connection "$PG_CONNECTION_STRING" \ + -pool-size 8 \ + -concurrency 1,8,16 \ + -iterations 30 \ + -session-memory-ceiling-bytes 67108864 \ + -pool-memory-ceiling-bytes 536870912 \ + -jsonl-output .coverage/graphbench-concurrency.jsonl +``` + +`-postgres-references` additionally captures an identical-SQL raw-pgx boundary +(pool wait, transaction, bind/prepare, first row, remaining decode, drain, and +allocations), a raw prepared round-trip, the C1 prepared round-trip, +endpoint validation, minimum graph-access ID floor, raw ordered-ID search, +path hydration from precomputed ordered edge IDs, and complete hand-written +PostgreSQL references for the active shortest-path and fixed-suffix expansion +targets. The main +case record remains the translated-CySQL boundary rather than a fixed ordinal +among the additive references. Component floors need not match the full query's +row count; complete references do. It also records +compile-stage timings and allocations. JSON/Markdown summaries include a +versioned exclusive-boundary cost table and its unexplained residual. The waterfall marks its translation +interval as overlapping optimization, so those fields must not be summed as an +additive attribution. + +Use `-postgres-reference-arms` to run only named tournament arms; it implies +`-postgres-references` and rejects unknown or duplicate names. Generated +fixed-suffix expansion cases expose `search_ordered_ids`, +`stepwise_forward_aa_ordered_ids`, `root_reuse_*`, `late_hydration_*`, +`factored_suffix_forward_*`, `suffix_seeded_reverse_*`, and +`backward_viability_forward_*` boundaries. Complete arms are exact-multiset +checked against the public CySQL observation. Ordered-ID arms retain +relationship IDs for trail uniqueness. Exactly three selected arms use a +six-round doubled Williams design that places every arm in every position twice +and balances every directed carryover pair twice. Exactly five arms use the +fixed ten-round Williams/carryover-balanced slot schedule; other arm counts +retain the historical alternating order: + +```text +0 1 4 2 3 +1 2 0 3 4 +2 3 1 4 0 +3 4 2 0 1 +4 0 3 1 2 +3 2 4 1 0 +4 3 0 2 1 +0 4 1 3 2 +1 0 2 4 3 +2 1 3 0 4 +``` + +The slots are the caller-selected arms, and rounds wrap after the tenth row. + +`-postgres-force-shortest-executor SP-S0` is the exact-incumbent control at the +same public distance or path boundary. It records selected/applied `SP-S0` and +executes the existing workspace harness, making containment regret and +candidate/reference comparisons explicit. + +`-postgres-force-shortest-executor SP-S0-DIRECT` is the tool-only direct-edge +preflight arm for structurally eligible bound-endpoint searches whose minimum +depth is one. A materialized indexed one-edge probe returns a valid singleton +witness immediately; a dependency-gated lateral branch invokes exact `SP-S0` +only when the probe is empty. Both branches share one SQL statement and +snapshot. Production `sp-static-v3` selection remains unchanged until the arm +passes exactness, zero-loop fallback, regret, resource, and concurrency gates. + +`-postgres-force-shortest-executor SP-S3-U-D` is a qualification-only seam for +eligible bounded singleton distance cases. It executes the repository-native +recursive AST directly, using compact `(next_id, depth)` state when both +endpoints are ID-only and retaining `(root_id, next_id, depth)` otherwise. It +reports the exact forced/applied target and rejects path-observed or otherwise +ineligible cases. It does not enable the executor in the public query API. + +`-postgres-force-shortest-executor SP-S3-U-E+MAT-M0` is the corresponding +qualification-only seam for eligible one-path observations. It emits +repository-native `(next_id, depth, edge_ids)` recursive state and hydrates the +ordered path directly from direction-specific edge endpoints. Distance-only, +directionless, correlated, optional, mutation, and other ineligible forms keep +the incumbent unless explicitly rejected by the tool request. Tool forcing +never broadens the structural correctness envelope. + +`-postgres-force-expansion-search EXPANSION-SUFFIX-SEEDED-REVERSE` is the +qualification-only seam for an eligible directed, bounded variable expansion +followed by exactly three fixed directed relationships. It emits the +repository-native suffix-seeded reverse recursive AST, preserves +relationship-trail uniqueness and exact suffix multiplicity, and supports +endpoint-ID and complete-path observations. The request fails closed when the +target is structurally ineligible or translation does not record the requested +strategy as applied. It is mutually exclusive with forced shortest execution. +Automatic suffix-seeded reverse dispatch remains disabled because query shape +does not bound suffix density or reverse fan-in. + +`-postgres-force-expansion-search EXPANSION-ENDPOINT-SEEDED-REVERSE` targets the production-qualified +fixed-prefix/terminal-expansion family. Its SQL has materialized 33-row endpoint and 4097-row reverse-state probes, +then mutually exclusive reverse and incumbent branches. Generated +`generated_endpoint_seeded_expansion_v1_d_e_q_w_o_x_m1_c_p

` fixtures independently vary +matching/other endpoints, productive/unproductive lanes, cycles, and payload. Edge multiplicity is fixed at one because +DAWGS storage uniquely keys edges by start, end, kind, and graph. Structured plan metrics +report probe rows, guard overflow, and whether the incumbent branch executed. + +Traversal telemetry is disabled by default. Opt in with +`-postgres-traversal-telemetry summary` or +`-postgres-traversal-telemetry diagnostic`; both modes require +`-pool-size 1` so the recorded backend identity cannot drift. Attachment runs +after the timed case, reference, raw-PGX, and concurrency blocks. Summary mode +uses only lightweight post-timing evidence and never performs the detailed +invocation-local replay. For a function-backed B arm whose outer plan cannot +prove its branch, it serializes `runtime_outcome_available=false` and leaves +runtime/applied/fallback facts unset. Diagnostic mode additionally retains the existing +`EXPLAIN (ANALYZE, TIMING OFF, FORMAT JSON)` plan replay. For SP-B1/B2 it also +replays the exact SQL in a separate Repeatable Read transaction on that same +physical connection, guarded by a unique invocation ID and the +`begin/read/clear_bidirectional_shortest_path_diagnostic_v1` session-local API; +ASP-B1/B2 uses the corresponding +`begin/read/clear_bidirectional_all_shortest_path_diagnostic_v1` API. +Cancellation and SQL errors roll the replay transaction back; replay duration +is never added to latency samples. + +An outer PostgreSQL `Function Scan` is not treated as internal traversal work. +SP/ASP B counters are retained only when the invocation ID, connection, +scheduler, caps, exactly-one singleton search call, level rows, and runtime +outcome all validate. A B candidate that +executes exact S4 fallback retains its measured candidate/fallback evidence but +is marked incomplete because nested S4 work is still opaque. Witness SP and all +ASP executions separately require complete hydration counters. Workspace-backed +B arms also require measured per-session and pool high-water bytes; declared +memory flags alone never qualify. These counters are not yet exposed, so those +records fail closed while retaining their validated search evidence. Other +function-backed SP/ASP arms are recorded as +`hidden_counters_unavailable`, never as zero work. The resource gate requires +`counter_status=complete` for candidate architectures even when no numeric cap +was declared. + +Traversal telemetry schema v2 gives guarded inline-predecessor evidence two non-interchangeable serialized +families. `ASP-I1-U-DAG+MAT-M0` emits `asp-i1-guarded-v1` and writes bounded +relation, output, and branch evidence under `diagnostic.counters.inline_asp`. +`SP-I1-C-WE+MAT-M0` emits the distinct +`sp-i1-canonical-guarded-v1` policy and writes the same-shaped evidence under +`diagnostic.counters.inline_shortest_path`; evidence from either namespace +cannot satisfy the other family. PostgreSQL's named candidate and fallback +marker CTEs must attribute exactly one arm, and the unselected output branch +must report zero rows. Parent-linked plan nodes also bind each branch body to +its direct inner executor; the selected executor must run and the unselected +executor must report zero loops. Canonical I1 reports `inline_canonical_witness` or +`inline_canonical_no_path` when its candidate marker executes, and +`exact_s4_fallback` with `SP-S4-C-WE+MAT-M0` when the fallback marker executes. +If any required named relation, marker, branch, or executor-loop counter is absent from the +plan replay, the diagnostic is `hidden_counters_unavailable`; absence is never +converted into a qualifying zero. +This adds fail-closed evidence for the default-off exact-query canary; it does +not change the automatic `sp-static-v5-contained` production selector. + +An emitted `orientation-probe-v1` policy requires orientation probes, selected +ordinary expansion, and hydration families. Its exact executed-candidate and +executed-incumbent marker rows must select one arm, the other must be zero, and +each named probe may execute at most once. Attribution uses only PostgreSQL's +single `Subplan Name: CTE ...` materialization body, never repeated consumer +CTE scans; the unselected traversal branch must also report zero loops. +Plan-derived partial evidence cannot qualify. +Telemetry attaches to every reference whose declared architecture is itself a +traversal or hydration boundary. Protocol, endpoint/root validation, and other +component probes remain intentionally unannotated; their missing attachment is +not missing traversal evidence. + +`-postgres-expansion-orientation-shadow` enables the tool-only +`orientation-probe-v1` shadow statement. It always executes the exact forward +incumbent and records the mutually exclusive SQL marker result separately as +`would_select_identity`; it never relabels that hypothetical choice as the +runtime or applied arm. A marker-first runtime receipt is emitted even when the +incumbent returns zero rows, and any cap+1 probe row is reflected in the shadow +overflow summary. The shadow flag is mutually exclusive with forced shortest- +path and forced expansion selectors. + +Build the matched selector-regret and probe-overhead report from separate +true-shadow, exact incumbent, and forced suffix-reverse artifacts plus the +host A/A calibration: + +```bash +go run ./cmd/graphbench \ + -orientation-shadow-artifact .coverage/orientation-shadow.jsonl \ + -orientation-incumbent-artifact .coverage/orientation-incumbent.jsonl \ + -orientation-reverse-artifact .coverage/orientation-reverse.jsonl \ + -orientation-aa .coverage/perf-aa-resolution.json \ + -orientation-output .coverage/orientation-selector.json \ + -orientation-protocol confirmation \ + -confidence-level 0.975 -seed 1 +``` + +The report requires exact matching observations, stable workload/SQL/binary +identities, one SQL-derived `would_select_identity`, and position-balanced +three-arm rounds. Selector regret must be within a `1.10` median-ratio upper +bound or the host A/A absolute floor. Shadow probe overhead must be within +`10%` or `100us`. Training records may inform the frozen selector; holdout +records are evaluation-only; diagnostic and legacy records are serialized but +excluded from qualification. Discovery uses 5-20 rounds, five warmups, and ten +samples per arm. Confirmation uses 10-20 rounds, 20 warmups, and 50 samples per +arm. + +`orientation-probe-v2` is a separate, immutable, tool-only experiment. It does +not reinterpret the v1 report or change the v1 exact-query production seam. +The v2 selector computes +`F2 = root_rows + maximum_depth * forward_degree_rows` and +`R2 = suffix_rows + boundary_rows + reverse_degree_rows`; it selects the exact +suffix-seeded reverse arm only when every cap+1 probe is complete and +`4 * R2 < 3 * F2`. Any probe or reverse-state overflow fails closed to the exact +forward arm. The checksum-bound v3 cohort has exactly eight training cases and +four holdouts. It independently varies maximum depth, fanout, reachable and +disconnected branches, reverse fan-in, suffix multiplicity, matching-root +multiplicity, zero depth, productive-boundary cycles and self-loops, payload, +and endpoint-ID versus complete-path observation. Holdouts use previously +unused depths 7, 11, 13, and 15 and must not be opened for threshold tuning. + +Capture the four artifacts with these exact arm labels. Every invocation also +requires `-postgres-repeatable-read`, `-postgres-traversal-telemetry summary` or +`diagnostic`, and `-pool-size 1`. + +| Artifact | Exact `-arm` label | Mode-specific flags | +| --- | --- | --- | +| Shadow | `shadow` | `-postgres-expansion-orientation-shadow -postgres-expansion-orientation-policy orientation-probe-v2` | +| Exact forward | `incumbent` | no orientation or forced-expansion flag | +| Exact reverse | `reverse` | `-postgres-force-expansion-search EXPANSION-SUFFIX-SEEDED-REVERSE` | +| Guarded selector | `guarded` | `-postgres-expansion-orientation-tournament -postgres-expansion-orientation-policy orientation-probe-v2` | + +Build GraphBench once from the clean source tree and invoke that exact binary +for every A/A, arm, and report command. Repeated `go run` builds do not prove a +single binary identity: + +```bash +CAPTURE=.coverage/orientation-v2-discovery +mkdir -p "$CAPTURE/bin" +go build -trimpath -o "$CAPTURE/bin/graphbench" ./cmd/graphbench +RUN_UUID="orientation-v2-discovery-$(git rev-parse HEAD)" +``` + +For example, the first shadow discovery round is captured with: + +```bash +"$CAPTURE/bin/graphbench" \ + -modes postgres_sql \ + -tags orientation-v2-training \ + -warmup-iterations 5 -iterations 10 -pool-size 1 \ + -round 1 -block 1 -run-uuid "$RUN_UUID" \ + -arm shadow -arm-order 1 \ + -postgres-repeatable-read \ + -postgres-traversal-telemetry diagnostic \ + -postgres-expansion-orientation-shadow \ + -postgres-expansion-orientation-policy orientation-probe-v2 \ + -jsonl-output "$CAPTURE/shadow.jsonl" -append-jsonl +``` + +Repeat the invocation for the other table rows and rotate `-arm-order` in each +subsequent round. `-run-uuid` is one series identity: reuse the same value across +all four arms and every appended round. Change `-round` and `-block`, but not the +UUID; append validation rejects a per-round UUID. Discovery selects only +`orientation-v2-training` and keeps the holdout timings closed. Its four +artifacts must contain exactly the canonical eight training cases, with no +holdout or diagnostic timing. After the formula is frozen, confirmation selects +`-tags orientation-v2-training,orientation-v2-holdout`, writes separate +confirmation artifacts containing exactly the canonical eight training plus +four holdout cases, and uses 20 warmups and 50 measured samples per arm and +round. + +Each matched round must give the four labels distinct `-arm-order` values from +1 through 4 and share the same nonzero `-block`, `-round`, and `-run-uuid`. +Rotate the positions across rounds so every arm occupies every position evenly; +the canonical four-round rotation is +`shadow/incumbent/reverse/guarded`, +`incumbent/reverse/guarded/shadow`, +`reverse/guarded/shadow/incumbent`, then +`guarded/shadow/incumbent/reverse`. The reporter rejects a position imbalance +greater than one, missing or extra cases, mismatched round sets, observation or +SQL drift, non-Repeatable-Read records, missing timed receipts on shadow or +guarded samples, and mixed source, dirty-diff, binary, corpus, host, or +PostgreSQL identities. +The shadow receipt branch is exactly `shadow_incumbent`. Guarded reverse +execution must report `suffix_seeded_reverse`; guarded forward selection and +overflow fallback both report `exact_forward_incumbent`, with +`fallback_executed=true` required only for overflow fallback. + +Capture the two A/A arms as separate append-safe exact-forward artifacts using +the same built binary, exact cohort tag, Repeatable Read, diagnostic traversal +telemetry, size-one pool, warmups, samples, and fixture reload protocol as the +incumbent arm. Use one A/A series UUID and alternate the two positions across +rounds. No orientation or forced-expansion flag is permitted. Then let +GraphBench validate the logical pair directly: + +```bash +"$CAPTURE/bin/graphbench" \ + -aa-artifact "$CAPTURE/aa-a.jsonl" \ + -aa-artifact "$CAPTURE/aa-b.jsonl" \ + -aa-output "$CAPTURE/aa.json" \ + -confidence-level 0.975 -seed 1 +``` + +Discovery is the only workflow that creates a freeze. Run it from a clean source +tree after capturing the exact canonical eight-case training artifacts and their +matching host A/A evidence. Both output flags are mandatory: the command writes +the training-only discovery report and a freeze manifest that binds its SHA-256 +together with the policy, formula, caps, source commit, clean dirty-diff, +binary, and canonical cohort declaration: + +```bash +"$CAPTURE/bin/graphbench" \ + -orientation-v2-shadow-artifact "$CAPTURE/shadow.jsonl" \ + -orientation-v2-incumbent-artifact "$CAPTURE/incumbent.jsonl" \ + -orientation-v2-reverse-artifact "$CAPTURE/reverse.jsonl" \ + -orientation-v2-guarded-artifact "$CAPTURE/guarded.jsonl" \ + -orientation-v2-aa "$CAPTURE/aa.json" \ + -orientation-v2-output "$CAPTURE/report.json" \ + -orientation-v2-freeze-output "$CAPTURE/freeze.json" \ + -orientation-v2-protocol discovery \ + -confidence-level 0.975 -seed 1 +``` + +Confirmation fails closed unless it receives that exact freeze manifest and +the discovery report whose digest the manifest binds. Its four timing artifacts +and matching host A/A report must cover exactly the canonical eight training and +four holdout cases: + +```bash +CONFIRMATION=.coverage/orientation-v2-confirmation +"$CAPTURE/bin/graphbench" \ + -orientation-v2-shadow-artifact "$CONFIRMATION/shadow.jsonl" \ + -orientation-v2-incumbent-artifact "$CONFIRMATION/incumbent.jsonl" \ + -orientation-v2-reverse-artifact "$CONFIRMATION/reverse.jsonl" \ + -orientation-v2-guarded-artifact "$CONFIRMATION/guarded.jsonl" \ + -orientation-v2-aa "$CONFIRMATION/aa.json" \ + -orientation-v2-freeze "$CAPTURE/freeze.json" \ + -orientation-v2-discovery-report "$CAPTURE/report.json" \ + -orientation-v2-output "$CONFIRMATION/report.json" \ + -orientation-v2-protocol confirmation \ + -confidence-level 0.975 -seed 1 +``` + +Every v2 A/A case carries separate checksums for its workload, the exact +PostgreSQL timing environment (including transaction isolation and normalized +ANALYZE state), and the exact validated fixture. Discovery and confirmation +reject missing or mismatched environment or fixture evidence. + +The forward-selected shadow/forward and guarded/selected overhead gates use a +`1.10` median-ratio upper bound or a `100us` absolute-gap ceiling. The +guarded/fastest regret gate uses the same ratio limit or the matching host A/A +absolute floor. Shadow overhead remains visible but is not +qualification-applicable when v2 selects reverse. Confirmation requires all +eight training and all four holdout cases to pass independently. No v2 +discovery or confirmation result has qualified yet; the flags and schema only +stage the experiment and do not authorize production rollout. + +The bounded same-statement fallback and keyset-continuation experiments are +retired. They are not exposed by GraphBench or production translation. Their +negative results remain under `docs/experiments`; the active `GFSE-BOUNDARY-*` +cases are optimization-neutral cardinality holdouts. + +Independent benchmark rounds can be accumulated with `-append-jsonl`. The +append path must be supplied with `-jsonl-output`; GraphBench rejects mismatched +run UUIDs, arms, binary/diff identities, and duplicate case rounds before +writing. This is the intended input shape for paired confirmation and the +round-stratified performance gate. + +Use `-reference-closure-artifact` with a capture containing the translated +raw-pgx boundary and one exact PostgreSQL full-comparator arm to generate a +seeded production/reference closure report. The report requires 10-20 matched +rounds, at least 20 untimed warmups and 50 measured samples per side in every +round, and exact public observations. It passes when the production/reference +median-ratio upper bound is at most 1.10 or the absolute median-gap interval is +within the greater of the case's within-session A/A resolution and +`-materiality-absolute` (100 microseconds by default). The report derives and +records A/A resolution independently for the production and reference raw +boundaries by splitting alternating samples within each round. Single selected +reference captures run production first in odd rounds and the reference first +in even rounds; the order is recorded on both boundaries and enforced by the +reporter: + +```bash +go run ./cmd/graphbench \ + -reference-closure-artifact .coverage/shortest-reference.jsonl \ + -reference-closure-arm s3_unidirectional_trail_cte \ + -reference-closure-output .coverage/shortest-reference-gate.json \ + -confidence-level 0.975 \ + -seed 1 +``` + +Fixed-suffix expansion JSON plans are retained in both text and structured +forms. Structured metrics include per-node planned/actual rows, loops, width, +timing, buffers, +relation/index identity, recursive rows, access-direction probe counts, and +hydration lookup loops. Derived fields state their provenance and do not present +fixture-derived per-depth counts as PostgreSQL measurements. Resource gate +version 1 applies the portable resource checks to the first upstream artifact +schema. + +The keyset-continuation v1 design and its GraphBench arm are retired. In the +10-round confirmation run, S513 had a 1.791 +median ratio (97.5% CI 1.752–1.875) and S600 had a 5.898 ratio (5.649–6.462) +against `complete_reference`. S511/S512 selected the existing bounded reverse +branch, so their improvements are not evidence for keyset continuation. The +resource gate passed without spill, local workspace, or WAL. See +`docs/experiments/guarded_suffix_keyset_continuation_v1.md` and its compact JSON +evidence. Generic `GFSE-BOUNDARY-*` holdouts preserve exact-limit, overflow, +path, multiplicity, and cyclic-trail coverage without retaining an executable +copy of the rejected arm. + +Supported generated singleton-shortest cases also run two additive comparators: +`s3_unidirectional_trail_cte` (legacy name +`complete_reference_s1_array_cte`) and `s3_bidirectional_trail_cte` (legacy name +`candidate_s2_bidirectional_cte`). New reference records declare a schema +version, architecture, implementation/state/observation shape, and semantic +validation level, raw-pgx timing boundary, normalized SQL fingerprint, and any +explicit A/A alias. A requested arm that is unavailable for a case fails the +run, and distinct architecture IDs with identical normalized SQL fail unless +the alias is declared. Full comparators are checked against untimed exact public +observations rather than row count alone. Distance S3-U uses node/depth frontier +state with no path or predecessor arrays. Historical readers preserve the old +labels in `legacy_name` while mapping them to `SP-S3-U-NE`/`SP-S3-B`. These remain +benchmark-only; S3-B is not evidence for the compact S2 architecture. + +Distance-only generated cases also expose `s1_array_bfs_distance`, a genuine +typed PL/pgSQL SP-S1 prototype. It keeps frontier and visited node IDs in +bounded arrays, records a fixed 100,000-node state ceiling, and restarts the +exact S3-U distance reference in the same statement on overflow. It is a +benchmark arm only and is never selected by production translation. + +Capture S3-U-D and SP-S1 together with 20 warmups and 50 observations, then +produce their seeded, order-balanced matched comparison with: + +```bash +go run ./cmd/graphbench \ + -reference-pair-artifact .coverage/shortest-alternatives.jsonl \ + -reference-pair-baseline s3_unidirectional_trail_cte \ + -reference-pair-candidate s1_array_bfs_distance \ + -reference-pair-output .coverage/shortest-alternatives.json \ + -confidence-level 0.975 \ + -seed 1 +``` + +The default confirmation pair reporter requires 10-20 independent rounds, 20 +warmups, 50 samples per arm per round, and distinct recorded measurement order. +`-reference-pair-protocol discovery` produces an explicitly labeled exploratory +report from 5-20 rounds, five warmups, and ten samples per arm; it cannot be +mistaken for confirmation evidence because the protocol and requirements are +written into the report. The reporter accepts two exact public-observation +comparators, two exact ordered-ID comparators, or two hydration-only arms +independently validated from the same precomputed exact path inputs; mixed +boundaries are rejected. Fixed-suffix expansion ordered-ID candidates are +checked against the canonical stepwise-forward node/edge-ID arrays before their +timing is retained. Reports show +candidate/baseline median and p95 ratios, absolute median change, and +within-session A/A resolution without turning architecture selection into a +post-hoc pass threshold. + +### Three- and five-arm reference tournaments + +Use the generic tournament reporter when a candidate family has three or five +exact PostgreSQL reference arms. The first declared arm is the incumbent: + +```bash +make perf_tournament \ + PERF_TOURNAMENT_ARTIFACT=.coverage/tournament.jsonl \ + PERF_TOURNAMENT_ARMS=expand_into_pair_join,expand_into_lower_degree_scan,expand_into_pair_cache \ + PERF_TOURNAMENT_PROTOCOL=confirmation +``` + +The reporter verifies exact public observations, immutable SQL/implementation +identity, the predeclared doubled-Williams measurement order, and per-round +sample floors. A confirmation is promotion-eligible only when one stable +candidate wins both training and frozen holdout, its median improvement clears +the configured 5% or 100us materiality floor, and its p95 ratio upper bound is +at most 1.05. Discovery reports are always non-promotional. + +Function-backed SP/ASP candidates and guarded orientation runs use a +session-local receipt around every timed invocation when `-pool-size 1` is in +effect. Arming and reading occur outside the measured interval. The receipt +binds the requested identity to the exact executed branch, fallback outcome, +and a singular record count. Multi-connection runs remain available for the +operational matrix, but their timing samples are intentionally not eligible as +per-invocation promotion evidence. + +### Promotion manifest + +Promotion is authorized only by a version-2 manifest that binds the candidate, +selector, source/binary/corpus SHA-256 digests, immutable caps, exact query +cohorts, training and frozen-holdout buckets, and checksummed A/A, +confirmation, performance, resource, reference-closure, and operational +reports. Version 1 is decoded only to reject it for new authorization. + +Every evidence report must repeat the manifest's complete authorization +identity. Generate the role-specific report first, then attach the identity +from a provisional manifest whose evidence map may still be empty: + +```bash +go run ./cmd/graphbench \ + -promotion-bind-manifest .coverage/promotion-provisional.json \ + -promotion-bind-role performance \ + -promotion-bind-input .coverage/performance-unbound.json \ + -promotion-bind-output .coverage/performance.json +``` + +Repeat this for `aa`, `confirmation`, `performance`, `resource`, +`reference_closure`, and `operational`, checksum the bound reports, and place +those digests in the final manifest. Then verify the complete closure without +opening a database connection: + +```bash +go run ./cmd/graphbench \ + -promotion-manifest .coverage/promotion.json \ + -promotion-manifest-output .coverage/promotion-verification.json +``` + +Verification fails closed for missing roles, mutated reports, path traversal, +non-passing evidence, invalid digests, absent caps, identity fields that differ +from the manifest, or buckets that do not bind both qualification splits. This +mode is mutually exclusive with benchmark, report, bind, and bundle operations. + +### Fixed-one-hop ExpandInto study + +Build the standalone three-arm fixed-one-hop report from records captured with +the `expand_into_one_hop` category and its exact PostgreSQL references: + +```bash +go run ./cmd/graphbench \ + -expand-into-artifact .coverage/expand-into.jsonl \ + -expand-into-output .coverage/expand-into-study.json \ + -expand-into-protocol discovery \ + -confidence-level 0.975 -seed 1 + +make perf_expand_into \ + PERF_EXPAND_INTO_ARTIFACT=.coverage/expand-into.jsonl \ + PERF_EXPAND_INTO_PROTOCOL=confirmation +``` + +`discovery` requires 5-20 independently reloaded rounds, five warmups, and ten +samples per arm per round. `confirmation` requires 10-20 rounds, 20 warmups, +and 50 samples per arm per round. Both protocols require the frozen doubled +Williams order for `expand_into_pair_join`, `expand_into_lower_degree_scan`, and +`expand_into_pair_cache`, exact public observations, stable implementation/SQL +identities, and persisted plan-cache/operator evidence. Confirmation reports +also require one stable non-direct winner across training and frozen holdout, +the configured 5% or 100us materiality floor, and p95 containment at 1.05. +Even a passing report does not activate a production strategy; discovery +remains evidence-only. + +Path-observed singleton cases additionally capture benchmark-only M0 and M1 +materializer arms. Whole-query comparison uses each architecture's minimal +state: `SP-S3-U-E+MAT-M0` carries edge IDs only and derives node order from the +directed edge endpoints, while `SP-S3-U-NE+MAT-M1` carries node and edge IDs and +hydrates both streams independently by ordinality. Outbound and inbound M0 use +distinct implementation identities. Separate +hydration-only arms use precomputed IDs so search cost stays outside the timed +materializer boundary. These arms are exact-result checked but do not change +production path rendering. Odd benchmark rounds execute references in declared +order and even rounds reverse that order, balancing which M0/M1 arm runs first +across the required independently reloaded rounds. + +Every PostgreSQL dataset reload truncates the active relationship and node +partitions together. Other backends delete relationships before nodes. PostgreSQL then checks +the physical row counts in the active `node_` and `edge_` +partitions against the fixture declaration before vacuuming or measuring. A +stale/orphan row therefore fails the run instead of silently contaminating scan +and count cases. Fixture records also retain active child-partition sizes rather +than the zero-sized partitioned-parent relations. + +```bash +go run ./cmd/graphbench \ + -modes postgres_sql -postgres-references \ + -cases 'GSP-D01-F001_path,GSP-D02-F016_path,GSP-D04-F128_path,GSP-D08-F001_path_inbound,GSP-D16-F016_path,GSP-D32-F512_path,GSP-D64-F1000_path' \ + -warmup-iterations 20 -iterations 50 -pool-size 1 \ + -pg-connection "$PG_CONNECTION_STRING" \ + -jsonl-output .coverage/materializer-round-1.jsonl +``` + +The optimizer also emits a typed `ShortestPathExecutorDecision` for every +shortest traversal. It records a machine-readable structural-eligibility result, +SP family and planned candidate identities, observation mode, minimum/maximum +depth, selected/fallback executor, selector version/mode, limits, and stable +fallback code. These fields are also copied into each exact target outcome. +Call count and read-only status are statement-wide, including shortest calls or +mutations separated by `WITH`. Selector `sp-static-v5-contained` chooses +`SP-S3-U-D` for qualified distance observations, bounded +`SP-S3-U-E+MAT-M0` for directed single-kind one-path observations, and +canonical `SP-S4-C-WE+MAT-M0` for deep inbound, multi-kind, or untyped witness +work. Qualification requires one directed three-element shortest-path +traversal, a supported bounded depth, one static ID equality per endpoint, no +relationship variable or predicate, no path predicate, one uncorrelated +endpoint pair, one statement-wide shortest call, and a read-only statement. +The selector also records graph direction, physical expansion +column, relationship-kind count, wildcard state, and a static topology class. +Deep `end_id` distance expansion selects canonical `SP-S4-C-D`. S4 uses compact +ID state, a bounded ceiling, and exact same-statement overflow fallback. +`asp-static-v1` selects `ASP-A1-DAG` for the narrow singleton all-shortest +envelope and retains all minimum-depth predecessor edges before enumeration. +`ASP-I1-U-DAG+MAT-M0` is a distinct inline predecessor-DAG comparator and a +default-off exact-query production canary. Its guarded statement records the +executed candidate/no-path/A1-fallback branch, uses immutable manifest caps, +and requires Repeatable Read or Serializable isolation. Forced executors +remain qualification seams. +`SP-I1-C-WE+MAT-M0` is the corresponding guarded canonical-predecessor witness +canary, with four cap+1 gates, inline M0 hydration, exact S4 fallback, and an +ordered runtime fallback event chain. Its target outcome names the exact +candidate/fallback pair and emitted `sp-i1-canonical-guarded-v1` policy, while +diagnostic resource evidence remains isolated from the ASP I1 counter family. +It remains default-off; `sp-static-v5-contained` continues to select the +automatic S3/S4 production paths. The evidence-gated `sp-static-v6` canary +identity accepts only the qualified inbound, typed, single-kind, one-path +`min=1`/`max=64` bucket. Outbound, untyped, multi-kind, and different-depth +manifests fail closed at verification, provisional capture, driver admission, +and translation. + +### Frozen canonical-I1 qualification + +The `sp-i1-inbound-v1` study is a dedicated two-arm comparison between exact +forced `SP-S4-C-WE+MAT-M0` and guarded forced +`SP-I1-C-WE+MAT-M0`. Its fresh cohort contains four training cases at depths 4 +and 16 and three unopened holdouts at depths 8 and 32. Every case uses the +same typed inbound one-path query with `min=1`, `max=64`, one `Traverse` kind, +exact path observations, and forbidden fallback. GraphBench excludes these +protocol-only holdouts from ordinary default, category, dataset, and generic-tag +selection. Only the exact holdout protocol tag (or an exact holdout case name) +enters the protected authorization path. Exact-name selection still fails +closed because the only executable confirmation selection is the complete +four-training/three-holdout cohort with a passing training freeze. The frozen +performance study executes PostgreSQL only; Neo4j remains part of the declared +cross-backend semantic contract, not an authorized holdout timing arm. + +Build GraphBench once from a clean committed tree. Keep the binary and all +outputs under ignored `.coverage`; repeated `go run` invocations have different +binary identities and cannot satisfy the freeze: + +```bash +CAPTURE=.coverage/sp-i1-inbound-v1 +mkdir -p "$CAPTURE/bin" +go build -trimpath -o "$CAPTURE/bin/graphbench" ./cmd/graphbench +BIN="$CAPTURE/bin/graphbench" +DISCOVERY_UUID="sp-i1-discovery-$(git rev-parse HEAD)" +``` + +Discovery opens only the four training declarations. Capture 5-20 paired +rounds with at least 5 warmups and 10 samples per arm per round. Use the same +UUID for both artifacts and all rounds. Odd rounds put S4 first; even rounds +put canonical I1 first. For round 1, the two commands are: + +```bash +"$BIN" \ + -modes postgres_sql \ + -tags sp-i1-inbound-v1-training \ + -round 1 -block 1 -run-uuid "$DISCOVERY_UUID" \ + -arm sp-i1-s4 -arm-order 1 \ + -warmup-iterations 5 -iterations 10 -pool-size 1 \ + -postgres-force-shortest-executor SP-S4-C-WE+MAT-M0 \ + -postgres-repeatable-read \ + -postgres-traversal-telemetry diagnostic \ + -pg-connection "$PG_CONNECTION_STRING" \ + -jsonl-output "$CAPTURE/discovery-s4.jsonl" -append-jsonl + +"$BIN" \ + -modes postgres_sql \ + -tags sp-i1-inbound-v1-training \ + -round 1 -block 1 -run-uuid "$DISCOVERY_UUID" \ + -arm sp-i1-candidate -arm-order 2 \ + -warmup-iterations 5 -iterations 10 -pool-size 1 \ + -postgres-force-shortest-executor SP-I1-C-WE+MAT-M0 \ + -postgres-repeatable-read \ + -postgres-traversal-telemetry diagnostic \ + -pg-connection "$PG_CONNECTION_STRING" \ + -jsonl-output "$CAPTURE/discovery-i1.jsonl" -append-jsonl +``` + +After all training rounds, bind resource-gate v5 to the exact candidate JSONL, +then write the discovery report and freeze: + +```bash +"$BIN" \ + -resource-artifact "$CAPTURE/discovery-i1.jsonl" \ + -resource-output "$CAPTURE/discovery-i1-resource.json" + +"$BIN" \ + -sp-i1-baseline-artifact "$CAPTURE/discovery-s4.jsonl" \ + -sp-i1-candidate-artifact "$CAPTURE/discovery-i1.jsonl" \ + -sp-i1-resource-report "$CAPTURE/discovery-i1-resource.json" \ + -sp-i1-protocol discovery \ + -sp-i1-output "$CAPTURE/discovery-report.json" \ + -sp-i1-freeze-output "$CAPTURE/discovery-freeze.json" +``` + +For structurally valid evidence, the reporter preserves the discovery result +and freeze even when a statistical or resource disposition fails. Identity, +path, and source-validation failures do not write an artifact. A failed freeze +cannot authorize holdout capture. A passing freeze binds the clean source archive, commit, +binary, query, training/full declarations and resolved selections, training +artifacts, resource report, and the promotion-form cap names +`state_limit`, `predecessor_limit`, `enumeration_limit`, and +`output_bytes_limit`. Resource evidence uses the corresponding telemetry names +`state_rows`, `predecessor_rows`, `output_rows`, and `output_bytes`. The CLI +fixes the bootstrap seed at `1` and confidence at `0.975`, uses 10,000 +resamples, and freezes all three settings. Schedule validation checks the +recorded invocation timestamps as well as the declared alternating order. +Resource-gate v5 binds every decision to the exact candidate arm, round, +block, run UUID, runtime receipt, and diagnostic counters. +The qualification validator requires `planned_candidates` to preserve the +translator's complete shortest-path executor search space. The exact study +arms are bound independently through selected, applied, emitted, and timed +runtime-receipt identities; a reduced two-arm planned list is invalid evidence. +Every warm sample also carries a unique session-local runtime invocation ID, +repeated on its receipt events; duplicate reuse anywhere in the paired study +is rejected. Fixture +and PostgreSQL comparison is deliberately strict, including byte-identical +node and edge relation sizes across paired arms and rounds. + +Only after discovery passes may confirmation open the full four-training and +three-holdout cohort. Every capture command must provide the freeze and its +checksummed discovery report before database setup. Confirmation requires +10-20 paired rounds, at least 20 warmups, 50 samples per arm per round, pool +size 1, diagnostic telemetry, Repeatable Read, an explicit shared UUID, block +equal to round, and the exact alternating labels/order. For confirmation round +1, create a fresh series UUID, add these authorization and cohort flags to the +two discovery commands, increase the sample settings, and write separate +artifacts. Reuse that confirmation UUID across both arms and every confirmation +round: + +```text +CONFIRMATION_UUID="sp-i1-confirmation-$(git rev-parse HEAD)" +-run-uuid "$CONFIRMATION_UUID" +-tags sp-i1-inbound-v1-training,sp-i1-inbound-v1-holdout +-sp-i1-freeze .coverage/sp-i1-inbound-v1/discovery-freeze.json +-sp-i1-discovery-report .coverage/sp-i1-inbound-v1/discovery-report.json +-sp-i1-training-baseline-artifact .coverage/sp-i1-inbound-v1/discovery-s4.jsonl +-sp-i1-training-candidate-artifact .coverage/sp-i1-inbound-v1/discovery-i1.jsonl +-sp-i1-training-resource-report .coverage/sp-i1-inbound-v1/discovery-i1-resource.json +-warmup-iterations 20 -iterations 50 +``` + +Use `sp-i1-s4` at order 1 and `sp-i1-candidate` at order 2 on odd rounds; +reverse those orders on even rounds. Rounds after the first must use +`-append-jsonl`. GraphBench rejects partial or extra cohorts, a changed tag or +case declaration, source/binary drift, insufficient capture settings, path +aliasing with freeze inputs, supplemental arms, and any attempt to enter an +unrelated report mode with holdout authorization flags. +Before every protected capture, GraphBench reloads those three training inputs, +checks their frozen digests, and recomputes the discovery statistics and +resource decisions before opening the database. + +Create resource-gate v5 from the complete confirmation I1 artifact, then issue +the final report with the frozen discovery inputs: + +```bash +"$BIN" \ + -resource-artifact "$CAPTURE/confirmation-i1.jsonl" \ + -resource-output "$CAPTURE/confirmation-i1-resource.json" + +"$BIN" \ + -sp-i1-baseline-artifact "$CAPTURE/confirmation-s4.jsonl" \ + -sp-i1-candidate-artifact "$CAPTURE/confirmation-i1.jsonl" \ + -sp-i1-resource-report "$CAPTURE/confirmation-i1-resource.json" \ + -sp-i1-freeze "$CAPTURE/discovery-freeze.json" \ + -sp-i1-discovery-report "$CAPTURE/discovery-report.json" \ + -sp-i1-training-baseline-artifact "$CAPTURE/discovery-s4.jsonl" \ + -sp-i1-training-candidate-artifact "$CAPTURE/discovery-i1.jsonl" \ + -sp-i1-training-resource-report "$CAPTURE/discovery-i1-resource.json" \ + -sp-i1-protocol confirmation \ + -sp-i1-output "$CAPTURE/confirmation-report.json" +``` + +Each case passes only when the candidate has complete per-sample timed runtime +receipts with no fallback or overflow, exact observations match S4, resource +evidence passes all four limits, the median-ratio upper bound is at most `0.95` +or the median-saving lower bound is at least `100us`, and the p95-ratio upper +bound is at most `1.05`. The study does not change the automatic production +selector; a passing report is input to later canary, rollback, and promotion +closure. + +The clean `6d56a609` confirmation completed 10 paired rounds and 500 timed +samples per arm/case. All four training and three holdout cases passed with +zero candidate fallbacks; median reductions were 75.9-94.2% and p95 reductions +were 70.2-89.7%. Resource-gate v5 passed all 70 candidate case-round records, +with maxima of 281 state rows, 280 predecessor rows, 33 output rows, and 9,075 +output bytes. This closes the frozen cohort; it does not replace the production +statement, reference-closure, and operational evidence required by a promotion +manifest. + +Use `-postgres-production-manifest` to measure the exact guarded production +statement from a provisional version-2 manifest before the evidence map can be +closed. The runner validates the candidate/fallback pair, selector, +family-specific immutable caps, unique exact query digests, and bucket match. +Guarded SP/ASP candidates require their four positive shortest-path caps. +`orientation-probe-v1` instead requires the optimizer's exact +`root_row_limit=512`, `reverse_seed_row_limit=512`, +`directional_degree_row_limit=16384`, and `state_limit=4096` contract, the +`EXPANSION-STEPWISE-FORWARD` fallback, and the `guarded_dual_arm` boundary; its +production options enable expansion orientation without selecting a +shortest-path executor. The runner executes each statement under Repeatable +Read and retains per-sample runtime +receipts. This flag is mutually exclusive with tool-forced and shadow modes; +evidence may be empty only because the capture is producing that evidence. +Final rollout still requires the ordinary complete manifest verifier. +Use `-postgres-repeatable-read` on the incumbent arm so a matched comparison +measures both sides under the stable-snapshot admission contract. A production +manifest implies this option and cannot be combined with it explicitly. + +## Existing graph non-mutating mode + +`-existing-graph` runs a selected PostgreSQL corpus without asserting schema, +clearing/loading fixtures, vacuuming, or creating persistent helpers. It +requires a versioned logical-key anchor manifest and refuses `write_scenario` +or mutation keywords before runner construction. It deliberately uses +read-write PostgreSQL sessions so session-local workspace setup, reset, and +statistics match production. Example: + +```json +{ + "version": 1, + "graph": "integration_test", + "content_identity": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "anchors": { + "outbound_source": {"logical_key": "sanitized-source", "kind": "Source"}, + "outbound_target": {"logical_key": "sanitized-target", "kind": "Target"} + } +} +``` + +```bash +go run ./cmd/graphbench \ + -existing-graph \ + -anchor-manifest anchors.json \ + -cases LIVE-outbound-distance \ + -checkpoint .coverage/live/checkpoint.json \ + -progress .coverage/live/progress.jsonl \ + -jsonl-output .coverage/live/results.jsonl +``` + +Anchor values are used only at runtime. Durable records replace them with +one-way hashes, omit rendered parameters and Cypher, and redact observed-row +and error payloads in both primary and nested reference outcomes. The runner captures +before/after graph cardinalities, relation sizes, PostgreSQL settings, and +schema/index fingerprints. Artifact schema v2 records a digest of the complete +workload, fixture identity, corpus, and run configuration. Each completed record +is checkpointed by stable backend/dataset/case/workload identity using an atomic +rename; `-resume` accepts only a matching manifest, corpus, and run identity and +preserves the original run UUID. + +Legacy graphs without `logical_key` properties may instead use a runtime-only +physical anchor with a content proof: + +```json +{"physical_id": 42, "content_sha256": "sha256:<64 lowercase hex characters>", "kind": "Entity"} +``` + +The digest is SHA-256 over PostgreSQL's canonical `kind_ids::text`, a newline, +and `properties::text` for that node. The runner accepts the ID only after the +digest and optional kind match, then removes the ID and manifest values from +durable records. Each anchor must use exactly one of `logical_key` or +`physical_id`; a physical anchor always requires its content digest. +For exact path observations on a legacy graph, include content-proved anchors +for intermediate nodes as well as parameter endpoints so stable path identity +can be reconstructed without persisting physical IDs. + +Existing-graph runs require the target database to have the DAWGS schema and +workspace functions from the current checkout already deployed. The runner +does not assert or upgrade schema in this mode because doing so would violate +its non-mutating existing-graph contract. + +Adaptive discovery is explicit: + +```bash +go run ./cmd/graphbench \ + -existing-graph -anchor-manifest anchors.json \ + -discovery -timeout-classes 100ms,1s,10s \ + -discovery-sample-floor 1 \ + -checkpoint .coverage/live/checkpoint.json +``` + +Every timeout and sample reduction stays in the case record. Adaptive artifacts +are refused by the complete performance gate. Confirmation omits `-discovery` +and uses fixed timeouts, arm order, warmups, and samples. + +The independent state/resource report is produced with +`-resource-artifact results.jsonl -resource-output resources.json`. Schema v5 +records the SHA-256 digest of the exact input JSONL so +promotion evidence can verify that resource decisions remain bound to their +capture. For non-stress portable PostgreSQL candidates it rejects temp spill, +local workspace, and WAL for non-mutating reads. S4 and ASP explicitly permit their +session-local compact workspace but still reject executor temp-file spill and +WAL; exact incumbent fallback retains its documented temporary-workspace +contract. `SP-S0-DIRECT` records are +attributed from the measured fallback function loops, so workspace use is +accepted only when the incumbent branch actually ran. Exact full-comparator +reference arms receive independent resource cases rather than inheriting the +outer production result. + +Shortest tournament references are independently selectable with +`-postgres-reference-arms s4_canonical_source_distance`, +`s4_canonical_source_witness_m0`, `sp_b1_strict_alternating_distance`, +`sp_b1_strict_alternating_witness_m0`, +`sp_b2_smaller_frontier_distance`, +`sp_b2_smaller_frontier_witness_m0`, `asp_a1_stored_helper_m0`, +`asp_i1_inline_predecessor_dag_m0`, +`asp_b1_bidirectional_dag_strict_m0`, and +`asp_b2_bidirectional_dag_smaller_frontier_m0`. They are exact full-query comparators at the same +public observation boundary, not production selectors. S4 canonicalizes inbound +search to physical `start_id -> end_id`; B1 alternates one accepted node per +side, while B2 expands the smaller complete current level with a deterministic +forward tie-break. Both candidates retain ID-only state, reconstruct one stable +witness late, and fall back to exact S4 before output if a seen, frontier, or +predecessor cap overflows. Their multi-statement functions reject Read Committed; +GraphBench runs any selected B1/B2 production or reference arm at Repeatable +Read so candidate search and fallback share one transaction snapshot. The ASP +arms retain every relationship-distinct shortest-depth predecessor, select one +canonical completed meeting cut, and separately cap discovery state, frontier, +predecessors, saturating path count, enumerated rows, and output bytes before +exact A1 fallback. SP and ASP identities are forceable with +`-postgres-force-shortest-executor`; automatic selection remains on S3/S4 for +SP and A1 for ASP. Activation evidence still requires the saved +plan/resource, holdout, concurrency, cancellation, and reference-closure gates. + +`-backend-delta-artifact combined.jsonl -backend-delta-output deltas.json` +produces matched PostgreSQL/Neo4j median and p95 ratios only when both records +exist, and reports logical-observation agreement. The report is explicitly +descriptive and never participates in PostgreSQL pass/fail selection. +Every other shape retains `SP-S0` and its specific fallback code. + +Ordinary variable expansions with fixed continuations similarly emit a typed +`ExpansionSearchStrategyDecision`. It records suffix bounds, logical direction, +observation mode, depth bounds, structural facts, selection mode, and stable +fallback codes. It also reports the fixed-suffix expansion family, planned +candidate set, selector version, and distinct correlated-suffix/cross-region +fallback reasons. +Factored-suffix and backward-viability SQL remains reference-only. +`EXPANSION-SUFFIX-SEEDED-REVERSE` has a repository-native emitter for +qualification, but it is not selected by the public query API. Structurally +eligible forms select `EXPANSION-STEPWISE-FORWARD` with +`tournament_unqualified`. + ## Outputs JSONL output contains one `CaseResult` record per case and execution mode. @@ -69,6 +1192,60 @@ Markdown and JSON summaries aggregate mode status counts, per-case timings, row counts, fallback reasons, and baseline regressions or improvements when a baseline capture is supplied. -PostgreSQL records include translated SQL and `EXPLAIN (ANALYZE, BUFFERS, -TIMING OFF, FORMAT JSON)` metrics. Neo4j records include plan operator names +PostgreSQL case records also include aggregate query-text-free parse-cache +counters. Optimization diagnostics retain target-specific selected, applied, +and skipped identities; compile-time records do not claim a runtime branch. + +Each timing record retains the unsorted cold and warm latency samples with round, +iteration, case, dataset, backend, and connection/session fields so confidence +interval and regression tooling does not have to reconstruct observations from +summary percentiles. Read cases run untimed preflight and postflight queries +and compare their complete row multisets, including duplicate rows, around the +timed block. For declared `id_rows`, `path_set`, and scalar results, recorded +`observed_rows` use stable fixture identities, retain relationship order, +kinds, and properties, and reject relationship reuse within a path. GraphBench +compares those stable result kinds across backends. Other result kinds still +receive per-backend preflight/postflight checks, but are not compared across +backends because they may contain backend-generated relationship IDs. +PostgreSQL fixture loads are followed by `VACUUM (ANALYZE)` through +the pool; a maintenance failure aborts the benchmark. +The PostgreSQL runner defaults to a one-connection pool and records +`pg_backend_pid()` as the serial sample connection identifier. Concurrency +blocks record the physical PID of every direct pool acquisition so per-session +cold state and pool queuing remain visible. + +Write records additionally report matched and affected counts and each +post-state observation. The recorded duration covers the mutation query; setup, +verification, and rollback are outside that duration. + +PostgreSQL records include translated SQL and its fingerprint, server settings, +fixture checksum/cardinalities, and `EXPLAIN (ANALYZE, BUFFERS, TIMING OFF)` +shared/local/temp metrics plus `EXPLAIN (ANALYZE, BUFFERS, WAL, SETTINGS, +FORMAT JSON)` for reads. Neo4j records include plan operator names when an `EXPLAIN` plan can be captured. + +## PostgreSQL scale-plan correctness gate + +The PostgreSQL-only `TestPostgreSQLScalePlanInvariants` test loads the same +scale corpus and fixture as the command. It executes all required Cypher scale +representatives, requires their declared cardinalities and mutation post-state, +and verifies that the captured plan came from `EXPLAIN ANALYZE`. Stable +assertions cover relationship/node mutation targets, branch-local logical +structure, temporal filtering, and anchored edge-index orientation. The test +uses rollback isolation for writes and runs automatically under +`make test_all` when `CONNECTION_STRING` selects PostgreSQL. + +Run only the scale-plan gate with: + +```bash +DAWGS_INTEGRATION_ALLOW_DESTRUCTIVE=1 \ +DAWGS_INTEGRATION_DISPOSABLE_TARGETS="postgresql://localhost:65432/dawgs" \ + CONNECTION_STRING="$PG_CONNECTION_STRING" \ + go test -tags manual_integration ./cmd/graphbench \ + -run 'Test(PostgreSQLScalePlanInvariants|ScaleCorpusRequiredRepresentativesDeclareCardinality)' \ + -count=1 +``` + +The non-integration cardinality test also guarantees that every required stable +query-form ID remains represented in the scale corpus and declares an expected +read or write cardinality. diff --git a/cmd/graphbench/aa_report.go b/cmd/graphbench/aa_report.go new file mode 100644 index 00000000..15730e5b --- /dev/null +++ b/cmd/graphbench/aa_report.go @@ -0,0 +1,404 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +// aaReportVersion identifies the serialized schema revision for A/A report. +const aaReportVersion = 3 + +// AAMetricResolution captures relative and absolute within-arm noise for one latency quantile. +type AAMetricResolution struct { + // Ratio reports the candidate-to-baseline latency ratio. + Ratio RatioInterval `json:"ratio"` + // RatioResolution records the relative A/A noise floor for ratio classification. + RatioResolution float64 `json:"ratio_resolution"` + // AbsoluteChange reports the paired candidate-minus-baseline A/A duration interval. + AbsoluteChange DurationInterval `json:"absolute_change"` + // AbsoluteResolution records the absolute A/A noise floor used for materiality decisions. + AbsoluteResolution time.Duration `json:"absolute_resolution"` +} + +// AAResolutionCase reports matched sample counts and median and P95 noise floors for one case. +type AAResolutionCase struct { + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // Backend identifies the execution backend. + Backend ExecutionMode `json:"backend"` + // WorkloadSHA256 binds the resolution to the exact logical workload declaration. + WorkloadSHA256 string `json:"workload_sha256"` + // PostgresEnvironmentSHA256 binds PostgreSQL A/A noise to the exact timing + // environment, including transaction isolation and normalized analyze state. + PostgresEnvironmentSHA256 string `json:"postgres_environment_sha256,omitempty"` + // FixtureSHA256 binds PostgreSQL A/A noise to the exact validated fixture. + FixtureSHA256 string `json:"fixture_sha256,omitempty"` + // Rounds records the number of independent measurement rounds. + Rounds int `json:"rounds"` + // SamplesPerArm records matched timing samples available from each A/A arm. + SamplesPerArm int `json:"samples_per_arm"` + // P50 records relative and absolute A/A noise at median latency. + P50 AAMetricResolution `json:"p50"` + // P95 records relative and absolute A/A noise at 95th-percentile latency. + P95 AAMetricResolution `json:"p95"` + // P99Gated reports whether the sample count is sufficient to enforce the P99 noise threshold. + P99Gated bool `json:"p99_gated"` + // P99Reason explains why P99 gating was applied or omitted. + P99Reason string `json:"p99_reason,omitempty"` +} + +// AAResolutionReport contains per-case A/A noise floors and the artifact identity used to derive them. +type AAResolutionReport struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // Seed controls deterministic random sampling. + Seed int64 `json:"seed"` + // Confidence sets the confidence level used for statistical intervals. + Confidence float64 `json:"confidence_level"` + // ArtifactSHA256 identifies the exact input artifact summarized by the report. + ArtifactSHA256 string `json:"artifact_sha256"` + // HostFingerprint identifies the host whose timing noise this report measures. + HostFingerprint string `json:"host_fingerprint"` + // MinimumRounds records the independent-round floor enforced by this report. + MinimumRounds int `json:"minimum_rounds"` + // MinimumSamplesPerArmPerRound records the sample floor enforced after splitting A/A arms. + MinimumSamplesPerArmPerRound int `json:"minimum_samples_per_arm_per_round"` + // OrderBalanced reports that the two explicitly executed A/A arms have complementary balanced first position. + OrderBalanced bool `json:"order_balanced"` + // MinimumP99SamplesPerArm sets the per-arm sample floor required before P99 gating. + MinimumP99SamplesPerArm int `json:"minimum_p99_samples_per_arm"` + // Cases contains per-workload A/A noise estimates and resolution thresholds. + Cases []AAResolutionCase `json:"cases"` +} + +// buildAAResolutionReport splits matched A/A samples and estimates per-case median and P95 noise floors. +func buildAAResolutionReport(records []CaseResult, options PerfGateOptions) (AAResolutionReport, error) { + if options.Confidence <= 0 || options.Confidence >= 1 { + return AAResolutionReport{}, fmt.Errorf("confidence level must be between 0 and 1") + } + if options.BootstrapCount == 0 { + options.BootstrapCount = defaultBootstrapCount + } + if options.BootstrapCount < 1 { + return AAResolutionReport{}, fmt.Errorf("bootstrap count must be positive") + } + hostFingerprint, err := artifactHostFingerprint(records) + if err != nil { + return AAResolutionReport{}, err + } + + all, err := collectExplicitAASeries(records) + if err != nil { + return AAResolutionReport{}, err + } + keys := make([]performanceKey, 0, len(all)) + for key := range all { + if key.backend == ModePostgresSQL { + keys = append(keys, key) + } + } + sort.Slice(keys, func(i, j int) bool { + if keys[i].dataset != keys[j].dataset { + return keys[i].dataset < keys[j].dataset + } + return keys[i].name < keys[j].name + }) + if len(keys) == 0 { + return AAResolutionReport{}, fmt.Errorf("artifact has no successful PostgreSQL warm samples") + } + + report := AAResolutionReport{ + Version: aaReportVersion, + Seed: options.Seed, + Confidence: options.Confidence, + HostFingerprint: hostFingerprint, + MinimumRounds: minimumGateRounds, + MinimumSamplesPerArmPerRound: 10, + OrderBalanced: true, + MinimumP99SamplesPerArm: 10_000, + } + for idx, key := range keys { + var ( + armA, armB = all[key][0], all[key][1] + seed = options.Seed + int64(idx)*7919 + ) + + armA, armB = matchedRounds(armA, armB) + if len(armA) < minimumGateRounds { + return AAResolutionReport{}, fmt.Errorf("%s/%s requires at least %d A/A rounds, got %d", key.dataset, key.name, minimumGateRounds, len(armA)) + } + for _, round := range sortedRounds(armA) { + if len(armA[round]) < report.MinimumSamplesPerArmPerRound || len(armB[round]) < report.MinimumSamplesPerArmPerRound { + return AAResolutionReport{}, fmt.Errorf("%s/%s round %d requires at least %d samples per A/A arm, got %d/%d", key.dataset, key.name, round, report.MinimumSamplesPerArmPerRound, len(armA[round]), len(armB[round])) + } + } + + var ( + p50 = bootstrapRoundMedianRatio(armA, armB, seed, options) + p95 = bootstrapStratifiedP95Ratio(armA, armB, seed+1, options) + p50Change = negateDurationInterval(bootstrapRoundMedianSaving(armA, armB, seed+2, options)) + p95Change = bootstrapStratifiedQuantileChange(armA, armB, 0.95, seed+3, options) + armSamples = min(sampleCount(armA), sampleCount(armB)) + ) + + workloadSHA256, err := workloadSHA256ForKey(records, key) + if err != nil { + return AAResolutionReport{}, err + } + postgresEnvironmentSHA256, err := postgresTimingEnvironmentSHA256ForKey(records, key) + if err != nil { + return AAResolutionReport{}, err + } + fixtureSHA256, err := fixtureSHA256ForKey(records, key) + if err != nil { + return AAResolutionReport{}, err + } + entry := AAResolutionCase{ + Dataset: key.dataset, + Name: key.name, + Backend: key.backend, + WorkloadSHA256: workloadSHA256, + PostgresEnvironmentSHA256: postgresEnvironmentSHA256, + FixtureSHA256: fixtureSHA256, + Rounds: len(armA), + SamplesPerArm: armSamples, + P50: aaMetricResolution(p50, p50Change), + P95: aaMetricResolution(p95, p95Change), + P99Gated: armSamples >= 10_000, + } + if !entry.P99Gated { + entry.P99Reason = fmt.Sprintf("diagnostic only: need at least 10000 samples per A/A arm, got %d", armSamples) + } + + report.Cases = append(report.Cases, entry) + } + + return report, nil +} + +// collectExplicitAASeries requires two independently executed arms with +// identical SQL and balanced block order. Splitting one timing stream into +// synthetic labels understates reload, connection, and first-order carryover +// noise and is therefore deliberately refused by the promotion-grade report. +func collectExplicitAASeries(records []CaseResult) (map[performanceKey][2]roundSamples, error) { + type armIdentity struct { + SQLFingerprint string + WorkloadSHA256 string + } + type armSeries struct { + identity armIdentity + samples roundSamples + orders map[int]int + blocks map[int]int + runUUIDs map[int]string + } + + byKey := map[performanceKey]map[string]*armSeries{} + for _, record := range records { + if record.Status != StatusOK || record.ExecutionMode != ModePostgresSQL { + continue + } + key := performanceKey{dataset: record.Dataset, name: record.Name, backend: record.ExecutionMode} + for _, sample := range record.Stats.Samples { + if sample.Classification != "warm" || sample.Duration <= 0 { + continue + } + if sample.Round < 1 || sample.Block < 1 || sample.Arm == "" || sample.Arm == "unlabeled" || sample.ArmOrder < 1 || sample.RunUUID == "" { + return nil, fmt.Errorf("%s/%s has A/A sample without explicit round, block, arm, order, and run UUID", key.dataset, key.name) + } + arms := byKey[key] + if arms == nil { + arms = map[string]*armSeries{} + byKey[key] = arms + } + arm := arms[sample.Arm] + if arm == nil { + arm = &armSeries{ + identity: armIdentity{SQLFingerprint: record.SQLFingerprint, WorkloadSHA256: record.WorkloadSHA256}, + samples: roundSamples{}, orders: map[int]int{}, blocks: map[int]int{}, runUUIDs: map[int]string{}, + } + arms[sample.Arm] = arm + } + identity := armIdentity{SQLFingerprint: record.SQLFingerprint, WorkloadSHA256: record.WorkloadSHA256} + if arm.identity != identity || identity.SQLFingerprint == "" || identity.WorkloadSHA256 == "" { + return nil, fmt.Errorf("%s/%s arm %q changes or lacks executable/workload identity", key.dataset, key.name, sample.Arm) + } + if prior, found := arm.orders[sample.Round]; found && prior != sample.ArmOrder { + return nil, fmt.Errorf("%s/%s arm %q round %d changes order", key.dataset, key.name, sample.Arm, sample.Round) + } + if prior, found := arm.blocks[sample.Round]; found && prior != sample.Block { + return nil, fmt.Errorf("%s/%s arm %q round %d changes block", key.dataset, key.name, sample.Arm, sample.Round) + } + if prior, found := arm.runUUIDs[sample.Round]; found && prior != sample.RunUUID { + return nil, fmt.Errorf("%s/%s arm %q round %d changes run UUID", key.dataset, key.name, sample.Arm, sample.Round) + } + arm.orders[sample.Round] = sample.ArmOrder + arm.blocks[sample.Round] = sample.Block + arm.runUUIDs[sample.Round] = sample.RunUUID + arm.samples[sample.Round] = append(arm.samples[sample.Round], sample.Duration) + } + } + + result := map[performanceKey][2]roundSamples{} + for key, arms := range byKey { + if len(arms) != 2 { + return nil, fmt.Errorf("%s/%s requires exactly two explicit A/A arms, got %d", key.dataset, key.name, len(arms)) + } + names := make([]string, 0, 2) + for name := range arms { + names = append(names, name) + } + sort.Strings(names) + left, right := arms[names[0]], arms[names[1]] + if left.identity != right.identity { + return nil, fmt.Errorf("%s/%s A/A arms do not have identical SQL and workload identities", key.dataset, key.name) + } + leftSamples, rightSamples := matchedRounds(left.samples, right.samples) + leftFirst := 0 + for _, round := range sortedRounds(leftSamples) { + if left.blocks[round] != right.blocks[round] || left.runUUIDs[round] != right.runUUIDs[round] { + return nil, fmt.Errorf("%s/%s round %d has mismatched A/A block or run identity", key.dataset, key.name, round) + } + if !((left.orders[round] == 1 && right.orders[round] == 2) || (left.orders[round] == 2 && right.orders[round] == 1)) { + return nil, fmt.Errorf("%s/%s round %d lacks a complete two-arm A/A order", key.dataset, key.name, round) + } + if left.orders[round] == 1 { + leftFirst++ + } + } + if rightFirst := len(leftSamples) - leftFirst; leftFirst-rightFirst > 1 || rightFirst-leftFirst > 1 { + return nil, fmt.Errorf("%s/%s A/A order is not balanced: %d/%d", key.dataset, key.name, leftFirst, rightFirst) + } + result[key] = [2]roundSamples{leftSamples, rightSamples} + } + return result, nil +} + +// aaMetricResolution returns the larger relative and absolute confidence-bound deviations observed between paired A/A samples. +func aaMetricResolution(interval RatioInterval, absoluteChange DurationInterval) AAMetricResolution { + resolution := math.Max(math.Abs(1-interval.Lower), math.Abs(interval.Upper-1)) + return AAMetricResolution{ + Ratio: interval, + RatioResolution: resolution, + AbsoluteChange: absoluteChange, + AbsoluteResolution: max(absDuration(absoluteChange.Lower), absDuration(absoluteChange.Upper)), + } +} + +// writeAAResolutionReport writes an A/A resolution report as indented JSON. +func writeAAResolutionReport(path string, report AAResolutionReport) (err error) { + var output *os.File + if path == "" { + output = os.Stdout + } else { + if err := ensureOutputDir(path); err != nil { + return err + } + output, err = os.Create(path) + if err != nil { + return err + } + defer func() { + if closeErr := output.Close(); err == nil && closeErr != nil { + err = closeErr + } + }() + } + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(report) +} + +// createAAResolutionReport loads one or more immutable arm artifacts, builds +// their joint A/A resolution report, and writes the result. +func createAAResolutionReport(artifactPaths []string, outputPath string, options PerfGateOptions) error { + records, artifactSHA256, err := loadAAResolutionArtifacts(artifactPaths) + if err != nil { + return err + } + report, err := buildAAResolutionReport(records, options) + if err != nil { + return err + } + report.ArtifactSHA256 = artifactSHA256 + return writeAAResolutionReport(outputPath, report) +} + +// loadAAResolutionArtifacts combines separately captured A/A arms without +// weakening appendJSONLFile's one-arm run-series identity. A single input keeps +// the historical raw-file checksum. Multiple inputs use a domain-separated, +// order-independent digest of their exact file checksums. +func loadAAResolutionArtifacts(paths []string) ([]CaseResult, string, error) { + if len(paths) == 0 { + return nil, "", fmt.Errorf("at least one A/A artifact is required") + } + + var ( + records []CaseResult + digests = make([]string, 0, len(paths)) + seen = make(map[string]struct{}, len(paths)) + ) + for _, path := range paths { + path = strings.TrimSpace(path) + if path == "" { + return nil, "", fmt.Errorf("A/A artifact path must not be empty") + } + cleaned := filepath.Clean(path) + if _, duplicate := seen[cleaned]; duplicate { + return nil, "", fmt.Errorf("duplicate A/A artifact %q", path) + } + seen[cleaned] = struct{}{} + + current, err := readJSONLFile(path) + if err != nil { + return nil, "", fmt.Errorf("read A/A artifact %q: %w", path, err) + } + digest, err := fileSHA256(path) + if err != nil { + return nil, "", fmt.Errorf("checksum A/A artifact %q: %w", path, err) + } + records = append(records, current...) + digests = append(digests, digest) + } + if len(digests) == 1 { + return records, digests[0], nil + } + + sort.Strings(digests) + hasher := sha256.New() + _, _ = hasher.Write([]byte("graphbench-aa-artifact-set-v1\n")) + for _, digest := range digests { + _, _ = hasher.Write([]byte(digest)) + _, _ = hasher.Write([]byte{'\n'}) + } + return records, hex.EncodeToString(hasher.Sum(nil)), nil +} + +// loadAAResolutionReport decodes a host A/A report and returns the report file's checksum. +func loadAAResolutionReport(path string) (*AAResolutionReport, string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, "", err + } + report := &AAResolutionReport{} + if err := json.Unmarshal(raw, report); err != nil { + return nil, "", fmt.Errorf("decode A/A report: %w", err) + } + digest := sha256.Sum256(raw) + return report, hex.EncodeToString(digest[:]), nil +} diff --git a/cmd/graphbench/aa_report_test.go b/cmd/graphbench/aa_report_test.go new file mode 100644 index 00000000..41d56d2a --- /dev/null +++ b/cmd/graphbench/aa_report_test.go @@ -0,0 +1,93 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestBuildAAResolutionReportUsesExplicitMatchedArmsAndKeepsP99Diagnostic verifies that independently executed balanced arms produce a promotion-grade noise floor while P99 remains explicitly non-gating. +func TestBuildAAResolutionReportUsesExplicitMatchedArmsAndKeepsP99Diagnostic(t *testing.T) { + records := explicitAARecords(t, 5, 20) + report, err := buildAAResolutionReport(records, PerfGateOptions{ + Seed: 1, + Confidence: 0.95, + BootstrapCount: 100, + }) + + require.NoError(t, err) + require.Len(t, report.Cases, 1) + require.Equal(t, aaReportVersion, report.Version) + require.True(t, validSHA256(report.HostFingerprint)) + require.True(t, report.OrderBalanced) + require.Equal(t, 100, report.Cases[0].SamplesPerArm) + require.InDelta(t, 1, report.Cases[0].P50.Ratio.Estimate, 0.0001) + require.False(t, report.Cases[0].P99Gated) + require.Contains(t, report.Cases[0].P99Reason, "diagnostic only") +} + +// TestBuildAAResolutionReportRejectsSyntheticSingleStream verifies unlabeled samples cannot be relabeled after timing to manufacture A/A evidence. +func TestBuildAAResolutionReportRejectsSyntheticSingleStream(t *testing.T) { + record := perfGateRecord("case", ModePostgresSQL, time.Millisecond, 5, 40) + _, err := buildAAResolutionReport([]CaseResult{record}, PerfGateOptions{Seed: 1, Confidence: 0.95, BootstrapCount: 100}) + require.ErrorContains(t, err, "without explicit round, block, arm, order, and run UUID") +} + +// TestCreateAAResolutionReportAcceptsSeparateArmArtifacts verifies the native +// multi-input path combines two immutable append-series arms and binds both +// exact files into one report checksum. +func TestCreateAAResolutionReportAcceptsSeparateArmArtifacts(t *testing.T) { + paths := []string{filepath.Join(t.TempDir(), "aa-a.jsonl"), filepath.Join(t.TempDir(), "aa-b.jsonl")} + records := explicitAARecords(t, 5, 10) + var left, right []CaseResult + for _, record := range records { + if record.Stats.Samples[0].Arm == "aa-a" { + left = append(left, record) + } else { + right = append(right, record) + } + } + require.NoError(t, writeJSONLFile(paths[0], left)) + require.NoError(t, writeJSONLFile(paths[1], right)) + + output := filepath.Join(t.TempDir(), "aa.json") + require.NoError(t, createAAResolutionReport(paths, output, PerfGateOptions{ + Seed: 1, Confidence: 0.95, BootstrapCount: 100, + })) + + report, _, err := loadAAResolutionReport(output) + require.NoError(t, err) + require.Len(t, report.Cases, 1) + require.True(t, validSHA256(report.ArtifactSHA256)) + leftDigest, err := fileSHA256(paths[0]) + require.NoError(t, err) + require.NotEqual(t, leftDigest, report.ArtifactSHA256) +} + +func explicitAARecords(t *testing.T, rounds, samples int) []CaseResult { + t.Helper() + var records []CaseResult + for round := 1; round <= rounds; round++ { + for armIndex, arm := range []string{"aa-a", "aa-b"} { + record := perfGateRecord("case", ModePostgresSQL, time.Millisecond, 1, samples) + record.SQLFingerprint = "identical-sql" + record.WorkloadSHA256 = "identical-workload" + for idx := range record.Stats.Samples { + record.Stats.Samples[idx].Round = round + record.Stats.Samples[idx].Block = round + record.Stats.Samples[idx].Arm = arm + record.Stats.Samples[idx].ArmOrder = 1 + (armIndex+round-1)%2 + record.Stats.Samples[idx].RunUUID = "aa-run" + } + records = append(records, record) + } + } + return records +} diff --git a/cmd/graphbench/backend_delta.go b/cmd/graphbench/backend_delta.go new file mode 100644 index 00000000..1c474d68 --- /dev/null +++ b/cmd/graphbench/backend_delta.go @@ -0,0 +1,168 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "os" + "slices" + "sort" + "time" +) + +// BackendDeltaReport contains descriptive PostgreSQL-to-Neo4j correctness and latency deltas for matched records. +type BackendDeltaReport struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // Notice states that backend deltas are descriptive and not release-gate evidence. + Notice string `json:"notice"` + // Cases contains matched PostgreSQL-to-Neo4j comparisons in deterministic report order. + Cases []BackendDeltaCase `json:"cases"` +} + +// BackendDeltaCase compares one matched PostgreSQL and Neo4j case round without assigning release-gate status. +type BackendDeltaCase struct { + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // Round identifies the measurement round. + Round int `json:"round,omitempty"` + // Complete reports whether both backend records were present. + Complete bool `json:"complete"` + // IncompleteReason identifies the absent backend side. + IncompleteReason string `json:"incomplete_reason,omitempty"` + // PostgresStatus records the PostgreSQL execution status for the matched round. + PostgresStatus string `json:"postgres_status"` + // Neo4jStatus records the Neo4j execution status for the matched round. + Neo4jStatus string `json:"neo4j_status"` + // PostgresMedian records PostgreSQL median latency for the matched round. + PostgresMedian time.Duration `json:"postgres_median,omitempty"` + // PostgresP95 records PostgreSQL P95 latency for the matched round. + PostgresP95 time.Duration `json:"postgres_p95,omitempty"` + // Neo4jMedian records Neo4j median latency for the matched round. + Neo4jMedian time.Duration `json:"neo4j_median,omitempty"` + // Neo4jP95 records Neo4j P95 latency for the matched round. + Neo4jP95 time.Duration `json:"neo4j_p95,omitempty"` + // MedianNeo4jOverPG reports the Neo4j-to-PostgreSQL median latency ratio. + MedianNeo4jOverPG float64 `json:"median_neo4j_over_postgres,omitempty"` + // P95Neo4jOverPG reports the Neo4j-to-PostgreSQL P95 latency ratio. + P95Neo4jOverPG float64 `json:"p95_neo4j_over_postgres,omitempty"` + // ObservationsComparable reports whether both backend records contain stable observations at the same boundary. + ObservationsComparable bool `json:"observations_comparable"` + // ObservationsMatch reports whether comparable backend row counts and normalized observations are equal. + ObservationsMatch bool `json:"observations_match"` +} + +// createBackendDeltaReport matches PostgreSQL and Neo4j records and writes descriptive latency and correctness deltas. +func createBackendDeltaReport(artifact, output string) error { + records, err := readJSONLFile(artifact) + if err != nil { + return err + } + + // key identifies one dataset, case, and round during backend matching. + type key struct { + // dataset names the fixture shared by the matched backend records. + dataset string + // name identifies the workload case matched across backends. + name string + // round identifies the measurement round used to balance execution order. + round int + } + + postgres, neo4j := map[key]CaseResult{}, map[key]CaseResult{} + for _, record := range records { + round := 0 + if record.Environment != nil { + round = record.Environment.Round + } + nextKey := key{ + dataset: record.Dataset, + name: record.Name, + round: round, + } + + switch record.ExecutionMode { + case ModePostgresSQL: + if _, duplicate := postgres[nextKey]; duplicate { + return fmt.Errorf("backend-delta artifact has duplicate PostgreSQL record for %s/%s round %d", nextKey.dataset, nextKey.name, nextKey.round) + } + postgres[nextKey] = record + case ModeNeo4j: + if _, duplicate := neo4j[nextKey]; duplicate { + return fmt.Errorf("backend-delta artifact has duplicate Neo4j record for %s/%s round %d", nextKey.dataset, nextKey.name, nextKey.round) + } + neo4j[nextKey] = record + } + } + + report := BackendDeltaReport{ + Version: 2, + Notice: "Descriptive only: PostgreSQL release gates compare PostgreSQL predecessors and exact PostgreSQL references, not Neo4j latency.", + } + keys := make(map[key]struct{}, len(postgres)+len(neo4j)) + for nextKey := range postgres { + keys[nextKey] = struct{}{} + } + for nextKey := range neo4j { + keys[nextKey] = struct{}{} + } + for nextKey := range keys { + pgRecord, pgFound := postgres[nextKey] + neoRecord, neoFound := neo4j[nextKey] + observationsComparable := pgRecord.StableObservation && neoRecord.StableObservation + next := BackendDeltaCase{ + Dataset: nextKey.dataset, + Name: nextKey.name, + Round: nextKey.round, + Complete: pgFound && neoFound, + PostgresStatus: pgRecord.Status, + Neo4jStatus: neoRecord.Status, + PostgresMedian: pgRecord.Stats.Median, + PostgresP95: pgRecord.Stats.P95, + Neo4jMedian: neoRecord.Stats.Median, + Neo4jP95: neoRecord.Stats.P95, + ObservationsComparable: observationsComparable, + ObservationsMatch: observationsComparable && pgRecord.RowCount == neoRecord.RowCount && slices.Equal(pgRecord.ObservedRows, neoRecord.ObservedRows), + } + switch { + case !pgFound: + next.IncompleteReason = "missing_postgres" + case !neoFound: + next.IncompleteReason = "missing_neo4j" + } + + if next.Complete && next.PostgresMedian > 0 && next.Neo4jMedian > 0 { + next.MedianNeo4jOverPG = float64(next.Neo4jMedian) / float64(next.PostgresMedian) + } + if next.Complete && next.PostgresP95 > 0 && next.Neo4jP95 > 0 { + next.P95Neo4jOverPG = float64(next.Neo4jP95) / float64(next.PostgresP95) + } + report.Cases = append(report.Cases, next) + } + + if len(report.Cases) == 0 { + return fmt.Errorf("backend-delta artifact has no PostgreSQL or Neo4j cases") + } + sort.Slice(report.Cases, func(i, j int) bool { + if report.Cases[i].Dataset != report.Cases[j].Dataset { + return report.Cases[i].Dataset < report.Cases[j].Dataset + } + if report.Cases[i].Name != report.Cases[j].Name { + return report.Cases[i].Name < report.Cases[j].Name + } + return report.Cases[i].Round < report.Cases[j].Round + }) + raw, err := json.MarshalIndent(report, "", " ") + if err != nil { + return err + } + if output == "" { + _, err = os.Stdout.Write(append(raw, '\n')) + return err + } + return os.WriteFile(output, append(raw, '\n'), 0o644) +} diff --git a/cmd/graphbench/backend_delta_test.go b/cmd/graphbench/backend_delta_test.go new file mode 100644 index 00000000..ca59ca8e --- /dev/null +++ b/cmd/graphbench/backend_delta_test.go @@ -0,0 +1,167 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestBackendDeltaReportIsDescriptiveAndRequiresMatchedObservations verifies that equal stable rows make backend timings comparable while the report remains explicitly non-gating. +func TestBackendDeltaReportIsDescriptiveAndRequiresMatchedObservations(t *testing.T) { + root := t.TempDir() + artifact, output := filepath.Join(root, "records.jsonl"), filepath.Join(root, "delta.json") + records := []CaseResult{ + { + Dataset: "fixture", + Name: "case", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + RowCount: 1, + StableObservation: true, + ObservedRows: []string{"one"}, + Stats: DurationStats{ + Median: time.Millisecond, + P95: 2 * time.Millisecond, + }, + }, + { + Dataset: "fixture", + Name: "case", + ExecutionMode: ModeNeo4j, + Status: StatusOK, + RowCount: 1, + StableObservation: true, + ObservedRows: []string{"one"}, + Stats: DurationStats{ + Median: 2 * time.Millisecond, + P95: 3 * time.Millisecond, + }, + }, + } + require.NoError(t, writeJSONLFile(artifact, records)) + require.NoError(t, createBackendDeltaReport(artifact, output)) + raw, err := os.ReadFile(output) + require.NoError(t, err) + var report BackendDeltaReport + require.NoError(t, json.Unmarshal(raw, &report)) + require.Len(t, report.Cases, 1) + require.True(t, report.Cases[0].ObservationsComparable) + require.True(t, report.Cases[0].ObservationsMatch) + require.Equal(t, 2.0, report.Cases[0].MedianNeo4jOverPG) + require.Contains(t, report.Notice, "Descriptive only") +} + +// TestBackendDeltaReportComparesPersistedObservations verifies that differing canonical row payloads are reported as a semantic mismatch even when row counts agree. +func TestBackendDeltaReportComparesPersistedObservations(t *testing.T) { + root := t.TempDir() + artifact, output := filepath.Join(root, "records.jsonl"), filepath.Join(root, "delta.json") + records := []CaseResult{ + { + Dataset: "fixture", + Name: "case", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + RowCount: 1, + StableObservation: true, + ObservedRows: []string{"postgres"}, + }, + { + Dataset: "fixture", + Name: "case", + ExecutionMode: ModeNeo4j, + Status: StatusOK, + RowCount: 1, + StableObservation: true, + ObservedRows: []string{"neo4j"}, + }, + } + require.NoError(t, writeJSONLFile(artifact, records)) + require.NoError(t, createBackendDeltaReport(artifact, output)) + raw, err := os.ReadFile(output) + require.NoError(t, err) + var report BackendDeltaReport + require.NoError(t, json.Unmarshal(raw, &report)) + require.Len(t, report.Cases, 1) + require.False(t, report.Cases[0].ObservationsMatch) +} + +// TestBackendDeltaReportDoesNotTreatAbsentObservationsAsMatching verifies that matching cardinalities cannot establish comparability without persisted stable row observations. +func TestBackendDeltaReportDoesNotTreatAbsentObservationsAsMatching(t *testing.T) { + root := t.TempDir() + artifact, output := filepath.Join(root, "records.jsonl"), filepath.Join(root, "delta.json") + records := []CaseResult{ + {Dataset: "fixture", Name: "case", ExecutionMode: ModePostgresSQL, Status: StatusOK, RowCount: 1}, + {Dataset: "fixture", Name: "case", ExecutionMode: ModeNeo4j, Status: StatusOK, RowCount: 1}, + } + require.NoError(t, writeJSONLFile(artifact, records)) + require.NoError(t, createBackendDeltaReport(artifact, output)) + raw, err := os.ReadFile(output) + require.NoError(t, err) + var report BackendDeltaReport + require.NoError(t, json.Unmarshal(raw, &report)) + require.False(t, report.Cases[0].ObservationsComparable) + require.False(t, report.Cases[0].ObservationsMatch) +} + +// TestBackendDeltaReportPreservesRepeatedRounds verifies that matched backend observations remain separate, ordered report cases for each measurement round. +func TestBackendDeltaReportPreservesRepeatedRounds(t *testing.T) { + root := t.TempDir() + artifact, output := filepath.Join(root, "records.jsonl"), filepath.Join(root, "delta.json") + var records []CaseResult + for round := 1; round <= 2; round++ { + for _, mode := range []ExecutionMode{ModePostgresSQL, ModeNeo4j} { + records = append(records, CaseResult{ + Dataset: "fixture", + Name: "case", + ExecutionMode: mode, + Status: StatusOK, + StableObservation: true, + ObservedRows: []string{"one"}, + RowCount: 1, + Environment: &RunEnvironment{Round: round}, + Stats: DurationStats{Median: time.Duration(round) * time.Millisecond}, + }) + } + } + require.NoError(t, writeJSONLFile(artifact, records)) + require.NoError(t, createBackendDeltaReport(artifact, output)) + raw, err := os.ReadFile(output) + require.NoError(t, err) + var report BackendDeltaReport + require.NoError(t, json.Unmarshal(raw, &report)) + require.Len(t, report.Cases, 2) + require.Equal(t, 1, report.Cases[0].Round) + require.Equal(t, 2, report.Cases[1].Round) +} + +// TestBackendDeltaReportPreservesIncompletePairs verifies a missing backend +// remains visible instead of disappearing from an intersection-only report. +func TestBackendDeltaReportPreservesIncompletePairs(t *testing.T) { + root := t.TempDir() + artifact, output := filepath.Join(root, "records.jsonl"), filepath.Join(root, "delta.json") + records := []CaseResult{{ + Dataset: "fixture", + Name: "postgres-only", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + }} + require.NoError(t, writeJSONLFile(artifact, records)) + require.NoError(t, createBackendDeltaReport(artifact, output)) + raw, err := os.ReadFile(output) + require.NoError(t, err) + var report BackendDeltaReport + require.NoError(t, json.Unmarshal(raw, &report)) + require.Equal(t, 2, report.Version) + require.Len(t, report.Cases, 1) + require.False(t, report.Cases[0].Complete) + require.Equal(t, "missing_neo4j", report.Cases[0].IncompleteReason) + require.Zero(t, report.Cases[0].MedianNeo4jOverPG) + require.Zero(t, report.Cases[0].P95Neo4jOverPG) +} diff --git a/cmd/graphbench/bundle.go b/cmd/graphbench/bundle.go new file mode 100644 index 00000000..2f38a802 --- /dev/null +++ b/cmd/graphbench/bundle.go @@ -0,0 +1,917 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bufio" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" +) + +// captureBundleVersion identifies the serialized schema revision for capture bundle. +const captureBundleVersion = 3 + +const captureBundleChecksumFile = "checksums.sha256" + +// CaptureBundleManifest inventories the benchmark artifacts and source provenance copied into a portable bundle. +type CaptureBundleManifest struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // Environment captures the environment in which the measurement ran. + Environment RunEnvironment `json:"environment"` + // RecordCount records case-result records included in the capture bundle. + RecordCount int `json:"record_count"` + // CorpusDeclaration contains the exact selected corpus declaration bundled for replay. + CorpusDeclaration string `json:"corpus_declaration"` + // RawArtifact identifies the uncopied artifact used as bundle input. + RawArtifact string `json:"raw_artifact"` + // Executable captures executable path, digest, and build metadata. + Executable string `json:"executable"` + // SourcePatch contains the tracked working-tree patch preserved as source provenance. + SourcePatch string `json:"source_patch"` + // UntrackedManifest names the bundle-relative JSON inventory of copied untracked sources. + UntrackedManifest string `json:"untracked_manifest"` + // SourceClean reports whether the captured working-tree fingerprint contains no tracked or untracked changes. + SourceClean bool `json:"source_clean"` + // Evidence contains named, checksummed gate and plan artifacts copied into the bundle. + Evidence []CaptureBundleEvidence `json:"evidence,omitempty"` +} + +// CaptureCorpusDeclaration preserves every selected workload field needed to +// reconstruct the exact benchmark corpus rather than only its backend index. +type CaptureCorpusDeclaration struct { + Version int `json:"version"` + Cases []ScaleCase `json:"cases"` +} + +// CaptureBundleEvidence identifies one auxiliary plan, A/A, correctness, resource, or decision artifact. +type CaptureBundleEvidence struct { + // Name is a stable, user-supplied evidence identity. + Name string `json:"name"` + // SourceSHA256 identifies the exact input bytes before copying. + SourceSHA256 string `json:"source_sha256"` + // Copy names the bundle-relative payload path. + Copy string `json:"copy"` +} + +// CaptureBundleEvidenceInput supplies one auxiliary artifact to a capture bundle. +type CaptureBundleEvidenceInput struct { + // Name is serialized as the evidence identity and file name stem. + Name string + // Path locates the source artifact copied into the bundle. + Path string +} + +// CaptureBundleVerification is the fail-closed result of validating a portable bundle. +type CaptureBundleVerification struct { + // Version identifies this verification result schema. + Version int `json:"version"` + // ManifestVersion is the bundle schema version read from manifest.json. + ManifestVersion int `json:"manifest_version"` + // SourceClean reports the source state declared by the bundle manifest. + SourceClean bool `json:"source_clean"` + // CheckedFiles records how many checksummed payload files were verified. + CheckedFiles int `json:"checked_files"` + // RecordCount records how many JSONL case records were decoded and matched to the manifest. + RecordCount int `json:"record_count"` + // Passed reports whether every structural, checksum, and provenance invariant succeeded. + Passed bool `json:"passed"` + // Reasons contains stable validation failures when Passed is false. + Reasons []string `json:"reasons,omitempty"` +} + +// UntrackedSource describes an untracked source file copied into an artifact bundle. +type UntrackedSource struct { + // Path records the untracked source path relative to the repository root. + Path string `json:"path"` + // SHA256 verifies the copied file's contents without depending on its path. + SHA256 string `json:"sha256"` + // Copy identifies the bundle-relative copy of an untracked source file. + Copy string `json:"copy"` +} + +// writeCaptureBundle copies run artifacts and provenance into a checksummed portable bundle. +func writeCaptureBundle(root string, corpus ScaleCorpus, records []CaseResult, environment RunEnvironment) error { + return writeCaptureBundleWithEvidence(root, corpus, records, environment, nil) +} + +// writeCaptureBundleWithEvidence copies run artifacts, auxiliary evidence, and provenance into a checksummed portable bundle. +func writeCaptureBundleWithEvidence(root string, corpus ScaleCorpus, records []CaseResult, environment RunEnvironment, evidenceInputs []CaptureBundleEvidenceInput) error { + root = filepath.Clean(root) + if root == "." || root == string(filepath.Separator) { + return fmt.Errorf("bundle directory must be a dedicated path") + } + if err := validateCaptureBundleDestination(root); err != nil { + return err + } + currentFingerprint, err := calculateWorkingTreeSHA256(root) + if err != nil { + return fmt.Errorf("fingerprint current source before bundle capture: %w", err) + } + if !isLowerHexSHA256(environment.DirtyDiffSHA256) || currentFingerprint != environment.DirtyDiffSHA256 { + return fmt.Errorf("current source fingerprint %s differs from run environment fingerprint %s", currentFingerprint, environment.DirtyDiffSHA256) + } + untracked, err := listUntrackedSources(root) + if err != nil { + return err + } + for _, dir := range []string{root, filepath.Join(root, "artifacts"), filepath.Join(root, "bin"), filepath.Join(root, "source-untracked")} { + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + } + + patch, err := exec.Command("git", "diff", "--binary", "HEAD", "--").Output() + if err != nil { + return fmt.Errorf("capture tracked source patch: %w", err) + } + if err := os.WriteFile(filepath.Join(root, "source.patch"), patch, 0o644); err != nil { + return err + } + + untrackedManifest := make([]UntrackedSource, 0, len(untracked)) + for _, source := range untracked { + destination := filepath.Join(root, "source-untracked", source) + if err := copyRegularFile(source, destination, 0o644); err != nil { + return fmt.Errorf("copy untracked source %s: %w", source, err) + } + checksum, err := fileSHA256(source) + if err != nil { + return err + } + untrackedManifest = append(untrackedManifest, UntrackedSource{ + Path: filepath.ToSlash(source), + SHA256: checksum, + Copy: filepath.ToSlash(filepath.Join("source-untracked", source)), + }) + } + if err := writeIndentedJSON(filepath.Join(root, "source-untracked-manifest.json"), untrackedManifest); err != nil { + return err + } + capturedFingerprint, err := capturedWorkingTreeSHA256(patch, untrackedManifest, root) + if err != nil { + return fmt.Errorf("fingerprint captured source: %w", err) + } + if !isLowerHexSHA256(environment.DirtyDiffSHA256) || capturedFingerprint != environment.DirtyDiffSHA256 { + return fmt.Errorf("captured source fingerprint %s differs from run environment fingerprint %s", capturedFingerprint, environment.DirtyDiffSHA256) + } + currentFingerprint, err = calculateWorkingTreeSHA256(root) + if err != nil { + return fmt.Errorf("fingerprint current source after bundle capture: %w", err) + } + if currentFingerprint != environment.DirtyDiffSHA256 { + return fmt.Errorf("source changed during bundle capture: current fingerprint %s differs from run environment fingerprint %s", currentFingerprint, environment.DirtyDiffSHA256) + } + + executable, err := os.Executable() + if err != nil { + return err + } + binaryName := "graphbench-" + environment.BinarySHA256 + if err := copyRegularFile(executable, filepath.Join(root, "bin", binaryName), 0o755); err != nil { + return fmt.Errorf("copy executable: %w", err) + } + if err := copyRegularFile("go.mod", filepath.Join(root, "go.mod"), 0o644); err != nil { + return err + } + if err := copyRegularFile("go.sum", filepath.Join(root, "go.sum"), 0o644); err != nil { + return err + } + cases := append([]ScaleCase(nil), corpus.Cases...) + sort.Slice(cases, func(i, j int) bool { + if cases[i].Source != cases[j].Source { + return cases[i].Source < cases[j].Source + } + if cases[i].Dataset != cases[j].Dataset { + return cases[i].Dataset < cases[j].Dataset + } + return cases[i].Name < cases[j].Name + }) + if err := writeIndentedJSON(filepath.Join(root, "corpus-declaration.json"), CaptureCorpusDeclaration{Version: 2, Cases: cases}); err != nil { + return err + } + if err := writeBundleJSONL(filepath.Join(root, "combined.jsonl"), records); err != nil { + return err + } + evidence, err := copyCaptureBundleEvidence(root, evidenceInputs) + if err != nil { + return err + } + + manifest := CaptureBundleManifest{ + Version: captureBundleVersion, + Environment: environment, + RecordCount: len(records), + CorpusDeclaration: "corpus-declaration.json", + RawArtifact: "combined.jsonl", + Executable: filepath.ToSlash(filepath.Join("bin", binaryName)), + SourcePatch: "source.patch", + UntrackedManifest: "source-untracked-manifest.json", + SourceClean: environment.DirtyDiffSHA256 == cleanWorkingTreeSHA256(), + Evidence: evidence, + } + if err := writeIndentedJSON(filepath.Join(root, "manifest.json"), manifest); err != nil { + return err + } + if err := writeBundleChecksums(root); err != nil { + return err + } + verification, err := verifyCaptureBundle(root, false) + if err != nil { + return err + } + if !verification.Passed { + return fmt.Errorf("verify capture bundle: %s", strings.Join(verification.Reasons, "; ")) + } + return nil +} + +// capturedWorkingTreeSHA256 reconstructs the exact byte framing used by +// workingTreeSHA256 from the patch and copied untracked payloads in a bundle. +func capturedWorkingTreeSHA256(patch []byte, untracked []UntrackedSource, root string) (string, error) { + digest := sha256.New() + writeWorkingTreePatchFingerprint(digest, patch) + entries := append([]UntrackedSource(nil), untracked...) + sort.Slice(entries, func(i, j int) bool { return entries[i].Path < entries[j].Path }) + seenPaths := map[string]struct{}{} + seenCopies := map[string]struct{}{} + for index, source := range entries { + if !validUntrackedSourcePath(source.Path) { + return "", fmt.Errorf("untracked source %d has invalid path %q", index, source.Path) + } + if _, duplicate := seenPaths[source.Path]; duplicate { + return "", fmt.Errorf("untracked source path %q is duplicated", source.Path) + } + seenPaths[source.Path] = struct{}{} + if !isLowerHexSHA256(source.SHA256) { + return "", fmt.Errorf("untracked source %q has invalid SHA-256", source.Path) + } + expectedCopy := filepath.ToSlash(filepath.Join("source-untracked", filepath.FromSlash(source.Path))) + if source.Copy != expectedCopy { + return "", fmt.Errorf("untracked source %q has noncanonical copy %q; expected %q", source.Path, source.Copy, expectedCopy) + } + copyPath, err := resolveBundlePath(root, source.Copy) + if err != nil { + return "", fmt.Errorf("untracked source %q copy: %w", source.Path, err) + } + if _, duplicate := seenCopies[source.Copy]; duplicate { + return "", fmt.Errorf("untracked source copy %q is duplicated", source.Copy) + } + seenCopies[source.Copy] = struct{}{} + content, err := os.ReadFile(copyPath) + if err != nil { + return "", fmt.Errorf("read untracked source %q copy: %w", source.Path, err) + } + actual := fmt.Sprintf("%x", sha256.Sum256(content)) + if actual != source.SHA256 { + return "", fmt.Errorf("untracked source %q digest does not match its copy", source.Path) + } + writeWorkingTreeUntrackedFingerprint(digest, source.Path, content) + } + return fmt.Sprintf("%x", digest.Sum(nil)), nil +} + +func validUntrackedSourcePath(path string) bool { + if path == "" || filepath.IsAbs(path) || path != filepath.ToSlash(path) { + return false + } + clean := filepath.Clean(filepath.FromSlash(path)) + return clean != "." && clean != ".." && !strings.HasPrefix(clean, ".."+string(filepath.Separator)) && filepath.ToSlash(clean) == path +} + +// validateCaptureBundleDestination rejects symlinks, non-directories, and stale +// payloads so every checksum inventory is constructed in a fresh destination. +func validateCaptureBundleDestination(root string) error { + info, err := os.Lstat(root) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return fmt.Errorf("inspect bundle destination: %w", err) + } + if !info.IsDir() { + return fmt.Errorf("bundle destination must be a directory") + } + entries, err := os.ReadDir(root) + if err != nil { + return fmt.Errorf("inspect bundle destination: %w", err) + } + if len(entries) != 0 { + return fmt.Errorf("bundle destination must not already contain files") + } + return nil +} + +// copyCaptureBundleEvidence validates stable names and copies every auxiliary artifact into the bundle. +func copyCaptureBundleEvidence(root string, inputs []CaptureBundleEvidenceInput) ([]CaptureBundleEvidence, error) { + seen := map[string]struct{}{} + evidence := make([]CaptureBundleEvidence, 0, len(inputs)) + for _, input := range inputs { + name := strings.TrimSpace(input.Name) + if !validBundleEvidenceName(name) { + return nil, fmt.Errorf("invalid capture bundle evidence name %q", input.Name) + } + if _, duplicate := seen[name]; duplicate { + return nil, fmt.Errorf("duplicate capture bundle evidence name %q", name) + } + seen[name] = struct{}{} + info, err := os.Lstat(input.Path) + if err != nil { + return nil, fmt.Errorf("stat capture bundle evidence %q: %w", name, err) + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("capture bundle evidence %q is not a regular file", name) + } + extension := strings.ToLower(filepath.Ext(input.Path)) + if extension == "" || len(extension) > 10 { + extension = ".artifact" + } + relative := filepath.ToSlash(filepath.Join("artifacts", name+extension)) + if err := copyRegularFile(input.Path, filepath.Join(root, filepath.FromSlash(relative)), 0o644); err != nil { + return nil, fmt.Errorf("copy capture bundle evidence %q: %w", name, err) + } + digest, err := fileSHA256(input.Path) + if err != nil { + return nil, err + } + evidence = append(evidence, CaptureBundleEvidence{Name: name, SourceSHA256: digest, Copy: relative}) + } + sort.Slice(evidence, func(i, j int) bool { return evidence[i].Name < evidence[j].Name }) + return evidence, nil +} + +// validBundleEvidenceName accepts stable path-independent artifact identities. +func validBundleEvidenceName(name string) bool { + if name == "" { + return false + } + for _, char := range name { + if (char < 'a' || char > 'z') && (char < '0' || char > '9') && char != '-' && char != '_' { + return false + } + } + return true +} + +// cleanWorkingTreeSHA256 returns the fingerprint emitted by workingTreeSHA256 for a clean source tree. +func cleanWorkingTreeSHA256() string { + return fmt.Sprintf("%x", sha256.Sum256(nil)) +} + +// listUntrackedSources returns untracked repository files eligible for inclusion in the bundle. +func listUntrackedSources(bundleRoot string) ([]string, error) { + gitPaths, err := gitUntrackedPaths() + if err != nil { + return nil, err + } + absRoot, _ := filepath.Abs(bundleRoot) + var paths []string + for _, path := range gitPaths { + absPath, err := filepath.Abs(path) + if err != nil { + return nil, err + } + if absPath == absRoot || strings.HasPrefix(absPath, absRoot+string(filepath.Separator)) { + continue + } + info, err := os.Lstat(path) + if err != nil { + return nil, err + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("untracked source %q is not a regular file", path) + } + paths = append(paths, filepath.Clean(path)) + } + return paths, nil +} + +// copyRegularFile copies one regular file to a newly created bundle path with the requested mode. +func copyRegularFile(source, destination string, mode os.FileMode) (err error) { + info, err := os.Lstat(source) + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return fmt.Errorf("source is not a regular file") + } + input, err := os.Open(source) + if err != nil { + return err + } + defer input.Close() + if err := os.MkdirAll(filepath.Dir(destination), 0o755); err != nil { + return err + } + output, err := os.OpenFile(destination, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode) + if err != nil { + return err + } + defer func() { + if closeErr := output.Close(); err == nil && closeErr != nil { + err = closeErr + } + }() + _, err = io.Copy(output, input) + return err +} + +// writeIndentedJSON writes one value as indented JSON with a trailing newline. +func writeIndentedJSON(path string, value any) (err error) { + output, err := os.Create(path) + if err != nil { + return err + } + defer func() { + if closeErr := output.Close(); err == nil && closeErr != nil { + err = closeErr + } + }() + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(value) +} + +// writeBundleJSONL writes case records as JSON Lines inside an artifact bundle. +func writeBundleJSONL(path string, records []CaseResult) (err error) { + output, err := os.Create(path) + if err != nil { + return err + } + defer func() { + if closeErr := output.Close(); err == nil && closeErr != nil { + err = closeErr + } + }() + return writeJSONL(output, records) +} + +// writeBundleChecksums writes sorted SHA-256 entries for every bundled file except the checksum file. +func writeBundleChecksums(root string) error { + var paths []string + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() || path == filepath.Join(root, captureBundleChecksumFile) { + return nil + } + paths = append(paths, path) + return nil + }) + if err != nil { + return err + } + sort.Strings(paths) + var lines strings.Builder + for _, path := range paths { + checksum, err := fileSHA256(path) + if err != nil { + return err + } + relative, err := filepath.Rel(root, path) + if err != nil { + return err + } + fmt.Fprintf(&lines, "%s %s\n", checksum, filepath.ToSlash(relative)) + } + return os.WriteFile(filepath.Join(root, captureBundleChecksumFile), []byte(lines.String()), 0o644) +} + +// verifyCaptureBundle validates bundle structure, every payload checksum, source provenance, and record count. +// When requireCleanSource is true, diagnostic bundles carrying a patch or untracked source are rejected. +func verifyCaptureBundle(root string, requireCleanSource bool) (CaptureBundleVerification, error) { + report := CaptureBundleVerification{Version: 1, Passed: true} + root = filepath.Clean(root) + rootInfo, err := os.Stat(root) + if err != nil { + return report, fmt.Errorf("stat capture bundle: %w", err) + } + if !rootInfo.IsDir() { + return report, fmt.Errorf("capture bundle path is not a directory: %s", root) + } + + checksums, reasons, err := readBundleChecksums(root) + if err != nil { + return report, err + } + report.Reasons = append(report.Reasons, reasons...) + for relative, expected := range checksums { + path, pathErr := resolveBundlePath(root, relative) + if pathErr != nil { + report.Reasons = append(report.Reasons, pathErr.Error()) + continue + } + info, statErr := os.Lstat(path) + if statErr != nil { + report.Reasons = append(report.Reasons, fmt.Sprintf("checksummed file %q is unavailable: %v", relative, statErr)) + continue + } + if !info.Mode().IsRegular() { + report.Reasons = append(report.Reasons, fmt.Sprintf("checksummed path %q is not a regular file", relative)) + continue + } + actual, digestErr := fileSHA256(path) + if digestErr != nil { + report.Reasons = append(report.Reasons, fmt.Sprintf("checksum %q: %v", relative, digestErr)) + continue + } + if actual != expected { + report.Reasons = append(report.Reasons, fmt.Sprintf("checksum mismatch for %q", relative)) + continue + } + report.CheckedFiles++ + } + + listedReasons, err := verifyBundleFileInventory(root, checksums) + if err != nil { + return report, err + } + report.Reasons = append(report.Reasons, listedReasons...) + + manifest, reasons := verifyBundleManifest(root, checksums) + report.ManifestVersion = manifest.Version + report.SourceClean = manifest.SourceClean + report.Reasons = append(report.Reasons, reasons...) + report.Reasons = append(report.Reasons, verifyBundleCorpus(root, manifest)...) + if requireCleanSource && !manifest.SourceClean { + report.Reasons = append(report.Reasons, "bundle source is not clean") + } + + recordCount, reasons := verifyBundleRecords(root, manifest) + report.RecordCount = recordCount + report.Reasons = append(report.Reasons, reasons...) + report.Passed = len(report.Reasons) == 0 + return report, nil +} + +func verifyBundleCorpus(root string, manifest CaptureBundleManifest) []string { + path, err := resolveBundlePath(root, manifest.CorpusDeclaration) + if err != nil { + return []string{err.Error()} + } + content, err := os.ReadFile(path) + if err != nil { + return []string{fmt.Sprintf("read corpus declaration: %v", err)} + } + var declaration CaptureCorpusDeclaration + decoder := json.NewDecoder(strings.NewReader(string(content))) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&declaration); err != nil { + return []string{fmt.Sprintf("decode corpus declaration: %v", err)} + } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + return []string{"corpus declaration contains trailing JSON data"} + } + if declaration.Version != 2 { + return []string{fmt.Sprintf("unsupported corpus declaration version %d", declaration.Version)} + } + identity := corpusIdentity(ScaleCorpus{Cases: declaration.Cases}) + if identity != manifest.Environment.CorpusSHA256 { + return []string{fmt.Sprintf("corpus declaration identity %s differs from manifest %s", identity, manifest.Environment.CorpusSHA256)} + } + return nil +} + +// createCaptureBundleVerification validates a portable bundle, writes its complete +// verification result, and reports whether it passed every requested invariant. +func createCaptureBundleVerification(root, outputPath string, requireCleanSource bool) (passed bool, err error) { + if outputPath != "" { + absoluteRoot, rootErr := filepath.Abs(filepath.Clean(root)) + absoluteOutput, outputErr := filepath.Abs(filepath.Clean(outputPath)) + if rootErr != nil { + return false, rootErr + } + if outputErr != nil { + return false, outputErr + } + if absoluteOutput == absoluteRoot || strings.HasPrefix(absoluteOutput, absoluteRoot+string(filepath.Separator)) { + return false, fmt.Errorf("bundle verification output must be outside the verified bundle") + } + } + report, err := verifyCaptureBundle(root, requireCleanSource) + if err != nil { + return false, err + } + + var output *os.File + if outputPath == "" { + output = os.Stdout + } else { + if err := ensureOutputDir(outputPath); err != nil { + return false, err + } + output, err = os.Create(outputPath) + if err != nil { + return false, err + } + defer func() { + if closeErr := output.Close(); err == nil && closeErr != nil { + err = closeErr + passed = false + } + }() + } + + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + if err := encoder.Encode(report); err != nil { + return false, err + } + return report.Passed, nil +} + +// readBundleChecksums parses the deterministic SHA-256 manifest without trusting its paths. +func readBundleChecksums(root string) (map[string]string, []string, error) { + path := filepath.Join(root, captureBundleChecksumFile) + input, err := os.Open(path) + if err != nil { + return nil, nil, fmt.Errorf("open capture bundle checksums: %w", err) + } + defer input.Close() + + checksums := map[string]string{} + var reasons []string + scanner := bufio.NewScanner(input) + lineNumber := 0 + for scanner.Scan() { + lineNumber++ + line := scanner.Text() + separator := strings.Index(line, " ") + if separator != 64 || len(line) <= separator+2 { + reasons = append(reasons, fmt.Sprintf("malformed checksum line %d", lineNumber)) + continue + } + digest := line[:separator] + relative := line[separator+2:] + if !isLowerHexSHA256(digest) { + reasons = append(reasons, fmt.Sprintf("invalid SHA-256 on checksum line %d", lineNumber)) + continue + } + if _, duplicate := checksums[relative]; duplicate { + reasons = append(reasons, fmt.Sprintf("duplicate checksum path %q", relative)) + continue + } + checksums[relative] = digest + } + if err := scanner.Err(); err != nil { + return nil, nil, fmt.Errorf("read capture bundle checksums: %w", err) + } + if len(checksums) == 0 { + reasons = append(reasons, "capture bundle checksum manifest is empty") + } + return checksums, reasons, nil +} + +// isLowerHexSHA256 reports whether value is one canonical lowercase SHA-256 digest. +func isLowerHexSHA256(value string) bool { + if len(value) != 64 { + return false + } + for _, char := range value { + if (char < '0' || char > '9') && (char < 'a' || char > 'f') { + return false + } + } + return true +} + +// resolveBundlePath rejects absolute, parent, platform-ambiguous, and checksum-self references. +func resolveBundlePath(root, relative string) (string, error) { + if relative == "" || filepath.IsAbs(relative) || relative != filepath.ToSlash(relative) { + return "", fmt.Errorf("invalid bundle-relative path %q", relative) + } + clean := filepath.Clean(filepath.FromSlash(relative)) + if clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) || relative == captureBundleChecksumFile { + return "", fmt.Errorf("invalid bundle-relative path %q", relative) + } + return filepath.Join(root, clean), nil +} + +// verifyBundleFileInventory rejects unchecksummed payload files and missing checksum entries. +func verifyBundleFileInventory(root string, checksums map[string]string) ([]string, error) { + var reasons []string + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() { + if path != root { + info, err := entry.Info() + if err != nil { + return err + } + if info.Mode()&os.ModeSymlink != 0 { + reasons = append(reasons, fmt.Sprintf("bundle contains symlink directory %q", path)) + return filepath.SkipDir + } + } + return nil + } + relative, err := filepath.Rel(root, path) + if err != nil { + return err + } + relative = filepath.ToSlash(relative) + if relative == captureBundleChecksumFile { + return nil + } + if _, listed := checksums[relative]; !listed { + reasons = append(reasons, fmt.Sprintf("unchecksummed bundle file %q", relative)) + } + return nil + }) + return reasons, err +} + +// verifyBundleManifest decodes the manifest and validates every referenced payload identity. +func verifyBundleManifest(root string, checksums map[string]string) (CaptureBundleManifest, []string) { + var manifest CaptureBundleManifest + var reasons []string + manifestPath, present := checksums["manifest.json"] + if !present || manifestPath == "" { + return manifest, []string{"manifest.json is not checksummed"} + } + content, err := os.ReadFile(filepath.Join(root, "manifest.json")) + if err != nil { + return manifest, []string{fmt.Sprintf("read manifest.json: %v", err)} + } + decoder := json.NewDecoder(strings.NewReader(string(content))) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&manifest); err != nil { + return manifest, []string{fmt.Sprintf("decode manifest.json: %v", err)} + } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + return manifest, []string{"manifest.json contains trailing JSON data"} + } + if manifest.Version != captureBundleVersion { + reasons = append(reasons, fmt.Sprintf("unsupported capture bundle version %d", manifest.Version)) + } + for label, relative := range map[string]string{ + "corpus declaration": manifest.CorpusDeclaration, + "raw artifact": manifest.RawArtifact, + "executable": manifest.Executable, + "source patch": manifest.SourcePatch, + "untracked manifest": manifest.UntrackedManifest, + } { + if _, err := resolveBundlePath(root, relative); err != nil { + reasons = append(reasons, fmt.Sprintf("%s: %v", label, err)) + continue + } + if _, exists := checksums[relative]; !exists { + reasons = append(reasons, fmt.Sprintf("%s %q is not checksummed", label, relative)) + } + } + evidenceNames := map[string]struct{}{} + for _, artifact := range manifest.Evidence { + if !validBundleEvidenceName(artifact.Name) { + reasons = append(reasons, fmt.Sprintf("invalid evidence name %q", artifact.Name)) + } + if _, duplicate := evidenceNames[artifact.Name]; duplicate { + reasons = append(reasons, fmt.Sprintf("duplicate evidence name %q", artifact.Name)) + } + evidenceNames[artifact.Name] = struct{}{} + path, pathErr := resolveBundlePath(root, artifact.Copy) + if pathErr != nil { + reasons = append(reasons, fmt.Sprintf("evidence %q: %v", artifact.Name, pathErr)) + continue + } + listedDigest, listed := checksums[artifact.Copy] + if !listed { + reasons = append(reasons, fmt.Sprintf("evidence %q copy %q is not checksummed", artifact.Name, artifact.Copy)) + continue + } + if !isLowerHexSHA256(artifact.SourceSHA256) || listedDigest != artifact.SourceSHA256 { + reasons = append(reasons, fmt.Sprintf("evidence %q source identity does not match its bundled copy", artifact.Name)) + continue + } + if digest, digestErr := fileSHA256(path); digestErr != nil || digest != artifact.SourceSHA256 { + reasons = append(reasons, fmt.Sprintf("evidence %q payload identity is invalid", artifact.Name)) + } + } + if manifest.Environment.BinarySHA256 == "" || manifest.Environment.BinarySHA256 == "unknown" { + reasons = append(reasons, "manifest has no concrete executable SHA-256") + } else if executablePath, err := resolveBundlePath(root, manifest.Executable); err == nil { + if digest, digestErr := fileSHA256(executablePath); digestErr != nil || digest != manifest.Environment.BinarySHA256 { + reasons = append(reasons, "manifest executable identity does not match bundled executable") + } + } + if manifest.Environment.SourceCommit == "" || manifest.Environment.SourceCommit == "unknown" { + reasons = append(reasons, "manifest has no concrete source commit") + } + if manifest.SourceClean && manifest.Environment.DirtyDiffSHA256 != cleanWorkingTreeSHA256() { + reasons = append(reasons, "clean-source declaration contradicts dirty source fingerprint") + } + if manifest.SourceClean { + patchPath, patchErr := resolveBundlePath(root, manifest.SourcePatch) + if patchErr == nil { + if patchInfo, err := os.Stat(patchPath); err != nil || patchInfo.Size() != 0 { + reasons = append(reasons, "clean-source bundle contains a non-empty source patch") + } + } + untrackedPath, untrackedErr := resolveBundlePath(root, manifest.UntrackedManifest) + if untrackedErr == nil { + var untracked []UntrackedSource + content, err := os.ReadFile(untrackedPath) + if err != nil || json.Unmarshal(content, &untracked) != nil || len(untracked) != 0 { + reasons = append(reasons, "clean-source bundle contains untracked source entries") + } + } + } + patchPath, patchErr := resolveBundlePath(root, manifest.SourcePatch) + untrackedPath, untrackedErr := resolveBundlePath(root, manifest.UntrackedManifest) + if patchErr == nil && untrackedErr == nil { + patch, readPatchErr := os.ReadFile(patchPath) + untracked, decodeReasons := readUntrackedSourceManifest(untrackedPath) + reasons = append(reasons, decodeReasons...) + if readPatchErr != nil { + reasons = append(reasons, fmt.Sprintf("read bundled source patch: %v", readPatchErr)) + } else if len(decodeReasons) == 0 { + fingerprint, fingerprintErr := capturedWorkingTreeSHA256(patch, untracked, root) + if fingerprintErr != nil { + reasons = append(reasons, "reconstruct bundled source fingerprint: "+fingerprintErr.Error()) + } else { + if !isLowerHexSHA256(manifest.Environment.DirtyDiffSHA256) || fingerprint != manifest.Environment.DirtyDiffSHA256 { + reasons = append(reasons, "manifest dirty source fingerprint does not match bundled patch and untracked sources") + } + if manifest.SourceClean != (fingerprint == cleanWorkingTreeSHA256()) { + reasons = append(reasons, "source_clean declaration does not match bundled source fingerprint") + } + } + } + manifestCopies := map[string]struct{}{} + for _, source := range untracked { + manifestCopies[source.Copy] = struct{}{} + if _, listed := checksums[source.Copy]; !listed { + reasons = append(reasons, fmt.Sprintf("untracked source %q copy %q is not checksummed", source.Path, source.Copy)) + } + } + for relative := range checksums { + if strings.HasPrefix(relative, "source-untracked/") { + if _, declared := manifestCopies[relative]; !declared { + reasons = append(reasons, fmt.Sprintf("checksummed untracked source copy %q has no manifest entry", relative)) + } + } + } + } + return manifest, reasons +} + +func readUntrackedSourceManifest(path string) ([]UntrackedSource, []string) { + content, err := os.ReadFile(path) + if err != nil { + return nil, []string{fmt.Sprintf("read untracked source manifest: %v", err)} + } + if len(strings.TrimSpace(string(content))) == 0 || strings.TrimSpace(string(content))[0] != '[' { + return nil, []string{"untracked source manifest must be a JSON array"} + } + var sources []UntrackedSource + decoder := json.NewDecoder(strings.NewReader(string(content))) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&sources); err != nil { + return nil, []string{fmt.Sprintf("decode untracked source manifest: %v", err)} + } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + return nil, []string{"untracked source manifest contains trailing JSON data"} + } + return sources, nil +} + +// verifyBundleRecords decodes the JSONL payload and binds every record to the manifest environment. +func verifyBundleRecords(root string, manifest CaptureBundleManifest) (int, []string) { + artifactPath, err := resolveBundlePath(root, manifest.RawArtifact) + if err != nil { + return 0, []string{err.Error()} + } + records, err := readJSONLFile(artifactPath) + if err != nil { + return 0, []string{fmt.Sprintf("decode bundled records: %v", err)} + } + var reasons []string + if len(records) != manifest.RecordCount { + reasons = append(reasons, fmt.Sprintf("manifest record count %d does not match artifact count %d", manifest.RecordCount, len(records))) + } + for index, record := range records { + if record.Environment == nil { + reasons = append(reasons, fmt.Sprintf("record %d has no environment provenance", index)) + continue + } + if record.Environment.BinarySHA256 != manifest.Environment.BinarySHA256 || + record.Environment.SourceCommit != manifest.Environment.SourceCommit || + record.Environment.DirtyDiffSHA256 != manifest.Environment.DirtyDiffSHA256 || + record.Environment.CorpusSHA256 != manifest.Environment.CorpusSHA256 { + reasons = append(reasons, fmt.Sprintf("record %d provenance does not match bundle manifest", index)) + } + } + return len(records), reasons +} diff --git a/cmd/graphbench/bundle_test.go b/cmd/graphbench/bundle_test.go new file mode 100644 index 00000000..cdd30084 --- /dev/null +++ b/cmd/graphbench/bundle_test.go @@ -0,0 +1,294 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestVerifyCaptureBundleValidatesChecksumsAndProvenance exercises the portable bundle verifier without depending on a built graphbench executable. +func TestVerifyCaptureBundleValidatesChecksumsAndProvenance(t *testing.T) { + root := t.TempDir() + environment := RunEnvironment{ + ArtifactSchemaVersion: 2, + CorpusSHA256: corpusIdentity(ScaleCorpus{}), + SourceCommit: "commit", + DirtyDiffSHA256: cleanWorkingTreeSHA256(), + BinarySHA256: "placeholder", + } + record := CaseResult{ + Environment: &environment, + Dataset: "fixture", + Name: "case", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + } + require.NoError(t, os.MkdirAll(filepath.Join(root, "bin"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(root, "bin", "graphbench"), []byte("binary"), 0o755)) + binarySHA, err := fileSHA256(filepath.Join(root, "bin", "graphbench")) + require.NoError(t, err) + environment.BinarySHA256 = binarySHA + record.Environment.BinarySHA256 = binarySHA + + require.NoError(t, os.WriteFile(filepath.Join(root, "source.patch"), nil, 0o644)) + require.NoError(t, writeIndentedJSON(filepath.Join(root, "source-untracked-manifest.json"), []UntrackedSource{})) + require.NoError(t, writeIndentedJSON(filepath.Join(root, "corpus-declaration.json"), CaptureCorpusDeclaration{Version: 2})) + require.NoError(t, writeBundleJSONL(filepath.Join(root, "combined.jsonl"), []CaseResult{record})) + require.NoError(t, writeIndentedJSON(filepath.Join(root, "manifest.json"), CaptureBundleManifest{ + Version: captureBundleVersion, + Environment: environment, + RecordCount: 1, + CorpusDeclaration: "corpus-declaration.json", + RawArtifact: "combined.jsonl", + Executable: "bin/graphbench", + SourcePatch: "source.patch", + UntrackedManifest: "source-untracked-manifest.json", + SourceClean: true, + })) + require.NoError(t, writeBundleChecksums(root)) + + report, err := verifyCaptureBundle(root, true) + require.NoError(t, err) + require.True(t, report.Passed, report.Reasons) + require.Equal(t, 6, report.CheckedFiles) + require.Equal(t, 1, report.RecordCount) + + outputPath := filepath.Join(t.TempDir(), "verification.json") + passed, err := createCaptureBundleVerification(root, outputPath, true) + require.NoError(t, err) + require.True(t, passed) + content, err := os.ReadFile(outputPath) + require.NoError(t, err) + var written CaptureBundleVerification + require.NoError(t, json.Unmarshal(content, &written)) + require.Equal(t, report, written) + + _, err = createCaptureBundleVerification(root, filepath.Join(root, "verification.json"), true) + require.ErrorContains(t, err, "must be outside the verified bundle") + + manifestPath := filepath.Join(root, "manifest.json") + manifestContent, err := os.ReadFile(manifestPath) + require.NoError(t, err) + require.NoError(t, os.WriteFile(manifestPath, append(manifestContent, []byte("{}\n")...), 0o644)) + require.NoError(t, writeBundleChecksums(root)) + report, err = verifyCaptureBundle(root, true) + require.NoError(t, err) + require.False(t, report.Passed) + require.Contains(t, report.Reasons, "manifest.json contains trailing JSON data") +} + +// TestVerifyCaptureBundleFailsClosedOnTamperingDirtySourceAndUnlistedFiles covers the three qualification boundaries a checksum-only writer cannot enforce. +func TestVerifyCaptureBundleFailsClosedOnTamperingDirtySourceAndUnlistedFiles(t *testing.T) { + root := t.TempDir() + environment := RunEnvironment{ + ArtifactSchemaVersion: 2, + CorpusSHA256: corpusIdentity(ScaleCorpus{}), + SourceCommit: "commit", + DirtyDiffSHA256: "dirty", + BinarySHA256: "placeholder", + } + require.NoError(t, os.WriteFile(filepath.Join(root, "binary"), []byte("binary"), 0o755)) + binarySHA, err := fileSHA256(filepath.Join(root, "binary")) + require.NoError(t, err) + environment.BinarySHA256 = binarySHA + recordEnvironment := environment + record := CaseResult{Environment: &recordEnvironment, Dataset: "fixture", Name: "case", ExecutionMode: ModePostgresSQL, Status: StatusOK} + require.NoError(t, os.WriteFile(filepath.Join(root, "source.patch"), []byte("diff"), 0o644)) + require.NoError(t, writeIndentedJSON(filepath.Join(root, "untracked.json"), []UntrackedSource{})) + require.NoError(t, writeIndentedJSON(filepath.Join(root, "corpus.json"), CaptureCorpusDeclaration{Version: 2})) + require.NoError(t, writeBundleJSONL(filepath.Join(root, "records.jsonl"), []CaseResult{record})) + require.NoError(t, writeIndentedJSON(filepath.Join(root, "manifest.json"), CaptureBundleManifest{ + Version: captureBundleVersion, + Environment: environment, + RecordCount: 1, + CorpusDeclaration: "corpus.json", + RawArtifact: "records.jsonl", + Executable: "binary", + SourcePatch: "source.patch", + UntrackedManifest: "untracked.json", + SourceClean: false, + })) + require.NoError(t, writeBundleChecksums(root)) + require.NoError(t, os.WriteFile(filepath.Join(root, "records.jsonl"), []byte("tampered\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(root, "unlisted"), []byte("payload"), 0o644)) + + report, err := verifyCaptureBundle(root, true) + require.NoError(t, err) + require.False(t, report.Passed) + require.Contains(t, report.Reasons, "checksum mismatch for \"records.jsonl\"") + require.Contains(t, report.Reasons, "unchecksummed bundle file \"unlisted\"") + require.Contains(t, report.Reasons, "bundle source is not clean") + + outputPath := filepath.Join(t.TempDir(), "failed-verification.json") + passed, err := createCaptureBundleVerification(root, outputPath, true) + require.NoError(t, err) + require.False(t, passed) + content, err := os.ReadFile(outputPath) + require.NoError(t, err) + var written CaptureBundleVerification + require.NoError(t, json.Unmarshal(content, &written)) + require.False(t, written.Passed) + require.NotEmpty(t, written.Reasons) +} + +// TestResolveBundlePathRejectsTraversal verifies checksum manifests cannot escape the capture root. +func TestResolveBundlePathRejectsTraversal(t *testing.T) { + _, err := resolveBundlePath(t.TempDir(), "../escape") + require.ErrorContains(t, err, "invalid bundle-relative path") +} + +// TestCopyCaptureBundleEvidenceUsesStableNamesAndDigests verifies auxiliary plan/gate inputs are copied without retaining host paths. +func TestCopyCaptureBundleEvidenceUsesStableNamesAndDigests(t *testing.T) { + root := t.TempDir() + input := filepath.Join(t.TempDir(), "aa-report.json") + require.NoError(t, os.WriteFile(input, []byte(`{"version":1}`), 0o644)) + + evidence, err := copyCaptureBundleEvidence(root, []CaptureBundleEvidenceInput{{Name: "host-aa", Path: input}}) + require.NoError(t, err) + require.Len(t, evidence, 1) + require.Equal(t, "host-aa", evidence[0].Name) + require.Equal(t, "artifacts/host-aa.json", evidence[0].Copy) + require.FileExists(t, filepath.Join(root, "artifacts", "host-aa.json")) + require.NotContains(t, evidence[0].Copy, filepath.Dir(input)) + + _, err = copyCaptureBundleEvidence(root, []CaptureBundleEvidenceInput{{Name: "../escape", Path: input}}) + require.ErrorContains(t, err, "invalid capture bundle evidence name") + + symlink := filepath.Join(t.TempDir(), "outside.json") + require.NoError(t, os.Symlink(input, symlink)) + _, err = copyCaptureBundleEvidence(root, []CaptureBundleEvidenceInput{{Name: "symlink", Path: symlink}}) + require.ErrorContains(t, err, "is not a regular file") +} + +// TestWriteCaptureBundleRejectsNonemptyDestination verifies stale payloads cannot leak into a newly checksummed bundle inventory. +func TestWriteCaptureBundleRejectsNonemptyDestination(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "stale.json"), []byte("stale"), 0o644)) + + err := writeCaptureBundleWithEvidence(root, ScaleCorpus{}, nil, RunEnvironment{}, nil) + require.ErrorContains(t, err, "must not already contain files") + require.FileExists(t, filepath.Join(root, "stale.json")) +} + +func TestWriteCaptureBundleRejectsStaleRunEnvironmentFingerprint(t *testing.T) { + root := filepath.Join(t.TempDir(), "bundle") + err := writeCaptureBundleWithEvidence(root, ScaleCorpus{}, nil, RunEnvironment{ + DirtyDiffSHA256: strings.Repeat("0", 64), + }, nil) + require.ErrorContains(t, err, "current source fingerprint") + require.NoDirExists(t, root) +} + +func TestParseNULTerminatedPathsPreservesWhitespace(t *testing.T) { + require.Equal(t, []string{"dir/name with spaces.go", "line\nbreak.go"}, parseNULTerminatedPaths([]byte("dir/name with spaces.go\x00line\nbreak.go\x00"))) +} + +// TestCopyRegularFileRejectsSymlink verifies the shared source copier cannot follow an untracked-source symlink outside the repository. +func TestCopyRegularFileRejectsSymlink(t *testing.T) { + source := filepath.Join(t.TempDir(), "outside") + link := filepath.Join(t.TempDir(), "untracked-link") + require.NoError(t, os.WriteFile(source, []byte("outside"), 0o644)) + require.NoError(t, os.Symlink(source, link)) + + err := copyRegularFile(link, filepath.Join(t.TempDir(), "copy"), 0o644) + require.ErrorContains(t, err, "source is not a regular file") +} + +func TestVerifyCaptureBundleBindsDirtyFingerprintToPatchAndUntrackedCopies(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(root, "source-untracked", "pkg"), 0o755)) + patch := []byte("diff --git a/a.go b/a.go\n") + content := []byte("package pkg\n") + require.NoError(t, os.WriteFile(filepath.Join(root, "source.patch"), patch, 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(root, "source-untracked", "pkg", "new.go"), content, 0o644)) + contentSHA := fmt.Sprintf("%x", sha256.Sum256(content)) + untracked := []UntrackedSource{{Path: "pkg/new.go", SHA256: contentSHA, Copy: "source-untracked/pkg/new.go"}} + require.NoError(t, writeIndentedJSON(filepath.Join(root, "untracked.json"), untracked)) + fingerprint, err := capturedWorkingTreeSHA256(patch, untracked, root) + require.NoError(t, err) + + require.NoError(t, os.WriteFile(filepath.Join(root, "binary"), []byte("binary"), 0o755)) + binarySHA, err := fileSHA256(filepath.Join(root, "binary")) + require.NoError(t, err) + environment := RunEnvironment{SourceCommit: "commit", DirtyDiffSHA256: fingerprint, BinarySHA256: binarySHA, CorpusSHA256: corpusIdentity(ScaleCorpus{})} + recordEnvironment := environment + require.NoError(t, writeBundleJSONL(filepath.Join(root, "records.jsonl"), []CaseResult{{Environment: &recordEnvironment}})) + require.NoError(t, writeIndentedJSON(filepath.Join(root, "corpus.json"), CaptureCorpusDeclaration{Version: 2})) + require.NoError(t, writeIndentedJSON(filepath.Join(root, "manifest.json"), CaptureBundleManifest{ + Version: captureBundleVersion, Environment: environment, RecordCount: 1, + CorpusDeclaration: "corpus.json", RawArtifact: "records.jsonl", Executable: "binary", + SourcePatch: "source.patch", UntrackedManifest: "untracked.json", SourceClean: false, + })) + require.NoError(t, writeBundleChecksums(root)) + + report, err := verifyCaptureBundle(root, false) + require.NoError(t, err) + require.True(t, report.Passed, report.Reasons) + + manifestPath := filepath.Join(root, "manifest.json") + var manifest CaptureBundleManifest + raw, err := os.ReadFile(manifestPath) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(raw, &manifest)) + manifest.Environment.DirtyDiffSHA256 = strings.Repeat("0", 64) + require.NoError(t, writeIndentedJSON(manifestPath, manifest)) + require.NoError(t, writeBundleChecksums(root)) + report, err = verifyCaptureBundle(root, false) + require.NoError(t, err) + require.False(t, report.Passed) + require.Contains(t, report.Reasons, "manifest dirty source fingerprint does not match bundled patch and untracked sources") +} + +func TestVerifyCaptureBundleRejectsMalformedOrUnchecksummedUntrackedEntries(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(root, "source-untracked"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(root, "source.patch"), nil, 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(root, "source-untracked", "new.go"), []byte("package p\n"), 0o644)) + require.NoError(t, writeIndentedJSON(filepath.Join(root, "untracked.json"), []UntrackedSource{{ + Path: "../escape.go", SHA256: "bad", Copy: "source-untracked/new.go", + }})) + require.NoError(t, os.WriteFile(filepath.Join(root, "binary"), []byte("binary"), 0o755)) + binarySHA, err := fileSHA256(filepath.Join(root, "binary")) + require.NoError(t, err) + environment := RunEnvironment{SourceCommit: "commit", DirtyDiffSHA256: strings.Repeat("0", 64), BinarySHA256: binarySHA, CorpusSHA256: corpusIdentity(ScaleCorpus{})} + recordEnvironment := environment + require.NoError(t, writeBundleJSONL(filepath.Join(root, "records.jsonl"), []CaseResult{{Environment: &recordEnvironment}})) + require.NoError(t, writeIndentedJSON(filepath.Join(root, "corpus.json"), CaptureCorpusDeclaration{Version: 2})) + require.NoError(t, writeIndentedJSON(filepath.Join(root, "manifest.json"), CaptureBundleManifest{ + Version: captureBundleVersion, Environment: environment, RecordCount: 1, + CorpusDeclaration: "corpus.json", RawArtifact: "records.jsonl", Executable: "binary", + SourcePatch: "source.patch", UntrackedManifest: "untracked.json", SourceClean: false, + })) + require.NoError(t, writeBundleChecksums(root)) + checksums, _, err := readBundleChecksums(root) + require.NoError(t, err) + delete(checksums, "source-untracked/new.go") + var lines strings.Builder + paths := make([]string, 0, len(checksums)) + for path := range checksums { + paths = append(paths, path) + } + sort.Strings(paths) + for _, path := range paths { + fmt.Fprintf(&lines, "%s %s\n", checksums[path], path) + } + require.NoError(t, os.WriteFile(filepath.Join(root, captureBundleChecksumFile), []byte(lines.String()), 0o644)) + + report, err := verifyCaptureBundle(root, false) + require.NoError(t, err) + require.False(t, report.Passed) + require.Contains(t, strings.Join(report.Reasons, "\n"), "invalid path") + require.Contains(t, report.Reasons, "untracked source \"../escape.go\" copy \"source-untracked/new.go\" is not checksummed") +} diff --git a/cmd/graphbench/concurrency.go b/cmd/graphbench/concurrency.go new file mode 100644 index 00000000..d6c5d8c6 --- /dev/null +++ b/cmd/graphbench/concurrency.go @@ -0,0 +1,181 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "fmt" + "sort" + "strconv" + "sync" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// measurePostgresConcurrency runs requested concurrency levels and records latency and connection reuse. +func measurePostgresConcurrency( + ctx context.Context, + pool *pgxpool.Pool, + sqlQuery string, + parameters map[string]any, + poolSize int, + levels []int, + iterations int, + isolation ...pgx.TxIsoLevel, +) ([]ConcurrencyBlock, error) { + blocks := make([]ConcurrencyBlock, 0, len(levels)) + for _, concurrency := range levels { + block, err := measurePostgresConcurrencyBlock(ctx, pool, sqlQuery, parameters, poolSize, concurrency, iterations, isolation...) + if err != nil { + return nil, fmt.Errorf("concurrency %d: %w", concurrency, err) + } + blocks = append(blocks, block) + } + return blocks, nil +} + +// measurePostgresConcurrencyBlock coordinates workers for one concurrency level and aggregates their samples. +func measurePostgresConcurrencyBlock( + ctx context.Context, + pool *pgxpool.Pool, + sqlQuery string, + parameters map[string]any, + poolSize, concurrency, iterations int, + isolation ...pgx.TxIsoLevel, +) (ConcurrencyBlock, error) { + var ( + startBarrier = make(chan struct{}) + wg sync.WaitGroup + mutex sync.Mutex + samples = make([]ConcurrencySample, 0, concurrency*iterations) + errorsSeen []error + seenPID = map[uint32]struct{}{} + ) + blockStart := time.Now() + for worker := range concurrency { + wg.Add(1) + go func() { + defer wg.Done() + <-startBarrier + for iteration := range iterations { + sample, pid, err := measurePostgresConcurrentIteration(ctx, pool, sqlQuery, parameters, worker+1, iteration+1, isolation...) + mutex.Lock() + if err != nil { + errorsSeen = append(errorsSeen, err) + mutex.Unlock() + return + } + if _, found := seenPID[pid]; found { + sample.Classification = "warm-session" + } else { + seenPID[pid] = struct{}{} + sample.Classification = "cold-session" + } + samples = append(samples, sample) + mutex.Unlock() + } + }() + } + close(startBarrier) + wg.Wait() + wall := time.Since(blockStart) + if len(errorsSeen) > 0 { + return ConcurrencyBlock{}, errorsSeen[0] + } + sort.Slice(samples, func(i, j int) bool { + if samples[i].Worker != samples[j].Worker { + return samples[i].Worker < samples[j].Worker + } + return samples[i].Iteration < samples[j].Iteration + }) + return ConcurrencyBlock{ + Concurrency: concurrency, + PoolSize: poolSize, + Operations: len(samples), + Wall: wall, + QPS: float64(len(samples)) / wall.Seconds(), + Samples: samples, + }, nil +} + +// measurePostgresConcurrentIteration executes one timed query in a transaction and records its backend process ID. +func measurePostgresConcurrentIteration( + ctx context.Context, + pool *pgxpool.Pool, + sqlQuery string, + parameters map[string]any, + worker, iteration int, + isolation ...pgx.TxIsoLevel, +) (ConcurrencySample, uint32, error) { + totalStart := time.Now() + acquireStart := time.Now() + conn, err := pool.Acquire(ctx) + if err != nil { + return ConcurrencySample{}, 0, err + } + defer conn.Release() + + poolWait := time.Since(acquireStart) + pid := conn.Conn().PgConn().PID() + + txStart := time.Now() + // DAWGS read queries may create and reset session-local workspace tables. + // Keep the transaction read-write, matching drivers/pg ReadTransaction, + // while rolling it back after the measurement. + txOptions := postgresConcurrencyTxOptions(isolation...) + tx, err := conn.BeginTx(ctx, txOptions) + if err != nil { + return ConcurrencySample{}, 0, err + } + defer func() { _ = tx.Rollback(ctx) }() + + transactionDuration := time.Since(txStart) + + queryArgs := []any{pgx.QueryExecModeCacheStatement, pgx.QueryResultFormats{pgx.BinaryFormatCode}} + if len(parameters) > 0 { + queryArgs = append(queryArgs, pgx.NamedArgs(parameters)) + } + executeStart := time.Now() + rows, err := tx.Query(ctx, sqlQuery, queryArgs...) + if err != nil { + return ConcurrencySample{}, 0, err + } + for rows.Next() { + if _, err := rows.Values(); err != nil { + rows.Close() + return ConcurrencySample{}, 0, err + } + } + rows.Close() + if err := rows.Err(); err != nil { + return ConcurrencySample{}, 0, err + } + executeDuration := time.Since(executeStart) + if err := tx.Rollback(ctx); err != nil { + return ConcurrencySample{}, 0, err + } + + return ConcurrencySample{ + Worker: worker, + Iteration: iteration, + ConnectionID: strconv.FormatUint(uint64(pid), 10), + PoolWait: poolWait, + Transaction: transactionDuration, + ExecuteDrain: executeDuration, + Total: time.Since(totalStart), + }, pid, nil +} + +// postgresConcurrencyTxOptions returns transaction options that preserve session-local workspace maintenance. +func postgresConcurrencyTxOptions(isolation ...pgx.TxIsoLevel) pgx.TxOptions { + options := pgx.TxOptions{AccessMode: pgx.ReadWrite} + if len(isolation) > 0 { + options.IsoLevel = isolation[0] + } + return options +} diff --git a/cmd/graphbench/concurrency_test.go b/cmd/graphbench/concurrency_test.go new file mode 100644 index 00000000..fbfdc75d --- /dev/null +++ b/cmd/graphbench/concurrency_test.go @@ -0,0 +1,20 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" + + "github.com/jackc/pgx/v5" + "github.com/stretchr/testify/require" +) + +// TestPostgresConcurrencyTransactionsPermitSessionWorkspaceMaintenance verifies that concurrent benchmark transactions are read-write so session-scoped workspace tables can be maintained. +func TestPostgresConcurrencyTransactionsPermitSessionWorkspaceMaintenance(t *testing.T) { + require.Equal(t, pgx.ReadWrite, postgresConcurrencyTxOptions().AccessMode) + require.Empty(t, postgresConcurrencyTxOptions().IsoLevel) + require.Equal(t, pgx.RepeatableRead, postgresConcurrencyTxOptions(pgx.RepeatableRead).IsoLevel) +} diff --git a/cmd/graphbench/confirm_report.go b/cmd/graphbench/confirm_report.go new file mode 100644 index 00000000..bda02deb --- /dev/null +++ b/cmd/graphbench/confirm_report.go @@ -0,0 +1,614 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "math/rand" + "os" + "regexp" + "sort" + "strings" + "time" +) + +// confirmationReportVersion identifies the JSON schema emitted by confirmation reports. +const confirmationReportVersion = 4 + +// ConfirmationOptions selects the paired artifacts, cases, confidence level, and bootstrap seed used for confirmation. +type ConfirmationOptions struct { + // Seed controls deterministic random sampling. + Seed int64 + // Confidence sets the confidence level used for statistical intervals. + Confidence float64 + // BootstrapCount sets the number of bootstrap resamples. + BootstrapCount int + // CaseNames restricts confirmation to the named workloads when nonempty. + CaseNames []string +} + +// ConfirmationMetric combines ratio, absolute-change, noise-floor, and classification evidence for one metric. +type ConfirmationMetric struct { + // Ratio reports the candidate-to-baseline latency ratio. + Ratio RatioInterval `json:"ratio"` + // AbsoluteChange reports the estimated absolute duration change and confidence bounds. + AbsoluteChange DurationInterval `json:"absolute_change"` + // NoiseRatio records the relative A/A noise floor used for classification. + NoiseRatio float64 `json:"noise_ratio"` + // NoiseAbsolute records the absolute A/A noise floor used for classification. + NoiseAbsolute time.Duration `json:"noise_absolute"` + // Classification records the assigned measurement or result class. + Classification string `json:"classification"` +} + +// ConfirmationCase reports comparability, timing deltas, and the final disposition for one confirmed case. +type ConfirmationCase struct { + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // Backend identifies the execution backend. + Backend ExecutionMode `json:"backend"` + // Tier identifies whether latency is promotion-gated or stress-diagnostic. + Tier string `json:"tier"` + // QualificationSplit identifies training, frozen holdout, or diagnostic evidence. + QualificationSplit string `json:"qualification_split"` + // TimingGated reports whether timing evidence contributes to promotion. + TimingGated bool `json:"timing_gated"` + // MatchedRounds records rounds containing both left- and right-arm samples. + MatchedRounds int `json:"matched_rounds"` + // LeftSamples records warm samples accepted from the left confirmation arm. + LeftSamples int `json:"left_samples"` + // RightSamples records warm samples accepted from the right confirmation arm. + RightSamples int `json:"right_samples"` + // Comparable reports whether the paired measurements satisfy comparison prerequisites. + Comparable bool `json:"comparable"` + // Comparability lists reasons paired confirmation records are or are not comparable. + Comparability []string `json:"comparability_reasons,omitempty"` + // P50 contains median ratio, absolute-change, noise, and classification evidence. + P50 ConfirmationMetric `json:"p50"` + // P95 contains 95th-percentile ratio, absolute-change, noise, and classification evidence. + P95 ConfirmationMetric `json:"p95"` + // Disposition records the confirmation classification assigned to the case. + Disposition string `json:"disposition"` + // RightRuntimeReceiptChains preserves the candidate/right arm's complete + // measured runtime branch chains. + RightRuntimeReceiptChains [][]RuntimeReceiptEvent `json:"right_runtime_receipt_chains,omitempty"` +} + +// ConfirmationReport contains paired-arm identities, A/A noise evidence, and per-case confirmation decisions. +type ConfirmationReport struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // Kind identifies the serialized confirmation-report format. + Kind string `json:"kind"` + // Seed controls deterministic random sampling. + Seed int64 `json:"seed"` + // Confidence sets the confidence level used for statistical intervals. + Confidence float64 `json:"confidence_level"` + // LeftArm identifies the artifact treated as the left confirmation arm. + LeftArm string `json:"left_arm"` + // RightArm identifies the artifact treated as the right confirmation arm. + RightArm string `json:"right_arm"` + // LeftSHA256 identifies the exact left-arm artifact evaluated by the report. + LeftSHA256 string `json:"left_sha256"` + // RightSHA256 identifies the exact right-arm artifact evaluated by the report. + RightSHA256 string `json:"right_sha256"` + // AAReport contains A/A noise evidence used to classify confirmation differences. + AAReport string `json:"aa_report,omitempty"` + // AAReportSHA256 identifies the exact A/A report used for classification. + AAReportSHA256 string `json:"aa_report_sha256,omitempty"` + // PromotionEligible reports whether every timing-gated causal case is comparable and P95-non-inferior. + PromotionEligible bool `json:"promotion_eligible"` + // QualificationRequired reports whether the artifact contains a prioritized traversal candidate that requires independent training and frozen-holdout confirmation. + QualificationRequired bool `json:"qualification_required"` + // TrainingCases records prioritized traversal cases confirmed on the selector-training partition. + TrainingCases int `json:"training_cases"` + // HoldoutCases records prioritized traversal cases confirmed on the frozen topology holdout. + HoldoutCases int `json:"holdout_cases"` + // TrainingPassed reports whether every observed prioritized training case cleared confirmation. + TrainingPassed bool `json:"training_passed"` + // HoldoutPassed reports whether every observed prioritized holdout case cleared confirmation. + HoldoutPassed bool `json:"holdout_passed"` + // QualificationPassed reports whether nonempty training and holdout partitions independently cleared confirmation. + QualificationPassed bool `json:"qualification_passed"` + // QualificationFamilies contains the independent split disposition for each concrete traversal candidate family. + QualificationFamilies []TraversalQualificationStatus `json:"qualification_families,omitempty"` + // Cases contains paired-arm evidence and the resulting disposition for each confirmed workload. + Cases []ConfirmationCase `json:"cases"` +} + +// createConfirmationReport loads both benchmark arms and optional A/A evidence, builds their comparison, and writes the resulting report. +func createConfirmationReport(leftPath, rightPath, aaPath, outputPath string, options ConfirmationOptions) error { + left, err := readJSONLFile(leftPath) + if err != nil { + return fmt.Errorf("read left artifact: %w", err) + } + right, err := readJSONLFile(rightPath) + if err != nil { + return fmt.Errorf("read right artifact: %w", err) + } + var aa *AAResolutionReport + aaSHA256 := "" + if aaPath != "" { + aa, aaSHA256, err = loadAAResolutionReport(aaPath) + if err != nil { + return fmt.Errorf("read A/A report: %w", err) + } + } + report, err := buildConfirmationReport(left, right, aa, options) + if err != nil { + return err + } + report.LeftSHA256, err = fileSHA256(leftPath) + if err != nil { + return err + } + report.RightSHA256, err = fileSHA256(rightPath) + if err != nil { + return err + } + report.AAReport = aaPath + report.AAReportSHA256 = aaSHA256 + return writeConfirmationReport(outputPath, report) +} + +// buildConfirmationReport pairs comparable cases, derives confidence intervals and noise-adjusted classifications, and records why incomparable cases were skipped. +func buildConfirmationReport(left, right []CaseResult, aa *AAResolutionReport, options ConfirmationOptions) (ConfirmationReport, error) { + if options.Confidence <= 0 || options.Confidence >= 1 { + return ConfirmationReport{}, fmt.Errorf("confidence level must be between 0 and 1") + } + if options.BootstrapCount == 0 { + options.BootstrapCount = defaultBootstrapCount + } + if options.BootstrapCount < 1 { + return ConfirmationReport{}, fmt.Errorf("bootstrap count must be positive") + } + leftSeries, rightSeries := collectWarmSeries(left), collectWarmSeries(right) + blockAA := sameExecutable(left, right) + if !blockAA && len(options.CaseNames) == 0 { + return ConfirmationReport{}, fmt.Errorf("causal confirmation requires exact primary case names") + } + if len(options.CaseNames) > 0 && len(options.CaseNames) <= 2 && options.Confidence < 0.975 { + options.Confidence = 0.975 + } + keys := make([]performanceKey, 0) + for key := range leftSeries { + if key.backend == ModePostgresSQL && rightSeries[key] != nil { + keys = append(keys, key) + } + } + sort.Slice(keys, func(i, j int) bool { + if keys[i].dataset != keys[j].dataset { + return keys[i].dataset < keys[j].dataset + } + return keys[i].name < keys[j].name + }) + if len(options.CaseNames) > 0 { + requested := map[string]bool{} + for _, name := range options.CaseNames { + requested[name] = false + } + filtered := keys[:0] + for _, key := range keys { + if _, ok := requested[key.name]; ok { + requested[key.name] = true + filtered = append(filtered, key) + } + } + for name, found := range requested { + if !found { + return ConfirmationReport{}, fmt.Errorf("unknown confirmation case %q", name) + } + } + keys = filtered + } + if len(keys) == 0 { + return ConfirmationReport{}, fmt.Errorf("artifacts have no matched PostgreSQL warm series") + } + tiers := make(map[performanceKey]string, len(keys)) + splits := make(map[performanceKey]string, len(keys)) + requiresAA := false + for _, key := range keys { + tier, err := timingTier(key, left, right) + if err != nil { + return ConfirmationReport{}, err + } + tiers[key] = tier + split, err := qualificationSplit(key, left, right) + if err != nil { + return ConfirmationReport{}, err + } + splits[key] = split + if !blockAA && tier != "stress" && promotionTimingSplit(split) { + requiresAA = true + } + } + if requiresAA { + if err := validateAAResolutionEvidence(aa, left, options.Confidence); err != nil { + return ConfirmationReport{}, fmt.Errorf("left-arm A/A evidence: %w", err) + } + if err := validateAAResolutionEvidence(aa, right, options.Confidence); err != nil { + return ConfirmationReport{}, fmt.Errorf("right-arm A/A evidence: %w", err) + } + } else if aa != nil { + if err := validateAAResolutionEvidence(aa, left, options.Confidence); err != nil { + return ConfirmationReport{}, err + } + } + + report := ConfirmationReport{ + Version: confirmationReportVersion, + Kind: "causal_confirmation", + Seed: options.Seed, + Confidence: options.Confidence, + } + report.LeftArm = artifactArm(left) + report.RightArm = artifactArm(right) + if blockAA { + report.Kind = "block_reload_aa" + } + report.PromotionEligible = !blockAA && requiresAA + report.TrainingPassed = true + report.HoldoutPassed = true + qualification := map[string]*TraversalQualificationStatus{} + gateOptions := PerfGateOptions{ + Seed: options.Seed, + Confidence: options.Confidence, + BootstrapCount: options.BootstrapCount, + } + for idx, key := range keys { + leftRounds, rightRounds := matchedRounds(leftSeries[key], rightSeries[key]) + timingGated := tiers[key] != "stress" && promotionTimingSplit(splits[key]) && !blockAA + if timingGated && (len(leftRounds) < 10 || len(leftRounds) > 20) { + return ConfirmationReport{}, fmt.Errorf("%s/%s requires 10-20 matched rounds, got %d", key.dataset, key.name, len(leftRounds)) + } + for _, round := range sortedRounds(leftRounds) { + if timingGated && (len(leftRounds[round]) < 50 || len(rightRounds[round]) < 50) { + return ConfirmationReport{}, fmt.Errorf("%s/%s round %d requires at least 50 warm samples per arm", key.dataset, key.name, round) + } + } + if timingGated { + if err := validatePairedOrderEvidence(left, right, key, sortedRounds(leftRounds), 20); err != nil { + return ConfirmationReport{}, fmt.Errorf("invalid confirmation evidence: %w", err) + } + } + seed := options.Seed + int64(idx)*7919 + p50Ratio := bootstrapRoundMedianRatio(leftRounds, rightRounds, seed, gateOptions) + p50Change := negateDurationInterval(bootstrapRoundMedianSaving(leftRounds, rightRounds, seed+1, gateOptions)) + p95Ratio := bootstrapStratifiedP95Ratio(leftRounds, rightRounds, seed+2, gateOptions) + p95Change := bootstrapStratifiedQuantileChange(leftRounds, rightRounds, 0.95, seed+3, gateOptions) + p50NoiseRatio, p50NoiseAbsolute := minimumTimingNoiseRatio, minimumTimingNoiseAbsolute + p95NoiseRatio, p95NoiseAbsolute := minimumTimingNoiseRatio, minimumTimingNoiseAbsolute + if aa != nil { + if ratio, absolute, floorErr := aaTimingFloor(aa, key, false, 0); floorErr == nil { + p50NoiseRatio, p50NoiseAbsolute = ratio, absolute + } else if timingGated { + return ConfirmationReport{}, floorErr + } + if ratio, absolute, floorErr := aaTimingFloor(aa, key, true, 0); floorErr == nil { + p95NoiseRatio, p95NoiseAbsolute = ratio, absolute + } else if timingGated { + return ConfirmationReport{}, floorErr + } + } + comparable, reasons := confirmationComparable(left, right, key) + entry := ConfirmationCase{ + Dataset: key.dataset, + Name: key.name, + Backend: key.backend, + Tier: tiers[key], + QualificationSplit: splits[key], + TimingGated: timingGated, + MatchedRounds: len(leftRounds), + LeftSamples: sampleCount(leftRounds), + RightSamples: sampleCount(rightRounds), + Comparable: comparable, + Comparability: reasons, + RightRuntimeReceiptChains: caseRuntimeReceiptChains(right, key), + P50: classifyConfirmationMetric(p50Ratio, p50Change, p50NoiseRatio, p50NoiseAbsolute), + P95: classifyConfirmationMetric(p95Ratio, p95Change, p95NoiseRatio, p95NoiseAbsolute), + } + entry.Disposition = entry.P95.Classification + if tiers[key] == "stress" { + entry.Disposition = "stress_diagnostic" + } + if splits[key] == "diagnostic" { + entry.Disposition = "qualification_diagnostic" + } + if !comparable { + entry.Disposition = "fingerprint_mismatch" + } + if entry.TimingGated && (!entry.Comparable || entry.P95.Classification != "cleared_non_inferior") { + report.PromotionEligible = false + } + if prioritizedTraversalKey(key, left, right) && entry.TimingGated { + report.QualificationRequired = true + passed := entry.Comparable && entry.P95.Classification == "cleared_non_inferior" + family := traversalQualificationFamily(key, left, right) + status := qualification[family] + if status == nil { + status = &TraversalQualificationStatus{Family: family, TrainingPassed: true, HoldoutPassed: true} + qualification[family] = status + } + switch entry.QualificationSplit { + case "training": + report.TrainingCases++ + report.TrainingPassed = report.TrainingPassed && passed + status.TrainingCases++ + status.TrainingPassed = status.TrainingPassed && passed + case "holdout": + report.HoldoutCases++ + report.HoldoutPassed = report.HoldoutPassed && passed + status.HoldoutCases++ + status.HoldoutPassed = status.HoldoutPassed && passed + } + } + report.Cases = append(report.Cases, entry) + } + if report.QualificationRequired { + families := make([]string, 0, len(qualification)) + for family := range qualification { + families = append(families, family) + } + sort.Strings(families) + for _, family := range families { + status := qualification[family] + status.TrainingPassed = status.TrainingPassed && status.TrainingCases > 0 + status.HoldoutPassed = status.HoldoutPassed && status.HoldoutCases > 0 + status.Passed = status.TrainingPassed && status.HoldoutPassed + report.TrainingPassed = report.TrainingPassed && status.TrainingPassed + report.HoldoutPassed = report.HoldoutPassed && status.HoldoutPassed + report.QualificationFamilies = append(report.QualificationFamilies, *status) + } + report.QualificationPassed = report.TrainingPassed && report.HoldoutPassed + report.PromotionEligible = report.PromotionEligible && report.QualificationPassed + } else { + report.TrainingPassed = false + report.HoldoutPassed = false + } + return report, nil +} + +// classifyConfirmationMetric labels a confidence interval as regression, improvement, or inconclusive only when both relative and absolute noise floors are crossed. +func classifyConfirmationMetric(ratio RatioInterval, change DurationInterval, noiseRatio float64, noiseAbsolute time.Duration) ConfirmationMetric { + classification := "inconclusive" + if ratio.Lower > 1+noiseRatio && change.Lower > noiseAbsolute { + classification = "confirmed" + } + if ratio.Upper <= 1+noiseRatio && change.Upper <= noiseAbsolute { + classification = "cleared_non_inferior" + } + return ConfirmationMetric{ + Ratio: ratio, + AbsoluteChange: change, + NoiseRatio: noiseRatio, + NoiseAbsolute: noiseAbsolute, + Classification: classification, + } +} + +// bootstrapStratifiedQuantileChange estimates a quantile delta and confidence interval by resampling within matching benchmark rounds. +func bootstrapStratifiedQuantileChange(left, right roundSamples, probability float64, seed int64, options PerfGateOptions) DurationInterval { + rounds := sortedRounds(left) + estimate := durationQuantile(flattenSamples(right, rounds), probability) - durationQuantile(flattenSamples(left, rounds), probability) + rng := rand.New(rand.NewSource(seed)) // #nosec G404 -- deterministic statistical resampling + changes := make([]float64, options.BootstrapCount) + for idx := range changes { + var sampledLeft, sampledRight []time.Duration + for _, round := range rounds { + sampledLeft = append(sampledLeft, resampleDurations(rng, left[round])...) + sampledRight = append(sampledRight, resampleDurations(rng, right[round])...) + } + changes[idx] = durationQuantile(sampledRight, probability) - durationQuantile(sampledLeft, probability) + } + interval := confidenceInterval(estimate, changes, options.Confidence) + return DurationInterval{ + Estimate: time.Duration(interval.Estimate), + Lower: time.Duration(interval.Lower), + Upper: time.Duration(interval.Upper), + } +} + +// negateDurationInterval reverses interval direction and swaps its bounds so left/right arm normalization preserves a valid ordered interval. +func negateDurationInterval(value DurationInterval) DurationInterval { + return DurationInterval{ + Estimate: -value.Estimate, + Lower: -value.Upper, + Upper: -value.Lower, + } +} + +// confirmationComparable compares two confirmation records and returns every reason they cannot be paired. +func confirmationComparable(left, right []CaseResult, key performanceKey) (bool, []string) { + leftRecords := matchingRecords(left, key) + rightRecords := matchingRecords(right, key) + var reasons []string + if len(leftRecords) == 0 || len(rightRecords) == 0 { + reasons = append(reasons, "missing record") + return false, reasons + } + leftRecord, rightRecord := leftRecords[0], rightRecords[0] + reasons = append(reasons, confirmationArmConsistency(leftRecords)...) + reasons = append(reasons, confirmationArmConsistency(rightRecords)...) + if leftRecord.Status != StatusOK || rightRecord.Status != StatusOK { + reasons = append(reasons, "non-ok status") + } + if leftRecord.Fixture == nil || rightRecord.Fixture == nil || leftRecord.Fixture.Checksum != rightRecord.Fixture.Checksum { + reasons = append(reasons, "fixture checksum differs") + } + if fmt.Sprint(leftRecord.ObservedRows) != fmt.Sprint(rightRecord.ObservedRows) { + reasons = append(reasons, "exact observations differ") + } + if leftRecord.RowCount != rightRecord.RowCount { + reasons = append(reasons, "row count differs") + } + if !comparablePostgresEnvironment(leftRecord.PostgresEnvironment, rightRecord.PostgresEnvironment) { + reasons = append(reasons, "PostgreSQL settings or relation sizes differ") + } + return len(reasons) == 0, uniqueStrings(reasons) +} + +// confirmationArmConsistency reports within-arm drift in environment, executable, and normalized PostgreSQL plan shape. +func confirmationArmConsistency(records []CaseResult) []string { + if len(records) == 0 { + return []string{"missing record"} + } + + baseline := records[0] + var reasons []string + for _, record := range records[1:] { + if record.Status != StatusOK { + reasons = append(reasons, "non-ok status") + } + if record.SQLFingerprint != baseline.SQLFingerprint { + reasons = append(reasons, "SQL fingerprint changes within arm") + } + if record.Fixture == nil || baseline.Fixture == nil || record.Fixture.Checksum != baseline.Fixture.Checksum { + reasons = append(reasons, "fixture checksum differs") + } + if fmt.Sprint(record.ObservedRows) != fmt.Sprint(baseline.ObservedRows) { + reasons = append(reasons, "exact observations differ") + } + if record.RowCount != baseline.RowCount { + reasons = append(reasons, "row count differs") + } + if !comparablePostgresEnvironment(baseline.PostgresEnvironment, record.PostgresEnvironment) { + reasons = append(reasons, "PostgreSQL settings or relation sizes differ") + } + if postgresPlanShapeSHA256(record.PostgresPlan) != postgresPlanShapeSHA256(baseline.PostgresPlan) { + reasons = append(reasons, "intended plan shape changes within arm") + } + } + return reasons +} + +var ( + // volatilePlanDetails matches planner cost and runtime annotations that do not define structural plan shape. + volatilePlanDetails = regexp.MustCompile(`\s+\((?:cost|actual)[^)]*\)`) + + // volatilePlanIDs matches generated bigint constants so dataset-specific IDs do not perturb plan-shape hashes. + volatilePlanIDs = regexp.MustCompile(`'[0-9]+'::bigint`) + + // volatilePlanLine matches resource and timing summary lines excluded from structural plan-shape hashes. + volatilePlanLine = regexp.MustCompile(`^(?:Buffers|Planning Time|Execution Time):`) +) + +// postgresPlanShapeSHA256 hashes structural EXPLAIN lines after removing costs, runtime counters, transient IDs, and timing details; confirmation compares plan shape without treating volatile measurements as structural changes. +func postgresPlanShapeSHA256(plan []string) string { + digest := sha256.New() + for _, line := range plan { + line = volatilePlanDetails.ReplaceAllString(line, "") + line = volatilePlanIDs.ReplaceAllString(line, "'$id'::bigint") + line = strings.TrimSpace(line) + if line == "" || volatilePlanLine.MatchString(line) { + continue + } + fmt.Fprintln(digest, line) + } + return hex.EncodeToString(digest.Sum(nil)) +} + +// matchingRecords selects successful measured records for one dataset, case, backend, and executor identity. +func matchingRecords(records []CaseResult, key performanceKey) []CaseResult { + var matched []CaseResult + for _, record := range records { + if record.Dataset == key.dataset && record.Name == key.name && record.ExecutionMode == key.backend { + matched = append(matched, record) + } + } + return matched +} + +// comparablePostgresEnvironment requires server version and normalized settings to match while tolerating absent environment metadata on both arms. +func comparablePostgresEnvironment(left, right *PostgresEnvironment) bool { + if left == nil || right == nil { + return left == nil && right == nil + } + return left.PlanCacheMode == right.PlanCacheMode && left.TransactionIsolation == right.TransactionIsolation && left.WorkMem == right.WorkMem && left.TempFileLimit == right.TempFileLimit && + left.GraphPartitionCount == right.GraphPartitionCount && left.NodeRelationBytes == right.NodeRelationBytes && left.EdgeRelationBytes == right.EdgeRelationBytes + +} + +// uniqueStrings removes duplicate diagnostic reasons while preserving their first-seen order. +func uniqueStrings(values []string) []string { + seen := map[string]struct{}{} + result := make([]string, 0, len(values)) + for _, value := range values { + if _, found := seen[value]; found { + continue + } + seen[value] = struct{}{} + result = append(result, value) + } + return result +} + +// artifactArm returns the first recorded benchmark arm label, or "unknown" when an artifact lacks environment metadata. +func artifactArm(records []CaseResult) string { + for _, record := range records { + if record.Environment != nil { + return record.Environment.Arm + } + } + return "unknown" +} + +// sameExecutable identifies a true block/reload A/A treatment. A shared +// executable alone is insufficient because one GraphBench binary can emit +// different forced executors and SQL statements. +func sameExecutable(left, right []CaseResult) bool { + leftIdentity := effectiveTreatmentIdentity(left) + return leftIdentity != "" && leftIdentity == effectiveTreatmentIdentity(right) +} + +func effectiveTreatmentIdentity(records []CaseResult) string { + if len(records) == 0 || records[0].Environment == nil || records[0].Environment.BinarySHA256 == "" { + return "" + } + identity := []string{"binary=" + records[0].Environment.BinarySHA256} + for _, argument := range records[0].Environment.Invocation { + if strings.Contains(argument, "postgres-force-shortest-executor") || + strings.Contains(argument, "postgres-force-expansion-strategy") || + strings.Contains(argument, "postgres-expansion-orientation") || + strings.Contains(argument, "reference-arm") { + identity = append(identity, "option="+argument) + } + } + fingerprints := make([]string, 0, len(records)) + for _, record := range records { + fingerprints = append(fingerprints, record.Dataset+"/"+record.Name+"="+record.SQLFingerprint) + } + sort.Strings(fingerprints) + identity = append(identity, fingerprints...) + digest := sha256.Sum256([]byte(strings.Join(identity, "\n"))) + return hex.EncodeToString(digest[:]) +} + +// writeConfirmationReport emits indented JSON to stdout or atomically replaces the requested output file. +func writeConfirmationReport(path string, report ConfirmationReport) (err error) { + output := os.Stdout + if path != "" { + if err := ensureOutputDir(path); err != nil { + return err + } + output, err = os.Create(path) + if err != nil { + return err + } + defer func() { + if closeErr := output.Close(); err == nil && closeErr != nil { + err = closeErr + } + }() + } + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(report) +} diff --git a/cmd/graphbench/confirm_report_test.go b/cmd/graphbench/confirm_report_test.go new file mode 100644 index 00000000..588f9387 --- /dev/null +++ b/cmd/graphbench/confirm_report_test.go @@ -0,0 +1,232 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestBuildConfirmationReportClassifiesFreshMatchedP95 verifies that distinct predecessor and candidate binaries with a measurable P95 increase produce a comparable causal confirmation. +func TestBuildConfirmationReportClassifiesFreshMatchedP95(t *testing.T) { + left := []CaseResult{confirmationRecord("alert", "predecessor", "binary-a", 10*time.Millisecond)} + right := []CaseResult{confirmationRecord("alert", "candidate", "binary-b", 13*time.Millisecond)} + stampPairedEvidence(left, right, 20) + + report, err := buildConfirmationReport(left, right, testAAReportForRecords(t, left), ConfirmationOptions{ + Seed: 7, + Confidence: 0.95, + BootstrapCount: 100, + CaseNames: []string{"alert"}, + }) + + require.NoError(t, err) + require.Equal(t, "causal_confirmation", report.Kind) + require.Equal(t, "confirmed", report.Cases[0].P95.Classification) + require.Equal(t, 3*time.Millisecond, report.Cases[0].P95.AbsoluteChange.Estimate) + require.True(t, report.Cases[0].Comparable) + require.False(t, report.PromotionEligible) +} + +// TestBuildConfirmationReportRecognizesSameBinaryBlockAA verifies that identical binaries are classified as a reload control and clear a non-inferior result. +func TestBuildConfirmationReportRecognizesSameBinaryBlockAA(t *testing.T) { + left := []CaseResult{confirmationRecord("control", "block-a", "same", 10*time.Millisecond)} + right := []CaseResult{confirmationRecord("control", "block-b", "same", 10*time.Millisecond)} + stampPairedEvidence(left, right, 20) + + report, err := buildConfirmationReport(left, right, nil, ConfirmationOptions{ + Seed: 1, + Confidence: 0.95, + BootstrapCount: 50, + }) + require.NoError(t, err) + require.Equal(t, "block_reload_aa", report.Kind) + require.Equal(t, "cleared_non_inferior", report.Cases[0].Disposition) +} + +// TestBuildConfirmationReportAllowsIntentionalCrossArmSQLAndPlanChanges verifies that implementation changes between predecessor and candidate arms do not invalidate an otherwise controlled comparison. +func TestBuildConfirmationReportAllowsIntentionalCrossArmSQLAndPlanChanges(t *testing.T) { + left := []CaseResult{confirmationRecord("changed", "predecessor", "binary-a", 10*time.Millisecond)} + right := []CaseResult{confirmationRecord("changed", "candidate", "binary-b", 5*time.Millisecond)} + left[0].SQLFingerprint = "incumbent-sql" + right[0].SQLFingerprint = "candidate-sql" + left[0].PostgresPlan = []string{"CTE Scan on incumbent"} + right[0].PostgresPlan = []string{"Recursive Union"} + stampPairedEvidence(left, right, 20) + + report, err := buildConfirmationReport(left, right, testAAReportForRecords(t, left), ConfirmationOptions{ + Seed: 1, + Confidence: 0.95, + BootstrapCount: 50, + CaseNames: []string{"changed"}, + }) + require.NoError(t, err) + require.True(t, report.Cases[0].Comparable) + require.True(t, report.PromotionEligible) +} + +// TestConfirmationComparableRejectsFingerprintChangeWithinArm verifies that SQL drift among repetitions of one arm makes the confirmation comparison invalid. +func TestConfirmationComparableRejectsFingerprintChangeWithinArm(t *testing.T) { + left := []CaseResult{ + confirmationRecord("changed", "predecessor", "binary-a", 10*time.Millisecond), + confirmationRecord("changed", "predecessor", "binary-a", 10*time.Millisecond), + } + right := []CaseResult{confirmationRecord("changed", "candidate", "binary-b", 5*time.Millisecond)} + left[1].SQLFingerprint = "unstable-sql" + + comparable, reasons := confirmationComparable(left, right, performanceKey{ + dataset: left[0].Dataset, + name: "changed", + backend: ModePostgresSQL, + }) + require.False(t, comparable) + require.Contains(t, reasons, "SQL fingerprint changes within arm") +} + +// TestPostgresPlanShapeIgnoresReloadedEntityIDs verifies that literal database IDs and timing noise do not alter the normalized PostgreSQL plan fingerprint. +func TestPostgresPlanShapeIgnoresReloadedEntityIDs(t *testing.T) { + left := []string{"Index Cond: (id = '4624444'::bigint)", "Planning Time: 0.408 ms", "Execution Time: 0.224 ms"} + right := []string{"Index Cond: (id = '4630087'::bigint)", "Planning Time: 0.189 ms", "Execution Time: 0.093 ms"} + require.Equal(t, postgresPlanShapeSHA256(left), postgresPlanShapeSHA256(right)) +} + +// TestBuildConfirmationReportRejectsUnknownExactCase verifies that an exact selector must resolve to an observed case instead of yielding an empty confirmation report. +func TestBuildConfirmationReportRejectsUnknownExactCase(t *testing.T) { + record := confirmationRecord("present", "arm", "binary", time.Millisecond) + _, err := buildConfirmationReport([]CaseResult{record}, []CaseResult{record}, nil, ConfirmationOptions{ + Seed: 1, + Confidence: 0.95, + BootstrapCount: 10, + CaseNames: []string{"missing"}, + }) + require.ErrorContains(t, err, "unknown confirmation case") +} + +// TestBuildConfirmationReportRequiresHostAAForCausalPromotion verifies a fresh binary comparison fails closed without per-case host calibration. +func TestBuildConfirmationReportRequiresHostAAForCausalPromotion(t *testing.T) { + left := []CaseResult{confirmationRecord("changed", "predecessor", "binary-a", time.Millisecond)} + right := []CaseResult{confirmationRecord("changed", "candidate", "binary-b", 900*time.Microsecond)} + stampPairedEvidence(left, right, 20) + + _, err := buildConfirmationReport(left, right, nil, ConfirmationOptions{ + Confidence: defaultConfidenceLevel, BootstrapCount: 10, CaseNames: []string{"changed"}, + }) + + require.ErrorContains(t, err, "host A/A resolution report is required") +} + +func TestSameExecutableRequiresSameEffectiveTreatment(t *testing.T) { + left := []CaseResult{confirmationRecord("changed", "a1", "shared-binary", time.Millisecond)} + right := []CaseResult{confirmationRecord("changed", "i1", "shared-binary", time.Millisecond)} + left[0].SQLFingerprint = "a1-sql" + right[0].SQLFingerprint = "i1-sql" + require.False(t, sameExecutable(left, right)) + + right[0].SQLFingerprint = left[0].SQLFingerprint + right[0].Environment.Invocation = append(right[0].Environment.Invocation, "--postgres-force-shortest-executor=ASP-I1-U-DAG+MAT-M0") + require.False(t, sameExecutable(left, right)) + + right[0].Environment.Invocation = append([]string(nil), left[0].Environment.Invocation...) + require.True(t, sameExecutable(left, right)) +} + +// TestBuildConfirmationReportKeepsStressTimingDiagnostic verifies stress comparisons remain descriptive and need no promotion calibration. +func TestBuildConfirmationReportKeepsStressTimingDiagnostic(t *testing.T) { + left := []CaseResult{confirmationRecord("stress", "predecessor", "binary-a", time.Millisecond)} + right := []CaseResult{confirmationRecord("stress", "candidate", "binary-b", 10*time.Millisecond)} + left[0].Shape.FixtureTier = "stress" + right[0].Shape.FixtureTier = "stress" + + report, err := buildConfirmationReport(left, right, nil, ConfirmationOptions{ + Confidence: defaultConfidenceLevel, BootstrapCount: 10, CaseNames: []string{"stress"}, + }) + + require.NoError(t, err) + require.False(t, report.PromotionEligible) + require.False(t, report.Cases[0].TimingGated) + require.Equal(t, "stress_diagnostic", report.Cases[0].Disposition) +} + +// TestBuildConfirmationReportKeepsDiagnosticSplitOutOfPromotion verifies a +// normal-tier boundary case remains evaluation-only by declaration. +func TestBuildConfirmationReportKeepsDiagnosticSplitOutOfPromotion(t *testing.T) { + left := []CaseResult{confirmationRecord("boundary", "predecessor", "binary-a", time.Millisecond)} + right := []CaseResult{confirmationRecord("boundary", "candidate", "binary-b", 10*time.Millisecond)} + left[0].Shape.QualificationSplit = "diagnostic" + right[0].Shape.QualificationSplit = "diagnostic" + + report, err := buildConfirmationReport(left, right, nil, ConfirmationOptions{ + Confidence: defaultConfidenceLevel, BootstrapCount: 10, CaseNames: []string{"boundary"}, + }) + + require.NoError(t, err) + require.False(t, report.PromotionEligible) + require.False(t, report.Cases[0].TimingGated) + require.Equal(t, "qualification_diagnostic", report.Cases[0].Disposition) +} + +// TestBuildConfirmationReportRequiresIndependentTraversalHoldout verifies a +// clean training result cannot qualify a traversal candidate without an +// independently named frozen-holdout case. +func TestBuildConfirmationReportRequiresIndependentTraversalHoldout(t *testing.T) { + left := []CaseResult{ + confirmationRecord("sp-training", "predecessor", "binary-a", 10*time.Millisecond), + confirmationRecord("sp-holdout", "predecessor", "binary-a", 10*time.Millisecond), + } + right := []CaseResult{ + confirmationRecord("sp-training", "candidate", "binary-b", 5*time.Millisecond), + confirmationRecord("sp-holdout", "candidate", "binary-b", 5*time.Millisecond), + } + for _, records := range [][]CaseResult{left, right} { + records[0].Category = "generated_shortest_path_v2" + records[0].Shape.QualificationSplit = "training" + records[1].Category = "generated_shortest_path_v2" + records[1].Shape.QualificationSplit = "holdout" + } + stampPairedEvidence(left, right, 20) + + report, err := buildConfirmationReport(left, right, testAAReportForRecords(t, left), ConfirmationOptions{ + Seed: 1, Confidence: defaultConfidenceLevel, BootstrapCount: 50, CaseNames: []string{"sp-training", "sp-holdout"}, + }) + require.NoError(t, err) + require.True(t, report.QualificationRequired) + require.True(t, report.TrainingPassed) + require.True(t, report.HoldoutPassed) + require.True(t, report.QualificationPassed) + require.True(t, report.PromotionEligible) + + left = left[:1] + right = right[:1] + report, err = buildConfirmationReport(left, right, testAAReportForRecords(t, left), ConfirmationOptions{ + Seed: 1, Confidence: defaultConfidenceLevel, BootstrapCount: 50, CaseNames: []string{"sp-training"}, + }) + require.NoError(t, err) + require.True(t, report.TrainingPassed) + require.False(t, report.HoldoutPassed) + require.False(t, report.QualificationPassed) + require.False(t, report.PromotionEligible) +} + +// confirmationRecord returns a stable PostgreSQL observation annotated with the requested arm and binary identity. +func confirmationRecord(name, arm, binary string, duration time.Duration) CaseResult { + record := perfGateRecord(name, ModePostgresSQL, duration, 10, 50) + record.SQLFingerprint = "sql" + record.ObservedRows = []string{"[1]"} + record.Fixture = &FixtureMetadata{Checksum: "fixture"} + record.Environment = &RunEnvironment{ + Arm: arm, + BinarySHA256: binary, + GOOS: "linux", + GOARCH: "amd64", + CPUCount: 8, + CPUModel: "test-cpu", + Kernel: "test-kernel", + CgroupCPU: "max 100000", + } + return record +} diff --git a/cmd/graphbench/corpus.go b/cmd/graphbench/corpus.go index 7d1c9075..a83de262 100644 --- a/cmd/graphbench/corpus.go +++ b/cmd/graphbench/corpus.go @@ -21,9 +21,12 @@ import ( "fmt" "os" "path/filepath" + "slices" "sort" + "strings" ) +// loadScaleCorpus loads all scale-case JSON files and rejects duplicate or invalid declarations. func loadScaleCorpus(root string) (ScaleCorpus, error) { casePaths, err := filepath.Glob(filepath.Join(root, "cases", "*.json")) if err != nil { @@ -45,6 +48,7 @@ func loadScaleCorpus(root string) (ScaleCorpus, error) { source := filepath.ToSlash(path) for idx, testCase := range file.Cases { testCase.Source = source + normalizeFallbackExpectation(&testCase) if err := validateScaleCase(testCase); err != nil { return ScaleCorpus{}, fmt.Errorf("%s case %d: %w", source, idx, err) } @@ -56,6 +60,24 @@ func loadScaleCorpus(root string) (ScaleCorpus, error) { return corpus, nil } +func normalizeFallbackExpectation(testCase *ScaleCase) { + if testCase == nil || testCase.Shape.FallbackExpectation != "" || !requiresQualificationSplit(*testCase) { + return + } + testCase.Shape.FallbackExpectation = "forbidden" + if testCase.Shape.FixtureTier == "stress" { + testCase.Shape.FallbackExpectation = "allowed" + } + for _, tag := range testCase.Tags { + normalized := strings.ToLower(tag) + if strings.Contains(normalized, "fallback") || strings.Contains(normalized, "overflow") { + testCase.Shape.FallbackExpectation = "required" + return + } + } +} + +// validateScaleCase checks case identity, modes, parameters, expectations, and workload shape. func validateScaleCase(testCase ScaleCase) error { if testCase.Name == "" { return fmt.Errorf("name is required") @@ -78,10 +100,133 @@ func validateScaleCase(testCase ScaleCase) error { return fmt.Errorf("unsupported candidate mode %q", mode) } } + for mode, reason := range testCase.UnsupportedModes { + if !mode.Valid() { + return fmt.Errorf("invalid unsupported mode %q", mode) + } + if reason == "" { + return fmt.Errorf("unsupported mode %q requires a reason", mode) + } + if testCase.Supports(mode) { + return fmt.Errorf("mode %q cannot be both candidate and unsupported", mode) + } + } + if testCase.Shape.RelationshipKindCount < 0 { + return fmt.Errorf("shape.relationship_kind_count must not be negative") + } + if tier := testCase.Shape.FixtureTier; tier != "" && tier != "normal" && tier != "envelope" && tier != "stress" { + return fmt.Errorf("shape.fixture_tier must be normal, envelope, or stress") + } + if split := testCase.Shape.QualificationSplit; split != "" && split != "training" && split != "holdout" && split != "diagnostic" { + return fmt.Errorf("shape.qualification_split must be training, holdout, or diagnostic") + } + if expectation := testCase.Shape.FallbackExpectation; expectation != "" && expectation != "forbidden" && expectation != "required" && expectation != "allowed" { + return fmt.Errorf("shape.fallback_expectation must be forbidden, required, or allowed") + } + if requiresQualificationSplit(testCase) && testCase.Shape.QualificationSplit == "" { + return fmt.Errorf("shape.qualification_split is required for traversal qualification cases") + } + if testCase.Shape.FixtureTier == "stress" && testCase.Shape.QualificationSplit != "diagnostic" && requiresQualificationSplit(testCase) { + return fmt.Errorf("stress traversal qualification cases must use shape.qualification_split diagnostic") + } + if slices.Contains(testCase.Tags, "holdout") && testCase.Shape.QualificationSplit != "holdout" { + return fmt.Errorf("holdout-tagged cases must use shape.qualification_split holdout") + } + if testCase.Shape.QualificationSplit == "holdout" && !slices.Contains(testCase.Tags, "holdout") { + return fmt.Errorf("shape.qualification_split holdout requires the holdout tag") + } + if direction := testCase.Shape.Direction; direction != "" && direction != "outbound" && direction != "inbound" && direction != "directionless" && direction != "mirrored" { + return fmt.Errorf("shape.direction must be outbound, inbound, directionless, or mirrored") + } + + if len(testCase.Expected.IDRows) > 0 { + if testCase.Expected.ResultKind != "id_rows" { + return fmt.Errorf("expected.id_rows requires result_kind id_rows") + } + if testCase.Expected.RowCount == nil || int64(len(testCase.Expected.IDRows)) != *testCase.Expected.RowCount { + return fmt.Errorf("expected.id_rows must contain exactly row_count rows") + } + } + if len(testCase.Expected.PathRows) > 0 { + if testCase.Expected.ResultKind != "path_set" { + return fmt.Errorf("expected.path_rows requires result_kind path_set") + } + if testCase.Expected.RowCount == nil || int64(len(testCase.Expected.PathRows)) != *testCase.Expected.RowCount { + return fmt.Errorf("expected.path_rows must contain exactly row_count rows") + } + for idx, path := range testCase.Expected.PathRows { + if len(path.Nodes) != len(path.RelationshipKinds)+1 { + return fmt.Errorf("expected.path_rows[%d] must have one more node than relationship kind", idx) + } + if slices.Contains(testCase.Tags, "fixed-suffix-expansion-v3") && len(path.RelationshipKeys) != len(path.RelationshipKinds) { + return fmt.Errorf("fixed-suffix v3 expected.path_rows[%d] must identify every relationship", idx) + } + } + } + if slices.Contains(testCase.Tags, "fixed-suffix-expansion-v3") && testCase.Expected.ResultKind == "path_set" && len(testCase.Expected.PathRows) == 0 { + return fmt.Errorf("fixed-suffix v3 path_set cases require exact expected.path_rows") + } + + if testCase.WriteScenario != nil { + if err := validateWriteScenario(*testCase.WriteScenario); err != nil { + return err + } + } + + return nil +} + +// requiresQualificationSplit identifies traversal-program declarations whose +// training/holdout boundary is part of their immutable workload identity. +// Older general-purpose scale cases remain loadable while each prioritized +// traversal family is migrated deliberately. +func requiresQualificationSplit(testCase ScaleCase) bool { + switch testCase.Category { + case "generated_shortest_path_v2", "expand_into_one_hop", "generated_endpoint_seeded_expansion": + return true + case "generated_fixed_suffix_expansion": + return slices.Contains(testCase.Tags, "fixed-suffix-expansion-v2") || + slices.Contains(testCase.Tags, "fixed-suffix-expansion-v3") || + slices.Contains(testCase.Tags, "fixed-suffix-expansion-boundary") + default: + return slices.Contains(testCase.Tags, "traversal-qualification") + } +} + +// validateWriteScenario checks mutation expectations and post-state query completeness. +func validateWriteScenario(scenario WriteScenario) error { + if scenario.SelectionCypher == "" { + return fmt.Errorf("write_scenario.selection_cypher is required") + } + if scenario.ExpectedMatched == nil { + return fmt.Errorf("write_scenario.expected_matched is required") + } + if scenario.ExpectedAffected == nil { + return fmt.Errorf("write_scenario.expected_affected is required") + } + if scenario.AffectedEntity != "node" && scenario.AffectedEntity != "relationship" { + return fmt.Errorf("write_scenario.affected_entity must be node or relationship") + } + if len(scenario.PostState) == 0 { + return fmt.Errorf("write_scenario.post_state is required") + } + + for idx, postState := range scenario.PostState { + if postState.Name == "" { + return fmt.Errorf("write_scenario.post_state[%d].name is required", idx) + } + if postState.Cypher == "" { + return fmt.Errorf("write_scenario.post_state[%d].cypher is required", idx) + } + if postState.Expected.RowCount == nil && postState.Expected.ScalarInt == nil { + return fmt.Errorf("write_scenario.post_state[%d].expected requires row_count or scalar_int", idx) + } + } return nil } +// decodeJSONFile reads a JSON file and decodes it into the supplied destination. func decodeJSONFile(path string, target any) error { raw, err := os.ReadFile(path) if err != nil { @@ -94,6 +239,7 @@ func decodeJSONFile(path string, target any) error { return nil } +// scaleCorpusDatasets returns unique corpus dataset names in sorted order. func scaleCorpusDatasets(corpus ScaleCorpus) []string { var ( seen = map[string]struct{}{} @@ -113,6 +259,7 @@ func scaleCorpusDatasets(corpus ScaleCorpus) []string { return datasets } +// scaleCasesByDataset indexes scale cases by dataset while preserving corpus order. func scaleCasesByDataset(corpus ScaleCorpus) map[string][]ScaleCase { grouped := map[string][]ScaleCase{} for _, testCase := range corpus.Cases { diff --git a/cmd/graphbench/corpus_test.go b/cmd/graphbench/corpus_test.go index 211b2084..c5b6b1bd 100644 --- a/cmd/graphbench/corpus_test.go +++ b/cmd/graphbench/corpus_test.go @@ -17,11 +17,15 @@ package main import ( + "fmt" "testing" + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/testutil" "github.com/stretchr/testify/require" ) +// TestLoadScaleCorpus verifies that every loaded case identifies its source, declares PostgreSQL support status, and excludes the reference-only AGE mode. func TestLoadScaleCorpus(t *testing.T) { corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") require.NoError(t, err) @@ -29,17 +33,258 @@ func TestLoadScaleCorpus(t *testing.T) { for _, testCase := range corpus.Cases { require.NotEqual(t, "", testCase.Source) - require.True(t, testCase.Supports(ModePostgresSQL), "postgres_sql should be part of the initial corpus for %s", testCase.Name) + _, explicitlyUnsupported := testCase.UnsupportedReason(ModePostgresSQL) + require.True(t, testCase.Supports(ModePostgresSQL) || explicitlyUnsupported, + "postgres_sql should be a candidate or explicitly unsupported for %s", testCase.Name) require.False(t, testCase.Supports(ExecutionMode("age")), "AGE is a reference design only for %s", testCase.Name) } } +// TestValidateScaleCaseRequiresConsistentUnsupportedModes verifies that a backend cannot be both runnable and unsupported and that every exclusion has a reason. +func TestValidateScaleCaseRequiresConsistentUnsupportedModes(t *testing.T) { + testCase := ScaleCase{ + Name: "directionless", + Dataset: "base", + Category: "shortest_path", + Cypher: "MATCH p = shortestPath((a)-[*]-(b)) RETURN p", + CandidateModes: []ExecutionMode{ModeNeo4j}, + UnsupportedModes: map[ExecutionMode]string{ModePostgresSQL: "translator does not support this form"}, + } + + require.NoError(t, validateScaleCase(testCase)) + testCase.CandidateModes = append(testCase.CandidateModes, ModePostgresSQL) + require.ErrorContains(t, validateScaleCase(testCase), "both candidate and unsupported") + testCase.CandidateModes = []ExecutionMode{ModeNeo4j} + testCase.UnsupportedModes[ModePostgresSQL] = "" + require.ErrorContains(t, validateScaleCase(testCase), "requires a reason") +} + +// TestValidateScaleCaseFreezesTraversalQualificationSplit verifies prioritized +// traversal cases cannot silently move between training, holdout, and +// diagnostic evidence after selector thresholds are chosen. +func TestValidateScaleCaseFreezesTraversalQualificationSplit(t *testing.T) { + testCase := ScaleCase{ + Name: "qualified", Dataset: "generated", Category: "generated_shortest_path_v2", + Cypher: "MATCH p = shortestPath((s)-[*]->(e)) RETURN p", + CandidateModes: []ExecutionMode{ModePostgresSQL}, + Shape: WorkloadShape{FixtureTier: "normal"}, + } + + require.ErrorContains(t, validateScaleCase(testCase), "qualification_split is required") + testCase.Shape.QualificationSplit = "training" + require.NoError(t, validateScaleCase(testCase)) + + testCase.Tags = []string{"holdout"} + require.ErrorContains(t, validateScaleCase(testCase), "holdout-tagged") + testCase.Shape.QualificationSplit = "holdout" + require.NoError(t, validateScaleCase(testCase)) + + testCase.Tags = nil + require.ErrorContains(t, validateScaleCase(testCase), "requires the holdout tag") + testCase.Shape = WorkloadShape{FixtureTier: "stress", QualificationSplit: "training"} + require.ErrorContains(t, validateScaleCase(testCase), "stress traversal") + testCase.Shape.QualificationSplit = "diagnostic" + require.NoError(t, validateScaleCase(testCase)) +} + +// TestValidateScaleCaseRequiresExactFixedSuffixV3Paths prevents a costly v2 +// capture from reaching report time without an independent stable path oracle. +func TestValidateScaleCaseRequiresExactFixedSuffixV3Paths(t *testing.T) { + rowCount := int64(1) + testCase := ScaleCase{ + Name: "v3-path", Dataset: "generated", Category: "generated_fixed_suffix_expansion", + Cypher: "MATCH p = (s)-[*]->(e) RETURN p", + CandidateModes: []ExecutionMode{ModePostgresSQL}, + Tags: []string{"fixed-suffix-expansion-v3"}, + Shape: WorkloadShape{FixtureTier: "normal", QualificationSplit: "training"}, + Expected: ExpectedResult{RowCount: &rowCount, ResultKind: "path_set"}, + } + + require.ErrorContains(t, validateScaleCase(testCase), "require exact expected.path_rows") + testCase.Expected.PathRows = []ExpectedPath{{Nodes: []string{"s", "e"}, RelationshipKinds: []string{"Expand"}}} + require.ErrorContains(t, validateScaleCase(testCase), "identify every relationship") + testCase.Expected.PathRows[0].RelationshipKeys = []string{"expand-1"} + require.NoError(t, validateScaleCase(testCase)) +} + +// TestScaleCorpusDatasets verifies that corpus dataset discovery removes repeated names and returns a deterministic lexical order. func TestScaleCorpusDatasets(t *testing.T) { - corpus := ScaleCorpus{Cases: []ScaleCase{ - {Name: "a", Dataset: "base", Category: "counts", Cypher: "return 1", CandidateModes: []ExecutionMode{ModePostgresSQL}}, - {Name: "b", Dataset: "adcs_fanout", Category: "counts", Cypher: "return 1", CandidateModes: []ExecutionMode{ModePostgresSQL}}, - {Name: "c", Dataset: "base", Category: "counts", Cypher: "return 1", CandidateModes: []ExecutionMode{ModePostgresSQL}}, - }} + corpus := ScaleCorpus{ + Cases: []ScaleCase{ + {Name: "a", Dataset: "base", Category: "counts", Cypher: "return 1", CandidateModes: []ExecutionMode{ModePostgresSQL}}, + { + Name: "b", + Dataset: "fixed_suffix_expansion_fanout", + Category: "counts", + Cypher: "return 1", + CandidateModes: []ExecutionMode{ModePostgresSQL}, + }, + {Name: "c", Dataset: "base", Category: "counts", Cypher: "return 1", CandidateModes: []ExecutionMode{ModePostgresSQL}}, + }, + } - require.Equal(t, []string{"adcs_fanout", "base"}, scaleCorpusDatasets(corpus)) + require.Equal(t, []string{"base", "fixed_suffix_expansion_fanout"}, scaleCorpusDatasets(corpus)) +} + +// TestGeneratedReconciliationDatasetRegistersThirtyKinds verifies that reconciliation generation exposes every RecKind01 through RecKind30 relationship kind. +func TestGeneratedReconciliationDatasetRegistersThirtyKinds(t *testing.T) { + doc, err := parseDataset("unused", testutil.ReconciliationScaleDataset) + require.NoError(t, err) + _, edgeKinds := doc.Graph.Kinds() + + for idx := 1; idx <= 30; idx++ { + require.Contains(t, edgeKinds, graph.StringKind(fmt.Sprintf("RecKind%02d", idx))) + } +} + +// TestGeneratedTrustPruningDatasetRegistersProductionShapes verifies that trust-pruning fixtures contain the domain, candidate, same-forest, cross-forest, and batch labels used by production queries. +func TestGeneratedTrustPruningDatasetRegistersProductionShapes(t *testing.T) { + doc, err := parseDataset("unused", testutil.TrustPruningScaleDataset) + require.NoError(t, err) + nodeKinds, edgeKinds := doc.Graph.Kinds() + + require.Contains(t, nodeKinds, graph.StringKind("Domain")) + require.Contains(t, nodeKinds, graph.StringKind("PruneCandidate")) + require.Contains(t, edgeKinds, graph.StringKind("SameForestTrust")) + require.Contains(t, edgeKinds, graph.StringKind("CrossForestTrust")) + require.Contains(t, edgeKinds, graph.StringKind("PruneBatch")) +} + +// TestGeneratedHopDatasetRegistersThirtyKindsAndEndpointSets verifies that hop fixtures expose both endpoint node classes, all thirty numbered relationship kinds, and the set-membership edge. +func TestGeneratedHopDatasetRegistersThirtyKindsAndEndpointSets(t *testing.T) { + doc, err := parseDataset("unused", testutil.HopScaleDataset) + require.NoError(t, err) + nodeKinds, edgeKinds := doc.Graph.Kinds() + + require.Contains(t, nodeKinds, graph.StringKind("HopIDEndpoint")) + require.Contains(t, nodeKinds, graph.StringKind("HopTemplate")) + for idx := 1; idx <= 30; idx++ { + require.Contains(t, edgeKinds, graph.StringKind(fmt.Sprintf("HopKind%02d", idx))) + } + require.Contains(t, edgeKinds, graph.StringKind("HopSetEdge")) +} + +// TestGeneratedScanLookupDatasetRegistersWideAndLargeShapes verifies that scan fixtures contain the base, role, and hydration nodes plus every relationship kind used by wide lookup plans. +func TestGeneratedScanLookupDatasetRegistersWideAndLargeShapes(t *testing.T) { + doc, err := parseDataset("unused", testutil.ScanLookupScaleDataset) + require.NoError(t, err) + nodeKinds, edgeKinds := doc.Graph.Kinds() + + require.Contains(t, nodeKinds, graph.StringKind("ADBase")) + require.Contains(t, nodeKinds, graph.StringKind("AZRole")) + require.Contains(t, nodeKinds, graph.StringKind("Hydrate")) + require.Contains(t, edgeKinds, graph.StringKind("ScanPostProcessed")) + require.Contains(t, edgeKinds, graph.StringKind("Contains")) + for idx := 1; idx <= 9; idx++ { + require.Contains(t, edgeKinds, graph.StringKind(fmt.Sprintf("ScanEdge%02d", idx))) + } +} + +// TestGeneratedShortestPathDatasetRegistersMatrixShapes verifies that shortest-path generation produces nonempty nodes and both generic and typed traversal relationships. +func TestGeneratedShortestPathDatasetRegistersMatrixShapes(t *testing.T) { + doc, err := parseDataset("unused", testutil.ShortestPathScaleDataset) + require.NoError(t, err) + nodeKinds, edgeKinds := doc.Graph.Kinds() + + require.Contains(t, nodeKinds, graph.StringKind("ShortestNode")) + require.Contains(t, edgeKinds, graph.StringKind("Traverse")) + require.Contains(t, edgeKinds, graph.StringKind("TypedTraverse")) + require.NotEmpty(t, doc.Graph.Nodes) +} + +// TestGeneratedFixedSuffixExpansionDatasetRegistersSuffixAndDecoyShapes verifies that fixed-suffix fixtures register all path stages and the wrong-entry decoy needed to detect over-broad matching. +func TestGeneratedFixedSuffixExpansionDatasetRegistersSuffixAndDecoyShapes(t *testing.T) { + doc, err := parseDataset("unused", testutil.FixedSuffixExpansionScaleDataset) + require.NoError(t, err) + nodeKinds, edgeKinds := doc.Graph.Kinds() + + for _, kind := range []string{"ExpansionRoot", "ExpansionNode", "SuffixHead", "SuffixMiddle", "SuffixTerminal"} { + require.Contains(t, nodeKinds, graph.StringKind(kind)) + } + for _, kind := range []string{"Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix", "WrongEnterSuffix"} { + require.Contains(t, edgeKinds, graph.StringKind(kind)) + } +} + +// TestValidateScaleCaseRequiresCompleteWriteScenario verifies that destructive cases include at least one post-state assertion after selection and affected-count expectations. +func TestValidateScaleCaseRequiresCompleteWriteScenario(t *testing.T) { + zero := int64(0) + testCase := ScaleCase{ + Name: "write", + Dataset: "base", + Category: "delete", + Cypher: "MATCH (n) DELETE n", + CandidateModes: []ExecutionMode{ModePostgresSQL}, + WriteScenario: &WriteScenario{ + SelectionCypher: "MATCH (n) RETURN n", + AffectedEntity: "node", + ExpectedMatched: &zero, + ExpectedAffected: &zero, + PostState: []ScaleStateQuery{{ + Name: "survivors", + Cypher: "MATCH (n) RETURN n", + Expected: ExpectedResult{RowCount: &zero}, + }}, + }, + } + + require.NoError(t, validateScaleCase(testCase)) + testCase.WriteScenario.PostState = nil + require.ErrorContains(t, validateScaleCase(testCase), "post_state is required") +} + +// TestSelectScaleCorpusUsesExactSelectorsAndMarksDiagnostics verifies that partial dataset/tag selection records omitted declarations, marks the manifest diagnostic-only, and rejects unresolved exact selectors. +func TestSelectScaleCorpusUsesExactSelectorsAndMarksDiagnostics(t *testing.T) { + corpus := ScaleCorpus{ + Cases: []ScaleCase{ + { + Name: "lookup", + Dataset: "base", + Category: "lookup", + Tags: []string{"primary"}, + CandidateModes: []ExecutionMode{ModePostgresSQL}, + }, + { + Name: "control", + Dataset: "base", + Category: "lookup", + Tags: []string{"control"}, + CandidateModes: []ExecutionMode{ModePostgresSQL, ModeNeo4j}, + }, + { + Name: "other", + Dataset: "other", + Category: "count", + CandidateModes: []ExecutionMode{ModePostgresSQL}, + }, + }, + } + + selected, manifest, err := selectScaleCorpus(corpus, CorpusSelectors{ + Datasets: []string{"base"}, + Tags: []string{"primary", "control"}, + }) + require.NoError(t, err) + require.Len(t, selected.Cases, 2) + require.True(t, manifest.DiagnosticOnly) + require.Equal(t, 1, manifest.OmittedDeclarationCount) + require.NotEmpty(t, manifest.DeclarationSHA256) + + _, _, err = selectScaleCorpus(corpus, CorpusSelectors{Cases: []string{"missing"}}) + require.ErrorContains(t, err, "unknown case selector") +} + +// TestSelectScaleCorpusRejectsAmbiguousExactNames verifies that a bare case selector cannot choose between identically named cases from different datasets. +func TestSelectScaleCorpusRejectsAmbiguousExactNames(t *testing.T) { + corpus := ScaleCorpus{ + Cases: []ScaleCase{{ + Name: "same", + Dataset: "one", + }, { + Name: "same", + Dataset: "two", + }}, + } + _, _, err := selectScaleCorpus(corpus, CorpusSelectors{Cases: []string{"same"}}) + require.ErrorContains(t, err, "ambiguous case selector") } diff --git a/cmd/graphbench/datasets.go b/cmd/graphbench/datasets.go index af400ca8..c24bf5f5 100644 --- a/cmd/graphbench/datasets.go +++ b/cmd/graphbench/datasets.go @@ -18,16 +18,24 @@ package main import ( "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" "fmt" "os" "path/filepath" + "strings" + "github.com/specterops/dawgs/drivers/pg" "github.com/specterops/dawgs/graph" "github.com/specterops/dawgs/opengraph" + "github.com/specterops/dawgs/testutil" ) +// defaultGraphName names the isolated graph populated with benchmark fixtures. const defaultGraphName = "integration_test" +// scanDatasetKinds enumerates dataset kinds without changing the source data. func scanDatasetKinds(datasetDir string, datasetNames []string) (graph.Kinds, graph.Kinds, error) { var nodeKinds, edgeKinds graph.Kinds @@ -45,7 +53,12 @@ func scanDatasetKinds(datasetDir string, datasetNames []string) (graph.Kinds, gr return nodeKinds, edgeKinds, nil } +// parseDataset decodes a fixture document or dispatches to the requested generated dataset builder. func parseDataset(datasetDir, name string) (opengraph.Document, error) { + if fixture := generatedDataset(name); fixture != nil { + return opengraph.Document{Graph: *fixture}, nil + } + path := filepath.Join(datasetDir, name+".json") f, err := os.Open(path) if err != nil { @@ -61,7 +74,12 @@ func parseDataset(datasetDir, name string) (opengraph.Document, error) { return doc, nil } +// loadDataset decodes and loads a named fixture dataset into an empty graph. func loadDataset(ctx context.Context, db graph.Database, datasetDir, name string) (opengraph.IDMap, error) { + if fixture := generatedDataset(name); fixture != nil { + return opengraph.WriteGraph(ctx, db, fixture) + } + path := filepath.Join(datasetDir, name+".json") f, err := os.Open(path) if err != nil { @@ -77,12 +95,635 @@ func loadDataset(ctx context.Context, db graph.Database, datasetDir, name string return idMap, nil } +// generatedDataset constructs a named generated fixture and its shape-specific expectations. +func generatedDataset(name string) *opengraph.Graph { + if config, ok := parseEndpointSeededExpansionDatasetName(name); ok { + return testutil.NewEndpointSeededExpansionScaleFixture(config) + } + if config, ok := parseShortestPathV2DatasetName(name); ok { + return testutil.NewShortestPathScaleV2Fixture(config) + } + var shortestDepth, shortestFanout int + if matched, _ := fmt.Sscanf(name, testutil.ShortestPathScaleDataset+"_d%d_f%d", &shortestDepth, &shortestFanout); matched == 2 && shortestDepth >= 1 && shortestFanout >= 1 && name == fmt.Sprintf(testutil.ShortestPathScaleDataset+"_d%d_f%d", shortestDepth, shortestFanout) { + return testutil.NewShortestPathScaleFixture(testutil.ShortestPathScaleConfig{ + Depth: shortestDepth, + Fanout: shortestFanout, + }) + } + var expansionDepth, expansionFanout, validSuffixEvery, expansionPayload int + if matched, _ := fmt.Sscanf(name, testutil.FixedSuffixExpansionScaleDataset+"_d%d_f%d_v%d_p%d", &expansionDepth, &expansionFanout, &validSuffixEvery, &expansionPayload); matched == 4 && expansionDepth >= 0 && expansionFanout >= 1 && validSuffixEvery >= 1 && expansionPayload >= 0 && name == fmt.Sprintf(testutil.FixedSuffixExpansionScaleDataset+"_d%d_f%d_v%d_p%d", expansionDepth, expansionFanout, validSuffixEvery, expansionPayload) { + return testutil.NewFixedSuffixExpansionScaleFixture(testutil.FixedSuffixExpansionScaleConfig{ + ExpansionDepth: expansionDepth, + Fanout: expansionFanout, + ValidSuffixEvery: validSuffixEvery, + PropertyPayloadSize: expansionPayload, + }) + } + if config, ok := parseFixedSuffixExpansionV3DatasetName(name); ok { + return testutil.NewFixedSuffixExpansionScaleFixture(config) + } + if config, ok := parseFixedSuffixExpansionV2DatasetName(name); ok { + return testutil.NewFixedSuffixExpansionScaleFixture(config) + } + switch name { + case testutil.ReconciliationScaleDataset: + return testutil.NewReconciliationScaleFixture(128) + case testutil.TrustPruningScaleDataset: + return testutil.NewTrustPruningScaleFixture(128) + case testutil.HopScaleDataset: + return testutil.NewHopScaleFixture(128) + case testutil.ScanLookupScaleDataset: + return testutil.NewScanLookupScaleFixture(128) + case testutil.ShortestPathScaleDataset: + return testutil.NewShortestPathScaleFixture(testutil.ShortestPathScaleConfig{ + Depth: 16, + Fanout: 128, + }) + case testutil.FixedSuffixExpansionScaleDataset: + return testutil.NewFixedSuffixExpansionScaleFixture(testutil.FixedSuffixExpansionScaleConfig{ + ExpansionDepth: 8, + Fanout: 100, + ValidSuffixEvery: 10, + PropertyPayloadSize: 4096, + }) + default: + return nil + } +} + +// FixtureMetadata captures fixture cardinalities, checksums, and generated-shape expectations. +type FixtureMetadata struct { + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Checksum identifies the fixture's canonical logical node and relationship contents. + Checksum string `json:"checksum"` + // NodeCount records logical fixture nodes declared or loaded. + NodeCount int `json:"node_count"` + // EdgeCount records logical fixture relationships declared or loaded. + EdgeCount int `json:"edge_count"` + // PhysicalValidated reports whether live database counts and checksum matched fixture metadata. + PhysicalValidated bool `json:"physical_cardinality_validated,omitempty"` + // PhysicalNodeCount records physical node rows present in the backend fixture. + PhysicalNodeCount int64 `json:"physical_node_count,omitempty"` + // PhysicalEdgeCount records physical relationship rows present in the backend fixture. + PhysicalEdgeCount int64 `json:"physical_edge_count,omitempty"` + // NodeRelationBytes records the physical size of the graph's node relation. + NodeRelationBytes int64 `json:"node_relation_bytes,omitempty"` + // EdgeRelationBytes records the physical size of the graph's relationship relation. + EdgeRelationBytes int64 `json:"edge_relation_bytes,omitempty"` + // Configuration captures the generator parameters that define the fixture shape. + Configuration string `json:"configuration,omitempty"` + // Shortest contains expectations derived from a generated shortest-path fixture. + Shortest *ShortestFixtureExpectations `json:"shortest,omitempty"` + // FixedSuffixExpansion contains expectations derived from a fixed-suffix expansion fixture. + FixedSuffixExpansion *FixedSuffixExpansionFixtureExpectations `json:"fixed_suffix_expansion,omitempty"` + // EndpointSeededExpansion contains expectations derived from an endpoint-seeded expansion fixture. + EndpointSeededExpansion *EndpointSeededExpansionFixtureExpectations `json:"endpoint_seeded_expansion,omitempty"` +} + +// ShortestFixtureExpectations records expected distances, witnesses, and intermediate state for shortest-path fixtures. +type ShortestFixtureExpectations struct { + // RootForwardDegree records outgoing relationships incident to the traversal root. + RootForwardDegree int64 `json:"root_forward_degree"` + // RootReverseDegree records incoming relationships incident to the traversal root. + RootReverseDegree int64 `json:"root_reverse_degree"` + // MaximumIntermediateForwardByLevel maps traversal depth to the largest expected forward frontier. + MaximumIntermediateForwardByLevel map[string]int64 `json:"maximum_intermediate_forward_by_level"` + // MaximumIntermediateReverseByLevel maps traversal depth to the largest expected reverse frontier. + MaximumIntermediateReverseByLevel map[string]int64 `json:"maximum_intermediate_reverse_by_level"` + // PhysicalTraversableEdgesByKind maps relationship kind to physical traversable edge count. + PhysicalTraversableEdgesByKind map[string]int64 `json:"physical_traversable_edges_by_kind"` + // DistinctReachableNodesByLevel maps traversal depth to distinct reachable node count. + DistinctReachableNodesByLevel map[string]int64 `json:"distinct_reachable_nodes_by_level"` + // ExpectedMinimumDistance records the shortest expected hop count between endpoints. + ExpectedMinimumDistance int64 `json:"expected_minimum_distance"` + // ExpectedOnePathCardinality records the expected number of valid single shortest-path witnesses. + ExpectedOnePathCardinality int64 `json:"expected_one_path_cardinality"` + // ExpectedAllShortestCardinality records the expected number of all-shortest-path results. + ExpectedAllShortestCardinality int64 `json:"expected_all_shortest_cardinality"` + // ExpectedPredecessorEdges records predecessor edges expected in the shortest-path DAG. + ExpectedPredecessorEdges int64 `json:"expected_relationship_distinct_predecessor_edges"` + // DisconnectedStateCardinality records recursive states belonging to disconnected shortest-path regions. + DisconnectedStateCardinality int64 `json:"disconnected_state_cardinality"` + // ParallelPhysicalEdges records physical parallel relationships in the generated fixture. + ParallelPhysicalEdges int64 `json:"parallel_physical_edges"` + // ParallelDistinctTargets records distinct targets reached by parallel fixture edges. + ParallelDistinctTargets int64 `json:"parallel_distinct_targets"` +} + +// FixedSuffixExpansionFixtureExpectations records expected state and output sizes for fixed-suffix expansion fixtures. +type FixedSuffixExpansionFixtureExpectations struct { + // RootSourceRows records rows selected as expansion roots. + RootSourceRows int64 `json:"root_source_rows"` + // DistinctRoots records unique root nodes in the generated fixture. + DistinctRoots int64 `json:"distinct_roots"` + // ForwardExpansionStates records recursive states visited by forward fixed-suffix expansion. + ForwardExpansionStates int64 `json:"forward_expansion_states"` + // SuffixRows records rows belonging to the fixed suffix of generated paths. + SuffixRows int64 `json:"suffix_rows"` + // DistinctBoundaries records unique terminal boundaries in the generated fixture. + DistinctBoundaries int64 `json:"distinct_boundaries"` + // ReachableBoundaries records terminal boundaries reachable in the generated fixture. + ReachableBoundaries int64 `json:"reachable_boundaries"` + // DisconnectedBoundaries records terminal boundaries intentionally disconnected from traversal roots. + DisconnectedBoundaries int64 `json:"disconnected_boundaries"` + // ExpectedReverseStates records the reverse-search states expected from the generated fixture. + ExpectedReverseStates int64 `json:"expected_reverse_states"` + // CompleteOutputTrails records output trails before fixture eligibility filters are applied. + CompleteOutputTrails int64 `json:"complete_output_trails"` + // ProductiveBoundaryCycleEdges records the two relationship-distinct Expand + // relationships forming the optional productive-boundary cycle. + ProductiveBoundaryCycleEdges int64 `json:"productive_boundary_cycle_edges,omitempty"` + // ProductiveBoundarySelfLoopEdges records the optional productive-boundary + // Expand self-loop. + ProductiveBoundarySelfLoopEdges int64 `json:"productive_boundary_self_loop_edges,omitempty"` +} + +// EndpointSeededExpansionFixtureExpectations records expected state and output sizes for endpoint-seeded expansion fixtures. +type EndpointSeededExpansionFixtureExpectations struct { + // MatchingEndpoints records endpoints satisfying the generated fixture predicate. + MatchingEndpoints int64 `json:"matching_endpoints"` + // OtherEndpoints records nonmatching endpoint nodes in an endpoint-seeded fixture. + OtherEndpoints int64 `json:"other_endpoints"` + // EligiblePrefixRows records prefix rows that can connect to the required suffix. + EligiblePrefixRows int64 `json:"eligible_prefix_rows"` + // MatchingIneligibleLanes records matching lanes excluded by endpoint eligibility filters. + MatchingIneligibleLanes int64 `json:"matching_ineligible_lanes"` + // ExpectedReverseStates records the reverse-search states expected from the generated fixture. + ExpectedReverseStates int64 `json:"expected_reverse_states"` + // ExpectedOutputTrails records result trails expected from the generated expansion fixture. + ExpectedOutputTrails int64 `json:"expected_output_trails"` +} + +// fixtureMetadata derives fixture counts, checksums, and generated-shape expectations from a graph. +func fixtureMetadata(datasetDir, name string) (FixtureMetadata, error) { + doc, err := parseDataset(datasetDir, name) + if err != nil { + return FixtureMetadata{}, err + } + raw, err := json.Marshal(doc.Graph) + if err != nil { + return FixtureMetadata{}, fmt.Errorf("encode dataset %s for checksum: %w", name, err) + } + digest := sha256.Sum256(raw) + configuration := "file" + if generatedDataset(name) != nil { + configuration = name + } + metadata := FixtureMetadata{ + Dataset: name, + Checksum: hex.EncodeToString(digest[:]), + NodeCount: len(doc.Graph.Nodes), + EdgeCount: len(doc.Graph.Edges), + Configuration: configuration, + } + if config, ok := parseFixedSuffixExpansionV3DatasetName(name); ok { + metadata.FixedSuffixExpansion = fixedSuffixExpansionV3FixtureExpectations(doc.Graph, config) + } else if config, ok := parseFixedSuffixExpansionV2DatasetName(name); ok { + metadata.FixedSuffixExpansion = fixedSuffixExpansionV2FixtureExpectations(config) + } + if config, ok := parseShortestPathV2DatasetName(name); ok { + metadata.Shortest = shortestFixtureExpectations(doc.Graph, config) + } + if config, ok := parseEndpointSeededExpansionDatasetName(name); ok { + metadata.EndpointSeededExpansion = endpointSeededExpansionFixtureExpectations(doc.Graph, config) + } + return metadata, nil +} + +// parseEndpointSeededExpansionDatasetName decodes and validates every scale parameter embedded in an endpoint-seeded dataset name. +func parseEndpointSeededExpansionDatasetName(name string) (testutil.EndpointSeededExpansionScaleConfig, bool) { + var depth, matchingEndpoints, otherEndpoints, matchingEligible, otherEligible, matchingIneligible, parallel, cycle, payload int + format := testutil.EndpointSeededExpansionScaleDataset + "_d%d_e%d_q%d_w%d_o%d_x%d_m%d_c%d_p%d" + matched, _ := fmt.Sscanf(name, format, &depth, &matchingEndpoints, &otherEndpoints, &matchingEligible, &otherEligible, &matchingIneligible, ¶llel, &cycle, &payload) + config := testutil.EndpointSeededExpansionScaleConfig{ + Depth: depth, MatchingEndpoints: matchingEndpoints, OtherEndpoints: otherEndpoints, + MatchingEligibleLanes: matchingEligible, OtherEligibleLanes: otherEligible, + MatchingIneligibleLanes: matchingIneligible, ParallelEdges: parallel, + AddCycle: cycle == 1, PropertyPayloadSize: payload, + } + if matched != 9 || (cycle != 0 && cycle != 1) || testutil.ValidateEndpointSeededExpansionScaleConfig(config) != nil || name != endpointSeededExpansionDatasetName(config) { + return testutil.EndpointSeededExpansionScaleConfig{}, false + } + return config, true +} + +// endpointSeededExpansionDatasetName encodes endpoint-seeded scale parameters in their canonical dataset name. +func endpointSeededExpansionDatasetName(config testutil.EndpointSeededExpansionScaleConfig) string { + cycle := 0 + if config.AddCycle { + cycle = 1 + } + return fmt.Sprintf(testutil.EndpointSeededExpansionScaleDataset+"_d%d_e%d_q%d_w%d_o%d_x%d_m%d_c%d_p%d", + config.Depth, config.MatchingEndpoints, config.OtherEndpoints, config.MatchingEligibleLanes, + config.OtherEligibleLanes, config.MatchingIneligibleLanes, config.ParallelEdges, cycle, config.PropertyPayloadSize) +} + +// endpointSeededExpansionFixtureExpectations derives reverse-search state and output counts from an endpoint-seeded fixture. +func endpointSeededExpansionFixtureExpectations(fixture opengraph.Graph, config testutil.EndpointSeededExpansionScaleConfig) *EndpointSeededExpansionFixtureExpectations { + incoming := map[string][]int{} + matching := map[string]bool{} + eligibleUsers := map[string]bool{} + for _, node := range fixture.Nodes { + if objectID, ok := node.Properties["objectid"].(string); ok && strings.HasSuffix(objectID, "-512") { + matching[node.ID] = true + } + } + for edgeIdx, edge := range fixture.Edges { + if edge.Kind == "MemberOf" { + incoming[edge.EndID] = append(incoming[edge.EndID], edgeIdx) + } else if edge.Kind == "HasSession" { + eligibleUsers[edge.EndID] = true + } + } + var ( + states, outputs int64 + visit func(string, int, map[int]bool) + ) + visit = func(nodeID string, depth int, used map[int]bool) { + states++ + if depth > 0 && eligibleUsers[nodeID] { + outputs++ + } + if depth == 64 { + return + } + for _, edgeIdx := range incoming[nodeID] { + if used[edgeIdx] { + continue + } + used[edgeIdx] = true + visit(fixture.Edges[edgeIdx].StartID, depth+1, used) + delete(used, edgeIdx) + } + } + for endpoint := range matching { + visit(endpoint, 0, map[int]bool{}) + } + return &EndpointSeededExpansionFixtureExpectations{ + MatchingEndpoints: int64(config.MatchingEndpoints), OtherEndpoints: int64(config.OtherEndpoints), + EligiblePrefixRows: int64(config.MatchingEligibleLanes + config.OtherEligibleLanes), + MatchingIneligibleLanes: int64(config.MatchingIneligibleLanes), + ExpectedReverseStates: states, ExpectedOutputTrails: outputs, + } +} + +// parseShortestPathV2DatasetName decodes and validates every scale parameter embedded in a shortest-path dataset name. +func parseShortestPathV2DatasetName(name string) (testutil.ShortestPathScaleV2Config, bool) { + var ( + depth, rootOut, rootIn, intermediateOut, intermediateIn, level int + kinds, targets, diamond, disconnected, payload, cycle, selfLoop int + ) + + format := testutil.ShortestPathScaleV2Dataset + "_d%d_o%d_r%d_fo%d_fi%d_l%d_k%d_t%d_w%d_x%d_p%d_c%d_s%d" + matched, _ := fmt.Sscanf(name, format, &depth, &rootOut, &rootIn, &intermediateOut, &intermediateIn, &level, &kinds, &targets, &diamond, &disconnected, &payload, &cycle, &selfLoop) + if matched != 13 || (cycle != 0 && cycle != 1) || (selfLoop != 0 && selfLoop != 1) { + return testutil.ShortestPathScaleV2Config{}, false + } + config := testutil.ShortestPathScaleV2Config{ + Depth: depth, + ForwardRootFanOut: rootOut, + ReverseRootFanIn: rootIn, + IntermediateFanOut: intermediateOut, + IntermediateReverseFanIn: intermediateIn, + FanInLevel: level, + ParallelKindCount: kinds, + ParallelTargetCount: targets, + DiamondWidth: diamond, + DisconnectedWidth: disconnected, + PropertyPayloadSize: payload, + AddCycle: cycle == 1, + AddSelfLoop: selfLoop == 1, + } + if err := testutil.ValidateShortestPathScaleV2Config(config); err != nil || name != shortestPathV2DatasetName(config) { + return testutil.ShortestPathScaleV2Config{}, false + } + return config, true +} + +// shortestPathV2DatasetName encodes shortest-path scale parameters in their canonical dataset name. +func shortestPathV2DatasetName(config testutil.ShortestPathScaleV2Config) string { + cycle, selfLoop := 0, 0 + if config.AddCycle { + cycle = 1 + } + if config.AddSelfLoop { + selfLoop = 1 + } + return fmt.Sprintf(testutil.ShortestPathScaleV2Dataset+"_d%d_o%d_r%d_fo%d_fi%d_l%d_k%d_t%d_w%d_x%d_p%d_c%d_s%d", + config.Depth, config.ForwardRootFanOut, config.ReverseRootFanIn, + config.IntermediateFanOut, config.IntermediateReverseFanIn, config.FanInLevel, + config.ParallelKindCount, config.ParallelTargetCount, config.DiamondWidth, + config.DisconnectedWidth, config.PropertyPayloadSize, cycle, selfLoop) +} + +// shortestFixtureExpectations derives shortest distance, path cardinality, and intermediate-state expectations from a fixture. +func shortestFixtureExpectations(fixture opengraph.Graph, config testutil.ShortestPathScaleV2Config) *ShortestFixtureExpectations { + expectations := &ShortestFixtureExpectations{ + MaximumIntermediateForwardByLevel: map[string]int64{}, + MaximumIntermediateReverseByLevel: map[string]int64{}, + PhysicalTraversableEdgesByKind: map[string]int64{}, + DistinctReachableNodesByLevel: map[string]int64{}, + ExpectedMinimumDistance: int64(config.Depth), + ExpectedOnePathCardinality: 1, + ExpectedAllShortestCardinality: 1, + ExpectedPredecessorEdges: int64(config.Depth), + DisconnectedStateCardinality: int64(config.DisconnectedWidth + 1), + ParallelPhysicalEdges: int64(config.ParallelKindCount * config.ParallelTargetCount), + ParallelDistinctTargets: int64(config.ParallelTargetCount), + } + outgoing, incoming := map[string][]string{}, map[string][]string{} + for _, edge := range fixture.Edges { + expectations.PhysicalTraversableEdgesByKind[edge.Kind]++ + outgoing[edge.StartID] = append(outgoing[edge.StartID], edge.EndID) + incoming[edge.EndID] = append(incoming[edge.EndID], edge.StartID) + } + expectations.RootForwardDegree = int64(len(outgoing["sp-v2-start"])) + expectations.RootReverseDegree = int64(len(incoming["sp-v2-inbound-root"])) + for level := 1; level < config.Depth; level++ { + id, key := fmt.Sprintf("sp-v2-linear-%02d", level), fmt.Sprintf("%d", level) + expectations.MaximumIntermediateForwardByLevel[key] = int64(len(outgoing[id])) + expectations.MaximumIntermediateReverseByLevel[key] = int64(len(incoming[fmt.Sprintf("sp-v2-inbound-linear-%02d", level)])) + } + seen := map[string]bool{"sp-v2-start": true} + frontier := []string{"sp-v2-start"} + for level := 0; len(frontier) > 0 && level <= 64; level++ { + expectations.DistinctReachableNodesByLevel[fmt.Sprintf("%d", level)] = int64(len(frontier)) + next := []string{} + for _, source := range frontier { + for _, target := range outgoing[source] { + if !seen[target] { + seen[target] = true + next = append(next, target) + } + } + } + frontier = next + } + return expectations +} + +// parseFixedSuffixExpansionV3DatasetName decodes the exact fixed-suffix +// grammar with independently encoded matching roots and productive-boundary +// cycle/self-loop controls. +func parseFixedSuffixExpansionV3DatasetName(name string) (testutil.FixedSuffixExpansionScaleConfig, bool) { + var depth, fanout, reachable, disconnected, fanIn, multiplicity, roots, zeroDepth, cycle, selfLoop, payload int + format := testutil.FixedSuffixExpansionScaleV3Dataset + "_d%d_f%d_r%d_x%d_i%d_m%d_q%d_z%d_c%d_s%d_p%d" + matched, _ := fmt.Sscanf(name, format, &depth, &fanout, &reachable, &disconnected, &fanIn, &multiplicity, &roots, &zeroDepth, &cycle, &selfLoop, &payload) + if matched != 11 || (zeroDepth != 0 && zeroDepth != 1) || (cycle != 0 && cycle != 1) || (selfLoop != 0 && selfLoop != 1) { + return testutil.FixedSuffixExpansionScaleConfig{}, false + } + + rootSuffix := zeroDepth == 1 + config := testutil.FixedSuffixExpansionScaleConfig{ + ExpansionDepth: depth, + Fanout: fanout, + ExactReachableSuffixSources: &reachable, + DisconnectedSuffixSources: disconnected, + ReverseFanIn: fanIn, + SuffixPathsPerBoundary: multiplicity, + RootMatchCount: roots, + RootHasZeroDepthSuffix: &rootSuffix, + AddProductiveBoundaryCycle: cycle == 1, + AddProductiveBoundarySelfLoop: selfLoop == 1, + PropertyPayloadSize: payload, + } + if testutil.ValidateFixedSuffixExpansionScaleV3Config(config) != nil || name != fixedSuffixExpansionV3DatasetName(config) { + return testutil.FixedSuffixExpansionScaleConfig{}, false + } + return config, true +} + +// fixedSuffixExpansionV3DatasetName encodes every v3 fixture dimension in its +// canonical, round-trippable dataset name. +func fixedSuffixExpansionV3DatasetName(config testutil.FixedSuffixExpansionScaleConfig) string { + reachable, zeroDepth, cycle, selfLoop := 0, 0, 0, 0 + if config.ExactReachableSuffixSources != nil { + reachable = *config.ExactReachableSuffixSources + } + if config.RootHasZeroDepthSuffix != nil && *config.RootHasZeroDepthSuffix { + zeroDepth = 1 + } + if config.AddProductiveBoundaryCycle { + cycle = 1 + } + if config.AddProductiveBoundarySelfLoop { + selfLoop = 1 + } + return fmt.Sprintf(testutil.FixedSuffixExpansionScaleV3Dataset+"_d%d_f%d_r%d_x%d_i%d_m%d_q%d_z%d_c%d_s%d_p%d", + config.ExpansionDepth, config.Fanout, reachable, config.DisconnectedSuffixSources, + config.ReverseFanIn, config.SuffixPathsPerBoundary, config.RootMatchCount, + zeroDepth, cycle, selfLoop, config.PropertyPayloadSize) +} + +// parseFixedSuffixExpansionV2DatasetName decodes and validates every scale parameter embedded in a fixed-suffix dataset name. +func parseFixedSuffixExpansionV2DatasetName(name string) (testutil.FixedSuffixExpansionScaleConfig, bool) { + var depth, fanout, reachable, disconnected, fanIn, multiplicity, zeroDepth, payload int + format := testutil.FixedSuffixExpansionScaleDataset + "_v2_d%d_f%d_r%d_x%d_i%d_m%d_z%d_p%d" + matched, _ := fmt.Sscanf(name, format, &depth, &fanout, &reachable, &disconnected, &fanIn, &multiplicity, &zeroDepth, &payload) + if matched != 8 || depth < 0 || fanout < 1 || reachable < 0 || reachable > fanout || disconnected < 0 || fanIn < 0 || multiplicity < 1 || (zeroDepth != 0 && zeroDepth != 1) || payload < 0 || name != fmt.Sprintf(format, depth, fanout, reachable, disconnected, fanIn, multiplicity, zeroDepth, payload) { + return testutil.FixedSuffixExpansionScaleConfig{}, false + } + rootSuffix := zeroDepth == 1 + return testutil.FixedSuffixExpansionScaleConfig{ + ExpansionDepth: depth, + Fanout: fanout, + ExactReachableSuffixSources: &reachable, + DisconnectedSuffixSources: disconnected, + ReverseFanIn: fanIn, + SuffixPathsPerBoundary: multiplicity, + RootMatchCount: 1, + RootHasZeroDepthSuffix: &rootSuffix, + PropertyPayloadSize: payload, + }, true +} + +// fixedSuffixExpansionV2FixtureExpectations preserves the exact v2 metadata +// contract for every existing fixture name. +func fixedSuffixExpansionV2FixtureExpectations(config testutil.FixedSuffixExpansionScaleConfig) *FixedSuffixExpansionFixtureExpectations { + reachable := 0 + if config.ExactReachableSuffixSources != nil { + reachable = *config.ExactReachableSuffixSources + } + rootSuffix := config.RootHasZeroDepthSuffix != nil && *config.RootHasZeroDepthSuffix + zero := 0 + if rootSuffix { + zero = 1 + } + multiplicity := max(config.SuffixPathsPerBoundary, 1) + rootCount := max(config.RootMatchCount, 1) + productiveFanIn := 0 + if zero+reachable > 0 { + productiveFanIn = config.ReverseFanIn + } + return &FixedSuffixExpansionFixtureExpectations{ + RootSourceRows: int64(rootCount), + DistinctRoots: int64(rootCount), + ForwardExpansionStates: int64(rootCount + config.Fanout*config.ExpansionDepth), + SuffixRows: int64((zero + reachable + config.DisconnectedSuffixSources) * multiplicity), + DistinctBoundaries: int64(zero + reachable + config.DisconnectedSuffixSources), + ReachableBoundaries: int64(zero + reachable), + DisconnectedBoundaries: int64(config.DisconnectedSuffixSources), + ExpectedReverseStates: int64(zero + reachable*(config.ExpansionDepth+1) + config.DisconnectedSuffixSources + productiveFanIn), + CompleteOutputTrails: int64((zero + reachable) * multiplicity), + } +} + +// fixedSuffixExpansionV3FixtureExpectations derives exact forward and reverse +// relationship-distinct states and output trails from a v3 fixture graph. +func fixedSuffixExpansionV3FixtureExpectations(fixture opengraph.Graph, config testutil.FixedSuffixExpansionScaleConfig) *FixedSuffixExpansionFixtureExpectations { + type adjacentEdge struct { + index int + next string + } + + nodeKinds := map[string]map[string]bool{} + roots := []string{} + for _, node := range fixture.Nodes { + kinds := map[string]bool{} + for _, kind := range node.Kinds { + kinds[kind] = true + } + nodeKinds[node.ID] = kinds + if kinds["ExpansionRoot"] && node.Properties["root_key"] == "generated-fse-root" { + roots = append(roots, node.ID) + } + } + + expandForward := map[string][]adjacentEdge{} + expandReverse := map[string][]adjacentEdge{} + edgesByStart := map[string][]int{} + for edgeIdx, edge := range fixture.Edges { + edgesByStart[edge.StartID] = append(edgesByStart[edge.StartID], edgeIdx) + if edge.Kind == "Expand" { + expandForward[edge.StartID] = append(expandForward[edge.StartID], adjacentEdge{index: edgeIdx, next: edge.EndID}) + expandReverse[edge.EndID] = append(expandReverse[edge.EndID], adjacentEdge{index: edgeIdx, next: edge.StartID}) + } + } + + suffixPaths := map[string]int64{} + for _, enter := range fixture.Edges { + if enter.Kind != "EnterSuffix" || !nodeKinds[enter.EndID]["SuffixHead"] { + continue + } + for _, continueIdx := range edgesByStart[enter.EndID] { + continuation := fixture.Edges[continueIdx] + if continuation.Kind != "ContinueSuffix" || !nodeKinds[continuation.EndID]["SuffixMiddle"] { + continue + } + for _, completeIdx := range edgesByStart[continuation.EndID] { + completion := fixture.Edges[completeIdx] + if completion.Kind == "CompleteSuffix" && nodeKinds[completion.EndID]["SuffixTerminal"] { + suffixPaths[enter.StartID]++ + } + } + } + } + + used := make([]bool, len(fixture.Edges)) + var enumerate func(map[string][]adjacentEdge, string, int, func(string)) int64 + enumerate = func(adjacency map[string][]adjacentEdge, nodeID string, depth int, observe func(string)) int64 { + states := int64(1) + observe(nodeID) + if depth == config.ExpansionDepth { + return states + } + for _, edge := range adjacency[nodeID] { + if used[edge.index] { + continue + } + used[edge.index] = true + states += enumerate(adjacency, edge.next, depth+1, observe) + used[edge.index] = false + } + return states + } + + boundaryVisits := map[string]int64{} + forwardStates := int64(0) + for _, root := range roots { + forwardStates += enumerate(expandForward, root, 0, func(nodeID string) { + if suffixPaths[nodeID] > 0 { + boundaryVisits[nodeID]++ + } + }) + } + + reverseStates := int64(0) + for boundary := range suffixPaths { + reverseStates += enumerate(expandReverse, boundary, 0, func(string) {}) + } + + suffixRows, outputTrails := int64(0), int64(0) + for boundary, pathCount := range suffixPaths { + suffixRows += pathCount + outputTrails += boundaryVisits[boundary] * pathCount + } + cycleEdges, selfLoopEdges := int64(0), int64(0) + if config.AddProductiveBoundaryCycle { + cycleEdges = 2 + } + if config.AddProductiveBoundarySelfLoop { + selfLoopEdges = 1 + } + return &FixedSuffixExpansionFixtureExpectations{ + RootSourceRows: int64(len(roots)), + DistinctRoots: int64(len(roots)), + ForwardExpansionStates: forwardStates, + SuffixRows: suffixRows, + DistinctBoundaries: int64(len(suffixPaths)), + ReachableBoundaries: int64(len(boundaryVisits)), + DisconnectedBoundaries: int64(len(suffixPaths) - len(boundaryVisits)), + ExpectedReverseStates: reverseStates, + CompleteOutputTrails: outputTrails, + ProductiveBoundaryCycleEdges: cycleEdges, + ProductiveBoundarySelfLoopEdges: selfLoopEdges, + } +} + +// clearGraph removes relationships before nodes, using PostgreSQL partition truncation when available. func clearGraph(ctx context.Context, db graph.Database) error { + if pgDriver, isPostgres := db.(*pg.Driver); isPostgres { + graphTarget, hasDefaultGraph := pgDriver.DefaultGraph() + if !hasDefaultGraph { + return fmt.Errorf("PostgreSQL default graph is not set") + } + + return clearPostgresGraph(ctx, db, graphTarget.ID) + } + + return db.WriteTransaction(ctx, func(tx graph.Transaction) error { + if err := tx.Relationships().Delete(); err != nil { + return fmt.Errorf("delete relationships: %w", err) + } + + if err := tx.Nodes().Delete(); err != nil { + return fmt.Errorf("delete nodes: %w", err) + } + + return nil + }) +} + +// clearPostgresGraph truncates one PostgreSQL graph's edge and node partitions in a transaction. +func clearPostgresGraph(ctx context.Context, db graph.Database, graphID int32) error { return db.WriteTransaction(ctx, func(tx graph.Transaction) error { - return tx.Nodes().Delete() + // Truncate the active child partitions together. The high-level + // relationship query cannot see an already-orphaned edge, while DELETE + // leaves heap and index size dependent on earlier benchmark fixtures. + // Naming the children also avoids the node parent's cross-graph trigger. + statement := fmt.Sprintf("truncate table edge_%d, node_%d", graphID, graphID) + result := tx.Raw(statement, nil) + result.Close() + if err := result.Error(); err != nil { + return fmt.Errorf("execute PostgreSQL graph reset: %w", err) + } + + return nil }) } +// benchmarkSchema returns the SQL schema used to isolate benchmark preparation from timed execution. func benchmarkSchema(nodeKinds, edgeKinds graph.Kinds) graph.Schema { return graph.Schema{ Graphs: []graph.Graph{{ @@ -94,24 +735,102 @@ func benchmarkSchema(nodeKinds, edgeKinds graph.Kinds) graph.Schema { } } +// resolveCaseParams resolves a scale case's scalar, node-key, node-list, and generated-node parameters. func resolveCaseParams(testCase ScaleCase, idMap opengraph.IDMap) (map[string]any, error) { - params := make(map[string]any, len(testCase.Params)+len(testCase.NodeParams)) - for key, value := range testCase.Params { + return resolveParams(testCase.Name, testCase.Params, testCase.NodeParams, testCase.NodeListParams, testCase.GeneratedNodeListParams, idMap) +} + +// resolveParams copies literal parameters and replaces symbolic node keys with database identifiers. +func resolveParams(caseName string, rawParams map[string]any, nodeParams map[string]string, nodeListParams map[string][]string, generatedNodeListParams map[string]testutil.GeneratedNodeListParam, idMap opengraph.IDMap) (map[string]any, error) { + params := make(map[string]any, len(rawParams)+len(nodeParams)+len(nodeListParams)+len(generatedNodeListParams)) + for key, value := range rawParams { params[key] = value } - for paramName, nodeName := range testCase.NodeParams { + for paramName, nodeName := range nodeParams { id, found := idMap[nodeName] if !found { - return nil, fmt.Errorf("case %s references unknown dataset node %q", testCase.Name, nodeName) + return nil, fmt.Errorf("case %s references unknown dataset node %q", caseName, nodeName) } params[paramName] = id.Int64() } + for paramName, nodeNames := range nodeListParams { + ids := make([]int64, len(nodeNames)) + for idx, nodeName := range nodeNames { + id, found := idMap[nodeName] + if !found { + return nil, fmt.Errorf("case %s references unknown dataset node %q in list parameter %q", caseName, nodeName, paramName) + } + + ids[idx] = id.Int64() + } + + params[paramName] = ids + } + + for paramName, spec := range generatedNodeListParams { + if spec.Count < 0 { + return nil, fmt.Errorf("case %s generated node list parameter %q has negative count", caseName, paramName) + } + + nodeNames := append([]string(nil), spec.Include...) + nodeNames = append(nodeNames, testutil.FixtureNames(spec.Prefix, spec.Count)...) + ids := make([]int64, len(nodeNames)) + for idx, nodeName := range nodeNames { + id, found := idMap[nodeName] + if !found { + return nil, fmt.Errorf("case %s references unknown dataset node %q in generated list parameter %q", caseName, nodeName, paramName) + } + ids[idx] = id.Int64() + } + params[paramName] = ids + } + if len(params) == 0 { return nil, nil } return params, nil } + +// resolveWriteScenario resolves selection and post-state parameters while preserving the write expectation contract. +func resolveWriteScenario(testCase ScaleCase, idMap opengraph.IDMap) (resolvedWriteScenario, error) { + if testCase.WriteScenario == nil { + return resolvedWriteScenario{}, nil + } + + scenario := testCase.WriteScenario + if scenario.ExpectedMatched == nil || scenario.ExpectedAffected == nil { + return resolvedWriteScenario{}, fmt.Errorf("case %s has an incomplete write scenario", testCase.Name) + } + selectionParams, err := resolveParams(testCase.Name+" selection", scenario.Params, scenario.NodeParams, scenario.NodeListParams, scenario.GeneratedNodeListParams, idMap) + if err != nil { + return resolvedWriteScenario{}, err + } + + resolved := resolvedWriteScenario{ + SelectionCypher: scenario.SelectionCypher, + SelectionParams: selectionParams, + AffectedEntity: scenario.AffectedEntity, + ExpectedMatched: *scenario.ExpectedMatched, + ExpectedAffected: *scenario.ExpectedAffected, + } + + for _, postState := range scenario.PostState { + params, err := resolveParams(testCase.Name+" post-state "+postState.Name, postState.Params, postState.NodeParams, postState.NodeListParams, postState.GeneratedNodeListParams, idMap) + if err != nil { + return resolvedWriteScenario{}, err + } + + resolved.PostState = append(resolved.PostState, resolvedStateQuery{ + Name: postState.Name, + Cypher: postState.Cypher, + Params: params, + Expected: postState.Expected, + }) + } + + return resolved, nil +} diff --git a/cmd/graphbench/datasets_test.go b/cmd/graphbench/datasets_test.go new file mode 100644 index 00000000..141512ba --- /dev/null +++ b/cmd/graphbench/datasets_test.go @@ -0,0 +1,383 @@ +package main + +import ( + "context" + "errors" + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/testutil" + "github.com/stretchr/testify/require" +) + +// TestGeneratedFixedSuffixExpansionV2DatasetCarriesExactExpectations verifies that a canonical encoded name derives exact forward, reverse, boundary, suffix-row, and output-trail counts. +func TestGeneratedFixedSuffixExpansionV2DatasetCarriesExactExpectations(t *testing.T) { + name := "generated_fixed_suffix_expansion_v2_d16_f1000_r1_x1_i0_m2_z1_p0" + config, ok := parseFixedSuffixExpansionV2DatasetName(name) + require.True(t, ok) + require.Equal(t, 16, config.ExpansionDepth) + require.Equal(t, 1, *config.ExactReachableSuffixSources) + require.Equal(t, 2, config.SuffixPathsPerBoundary) + + metadata, err := fixtureMetadata("unused", name) + require.NoError(t, err) + require.NotNil(t, metadata.FixedSuffixExpansion) + require.Equal(t, int64(16_001), metadata.FixedSuffixExpansion.ForwardExpansionStates) + require.Equal(t, int64(6), metadata.FixedSuffixExpansion.SuffixRows) + require.Equal(t, int64(3), metadata.FixedSuffixExpansion.DistinctBoundaries) + require.Equal(t, int64(19), metadata.FixedSuffixExpansion.ExpectedReverseStates) + require.Equal(t, int64(4), metadata.FixedSuffixExpansion.CompleteOutputTrails) +} + +// TestGeneratedFixedSuffixExpansionV3DatasetRoundTripsAllBoundaryControls +// verifies independent root multiplicity and every canonical cycle/self-loop +// combination, including exact relationship-distinct state and output counts. +func TestGeneratedFixedSuffixExpansionV3DatasetRoundTripsAllBoundaryControls(t *testing.T) { + for _, testCase := range []struct { + name string + cycle bool + selfLoop bool + forwardStates int64 + reverseStates int64 + outputTrails int64 + }{ + {name: "neither", forwardStates: 5, reverseStates: 1, outputTrails: 1}, + {name: "cycle", cycle: true, forwardStates: 7, reverseStates: 3, outputTrails: 2}, + {name: "self-loop", selfLoop: true, forwardStates: 7, reverseStates: 2, outputTrails: 2}, + {name: "both", cycle: true, selfLoop: true, forwardStates: 10, reverseStates: 5, outputTrails: 3}, + } { + t.Run(testCase.name, func(t *testing.T) { + reachable := 0 + zeroDepth := true + config := testutil.FixedSuffixExpansionScaleConfig{ + ExpansionDepth: 2, + Fanout: 1, + ExactReachableSuffixSources: &reachable, + SuffixPathsPerBoundary: 1, + RootMatchCount: 3, + RootHasZeroDepthSuffix: &zeroDepth, + AddProductiveBoundaryCycle: testCase.cycle, + AddProductiveBoundarySelfLoop: testCase.selfLoop, + } + name := fixedSuffixExpansionV3DatasetName(config) + parsed, ok := parseFixedSuffixExpansionV3DatasetName(name) + require.True(t, ok) + require.Equal(t, config, parsed) + + metadata, err := fixtureMetadata("unused", name) + require.NoError(t, err) + require.NotNil(t, metadata.FixedSuffixExpansion) + require.Equal(t, int64(3), metadata.FixedSuffixExpansion.RootSourceRows) + require.Equal(t, testCase.forwardStates, metadata.FixedSuffixExpansion.ForwardExpansionStates) + require.Equal(t, testCase.reverseStates, metadata.FixedSuffixExpansion.ExpectedReverseStates) + require.Equal(t, testCase.outputTrails, metadata.FixedSuffixExpansion.CompleteOutputTrails) + if testCase.cycle { + require.Equal(t, int64(2), metadata.FixedSuffixExpansion.ProductiveBoundaryCycleEdges) + } else { + require.Zero(t, metadata.FixedSuffixExpansion.ProductiveBoundaryCycleEdges) + } + if testCase.selfLoop { + require.Equal(t, int64(1), metadata.FixedSuffixExpansion.ProductiveBoundarySelfLoopEdges) + } else { + require.Zero(t, metadata.FixedSuffixExpansion.ProductiveBoundarySelfLoopEdges) + } + }) + } +} + +// TestGeneratedEndpointSeededExpansionDatasetRoundTripsWithExactExpectations verifies lossless name encoding and the expected endpoint, prefix, output, and reverse-search cardinalities. +func TestGeneratedEndpointSeededExpansionDatasetRoundTripsWithExactExpectations(t *testing.T) { + config := testutil.EndpointSeededExpansionScaleConfig{ + Depth: 3, MatchingEndpoints: 2, OtherEndpoints: 1, + MatchingEligibleLanes: 2, OtherEligibleLanes: 1, MatchingIneligibleLanes: 1, + ParallelEdges: 1, AddCycle: false, PropertyPayloadSize: 8, + } + name := endpointSeededExpansionDatasetName(config) + parsed, ok := parseEndpointSeededExpansionDatasetName(name) + require.True(t, ok) + require.Equal(t, config, parsed) + metadata, err := fixtureMetadata("unused", name) + require.NoError(t, err) + require.NotNil(t, metadata.EndpointSeededExpansion) + require.Equal(t, int64(2), metadata.EndpointSeededExpansion.MatchingEndpoints) + require.Equal(t, int64(3), metadata.EndpointSeededExpansion.EligiblePrefixRows) + require.Equal(t, int64(2), metadata.EndpointSeededExpansion.ExpectedOutputTrails) + require.Greater(t, metadata.EndpointSeededExpansion.ExpectedReverseStates, metadata.EndpointSeededExpansion.ExpectedOutputTrails) +} + +// TestGeneratedEndpointSeededExpansionRejectsInvalidNames verifies that zero dimensions, padded numbers, invalid booleans, and inconsistent parallelism cannot select a generated fixture. +func TestGeneratedEndpointSeededExpansionRejectsInvalidNames(t *testing.T) { + for _, name := range []string{ + "generated_endpoint_seeded_expansion_v1_d0_e1_q0_w1_o0_x0_m1_c0_p0", + "generated_endpoint_seeded_expansion_v1_d3_e0_q0_w1_o0_x0_m1_c0_p0", + "generated_endpoint_seeded_expansion_v1_d3_e1_q0_w1_o0_x0_m0_c0_p0", + "generated_endpoint_seeded_expansion_v1_d03_e1_q0_w1_o0_x0_m1_c0_p0", + "generated_endpoint_seeded_expansion_v1_d3_e1_q0_w1_o0_x0_m1_c2_p0", + "generated_endpoint_seeded_expansion_v1_d3_e1_q0_w1_o0_x0_m2_c0_p0", + } { + _, ok := parseEndpointSeededExpansionDatasetName(name) + require.False(t, ok, name) + require.Nil(t, generatedDataset(name), name) + } +} + +// TestGeneratedShortestPathV2DatasetRoundTripsAndCarriesExactExpectations verifies lossless configuration naming and exact topology metrics for branching, parallel, disconnected, cyclic, and self-loop shapes. +func TestGeneratedShortestPathV2DatasetRoundTripsAndCarriesExactExpectations(t *testing.T) { + config := testutil.ShortestPathScaleV2Config{ + Depth: 3, + ForwardRootFanOut: 2, + ReverseRootFanIn: 2, + IntermediateFanOut: 1, + IntermediateReverseFanIn: 4, + FanInLevel: 2, + ParallelKindCount: 3, + ParallelTargetCount: 2, + DiamondWidth: 2, + DisconnectedWidth: 3, + PropertyPayloadSize: 8, + AddCycle: true, + AddSelfLoop: true, + } + name := shortestPathV2DatasetName(config) + parsed, ok := parseShortestPathV2DatasetName(name) + require.True(t, ok) + require.Equal(t, config, parsed) + + metadata, err := fixtureMetadata("unused", name) + require.NoError(t, err) + require.NotNil(t, metadata.Shortest) + require.Equal(t, 32, metadata.NodeCount) + require.Equal(t, 33, metadata.EdgeCount) + require.Equal(t, int64(5), metadata.Shortest.RootForwardDegree) + require.Equal(t, int64(3), metadata.Shortest.RootReverseDegree) + require.Equal(t, int64(2), metadata.Shortest.MaximumIntermediateForwardByLevel["2"]) + require.Equal(t, int64(5), metadata.Shortest.MaximumIntermediateReverseByLevel["2"]) + require.Equal(t, int64(23), metadata.Shortest.PhysicalTraversableEdgesByKind["Traverse"]) + require.Equal(t, int64(6), metadata.Shortest.ParallelPhysicalEdges) + require.Equal(t, int64(2), metadata.Shortest.ParallelDistinctTargets) + require.Equal(t, int64(3), metadata.Shortest.ExpectedMinimumDistance) + require.Equal(t, int64(3), metadata.Shortest.ExpectedPredecessorEdges) + require.Equal(t, int64(4), metadata.Shortest.DisconnectedStateCardinality) + require.Equal(t, int64(5), metadata.Shortest.DistinctReachableNodesByLevel["1"]) + require.NotEmpty(t, metadata.Checksum) +} + +// TestGeneratedShortestPathV2DatasetRejectsInvalidOrNonCanonicalNames verifies that inconsistent levels, empty required dimensions, padded or negative values, invalid booleans, and trailing tokens are rejected. +func TestGeneratedShortestPathV2DatasetRejectsInvalidOrNonCanonicalNames(t *testing.T) { + for _, name := range []string{ + "generated_shortest_paths_v2_d3_o2_r2_fo1_fi4_l3_k3_t2_w2_x3_p8_c1_s1", + "generated_shortest_paths_v2_d3_o2_r2_fo1_fi4_l2_k3_t0_w2_x3_p8_c1_s1", + "generated_shortest_paths_v2_d03_o2_r2_fo1_fi4_l2_k3_t2_w2_x3_p8_c1_s1", + "generated_shortest_paths_v2_d3_o2_r2_fo1_fi4_l2_k3_t2_w2_x3_p8_c2_s1", + "generated_shortest_paths_v2_d3_o2_r2_fo1_fi4_l2_k3_t2_w2_x3_p8_c1_s1_unknown", + "generated_shortest_paths_v2_d-1_o2_r2_fo1_fi4_l2_k3_t2_w2_x3_p8_c1_s1", + } { + _, ok := parseShortestPathV2DatasetName(name) + require.False(t, ok, name) + require.Nil(t, generatedDataset(name), name) + } +} + +// TestGeneratedFixedSuffixExpansionV2DatasetRejectsInvalidOrNonCanonicalNames verifies that impossible reachability, zero multiplicity, invalid booleans, and padded dimensions cannot identify a fixture. +func TestGeneratedFixedSuffixExpansionV2DatasetRejectsInvalidOrNonCanonicalNames(t *testing.T) { + for _, name := range []string{ + "generated_fixed_suffix_expansion_v2_d16_f1000_r1001_x1_i0_m1_z1_p0", + "generated_fixed_suffix_expansion_v2_d16_f1000_r1_x1_i0_m0_z1_p0", + "generated_fixed_suffix_expansion_v2_d16_f1000_r1_x1_i0_m1_z2_p0", + "generated_fixed_suffix_expansion_v2_d016_f1000_r1_x1_i0_m1_z1_p0", + } { + _, ok := parseFixedSuffixExpansionV2DatasetName(name) + require.False(t, ok, name) + require.Nil(t, generatedDataset(name), name) + } +} + +// TestGeneratedFixedSuffixExpansionV3DatasetRejectsInvalidOrNonCanonicalNames +// verifies strict roots, booleans, productive-boundary requirements, exact +// depth-zero reachability, canonical numbers, and complete token consumption. +func TestGeneratedFixedSuffixExpansionV3DatasetRejectsInvalidOrNonCanonicalNames(t *testing.T) { + for _, name := range []string{ + "generated_fixed_suffix_expansion_v3_d2_f1_r0_x0_i0_m1_q0_z1_c0_s0_p0", + "generated_fixed_suffix_expansion_v3_d2_f1_r0_x0_i0_m1_q1_z1_c2_s0_p0", + "generated_fixed_suffix_expansion_v3_d2_f1_r0_x0_i0_m1_q1_z1_c0_s2_p0", + "generated_fixed_suffix_expansion_v3_d2_f1_r0_x0_i0_m1_q1_z0_c1_s0_p0", + "generated_fixed_suffix_expansion_v3_d2_f1_r0_x0_i1_m1_q1_z0_c0_s0_p0", + "generated_fixed_suffix_expansion_v3_d0_f1_r1_x0_i0_m1_q1_z0_c0_s0_p0", + "generated_fixed_suffix_expansion_v3_d02_f1_r0_x0_i0_m1_q1_z1_c0_s0_p0", + "generated_fixed_suffix_expansion_v3_d2_f1_r0_x0_i0_m1_q1_z1_c0_s0_p0_unknown", + } { + _, ok := parseFixedSuffixExpansionV3DatasetName(name) + require.False(t, ok, name) + require.Nil(t, generatedDataset(name), name) + } +} + +// TestClearGraphDeletesRelationshipsBeforeNodes verifies that cleanup removes relationships before nodes so attached edges cannot block node deletion. +func TestClearGraphDeletesRelationshipsBeforeNodes(t *testing.T) { + database := &clearGraphTestDatabase{} + + require.NoError(t, clearGraph(context.Background(), database)) + require.Equal(t, []string{"relationships", "nodes"}, database.deletes) +} + +// TestClearGraphStopsWhenRelationshipDeleteFails verifies that a relationship deletion error is wrapped and prevents the subsequent node deletion. +func TestClearGraphStopsWhenRelationshipDeleteFails(t *testing.T) { + database := &clearGraphTestDatabase{relationshipError: errors.New("relationship failure")} + + err := clearGraph(context.Background(), database) + require.ErrorContains(t, err, "delete relationships: relationship failure") + require.Equal(t, []string{"relationships"}, database.deletes) +} + +// TestClearGraphReportsNodeDeleteFailure verifies that cleanup reports a node deletion failure only after relationships have been removed. +func TestClearGraphReportsNodeDeleteFailure(t *testing.T) { + database := &clearGraphTestDatabase{nodeError: errors.New("node failure")} + + err := clearGraph(context.Background(), database) + require.ErrorContains(t, err, "delete nodes: node failure") + require.Equal(t, []string{"relationships", "nodes"}, database.deletes) +} + +// TestClearPostgresGraphTruncatesPhysicalPartitionsTogether verifies that PostgreSQL cleanup issues one parameter-free TRUNCATE for both graph-specific physical tables. +func TestClearPostgresGraphTruncatesPhysicalPartitionsTogether(t *testing.T) { + database := &clearPostgresGraphTestDatabase{} + + require.NoError(t, clearPostgresGraph(context.Background(), database, 42)) + require.Equal(t, []string{"truncate table edge_42, node_42"}, database.statements) + require.Equal(t, []map[string]any{nil}, database.parameters) +} + +// TestClearPostgresGraphRollsBackAfterRawDeleteFailure verifies that a failed physical-table reset is surfaced from the enclosing write transaction. +func TestClearPostgresGraphRollsBackAfterRawDeleteFailure(t *testing.T) { + database := &clearPostgresGraphTestDatabase{failAt: 1} + + err := clearPostgresGraph(context.Background(), database, 42) + require.ErrorContains(t, err, "execute PostgreSQL graph reset") + require.Equal(t, []string{"truncate table edge_42, node_42"}, database.statements) +} + +// clearGraphTestDatabase supplies a fake transaction for graph-cleanup tests. +type clearGraphTestDatabase struct { + // Database supplies methods irrelevant to the cleanup interaction under test. + graph.Database + + // deletes records whether relationship or node deletion was requested first. + deletes []string + + // relationshipError is returned when cleanup attempts relationship deletion. + relationshipError error + + // nodeError is returned when cleanup attempts node deletion. + nodeError error +} + +// WriteTransaction routes cleanup through a transaction that shares the deletion trace and injected failures. +func (s *clearGraphTestDatabase) WriteTransaction(_ context.Context, delegate graph.TransactionDelegate, _ ...graph.TransactionOption) error { + return delegate(&clearGraphTestTransaction{database: s}) +} + +// clearGraphTestTransaction routes relationship and node queries to graph-cleanup fakes. +type clearGraphTestTransaction struct { + // Transaction supplies methods outside the cleanup query surface. + graph.Transaction + + // database owns the call trace and injected deletion failures. + database *clearGraphTestDatabase +} + +// Relationships returns a deletion recorder backed by the owning database trace. +func (s *clearGraphTestTransaction) Relationships() graph.RelationshipQuery { + return &clearGraphTestRelationshipQuery{database: s.database} +} + +// Nodes returns a deletion recorder backed by the owning database trace. +func (s *clearGraphTestTransaction) Nodes() graph.NodeQuery { + return &clearGraphTestNodeQuery{database: s.database} +} + +// clearGraphTestRelationshipQuery records relationship deletion and injects configured failures. +type clearGraphTestRelationshipQuery struct { + // RelationshipQuery supplies methods other than the deletion operation under test. + graph.RelationshipQuery + + // database receives the relationship deletion trace and supplies its error. + database *clearGraphTestDatabase +} + +// Delete records the deletion request and returns the configured failure. +func (s *clearGraphTestRelationshipQuery) Delete() error { + s.database.deletes = append(s.database.deletes, "relationships") + return s.database.relationshipError +} + +// clearGraphTestNodeQuery records node deletion and injects configured failures. +type clearGraphTestNodeQuery struct { + // NodeQuery supplies methods other than the deletion operation under test. + graph.NodeQuery + + // database receives the node deletion trace and supplies its error. + database *clearGraphTestDatabase +} + +// Delete records the deletion request and returns the configured failure. +func (s *clearGraphTestNodeQuery) Delete() error { + s.database.deletes = append(s.database.deletes, "nodes") + return s.database.nodeError +} + +// clearPostgresGraphTestDatabase supplies a fake raw transaction for PostgreSQL cleanup tests. +type clearPostgresGraphTestDatabase struct { + // Database supplies methods irrelevant to the raw cleanup interaction. + graph.Database + + // statements records every raw SQL statement issued by cleanup. + statements []string + + // parameters records the bind arguments paired with each captured statement. + parameters []map[string]any + + // failAt selects the one-based raw call that returns a terminal result error. + failAt int +} + +// WriteTransaction routes PostgreSQL cleanup through a raw-SQL recorder owned by the fake database. +func (s *clearPostgresGraphTestDatabase) WriteTransaction(_ context.Context, delegate graph.TransactionDelegate, _ ...graph.TransactionOption) error { + return delegate(&clearPostgresGraphTestTransaction{database: s}) +} + +// clearPostgresGraphTestTransaction records PostgreSQL cleanup SQL and returns a configured result. +type clearPostgresGraphTestTransaction struct { + // Transaction supplies methods outside the raw SQL cleanup surface. + graph.Transaction + + // database owns the captured statements, parameters, and failure injection. + database *clearPostgresGraphTestDatabase +} + +// Raw captures one statement and its parameters, injecting a terminal error on the selected call. +func (s *clearPostgresGraphTestTransaction) Raw(statement string, parameters map[string]any) graph.Result { + s.database.statements = append(s.database.statements, statement) + s.database.parameters = append(s.database.parameters, parameters) + if s.database.failAt > 0 && len(s.database.statements) == s.database.failAt { + return &clearPostgresGraphTestResult{err: errors.New("raw delete failure")} + } + + return &clearPostgresGraphTestResult{} +} + +// clearPostgresGraphTestResult exposes a configured terminal raw-statement failure to cleanup code. +type clearPostgresGraphTestResult struct { + // Result supplies result methods that cleanup does not exercise. + graph.Result + + // err is exposed as the terminal raw-statement failure. + err error +} + +// Error returns the configured terminal iterator error. +func (s *clearPostgresGraphTestResult) Error() error { + return s.err +} + +// Close satisfies graph.Result; this fake has no close state to record. +func (s *clearPostgresGraphTestResult) Close() {} diff --git a/cmd/graphbench/destructive_guard_test.go b/cmd/graphbench/destructive_guard_test.go new file mode 100644 index 00000000..23285f53 --- /dev/null +++ b/cmd/graphbench/destructive_guard_test.go @@ -0,0 +1,26 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "testing" + + "github.com/specterops/dawgs/databaseguard" + "github.com/stretchr/testify/require" +) + +// TestDestructiveRunnersRequireTargetAuthorization verifies that neither PostgreSQL nor Neo4j runners can initialize against an unapproved destructive target. +func TestDestructiveRunnersRequireTargetAuthorization(t *testing.T) { + t.Setenv(databaseguard.AllowDestructiveEnv, "") + t.Setenv(databaseguard.DisposableTargetsEnv, "") + + _, err := newPostgresSQLRunner(context.Background(), "", "postgresql://user:secret@localhost/dawgs", ScaleCorpus{}, 1, 1, nil, false, nil, "", "") + require.ErrorContains(t, err, "refuse destructive PostgreSQL") + + _, err = newNeo4jRunner(context.Background(), "", "neo4j://user:secret@localhost", ScaleCorpus{}) + require.ErrorContains(t, err, "refuse destructive Neo4j") +} diff --git a/cmd/graphbench/dormant_forms_guard_test.go b/cmd/graphbench/dormant_forms_guard_test.go new file mode 100644 index 00000000..63482cc2 --- /dev/null +++ b/cmd/graphbench/dormant_forms_guard_test.go @@ -0,0 +1,44 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestDormantFormsStayOutOfScaleCorpus verifies that active scale-case names and tags never publish FUTURE-prefixed query forms. +func TestDormantFormsStayOutOfScaleCorpus(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + + for _, testCase := range corpus.Cases { + requireNoDormantQueryFormID(t, testCase.Source+" name", testCase.Name) + for _, tag := range testCase.Tags { + requireNoDormantQueryFormID(t, testCase.Source+" tag", tag) + } + } +} + +// requireNoDormantQueryFormID rejects a case field containing the reserved FUTURE marker, independent of letter case. +func requireNoDormantQueryFormID(t *testing.T, field, value string) { + t.Helper() + require.False(t, strings.Contains(strings.ToUpper(value), "FUTURE-"), + "%s %q places a dormant query form in the active scale corpus", field, value) +} diff --git a/cmd/graphbench/environment.go b/cmd/graphbench/environment.go new file mode 100644 index 00000000..7bf48a87 --- /dev/null +++ b/cmd/graphbench/environment.go @@ -0,0 +1,338 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strings" + "time" +) + +// RunEnvironment captures source, host, invocation, fixture, and protocol identity for a benchmark run. +type RunEnvironment struct { + // ArtifactSchemaVersion identifies the benchmark artifact schema emitted by the run. + ArtifactSchemaVersion int `json:"artifact_schema_version"` + // CorpusSHA256 binds run provenance to the exact canonical workload declarations. + CorpusSHA256 string `json:"corpus_sha256,omitempty"` + // RunIdentitySHA256 binds resumable records to execution settings that affect comparability. + RunIdentitySHA256 string `json:"run_identity_sha256,omitempty"` + // SourceCommit identifies the source commit used to build the benchmark executable. + SourceCommit string `json:"source_commit"` + // DirtyDiffSHA256 identifies uncommitted source changes present during the run. + DirtyDiffSHA256 string `json:"dirty_diff_sha256"` + // BinarySHA256 identifies the benchmark executable used for the run. + BinarySHA256 string `json:"binary_sha256"` + // GOOS records the target operating system of the benchmark executable. + GOOS string `json:"goos"` + // GOARCH records the target architecture of the benchmark executable. + GOARCH string `json:"goarch"` + // GoVersion records the Go toolchain version used to build the executable. + GoVersion string `json:"go_version"` + // CPUCount records logical CPUs visible to the benchmark process. + CPUCount int `json:"cpu_count"` + // CPUModel records the host processor model for reproducibility. + CPUModel string `json:"cpu_model,omitempty"` + // Kernel records the host kernel release for reproducibility. + Kernel string `json:"kernel,omitempty"` + // CgroupCPU records the process cgroup CPU allocation context. + CgroupCPU string `json:"cgroup_cpu,omitempty"` + // CgroupMemory records the process cgroup memory limit and usage context. + CgroupMemory string `json:"cgroup_memory,omitempty"` + // CPUGovernor records the active CPU frequency governor. + CPUGovernor string `json:"cpu_governor,omitempty"` + // CPUFrequency records the observed CPU frequency policy. + CPUFrequency string `json:"cpu_frequency,omitempty"` + // HostLoad records host load averages observed during the run. + HostLoad string `json:"host_load,omitempty"` + // Invocation records the sanitized command invocation used for the run. + Invocation []string `json:"invocation"` + // BuildCommand records the reproducible command used to build the benchmark executable. + BuildCommand string `json:"build_command"` + // RunUUID groups records produced by the same resumable benchmark run series. + RunUUID string `json:"run_uuid"` + // Arm identifies the measurement arm that produced the sample. + Arm string `json:"arm"` + // ArmOrder records the arm's position within its balanced measurement block. + ArmOrder int `json:"arm_order,omitempty"` + // Block identifies the measurement block used to control carryover effects. + Block int `json:"block"` + // Round identifies the measurement round. + Round int `json:"round"` + // StartedAt records when the benchmark run began. + StartedAt time.Time `json:"started_at"` + // EndedAt records when the benchmark run finished. + EndedAt time.Time `json:"ended_at"` + // WarmupIterations records the untimed iterations run before measurement. + WarmupIterations int `json:"warmup_iterations"` + // Selection captures the exact workload selection applied to the run. + Selection *SelectionManifest `json:"selection,omitempty"` + // PoolSize sets the database connection-pool size. + PoolSize int `json:"pool_size"` + // Concurrency records the worker counts exercised during the run. + Concurrency []int `json:"concurrency,omitempty"` + // SessionMemoryCeilingBytes sets the per-session memory ceiling in bytes. + SessionMemoryCeilingBytes int64 `json:"session_memory_ceiling_bytes,omitempty"` + // PoolMemoryCeilingBytes sets the aggregate pool memory ceiling in bytes. + PoolMemoryCeilingBytes int64 `json:"pool_memory_ceiling_bytes,omitempty"` + // ExistingGraph selects read-only execution against a pre-existing graph. + ExistingGraph bool `json:"existing_graph,omitempty"` + // Protocol identifies the measurement protocol. + Protocol string `json:"protocol,omitempty"` +} + +// PostgresEnvironment captures PostgreSQL settings, relation sizes, and schema fingerprints required for comparability. +type PostgresEnvironment struct { + // Version identifies the serialized schema revision. + Version string `json:"version"` + // Database names the PostgreSQL database whose settings and schema were captured. + Database string `json:"database"` + // PlanCacheMode records PostgreSQL plan_cache_mode for environment comparability. + PlanCacheMode string `json:"plan_cache_mode"` + // TransactionIsolation records the isolation applied to measured read + // transactions. Tool and provisional guarded orientation evidence uses + // Repeatable Read even when the server default differs. + TransactionIsolation string `json:"transaction_isolation"` + // WorkMem records PostgreSQL work_mem for environment comparability. + WorkMem string `json:"work_mem"` + // TempFileLimit records the configured PostgreSQL temporary-file ceiling. + TempFileLimit string `json:"temp_file_limit"` + // GraphPartitionCount records physical PostgreSQL graph partitions included in relation-size evidence. + GraphPartitionCount int64 `json:"graph_partition_count"` + // PostmasterStartedAt records PostgreSQL server start time for restart detection. + PostmasterStartedAt time.Time `json:"postmaster_started_at,omitempty"` + // DatabaseOID identifies the PostgreSQL database across environment and restart comparisons. + DatabaseOID int64 `json:"database_oid,omitempty"` + // Autovacuum records PostgreSQL autovacuum settings relevant to comparability. + Autovacuum string `json:"autovacuum,omitempty"` + // NodeRelationBytes records the physical size of the graph's node relation. + NodeRelationBytes int64 `json:"node_relation_bytes,omitempty"` + // EdgeRelationBytes records the physical size of the graph's relationship relation. + EdgeRelationBytes int64 `json:"edge_relation_bytes,omitempty"` + // AnalyzeState records PostgreSQL analyze statistics state for the fixture. + AnalyzeState string `json:"analyze_state,omitempty"` + // SchemaFingerprint identifies the normalized PostgreSQL graph schema definition. + SchemaFingerprint string `json:"schema_fingerprint,omitempty"` + // IndexFingerprint identifies the normalized database index configuration. + IndexFingerprint string `json:"index_fingerprint,omitempty"` +} + +// resolveRunEnvironment captures reproducibility metadata, invocation, fixture selection, and run timestamps. +func resolveRunEnvironment(cfg config, args []string, selection SelectionManifest, startedAt, endedAt time.Time) RunEnvironment { + runUUID := cfg.RunUUID + if runUUID == "" { + runUUID = newRunUUID() + } + return RunEnvironment{ + ArtifactSchemaVersion: 2, + SourceCommit: commandOutput("git", "rev-parse", "HEAD"), + DirtyDiffSHA256: workingTreeSHA256(), + BinarySHA256: executableSHA256(), + GOOS: runtime.GOOS, + GOARCH: runtime.GOARCH, + GoVersion: runtime.Version(), + CPUCount: runtime.NumCPU(), + CPUModel: cpuModel(), + Kernel: commandOutput("uname", "-srvm"), + CgroupCPU: firstReadableFile("/sys/fs/cgroup/cpu.max", "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"), + CgroupMemory: firstReadableFile("/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"), + CPUGovernor: firstReadableFile("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor"), + CPUFrequency: firstReadableFile("/sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq"), + HostLoad: firstReadableFile("/proc/loadavg"), + Invocation: sanitizedInvocation(args), + BuildCommand: cfg.BuildCommand, + RunUUID: runUUID, + Arm: cfg.Arm, + ArmOrder: cfg.ArmOrder, + Block: cfg.Block, + Round: cfg.Round, + StartedAt: startedAt.UTC(), + EndedAt: endedAt.UTC(), + WarmupIterations: cfg.WarmupIterations, + Selection: &selection, + PoolSize: cfg.PoolSize, + Concurrency: append([]int(nil), cfg.Concurrency...), + SessionMemoryCeilingBytes: cfg.SessionMemoryCeilingBytes, + PoolMemoryCeilingBytes: cfg.PoolMemoryCeilingBytes, + ExistingGraph: cfg.ExistingGraph, + Protocol: benchmarkProtocol(cfg), + } +} + +// benchmarkProtocol returns the stable name of the measurement protocol selected by the command. +func benchmarkProtocol(cfg config) string { + if cfg.Discovery { + return "adaptive_discovery" + } + return "fixed_confirmation" +} + +// newRunUUID generates a random RFC 4122 version 4 run identifier. +func newRunUUID() string { + var value [16]byte + if _, err := rand.Read(value[:]); err != nil { + return fmt.Sprintf("fallback-%d", time.Now().UnixNano()) + } + value[6] = (value[6] & 0x0f) | 0x40 + value[8] = (value[8] & 0x3f) | 0x80 + return fmt.Sprintf("%x-%x-%x-%x-%x", value[0:4], value[4:6], value[6:8], value[8:10], value[10:16]) +} + +// cpuModel returns the host CPU model reported by the operating system. +func cpuModel() string { + raw, err := os.ReadFile("/proc/cpuinfo") + if err != nil { + return "unknown" + } + for _, line := range strings.Split(string(raw), "\n") { + if name, value, found := strings.Cut(line, ":"); found && strings.TrimSpace(name) == "model name" { + return strings.TrimSpace(value) + } + } + return "unknown" +} + +// firstReadableFile returns trimmed contents of the first readable path. +func firstReadableFile(paths ...string) string { + for _, path := range paths { + if raw, err := os.ReadFile(path); err == nil { + return strings.TrimSpace(string(raw)) + } + } + return "unknown" +} + +// sanitizedInvocation returns command arguments with connection-string credentials redacted. +func sanitizedInvocation(args []string) []string { + const redacted = "" + connectionFlags := []string{"-connection", "-pg-connection", "-neo4j-connection"} + result := append([]string(nil), args...) + for idx := range result { + for _, name := range connectionFlags { + if result[idx] == name && idx+1 < len(result) { + result[idx+1] = redacted + break + } + if strings.HasPrefix(result[idx], name+"=") { + result[idx] = name + "=" + redacted + break + } + } + } + return result +} + +// commandOutput runs a provenance command and returns its trimmed standard output. +func commandOutput(name string, args ...string) string { + output, err := exec.Command(name, args...).Output() + if err != nil { + return "unknown" + } + return strings.TrimSpace(string(output)) +} + +// workingTreeSHA256 hashes the tracked Git diff together with sorted untracked paths and contents. +func workingTreeSHA256() string { + fingerprint, err := calculateWorkingTreeSHA256("") + if err != nil { + return "unknown" + } + return fingerprint +} + +func calculateWorkingTreeSHA256(excludedRoot string) (string, error) { + digest := sha256.New() + output, err := exec.Command("git", "diff", "--binary", "HEAD", "--").Output() + if err != nil { + return "", fmt.Errorf("capture tracked source diff: %w", err) + } + writeWorkingTreePatchFingerprint(digest, output) + paths, err := gitUntrackedPaths() + if err != nil { + return "", err + } + excludedAbsolute := "" + if excludedRoot != "" { + excludedAbsolute, err = filepath.Abs(excludedRoot) + if err != nil { + return "", fmt.Errorf("resolve excluded source root: %w", err) + } + } + for _, path := range paths { + if excludedAbsolute != "" { + absolute, err := filepath.Abs(path) + if err != nil { + return "", fmt.Errorf("resolve untracked source %q: %w", path, err) + } + if absolute == excludedAbsolute || strings.HasPrefix(absolute, excludedAbsolute+string(filepath.Separator)) { + continue + } + } + content, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("read untracked source %q: %w", path, err) + } + writeWorkingTreeUntrackedFingerprint(digest, filepath.ToSlash(path), content) + } + return hex.EncodeToString(digest.Sum(nil)), nil +} + +func gitUntrackedPaths() ([]string, error) { + output, err := exec.Command("git", "ls-files", "-z", "--others", "--exclude-standard").Output() + if err != nil { + return nil, fmt.Errorf("list untracked source: %w", err) + } + paths := parseNULTerminatedPaths(output) + sort.Strings(paths) + return paths, nil +} + +func parseNULTerminatedPaths(output []byte) []string { + fields := strings.Split(string(output), "\x00") + paths := make([]string, 0, len(fields)) + for _, path := range fields { + if path != "" { + paths = append(paths, path) + } + } + return paths +} + +func writeWorkingTreePatchFingerprint(digest io.Writer, patch []byte) { + _, _ = digest.Write(patch) +} + +func writeWorkingTreeUntrackedFingerprint(digest io.Writer, path string, content []byte) { + _, _ = fmt.Fprintf(digest, "untracked:%s\x00", path) + _, _ = digest.Write(content) +} + +// executableSHA256 returns the SHA-256 digest of the running benchmark executable. +func executableSHA256() string { + path, err := os.Executable() + if err != nil { + return "unknown" + } + checksum, err := fileSHA256(path) + if err != nil { + return "unknown" + } + return checksum +} + +// sqlFingerprint returns the SHA-256 digest of the supplied SQL text exactly as provided. +func sqlFingerprint(sql string) string { + digest := sha256.Sum256([]byte(sql)) + return hex.EncodeToString(digest[:]) +} diff --git a/cmd/graphbench/environment_test.go b/cmd/graphbench/environment_test.go new file mode 100644 index 00000000..35624c9e --- /dev/null +++ b/cmd/graphbench/environment_test.go @@ -0,0 +1,39 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestSQLFingerprintIsStableAndContentSensitive verifies that identical SQL yields a repeatable 256-bit digest while a query change alters that digest. +func TestSQLFingerprintIsStableAndContentSensitive(t *testing.T) { + require.Equal(t, sqlFingerprint("select 1"), sqlFingerprint("select 1")) + require.NotEqual(t, sqlFingerprint("select 1"), sqlFingerprint("select 2")) + require.Len(t, sqlFingerprint("select 1"), 64) +} + +// TestSanitizedInvocationRedactsConnectionStrings verifies redaction for split and inline connection flags while preserving unrelated arguments and the caller's input slice. +func TestSanitizedInvocationRedactsConnectionStrings(t *testing.T) { + args := []string{ + "graphbench", + "-connection", "postgres://user:secret@host/database", + "-pg-connection=postgres://user:secret@host/database", + "-neo4j-connection", "neo4j://user:secret@host", + "-iterations", "30", + } + + require.Equal(t, []string{ + "graphbench", + "-connection", "", + "-pg-connection=", + "-neo4j-connection", "", + "-iterations", "30", + }, sanitizedInvocation(args)) + require.Contains(t, args[2], "secret", "the caller's argument slice must not be mutated") +} diff --git a/cmd/graphbench/expand_into_report.go b/cmd/graphbench/expand_into_report.go new file mode 100644 index 00000000..2235579f --- /dev/null +++ b/cmd/graphbench/expand_into_report.go @@ -0,0 +1,498 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "os" + "sort" + "strings" + "time" +) + +const expandIntoStudyReportVersion = 2 + +var expandIntoStudyArms = []string{ + "expand_into_pair_join", + "expand_into_lower_degree_scan", + "expand_into_pair_cache", +} + +// ExpandIntoStudyOptions selects the discovery or confirmation evidence protocol. +type ExpandIntoStudyOptions struct { + Seed int64 + Confidence float64 + BootstrapCount int + Protocol string + MaterialityRatio float64 + MaterialityAbsolute time.Duration + P95RatioLimit float64 +} + +// ExpandIntoStudyReport contains exact three-arm fixed-one-hop evidence. +type ExpandIntoStudyReport struct { + Version int `json:"version"` + ArtifactSHA256 string `json:"artifact_sha256"` + Protocol string `json:"protocol"` + Confidence float64 `json:"confidence_level"` + Passed bool `json:"passed"` + TrainingCases int `json:"training_cases"` + HoldoutCases int `json:"holdout_cases"` + TrainingPassed bool `json:"training_passed"` + HoldoutPassed bool `json:"holdout_passed"` + QualificationPassed bool `json:"qualification_passed"` + PromotionEligible bool `json:"promotion_eligible"` + Winner string `json:"winner,omitempty"` + Cases []ExpandIntoStudyCase `json:"cases"` +} + +// ExpandIntoStudyCase reports exactness, order balance, plan shape, and latency for one pair workload. +type ExpandIntoStudyCase struct { + Dataset string `json:"dataset"` + Name string `json:"name"` + Tier string `json:"tier,omitempty"` + QualificationSplit string `json:"qualification_split"` + Rounds int `json:"rounds"` + Winner string `json:"descriptive_median_winner,omitempty"` + QualifiedWinner string `json:"qualified_winner,omitempty"` + Passed bool `json:"passed"` + Reasons []string `json:"reasons,omitempty"` + ArmResults []ExpandIntoStudyArmEvidence `json:"arms"` +} + +// ExpandIntoStudyArmEvidence records one exact plan-study arm and its ratio to the direct pair join. +type ExpandIntoStudyArmEvidence struct { + Name string `json:"name"` + Architecture string `json:"architecture"` + ImplementationID string `json:"implementation_id"` + SQLFingerprint string `json:"sql_fingerprint"` + Samples int `json:"samples"` + Median time.Duration `json:"median"` + P95 time.Duration `json:"p95"` + MedianRatioToDirect *RatioInterval `json:"median_ratio_to_direct,omitempty"` + MedianSavingToDirect *DurationInterval `json:"median_saving_to_direct,omitempty"` + P95RatioToDirect *RatioInterval `json:"p95_ratio_to_direct,omitempty"` + Material bool `json:"material"` + P95Contained bool `json:"p95_contained"` + QualifiedWinner bool `json:"qualified_winner"` + PlanModes []ExpandIntoPlanMode `json:"plan_modes"` +} + +// ExpandIntoPlanMode summarizes the PostgreSQL shapes observed under one plan-cache mode. +type ExpandIntoPlanMode struct { + PlanCacheMode string `json:"plan_cache_mode"` + Fingerprints []string `json:"plan_fingerprints"` + OperatorFamilies []string `json:"operator_families"` + ParameterizedIndex bool `json:"parameterized_index"` + Memoize bool `json:"memoize"` + HashJoin bool `json:"hash_join"` +} + +type expandIntoArmSeries struct { + identity postgresReferenceSpec + samples roundSamples + plans map[string]map[string][]string +} + +// buildExpandIntoStudyReport validates all three exact arms and constructs descriptive crossover evidence. +func buildExpandIntoStudyReport(records []CaseResult, options ExpandIntoStudyOptions) (ExpandIntoStudyReport, error) { + protocol := options.Protocol + if protocol == "" { + protocol = referencePairProtocolDiscovery + } + minimumWarmups, minimumRounds, maximumRounds, minimumSamples := 5, 5, 20, 10 + if protocol == referencePairProtocolConfirmation { + minimumWarmups, minimumRounds, maximumRounds, minimumSamples = 20, 10, 20, 50 + } else if protocol != referencePairProtocolDiscovery { + return ExpandIntoStudyReport{}, fmt.Errorf("unsupported ExpandInto study protocol %q", protocol) + } + if options.Confidence <= 0 || options.Confidence >= 1 { + return ExpandIntoStudyReport{}, fmt.Errorf("confidence level must be between 0 and 1") + } + if options.BootstrapCount == 0 { + options.BootstrapCount = defaultBootstrapCount + } + if options.MaterialityRatio == 0 { + options.MaterialityRatio = .95 + } + if options.MaterialityRatio <= 0 || options.MaterialityRatio >= 1 { + return ExpandIntoStudyReport{}, fmt.Errorf("materiality ratio must be between 0 and 1") + } + if options.MaterialityAbsolute == 0 { + options.MaterialityAbsolute = 100 * time.Microsecond + } + if options.MaterialityAbsolute < 0 { + return ExpandIntoStudyReport{}, fmt.Errorf("materiality absolute must not be negative") + } + if options.P95RatioLimit == 0 { + options.P95RatioLimit = 1.05 + } + if options.P95RatioLimit <= 0 { + return ExpandIntoStudyReport{}, fmt.Errorf("p95 ratio limit must be positive") + } + + type key struct{ dataset, name string } + type caseSeries struct { + tier string + split string + arms map[string]*expandIntoArmSeries + rounds map[int]struct{} + planModes map[string]struct{} + problems map[string]struct{} + } + series := map[key]*caseSeries{} + for _, record := range records { + if record.ExecutionMode != ModePostgresSQL || record.Category != "expand_into_one_hop" { + continue + } + if record.Status != StatusOK || record.Environment == nil || record.Environment.WarmupIterations < minimumWarmups { + return ExpandIntoStudyReport{}, fmt.Errorf("%s/%s lacks a successful %d-warmup PostgreSQL record", record.Dataset, record.Name, minimumWarmups) + } + caseKey := key{record.Dataset, record.Name} + current := series[caseKey] + if current == nil { + current = &caseSeries{ + tier: record.Shape.FixtureTier, split: record.Shape.QualificationSplit, arms: map[string]*expandIntoArmSeries{}, rounds: map[int]struct{}{}, + planModes: map[string]struct{}{}, problems: map[string]struct{}{}, + } + series[caseKey] = current + } else if current.tier != record.Shape.FixtureTier { + return ExpandIntoStudyReport{}, fmt.Errorf("%s/%s changes fixture tier across rounds", record.Dataset, record.Name) + } else if current.split != record.Shape.QualificationSplit { + return ExpandIntoStudyReport{}, fmt.Errorf("%s/%s changes qualification split across rounds", record.Dataset, record.Name) + } + if current.split != "training" && current.split != "holdout" { + return ExpandIntoStudyReport{}, fmt.Errorf("%s/%s requires a training or holdout qualification split", record.Dataset, record.Name) + } + if record.Environment.Round < 1 { + return ExpandIntoStudyReport{}, fmt.Errorf("%s/%s has an invalid measurement round %d", record.Dataset, record.Name, record.Environment.Round) + } + if _, duplicate := current.rounds[record.Environment.Round]; duplicate { + return ExpandIntoStudyReport{}, fmt.Errorf("%s/%s has duplicate round %d", record.Dataset, record.Name, record.Environment.Round) + } + current.rounds[record.Environment.Round] = struct{}{} + cacheMode := "" + if record.PostgresEnvironment != nil { + cacheMode = record.PostgresEnvironment.PlanCacheMode + } + if cacheMode != "auto" && cacheMode != "force_custom_plan" && cacheMode != "force_generic_plan" { + current.problems[fmt.Sprintf("round %d has missing or unsupported plan_cache_mode %q", record.Environment.Round, cacheMode)] = struct{}{} + } else { + current.planModes[cacheMode] = struct{}{} + } + for _, armName := range expandIntoStudyArms { + reference := findReference(record.PostgresReferences, armName) + if reference == nil { + return ExpandIntoStudyReport{}, fmt.Errorf("%s/%s round %d lacks ExpandInto arm %s", record.Dataset, record.Name, record.Environment.Round, armName) + } + if !reference.FullComparator || reference.SemanticValidation != "exact_public_observation" || reference.RowCount != record.RowCount || !equalStrings(reference.ObservedRows, record.ObservedRows) { + return ExpandIntoStudyReport{}, fmt.Errorf("%s/%s arm %s is not an exact public comparator", record.Dataset, record.Name, armName) + } + if reference.Stats.WarmupIterations < minimumWarmups { + return ExpandIntoStudyReport{}, fmt.Errorf("%s/%s arm %s has fewer than %d warmups", record.Dataset, record.Name, armName, minimumWarmups) + } + arm := current.arms[armName] + identity := normalizedReferenceSpec(postgresReferenceSpec{ + name: reference.Name, architecture: reference.Architecture, implementationID: reference.ImplementationID, + stateShape: reference.StateShape, observationShape: reference.ObservationShape, + semanticValidation: reference.SemanticValidation, boundary: reference.Boundary, + fullComparator: reference.FullComparator, timingBoundary: reference.TimingBoundary, + sql: reference.SQL, + }) + if arm == nil { + arm = &expandIntoArmSeries{identity: identity, samples: roundSamples{}, plans: map[string]map[string][]string{}} + current.arms[armName] = arm + } else if arm.identity.architecture != identity.architecture || arm.identity.implementationID != identity.implementationID || normalizedSQLFingerprint(arm.identity.sql) != normalizedSQLFingerprint(identity.sql) { + return ExpandIntoStudyReport{}, fmt.Errorf("%s/%s arm %s identity changed across rounds", record.Dataset, record.Name, armName) + } + for _, sample := range reference.Stats.Samples { + if sample.Classification == "warm" && sample.Duration > 0 { + arm.samples[record.Environment.Round] = append(arm.samples[record.Environment.Round], sample.Duration) + } + } + if len(reference.PostgresPlan) == 0 { + current.problems[fmt.Sprintf("%s round %d has no persisted PostgreSQL plan", armName, record.Environment.Round)] = struct{}{} + } + planModeKey := cacheMode + if planModeKey == "" { + planModeKey = "unknown" + } + fingerprint := normalizedSQLFingerprint(strings.Join(reference.PostgresPlan, "\n")) + if arm.plans[planModeKey] == nil { + arm.plans[planModeKey] = map[string][]string{} + } + arm.plans[planModeKey][fingerprint] = append([]string(nil), reference.PostgresPlan...) + } + if err := validateExpandIntoRoundOrder(record.Environment.Round, record.PostgresReferences); err != nil { + return ExpandIntoStudyReport{}, fmt.Errorf("%s/%s: %w", record.Dataset, record.Name, err) + } + } + if len(series) == 0 { + return ExpandIntoStudyReport{}, fmt.Errorf("artifact has no PostgreSQL ExpandInto study records") + } + + report := ExpandIntoStudyReport{ + Version: expandIntoStudyReportVersion, + Protocol: protocol, + Confidence: options.Confidence, + Passed: true, + TrainingPassed: true, + HoldoutPassed: true, + } + keys := make([]key, 0, len(series)) + for caseKey := range series { + keys = append(keys, caseKey) + } + sort.Slice(keys, func(i, j int) bool { + return keys[i].dataset < keys[j].dataset || keys[i].dataset == keys[j].dataset && keys[i].name < keys[j].name + }) + gateOptions := PerfGateOptions{Seed: options.Seed, Confidence: options.Confidence, BootstrapCount: options.BootstrapCount} + qualifiedWinners := map[string]struct{}{} + for caseIndex, caseKey := range keys { + current := series[caseKey] + entry := ExpandIntoStudyCase{Dataset: caseKey.dataset, Name: caseKey.name, Tier: current.tier, QualificationSplit: current.split, Rounds: len(current.rounds), Passed: true} + for problem := range current.problems { + entry.Reasons = append(entry.Reasons, problem) + } + sort.Strings(entry.Reasons) + if len(entry.Reasons) > 0 { + entry.Passed = false + } + if entry.Rounds < minimumRounds || entry.Rounds > maximumRounds { + entry.Passed = false + entry.Reasons = append(entry.Reasons, fmt.Sprintf("requires %d-%d rounds, got %d", minimumRounds, maximumRounds, entry.Rounds)) + } + if protocol == referencePairProtocolConfirmation { + for _, mode := range []string{"auto", "force_custom_plan", "force_generic_plan"} { + if _, present := current.planModes[mode]; !present { + entry.Passed = false + entry.Reasons = append(entry.Reasons, "confirmation requires plan_cache_mode="+mode) + } + } + } + direct := current.arms[expandIntoStudyArms[0]].samples + winnerMedian := time.Duration(1<<63 - 1) + qualifiedWinnerMedian := time.Duration(1<<63 - 1) + for armIndex, armName := range expandIntoStudyArms { + arm := current.arms[armName] + for _, round := range sortedRoundSet(current.rounds) { + if len(arm.samples[round]) < minimumSamples { + entry.Passed = false + entry.Reasons = append(entry.Reasons, fmt.Sprintf("%s round %d requires %d samples, got %d", armName, round, minimumSamples, len(arm.samples[round]))) + } + } + flat := flattenSamples(arm.samples, sortedRounds(arm.samples)) + evidence := ExpandIntoStudyArmEvidence{ + Name: armName, Architecture: arm.identity.architecture, ImplementationID: arm.identity.implementationID, + SQLFingerprint: normalizedSQLFingerprint(arm.identity.sql), Samples: len(flat), + Median: time.Duration(durationQuantile(flat, .50)), P95: time.Duration(durationQuantile(flat, .95)), + PlanModes: expandIntoPlanModes(arm.plans), + } + if evidence.Median < winnerMedian { + winnerMedian, entry.Winner = evidence.Median, armName + } + if armName != expandIntoStudyArms[0] { + baseline, candidate := matchedRounds(direct, arm.samples) + if len(baseline) > 0 { + seed := options.Seed + int64(caseIndex*31+armIndex)*7919 + median := bootstrapRoundMedianRatio(baseline, candidate, seed, gateOptions) + evidence.MedianRatioToDirect = &median + saving := bootstrapRoundMedianSaving(baseline, candidate, seed+1, gateOptions) + evidence.MedianSavingToDirect = &saving + if sampleCount(baseline) >= minimumP95Samples && sampleCount(candidate) >= minimumP95Samples { + p95 := bootstrapStratifiedP95Ratio(baseline, candidate, seed+2, gateOptions) + evidence.P95RatioToDirect = &p95 + } + evidence.Material = median.Upper <= options.MaterialityRatio || saving.Lower >= options.MaterialityAbsolute + evidence.P95Contained = evidence.P95RatioToDirect != nil && evidence.P95RatioToDirect.Upper <= options.P95RatioLimit + evidence.QualifiedWinner = evidence.Material && evidence.P95Contained + if evidence.QualifiedWinner && evidence.Median < qualifiedWinnerMedian { + qualifiedWinnerMedian, entry.QualifiedWinner = evidence.Median, armName + } + } + } + entry.ArmResults = append(entry.ArmResults, evidence) + } + if protocol == referencePairProtocolConfirmation && entry.QualifiedWinner == "" { + entry.Passed = false + entry.Reasons = append(entry.Reasons, "no non-incumbent arm materially beats the direct pair join with p95 containment") + } + if protocol == referencePairProtocolConfirmation && entry.Passed { + qualifiedWinners[entry.QualifiedWinner] = struct{}{} + } + if !entry.Passed { + report.Passed = false + } + switch entry.QualificationSplit { + case "training": + report.TrainingCases++ + report.TrainingPassed = report.TrainingPassed && entry.Passed + case "holdout": + report.HoldoutCases++ + report.HoldoutPassed = report.HoldoutPassed && entry.Passed + } + report.Cases = append(report.Cases, entry) + } + report.TrainingPassed = report.TrainingCases > 0 && report.TrainingPassed + report.HoldoutPassed = report.HoldoutCases > 0 && report.HoldoutPassed + report.QualificationPassed = protocol == referencePairProtocolConfirmation && + report.TrainingPassed && report.HoldoutPassed && len(qualifiedWinners) == 1 + if len(qualifiedWinners) == 1 { + for winner := range qualifiedWinners { + report.Winner = winner + } + } + report.PromotionEligible = report.QualificationPassed + if protocol == referencePairProtocolConfirmation && !report.QualificationPassed { + report.Passed = false + } + return report, nil +} + +// sortedRoundSet returns declared measurement rounds in stable order, including +// rounds whose arms contain no usable warm sample. +func sortedRoundSet(rounds map[int]struct{}) []int { + ordered := make([]int, 0, len(rounds)) + for round := range rounds { + ordered = append(ordered, round) + } + sort.Ints(ordered) + return ordered +} + +func equalStrings(left, right []string) bool { + if len(left) != len(right) { + return false + } + for idx := range left { + if left[idx] != right[idx] { + return false + } + } + return true +} + +// validateExpandIntoRoundOrder enforces the predeclared doubled Williams schedule relative to the three selected arms. +func validateExpandIntoRoundOrder(round int, references []PostgresReferenceResult) error { + base := make([]postgresReferenceSpec, len(expandIntoStudyArms)) + for idx, name := range expandIntoStudyArms { + base[idx] = postgresReferenceSpec{name: name} + } + expected := referenceSpecsForRound(base, round) + byName := map[string]int{} + for _, reference := range references { + if containsString(expandIntoStudyArms, reference.Name) { + byName[reference.Name] = reference.MeasurementOrder + } + } + for _, name := range expandIntoStudyArms { + if byName[name] <= 0 { + return fmt.Errorf("round %d is missing measurement order for %s", round, name) + } + } + for idx := 1; idx < len(expected); idx++ { + if byName[expected[idx-1].name] >= byName[expected[idx].name] { + return fmt.Errorf("round %d lacks the declared three-arm carryover order", round) + } + } + return nil +} + +func containsString(values []string, value string) bool { + for _, candidate := range values { + if candidate == value { + return true + } + } + return false +} + +// expandIntoPlanModes classifies parameterized index, Memoize, and hash alternatives per plan-cache mode. +func expandIntoPlanModes(plans map[string]map[string][]string) []ExpandIntoPlanMode { + var modes []ExpandIntoPlanMode + for mode, byFingerprint := range plans { + evidence := ExpandIntoPlanMode{PlanCacheMode: mode} + operators := map[string]struct{}{} + for fingerprint, plan := range byFingerprint { + evidence.Fingerprints = append(evidence.Fingerprints, fingerprint) + joined := strings.ToLower(strings.Join(plan, "\n")) + evidence.ParameterizedIndex = evidence.ParameterizedIndex || strings.Contains(joined, "index scan") && (strings.Contains(joined, "start_id") || strings.Contains(joined, "end_id")) + evidence.Memoize = evidence.Memoize || strings.Contains(joined, "memoize") + evidence.HashJoin = evidence.HashJoin || strings.Contains(joined, "hash join") + for _, line := range plan { + operator := expandIntoPlanOperator(line) + if operator != "" { + operators[operator] = struct{}{} + } + } + } + for operator := range operators { + evidence.OperatorFamilies = append(evidence.OperatorFamilies, operator) + } + sort.Strings(evidence.Fingerprints) + sort.Strings(evidence.OperatorFamilies) + modes = append(modes, evidence) + } + sort.Slice(modes, func(i, j int) bool { return modes[i].PlanCacheMode < modes[j].PlanCacheMode }) + return modes +} + +// expandIntoPlanOperator removes EXPLAIN decorations while retaining the physical operator family. +func expandIntoPlanOperator(line string) string { + line = strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(line), "->")) + if line == "" || strings.HasPrefix(line, "Filter:") || strings.HasPrefix(line, "Index Cond:") || strings.HasPrefix(line, "Join Filter:") { + return "" + } + if index := strings.Index(line, " ("); index >= 0 { + line = line[:index] + } + if index := strings.Index(line, " on "); index >= 0 { + line = line[:index] + } + if index := strings.Index(line, " using "); index >= 0 { + line = line[:index] + } + return strings.TrimSpace(line) +} + +// createExpandIntoStudyReport reads, validates, fingerprints, and writes a three-arm study artifact. +func createExpandIntoStudyReport(artifactPath, outputPath string, options ExpandIntoStudyOptions) error { + records, err := readJSONLFile(artifactPath) + if err != nil { + return err + } + report, err := buildExpandIntoStudyReport(records, options) + if err != nil { + return err + } + report.ArtifactSHA256, err = fileSHA256(artifactPath) + if err != nil { + return err + } + var output *os.File + if outputPath == "" { + output = os.Stdout + } else { + if err := ensureOutputDir(outputPath); err != nil { + return err + } + output, err = os.Create(outputPath) + if err != nil { + return err + } + defer output.Close() + } + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + if err := encoder.Encode(report); err != nil { + return err + } + if !report.Passed { + return fmt.Errorf("ExpandInto %s evidence did not pass its declared protocol", report.Protocol) + } + return nil +} diff --git a/cmd/graphbench/expand_into_report_test.go b/cmd/graphbench/expand_into_report_test.go new file mode 100644 index 00000000..82835df8 --- /dev/null +++ b/cmd/graphbench/expand_into_report_test.go @@ -0,0 +1,217 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestBuildExpandIntoStudyReportValidatesThreeArmEvidence verifies exactness, Williams order, ratios, winners, and physical-plan classification. +func TestBuildExpandIntoStudyReportValidatesThreeArmEvidence(t *testing.T) { + var records []CaseResult + for round := 1; round <= 5; round++ { + orderSpecs := make([]postgresReferenceSpec, len(expandIntoStudyArms)) + for idx, name := range expandIntoStudyArms { + orderSpecs[idx].name = name + } + ordered := referenceSpecsForRound(orderSpecs, round) + orders := map[string]int{} + for idx, spec := range ordered { + orders[spec.name] = idx + 2 + } + record := CaseResult{ + Environment: &RunEnvironment{Round: round, WarmupIterations: 5}, + PostgresEnvironment: &PostgresEnvironment{PlanCacheMode: "force_custom_plan"}, + Dataset: "expand_into", Name: "pair", Category: "expand_into_one_hop", + Shape: WorkloadShape{FixtureTier: "normal", QualificationSplit: "training"}, ExecutionMode: ModePostgresSQL, Status: StatusOK, + RowCount: 1, ObservedRows: []string{`["edge"]`}, + } + for idx, name := range expandIntoStudyArms { + duration := time.Duration(100-idx*10) * time.Microsecond + var samples []LatencySample + for sample := 0; sample < 10; sample++ { + samples = append(samples, LatencySample{Classification: "warm", Duration: duration + time.Duration(sample)}) + } + plan := []string{"Nested Loop (cost=0.00..1.00 rows=1 width=8)", " -> Index Scan using edge_start_id_idx on edge (cost=0.00..1.00 rows=1 width=8)", " Index Cond: (start_id = input_pairs.start_id)"} + if name == "expand_into_pair_cache" { + plan = []string{"Hash Join (cost=0.00..1.00 rows=1 width=8)", " -> Memoize (cost=0.00..1.00 rows=1 width=8)"} + } + record.PostgresReferences = append(record.PostgresReferences, PostgresReferenceResult{ + SchemaVersion: postgresReferenceSchemaVersion, Name: name, Architecture: "architecture-" + name, + ImplementationID: name + "-v1", StateShape: "state", ObservationShape: "relationships", + SemanticValidation: "exact_public_observation", Boundary: "relationships", TimingBoundary: "raw_pgx", + FullComparator: true, MeasurementOrder: orders[name], SQL: "select '" + name + "'", SQLFingerprint: name, + RowCount: 1, ObservedRows: []string{`["edge"]`}, Stats: DurationStats{WarmupIterations: 5, Samples: samples}, + PostgresPlan: plan, + }) + } + records = append(records, record) + } + + report, err := buildExpandIntoStudyReport(records, ExpandIntoStudyOptions{Seed: 1, Confidence: .975, BootstrapCount: 100, Protocol: referencePairProtocolDiscovery}) + require.NoError(t, err) + require.True(t, report.Passed) + require.Equal(t, 1, report.TrainingCases) + require.Zero(t, report.HoldoutCases) + require.True(t, report.TrainingPassed) + require.False(t, report.HoldoutPassed) + require.False(t, report.QualificationPassed) + require.Len(t, report.Cases, 1) + entry := report.Cases[0] + require.Equal(t, "expand_into_pair_cache", entry.Winner) + require.Len(t, entry.ArmResults, 3) + require.Nil(t, entry.ArmResults[0].MedianRatioToDirect) + require.NotNil(t, entry.ArmResults[1].MedianRatioToDirect) + require.True(t, entry.ArmResults[0].PlanModes[0].ParameterizedIndex) + require.True(t, entry.ArmResults[2].PlanModes[0].Memoize) + require.True(t, entry.ArmResults[2].PlanModes[0].HashJoin) + require.Equal(t, "training", entry.QualificationSplit) + + artifactPath := filepath.Join(t.TempDir(), "expand-into.jsonl") + outputPath := filepath.Join(t.TempDir(), "expand-into.json") + require.NoError(t, writeJSONLFile(artifactPath, records)) + require.NoError(t, createExpandIntoStudyReport(artifactPath, outputPath, ExpandIntoStudyOptions{ + Seed: 1, Confidence: .975, BootstrapCount: 100, Protocol: referencePairProtocolDiscovery, + })) + content, err := os.ReadFile(outputPath) + require.NoError(t, err) + var written ExpandIntoStudyReport + require.NoError(t, json.Unmarshal(content, &written)) + require.True(t, written.Passed) + require.True(t, validSHA256(written.ArtifactSHA256)) + require.Equal(t, referencePairProtocolDiscovery, written.Protocol) + + var confirmationRecords []CaseResult + for round := 1; round <= 10; round++ { + record := records[(round-1)%len(records)] + record.Environment = &RunEnvironment{Round: round, WarmupIterations: 20} + planModes := []string{"auto", "force_custom_plan", "force_generic_plan"} + record.PostgresEnvironment = &PostgresEnvironment{PlanCacheMode: planModes[(round-1)%len(planModes)]} + record.PostgresReferences = append([]PostgresReferenceResult(nil), record.PostgresReferences...) + orderSpecs := make([]postgresReferenceSpec, len(expandIntoStudyArms)) + for idx, name := range expandIntoStudyArms { + orderSpecs[idx].name = name + } + orders := map[string]int{} + for idx, spec := range referenceSpecsForRound(orderSpecs, round) { + orders[spec.name] = idx + 2 + } + for idx := range record.PostgresReferences { + reference := &record.PostgresReferences[idx] + reference.MeasurementOrder = orders[reference.Name] + reference.Stats.WarmupIterations = 20 + duration := reference.Stats.Samples[0].Duration + reference.Stats.Samples = make([]LatencySample, 50) + for sample := range reference.Stats.Samples { + reference.Stats.Samples[sample] = LatencySample{Classification: "warm", Duration: duration + time.Duration(sample)} + } + } + confirmationRecords = append(confirmationRecords, record) + holdout := record + holdout.Name = "pair-holdout" + holdout.Shape.QualificationSplit = "holdout" + confirmationRecords = append(confirmationRecords, holdout) + } + confirmation, err := buildExpandIntoStudyReport(confirmationRecords, ExpandIntoStudyOptions{ + Seed: 1, Confidence: .975, BootstrapCount: 100, Protocol: referencePairProtocolConfirmation, + }) + require.NoError(t, err) + require.True(t, confirmation.Passed) + require.Equal(t, 1, confirmation.TrainingCases) + require.Equal(t, 1, confirmation.HoldoutCases) + require.True(t, confirmation.TrainingPassed) + require.True(t, confirmation.HoldoutPassed) + require.True(t, confirmation.QualificationPassed) + require.Equal(t, referencePairProtocolConfirmation, confirmation.Protocol) + var trainingOnly []CaseResult + for _, record := range confirmationRecords { + if record.Shape.QualificationSplit == "training" { + trainingOnly = append(trainingOnly, record) + } + } + trainingOnlyReport, err := buildExpandIntoStudyReport(trainingOnly, ExpandIntoStudyOptions{ + Seed: 1, Confidence: .975, BootstrapCount: 100, Protocol: referencePairProtocolConfirmation, + }) + require.NoError(t, err) + require.False(t, trainingOnlyReport.Passed) + require.True(t, trainingOnlyReport.TrainingPassed) + require.False(t, trainingOnlyReport.HoldoutPassed) + require.False(t, trainingOnlyReport.QualificationPassed) + + for idx := range confirmationRecords { + confirmationRecords[idx].PostgresEnvironment = &PostgresEnvironment{PlanCacheMode: "force_custom_plan"} + } + incompleteModes, err := buildExpandIntoStudyReport(confirmationRecords, ExpandIntoStudyOptions{ + Seed: 1, Confidence: .975, BootstrapCount: 100, Protocol: referencePairProtocolConfirmation, + }) + require.NoError(t, err) + require.False(t, incompleteModes.Passed) + require.Contains(t, incompleteModes.Cases[0].Reasons, "confirmation requires plan_cache_mode=auto") + require.Contains(t, incompleteModes.Cases[0].Reasons, "confirmation requires plan_cache_mode=force_generic_plan") +} + +// TestBuildExpandIntoStudyReportFailsClosedOnObservationOrOrderMismatch verifies plan evidence cannot qualify without exact rows and declared carryover order. +func TestBuildExpandIntoStudyReportFailsClosedOnObservationOrOrderMismatch(t *testing.T) { + record := CaseResult{ + Environment: &RunEnvironment{Round: 1, WarmupIterations: 5}, Dataset: "expand_into", Name: "pair", + Category: "expand_into_one_hop", Shape: WorkloadShape{FixtureTier: "normal", QualificationSplit: "training"}, + ExecutionMode: ModePostgresSQL, Status: StatusOK, RowCount: 1, ObservedRows: []string{"public"}, + } + for _, name := range expandIntoStudyArms { + record.PostgresReferences = append(record.PostgresReferences, PostgresReferenceResult{ + Name: name, Architecture: name, ImplementationID: name, FullComparator: true, + SemanticValidation: "exact_public_observation", RowCount: 1, ObservedRows: []string{"different"}, + Stats: DurationStats{WarmupIterations: 5}, MeasurementOrder: 2, + }) + } + _, err := buildExpandIntoStudyReport([]CaseResult{record}, ExpandIntoStudyOptions{Confidence: .975, Protocol: referencePairProtocolDiscovery}) + require.ErrorContains(t, err, "not an exact public comparator") +} + +// TestCreateExpandIntoStudyReportPersistsAndRejectsIncompleteEvidence verifies +// a durable diagnostic report cannot be mistaken for a successful gate. +func TestCreateExpandIntoStudyReportPersistsAndRejectsIncompleteEvidence(t *testing.T) { + record := CaseResult{ + Environment: &RunEnvironment{Round: 1, WarmupIterations: 5}, Dataset: "expand_into", Name: "pair", + Category: "expand_into_one_hop", Shape: WorkloadShape{FixtureTier: "normal", QualificationSplit: "training"}, ExecutionMode: ModePostgresSQL, + Status: StatusOK, RowCount: 1, ObservedRows: []string{`["edge"]`}, + } + orderSpecs := make([]postgresReferenceSpec, len(expandIntoStudyArms)) + for idx, name := range expandIntoStudyArms { + orderSpecs[idx].name = name + } + orders := map[string]int{} + for idx, spec := range referenceSpecsForRound(orderSpecs, 1) { + orders[spec.name] = idx + 2 + } + for _, name := range expandIntoStudyArms { + record.PostgresReferences = append(record.PostgresReferences, PostgresReferenceResult{ + Name: name, Architecture: name, ImplementationID: name + "-v1", StateShape: "state", + ObservationShape: "relationships", Boundary: "relationships", TimingBoundary: "raw_pgx", + FullComparator: true, SemanticValidation: "exact_public_observation", RowCount: 1, + ObservedRows: []string{`["edge"]`}, SQL: "select '" + name + "'", MeasurementOrder: orders[name], + Stats: DurationStats{WarmupIterations: 5, Samples: []LatencySample{{Classification: "warm", Duration: time.Millisecond}}}, + }) + } + artifactPath := filepath.Join(t.TempDir(), "incomplete.jsonl") + outputPath := filepath.Join(t.TempDir(), "report.json") + require.NoError(t, writeJSONLFile(artifactPath, []CaseResult{record})) + require.ErrorContains(t, createExpandIntoStudyReport(artifactPath, outputPath, ExpandIntoStudyOptions{ + Seed: 1, Confidence: .975, BootstrapCount: 100, Protocol: referencePairProtocolDiscovery, + }), "did not pass") + + content, err := os.ReadFile(outputPath) + require.NoError(t, err) + var report ExpandIntoStudyReport + require.NoError(t, json.Unmarshal(content, &report)) + require.False(t, report.Passed) +} diff --git a/cmd/graphbench/live_mode.go b/cmd/graphbench/live_mode.go new file mode 100644 index 00000000..a21cde97 --- /dev/null +++ b/cmd/graphbench/live_mode.go @@ -0,0 +1,598 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bufio" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "time" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/opengraph" +) + +// existingGraphCheckpointVersion identifies the serialized schema revision for existing graph checkpoint. +const existingGraphCheckpointVersion = 2 + +// mutationKeyword matches Cypher keywords that can mutate an existing graph. +var mutationKeyword = regexp.MustCompile(`(?i)\b(create|merge|delete|detach|set|remove|drop|alter|truncate|grant|revoke|call|foreach|load\s+csv)\b`) + +// ExistingGraphAnchorManifest authorizes read-only live-graph workloads against validated logical or redacted physical anchors. +type ExistingGraphAnchorManifest struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // Graph identifies the graph addressed by the artifact. + Graph string `json:"graph"` + // ContentIdentity binds resumable work to the logical contents of the live graph. + ContentIdentity string `json:"content_identity"` + // Anchors maps manifest anchor names to their logical or redacted physical identities. + Anchors map[string]ExistingGraphAnchor `json:"anchors"` + // Checksum records the digest of the validated manifest file. + Checksum string `json:"-"` +} + +// ExistingGraphAnchor maps a logical fixture key to either a logical or redacted physical identity. +type ExistingGraphAnchor struct { + // LogicalKey identifies an anchor using a corpus-visible fixture key. + LogicalKey string `json:"logical_key,omitempty"` + // PhysicalID selects a backend node directly when no corpus-visible logical key is available. + PhysicalID *int64 `json:"physical_id,omitempty"` + // ContentSHA256 identifies scrubbed physical anchor content without exposing it. + ContentSHA256 string `json:"content_sha256,omitempty"` + // Kind optionally requires the resolved anchor node to carry this graph kind. + Kind string `json:"kind,omitempty"` +} + +// ExistingGraphAttempt captures the applied deadline, collected samples, and outcome of one live-graph execution. +type ExistingGraphAttempt struct { + // Timeout records the deadline applied to this live-graph attempt; zero means no deadline. + Timeout time.Duration `json:"timeout"` + // WarmupSamples records untimed samples collected before live-graph measurement. + WarmupSamples int `json:"warmup_samples"` + // MeasuredSamples records timed samples collected for the live-graph attempt. + MeasuredSamples int `json:"measured_samples"` + // Status records the execution outcome. + Status string `json:"status"` + // Error records the failure message when the operation did not succeed. + Error string `json:"error,omitempty"` +} + +// ExistingGraphRun describes a resumable live-graph run and all attempts made in it. +type ExistingGraphRun struct { + // ManifestSHA256 identifies the anchor manifest that authorized the run. + ManifestSHA256 string `json:"manifest_sha256"` + // ContentIdentity binds resumable work to the logical contents of the live graph. + ContentIdentity string `json:"content_identity"` + // Protocol identifies the measurement protocol. + Protocol string `json:"protocol"` + // Adaptive indicates that adaptive discovery, rather than a fixed protocol, produced the record. + Adaptive bool `json:"adaptive"` + // Attempts lists live-graph attempts in execution order. + Attempts []ExistingGraphAttempt `json:"attempts,omitempty"` + // PreNodeCount records graph nodes present before the live-graph run. + PreNodeCount int64 `json:"pre_node_count"` + // PreEdgeCount records graph relationships present before the live-graph run. + PreEdgeCount int64 `json:"pre_edge_count"` + // PostNodeCount records graph nodes present after the live-graph run. + PostNodeCount int64 `json:"post_node_count"` + // PostEdgeCount records graph relationships present after the live-graph run. + PostEdgeCount int64 `json:"post_edge_count"` +} + +// ExistingGraphProgress is one append-only progress event emitted during a live-graph run. +type ExistingGraphProgress struct { + // At records when the progress event was emitted. + At time.Time `json:"at"` + // Stage identifies the stage reached by a live-graph progress event. + Stage string `json:"stage"` + // CaseKey identifies the dataset/case pair addressed by a progress event. + CaseKey string `json:"case_key,omitempty"` + // Detail contains the progress or failure detail safe to persist. + Detail string `json:"detail,omitempty"` +} + +// existingGraphCheckpoint binds completed live-graph cases to a corpus, run configuration, and fixture identity. +type existingGraphCheckpoint struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // ManifestSHA256 identifies the anchor manifest that authorized the run. + ManifestSHA256 string `json:"manifest_sha256"` + // CorpusSHA256 binds checkpoint records to the exact canonical workload declarations. + CorpusSHA256 string `json:"corpus_sha256"` + // RunSHA256 binds completed records to the exact resumable run configuration. + RunSHA256 string `json:"run_sha256"` + // Records contains completed CaseResults retained for resumable execution. + Records []CaseResult `json:"records"` +} + +// loadExistingGraphAnchorManifest reads and validates a live-graph anchor manifest and records its checksum. +func loadExistingGraphAnchorManifest(path string) (ExistingGraphAnchorManifest, error) { + raw, err := os.ReadFile(path) + if err != nil { + return ExistingGraphAnchorManifest{}, fmt.Errorf("read anchor manifest: %w", err) + } + var manifest ExistingGraphAnchorManifest + if err := json.Unmarshal(raw, &manifest); err != nil { + return ExistingGraphAnchorManifest{}, fmt.Errorf("decode anchor manifest: %w", err) + } + if manifest.Version != 1 { + return ExistingGraphAnchorManifest{}, fmt.Errorf("unsupported anchor manifest version %d", manifest.Version) + } + if len(manifest.Anchors) == 0 { + return ExistingGraphAnchorManifest{}, fmt.Errorf("anchor manifest must contain anchors") + } + if strings.TrimSpace(manifest.Graph) == "" { + return ExistingGraphAnchorManifest{}, fmt.Errorf("anchor manifest graph must not be empty") + } + if matched, _ := regexp.MatchString(`^sha256:[0-9a-f]{64}$`, manifest.ContentIdentity); !matched { + return ExistingGraphAnchorManifest{}, fmt.Errorf("anchor manifest content_identity must be a lowercase sha256 digest") + } + for name, anchor := range manifest.Anchors { + if strings.TrimSpace(name) == "" { + return ExistingGraphAnchorManifest{}, fmt.Errorf("anchor names must not be empty") + } + hasLogicalKey := strings.TrimSpace(anchor.LogicalKey) != "" + hasPhysicalID := anchor.PhysicalID != nil + if hasLogicalKey == hasPhysicalID { + return ExistingGraphAnchorManifest{}, fmt.Errorf("anchor %s must declare exactly one of logical_key or physical_id", name) + } + if hasPhysicalID { + if matched, _ := regexp.MatchString(`^sha256:[0-9a-f]{64}$`, anchor.ContentSHA256); !matched { + return ExistingGraphAnchorManifest{}, fmt.Errorf("physical anchor %s content_sha256 must be a lowercase sha256 digest", name) + } + } else if anchor.ContentSHA256 != "" { + return ExistingGraphAnchorManifest{}, fmt.Errorf("logical-key anchor %s must not declare content_sha256", name) + } + } + digest := sha256.Sum256(raw) + manifest.Checksum = hex.EncodeToString(digest[:]) + return manifest, nil +} + +// validateExistingGraphCorpus rejects mutations and anchors absent from the live-graph manifest. +func validateExistingGraphCorpus(corpus ScaleCorpus, manifest ExistingGraphAnchorManifest) error { + for _, testCase := range corpus.Cases { + if testCase.WriteScenario != nil { + return fmt.Errorf("existing-graph mode rejects write_scenario in case %s", testCase.Name) + } + if mutationKeyword.MatchString(stripCypherStringLiterals(testCase.Cypher)) { + return fmt.Errorf("existing-graph mode rejects mutation keyword in case %s", testCase.Name) + } + for _, anchor := range testCase.NodeParams { + if _, found := manifest.Anchors[anchor]; !found { + return fmt.Errorf("case %s references anchor %q absent from the manifest", testCase.Name, anchor) + } + } + for _, anchors := range testCase.NodeListParams { + for _, anchor := range anchors { + if _, found := manifest.Anchors[anchor]; !found { + return fmt.Errorf("case %s references anchor %q absent from the manifest", testCase.Name, anchor) + } + } + } + } + return nil +} + +// stripCypherStringLiterals replaces quoted Cypher contents with spaces before mutation-keyword scanning. +func stripCypherStringLiterals(query string) string { + var ( + result strings.Builder + quote rune + escaped bool + ) + + for _, value := range query { + if quote != 0 { + if escaped { + escaped = false + continue + } + if value == '\\' { + escaped = true + continue + } + if value == quote { + quote = 0 + } + result.WriteRune(' ') + continue + } + + if value == '\'' || value == '"' { + quote = value + result.WriteRune(' ') + continue + } + result.WriteRune(value) + } + + return result.String() +} + +// existingGraphCaseKey joins execution mode, dataset, and case name into the checkpoint lookup key. +func existingGraphCaseKey(mode ExecutionMode, testCase ScaleCase) string { + return strings.Join([]string{string(mode), testCase.Dataset, testCase.Name}, "/") +} + +// corpusIdentity hashes the canonical corpus declaration used to bind checkpoints to workloads. +func corpusIdentity(corpus ScaleCorpus) string { + cases := append([]ScaleCase(nil), corpus.Cases...) + sort.Slice(cases, func(i, j int) bool { + if cases[i].Source != cases[j].Source { + return cases[i].Source < cases[j].Source + } + if cases[i].Dataset != cases[j].Dataset { + return cases[i].Dataset < cases[j].Dataset + } + return cases[i].Name < cases[j].Name + }) + raw, _ := json.Marshal(struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // Cases contains the canonically ordered workload declarations bound into the corpus digest. + Cases []ScaleCase `json:"cases"` + }{Version: 2, Cases: cases}) + digest := sha256.Sum256(raw) + return hex.EncodeToString(digest[:]) +} + +// runConfigurationIdentity hashes execution-affecting configuration and environment fields for checkpoint compatibility. +func runConfigurationIdentity(cfg config, environment RunEnvironment) string { + payload := struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // SourceCommit identifies the source commit used to build the benchmark executable. + SourceCommit string `json:"source_commit"` + // DirtyDiffSHA256 identifies uncommitted source changes present during the run. + DirtyDiffSHA256 string `json:"dirty_diff_sha256"` + // BinarySHA256 identifies the benchmark executable used for the run. + BinarySHA256 string `json:"binary_sha256"` + // GOOS records the target operating system of the benchmark executable. + GOOS string `json:"goos"` + // GOARCH records the target architecture of the benchmark executable. + GOARCH string `json:"goarch"` + // GoVersion records the Go toolchain version used to build the executable. + GoVersion string `json:"go_version"` + // Modes records execution-mode order as part of resumable run identity. + Modes []ExecutionMode `json:"modes"` + // Iterations records the number of measured iterations. + Iterations int `json:"iterations"` + // WarmupIterations records the untimed iterations run before measurement. + WarmupIterations int `json:"warmup_iterations"` + // Round identifies the measurement round. + Round int `json:"round"` + // Block identifies the measurement block used to control carryover effects. + Block int `json:"block"` + // Arm identifies the measurement arm that produced the sample. + Arm string `json:"arm"` + // ArmOrder records the arm's position within its balanced measurement block. + ArmOrder int `json:"arm_order"` + // PoolSize sets the database connection-pool size. + PoolSize int `json:"pool_size"` + // Concurrency records the requested worker counts as part of resumable run identity. + Concurrency []int `json:"concurrency"` + // SessionMemoryCeilingBytes sets the per-session memory ceiling in bytes. + SessionMemoryCeilingBytes int64 `json:"session_memory_ceiling_bytes"` + // PoolMemoryCeilingBytes sets the aggregate pool memory ceiling in bytes. + PoolMemoryCeilingBytes int64 `json:"pool_memory_ceiling_bytes"` + // PostgresReferences records whether independent PostgreSQL references are enabled for the run identity. + PostgresReferences bool `json:"postgres_references"` + // PostgresReferenceArms lists independent PostgreSQL reference arms selected for measurement. + PostgresReferenceArms []string `json:"postgres_reference_arms"` + // PostgresForceShortest selects a forced shortest-path executor for diagnostic runs. + PostgresForceShortest string `json:"postgres_force_shortest"` + // PostgresForceExpansion selects a forced expansion search strategy for diagnostic runs. + PostgresForceExpansion string `json:"postgres_force_expansion"` + // PostgresRepeatableRead records the stable-snapshot timing contract. + PostgresRepeatableRead bool `json:"postgres_repeatable_read"` + // PostgresTraversalTelemetry selects the opt-in traversal evidence boundary. + PostgresTraversalTelemetry string `json:"postgres_traversal_telemetry"` + // PostgresExpansionOrientationShadow records the tool-only selector shadow mode. + PostgresExpansionOrientationShadow bool `json:"postgres_expansion_orientation_shadow"` + // PostgresExpansionOrientationTournament records the guarded selector mode. + PostgresExpansionOrientationTournament bool `json:"postgres_expansion_orientation_tournament"` + // PostgresExpansionOrientationPolicy records the immutable selector formula. + PostgresExpansionOrientationPolicy string `json:"postgres_expansion_orientation_policy"` + // Discovery enables adaptive live-graph discovery instead of the fixed confirmation protocol. + Discovery bool `json:"discovery"` + // TimeoutClasses lists the increasing per-attempt deadlines included in resumable run identity. + TimeoutClasses []time.Duration `json:"timeout_classes"` + // DiscoverySampleFloor sets the minimum live-graph samples required before adaptive discovery may stop. + DiscoverySampleFloor int `json:"discovery_sample_floor"` + }{ + Version: 1, + SourceCommit: environment.SourceCommit, + DirtyDiffSHA256: environment.DirtyDiffSHA256, + BinarySHA256: environment.BinarySHA256, + GOOS: environment.GOOS, + GOARCH: environment.GOARCH, + GoVersion: environment.GoVersion, + Modes: append([]ExecutionMode(nil), cfg.Modes...), + Iterations: cfg.Iterations, + WarmupIterations: cfg.WarmupIterations, + Round: cfg.Round, + Block: cfg.Block, + Arm: cfg.Arm, + ArmOrder: cfg.ArmOrder, + PoolSize: cfg.PoolSize, + Concurrency: append([]int(nil), cfg.Concurrency...), + SessionMemoryCeilingBytes: cfg.SessionMemoryCeilingBytes, + PoolMemoryCeilingBytes: cfg.PoolMemoryCeilingBytes, + PostgresReferences: cfg.PostgresReferences, + PostgresReferenceArms: append([]string(nil), cfg.PostgresReferenceArms...), + PostgresForceShortest: cfg.PostgresForceShortest, + PostgresForceExpansion: cfg.PostgresForceExpansion, + PostgresRepeatableRead: cfg.PostgresRepeatableRead, + PostgresTraversalTelemetry: cfg.PostgresTraversalTelemetry, + PostgresExpansionOrientationShadow: cfg.PostgresExpansionOrientationShadow, + PostgresExpansionOrientationTournament: cfg.PostgresExpansionOrientationTournament, + PostgresExpansionOrientationPolicy: cfg.PostgresExpansionOrientationPolicy, + Discovery: cfg.Discovery, + TimeoutClasses: append([]time.Duration(nil), cfg.TimeoutClasses...), + DiscoverySampleFloor: cfg.DiscoverySampleFloor, + } + raw, _ := json.Marshal(payload) + digest := sha256.Sum256(raw) + return hex.EncodeToString(digest[:]) +} + +// readExistingGraphCheckpoint reads a checkpoint, returning an empty checkpoint when the file does not exist. +func readExistingGraphCheckpoint(path, manifestHash, corpusHash, runHash string) ([]CaseResult, error) { + if path == "" { + return nil, nil + } + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var checkpoint existingGraphCheckpoint + if err := json.Unmarshal(raw, &checkpoint); err != nil { + return nil, fmt.Errorf("decode existing-graph checkpoint: %w", err) + } + if checkpoint.Version != existingGraphCheckpointVersion || checkpoint.ManifestSHA256 != manifestHash || checkpoint.CorpusSHA256 != corpusHash || checkpoint.RunSHA256 != runHash { + return nil, fmt.Errorf("existing-graph checkpoint identity does not match this run") + } + seen := map[string]struct{}{} + runUUID := "" + for _, record := range checkpoint.Records { + if record.WorkloadSHA256 == "" || record.Environment == nil || record.Environment.ArtifactSchemaVersion != 2 || record.Environment.CorpusSHA256 != corpusHash || record.Environment.RunIdentitySHA256 != runHash || record.Environment.RunUUID == "" { + return nil, fmt.Errorf("existing-graph checkpoint record identity does not match this run") + } + if runUUID == "" { + runUUID = record.Environment.RunUUID + } else if record.Environment.RunUUID != runUUID { + return nil, fmt.Errorf("existing-graph checkpoint contains multiple run UUIDs") + } + key := strings.Join([]string{string(record.ExecutionMode), record.Dataset, record.Name}, "/") + if _, found := seen[key]; found { + return nil, fmt.Errorf("existing-graph checkpoint contains duplicate record %s", key) + } + seen[key] = struct{}{} + } + return checkpoint.Records, nil +} + +// writeExistingGraphCheckpoint atomically persists live-graph completion state with restrictive permissions. +func writeExistingGraphCheckpoint(path, manifestHash, corpusHash, runHash string, records []CaseResult) error { + if path == "" { + return nil + } + checkpoint := existingGraphCheckpoint{ + Version: existingGraphCheckpointVersion, + ManifestSHA256: manifestHash, + CorpusSHA256: corpusHash, + RunSHA256: runHash, + Records: records, + } + raw, err := json.MarshalIndent(checkpoint, "", " ") + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + temporary, err := os.CreateTemp(filepath.Dir(path), ".graphbench-checkpoint-*") + if err != nil { + return err + } + temporaryName := temporary.Name() + defer os.Remove(temporaryName) + if _, err := temporary.Write(append(raw, '\n')); err != nil { + _ = temporary.Close() + return err + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return err + } + if err := temporary.Close(); err != nil { + return err + } + return os.Rename(temporaryName, path) +} + +// appendExistingGraphProgress appends one progress event as a durable JSON Lines record. +func appendExistingGraphProgress(path string, event ExistingGraphProgress) error { + if path == "" { + return nil + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + return err + } + defer file.Close() + event.At = time.Now().UTC() + return json.NewEncoder(file).Encode(event) +} + +// redactExistingGraphRecord removes raw parameters and Cypher text, pseudonymizes anchor values, and scrubs resolved IDs from diagnostics and plans before a live-run record is persisted. +func redactExistingGraphRecord(record *CaseResult, manifest ExistingGraphAnchorManifest, resolved map[string]graph.ID) { + if record == nil { + return + } + record.Params = nil + redacted := map[string]string{} + for parameter, name := range record.NodeParams { + anchor, found := manifest.Anchors[name] + if !found { + continue + } + seed := anchor.LogicalKey + if seed == "" { + seed = anchor.ContentSHA256 + } + digest := sha256.Sum256([]byte(seed)) + redacted[parameter] = "sha256:" + hex.EncodeToString(digest[:]) + } + record.NodeParams = redacted + record.NodeListParams = nil + record.Cypher = "" + record.ObservedRows = redactObservedRows(record.ObservedRows) + record.SQL = redactResolvedIDs(record.SQL, resolved) + for idx := range record.PostgresPlan { + record.PostgresPlan[idx] = redactResolvedIDs(record.PostgresPlan[idx], resolved) + } + if len(record.PostgresPlanJSON) > 0 { + record.PostgresPlanJSON = redactPlanJSON(record.PostgresPlanJSON, resolved) + } + record.Error = redactDiagnostic(record.Error) + for idx := range record.PostgresReferences { + reference := &record.PostgresReferences[idx] + reference.ObservedRows = redactObservedRows(reference.ObservedRows) + reference.SQL = redactResolvedIDs(reference.SQL, resolved) + for planIdx := range reference.PostgresPlan { + reference.PostgresPlan[planIdx] = redactResolvedIDs(reference.PostgresPlan[planIdx], resolved) + } + if len(reference.PostgresPlanJSON) > 0 { + reference.PostgresPlanJSON = redactPlanJSON(reference.PostgresPlanJSON, resolved) + } + } + if record.ExistingGraph != nil { + for idx := range record.ExistingGraph.Attempts { + record.ExistingGraph.Attempts[idx].Error = redactDiagnostic(record.ExistingGraph.Attempts[idx].Error) + } + } +} + +// redactObservedRows replaces each normalized observation with a SHA-256 digest for live-graph persistence. +func redactObservedRows(rows []string) []string { + for idx := range rows { + digest := sha256.Sum256([]byte(rows[idx])) + rows[idx] = "sha256:" + hex.EncodeToString(digest[:]) + } + return rows +} + +// redactDiagnostic replaces a nonempty diagnostic with its SHA-256 digest. +func redactDiagnostic(value string) string { + if value == "" { + return "" + } + digest := sha256.Sum256([]byte(value)) + return "sha256:" + hex.EncodeToString(digest[:]) +} + +// redactResolvedIDs replaces resolved physical node IDs and unmapped entity IDs with stable redaction markers. +func redactResolvedIDs(value string, resolved map[string]graph.ID) string { + for _, id := range resolved { + value = regexp.MustCompile(`\b`+regexp.QuoteMeta(fmt.Sprint(id))+`\b`).ReplaceAllString(value, "") + } + value = regexp.MustCompile(`unmapped-(node|edge|relationship):[0-9]+`).ReplaceAllString(value, "unmapped-$1:") + return value +} + +// redactPlanJSON recursively replaces resolved graph IDs in a PostgreSQL JSON plan so live-run artifacts cannot disclose dataset identifiers. +func redactPlanJSON(raw json.RawMessage, resolved map[string]graph.ID) json.RawMessage { + var value any + if err := json.Unmarshal(raw, &value); err != nil { + return nil + } + var redact func(any) any + redact = func(current any) any { + switch typed := current.(type) { + case string: + return redactResolvedIDs(typed, resolved) + case []any: + for idx := range typed { + typed[idx] = redact(typed[idx]) + } + case map[string]any: + for key := range typed { + typed[key] = redact(typed[key]) + } + } + return current + } + encoded, err := json.Marshal(redact(value)) + if err != nil { + return nil + } + return encoded +} + +// validateCompletedWorkloads rejects checkpoint entries that are unknown or bound to stale workload identities. +func validateCompletedWorkloads(completed map[string]string, corpus ScaleCorpus, fixture FixtureMetadata) error { + expectedKeys := map[string]struct{}{} + for _, testCase := range corpus.Cases { + if !testCase.Supports(ModePostgresSQL) { + continue + } + key := existingGraphCaseKey(ModePostgresSQL, testCase) + expectedKeys[key] = struct{}{} + checkpointWorkload, found := completed[key] + if !found { + continue + } + expected := newCaseResult(testCase, ModePostgresSQL, nil) + attachFixtureMetadata(&expected, fixture) + if checkpointWorkload == "" || checkpointWorkload != expected.WorkloadSHA256 { + return fmt.Errorf("existing-graph checkpoint workload identity does not match %s", key) + } + } + for key := range completed { + if _, found := expectedKeys[key]; !found { + return fmt.Errorf("existing-graph checkpoint contains unknown workload %s", key) + } + } + return nil +} + +// idMapForManifest builds an ID map from logical and redacted physical anchor identities. +func idMapForManifest(anchors map[string]graph.ID) opengraph.IDMap { + result := make(opengraph.IDMap, len(anchors)) + for name, id := range anchors { + result[name] = id + } + return result +} + +// scanCheckpointJSONL is deliberately strict: a truncated last line is not a +// completed record and therefore cannot be treated as resumable evidence. +func scanCheckpointJSONL(path string) error { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + var value map[string]any + if err := json.Unmarshal(scanner.Bytes(), &value); err != nil { + return err + } + } + return scanner.Err() +} diff --git a/cmd/graphbench/live_mode_test.go b/cmd/graphbench/live_mode_test.go new file mode 100644 index 00000000..23aa770e --- /dev/null +++ b/cmd/graphbench/live_mode_test.go @@ -0,0 +1,263 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "regexp" + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/testutil" + "github.com/stretchr/testify/require" +) + +// TestExistingGraphManifestCorpusSafetyAndRedaction verifies that live-graph mode rejects mutations and strips query, parameter, plan, row, reference, and error disclosures from artifacts. +func TestExistingGraphManifestCorpusSafetyAndRedaction(t *testing.T) { + manifest := ExistingGraphAnchorManifest{ + Version: 1, + Checksum: "manifest", + Anchors: map[string]ExistingGraphAnchor{ + "source": { + LogicalKey: "safe-source", + }, "target": { + LogicalKey: "safe-target", + }, + }, + } + readCase := ScaleCase{ + Name: "read", + Dataset: "live", + Category: "live", + Cypher: `MATCH (n) WHERE n.note = 'create is text' AND id(n) = $source RETURN n`, + NodeParams: map[string]string{"source": "source"}, + CandidateModes: []ExecutionMode{ModePostgresSQL}, + } + require.NoError(t, validateExistingGraphCorpus(ScaleCorpus{ + Cases: []ScaleCase{readCase}, + }, manifest)) + + writeCase := readCase + writeCase.Name = "write" + writeCase.Cypher = "MATCH (n) DELETE n" + require.ErrorContains(t, validateExistingGraphCorpus(ScaleCorpus{ + Cases: []ScaleCase{writeCase}, + }, manifest), "mutation keyword") + writeCase.Cypher = "MATCH (n) RETURN n" + writeCase.WriteScenario = &WriteScenario{} + require.ErrorContains(t, validateExistingGraphCorpus(ScaleCorpus{ + Cases: []ScaleCase{writeCase}, + }, manifest), "write_scenario") + + record := CaseResult{ + Cypher: readCase.Cypher, + Params: map[string]any{"source": 42}, + NodeParams: map[string]string{"source": "source"}, + ObservedRows: []string{"sensitive-property"}, + PostgresPlan: []string{"Index Cond: id = 42"}, + Error: "unmapped-node:77", + PostgresReferences: []PostgresReferenceResult{{ + ObservedRows: []string{"reference-sensitive-property"}, + }}, + ExistingGraph: &ExistingGraphRun{Attempts: []ExistingGraphAttempt{{ + Error: "attempt-sensitive-property 42", + }}}, + } + redactExistingGraphRecord(&record, manifest, map[string]graph.ID{"source": 42}) + require.Empty(t, record.Cypher) + require.Empty(t, record.Params) + require.Regexp(t, `^sha256:[0-9a-f]{64}$`, record.NodeParams["source"]) + require.NotContains(t, record.NodeParams["source"], "safe-source") + require.NotContains(t, record.ObservedRows[0], "sensitive-property") + require.NotContains(t, record.PostgresPlan[0], "42") + require.Regexp(t, `^sha256:[0-9a-f]{64}$`, record.Error) + require.NotContains(t, record.Error, "77") + require.Regexp(t, `^sha256:[0-9a-f]{64}$`, record.PostgresReferences[0].ObservedRows[0]) + require.NotContains(t, record.PostgresReferences[0].ObservedRows[0], "reference-sensitive-property") + require.Regexp(t, `^sha256:[0-9a-f]{64}$`, record.ExistingGraph.Attempts[0].Error) + require.NotContains(t, record.ExistingGraph.Attempts[0].Error, "attempt-sensitive-property") +} + +// TestExistingGraphManifestRequiresGraphAndLogicalContentIdentity verifies manifest checksums bind graph/content identity and that each anchor chooses exactly one complete logical or physical identity form. +func TestExistingGraphManifestRequiresGraphAndLogicalContentIdentity(t *testing.T) { + path := filepath.Join(t.TempDir(), "anchors.json") + valid := `{"version":1,"graph":"integration_test","content_identity":"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","anchors":{"source":{"logical_key":"safe-source"}}}` + require.NoError(t, os.WriteFile(path, []byte(valid), 0o600)) + manifest, err := loadExistingGraphAnchorManifest(path) + require.NoError(t, err) + require.Equal(t, "integration_test", manifest.Graph) + require.Regexp(t, `^[0-9a-f]{64}$`, manifest.Checksum) + + physical := `{"version":1,"graph":"integration_test","content_identity":"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","anchors":{"source":{"physical_id":42,"content_sha256":"sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"}}}` + require.NoError(t, os.WriteFile(path, []byte(physical), 0o600)) + manifest, err = loadExistingGraphAnchorManifest(path) + require.NoError(t, err) + require.Equal(t, int64(42), *manifest.Anchors["source"].PhysicalID) + + require.NoError(t, os.WriteFile(path, []byte(`{"version":1,"graph":"integration_test","content_identity":"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","anchors":{"source":{"physical_id":42}}}`), 0o600)) + _, err = loadExistingGraphAnchorManifest(path) + require.ErrorContains(t, err, "content_sha256") + + require.NoError(t, os.WriteFile(path, []byte(`{"version":1,"graph":"integration_test","content_identity":"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","anchors":{"source":{"logical_key":"safe-source","physical_id":42,"content_sha256":"sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"}}}`), 0o600)) + _, err = loadExistingGraphAnchorManifest(path) + require.ErrorContains(t, err, "exactly one") + + require.NoError(t, os.WriteFile(path, []byte(`{"version":1,"graph":"integration_test","anchors":{"source":{"logical_key":"safe-source"}}}`), 0o600)) + _, err = loadExistingGraphAnchorManifest(path) + require.ErrorContains(t, err, "content_identity") +} + +// TestPhysicalExistingGraphAnchorRedactionUsesContentIdentity verifies that artifacts replace a physical anchor ID with an opaque digest derived from its content identity. +func TestPhysicalExistingGraphAnchorRedactionUsesContentIdentity(t *testing.T) { + id := int64(42) + manifest := ExistingGraphAnchorManifest{ + Anchors: map[string]ExistingGraphAnchor{ + "source": { + PhysicalID: &id, + ContentSHA256: "sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789", + }, + }, + } + record := CaseResult{ + NodeParams: map[string]string{"source": "source"}, + } + redactExistingGraphRecord(&record, manifest, map[string]graph.ID{"source": graph.ID(id)}) + require.Regexp(t, `^sha256:[0-9a-f]{64}$`, record.NodeParams["source"]) + require.NotContains(t, record.NodeParams["source"], "42") +} + +// TestExistingGraphCheckpointIsIdentityBoundAndResumable verifies round-trip recovery only for matching manifest, corpus, and run identities and rejects duplicate completed records. +func TestExistingGraphCheckpointIsIdentityBoundAndResumable(t *testing.T) { + path := filepath.Join(t.TempDir(), "checkpoint.json") + records := []CaseResult{{ + Dataset: "live", + Name: "case", + WorkloadSHA256: "workload", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Environment: &RunEnvironment{ + ArtifactSchemaVersion: 2, + CorpusSHA256: "corpus", + RunIdentitySHA256: "run", + RunUUID: "run-uuid", + }, + }} + require.NoError(t, writeExistingGraphCheckpoint(path, "manifest", "corpus", "run", records)) + loaded, err := readExistingGraphCheckpoint(path, "manifest", "corpus", "run") + require.NoError(t, err) + require.Equal(t, records, loaded) + _, err = readExistingGraphCheckpoint(path, "other", "corpus", "run") + require.ErrorContains(t, err, "identity") + _, err = readExistingGraphCheckpoint(path, "manifest", "corpus", "other-run") + require.ErrorContains(t, err, "identity") + + raw, err := os.ReadFile(path) + require.NoError(t, err) + var checkpoint existingGraphCheckpoint + require.NoError(t, json.Unmarshal(raw, &checkpoint)) + require.Equal(t, existingGraphCheckpointVersion, checkpoint.Version) + + checkpoint.Records = append(checkpoint.Records, checkpoint.Records[0]) + raw, err = json.Marshal(checkpoint) + require.NoError(t, err) + require.NoError(t, os.WriteFile(path, raw, 0o600)) + _, err = readExistingGraphCheckpoint(path, "manifest", "corpus", "run") + require.ErrorContains(t, err, "duplicate record") +} + +// TestExistingGraphPlanRedactionPreservesJSONNumbers verifies that plan redaction replaces IDs inside text without corrupting numeric cardinality fields in the JSON document. +func TestExistingGraphPlanRedactionPreservesJSONNumbers(t *testing.T) { + raw := json.RawMessage(`[{"Plan":{"Plan Rows":42,"Index Cond":"id = 42"}}]`) + redacted := redactPlanJSON(raw, map[string]graph.ID{"source": 42}) + require.JSONEq(t, `[{"Plan":{"Plan Rows":42,"Index Cond":"id = "}}]`, string(redacted)) +} + +// TestExistingGraphProgressIsAppendOnlyJSONL verifies that successive progress events remain two independently parseable JSON Lines records. +func TestExistingGraphProgressIsAppendOnlyJSONL(t *testing.T) { + path := filepath.Join(t.TempDir(), "progress.jsonl") + require.NoError(t, appendExistingGraphProgress(path, ExistingGraphProgress{ + Stage: "case", + CaseKey: "one", + })) + require.NoError(t, appendExistingGraphProgress(path, ExistingGraphProgress{ + Stage: "plan", + CaseKey: "one", + })) + require.NoError(t, scanCheckpointJSONL(path)) + raw, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, 2, len(splitNonEmptyLines(string(raw)))) +} + +// TestCompleteGateRejectsAdaptiveExistingGraphArtifacts verifies that discovery-selected live-graph measurements cannot enter a complete performance gate. +func TestCompleteGateRejectsAdaptiveExistingGraphArtifacts(t *testing.T) { + records := []CaseResult{{ + ExistingGraph: &ExistingGraphRun{ + Adaptive: true, + }, + }} + require.ErrorContains(t, validatePerformanceArtifactSelections(records, records, false), "adaptive-discovery") +} + +// TestExistingGraphCorpusIdentityIsStable verifies that corpus identity is deterministic and changes when either query text or expected cardinality changes. +func TestExistingGraphCorpusIdentityIsStable(t *testing.T) { + zero := int64(0) + corpus := ScaleCorpus{ + Cases: []ScaleCase{{ + Name: "case", + Dataset: "live", + Category: "live", + Cypher: "RETURN 1", + Expected: ExpectedResult{ + RowCount: &zero, + }, + Params: testutil.Params{}, + CandidateModes: []ExecutionMode{ModePostgresSQL}, + }}, + } + require.Equal(t, corpusIdentity(corpus), corpusIdentity(corpus)) + changedQuery := corpus + changedQuery.Cases = append([]ScaleCase(nil), corpus.Cases...) + changedQuery.Cases[0].Cypher = "RETURN 2" + require.NotEqual(t, corpusIdentity(corpus), corpusIdentity(changedQuery)) + + changedExpected := corpus + changedExpected.Cases = append([]ScaleCase(nil), corpus.Cases...) + one := int64(1) + changedExpected.Cases[0].Expected.RowCount = &one + require.NotEqual(t, corpusIdentity(corpus), corpusIdentity(changedExpected)) +} + +// TestExistingGraphCompletedWorkloadsAreFixtureBound verifies that resume records are accepted only for known cases with the same fixture checksum and workload digest. +func TestExistingGraphCompletedWorkloadsAreFixtureBound(t *testing.T) { + corpus := ScaleCorpus{Cases: []ScaleCase{{ + Name: "case", + Dataset: "live", + Cypher: "RETURN 1", + CandidateModes: []ExecutionMode{ModePostgresSQL}, + }}} + fixture := FixtureMetadata{Dataset: "existing_graph", Checksum: "manifest:content:schema:index"} + expected := newCaseResult(corpus.Cases[0], ModePostgresSQL, nil) + attachFixtureMetadata(&expected, fixture) + completed := map[string]string{existingGraphCaseKey(ModePostgresSQL, corpus.Cases[0]): expected.WorkloadSHA256} + require.NoError(t, validateCompletedWorkloads(completed, corpus, fixture)) + + changedFixture := fixture + changedFixture.Checksum = "manifest:other-content:schema:index" + require.ErrorContains(t, validateCompletedWorkloads(completed, corpus, changedFixture), "workload identity") + require.ErrorContains(t, validateCompletedWorkloads(map[string]string{"postgres_sql/other/case": "digest"}, corpus, fixture), "unknown workload") +} + +// splitNonEmptyLines separates platform-independent line endings and discards empty records. +func splitNonEmptyLines(value string) []string { + var lines []string + for _, line := range regexp.MustCompile(`\r?\n`).Split(value, -1) { + if line != "" { + lines = append(lines, line) + } + } + return lines +} diff --git a/cmd/graphbench/main.go b/cmd/graphbench/main.go index bd18d1a3..625f5bb7 100644 --- a/cmd/graphbench/main.go +++ b/cmd/graphbench/main.go @@ -21,31 +21,291 @@ import ( "flag" "fmt" "io" + "math" "os" + "slices" + "strconv" "strings" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/databaseguard" + "github.com/specterops/dawgs/testutil" ) +// config contains graphbench command-line selections and safety settings. type config struct { - CorpusRoot string - DatasetDir string - Connection string - PGConnection string + // CorpusRoot locates scale-case and template declarations. + CorpusRoot string + // DatasetDir locates fixture datasets loaded for managed benchmark runs. + DatasetDir string + // Connection contains the backend connection string. + Connection string + // PGConnection contains the PostgreSQL connection string. + PGConnection string + // Neo4jConnection contains the Neo4j connection string. Neo4jConnection string - Modes []ExecutionMode - Iterations int - OutputJSONL string - Summary string - SummaryJSON string - Baseline string + // Modes lists backend execution modes requested for each benchmark round. + Modes []ExecutionMode + // Iterations records the number of measured iterations. + Iterations int + // WarmupIterations records the untimed iterations run before measurement. + WarmupIterations int + // Round identifies the measurement round. + Round int + // Block identifies the measurement block used to control carryover effects. + Block int + // Arm identifies the measurement arm that produced the sample. + Arm string + // ArmOrder records the arm's position within its balanced measurement block. + ArmOrder int + // RunUUID supplies an optional stable identity shared by every artifact in one run series. + RunUUID string + // Cases lists exact case names requested by the user. + Cases []string + // Datasets lists exact dataset selectors supplied by the user. + Datasets []string + // Categories lists workload categories used to filter the corpus. + Categories []string + // Tags lists exact tag selectors supplied by the user. + Tags []string + // OutputJSONL selects the benchmark-result JSON Lines destination. + OutputJSONL string + // AppendJSONL selects append-safe JSON Lines output instead of replacing the artifact. + AppendJSONL bool + // Summary selects the Markdown benchmark-summary destination. + Summary string + // SummaryJSON selects the JSON summary destination. + SummaryJSON string + // Baseline identifies the baseline version or result used for comparison. + Baseline string + // DAWGSVersion records the DAWGS source version attached to artifact provenance. + DAWGSVersion string + // GateBaseline selects the baseline JSON Lines artifact for performance gating. + GateBaseline string + // GateCandidate selects the candidate JSON Lines artifact for performance gating. + GateCandidate string + // GateOutput selects the performance-gate JSON report destination. + GateOutput string + // GateAA selects the host A/A resolution report required by production performance gating. + GateAA string + // GateSeed controls deterministic performance-gate bootstrap resampling. + GateSeed int64 + // Confidence sets the confidence level used for statistical intervals. + Confidence float64 + // Regression sets the largest candidate-to-baseline median ratio accepted by the gate. + Regression float64 + // GateTargets lists exact case names subject to performance gating. + GateTargets []string + // MaterialityRatio sets the relative change required before a difference is material. + MaterialityRatio float64 + // MaterialityAbsolute sets the absolute duration change required before a difference is material. + MaterialityAbsolute time.Duration + // DestructiveLock selects the lock-file path that serializes destructive runs. + DestructiveLock string + // AAArtifacts select one or more benchmark record files used to estimate + // within-arm noise. Repeating the input lets independently appended A/A arms + // remain immutable while the reporter validates them as one logical cohort. + AAArtifacts []string + // AAOutput selects the A/A resolution report destination. + AAOutput string + // ReferenceClosureArtifact selects benchmark records used for production-to-reference closure analysis. + ReferenceClosureArtifact string + // ReferenceClosureOutput selects the reference-closure report destination. + ReferenceClosureOutput string + // ReferenceClosureArm selects the independent reference arm compared with production. + ReferenceClosureArm string + // ReferencePairArtifact selects benchmark records containing the two reference arms to compare. + ReferencePairArtifact string + // ReferencePairOutput selects the paired-reference report destination. + ReferencePairOutput string + // ReferencePairBaseline selects the reference arm treated as the paired baseline. + ReferencePairBaseline string + // ReferencePairCandidate selects the reference arm compared with the paired baseline. + ReferencePairCandidate string + // ReferencePairProtocol selects confirmation or discovery sample requirements for paired references. + ReferencePairProtocol string + // ReferenceTournamentArtifact selects records containing a predeclared three- or five-arm tournament. + ReferenceTournamentArtifact string + // ReferenceTournamentOutput selects the tournament report destination. + ReferenceTournamentOutput string + // ReferenceTournamentArms lists tournament arms with the incumbent first. + ReferenceTournamentArms []string + // ReferenceTournamentProtocol selects confirmation or discovery tournament requirements. + ReferenceTournamentProtocol string + // PoolSize sets the database connection-pool size. + PoolSize int + // Concurrency lists opt-in worker counts for PostgreSQL concurrency measurements. + Concurrency []int + // SessionMemoryCeilingBytes sets the per-session memory ceiling in bytes. + SessionMemoryCeilingBytes int64 + // PoolMemoryCeilingBytes sets the aggregate pool memory ceiling in bytes. + PoolMemoryCeilingBytes int64 + // PostgresReferences enables independent PostgreSQL reference-arm measurement and persistence. + PostgresReferences bool + // PostgresReferenceArms lists independent PostgreSQL reference arms selected for measurement. + PostgresReferenceArms []string + // PostgresForceShortest selects a forced shortest-path executor for diagnostic runs. + PostgresForceShortest string + // PostgresProductionManifest selects a provisional version-2 manifest used + // to measure an exact guarded production statement before evidence closure. + PostgresProductionManifest string + // PostgresRepeatableRead measures the incumbent under the same stable + // snapshot contract required for guarded candidate admission. + PostgresRepeatableRead bool + // PostgresForceExpansion selects a forced expansion search strategy for diagnostic runs. + PostgresForceExpansion string + // PostgresTraversalTelemetry selects off, summary, or an untimed diagnostic replay. + PostgresTraversalTelemetry string + // PostgresExpansionOrientationShadow executes the incumbent while recording the orientation policy's SQL-visible choice. + PostgresExpansionOrientationShadow bool + // PostgresExpansionOrientationTournament executes the guarded selector's + // chosen arm in the same statement. + PostgresExpansionOrientationTournament bool + // PostgresExpansionOrientationPolicy selects an immutable tool-only + // orientation formula. Empty preserves orientation-probe-v1. + PostgresExpansionOrientationPolicy string + // ConfirmLeft selects the left artifact used for paired confirmation. + ConfirmLeft string + // ConfirmRight selects the right artifact used for paired confirmation. + ConfirmRight string + // ConfirmAA selects the A/A noise report used to classify confirmation deltas. + ConfirmAA string + // ConfirmOutput selects the paired confirmation report destination. + ConfirmOutput string + // ConfirmCases lists exact case names included in paired confirmation. + ConfirmCases []string + // DiagnosticGate marks output as diagnostic and therefore ineligible for a complete release-gate pass. + DiagnosticGate bool + // BundleDir selects the directory that receives portable artifacts and source provenance. + BundleDir string + // BundleEvidence lists named auxiliary artifacts copied into a newly captured bundle. + BundleEvidence []CaptureBundleEvidenceInput + // BundleVerify selects a portable bundle directory for standalone validation. + BundleVerify string + // BundleVerifyOutput selects the standalone bundle-verification JSON destination. + BundleVerifyOutput string + // BundleRequireClean rejects otherwise valid bundles captured from a dirty source tree. + BundleRequireClean bool + // PromotionManifest selects a complete evidence-closure manifest for standalone verification. + PromotionManifest string + // PromotionManifestOutput selects the verification report destination. + PromotionManifestOutput string + // PromotionBindManifest supplies the provisional manifest whose immutable + // identity is attached to one generated evidence report. + PromotionBindManifest string + // PromotionBindRole names the evidence role being bound. + PromotionBindRole string + // PromotionBindInput and PromotionBindOutput select the unbound and bound reports. + PromotionBindInput string + PromotionBindOutput string + // BuildCommand records the reproducible command used to build the benchmark executable. + BuildCommand string + // ExistingGraph selects read-only execution against a pre-existing graph. + ExistingGraph bool + // AnchorManifest selects the live-graph anchor manifest to validate and redact. + AnchorManifest string + // Checkpoint selects the persisted live-graph completion checkpoint. + Checkpoint string + // Resume allows live-graph execution to skip checkpointed workloads with matching identities. + Resume bool + // Progress selects the append-only live-graph progress JSON Lines destination. + Progress string + // Discovery enables adaptive live-graph discovery instead of the fixed confirmation protocol. + Discovery bool + // TimeoutClasses lists increasing per-attempt deadlines for adaptive live-graph discovery. + TimeoutClasses []time.Duration + // DiscoverySampleFloor sets the minimum live-graph samples required before adaptive discovery may stop. + DiscoverySampleFloor int + // ResourceArtifact selects benchmark records evaluated against plan-resource limits. + ResourceArtifact string + // ResourceOutput selects the resource-gate JSON report destination. + ResourceOutput string + // BackendDeltaArtifact selects records used for descriptive PostgreSQL-to-Neo4j comparison. + BackendDeltaArtifact string + // BackendDeltaOutput selects the cross-backend delta report destination. + BackendDeltaOutput string + // ExpandIntoArtifact selects records used to build the fixed-one-hop three-arm study report. + ExpandIntoArtifact string + // ExpandIntoOutput selects the ExpandInto study JSON destination. + ExpandIntoOutput string + // ExpandIntoProtocol selects discovery or confirmation evidence requirements. + ExpandIntoProtocol string + // OrientationShadowArtifact selects true-shadow orientation records. + OrientationShadowArtifact string + // OrientationIncumbentArtifact selects matched exact incumbent records. + OrientationIncumbentArtifact string + // OrientationReverseArtifact selects matched exact forced-reverse records. + OrientationReverseArtifact string + // OrientationAA selects host A/A timing resolution for selector regret. + OrientationAA string + // OrientationOutput selects the selector-regret and probe-overhead report destination. + OrientationOutput string + // OrientationProtocol selects discovery or confirmation evidence requirements. + OrientationProtocol string + // OrientationV2ShadowArtifact selects orientation-probe-v2 shadow records. + OrientationV2ShadowArtifact string + // OrientationV2IncumbentArtifact selects matched exact forward records. + OrientationV2IncumbentArtifact string + // OrientationV2ReverseArtifact selects matched exact reverse records. + OrientationV2ReverseArtifact string + // OrientationV2GuardedArtifact selects actual guarded dual-arm records. + OrientationV2GuardedArtifact string + // OrientationV2AA selects the host A/A timing-resolution report. + OrientationV2AA string + // OrientationV2Freeze binds confirmation to the preregistered discovery identity. + OrientationV2Freeze string + // OrientationV2DiscoveryReport supplies the checksummed training-only report bound by the freeze. + OrientationV2DiscoveryReport string + // OrientationV2FreezeOutput writes the preregistered identity after training-only discovery. + OrientationV2FreezeOutput string + // OrientationV2Output selects the four-arm qualification report destination. + OrientationV2Output string + // OrientationV2Protocol selects discovery or confirmation evidence requirements. + OrientationV2Protocol string + // SPI1BaselineArtifact selects exact S4 records for the staged inbound-I1 study. + SPI1BaselineArtifact string + // SPI1CandidateArtifact selects guarded canonical-I1 records for the staged study. + SPI1CandidateArtifact string + // SPI1ResourceReport supplies the candidate artifact's checksummed resource gate. + SPI1ResourceReport string + // SPI1Freeze binds confirmation reporting or holdout capture to training-only discovery. + SPI1Freeze string + // SPI1DiscoveryReport supplies the checksummed training-only report bound by the freeze. + SPI1DiscoveryReport string + // SPI1TrainingBaseline supplies the exact S4 training evidence named by the freeze. + SPI1TrainingBaseline string + // SPI1TrainingCandidate supplies the exact I1 training evidence named by the freeze. + SPI1TrainingCandidate string + // SPI1TrainingResource supplies the exact training resource report named by the freeze. + SPI1TrainingResource string + // SPI1FreezeOutput writes the training-only staged-study freeze manifest. + SPI1FreezeOutput string + // SPI1Output selects the staged S4-to-I1 qualification report destination. + SPI1Output string + // SPI1Protocol selects discovery or confirmation evidence requirements. + SPI1Protocol string } +// parseConfig parses graphbench flags and rejects unsafe or incomplete workflow combinations. func parseConfig(args []string, env func(string) string) (config, error) { flags := flag.NewFlagSet("graphbench", flag.ContinueOnError) flags.SetOutput(io.Discard) var ( - cfg config - rawModes string + cfg config + rawModes string + rawGateTargets string + rawConcurrency string + rawCases string + rawDatasets string + rawCategories string + rawTags string + rawConfirmCases string + rawReferenceArms string + rawTournamentArms string + rawTimeoutClasses string + rawBundleEvidence []string ) flags.StringVar(&cfg.CorpusRoot, "corpus-root", "benchmark/testdata/scale", "scale corpus root") @@ -55,27 +315,653 @@ func parseConfig(args []string, env func(string) string) (config, error) { flags.StringVar(&cfg.Neo4jConnection, "neo4j-connection", env("NEO4J_CONNECTION_STRING"), "Neo4j connection string") flags.StringVar(&rawModes, "modes", string(ModePostgresSQL), "comma-separated execution modes") flags.IntVar(&cfg.Iterations, "iterations", 3, "timed iterations per case") + flags.IntVar(&cfg.WarmupIterations, "warmup-iterations", 1, "fixed untimed warmup iterations per case") + flags.IntVar(&cfg.Round, "round", 1, "independent benchmark round identifier") + flags.IntVar(&cfg.Block, "block", 1, "matched benchmark block identifier") + flags.StringVar(&cfg.Arm, "arm", "unlabeled", "matched benchmark arm label") + flags.IntVar(&cfg.ArmOrder, "arm-order", 0, "one-based execution order inside the matched block (0 when unpaired)") + flags.StringVar(&cfg.RunUUID, "run-uuid", "", "run-series UUID (generated when empty)") + flags.StringVar(&rawCases, "cases", "", "comma-separated exact case names") + flags.StringVar(&rawDatasets, "datasets", "", "comma-separated exact dataset names") + flags.StringVar(&rawCategories, "categories", "", "comma-separated exact category names") + flags.StringVar(&rawTags, "tags", "", "comma-separated exact case tags") flags.StringVar(&cfg.OutputJSONL, "jsonl-output", "", "JSONL output path (default: stdout)") + flags.BoolVar(&cfg.AppendJSONL, "append-jsonl", false, "append a validated round to an existing JSONL run-series artifact") flags.StringVar(&cfg.Summary, "summary", "", "markdown summary output path") flags.StringVar(&cfg.SummaryJSON, "summary-json", "", "JSON summary output path") flags.StringVar(&cfg.Baseline, "baseline", "", "previous JSONL output for baseline comparison") + flags.StringVar(&cfg.DAWGSVersion, "dawgs-version", "", "DAWGS source version (auto-detected when empty)") + flags.StringVar(&cfg.GateBaseline, "gate-baseline", "", "baseline JSONL artifact for comparison-only mode") + flags.StringVar(&cfg.GateCandidate, "gate-candidate", "", "candidate JSONL artifact for comparison-only mode") + flags.StringVar(&cfg.GateOutput, "gate-output", "", "performance-gate JSON output path (default: stdout)") + flags.StringVar(&cfg.GateAA, "gate-aa", "", "host A/A resolution report required for production performance gating") + flags.Int64Var(&cfg.GateSeed, "seed", 1, "deterministic bootstrap seed") + flags.Float64Var(&cfg.Confidence, "confidence-level", defaultConfidenceLevel, "bootstrap confidence level") + flags.Float64Var(&cfg.Regression, "regression-threshold", minimumTimingNoiseRatio, "minimum allowed comparable-case regression ratio before host A/A noise") + flags.StringVar(&rawGateTargets, "gate-targets", "", "comma-separated PostgreSQL case names expected to improve materially") + flags.Float64Var(&cfg.MaterialityRatio, "materiality-ratio", 0.95, "target median-ratio upper bound") + flags.DurationVar(&cfg.MaterialityAbsolute, "materiality-absolute", 100*time.Microsecond, "target median-saving lower bound") + flags.StringVar(&cfg.DestructiveLock, "destructive-lock", ".coverage/graphbench.lock", "local lock file guarding destructive fixture reloads") + flags.Func("aa-artifact", "JSONL artifact used to calculate baseline A/A measurement resolution (repeat for separately captured arms)", func(value string) error { + value = strings.TrimSpace(value) + if value == "" { + return fmt.Errorf("aa-artifact path must not be empty") + } + cfg.AAArtifacts = append(cfg.AAArtifacts, value) + return nil + }) + flags.StringVar(&cfg.AAOutput, "aa-output", "", "A/A measurement-resolution JSON output path (default: stdout)") + flags.StringVar(&cfg.ReferenceClosureArtifact, "reference-closure-artifact", "", "JSONL artifact containing matched production raw-pgx and PostgreSQL reference samples") + flags.StringVar(&cfg.ReferenceClosureOutput, "reference-closure-output", "", "production/reference closure JSON output path (default: stdout)") + flags.StringVar(&cfg.ReferenceClosureArm, "reference-closure-arm", "s3_unidirectional_trail_cte", "PostgreSQL full-comparator reference arm") + flags.StringVar(&cfg.ReferencePairArtifact, "reference-pair-artifact", "", "JSONL artifact containing two matched PostgreSQL reference arms") + flags.StringVar(&cfg.ReferencePairOutput, "reference-pair-output", "", "matched PostgreSQL reference-pair JSON output path (default: stdout)") + flags.StringVar(&cfg.ReferencePairBaseline, "reference-pair-baseline", "", "baseline PostgreSQL reference arm") + flags.StringVar(&cfg.ReferencePairCandidate, "reference-pair-candidate", "", "candidate PostgreSQL reference arm") + flags.StringVar(&cfg.ReferencePairProtocol, "reference-pair-protocol", referencePairProtocolConfirmation, "reference-pair report protocol (confirmation or discovery)") + flags.StringVar(&cfg.ReferenceTournamentArtifact, "reference-tournament-artifact", "", "JSONL artifact containing a predeclared three- or five-arm PostgreSQL reference tournament") + flags.StringVar(&cfg.ReferenceTournamentOutput, "reference-tournament-output", "", "reference tournament JSON output path (default: stdout)") + flags.StringVar(&rawTournamentArms, "reference-tournament-arms", "", "comma-separated tournament arms with the incumbent first") + flags.StringVar(&cfg.ReferenceTournamentProtocol, "reference-tournament-protocol", referencePairProtocolConfirmation, "reference tournament protocol (confirmation or discovery)") + flags.IntVar(&cfg.PoolSize, "pool-size", 1, "PostgreSQL physical pool size") + flags.StringVar(&rawConcurrency, "concurrency", "", "comma-separated opt-in PostgreSQL concurrency smoke levels") + flags.Int64Var(&cfg.SessionMemoryCeilingBytes, "session-memory-ceiling-bytes", 0, "declared maximum performance workspace bytes per PostgreSQL session") + flags.Int64Var(&cfg.PoolMemoryCeilingBytes, "pool-memory-ceiling-bytes", 0, "declared maximum performance workspace bytes for the complete PostgreSQL pool") + flags.BoolVar(&cfg.PostgresReferences, "postgres-references", false, "capture C1 PostgreSQL component floors and full-query references") + flags.StringVar(&rawReferenceArms, "postgres-reference-arms", "", "comma-separated PostgreSQL reference arms (default: all applicable arms)") + flags.StringVar(&cfg.PostgresForceShortest, "postgres-force-shortest-executor", "", "tool-only forced PostgreSQL shortest executor (supported: SP-S0, SP-S0-DIRECT, SP-S3-U-D, SP-S3-U-E+MAT-M0, SP-S4-C-D, SP-S4-C-WE+MAT-M0, SP-I1-C-D, SP-I1-U-E+MAT-M0, SP-I1-C-WE+MAT-M0, SP-B1-C-ALT-NODE-D, SP-B1-C-ALT-NODE-WE+MAT-M0, SP-B2-C-MIN-LEVEL-D, SP-B2-C-MIN-LEVEL-WE+MAT-M0, ASP-A1-DAG, ASP-I1-U-DAG+MAT-M0, ASP-B1-DAG-ALT-NODE, ASP-B2-DAG-MIN-LEVEL)") + flags.StringVar(&cfg.PostgresProductionManifest, "postgres-production-manifest", "", "provisional version-2 manifest for exact guarded PostgreSQL candidate measurement") + flags.BoolVar(&cfg.PostgresRepeatableRead, "postgres-repeatable-read", false, "measure PostgreSQL under an explicit Repeatable Read transaction") + flags.StringVar(&cfg.PostgresForceExpansion, "postgres-force-expansion-search", "", "tool-only forced PostgreSQL expansion search (supported: EXPANSION-SUFFIX-SEEDED-REVERSE, EXPANSION-ENDPOINT-SEEDED-REVERSE)") + flags.StringVar(&cfg.PostgresTraversalTelemetry, "postgres-traversal-telemetry", postgresTraversalTelemetryOff, "PostgreSQL traversal telemetry level (off, summary, or diagnostic); replays run outside timed samples") + flags.BoolVar(&cfg.PostgresExpansionOrientationShadow, "postgres-expansion-orientation-shadow", false, "tool-only orientation-probe shadow mode; executes only the exact incumbent traversal arm") + flags.BoolVar(&cfg.PostgresExpansionOrientationTournament, "postgres-expansion-orientation-tournament", false, "tool-only guarded orientation-probe mode; executes the selected exact arm") + flags.StringVar(&cfg.PostgresExpansionOrientationPolicy, "postgres-expansion-orientation-policy", "", "tool-only immutable orientation policy (orientation-probe-v1 or orientation-probe-v2; default: v1)") + flags.StringVar(&cfg.ConfirmLeft, "confirm-left", "", "left JSONL artifact for paired confirmation mode") + flags.StringVar(&cfg.ConfirmRight, "confirm-right", "", "right JSONL artifact for paired confirmation mode") + flags.StringVar(&cfg.ConfirmAA, "confirm-aa", "", "optional block/reload A/A resolution report") + flags.StringVar(&cfg.ConfirmOutput, "confirm-output", "", "paired confirmation JSON output path (default: stdout)") + flags.StringVar(&rawConfirmCases, "confirm-cases", "", "comma-separated exact primary names for paired confirmation") + flags.BoolVar(&cfg.DiagnosticGate, "diagnostic-gate", false, "allow comparison of matching diagnostic-only subsets") + flags.StringVar(&cfg.BundleDir, "bundle-dir", "", "write a reconstructible capture bundle to this directory") + flags.Func("bundle-evidence", "named auxiliary bundle artifact as name=path (repeatable)", func(value string) error { + rawBundleEvidence = append(rawBundleEvidence, value) + return nil + }) + flags.StringVar(&cfg.BundleVerify, "bundle-verify", "", "standalone verification of a capture bundle directory") + flags.StringVar(&cfg.BundleVerifyOutput, "bundle-verify-output", "", "capture-bundle verification JSON output path (default: stdout)") + flags.BoolVar(&cfg.BundleRequireClean, "bundle-require-clean", false, "require standalone bundle verification to prove a clean source capture") + flags.StringVar(&cfg.PromotionManifest, "promotion-manifest", "", "verify a candidate promotion manifest and every bound evidence report") + flags.StringVar(&cfg.PromotionManifestOutput, "promotion-manifest-output", "", "promotion-manifest verification JSON destination (default: stdout)") + flags.StringVar(&cfg.PromotionBindManifest, "promotion-bind-manifest", "", "provisional promotion manifest supplying report identity") + flags.StringVar(&cfg.PromotionBindRole, "promotion-bind-role", "", "promotion evidence role to bind") + flags.StringVar(&cfg.PromotionBindInput, "promotion-bind-input", "", "unbound promotion evidence report") + flags.StringVar(&cfg.PromotionBindOutput, "promotion-bind-output", "", "identity-bound promotion evidence report") + flags.StringVar(&cfg.BuildCommand, "build-command", "go build -trimpath ./cmd/graphbench", "reproducible build command recorded in bundles") + flags.BoolVar(&cfg.ExistingGraph, "existing-graph", false, "run non-mutating PostgreSQL cases against an existing graph in read-write sessions without schema, load, clear, vacuum, or persistent writes") + flags.StringVar(&cfg.AnchorManifest, "anchor-manifest", "", "versioned logical-key anchor manifest for existing-graph mode") + flags.StringVar(&cfg.Checkpoint, "checkpoint", "", "atomic existing-graph checkpoint path") + flags.BoolVar(&cfg.Resume, "resume", false, "resume completed records from the matching existing-graph checkpoint") + flags.StringVar(&cfg.Progress, "progress", "", "append-only existing-graph progress JSONL path") + flags.BoolVar(&cfg.Discovery, "discovery", false, "label the run adaptive discovery rather than fixed confirmation") + flags.StringVar(&rawTimeoutClasses, "timeout-classes", "", "comma-separated predeclared per-case timeout classes used by discovery") + flags.IntVar(&cfg.DiscoverySampleFloor, "discovery-sample-floor", 1, "minimum measured samples after adaptive discovery reduction") + flags.StringVar(&cfg.ResourceArtifact, "resource-artifact", "", "JSONL artifact used to calculate the state/resource gate") + flags.StringVar(&cfg.ResourceOutput, "resource-output", "", "state/resource gate JSON output path (default: stdout)") + flags.StringVar(&cfg.BackendDeltaArtifact, "backend-delta-artifact", "", "JSONL artifact used for descriptive matched PostgreSQL/Neo4j deltas") + flags.StringVar(&cfg.BackendDeltaOutput, "backend-delta-output", "", "descriptive backend-delta JSON output path (default: stdout)") + flags.StringVar(&cfg.ExpandIntoArtifact, "expand-into-artifact", "", "JSONL artifact used to build the fixed-one-hop three-arm study report") + flags.StringVar(&cfg.ExpandIntoOutput, "expand-into-output", "", "ExpandInto study JSON output path (default: stdout)") + flags.StringVar(&cfg.ExpandIntoProtocol, "expand-into-protocol", referencePairProtocolDiscovery, "ExpandInto study protocol (discovery or confirmation)") + flags.StringVar(&cfg.OrientationShadowArtifact, "orientation-shadow-artifact", "", "true-shadow orientation JSONL artifact") + flags.StringVar(&cfg.OrientationIncumbentArtifact, "orientation-incumbent-artifact", "", "matched exact incumbent orientation JSONL artifact") + flags.StringVar(&cfg.OrientationReverseArtifact, "orientation-reverse-artifact", "", "matched exact forced-reverse orientation JSONL artifact") + flags.StringVar(&cfg.OrientationAA, "orientation-aa", "", "host A/A report used by orientation selector-regret analysis") + flags.StringVar(&cfg.OrientationOutput, "orientation-output", "", "orientation selector-regret and probe-overhead JSON output path (default: stdout)") + flags.StringVar(&cfg.OrientationProtocol, "orientation-protocol", referencePairProtocolConfirmation, "orientation report protocol (discovery or confirmation)") + flags.StringVar(&cfg.OrientationV2ShadowArtifact, "orientation-v2-shadow-artifact", "", "orientation-probe-v2 shadow JSONL artifact") + flags.StringVar(&cfg.OrientationV2IncumbentArtifact, "orientation-v2-incumbent-artifact", "", "matched exact forward orientation-v2 JSONL artifact") + flags.StringVar(&cfg.OrientationV2ReverseArtifact, "orientation-v2-reverse-artifact", "", "matched exact forced-reverse orientation-v2 JSONL artifact") + flags.StringVar(&cfg.OrientationV2GuardedArtifact, "orientation-v2-guarded-artifact", "", "matched actual guarded orientation-v2 JSONL artifact") + flags.StringVar(&cfg.OrientationV2AA, "orientation-v2-aa", "", "host A/A report used by orientation-v2 qualification") + flags.StringVar(&cfg.OrientationV2Freeze, "orientation-v2-freeze", "", "discovery freeze manifest required by orientation-v2 confirmation") + flags.StringVar(&cfg.OrientationV2DiscoveryReport, "orientation-v2-discovery-report", "", "training-only discovery report bound by the orientation-v2 freeze") + flags.StringVar(&cfg.OrientationV2FreezeOutput, "orientation-v2-freeze-output", "", "write the training-only orientation-v2 discovery freeze manifest") + flags.StringVar(&cfg.OrientationV2Output, "orientation-v2-output", "", "four-arm orientation-v2 qualification JSON output path (default: stdout)") + flags.StringVar(&cfg.OrientationV2Protocol, "orientation-v2-protocol", referencePairProtocolConfirmation, "orientation-v2 report protocol (discovery or confirmation)") + flags.StringVar(&cfg.SPI1BaselineArtifact, "sp-i1-baseline-artifact", "", "matched exact S4 JSONL artifact for staged inbound-I1 qualification") + flags.StringVar(&cfg.SPI1CandidateArtifact, "sp-i1-candidate-artifact", "", "matched guarded canonical-I1 JSONL artifact for staged inbound-I1 qualification") + flags.StringVar(&cfg.SPI1ResourceReport, "sp-i1-resource-report", "", "resource-gate report bound to the staged canonical-I1 artifact") + flags.StringVar(&cfg.SPI1Freeze, "sp-i1-freeze", "", "training-only freeze required by SP-I1 confirmation reporting and holdout capture") + flags.StringVar(&cfg.SPI1DiscoveryReport, "sp-i1-discovery-report", "", "training-only discovery report bound by the SP-I1 freeze") + flags.StringVar(&cfg.SPI1TrainingBaseline, "sp-i1-training-baseline-artifact", "", "exact S4 training artifact required to recompute a frozen SP-I1 discovery") + flags.StringVar(&cfg.SPI1TrainingCandidate, "sp-i1-training-candidate-artifact", "", "exact canonical-I1 training artifact required to recompute a frozen SP-I1 discovery") + flags.StringVar(&cfg.SPI1TrainingResource, "sp-i1-training-resource-report", "", "exact training resource report required to recompute a frozen SP-I1 discovery") + flags.StringVar(&cfg.SPI1FreezeOutput, "sp-i1-freeze-output", "", "write the staged SP-I1 training-only freeze manifest") + flags.StringVar(&cfg.SPI1Output, "sp-i1-output", "", "staged S4-to-I1 qualification JSON output path") + flags.StringVar(&cfg.SPI1Protocol, "sp-i1-protocol", referencePairProtocolConfirmation, "staged SP-I1 report protocol (discovery or confirmation)") if err := flags.Parse(args); err != nil { return config{}, err } + var err error + if cfg.BundleEvidence, err = parseCaptureBundleEvidenceInputs(rawBundleEvidence); err != nil { + return config{}, err + } if cfg.Iterations < 1 { return config{}, fmt.Errorf("iterations must be at least 1") } + if cfg.WarmupIterations < 0 { + return config{}, fmt.Errorf("warmup-iterations must not be negative") + } + if cfg.Round < 1 { + return config{}, fmt.Errorf("round must be at least 1") + } + if cfg.Block < 1 { + return config{}, fmt.Errorf("block must be at least 1") + } + if strings.TrimSpace(cfg.Arm) == "" { + return config{}, fmt.Errorf("arm must not be empty") + } + if cfg.ArmOrder < 0 { + return config{}, fmt.Errorf("arm-order must not be negative") + } + if cfg.PoolSize < 1 { + return config{}, fmt.Errorf("pool-size must be at least 1") + } + if cfg.PostgresTraversalTelemetry != postgresTraversalTelemetryOff && + cfg.PostgresTraversalTelemetry != postgresTraversalTelemetrySummary && + cfg.PostgresTraversalTelemetry != postgresTraversalTelemetryDiagnostic { + return config{}, fmt.Errorf("postgres-traversal-telemetry must be off, summary, or diagnostic") + } + if cfg.PostgresTraversalTelemetry != postgresTraversalTelemetryOff && cfg.PoolSize != 1 { + return config{}, fmt.Errorf("PostgreSQL traversal telemetry requires pool-size 1 to preserve connection identity") + } + if cfg.SessionMemoryCeilingBytes < 0 || cfg.PoolMemoryCeilingBytes < 0 { + return config{}, fmt.Errorf("memory ceilings must not be negative") + } + if cfg.SessionMemoryCeilingBytes > 0 && cfg.PoolMemoryCeilingBytes > 0 && cfg.SessionMemoryCeilingBytes*int64(cfg.PoolSize) > cfg.PoolMemoryCeilingBytes { + return config{}, fmt.Errorf("session memory ceiling times pool size exceeds pool memory ceiling") + } + for _, raw := range strings.Split(rawConcurrency, ",") { + if raw = strings.TrimSpace(raw); raw == "" { + continue + } + level, err := strconv.Atoi(raw) + if err != nil || level < 1 { + return config{}, fmt.Errorf("concurrency levels must be positive integers, got %q", raw) + } + if !slices.Contains(cfg.Concurrency, level) { + cfg.Concurrency = append(cfg.Concurrency, level) + } + } + if (cfg.GateBaseline == "") != (cfg.GateCandidate == "") { + return config{}, fmt.Errorf("gate-baseline and gate-candidate must be supplied together") + } + if cfg.GateAA != "" && cfg.GateBaseline == "" { + return config{}, fmt.Errorf("gate-aa requires gate-baseline and gate-candidate") + } + if (cfg.ConfirmLeft == "") != (cfg.ConfirmRight == "") { + return config{}, fmt.Errorf("confirm-left and confirm-right must be supplied together") + } + if cfg.ConfirmAA != "" && cfg.ConfirmLeft == "" { + return config{}, fmt.Errorf("confirm-aa requires confirm-left and confirm-right") + } + if cfg.ReferenceClosureOutput != "" && cfg.ReferenceClosureArtifact == "" { + return config{}, fmt.Errorf("reference-closure-output requires reference-closure-artifact") + } + if cfg.ReferencePairOutput != "" && cfg.ReferencePairArtifact == "" { + return config{}, fmt.Errorf("reference-pair-output requires reference-pair-artifact") + } + if cfg.ReferencePairArtifact != "" && (cfg.ReferencePairBaseline == "" || cfg.ReferencePairCandidate == "") { + return config{}, fmt.Errorf("reference-pair-artifact requires baseline and candidate arms") + } + if cfg.ReferencePairBaseline != "" && cfg.ReferencePairBaseline == cfg.ReferencePairCandidate { + return config{}, fmt.Errorf("reference-pair baseline and candidate must differ") + } + if cfg.ReferenceTournamentOutput != "" && cfg.ReferenceTournamentArtifact == "" { + return config{}, fmt.Errorf("reference-tournament-output requires reference-tournament-artifact") + } + if cfg.ReferenceTournamentArtifact != "" && len(cfg.ReferenceTournamentArms) == 0 && strings.TrimSpace(rawTournamentArms) == "" { + return config{}, fmt.Errorf("reference-tournament-artifact requires reference-tournament-arms") + } + if cfg.ReferenceTournamentProtocol != referencePairProtocolDiscovery && cfg.ReferenceTournamentProtocol != referencePairProtocolConfirmation { + return config{}, fmt.Errorf("reference-tournament-protocol must be discovery or confirmation") + } + if cfg.BundleVerifyOutput != "" && cfg.BundleVerify == "" { + return config{}, fmt.Errorf("bundle-verify-output requires bundle-verify") + } + if cfg.PromotionManifestOutput != "" && cfg.PromotionManifest == "" { + return config{}, fmt.Errorf("promotion-manifest-output requires promotion-manifest") + } + promotionBindConfigured := cfg.PromotionBindManifest != "" || cfg.PromotionBindRole != "" || cfg.PromotionBindInput != "" || cfg.PromotionBindOutput != "" + if promotionBindConfigured && (cfg.PromotionBindManifest == "" || cfg.PromotionBindRole == "" || cfg.PromotionBindInput == "" || cfg.PromotionBindOutput == "") { + return config{}, fmt.Errorf("promotion report binding requires manifest, role, input, and output") + } + if cfg.BundleRequireClean && cfg.BundleVerify == "" { + return config{}, fmt.Errorf("bundle-require-clean requires bundle-verify") + } + if len(cfg.BundleEvidence) > 0 && cfg.BundleDir == "" { + return config{}, fmt.Errorf("bundle-evidence requires bundle-dir") + } + if cfg.BundleVerify != "" && cfg.BundleDir != "" { + return config{}, fmt.Errorf("bundle-verify and bundle-dir are mutually exclusive") + } + if cfg.PromotionManifest != "" && (cfg.BundleVerify != "" || cfg.BundleDir != "") { + return config{}, fmt.Errorf("promotion-manifest verification is mutually exclusive with bundle operations") + } + if cfg.ExpandIntoOutput != "" && cfg.ExpandIntoArtifact == "" { + return config{}, fmt.Errorf("expand-into-output requires expand-into-artifact") + } + if cfg.ExpandIntoProtocol != referencePairProtocolDiscovery && cfg.ExpandIntoProtocol != referencePairProtocolConfirmation { + return config{}, fmt.Errorf("expand-into-protocol must be discovery or confirmation") + } + orientationInputs := []string{cfg.OrientationShadowArtifact, cfg.OrientationIncumbentArtifact, cfg.OrientationReverseArtifact, cfg.OrientationAA} + orientationConfigured := false + for _, input := range orientationInputs { + orientationConfigured = orientationConfigured || input != "" + } + if cfg.OrientationOutput != "" { + orientationConfigured = true + } + if orientationConfigured { + for _, input := range orientationInputs { + if input == "" { + return config{}, fmt.Errorf("orientation report requires shadow, incumbent, reverse, and A/A artifacts") + } + } + } + if cfg.OrientationProtocol != referencePairProtocolDiscovery && cfg.OrientationProtocol != referencePairProtocolConfirmation { + return config{}, fmt.Errorf("orientation-protocol must be discovery or confirmation") + } + orientationV2Inputs := []string{ + cfg.OrientationV2ShadowArtifact, cfg.OrientationV2IncumbentArtifact, cfg.OrientationV2ReverseArtifact, + cfg.OrientationV2GuardedArtifact, cfg.OrientationV2AA, + } + orientationV2Configured := cfg.OrientationV2Output != "" + for _, input := range orientationV2Inputs { + orientationV2Configured = orientationV2Configured || input != "" + } + if orientationV2Configured { + for _, input := range orientationV2Inputs { + if input == "" { + return config{}, fmt.Errorf("orientation-v2 report requires shadow, incumbent, reverse, guarded, and A/A artifacts") + } + } + } + if cfg.OrientationV2Protocol != referencePairProtocolDiscovery && cfg.OrientationV2Protocol != referencePairProtocolConfirmation { + return config{}, fmt.Errorf("orientation-v2-protocol must be discovery or confirmation") + } + if orientationV2Configured && cfg.OrientationV2Protocol == referencePairProtocolConfirmation && (cfg.OrientationV2Freeze == "" || cfg.OrientationV2DiscoveryReport == "") { + return config{}, fmt.Errorf("orientation-v2 confirmation requires orientation-v2-freeze and orientation-v2-discovery-report") + } + if orientationV2Configured && cfg.OrientationV2Protocol == referencePairProtocolDiscovery && (cfg.OrientationV2FreezeOutput == "" || cfg.OrientationV2Output == "") { + return config{}, fmt.Errorf("orientation-v2 discovery requires orientation-v2-output and orientation-v2-freeze-output") + } + if cfg.OrientationV2Freeze != "" && cfg.OrientationV2Protocol != referencePairProtocolConfirmation { + return config{}, fmt.Errorf("orientation-v2-freeze is only valid for confirmation") + } + if cfg.OrientationV2DiscoveryReport != "" && cfg.OrientationV2Protocol != referencePairProtocolConfirmation { + return config{}, fmt.Errorf("orientation-v2-discovery-report is only valid for confirmation") + } + if cfg.OrientationV2FreezeOutput != "" && cfg.OrientationV2Protocol != referencePairProtocolDiscovery { + return config{}, fmt.Errorf("orientation-v2-freeze-output is only valid for discovery") + } + if (cfg.OrientationV2Freeze != "" || cfg.OrientationV2DiscoveryReport != "" || cfg.OrientationV2FreezeOutput != "") && !orientationV2Configured { + return config{}, fmt.Errorf("orientation-v2-freeze requires orientation-v2 report mode") + } + spI1ReportInputs := []string{cfg.SPI1BaselineArtifact, cfg.SPI1CandidateArtifact, cfg.SPI1ResourceReport} + spI1TrainingInputs := []string{cfg.SPI1TrainingBaseline, cfg.SPI1TrainingCandidate, cfg.SPI1TrainingResource} + spI1ReportConfigured := cfg.SPI1Output != "" || cfg.SPI1FreezeOutput != "" + for _, input := range spI1ReportInputs { + spI1ReportConfigured = spI1ReportConfigured || input != "" + } + if cfg.SPI1Protocol != referencePairProtocolDiscovery && cfg.SPI1Protocol != referencePairProtocolConfirmation { + return config{}, fmt.Errorf("sp-i1-protocol must be discovery or confirmation") + } + if spI1ReportConfigured { + for _, input := range spI1ReportInputs { + if input == "" { + return config{}, fmt.Errorf("SP-I1 report requires baseline, candidate, and resource artifacts") + } + } + if cfg.SPI1Output == "" { + return config{}, fmt.Errorf("SP-I1 report requires sp-i1-output") + } + if cfg.SPI1Protocol == referencePairProtocolDiscovery && cfg.SPI1FreezeOutput == "" { + return config{}, fmt.Errorf("SP-I1 discovery requires sp-i1-freeze-output") + } + if cfg.SPI1Protocol == referencePairProtocolConfirmation && (cfg.SPI1Freeze == "" || cfg.SPI1DiscoveryReport == "") { + return config{}, fmt.Errorf("SP-I1 confirmation requires sp-i1-freeze and sp-i1-discovery-report") + } + } else if (cfg.SPI1Freeze == "") != (cfg.SPI1DiscoveryReport == "") { + return config{}, fmt.Errorf("SP-I1 holdout capture requires both sp-i1-freeze and sp-i1-discovery-report") + } + trainingInputCount := 0 + for _, input := range spI1TrainingInputs { + if input != "" { + trainingInputCount++ + } + } + if cfg.SPI1Freeze != "" && trainingInputCount != len(spI1TrainingInputs) { + return config{}, fmt.Errorf("SP-I1 frozen authorization requires all three exact training evidence artifacts") + } + if cfg.SPI1Freeze == "" && trainingInputCount != 0 { + return config{}, fmt.Errorf("SP-I1 training evidence inputs require a discovery freeze") + } + if !spI1ReportConfigured && cfg.SPI1Freeze != "" && cfg.SPI1Protocol != referencePairProtocolConfirmation { + return config{}, fmt.Errorf("SP-I1 holdout capture requires the confirmation protocol") + } + if cfg.SPI1FreezeOutput != "" && cfg.SPI1Protocol != referencePairProtocolDiscovery { + return config{}, fmt.Errorf("sp-i1-freeze-output is only valid for discovery") + } + if spI1ReportConfigured && cfg.SPI1Protocol == referencePairProtocolDiscovery && (cfg.SPI1Freeze != "" || cfg.SPI1DiscoveryReport != "") { + return config{}, fmt.Errorf("SP-I1 discovery creates a freeze and cannot consume confirmation inputs") + } + if spI1ReportConfigured && (cfg.OutputJSONL != "" || rawCases != "" || rawDatasets != "" || rawCategories != "" || rawTags != "") { + return config{}, fmt.Errorf("SP-I1 report mode cannot also execute or select benchmark cases") + } + if spI1ReportConfigured { + if err := validateDistinctSPI1Paths(map[string]string{ + "baseline artifact": cfg.SPI1BaselineArtifact, "candidate artifact": cfg.SPI1CandidateArtifact, + "resource report": cfg.SPI1ResourceReport, "freeze manifest": cfg.SPI1Freeze, + "discovery report": cfg.SPI1DiscoveryReport, "freeze output": cfg.SPI1FreezeOutput, + "training baseline artifact": cfg.SPI1TrainingBaseline, + "training candidate artifact": cfg.SPI1TrainingCandidate, + "training resource report": cfg.SPI1TrainingResource, + "report output": cfg.SPI1Output, + }); err != nil { + return config{}, err + } + } + modeCount := 0 + if cfg.GateBaseline != "" { + modeCount++ + } + if len(cfg.AAArtifacts) != 0 { + modeCount++ + } + if cfg.ConfirmLeft != "" { + modeCount++ + } + if cfg.ReferenceClosureArtifact != "" { + modeCount++ + } + if cfg.ReferencePairArtifact != "" { + modeCount++ + } + if cfg.ReferenceTournamentArtifact != "" { + modeCount++ + } + if cfg.ResourceArtifact != "" { + modeCount++ + } + if cfg.BackendDeltaArtifact != "" { + modeCount++ + } + if cfg.BundleVerify != "" { + modeCount++ + } + if cfg.PromotionManifest != "" { + modeCount++ + } + if promotionBindConfigured { + modeCount++ + } + if cfg.ExpandIntoArtifact != "" { + modeCount++ + } + if orientationConfigured { + modeCount++ + } + if orientationV2Configured { + modeCount++ + } + if spI1ReportConfigured { + modeCount++ + } + if !spI1ReportConfigured && cfg.SPI1Freeze != "" && modeCount > 0 { + return config{}, fmt.Errorf("SP-I1 holdout authorization cannot be combined with a standalone report mode") + } + if modeCount > 1 { + return config{}, fmt.Errorf("performance-gate, A/A, paired-confirmation, reference-closure, reference-pair, reference-tournament, resource-gate, backend-delta, bundle-verify, promotion-manifest, promotion-bind, ExpandInto-report, orientation-report, orientation-v2-report, and SP-I1-report modes are mutually exclusive") + } + if modeCount > 0 && cfg.BundleDir != "" { + return config{}, fmt.Errorf("standalone report modes and bundle-dir are mutually exclusive") + } + if len(cfg.AAArtifacts) != 0 && cfg.GateBaseline != "" { + return config{}, fmt.Errorf("aa-artifact and performance-gate mode are mutually exclusive") + } + if cfg.Confidence <= 0 || cfg.Confidence >= 1 || math.IsNaN(cfg.Confidence) || math.IsInf(cfg.Confidence, 0) { + return config{}, fmt.Errorf("confidence-level must be between 0 and 1") + } + if spI1ReportConfigured && (cfg.GateSeed != 1 || cfg.Confidence != defaultConfidenceLevel) { + return config{}, fmt.Errorf("SP-I1 reporting requires frozen seed 1 and confidence %.4f", defaultConfidenceLevel) + } + if cfg.Regression < 0 { + return config{}, fmt.Errorf("regression-threshold must not be negative") + } + if cfg.MaterialityRatio <= 0 || cfg.MaterialityRatio >= 1 { + return config{}, fmt.Errorf("materiality-ratio must be between 0 and 1") + } + if cfg.MaterialityAbsolute < 0 { + return config{}, fmt.Errorf("materiality-absolute must not be negative") + } + if cfg.AppendJSONL && cfg.OutputJSONL == "" { + return config{}, fmt.Errorf("append-jsonl requires jsonl-output") + } + if cfg.ResourceOutput != "" && cfg.ResourceArtifact == "" { + return config{}, fmt.Errorf("resource-output requires resource-artifact") + } + if cfg.BackendDeltaOutput != "" && cfg.BackendDeltaArtifact == "" { + return config{}, fmt.Errorf("backend-delta-output requires backend-delta-artifact") + } + if cfg.DiscoverySampleFloor < 1 { + return config{}, fmt.Errorf("discovery-sample-floor must be at least 1") + } + for _, raw := range strings.Split(rawTimeoutClasses, ",") { + if raw = strings.TrimSpace(raw); raw != "" { + timeout, err := time.ParseDuration(raw) + if err != nil || timeout <= 0 { + return config{}, fmt.Errorf("timeout classes must be positive durations, got %q", raw) + } + if len(cfg.TimeoutClasses) > 0 && timeout <= cfg.TimeoutClasses[len(cfg.TimeoutClasses)-1] { + return config{}, fmt.Errorf("timeout classes must be strictly increasing") + } + cfg.TimeoutClasses = append(cfg.TimeoutClasses, timeout) + } + } + for _, target := range strings.Split(rawGateTargets, ",") { + if target = strings.TrimSpace(target); target != "" { + cfg.GateTargets = append(cfg.GateTargets, target) + } + } + if cfg.Cases, err = parseUniqueCSV("case", rawCases); err != nil { + return config{}, err + } + if cfg.Datasets, err = parseUniqueCSV("dataset", rawDatasets); err != nil { + return config{}, err + } + if cfg.Categories, err = parseUniqueCSV("category", rawCategories); err != nil { + return config{}, err + } + if cfg.Tags, err = parseUniqueCSV("tag", rawTags); err != nil { + return config{}, err + } + if cfg.ConfirmCases, err = parseUniqueCSV("confirmation case", rawConfirmCases); err != nil { + return config{}, err + } + if cfg.PostgresReferenceArms, err = parseUniqueCSV("PostgreSQL reference arm", rawReferenceArms); err != nil { + return config{}, err + } + if cfg.ReferenceTournamentArms, err = parseUniqueCSV("reference tournament arm", rawTournamentArms); err != nil { + return config{}, err + } + if cfg.ReferenceTournamentArtifact != "" && len(cfg.ReferenceTournamentArms) != 3 && len(cfg.ReferenceTournamentArms) != 5 { + return config{}, fmt.Errorf("reference tournament requires exactly 3 or 5 arms") + } + for _, arm := range cfg.ReferenceTournamentArms { + if !validPostgresReferenceArm(arm) { + return config{}, fmt.Errorf("unknown PostgreSQL reference tournament arm %q", arm) + } + } + for _, arm := range cfg.PostgresReferenceArms { + if !validPostgresReferenceArm(arm) { + return config{}, fmt.Errorf("unknown PostgreSQL reference arm %q", arm) + } + } + if cfg.ReferenceClosureArtifact != "" && !validPostgresReferenceArm(cfg.ReferenceClosureArm) { + return config{}, fmt.Errorf("unknown PostgreSQL reference closure arm %q", cfg.ReferenceClosureArm) + } + if len(cfg.PostgresReferenceArms) > 0 { + cfg.PostgresReferences = true + } + if cfg.PostgresForceShortest != "" && !validForcedShortestPathExecutor(cfg.PostgresForceShortest) { + return config{}, fmt.Errorf("unsupported PostgreSQL forced shortest executor %q", cfg.PostgresForceShortest) + } + if cfg.PostgresForceExpansion != "" && cfg.PostgresForceExpansion != "EXPANSION-SUFFIX-SEEDED-REVERSE" && cfg.PostgresForceExpansion != "EXPANSION-ENDPOINT-SEEDED-REVERSE" { + return config{}, fmt.Errorf("unsupported PostgreSQL forced expansion search %q", cfg.PostgresForceExpansion) + } + if cfg.PostgresForceShortest != "" && cfg.PostgresForceExpansion != "" { + return config{}, fmt.Errorf("PostgreSQL shortest and expansion search forces are mutually exclusive") + } + orientationMode := cfg.PostgresExpansionOrientationShadow || cfg.PostgresExpansionOrientationTournament + if cfg.PostgresExpansionOrientationShadow && cfg.PostgresExpansionOrientationTournament { + return config{}, fmt.Errorf("PostgreSQL expansion orientation shadow and tournament modes are mutually exclusive") + } + if orientationMode && (cfg.PostgresForceShortest != "" || cfg.PostgresForceExpansion != "") { + return config{}, fmt.Errorf("PostgreSQL expansion orientation and forced traversal selectors are mutually exclusive") + } + if cfg.PostgresExpansionOrientationPolicy != "" && !orientationMode { + return config{}, fmt.Errorf("PostgreSQL expansion orientation policy requires shadow or tournament mode") + } + if cfg.PostgresExpansionOrientationPolicy != "" && + cfg.PostgresExpansionOrientationPolicy != string(optimize.ExpansionSearchPolicyOrientationProbeV1) && + cfg.PostgresExpansionOrientationPolicy != string(optimize.ExpansionSearchPolicyOrientationProbeV2) { + return config{}, fmt.Errorf("unsupported PostgreSQL expansion orientation policy %q", cfg.PostgresExpansionOrientationPolicy) + } + if (cfg.PostgresExpansionOrientationTournament || cfg.PostgresExpansionOrientationPolicy == string(optimize.ExpansionSearchPolicyOrientationProbeV2)) && !cfg.PostgresRepeatableRead { + return config{}, fmt.Errorf("guarded and orientation-probe-v2 measurements require postgres-repeatable-read") + } + if cfg.PostgresExpansionOrientationPolicy == string(optimize.ExpansionSearchPolicyOrientationProbeV2) && cfg.PostgresTraversalTelemetry == postgresTraversalTelemetryOff { + return config{}, fmt.Errorf("orientation-probe-v2 measurements require PostgreSQL traversal telemetry") + } + if cfg.PostgresProductionManifest != "" && (cfg.PostgresForceShortest != "" || cfg.PostgresForceExpansion != "" || orientationMode) { + return config{}, fmt.Errorf("PostgreSQL production manifest is mutually exclusive with forced and shadow translation modes") + } + if cfg.PostgresProductionManifest != "" && cfg.PostgresRepeatableRead { + return config{}, fmt.Errorf("PostgreSQL production manifest already implies Repeatable Read") + } + if cfg.GateBaseline != "" && !cfg.DiagnosticGate && cfg.GateAA == "" { + return config{}, fmt.Errorf("complete performance gate requires gate-aa host calibration evidence") + } modes, err := parseExecutionModes(rawModes) if err != nil { return config{}, err } cfg.Modes = modes + if cfg.ExistingGraph { + if cfg.AnchorManifest == "" { + return config{}, fmt.Errorf("existing-graph mode requires anchor-manifest") + } + if len(cfg.Modes) != 1 || cfg.Modes[0] != ModePostgresSQL { + return config{}, fmt.Errorf("existing-graph mode currently requires only postgres_sql mode") + } + if cfg.Resume && cfg.Checkpoint == "" { + return config{}, fmt.Errorf("resume requires checkpoint") + } + if len(cfg.TimeoutClasses) > 0 && !cfg.Discovery { + return config{}, fmt.Errorf("timeout-classes require discovery mode") + } + } else if cfg.Resume || cfg.AnchorManifest != "" || cfg.Checkpoint != "" || cfg.Progress != "" || cfg.Discovery || len(cfg.TimeoutClasses) > 0 { + return config{}, fmt.Errorf("existing-graph workflow flags require existing-graph mode") + } + if !spI1ReportConfigured && cfg.SPI1Freeze != "" { + if err := validateSPI1HoldoutCaptureConfig(cfg); err != nil { + return config{}, err + } + } return cfg, nil } +// validForcedShortestPathExecutor reports whether graphbench recognizes a +// production executor or a declared tournament identity. +func validForcedShortestPathExecutor(executor string) bool { + switch executor { + case "SP-S0", + "SP-S0-DIRECT", + "SP-S3-U-D", + "SP-S3-U-E+MAT-M0", + "SP-S4-C-D", + "SP-S4-C-WE+MAT-M0", + "SP-I1-C-D", + "SP-I1-U-E+MAT-M0", + "SP-I1-C-WE+MAT-M0", + "SP-B1-C-ALT-NODE-D", + "SP-B1-C-ALT-NODE-WE+MAT-M0", + "SP-B2-C-MIN-LEVEL-D", + "SP-B2-C-MIN-LEVEL-WE+MAT-M0", + "ASP-A1-DAG", + "ASP-I1-U-DAG+MAT-M0", + "ASP-B1-DAG-ALT-NODE", + "ASP-B2-DAG-MIN-LEVEL": + return true + default: + return false + } +} + +// parseCaptureBundleEvidenceInputs parses repeatable name=path bundle evidence +// declarations while keeping host paths out of serialized evidence identities. +func parseCaptureBundleEvidenceInputs(rawValues []string) ([]CaptureBundleEvidenceInput, error) { + inputs := make([]CaptureBundleEvidenceInput, 0, len(rawValues)) + seen := map[string]struct{}{} + for _, raw := range rawValues { + name, path, found := strings.Cut(raw, "=") + name = strings.TrimSpace(name) + path = strings.TrimSpace(path) + if !found || !validBundleEvidenceName(name) || path == "" { + return nil, fmt.Errorf("bundle-evidence must be a valid name=path declaration, got %q", raw) + } + if _, duplicate := seen[name]; duplicate { + return nil, fmt.Errorf("duplicate bundle-evidence name %q", name) + } + seen[name] = struct{}{} + inputs = append(inputs, CaptureBundleEvidenceInput{Name: name, Path: path}) + } + return inputs, nil +} + +// parseUniqueCSV splits comma-separated selectors, rejecting duplicates and empty elements. +func parseUniqueCSV(kind, raw string) ([]string, error) { + var values []string + seen := map[string]struct{}{} + for _, value := range strings.Split(raw, ",") { + value = strings.TrimSpace(value) + if value == "" { + continue + } + if _, duplicate := seen[value]; duplicate { + return nil, fmt.Errorf("duplicate %s selector %q", kind, value) + } + seen[value] = struct{}{} + values = append(values, value) + } + return values, nil +} + +func selectedCorpusContainsTag(corpus ScaleCorpus, tag string) bool { + for _, testCase := range corpus.Cases { + if slices.Contains(testCase.Tags, tag) { + return true + } + } + return false +} + +// parseExecutionModes parses a comma-separated mode list and rejects duplicates or unsupported values. func parseExecutionModes(raw string) ([]ExecutionMode, error) { var ( modes []ExecutionMode @@ -101,28 +987,336 @@ func parseExecutionModes(raw string) ([]ExecutionMode, error) { return modes, nil } +// fatal logs a formatted fatal error and terminates the command. func fatal(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) os.Exit(1) } +// main runs the graphbench command. func main() { cfg, err := parseConfig(os.Args[1:], os.Getenv) if err != nil { fatal("%v", err) } - - corpus, err := loadScaleCorpus(cfg.CorpusRoot) + if cfg.BundleVerify != "" { + passed, err := createCaptureBundleVerification(cfg.BundleVerify, cfg.BundleVerifyOutput, cfg.BundleRequireClean) + if err != nil { + fatal("verify capture bundle: %v", err) + } + if !passed { + fatal("capture bundle verification failed") + } + return + } + if cfg.PromotionManifest != "" { + passed, err := writePromotionManifestVerification(cfg.PromotionManifest, cfg.PromotionManifestOutput) + if err != nil { + fatal("verify promotion manifest: %v", err) + } + if !passed { + fatal("promotion manifest verification failed") + } + return + } + if cfg.PromotionBindManifest != "" { + if err := bindPromotionEvidenceReport(cfg.PromotionBindManifest, cfg.PromotionBindRole, cfg.PromotionBindInput, cfg.PromotionBindOutput); err != nil { + fatal("bind promotion evidence report: %v", err) + } + return + } + if cfg.OrientationShadowArtifact != "" { + passed, err := createOrientationSelectorReport( + cfg.OrientationShadowArtifact, + cfg.OrientationIncumbentArtifact, + cfg.OrientationReverseArtifact, + cfg.OrientationAA, + cfg.OrientationOutput, + OrientationSelectorReportOptions{ + Seed: cfg.GateSeed, + Confidence: cfg.Confidence, + Protocol: cfg.OrientationProtocol, + }, + ) + if err != nil { + fatal("calculate orientation selector report: %v", err) + } + if cfg.OrientationProtocol == referencePairProtocolConfirmation && !passed { + fatal("orientation selector qualification failed") + } + return + } + if cfg.OrientationV2ShadowArtifact != "" { + passed, err := createOrientationSelectorV2Report( + cfg.OrientationV2ShadowArtifact, + cfg.OrientationV2IncumbentArtifact, + cfg.OrientationV2ReverseArtifact, + cfg.OrientationV2GuardedArtifact, + cfg.OrientationV2AA, + cfg.OrientationV2Freeze, + cfg.OrientationV2DiscoveryReport, + cfg.OrientationV2FreezeOutput, + cfg.OrientationV2Output, + OrientationSelectorV2ReportOptions{ + Seed: cfg.GateSeed, + Confidence: cfg.Confidence, + Protocol: cfg.OrientationV2Protocol, + }, + ) + if err != nil { + fatal("calculate orientation-v2 selector report: %v", err) + } + if cfg.OrientationV2Protocol == referencePairProtocolConfirmation && !passed { + fatal("orientation-v2 selector qualification failed") + } + return + } + if cfg.SPI1BaselineArtifact != "" { + passed, err := createSPI1QualificationReport( + cfg.SPI1BaselineArtifact, + cfg.SPI1CandidateArtifact, + cfg.SPI1ResourceReport, + cfg.SPI1Freeze, + cfg.SPI1DiscoveryReport, + cfg.SPI1FreezeOutput, + cfg.SPI1Output, + SPI1QualificationOptions{ + Seed: cfg.GateSeed, + Confidence: cfg.Confidence, + Protocol: cfg.SPI1Protocol, + TrainingBaselinePath: cfg.SPI1TrainingBaseline, + TrainingCandidatePath: cfg.SPI1TrainingCandidate, + TrainingResourcePath: cfg.SPI1TrainingResource, + }, + ) + if err != nil { + fatal("calculate staged SP-I1 qualification: %v", err) + } + if cfg.SPI1Protocol == referencePairProtocolConfirmation && !passed { + fatal("staged SP-I1 qualification failed") + } + return + } + if cfg.ExpandIntoArtifact != "" { + if err := createExpandIntoStudyReport(cfg.ExpandIntoArtifact, cfg.ExpandIntoOutput, ExpandIntoStudyOptions{ + Seed: cfg.GateSeed, + Confidence: cfg.Confidence, + Protocol: cfg.ExpandIntoProtocol, + MaterialityRatio: cfg.MaterialityRatio, + MaterialityAbsolute: cfg.MaterialityAbsolute, + P95RatioLimit: 1.05, + }); err != nil { + fatal("calculate ExpandInto study: %v", err) + } + return + } + if cfg.GateBaseline != "" { + corpus, err := loadScaleCorpus(cfg.CorpusRoot) + if err != nil { + fatal("load gate corpus declaration: %v", err) + } + selected, _, err := selectRunnableScaleCorpus(corpus, CorpusSelectors{ + Cases: cfg.Cases, + Datasets: cfg.Datasets, + Categories: cfg.Categories, + Tags: cfg.Tags, + }) + if err != nil { + fatal("select gate corpus: %v", err) + } + passed, err := comparePerformanceArtifacts(cfg.GateBaseline, cfg.GateCandidate, cfg.GateOutput, PerfGateOptions{ + Seed: cfg.GateSeed, + Confidence: cfg.Confidence, + RegressionThreshold: cfg.Regression, + DeclaredBackends: selected.DeclaredBackends(), + TargetNames: cfg.GateTargets, + MaterialityRatio: cfg.MaterialityRatio, + MaterialityAbsolute: cfg.MaterialityAbsolute, + DiagnosticMode: cfg.DiagnosticGate, + AAReportPath: cfg.GateAA, + }) + if err != nil { + fatal("compare performance artifacts: %v", err) + } + if !passed { + fatal("performance gate failed") + } + return + } + if len(cfg.AAArtifacts) != 0 { + if err := createAAResolutionReport(cfg.AAArtifacts, cfg.AAOutput, PerfGateOptions{ + Seed: cfg.GateSeed, + Confidence: cfg.Confidence, + }); err != nil { + fatal("calculate A/A measurement resolution: %v", err) + } + return + } + if cfg.ConfirmLeft != "" { + if err := createConfirmationReport(cfg.ConfirmLeft, cfg.ConfirmRight, cfg.ConfirmAA, cfg.ConfirmOutput, ConfirmationOptions{ + Seed: cfg.GateSeed, + Confidence: cfg.Confidence, + CaseNames: cfg.ConfirmCases, + }); err != nil { + fatal("calculate paired confirmation: %v", err) + } + return + } + if cfg.ReferenceClosureArtifact != "" { + passed, err := createReferenceClosureReport(cfg.ReferenceClosureArtifact, cfg.ReferenceClosureOutput, ReferenceClosureOptions{ + Seed: cfg.GateSeed, + Confidence: cfg.Confidence, + ReferenceName: cfg.ReferenceClosureArm, + RatioUpperLimit: 1.10, + AbsoluteResolution: cfg.MaterialityAbsolute, + }) + if err != nil { + fatal("calculate production/reference closure: %v", err) + } + if !passed { + fatal("production/reference closure failed") + } + return + } + if cfg.ReferencePairArtifact != "" { + if err := createReferencePairReport(cfg.ReferencePairArtifact, cfg.ReferencePairOutput, ReferencePairOptions{ + Seed: cfg.GateSeed, + Confidence: cfg.Confidence, + BaselineName: cfg.ReferencePairBaseline, + CandidateName: cfg.ReferencePairCandidate, + Protocol: cfg.ReferencePairProtocol, + }); err != nil { + fatal("calculate matched reference pair: %v", err) + } + return + } + if cfg.ReferenceTournamentArtifact != "" { + passed, err := createReferenceTournamentReport(cfg.ReferenceTournamentArtifact, cfg.ReferenceTournamentOutput, ReferenceTournamentOptions{ + Seed: cfg.GateSeed, + Confidence: cfg.Confidence, + MaterialityRatio: cfg.MaterialityRatio, + MaterialityAbsolute: cfg.MaterialityAbsolute, + P95RatioLimit: 1.05, + Arms: cfg.ReferenceTournamentArms, + Protocol: cfg.ReferenceTournamentProtocol, + }) + if err != nil { + fatal("calculate reference tournament: %v", err) + } + if cfg.ReferenceTournamentProtocol == referencePairProtocolConfirmation && !passed { + fatal("reference tournament qualification failed") + } + return + } + if cfg.ResourceArtifact != "" { + passed, err := createResourceGateReport(cfg.ResourceArtifact, cfg.ResourceOutput) + if err != nil { + fatal("calculate state/resource gate: %v", err) + } + if !passed { + fatal("state/resource gate failed") + } + return + } + if cfg.BackendDeltaArtifact != "" { + if err := createBackendDeltaReport(cfg.BackendDeltaArtifact, cfg.BackendDeltaOutput); err != nil { + fatal("calculate descriptive backend deltas: %v", err) + } + return + } + fullCorpus, err := loadScaleCorpus(cfg.CorpusRoot) if err != nil { fatal("load corpus: %v", err) } + corpus, selection, err := selectRunnableScaleCorpus(fullCorpus, CorpusSelectors{ + Cases: cfg.Cases, + Datasets: cfg.Datasets, + Categories: cfg.Categories, + Tags: cfg.Tags, + }) + if err != nil { + fatal("select corpus: %v", err) + } + if selectedCorpusContainsTag(corpus, spI1HoldoutTag) || selectedCorpusContainsSPI1Holdout(corpus) || cfg.SPI1Freeze != "" { + if cfg.SPI1Freeze == "" || cfg.SPI1DiscoveryReport == "" { + fatal("SP-I1 holdout capture requires sp-i1-freeze and sp-i1-discovery-report before database setup") + } + if err := validateSPI1HoldoutCapture( + corpus, cfg.SPI1Freeze, cfg.SPI1DiscoveryReport, + cfg.SPI1TrainingBaseline, cfg.SPI1TrainingCandidate, cfg.SPI1TrainingResource, + ); err != nil { + fatal("authorize SP-I1 holdout capture: %v", err) + } + } + + if !cfg.ExistingGraph { + for _, mode := range cfg.Modes { + var connection string + switch mode { + case ModePostgresSQL: + connection = cfg.PGConnection + case ModeNeo4j: + connection = cfg.Neo4jConnection + default: + continue + } + if connection == "" { + connection = cfg.Connection + } + if connection == "" { + continue + } + if err := databaseguard.ValidateEnvironment(connection); err != nil { + fatal("refuse destructive GraphBench target: %v", err) + } + } + + runLock, err := acquireDestructiveRunLock(cfg.DestructiveLock) + if err != nil { + fatal("acquire destructive run lock: %v", err) + } + defer func() { + if err := runLock.Close(); err != nil { + fatal("release destructive run lock: %v", err) + } + }() + } var ( - ctx = context.Background() - records []CaseResult + ctx = context.Background() + records []CaseResult + existingManifest ExistingGraphAnchorManifest + startedAt = time.Now() ) + checkpointCorpusHash := corpusIdentity(corpus) + metadata := testutil.ResolveBaselineMetadata(cfg.DAWGSVersion) + environment := resolveRunEnvironment(cfg, os.Args, selection, startedAt, startedAt) + checkpointRunHash := runConfigurationIdentity(cfg, environment) + environment.CorpusSHA256 = checkpointCorpusHash + environment.RunIdentitySHA256 = checkpointRunHash + if cfg.ExistingGraph { + existingManifest, err = loadExistingGraphAnchorManifest(cfg.AnchorManifest) + if err != nil { + fatal("load existing-graph anchor manifest: %v", err) + } + if err := validateExistingGraphCorpus(corpus, existingManifest); err != nil { + fatal("validate existing-graph corpus: %v", err) + } + if cfg.Resume { + records, err = readExistingGraphCheckpoint(cfg.Checkpoint, existingManifest.Checksum, checkpointCorpusHash, checkpointRunHash) + if err != nil { + fatal("resume existing-graph checkpoint: %v", err) + } + for _, record := range records { + if record.Environment != nil && record.Environment.RunUUID != "" { + environment.RunUUID = record.Environment.RunUUID + break + } + } + } + } - for _, mode := range cfg.Modes { + for _, mode := range modesForRound(cfg.Modes, cfg.Round) { switch mode { case ModePostgresSQL: pgConnection := cfg.PGConnection @@ -133,12 +1327,49 @@ func main() { fatal("postgres_sql mode requires -pg-connection, -connection, PG_CONNECTION_STRING, or CONNECTION_STRING") } - runner, err := newPostgresSQLRunner(ctx, cfg.DatasetDir, pgConnection, corpus) + var existingOptions *existingGraphRunnerOptions + if cfg.ExistingGraph { + completed := map[string]string{} + for _, record := range records { + completed[existingGraphCaseKey(record.ExecutionMode, ScaleCase{Dataset: record.Dataset, Name: record.Name})] = record.WorkloadSHA256 + } + existingOptions = &existingGraphRunnerOptions{ + Manifest: existingManifest, + ProgressPath: cfg.Progress, + Discovery: cfg.Discovery, + TimeoutClasses: append([]time.Duration(nil), cfg.TimeoutClasses...), + SampleFloor: cfg.DiscoverySampleFloor, + Completed: completed, + OnRecord: func(record CaseResult) error { + setCaseRunMetadata(&record, metadata, environment) + records = append(records, record) + return writeExistingGraphCheckpoint(cfg.Checkpoint, existingManifest.Checksum, checkpointCorpusHash, checkpointRunHash, records) + }, + OnComplete: func(postNodes, postEdges int64) error { + for idx := range records { + if records[idx].ExistingGraph != nil { + records[idx].ExistingGraph.PostNodeCount = postNodes + records[idx].ExistingGraph.PostEdgeCount = postEdges + } + } + return writeExistingGraphCheckpoint(cfg.Checkpoint, existingManifest.Checksum, checkpointCorpusHash, checkpointRunHash, records) + }, + } + } + runner, err := newPostgresSQLRunnerWithExistingGraph(ctx, cfg.DatasetDir, pgConnection, corpus, cfg.PoolSize, cfg.Round, cfg.Concurrency, cfg.PostgresReferences, cfg.PostgresReferenceArms, cfg.PostgresForceShortest, cfg.PostgresForceExpansion, existingOptions) if err != nil { fatal("open postgres_sql runner: %v", err) } - - nextRecords, err := runner.Run(ctx, cfg.Iterations, corpus) + runner.traversalTelemetry = cfg.PostgresTraversalTelemetry + runner.repeatableRead = cfg.PostgresRepeatableRead + runner.toolOptions.EnableExpansionOrientationShadow = cfg.PostgresExpansionOrientationShadow + runner.toolOptions.EnableExpansionOrientationTournament = cfg.PostgresExpansionOrientationTournament + runner.toolOptions.ExpansionOrientationPolicy = optimize.ExpansionSearchPolicy(cfg.PostgresExpansionOrientationPolicy) + if err := runner.setProductionManifest(cfg.PostgresProductionManifest); err != nil { + _ = runner.Close(ctx) + fatal("configure PostgreSQL production candidate: %v", err) + } + nextRecords, err := runner.Run(ctx, cfg.WarmupIterations, cfg.Iterations, corpus) closeErr := runner.Close(ctx) if err != nil { fatal("run postgres_sql: %v", err) @@ -147,7 +1378,16 @@ func main() { fatal("close postgres_sql: %v", closeErr) } - records = append(records, nextRecords...) + if !cfg.ExistingGraph { + records = append(records, nextRecords...) + } else { + // OnRecord appends each completed record atomically. A resumed run + // may have no new records, while a complete run refreshes the final + // before/after cardinality proof below. + if err := writeExistingGraphCheckpoint(cfg.Checkpoint, existingManifest.Checksum, checkpointCorpusHash, checkpointRunHash, records); err != nil { + fatal("finalize existing-graph checkpoint: %v", err) + } + } case ModeNeo4j: neo4jConnection := cfg.Neo4jConnection @@ -163,7 +1403,7 @@ func main() { fatal("open neo4j runner: %v", err) } - nextRecords, err := runner.Run(ctx, cfg.Iterations, corpus) + nextRecords, err := runner.Run(ctx, cfg.WarmupIterations, cfg.Iterations, corpus) closeErr := runner.Close(ctx) if err != nil { fatal("run neo4j: %v", err) @@ -182,14 +1422,43 @@ func main() { } } + if err := validateBackendObservations(records); err != nil { + fatal("validate backend observations: %v", err) + } + + environment.EndedAt = time.Now().UTC() + for idx := range records { + if records[idx].Environment == nil { + setCaseRunMetadata(&records[idx], metadata, environment) + } else if records[idx].Environment.RunUUID == environment.RunUUID { + records[idx].Environment.EndedAt = environment.EndedAt + } + } + if cfg.ExistingGraph { + if err := writeExistingGraphCheckpoint(cfg.Checkpoint, existingManifest.Checksum, checkpointCorpusHash, checkpointRunHash, records); err != nil { + fatal("persist finalized existing-graph checkpoint: %v", err) + } + } + if cfg.Baseline != "" { if err := applyBaseline(cfg.Baseline, records); err != nil { fatal("compare baseline: %v", err) } } - if err := writeJSONLFile(cfg.OutputJSONL, records); err != nil { - fatal("write JSONL: %v", err) + var writeErr error + if cfg.AppendJSONL { + writeErr = appendJSONLFile(cfg.OutputJSONL, records) + } else { + writeErr = writeJSONLFile(cfg.OutputJSONL, records) + } + if writeErr != nil { + fatal("write JSONL: %v", writeErr) + } + if cfg.BundleDir != "" { + if err := writeCaptureBundleWithEvidence(cfg.BundleDir, corpus, records, environment, cfg.BundleEvidence); err != nil { + fatal("write capture bundle: %v", err) + } } summary := buildSummary(records) @@ -204,3 +1473,12 @@ func main() { } } } + +// modesForRound returns execution modes in alternating round order without mutating the configured slice. +func modesForRound(modes []ExecutionMode, round int) []ExecutionMode { + ordered := append([]ExecutionMode(nil), modes...) + if round%2 == 0 { + slices.Reverse(ordered) + } + return ordered +} diff --git a/cmd/graphbench/main_test.go b/cmd/graphbench/main_test.go new file mode 100644 index 00000000..15f6e75a --- /dev/null +++ b/cmd/graphbench/main_test.go @@ -0,0 +1,574 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestModesForRoundAlternatesBackendOrderWithoutMutatingConfig verifies odd/even round rotation without modifying the configured backend order. +func TestModesForRoundAlternatesBackendOrderWithoutMutatingConfig(t *testing.T) { + modes := []ExecutionMode{ModePostgresSQL, ModeNeo4j} + + require.Equal(t, []ExecutionMode{ModePostgresSQL, ModeNeo4j}, modesForRound(modes, 1)) + require.Equal(t, []ExecutionMode{ModeNeo4j, ModePostgresSQL}, modesForRound(modes, 2)) + require.Equal(t, []ExecutionMode{ModePostgresSQL, ModeNeo4j}, modes) +} + +// TestParseConfigRequiresCompleteGateInputs verifies that baseline gating cannot be enabled without its paired candidate artifact. +func TestParseConfigRequiresCompleteGateInputs(t *testing.T) { + _, err := parseConfig([]string{"-gate-baseline", "baseline.jsonl"}, func(string) string { return "" }) + + require.ErrorContains(t, err, "must be supplied together") +} + +// TestParseConfigDefaultsQualificationConfidence verifies every statistical workflow starts at the frozen 97.5% policy. +func TestParseConfigDefaultsQualificationConfidence(t *testing.T) { + cfg, err := parseConfig(nil, func(string) string { return "" }) + + require.NoError(t, err) + require.Equal(t, defaultConfidenceLevel, cfg.Confidence) + require.Equal(t, minimumTimingNoiseRatio, cfg.Regression) +} + +// TestParseConfigRequiresGateAAForPromotion verifies only explicit diagnostic comparisons may omit host calibration. +func TestParseConfigRequiresGateAAForPromotion(t *testing.T) { + _, err := parseConfig([]string{"-gate-baseline", "baseline.jsonl", "-gate-candidate", "candidate.jsonl"}, func(string) string { return "" }) + require.ErrorContains(t, err, "requires gate-aa") + + cfg, err := parseConfig([]string{ + "-gate-baseline", "baseline.jsonl", "-gate-candidate", "candidate.jsonl", "-gate-aa", "aa.json", + }, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, "aa.json", cfg.GateAA) +} + +// TestParseConfigAcceptsNamedBundleEvidence verifies repeatable name=path inputs are retained for capture without conflating their host paths with evidence names. +func TestParseConfigAcceptsNamedBundleEvidence(t *testing.T) { + cfg, err := parseConfig([]string{ + "-bundle-dir", "capture", + "-bundle-evidence", "host-aa=.coverage/aa.json", + "-bundle-evidence", "plan-delta=.coverage/plan-delta.json", + }, func(string) string { return "" }) + + require.NoError(t, err) + require.Equal(t, []CaptureBundleEvidenceInput{ + {Name: "host-aa", Path: ".coverage/aa.json"}, + {Name: "plan-delta", Path: ".coverage/plan-delta.json"}, + }, cfg.BundleEvidence) +} + +// TestParseConfigAcceptsStandaloneBundleVerification verifies portable verification can run without a benchmark connection and optionally enforce clean-source provenance. +func TestParseConfigAcceptsStandaloneBundleVerification(t *testing.T) { + cfg, err := parseConfig([]string{ + "-bundle-verify", "capture", + "-bundle-verify-output", "verification.json", + "-bundle-require-clean", + }, func(string) string { return "" }) + + require.NoError(t, err) + require.Equal(t, "capture", cfg.BundleVerify) + require.Equal(t, "verification.json", cfg.BundleVerifyOutput) + require.True(t, cfg.BundleRequireClean) +} + +func TestParseConfigAcceptsOnlyStandalonePromotionManifestVerification(t *testing.T) { + cfg, err := parseConfig([]string{ + "-promotion-manifest", "promotion.json", + "-promotion-manifest-output", "verification.json", + }, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, "promotion.json", cfg.PromotionManifest) + require.Equal(t, "verification.json", cfg.PromotionManifestOutput) + + for _, args := range [][]string{ + {"-promotion-manifest-output", "verification.json"}, + {"-promotion-manifest", "promotion.json", "-bundle-verify", "capture"}, + {"-promotion-manifest", "promotion.json", "-resource-artifact", "resources.jsonl"}, + } { + _, err := parseConfig(args, func(string) string { return "" }) + require.Error(t, err, args) + } +} + +// TestParseConfigRejectsMalformedOrOrphanedBundleFlags verifies capture and verification inputs fail before any artifact or database is touched. +func TestParseConfigRejectsMalformedOrOrphanedBundleFlags(t *testing.T) { + for _, args := range [][]string{ + {"-bundle-evidence", "host-aa=aa.json"}, + {"-bundle-dir", "capture", "-bundle-evidence", "missing-separator"}, + {"-bundle-dir", "capture", "-bundle-evidence", "../escape=aa.json"}, + {"-bundle-dir", "capture", "-bundle-evidence", "host-aa=one.json", "-bundle-evidence", "host-aa=two.json"}, + {"-bundle-verify-output", "verification.json"}, + {"-bundle-require-clean"}, + {"-bundle-verify", "capture", "-bundle-dir", "new-capture"}, + } { + _, err := parseConfig(args, func(string) string { return "" }) + require.Error(t, err, args) + } +} + +// TestParseConfigAcceptsExpandIntoStudyProtocols verifies standalone three-arm reports expose the frozen discovery and confirmation evidence contracts. +func TestParseConfigAcceptsExpandIntoStudyProtocols(t *testing.T) { + for _, protocol := range []string{referencePairProtocolDiscovery, referencePairProtocolConfirmation} { + t.Run(protocol, func(t *testing.T) { + cfg, err := parseConfig([]string{ + "-expand-into-artifact", "expand-into.jsonl", + "-expand-into-output", "expand-into.json", + "-expand-into-protocol", protocol, + }, func(string) string { return "" }) + + require.NoError(t, err) + require.Equal(t, "expand-into.jsonl", cfg.ExpandIntoArtifact) + require.Equal(t, "expand-into.json", cfg.ExpandIntoOutput) + require.Equal(t, protocol, cfg.ExpandIntoProtocol) + }) + } +} + +// TestParseConfigAcceptsOrientationSelectorReport verifies the matched shadow, +// incumbent, forced-reverse, and A/A artifacts form one standalone workflow. +func TestParseConfigAcceptsOrientationSelectorReport(t *testing.T) { + cfg, err := parseConfig([]string{ + "-orientation-shadow-artifact", "shadow.jsonl", + "-orientation-incumbent-artifact", "incumbent.jsonl", + "-orientation-reverse-artifact", "reverse.jsonl", + "-orientation-aa", "aa.json", + "-orientation-output", "orientation.json", + "-orientation-protocol", referencePairProtocolConfirmation, + }, func(string) string { return "" }) + + require.NoError(t, err) + require.Equal(t, "shadow.jsonl", cfg.OrientationShadowArtifact) + require.Equal(t, "incumbent.jsonl", cfg.OrientationIncumbentArtifact) + require.Equal(t, "reverse.jsonl", cfg.OrientationReverseArtifact) + require.Equal(t, "aa.json", cfg.OrientationAA) + require.Equal(t, "orientation.json", cfg.OrientationOutput) + require.Equal(t, referencePairProtocolConfirmation, cfg.OrientationProtocol) +} + +func TestParseConfigAcceptsOrientationSelectorV2Report(t *testing.T) { + cfg, err := parseConfig([]string{ + "-orientation-v2-shadow-artifact", "shadow-v2.jsonl", + "-orientation-v2-incumbent-artifact", "incumbent.jsonl", + "-orientation-v2-reverse-artifact", "reverse.jsonl", + "-orientation-v2-guarded-artifact", "guarded-v2.jsonl", + "-orientation-v2-aa", "aa.json", + "-orientation-v2-freeze", "orientation-v2-freeze.json", + "-orientation-v2-discovery-report", "orientation-v2-discovery.json", + "-orientation-v2-output", "orientation-v2.json", + "-orientation-v2-protocol", referencePairProtocolConfirmation, + }, func(string) string { return "" }) + + require.NoError(t, err) + require.Equal(t, "shadow-v2.jsonl", cfg.OrientationV2ShadowArtifact) + require.Equal(t, "incumbent.jsonl", cfg.OrientationV2IncumbentArtifact) + require.Equal(t, "reverse.jsonl", cfg.OrientationV2ReverseArtifact) + require.Equal(t, "guarded-v2.jsonl", cfg.OrientationV2GuardedArtifact) + require.Equal(t, "aa.json", cfg.OrientationV2AA) + require.Equal(t, "orientation-v2-freeze.json", cfg.OrientationV2Freeze) + require.Equal(t, "orientation-v2-discovery.json", cfg.OrientationV2DiscoveryReport) + require.Equal(t, "orientation-v2.json", cfg.OrientationV2Output) +} + +func TestParseConfigAcceptsOrientationSelectorV2DiscoveryFreeze(t *testing.T) { + cfg, err := parseConfig([]string{ + "-orientation-v2-shadow-artifact", "shadow-v2.jsonl", + "-orientation-v2-incumbent-artifact", "incumbent.jsonl", + "-orientation-v2-reverse-artifact", "reverse.jsonl", + "-orientation-v2-guarded-artifact", "guarded-v2.jsonl", + "-orientation-v2-aa", "aa.json", + "-orientation-v2-output", "orientation-v2-discovery.json", + "-orientation-v2-freeze-output", "orientation-v2-freeze.json", + "-orientation-v2-protocol", referencePairProtocolDiscovery, + }, func(string) string { return "" }) + + require.NoError(t, err) + require.Equal(t, referencePairProtocolDiscovery, cfg.OrientationV2Protocol) + require.Equal(t, "orientation-v2-discovery.json", cfg.OrientationV2Output) + require.Equal(t, "orientation-v2-freeze.json", cfg.OrientationV2FreezeOutput) +} + +func TestParseConfigRejectsIncompleteOrMixedOrientationSelectorV2Report(t *testing.T) { + complete := []string{ + "-orientation-v2-shadow-artifact", "shadow-v2.jsonl", + "-orientation-v2-incumbent-artifact", "incumbent.jsonl", + "-orientation-v2-reverse-artifact", "reverse.jsonl", + "-orientation-v2-guarded-artifact", "guarded-v2.jsonl", + "-orientation-v2-aa", "aa.json", + "-orientation-v2-freeze", "orientation-v2-freeze.json", + "-orientation-v2-discovery-report", "orientation-v2-discovery.json", + } + for _, args := range [][]string{ + {"-orientation-v2-shadow-artifact", "shadow-v2.jsonl"}, + { + "-orientation-v2-shadow-artifact", "shadow-v2.jsonl", "-orientation-v2-incumbent-artifact", "incumbent.jsonl", + "-orientation-v2-reverse-artifact", "reverse.jsonl", "-orientation-v2-guarded-artifact", "guarded-v2.jsonl", + "-orientation-v2-aa", "aa.json", "-orientation-v2-output", "report.json", + }, + append(append([]string(nil), complete...), "-orientation-v2-protocol", "exploratory"), + append(append([]string(nil), complete...), "-orientation-shadow-artifact", "shadow-v1.jsonl", "-orientation-incumbent-artifact", "incumbent.jsonl", "-orientation-reverse-artifact", "reverse.jsonl", "-orientation-aa", "aa.json"), + append(append([]string(nil), complete...), "-expand-into-artifact", "expand.jsonl"), + } { + _, err := parseConfig(args, func(string) string { return "" }) + require.Error(t, err, args) + } +} + +func TestParseConfigAcceptsSPI1StagedDiscoveryAndConfirmation(t *testing.T) { + discovery, err := parseConfig([]string{ + "-sp-i1-baseline-artifact", "s4-training.jsonl", + "-sp-i1-candidate-artifact", "i1-training.jsonl", + "-sp-i1-resource-report", "i1-training-resource.json", + "-sp-i1-output", "sp-i1-discovery.json", + "-sp-i1-freeze-output", "sp-i1-freeze.json", + "-sp-i1-protocol", referencePairProtocolDiscovery, + }, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, "s4-training.jsonl", discovery.SPI1BaselineArtifact) + require.Equal(t, "i1-training.jsonl", discovery.SPI1CandidateArtifact) + require.Equal(t, "i1-training-resource.json", discovery.SPI1ResourceReport) + require.Equal(t, "sp-i1-freeze.json", discovery.SPI1FreezeOutput) + + confirmation, err := parseConfig([]string{ + "-sp-i1-baseline-artifact", "s4-confirmation.jsonl", + "-sp-i1-candidate-artifact", "i1-confirmation.jsonl", + "-sp-i1-resource-report", "i1-confirmation-resource.json", + "-sp-i1-output", "sp-i1-confirmation.json", + "-sp-i1-freeze", "sp-i1-freeze.json", + "-sp-i1-discovery-report", "sp-i1-discovery.json", + "-sp-i1-training-baseline-artifact", "s4-training.jsonl", + "-sp-i1-training-candidate-artifact", "i1-training.jsonl", + "-sp-i1-training-resource-report", "i1-training-resource.json", + "-sp-i1-protocol", referencePairProtocolConfirmation, + }, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, "sp-i1-freeze.json", confirmation.SPI1Freeze) + require.Equal(t, "sp-i1-discovery.json", confirmation.SPI1DiscoveryReport) +} + +func TestParseConfigAcceptsSPI1HoldoutCaptureAuthorization(t *testing.T) { + cfg, err := parseConfig([]string{ + "-sp-i1-freeze", "sp-i1-freeze.json", + "-sp-i1-discovery-report", "sp-i1-discovery.json", + "-sp-i1-training-baseline-artifact", "s4-training.jsonl", + "-sp-i1-training-candidate-artifact", "i1-training.jsonl", + "-sp-i1-training-resource-report", "i1-training-resource.json", + "-tags", "sp-i1-inbound-v1-training,sp-i1-inbound-v1-holdout", + "-iterations", "50", + "-warmup-iterations", "20", + "-round", "1", + "-block", "1", + "-arm", "sp-i1-s4", + "-arm-order", "1", + "-run-uuid", "sp-i1-confirmation-run", + "-postgres-force-shortest-executor", "SP-S4-C-WE+MAT-M0", + "-postgres-repeatable-read", + "-postgres-traversal-telemetry", postgresTraversalTelemetryDiagnostic, + "-jsonl-output", "sp-i1-s4-confirmation.jsonl", + }, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, "sp-i1-freeze.json", cfg.SPI1Freeze) + require.Empty(t, cfg.SPI1BaselineArtifact) +} + +func TestParseConfigRejectsIncompleteOrMixedSPI1StagedWorkflow(t *testing.T) { + discovery := []string{ + "-sp-i1-baseline-artifact", "s4.jsonl", + "-sp-i1-candidate-artifact", "i1.jsonl", + "-sp-i1-resource-report", "resource.json", + "-sp-i1-output", "report.json", + "-sp-i1-freeze-output", "freeze.json", + "-sp-i1-protocol", referencePairProtocolDiscovery, + } + for _, args := range [][]string{ + {"-sp-i1-baseline-artifact", "s4.jsonl"}, + {"-sp-i1-freeze", "freeze.json"}, + append(append([]string(nil), discovery...), "-sp-i1-freeze", "old-freeze.json", "-sp-i1-discovery-report", "old-report.json"), + append(append([]string(nil), discovery...), "-sp-i1-protocol", "exploratory"), + append(append([]string(nil), discovery...), "-resource-artifact", "other.jsonl"), + append(append([]string(nil), discovery...), "-sp-i1-output", "s4.jsonl"), + append(append([]string(nil), discovery...), "-seed", "2"), + append(append([]string(nil), discovery...), "-confidence-level", "0.95"), + {"-sp-i1-freeze", "freeze.json", "-sp-i1-discovery-report", "discovery.json", "-resource-artifact", "candidate.jsonl"}, + } { + _, err := parseConfig(args, func(string) string { return "" }) + require.Error(t, err, args) + } +} + +func TestParseConfigAcceptsProductionManifestAndRejectsToolMixing(t *testing.T) { + cfg, err := parseConfig([]string{"-postgres-production-manifest", "provisional.json"}, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, "provisional.json", cfg.PostgresProductionManifest) + + _, err = parseConfig([]string{ + "-postgres-production-manifest", "provisional.json", + "-postgres-force-shortest-executor", "ASP-I1-U-DAG+MAT-M0", + }, func(string) string { return "" }) + require.ErrorContains(t, err, "mutually exclusive") + + cfg, err = parseConfig([]string{"-postgres-repeatable-read"}, func(string) string { return "" }) + require.NoError(t, err) + require.True(t, cfg.PostgresRepeatableRead) + _, err = parseConfig([]string{"-postgres-production-manifest", "provisional.json", "-postgres-repeatable-read"}, func(string) string { return "" }) + require.ErrorContains(t, err, "already implies Repeatable Read") +} + +// TestParseConfigRejectsIncompleteOrientationSelectorReport verifies the +// report cannot silently omit an exact comparator, A/A floor, or standalone +// workflow boundary. +func TestParseConfigRejectsIncompleteOrientationSelectorReport(t *testing.T) { + complete := []string{ + "-orientation-shadow-artifact", "shadow.jsonl", + "-orientation-incumbent-artifact", "incumbent.jsonl", + "-orientation-reverse-artifact", "reverse.jsonl", + "-orientation-aa", "aa.json", + } + for _, args := range [][]string{ + {"-orientation-shadow-artifact", "shadow.jsonl"}, + append(append([]string(nil), complete...), "-orientation-protocol", "exploratory"), + append(append([]string(nil), complete...), "-expand-into-artifact", "expand.jsonl"), + } { + _, err := parseConfig(args, func(string) string { return "" }) + require.Error(t, err, args) + } +} + +// TestParseConfigRejectsInvalidExpandIntoStudyMode verifies report output, protocol, and standalone-mode exclusivity fail closed. +func TestParseConfigRejectsInvalidExpandIntoStudyMode(t *testing.T) { + for _, args := range [][]string{ + {"-expand-into-output", "expand-into.json"}, + {"-expand-into-artifact", "expand-into.jsonl", "-expand-into-protocol", "exploratory"}, + {"-expand-into-artifact", "expand-into.jsonl", "-bundle-verify", "capture"}, + } { + _, err := parseConfig(args, func(string) string { return "" }) + require.Error(t, err, args) + } +} + +// TestParseConfigAcceptsPoolAndConcurrencySmokeLevels verifies numeric pool parsing and stable deduplication of requested concurrency levels. +func TestParseConfigAcceptsPoolAndConcurrencySmokeLevels(t *testing.T) { + cfg, err := parseConfig([]string{"-pool-size", "4", "-concurrency", "1,4,8,4"}, func(string) string { return "" }) + + require.NoError(t, err) + require.Equal(t, 4, cfg.PoolSize) + require.Equal(t, []int{1, 4, 8}, cfg.Concurrency) +} + +// TestParseConfigAcceptsReferencePairDiscoveryProtocol verifies that the discovery protocol flag selects the corresponding reference-pair workflow. +func TestParseConfigAcceptsReferencePairDiscoveryProtocol(t *testing.T) { + cfg, err := parseConfig([]string{"-reference-pair-protocol", "discovery"}, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, referencePairProtocolDiscovery, cfg.ReferencePairProtocol) +} + +// TestParseConfigAcceptsReferenceTournament verifies a predeclared arm order +// is preserved because the first arm defines the incumbent. +func TestParseConfigAcceptsReferenceTournament(t *testing.T) { + arms := "expand_into_pair_join,expand_into_lower_degree_scan,expand_into_pair_cache" + cfg, err := parseConfig([]string{ + "-reference-tournament-artifact", "tournament.jsonl", + "-reference-tournament-output", "tournament.json", + "-reference-tournament-arms", arms, + "-reference-tournament-protocol", referencePairProtocolConfirmation, + }, func(string) string { return "" }) + + require.NoError(t, err) + require.Equal(t, []string{"expand_into_pair_join", "expand_into_lower_degree_scan", "expand_into_pair_cache"}, cfg.ReferenceTournamentArms) + require.Equal(t, referencePairProtocolConfirmation, cfg.ReferenceTournamentProtocol) +} + +func TestParseConfigRejectsInvalidReferenceTournament(t *testing.T) { + for _, args := range [][]string{ + {"-reference-tournament-output", "tournament.json"}, + {"-reference-tournament-artifact", "tournament.jsonl"}, + {"-reference-tournament-artifact", "tournament.jsonl", "-reference-tournament-arms", "expand_into_pair_join,expand_into_pair_cache"}, + {"-reference-tournament-artifact", "tournament.jsonl", "-reference-tournament-arms", "expand_into_pair_join,unknown,expand_into_pair_cache"}, + } { + _, err := parseConfig(args, func(string) string { return "" }) + require.Error(t, err, args) + } +} + +// TestParseConfigRejectsPoolMemoryBelowPerSessionBudget verifies that the pool ceiling must cover the per-session budget for every configured connection. +func TestParseConfigRejectsPoolMemoryBelowPerSessionBudget(t *testing.T) { + _, err := parseConfig([]string{ + "-pool-size", "4", + "-session-memory-ceiling-bytes", "100", + "-pool-memory-ceiling-bytes", "399", + }, func(string) string { return "" }) + + require.ErrorContains(t, err, "session memory ceiling times pool size") +} + +// TestParseConfigAcceptsDiagnosticSelectorsAndRunMetadata verifies parsing of case filters, warmups, arm identity, and block metadata used to reproduce diagnostic runs. +func TestParseConfigAcceptsDiagnosticSelectorsAndRunMetadata(t *testing.T) { + cfg, err := parseConfig([]string{ + "-cases", "case-a,case-b", "-datasets", "fixture", "-categories", "lookup", "-tags", "primary,control", + "-warmup-iterations", "20", "-arm", "candidate", "-arm-order", "2", "-block", "7", "-run-uuid", "run-1", + }, func(string) string { return "" }) + + require.NoError(t, err) + require.Equal(t, []string{"case-a", "case-b"}, cfg.Cases) + require.Equal(t, 20, cfg.WarmupIterations) + require.Equal(t, "candidate", cfg.Arm) + require.Equal(t, 7, cfg.Block) +} + +// TestParseConfigRejectsDuplicateExactSelectors verifies that repeated exact case names are rejected before corpus selection. +func TestParseConfigRejectsDuplicateExactSelectors(t *testing.T) { + _, err := parseConfig([]string{"-cases", "case-a,case-a"}, func(string) string { return "" }) + require.ErrorContains(t, err, "duplicate case selector") +} + +// TestParseConfigAcceptsOnlyQualifiedForcedShortestExecutor verifies the supported shortest-executor allowlist and rejects an incomplete strategy name. +func TestParseConfigAcceptsOnlyQualifiedForcedShortestExecutor(t *testing.T) { + for _, executor := range []string{ + "SP-S0", + "SP-S0-DIRECT", + "SP-S3-U-D", + "SP-S3-U-E+MAT-M0", + "SP-S4-C-D", + "SP-S4-C-WE+MAT-M0", + "SP-I1-C-D", + "SP-I1-U-E+MAT-M0", + "SP-I1-C-WE+MAT-M0", + "SP-B1-C-ALT-NODE-D", + "SP-B1-C-ALT-NODE-WE+MAT-M0", + "SP-B2-C-MIN-LEVEL-D", + "SP-B2-C-MIN-LEVEL-WE+MAT-M0", + "ASP-A1-DAG", + "ASP-I1-U-DAG+MAT-M0", + "ASP-B1-DAG-ALT-NODE", + "ASP-B2-DAG-MIN-LEVEL", + } { + t.Run(executor, func(t *testing.T) { + cfg, err := parseConfig([]string{"-postgres-force-shortest-executor", executor}, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, executor, cfg.PostgresForceShortest) + }) + } + + _, err := parseConfig([]string{"-postgres-force-shortest-executor", "SP-S1"}, func(string) string { return "" }) + require.ErrorContains(t, err, "unsupported PostgreSQL forced shortest executor") +} + +// TestParseConfigExistingGraphWorkflow verifies that a fully specified live-graph discovery run retains checkpoint, resume, progress, timeout, and sampling settings. +func TestParseConfigExistingGraphWorkflow(t *testing.T) { + cfg, err := parseConfig([]string{ + "-existing-graph", "-anchor-manifest", "anchors.json", "-checkpoint", "checkpoint.json", + "-resume", "-progress", "progress.jsonl", "-discovery", "-timeout-classes", "100ms,1s", + "-discovery-sample-floor", "2", + }, func(string) string { return "" }) + require.NoError(t, err) + require.True(t, cfg.ExistingGraph) + require.True(t, cfg.Resume) + require.True(t, cfg.Discovery) + require.Equal(t, []time.Duration{100 * time.Millisecond, time.Second}, cfg.TimeoutClasses) + require.Equal(t, 2, cfg.DiscoverySampleFloor) +} + +// TestParseConfigRejectsUnsafeExistingGraphCombinations verifies that live-graph mode requires an anchor manifest and disallows mismatched backends or orphaned resume/discovery flags. +func TestParseConfigRejectsUnsafeExistingGraphCombinations(t *testing.T) { + for _, args := range [][]string{ + {"-existing-graph"}, + {"-existing-graph", "-anchor-manifest", "anchors.json", "-modes", "postgres_sql,neo4j"}, + {"-existing-graph", "-anchor-manifest", "anchors.json", "-resume"}, + {"-existing-graph", "-anchor-manifest", "anchors.json", "-timeout-classes", "1s"}, + {"-anchor-manifest", "anchors.json"}, + } { + _, err := parseConfig(args, func(string) string { return "" }) + require.Error(t, err, args) + } +} + +// TestParseConfigAcceptsOnlyQualifiedForcedExpansionSearch verifies the expansion-strategy allowlist and prevents simultaneous forced expansion and shortest-path strategies. +func TestParseConfigAcceptsOnlyQualifiedForcedExpansionSearch(t *testing.T) { + cfg, err := parseConfig([]string{"-postgres-force-expansion-search", "EXPANSION-SUFFIX-SEEDED-REVERSE"}, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, "EXPANSION-SUFFIX-SEEDED-REVERSE", cfg.PostgresForceExpansion) + cfg, err = parseConfig([]string{"-postgres-force-expansion-search", "EXPANSION-ENDPOINT-SEEDED-REVERSE"}, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, "EXPANSION-ENDPOINT-SEEDED-REVERSE", cfg.PostgresForceExpansion) + + _, err = parseConfig([]string{"-postgres-force-expansion-search", "unknown-strategy"}, func(string) string { return "" }) + require.ErrorContains(t, err, "unsupported PostgreSQL forced expansion search") + + _, err = parseConfig([]string{ + "-postgres-force-shortest-executor", "SP-S3-U-D", + "-postgres-force-expansion-search", "EXPANSION-SUFFIX-SEEDED-REVERSE", + }, func(string) string { return "" }) + require.ErrorContains(t, err, "mutually exclusive") +} + +// TestParseConfigRequiresOutputForJSONLAppend verifies that append mode names a destination and is retained once that destination is present. +func TestParseConfigRequiresOutputForJSONLAppend(t *testing.T) { + _, err := parseConfig([]string{"-append-jsonl"}, func(string) string { return "" }) + require.ErrorContains(t, err, "append-jsonl requires jsonl-output") + + cfg, err := parseConfig([]string{"-append-jsonl", "-jsonl-output", "rounds.jsonl"}, func(string) string { return "" }) + require.NoError(t, err) + require.True(t, cfg.AppendJSONL) +} + +// TestParseConfigAcceptsMultipleAAArtifacts verifies independently captured +// A/A arms can be passed to the reporter without an unvalidated external merge. +func TestParseConfigAcceptsMultipleAAArtifacts(t *testing.T) { + cfg, err := parseConfig([]string{ + "-aa-artifact", "aa-a.jsonl", + "-aa-artifact", "aa-b.jsonl", + "-aa-output", "aa.json", + }, func(string) string { return "" }) + + require.NoError(t, err) + require.Equal(t, []string{"aa-a.jsonl", "aa-b.jsonl"}, cfg.AAArtifacts) +} + +// TestParseConfigAcceptsReferenceClosureMode verifies reference-closure artifact parsing, confidence propagation, required output pairing, and exclusion of incompatible A/A mode. +func TestParseConfigAcceptsReferenceClosureMode(t *testing.T) { + cfg, err := parseConfig([]string{ + "-reference-closure-artifact", "reference.jsonl", + "-reference-closure-output", "report.json", + "-reference-closure-arm", "s3_unidirectional_trail_cte", + "-confidence-level", "0.975", + }, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, "reference.jsonl", cfg.ReferenceClosureArtifact) + require.Equal(t, 0.975, cfg.Confidence) + + _, err = parseConfig([]string{"-reference-closure-output", "report.json"}, func(string) string { return "" }) + require.ErrorContains(t, err, "requires reference-closure-artifact") + _, err = parseConfig([]string{"-reference-closure-artifact", "reference.jsonl", "-aa-artifact", "aa.jsonl"}, func(string) string { return "" }) + require.ErrorContains(t, err, "mutually exclusive") +} + +// TestParseConfigAcceptsReferencePairMode verifies that pair-report configuration retains its artifact and explicit baseline/candidate arm names. +func TestParseConfigAcceptsReferencePairMode(t *testing.T) { + cfg, err := parseConfig([]string{ + "-reference-pair-artifact", "pair.jsonl", + "-reference-pair-baseline", "s3", + "-reference-pair-candidate", "s1", + }, func(string) string { return "" }) + + require.NoError(t, err) + require.Equal(t, "pair.jsonl", cfg.ReferencePairArtifact) + require.Equal(t, "s3", cfg.ReferencePairBaseline) + require.Equal(t, "s1", cfg.ReferencePairCandidate) +} diff --git a/cmd/graphbench/measure.go b/cmd/graphbench/measure.go index 7aaa7a93..54ed7066 100644 --- a/cmd/graphbench/measure.go +++ b/cmd/graphbench/measure.go @@ -18,12 +18,80 @@ package main import ( "context" + "encoding/json" + "errors" "fmt" + "math" + "slices" + "sort" "time" "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/opengraph" ) +// errScaleWriteRollback signals the intentional rollback used to isolate a measured write. +var errScaleWriteRollback = errors.New("scale write rollback") + +// resolvedWriteScenario contains a write scenario after symbolic fixture parameters are resolved. +type resolvedWriteScenario struct { + // SelectionCypher contains the write-selection Cypher statement. + SelectionCypher string + // SelectionParams contains resolved parameters for the write-selection query. + SelectionParams map[string]any + // AffectedEntity identifies the entity class counted after a write. + AffectedEntity string + // ExpectedMatched sets the required number of matched entities. + ExpectedMatched int64 + // ExpectedAffected sets the required number of affected entities. + ExpectedAffected int64 + // PostState defines the state query evaluated after a write. + PostState []resolvedStateQuery +} + +// resolvedStateQuery contains a post-write state query after fixture parameters are resolved. +type resolvedStateQuery struct { + // Name labels the post-write state assertion in diagnostics and results. + Name string + // Cypher contains the Cypher statement under test. + Cypher string + // Params supplies literal query parameters. + Params map[string]any + // Expected defines the required observable result. + Expected ExpectedResult +} + +// writeMeasurement captures a write's matched and affected counts, duration, and post-state observations. +type writeMeasurement struct { + // Matched records entities matched by the write selection. + Matched int64 + // Affected records entities changed by the measured write. + Affected int64 + // Duration records elapsed time for this observation. + Duration time.Duration + // PostState contains the observed results of post-write validation queries. + PostState []StateQueryResult +} + +// timedReadAttestation is the runtime receipt captured outside a measured +// query's latency boundary for that exact invocation. +type timedReadAttestation struct { + InvocationID string + RequestedIdentity string + RuntimeIdentity string + RuntimeBranch string + FallbackExecuted *bool + Events []RuntimeReceiptEvent +} + +// timedReadAttestor arms and reads invocation-local runtime evidence. Begin +// and Complete execute outside the duration measurement. +type timedReadAttestor interface { + Begin(context.Context, int) error + Complete(context.Context, int) (timedReadAttestation, error) +} + +// countCypherRows executes a Cypher query and returns the number of result rows. func countCypherRows(tx graph.Transaction, cypher string, params map[string]any) (int64, error) { result := tx.Query(cypher, params) defer result.Close() @@ -33,39 +101,785 @@ func countCypherRows(tx graph.Transaction, cypher string, params map[string]any) rowCount++ } - return rowCount, result.Error() + if err := result.Error(); err != nil { + return 0, err + } + + return rowCount, nil +} + +// countRawRows executes a raw backend query and returns the number of result rows. +func countRawRows(tx graph.Transaction, sql string, params map[string]any) (int64, error) { + result := tx.Raw(sql, params) + defer result.Close() + + var rowCount int64 + for result.Next() { + rowCount++ + } + + if err := result.Error(); err != nil { + return 0, err + } + + return rowCount, nil +} + +// stableNodeObservation serializes a node using fixture-stable identity, kinds, and properties. +type stableNodeObservation struct { + // Identity contains the stable fixture identity emitted in observations. + Identity string `json:"identity"` + // Kinds lists stable node kinds in deterministic observation order. + Kinds []string `json:"kinds,omitempty"` + // Properties contains normalized property values. + Properties map[string]any `json:"properties,omitempty"` +} + +// stableRelationshipObservation serializes a relationship using stable endpoints, kind, identity, and properties. +type stableRelationshipObservation struct { + // Identity contains the stable fixture identity emitted in observations. + Identity string `json:"identity,omitempty"` + // Start contains the stable identity of the relationship's start node. + Start string `json:"start"` + // End contains the stable identity of the relationship's end node. + End string `json:"end"` + // Kind names the relationship kind preserved in the stable observation. + Kind string `json:"kind"` + // Properties contains normalized property values. + Properties map[string]any `json:"properties,omitempty"` +} + +// stablePathObservation serializes an ordered path as stable node and relationship observations. +type stablePathObservation struct { + // Nodes contains the stable node sequence. + Nodes []stableNodeObservation `json:"nodes"` + // Relationships contains the ordered stable relationship sequence in the path. + Relationships []stableRelationshipObservation `json:"relationships"` +} + +// reverseIDMap inverts fixture node-key mappings for stable result serialization. +func reverseIDMap(idMap opengraph.IDMap) map[graph.ID]string { + reversed := make(map[graph.ID]string, len(idMap)) + for name, id := range idMap { + reversed[id] = name + } + return reversed +} + +// stableIdentity maps a database identifier to its fixture key, falling back to its decimal representation. +func stableIdentity(id graph.ID, reversed map[graph.ID]string) string { + if name, found := reversed[id]; found { + return name + } + return fmt.Sprintf("unmapped-node:%d", id) +} + +// stableProperties returns properties with database identifiers replaced by stable fixture keys. +func stableProperties(properties *graph.Properties) map[string]any { + if properties == nil { + return nil + } + return properties.Map +} + +// stableNode converts a backend node to a fixture-stable serialized observation. +func stableNode(node *graph.Node, reversed map[graph.ID]string) stableNodeObservation { + kinds := node.Kinds.Strings() + sort.Strings(kinds) + return stableNodeObservation{ + Identity: stableIdentity(node.ID, reversed), + Kinds: kinds, + Properties: stableProperties(node.Properties), + } +} + +// stableRelationship converts a backend relationship to stable endpoints, kind, identity, and properties. +func stableRelationship(relationship *graph.Relationship, reversed map[graph.ID]string) stableRelationshipObservation { + kind := "" + if relationship.Kind != nil { + kind = relationship.Kind.String() + } + identity := "" + if relationship.Properties != nil { + if logicalKey, err := relationship.Properties.Get("logical_key").String(); err == nil { + identity = logicalKey + } + } + return stableRelationshipObservation{ + Identity: identity, + Start: stableIdentity(relationship.StartID, reversed), + End: stableIdentity(relationship.EndID, reversed), + Kind: kind, + Properties: stableProperties(relationship.Properties), + } +} + +// stablePath converts a backend path to stable ordered node and relationship observations. +func stablePath(path graph.Path, reversed map[graph.ID]string) (stablePathObservation, error) { + if len(path.Nodes) == 0 { + return stablePathObservation{}, fmt.Errorf("path has no nodes") + } + nodesByID := make(map[graph.ID]*graph.Node, len(path.Nodes)) + for _, node := range path.Nodes { + if node == nil { + return stablePathObservation{}, fmt.Errorf("path has a nil node") + } + nodesByID[node.ID] = node + } + + // Neo4j exposes the distinct node collection for cyclic paths while + // PostgreSQL exposes one node per traversal position. Reconstruct the public + // walk from the ordered relationships so cycles and self-loops normalize to + // the same repeated-node sequence on both backends. + orderedNodes := make([]*graph.Node, 1, len(path.Edges)+1) + orderedNodes[0] = path.Nodes[0] + currentID := path.Nodes[0].ID + for idx, relationship := range path.Edges { + if relationship == nil { + return stablePathObservation{}, fmt.Errorf("path relationship %d is nil", idx) + } + nextID := relationship.EndID + switch { + case relationship.StartID == currentID: + case relationship.EndID == currentID: + nextID = relationship.StartID + default: + return stablePathObservation{}, fmt.Errorf("path relationship %d is not contiguous with node ID %d", idx, currentID) + } + next, found := nodesByID[nextID] + if !found { + return stablePathObservation{}, fmt.Errorf("path relationship %d references missing node ID %d", idx, nextID) + } + orderedNodes = append(orderedNodes, next) + currentID = nextID + } + + observation := stablePathObservation{ + Nodes: make([]stableNodeObservation, len(orderedNodes)), + Relationships: make([]stableRelationshipObservation, len(path.Edges)), + } + for idx, node := range orderedNodes { + observation.Nodes[idx] = stableNode(node, reversed) + } + seenRelationships := make(map[graph.ID]struct{}, len(path.Edges)) + for idx, relationship := range path.Edges { + if _, duplicate := seenRelationships[relationship.ID]; duplicate { + return stablePathObservation{}, fmt.Errorf("path reuses relationship ID %d", relationship.ID) + } + seenRelationships[relationship.ID] = struct{}{} + observation.Relationships[idx] = stableRelationship(relationship, reversed) + } + return observation, nil +} + +// stableRowValues normalizes result values to stable scalar IDs or canonical path JSON. +func stableRowValues(values []any, mapper graph.ValueMapper, reversed map[graph.ID]string, scalarNodeIDs bool, pathValues bool) ([]any, error) { + stable := make([]any, len(values)) + for idx, value := range values { + switch typed := value.(type) { + case *graph.Node: + stable[idx] = stableNode(typed, reversed) + case graph.Node: + stable[idx] = stableNode(&typed, reversed) + case *graph.Relationship: + stable[idx] = stableRelationship(typed, reversed) + case graph.Relationship: + stable[idx] = stableRelationship(&typed, reversed) + case graph.Path: + path, err := stablePath(typed, reversed) + if err != nil { + return nil, err + } + stable[idx] = path + case *graph.Path: + path, err := stablePath(*typed, reversed) + if err != nil { + return nil, err + } + stable[idx] = path + default: + var relationship graph.Relationship + if mapper.Map(value, &relationship) { + stable[idx] = stableRelationship(&relationship, reversed) + continue + } + + var node graph.Node + if mapper.Map(value, &node) { + stable[idx] = stableNode(&node, reversed) + continue + } + + // The PostgreSQL path mapper accepts a map without path fields as an + // empty path, so only attempt this mapping when the result contract + // says the row contains paths. + if pathValues { + var path graph.Path + if mapper.Map(value, &path) { + observation, err := stablePath(path, reversed) + if err != nil { + return nil, err + } + stable[idx] = observation + continue + } + } + + if scalarNodeIDs { + if id, ok := scaleInt64(value); ok { + stable[idx] = stableIdentity(graph.ID(id), reversed) + continue + } + } + stable[idx] = value + } + } + return stable, nil +} + +// expectedPathRows serializes expected paths to the same canonical representation as observed paths. +func expectedPathRows(rows []ExpectedPath) ([]string, error) { + encoded := make([]string, len(rows)) + for idx, row := range rows { + value, err := json.Marshal(row) + if err != nil { + return nil, err + } + encoded[idx] = string(value) + } + + sort.Strings(encoded) + return encoded, nil +} + +// observedPathRows extracts and sorts canonical path observations from normalized rows. +func observedPathRows(rows []string) ([]string, error) { + encoded := make([]string, len(rows)) + for idx, row := range rows { + var values []json.RawMessage + if err := json.Unmarshal([]byte(row), &values); err != nil { + return nil, err + } + if len(values) != 1 { + return nil, fmt.Errorf("expected one path column, got %d", len(values)) + } + var path stablePathObservation + if err := json.Unmarshal(values[0], &path); err != nil { + return nil, err + } + signature := ExpectedPath{ + Nodes: make([]string, len(path.Nodes)), + RelationshipKinds: make([]string, len(path.Relationships)), + } + includeRelationshipKeys := false + for _, relationship := range path.Relationships { + includeRelationshipKeys = includeRelationshipKeys || relationship.Identity != "" + } + if includeRelationshipKeys { + signature.RelationshipKeys = make([]string, len(path.Relationships)) + } + for nodeIdx, node := range path.Nodes { + signature.Nodes[nodeIdx] = node.Identity + } + for relationshipIdx, relationship := range path.Relationships { + signature.RelationshipKinds[relationshipIdx] = relationship.Kind + if includeRelationshipKeys { + signature.RelationshipKeys[relationshipIdx] = relationship.Identity + } + } + value, err := json.Marshal(signature) + if err != nil { + return nil, err + } + encoded[idx] = string(value) + } + sort.Strings(encoded) + return encoded, nil +} + +// observeCypherRows executes Cypher and returns row count plus normalized observations. +func observeCypherRows(tx graph.Transaction, cypher string, params map[string]any, idMap opengraph.IDMap, scalarNodeIDs bool, pathValues bool) (int64, []string, error) { + result := tx.Query(cypher, params) + return observeResultRows(result, idMap, scalarNodeIDs, pathValues) +} + +// observeRawRows executes raw SQL and returns row count plus normalized observations. +func observeRawRows(tx graph.Transaction, sql string, params map[string]any, idMap opengraph.IDMap, scalarNodeIDs bool, pathValues bool) (int64, []string, error) { + result := tx.Raw(sql, params) + return observeResultRows(result, idMap, scalarNodeIDs, pathValues) +} + +// observeResultRows drains a result iterator into a count and sorted stable observations. +func observeResultRows(result graph.Result, idMap opengraph.IDMap, scalarNodeIDs bool, pathValues bool) (int64, []string, error) { + defer result.Close() + + var ( + rowCount int64 + rows []string + ) + for result.Next() { + rowCount++ + stableValues, err := stableRowValues(result.Values(), result.Mapper(), reverseIDMap(idMap), scalarNodeIDs, pathValues) + if err != nil { + return 0, nil, fmt.Errorf("stabilize observed row %d: %w", rowCount, err) + } + encoded, err := json.Marshal(stableValues) + if err != nil { + return 0, nil, fmt.Errorf("encode observed row %d: %w", rowCount, err) + } + rows = append(rows, string(encoded)) + } + if err := result.Error(); err != nil { + return 0, nil, err + } + + // Cypher does not promise row order without ORDER BY. Comparing sorted row + // encodings preserves multiplicity while avoiding a false mismatch when an + // otherwise identical plan returns rows in another order. + sort.Strings(rows) + return rowCount, rows, nil +} + +// validateExpectedObservations compares normalized rows with explicit scalar, ID-row, or path expectations. +func validateExpectedObservations(expected ExpectedResult, observed []string) error { + if len(expected.IDRows) > 0 { + expectedRows := make([]string, len(expected.IDRows)) + for idx, row := range expected.IDRows { + encoded, err := json.Marshal(row) + if err != nil { + return err + } + expectedRows[idx] = string(encoded) + } + sort.Strings(expectedRows) + if !slices.Equal(expectedRows, observed) { + return fmt.Errorf("stable ID rows differ: expected=%v observed=%v", expectedRows, observed) + } + } + if len(expected.PathRows) > 0 { + expectedRows, err := expectedPathRows(expected.PathRows) + if err != nil { + return err + } + observedRows, err := observedPathRows(observed) + if err != nil { + return err + } + if !slices.Equal(expectedRows, observedRows) { + return fmt.Errorf("stable path rows differ: expected=%v observed=%v", expectedRows, observedRows) + } + } + if expected.ScalarInt != nil { + expectedRow := fmt.Sprintf("[%d]", *expected.ScalarInt) + if len(observed) != 1 || observed[0] != expectedRow { + return fmt.Errorf("scalar result differs: expected=%s observed=%v", expectedRow, observed) + } + } + return nil } -func measureCypher(ctx context.Context, db graph.Database, cypher string, params map[string]any, iterations int) (int64, DurationStats, error) { +// observeCypher runs a Cypher query in a read transaction and returns stable observations. +func observeCypher(tx graph.Transaction, cypher string, params map[string]any) (StateQueryResult, error) { + result := tx.Query(cypher, params) + defer result.Close() + + var observation StateQueryResult + for result.Next() { + observation.RowCount++ + if observation.RowCount == 1 && len(result.Values()) > 0 { + if scalar, ok := scaleInt64(result.Values()[0]); ok { + observation.ScalarInt = &scalar + } + } + } + + if err := result.Error(); err != nil { + return StateQueryResult{}, err + } + + return observation, nil +} + +// resultContainsNodeIDs reports whether the expected result kind requires stable node-identifier mapping. +func resultContainsNodeIDs(expected ExpectedResult) bool { + return expected.ResultKind == "id_set" || expected.ResultKind == "id_rows" +} + +// resultContainsPaths reports whether expected observations require canonical path normalization. +func resultContainsPaths(expected ExpectedResult) bool { + return expected.ResultKind == "path_set" +} + +// measureCypher executes cypher and records its timing observations. +func measureCypher(ctx context.Context, db graph.Database, cypher string, params map[string]any, expected ExpectedResult, idMap opengraph.IDMap, iterations int) (int64, []string, DurationStats, error) { + return measureCypherWithWarmups(ctx, db, cypher, params, expected, idMap, 0, iterations) +} + +// measureCypherWithWarmups executes cypher with warmups and records its timing observations. +func measureCypherWithWarmups(ctx context.Context, db graph.Database, cypher string, params map[string]any, expected ExpectedResult, idMap opengraph.IDMap, warmupIterations, iterations int) (int64, []string, DurationStats, error) { + return measureReadWithWarmups(ctx, db, cypher, params, expected, idMap, warmupIterations, iterations, false) +} + +func measureCypherWithWarmupsOptions(ctx context.Context, db graph.Database, cypher string, params map[string]any, expected ExpectedResult, idMap opengraph.IDMap, warmupIterations, iterations int, options ...graph.TransactionOption) (int64, []string, DurationStats, error) { + return measureReadWithWarmupsAndAttestation(ctx, db, cypher, params, expected, idMap, warmupIterations, iterations, false, nil, options...) +} + +// measureRawSQLWithWarmups executes raw SQL with warmups and records its timing observations. +func measureRawSQLWithWarmups(ctx context.Context, db graph.Database, sql string, params map[string]any, expected ExpectedResult, idMap opengraph.IDMap, warmupIterations, iterations int) (int64, []string, DurationStats, error) { + return measureReadWithWarmups(ctx, db, sql, params, expected, idMap, warmupIterations, iterations, true) +} + +func measureRawSQLWithWarmupsOptions(ctx context.Context, db graph.Database, sql string, params map[string]any, expected ExpectedResult, idMap opengraph.IDMap, warmupIterations, iterations int, options ...graph.TransactionOption) (int64, []string, DurationStats, error) { + return measureReadWithWarmupsAndAttestation(ctx, db, sql, params, expected, idMap, warmupIterations, iterations, true, nil, options...) +} + +// measureRawSQLWithWarmupsAndAttestation preserves the ordinary raw-SQL +// measurement boundary while binding each timed sample to an exact runtime +// receipt armed immediately before and read immediately after execution. +func measureRawSQLWithWarmupsAndAttestation(ctx context.Context, db graph.Database, sql string, params map[string]any, expected ExpectedResult, idMap opengraph.IDMap, warmupIterations, iterations int, attestor timedReadAttestor) (int64, []string, DurationStats, error) { + return measureReadWithWarmupsAndAttestation(ctx, db, sql, params, expected, idMap, warmupIterations, iterations, true, attestor) +} + +// measureRawSQLWithWarmupsAndAttestationOptions measures a raw production +// statement under explicit graph transaction options while keeping receipt +// arming and reading outside the timed transaction. +func measureRawSQLWithWarmupsAndAttestationOptions(ctx context.Context, db graph.Database, sql string, params map[string]any, expected ExpectedResult, idMap opengraph.IDMap, warmupIterations, iterations int, attestor timedReadAttestor, options ...graph.TransactionOption) (int64, []string, DurationStats, error) { + return measureReadWithWarmupsAndAttestation(ctx, db, sql, params, expected, idMap, warmupIterations, iterations, true, attestor, options...) +} + +// measureReadWithWarmups executes read with warmups and records its timing observations. +func measureReadWithWarmups(ctx context.Context, db graph.Database, query string, params map[string]any, expected ExpectedResult, idMap opengraph.IDMap, warmupIterations, iterations int, raw bool) (int64, []string, DurationStats, error) { + return measureReadWithWarmupsAndAttestation(ctx, db, query, params, expected, idMap, warmupIterations, iterations, raw, nil) +} + +func measureReadWithWarmupsAndAttestation(ctx context.Context, db graph.Database, query string, params map[string]any, expected ExpectedResult, idMap opengraph.IDMap, warmupIterations, iterations int, raw bool, attestor timedReadAttestor, options ...graph.TransactionOption) (int64, []string, DurationStats, error) { if iterations < 1 { - return 0, DurationStats{}, fmt.Errorf("iterations must be at least 1") + return 0, nil, DurationStats{}, fmt.Errorf("iterations must be at least 1") + } + if warmupIterations < 0 { + return 0, nil, DurationStats{}, fmt.Errorf("warmup iterations must not be negative") + } + + coldStart := time.Now() + if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { + _, err := countReadRows(tx, query, params, raw) + return err + }, options...); err != nil { + return 0, nil, DurationStats{}, err + } + coldDuration := time.Since(coldStart) + for range warmupIterations { + if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { + _, err := countReadRows(tx, query, params, raw) + return err + }, options...); err != nil { + return 0, nil, DurationStats{}, err + } } - var warmupRows int64 + var ( + warmupRows int64 + preflightObserved []string + stabilizeNodeIDs = resultContainsNodeIDs(expected) + stabilizePaths = resultContainsPaths(expected) + ) if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { var err error - warmupRows, err = countCypherRows(tx, cypher, params) + warmupRows, preflightObserved, err = observeReadRows(tx, query, params, idMap, stabilizeNodeIDs, stabilizePaths, raw) return err - }); err != nil { - return 0, DurationStats{}, err + }, options...); err != nil { + return 0, nil, DurationStats{}, err } durations := make([]time.Duration, iterations) + attestations := make([]timedReadAttestation, iterations) for idx := range iterations { + if attestor != nil { + if err := attestor.Begin(ctx, idx+1); err != nil { + return 0, nil, DurationStats{}, fmt.Errorf("arm timed runtime attestation %d: %w", idx+1, err) + } + } start := time.Now() if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { - _, err := countCypherRows(tx, cypher, params) + _, err := countReadRows(tx, query, params, raw) return err - }); err != nil { - return 0, DurationStats{}, err + }, options...); err != nil { + if attestor != nil { + _, _ = attestor.Complete(context.WithoutCancel(ctx), idx+1) + } + return 0, nil, DurationStats{}, err } durations[idx] = time.Since(start) + if attestor != nil { + attestation, err := attestor.Complete(ctx, idx+1) + if err != nil { + return 0, nil, DurationStats{}, fmt.Errorf("read timed runtime attestation %d: %w", idx+1, err) + } + attestations[idx] = attestation + } + } + + var ( + postflightRows int64 + postflightObserved []string + ) + if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { + var err error + postflightRows, postflightObserved, err = observeReadRows(tx, query, params, idMap, stabilizeNodeIDs, stabilizePaths, raw) + return err + }, options...); err != nil { + return 0, nil, DurationStats{}, err + } + if postflightRows != warmupRows { + return 0, nil, DurationStats{}, fmt.Errorf("postflight row count changed: preflight=%d postflight=%d", warmupRows, postflightRows) + } + if !slices.Equal(preflightObserved, postflightObserved) { + return 0, nil, DurationStats{}, fmt.Errorf("postflight result changed despite stable row count") + } + if err := validateExpectedObservations(expected, preflightObserved); err != nil { + return 0, nil, DurationStats{}, err } stats, err := computeDurationStats(durations) if err != nil { - return 0, DurationStats{}, err + return 0, nil, DurationStats{}, err + } + stats.WarmupIterations = warmupIterations + if attestor != nil { + for idx := range attestations { + stats.Samples[idx].RuntimeInvocationID = attestations[idx].InvocationID + stats.Samples[idx].RequestedIdentity = attestations[idx].RequestedIdentity + stats.Samples[idx].RuntimeIdentity = attestations[idx].RuntimeIdentity + stats.Samples[idx].RuntimeBranch = attestations[idx].RuntimeBranch + stats.Samples[idx].FallbackExecuted = attestations[idx].FallbackExecuted + stats.Samples[idx].RuntimeAttestation = "timed_invocation" + stats.Samples[idx].RuntimeReceiptEvents = append([]RuntimeReceiptEvent(nil), attestations[idx].Events...) + } + } + + stats.Samples = append([]LatencySample{{ + Round: 1, + Iteration: 0, + Classification: "cold", + Duration: coldDuration, + }}, stats.Samples...) + + return warmupRows, preflightObserved, stats, nil +} + +// countReadRows dispatches to raw SQL or Cypher row counting according to raw. +func countReadRows(tx graph.Transaction, query string, params map[string]any, raw bool) (int64, error) { + if raw { + return countRawRows(tx, query, params) + } + return countCypherRows(tx, query, params) +} + +// observeReadRows dispatches a read observation to raw SQL or Cypher execution. +func observeReadRows(tx graph.Transaction, query string, params map[string]any, idMap opengraph.IDMap, scalarNodeIDs, pathValues, raw bool) (int64, []string, error) { + if raw { + return observeRawRows(tx, query, params, idMap, scalarNodeIDs, pathValues) + } + return observeCypherRows(tx, query, params, idMap, scalarNodeIDs, pathValues) +} + +// measureWriteCypher executes write cypher and records its timing observations. +func measureWriteCypher( + ctx context.Context, + db graph.Database, + cypher string, + params map[string]any, + scenario resolvedWriteScenario, + iterations int, +) (writeMeasurement, DurationStats, error) { + return measureWriteCypherWithWarmups(ctx, db, cypher, params, scenario, 0, iterations) +} + +// measureWriteCypherWithWarmups executes write cypher with warmups and records its timing observations. +func measureWriteCypherWithWarmups( + ctx context.Context, + db graph.Database, + cypher string, + params map[string]any, + scenario resolvedWriteScenario, + warmupIterations int, + iterations int, +) (writeMeasurement, DurationStats, error) { + if iterations < 1 { + return writeMeasurement{}, DurationStats{}, fmt.Errorf("iterations must be at least 1") + } + if warmupIterations < 0 { + return writeMeasurement{}, DurationStats{}, fmt.Errorf("warmup iterations must not be negative") + } + + // The first untimed execution remains the cold diagnostic. Additional + // configured warmups are also untimed and must preserve its semantics. + warmup, err := measureWriteIteration(ctx, db, cypher, params, scenario) + if err != nil { + return writeMeasurement{}, DurationStats{}, err + } + for idx := 0; idx < warmupIterations; idx++ { + next, err := measureWriteIteration(ctx, db, cypher, params, scenario) + if err != nil { + return writeMeasurement{}, DurationStats{}, err + } + if next.Matched != warmup.Matched || next.Affected != warmup.Affected { + return writeMeasurement{}, DurationStats{}, fmt.Errorf("warm-up iteration %d changed cardinality", idx+1) + } + } + + durations := make([]time.Duration, iterations) + for idx := range iterations { + measurement, err := measureWriteIteration(ctx, db, cypher, params, scenario) + if err != nil { + return writeMeasurement{}, DurationStats{}, err + } + if measurement.Matched != warmup.Matched || measurement.Affected != warmup.Affected { + return writeMeasurement{}, DurationStats{}, fmt.Errorf( + "write iteration %d changed cardinality: matched=%d affected=%d, warm-up matched=%d affected=%d", + idx+1, + measurement.Matched, + measurement.Affected, + warmup.Matched, + warmup.Affected, + ) + } + durations[idx] = measurement.Duration + } + + stats, err := computeDurationStats(durations) + if err != nil { + return writeMeasurement{}, DurationStats{}, err + } + stats.WarmupIterations = warmupIterations + + stats.Samples = append([]LatencySample{{ + Round: 1, + Iteration: 0, + Classification: "cold", + Duration: warmup.Duration, + }}, stats.Samples...) + + return warmup, stats, nil +} + +// measureWriteIteration executes write iteration and records its timing observations. +func measureWriteIteration( + ctx context.Context, + db graph.Database, + cypher string, + params map[string]any, + scenario resolvedWriteScenario, +) (writeMeasurement, error) { + var measurement writeMeasurement + + err := db.WriteTransaction(ctx, func(tx graph.Transaction) error { + matched, err := countCypherRows(tx, scenario.SelectionCypher, scenario.SelectionParams) + if err != nil { + return fmt.Errorf("count matched rows: %w", err) + } + measurement.Matched = matched + if matched != scenario.ExpectedMatched { + return fmt.Errorf("expected %d matched rows, got %d", scenario.ExpectedMatched, matched) + } + + before, err := countAffectedEntities(tx, scenario.AffectedEntity) + if err != nil { + return err + } + + start := time.Now() + if _, err := countCypherRows(tx, cypher, params); err != nil { + return fmt.Errorf("execute mutation: %w", err) + } + measurement.Duration = time.Since(start) + + after, err := countAffectedEntities(tx, scenario.AffectedEntity) + if err != nil { + return err + } + measurement.Affected = before - after + if measurement.Affected != scenario.ExpectedAffected { + return fmt.Errorf("expected %d affected %ss, got %d", scenario.ExpectedAffected, scenario.AffectedEntity, measurement.Affected) + } + + for _, stateQuery := range scenario.PostState { + observation, err := observeCypher(tx, stateQuery.Cypher, stateQuery.Params) + if err != nil { + return fmt.Errorf("post-state %q: %w", stateQuery.Name, err) + } + observation.Name = stateQuery.Name + if err := checkStateExpectation(observation, stateQuery.Expected); err != nil { + return fmt.Errorf("post-state %q: %w", stateQuery.Name, err) + } + measurement.PostState = append(measurement.PostState, observation) + } + + return errScaleWriteRollback + }) + if errors.Is(err, errScaleWriteRollback) { + return measurement, nil + } + if err != nil { + return writeMeasurement{}, err + } + + return writeMeasurement{}, fmt.Errorf("write scenario committed instead of rolling back") +} + +// countAffectedEntities returns the transaction-visible node or relationship count selected by entity. +func countAffectedEntities(tx graph.Transaction, entity string) (int64, error) { + switch entity { + case "node": + return tx.Nodes().Count() + case "relationship": + return tx.Relationships().Count() + default: + return 0, fmt.Errorf("unsupported affected entity %q", entity) + } +} + +// checkStateExpectation validates a post-write observation against its declared row-count and scalar expectations. +func checkStateExpectation(observation StateQueryResult, expected ExpectedResult) error { + if expected.RowCount != nil && observation.RowCount != *expected.RowCount { + return fmt.Errorf("expected %d rows, got %d", *expected.RowCount, observation.RowCount) + } + if expected.ScalarInt != nil { + if observation.ScalarInt == nil { + return fmt.Errorf("expected scalar integer %d, got no integer scalar", *expected.ScalarInt) + } + if *observation.ScalarInt != *expected.ScalarInt { + return fmt.Errorf("expected scalar integer %d, got %d", *expected.ScalarInt, *observation.ScalarInt) + } + } + + return nil +} + +// scaleInt64 converts supported integral numeric representations to int64 without unsigned overflow. +func scaleInt64(value any) (int64, bool) { + switch typedValue := value.(type) { + case int: + return int64(typedValue), true + case int32: + return int64(typedValue), true + case int64: + return typedValue, true + case uint: + if uint64(typedValue) <= math.MaxInt64 { + return int64(typedValue), true + } + case uint32: + return int64(typedValue), true + case uint64: + if typedValue <= math.MaxInt64 { + return int64(typedValue), true + } + case float64: + if math.Trunc(typedValue) == typedValue { + return int64(typedValue), true + } } - return warmupRows, stats, nil + return 0, false } diff --git a/cmd/graphbench/measure_test.go b/cmd/graphbench/measure_test.go new file mode 100644 index 00000000..3c1edd09 --- /dev/null +++ b/cmd/graphbench/measure_test.go @@ -0,0 +1,449 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "errors" + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/opengraph" + "github.com/stretchr/testify/require" +) + +// TestStableRowValuesReverseMapsNodeIDs verifies that scalar and node IDs become fixture keys while node-kind metadata is preserved. +func TestStableRowValuesReverseMapsNodeIDs(t *testing.T) { + values, err := stableRowValues( + []any{int64(101), graph.NewNode(102, nil, graph.StringKind("Group"))}, + graph.NewValueMapper(), + reverseIDMap(opengraph.IDMap{"start": 101, "end": 102}), + true, + false, + ) + require.NoError(t, err) + require.Equal(t, "start", values[0]) + require.Equal(t, stableNodeObservation{ + Identity: "end", + Kinds: []string{"Group"}, + }, values[1]) +} + +// TestResultContainsNodeIDs verifies that only ID-set and ID-row expectations request physical-to-logical ID normalization. +func TestResultContainsNodeIDs(t *testing.T) { + require.True(t, resultContainsNodeIDs(ExpectedResult{ResultKind: "id_set"})) + require.True(t, resultContainsNodeIDs(ExpectedResult{ResultKind: "id_rows"})) + require.False(t, resultContainsNodeIDs(ExpectedResult{ResultKind: "scalar"})) + require.False(t, resultContainsNodeIDs(ExpectedResult{ResultKind: "path_set"})) +} + +// TestStableRowValuesMapsNativePathValues verifies that driver-native paths normalize into logical node identities and directed relationship observations. +func TestStableRowValuesMapsNativePathValues(t *testing.T) { + start := graph.NewNode(1, nil, graph.StringKind("Start")) + end := graph.NewNode(2, nil, graph.StringKind("End")) + edge := graph.NewRelationship(3, 1, 2, nil, graph.StringKind("Edge")) + mapper := graph.NewValueMapper(func(value, target any) bool { + path, sourceOK := value.(string) + mapped, targetOK := target.(*graph.Path) + if sourceOK && targetOK && path == "native-path" { + *mapped = graph.Path{ + Nodes: []*graph.Node{start, end}, + Edges: []*graph.Relationship{edge}, + } + return true + } + return false + }) + + values, err := stableRowValues( + []any{"native-path"}, + mapper, + reverseIDMap(opengraph.IDMap{"start": 1, "end": 2}), + false, + true, + ) + + require.NoError(t, err) + require.Equal(t, stablePathObservation{ + Nodes: []stableNodeObservation{ + { + Identity: "start", + Kinds: []string{"Start"}, + }, + { + Identity: "end", + Kinds: []string{"End"}, + }, + }, + Relationships: []stableRelationshipObservation{{ + Start: "start", + End: "end", + Kind: "Edge", + }}, + }, values[0]) +} + +// TestStablePathReconstructsRepeatedCycleAndSelfLoopNodes verifies a backend +// path that supplies distinct nodes still produces the complete ordered Cypher +// walk, including repeated occurrences at cycles and self-loops. +func TestStablePathReconstructsRepeatedCycleAndSelfLoopNodes(t *testing.T) { + root := graph.NewNode(1, nil) + cycle := graph.NewNode(2, nil) + terminal := graph.NewNode(3, nil) + path := graph.Path{ + Nodes: []*graph.Node{root, cycle, terminal}, + Edges: []*graph.Relationship{ + graph.NewRelationship(10, 1, 2, nil, graph.StringKind("Expand")), + graph.NewRelationship(11, 2, 1, nil, graph.StringKind("Expand")), + graph.NewRelationship(12, 1, 1, nil, graph.StringKind("Expand")), + graph.NewRelationship(13, 1, 3, nil, graph.StringKind("Complete")), + }, + } + + observed, err := stablePath(path, reverseIDMap(opengraph.IDMap{ + "root": 1, "cycle": 2, "terminal": 3, + })) + + require.NoError(t, err) + require.Equal(t, []string{"root", "cycle", "root", "root", "terminal"}, []string{ + observed.Nodes[0].Identity, + observed.Nodes[1].Identity, + observed.Nodes[2].Identity, + observed.Nodes[3].Identity, + observed.Nodes[4].Identity, + }) +} + +// TestStablePathReconstructsInboundTraversal verifies relationship storage +// direction does not reverse the public path walk. +func TestStablePathReconstructsInboundTraversal(t *testing.T) { + root := graph.NewNode(1, nil) + terminal := graph.NewNode(2, nil) + observed, err := stablePath(graph.Path{ + Nodes: []*graph.Node{root, terminal}, + Edges: []*graph.Relationship{ + graph.NewRelationship(10, 2, 1, nil, graph.StringKind("Expand")), + }, + }, reverseIDMap(opengraph.IDMap{"root": 1, "terminal": 2})) + + require.NoError(t, err) + require.Equal(t, "root", observed.Nodes[0].Identity) + require.Equal(t, "terminal", observed.Nodes[1].Identity) +} + +// TestStablePathRejectsNoncontiguousRelationships verifies malformed backend +// path values cannot manufacture a stable observation. +func TestStablePathRejectsNoncontiguousRelationships(t *testing.T) { + _, err := stablePath(graph.Path{ + Nodes: []*graph.Node{graph.NewNode(1, nil), graph.NewNode(2, nil), graph.NewNode(3, nil)}, + Edges: []*graph.Relationship{ + graph.NewRelationship(10, 2, 3, nil, graph.StringKind("Expand")), + }, + }, nil) + + require.ErrorContains(t, err, "is not contiguous") +} + +// TestStableRowValuesRejectsRelationshipReuseWithinPath verifies that observation normalization rejects a trail containing the same physical relationship twice. +func TestStableRowValuesRejectsRelationshipReuseWithinPath(t *testing.T) { + start := graph.NewNode(1, nil) + end := graph.NewNode(2, nil) + relationship := graph.NewRelationship(10, 1, 2, nil, graph.StringKind("Edge")) + _, err := stableRowValues([]any{graph.Path{ + Nodes: []*graph.Node{start, end, start}, + Edges: []*graph.Relationship{relationship, relationship}, + }}, graph.NewValueMapper(), reverseIDMap(opengraph.IDMap{"start": 1, "end": 2}), false, true) + require.ErrorContains(t, err, "reuses relationship ID 10") +} + +// TestStableRelationshipUsesLogicalFixtureKeyAsCrossBackendIdentity verifies that a relationship's logical_key property, rather than its backend ID, identifies it across engines. +func TestStableRelationshipUsesLogicalFixtureKeyAsCrossBackendIdentity(t *testing.T) { + properties := graph.NewProperties().Set("logical_key", "branch-0001-level-02") + relationship := graph.NewRelationship(99, 1, 2, properties, graph.StringKind("MemberOf")) + + stable := stableRelationship(relationship, map[graph.ID]string{1: "start", 2: "end"}) + require.Equal(t, "branch-0001-level-02", stable.Identity) + require.Equal(t, "start", stable.Start) + require.Equal(t, "end", stable.End) +} + +// TestObserveCypherReturnsZeroValueOnResultError verifies that an iterator failure cannot leak a partially populated state observation. +func TestObserveCypherReturnsZeroValueOnResultError(t *testing.T) { + tx := &scaleWriteTestTransaction{ + database: &scaleWriteTestDatabase{}, + } + + observation, err := observeCypher(tx, "unexpected", nil) + + require.ErrorContains(t, err, "unexpected query") + require.Equal(t, StateQueryResult{}, observation) +} + +// TestMeasureWriteCypherRollsBackWarmupAndEveryIteration verifies matched/affected/post-state measurements, cold-versus-warm classification, and rollback after every sampled mutation. +func TestMeasureWriteCypherRollsBackWarmupAndEveryIteration(t *testing.T) { + database := &scaleWriteTestDatabase{ + nodes: 2, + relationships: 3, + deleteCount: 1, + } + postStateCount := int64(2) + scenario := resolvedWriteScenario{ + SelectionCypher: "selection", + AffectedEntity: "relationship", + ExpectedMatched: 1, + ExpectedAffected: 1, + PostState: []resolvedStateQuery{{ + Name: "surviving relationships", + Cypher: "relationship count", + Expected: ExpectedResult{ScalarInt: &postStateCount}, + }}, + } + + measurement, stats, err := measureWriteCypher(context.Background(), database, "delete", nil, scenario, 2) + + require.NoError(t, err) + require.Equal(t, int64(1), measurement.Matched) + require.Equal(t, int64(1), measurement.Affected) + require.Equal(t, int64(2), *measurement.PostState[0].ScalarInt) + require.Equal(t, 2, stats.Iterations) + require.Len(t, stats.Samples, 3) + require.Equal(t, "cold", stats.Samples[0].Classification) + require.Equal(t, "warm", stats.Samples[1].Classification) + require.Equal(t, 3, database.writeTransactions) + require.Equal(t, int64(3), database.relationships, "every write transaction must roll back") +} + +// TestMeasureWriteCypherRecordsConfiguredUntimedWarmups verifies that configured warmups execute transactions and update metadata without entering the timing sample set. +func TestMeasureWriteCypherRecordsConfiguredUntimedWarmups(t *testing.T) { + database := &scaleWriteTestDatabase{ + nodes: 2, + relationships: 3, + deleteCount: 1, + } + scenario := resolvedWriteScenario{ + SelectionCypher: "selection", + AffectedEntity: "relationship", + ExpectedMatched: 1, + ExpectedAffected: 1, + } + + _, stats, err := measureWriteCypherWithWarmups(context.Background(), database, "delete", nil, scenario, 2, 1) + require.NoError(t, err) + require.Equal(t, 2, stats.WarmupIterations) + require.Len(t, stats.Samples, 2, "configured warmups must not become samples") + require.Equal(t, 4, database.writeTransactions, "cold + two warmups + one timed transaction") +} + +// TestMeasureWriteCypherRejectsOverBroadMutation verifies that deleting more relationships than declared fails validation and leaves the fixture unchanged. +func TestMeasureWriteCypherRejectsOverBroadMutation(t *testing.T) { + database := &scaleWriteTestDatabase{ + nodes: 2, + relationships: 3, + deleteCount: 2, + } + scenario := resolvedWriteScenario{ + SelectionCypher: "selection", + AffectedEntity: "relationship", + ExpectedMatched: 1, + ExpectedAffected: 1, + PostState: []resolvedStateQuery{{ + Name: "survivors", + Cypher: "relationship count", + Expected: ExpectedResult{RowCount: int64Pointer(1)}, + }}, + } + + _, _, err := measureWriteCypher(context.Background(), database, "delete", nil, scenario, 1) + require.ErrorContains(t, err, "expected 1 affected relationships, got 2") + require.Equal(t, int64(3), database.relationships) +} + +// TestMeasureWriteCypherRejectsUnderBroadMutation verifies that deleting fewer relationships than declared fails validation and leaves the fixture unchanged. +func TestMeasureWriteCypherRejectsUnderBroadMutation(t *testing.T) { + database := &scaleWriteTestDatabase{ + nodes: 2, + relationships: 3, + deleteCount: 0, + } + scenario := resolvedWriteScenario{ + SelectionCypher: "selection", + AffectedEntity: "relationship", + ExpectedMatched: 1, + ExpectedAffected: 1, + PostState: []resolvedStateQuery{{ + Name: "survivors", + Cypher: "relationship count", + Expected: ExpectedResult{RowCount: int64Pointer(1)}, + }}, + } + + _, _, err := measureWriteCypher(context.Background(), database, "delete", nil, scenario, 1) + require.ErrorContains(t, err, "expected 1 affected relationships, got 0") + require.Equal(t, int64(3), database.relationships) +} + +// int64Pointer returns a pointer to the supplied integer for optional expectations. +func int64Pointer(value int64) *int64 { + return &value +} + +// scaleWriteTestDatabase models mutable entity counts and rollback boundaries for write measurements. +type scaleWriteTestDatabase struct { + // Database supplies methods outside the transaction interaction under test. + graph.Database + + // nodes is the mutable node cardinality visible to count queries. + nodes int64 + + // relationships is the mutable relationship cardinality restored on rollback. + relationships int64 + + // deleteCount controls how many relationships the synthetic mutation removes. + deleteCount int64 + + // writeTransactions counts cold, warmup, and measured transaction attempts. + writeTransactions int +} + +// WriteTransaction runs the delegate and restores entity counts when its sentinel error requests rollback. +func (s *scaleWriteTestDatabase) WriteTransaction(_ context.Context, delegate graph.TransactionDelegate, _ ...graph.TransactionOption) error { + s.writeTransactions++ + originalNodes := s.nodes + originalRelationships := s.relationships + err := delegate(&scaleWriteTestTransaction{database: s}) + if err != nil { + s.nodes = originalNodes + s.relationships = originalRelationships + } + + return err +} + +// scaleWriteTestTransaction interprets the synthetic selection, deletion, and post-state query names used by write measurements. +type scaleWriteTestTransaction struct { + // Transaction supplies operations outside the query and count surfaces under test. + graph.Transaction + + // database owns the mutable cardinalities affected by synthetic queries. + database *scaleWriteTestDatabase +} + +// Query maps synthetic query names to selection rows, cardinality mutation, post-state counts, or a terminal error. +func (s *scaleWriteTestTransaction) Query(cypher string, _ map[string]any) graph.Result { + switch cypher { + case "selection": + return &scaleWriteTestResult{rows: [][]any{{int64(1)}}} + case "delete": + s.database.relationships -= s.database.deleteCount + return &scaleWriteTestResult{} + case "relationship count": + return &scaleWriteTestResult{rows: [][]any{{s.database.relationships}}} + default: + return &scaleWriteTestResult{err: errors.New("unexpected query")} + } +} + +// Nodes returns the current node-cardinality snapshot used to compute affected entities. +func (s *scaleWriteTestTransaction) Nodes() graph.NodeQuery { + return &scaleWriteTestNodeQuery{count: s.database.nodes} +} + +// Relationships returns the current relationship-cardinality snapshot used to compute affected entities. +func (s *scaleWriteTestTransaction) Relationships() graph.RelationshipQuery { + return &scaleWriteTestRelationshipQuery{count: s.database.relationships} +} + +// scaleWriteTestNodeQuery exposes a fixed node cardinality through the graph query interface. +type scaleWriteTestNodeQuery struct { + // NodeQuery supplies query methods other than Count. + graph.NodeQuery + + // count is the node cardinality returned to mutation accounting. + count int64 +} + +// Count returns the node snapshot without a query failure. +func (s *scaleWriteTestNodeQuery) Count() (int64, error) { + return s.count, nil +} + +// scaleWriteTestRelationshipQuery exposes a fixed relationship cardinality through the graph query interface. +type scaleWriteTestRelationshipQuery struct { + // RelationshipQuery supplies query methods other than Count. + graph.RelationshipQuery + + // count is the relationship cardinality returned to mutation accounting. + count int64 +} + +// Count returns the relationship snapshot without a query failure. +func (s *scaleWriteTestRelationshipQuery) Count() (int64, error) { + return s.count, nil +} + +// scaleWriteTestResult iterates configured rows and errors for write-measurement tests. +type scaleWriteTestResult struct { + // rows contains the synthetic values exposed by iteration. + rows [][]any + + // idx is the one-based cursor position after a successful Next call. + idx int + + // err is returned after iteration completes. + err error +} + +// Next advances the one-based cursor while synthetic rows remain. +func (s *scaleWriteTestResult) Next() bool { + if s.idx >= len(s.rows) { + return false + } + s.idx++ + return true +} + +// Keys returns no column names because write-measurement observations consume values positionally. +func (s *scaleWriteTestResult) Keys() []string { + return nil +} + +// Values returns the current synthetic row or nil before and after valid iteration. +func (s *scaleWriteTestResult) Values() []any { + if s.idx == 0 || s.idx > len(s.rows) { + return nil + } + + return s.rows[s.idx-1] +} + +// Mapper returns the zero mapper because the synthetic rows contain primitive counts only. +func (s *scaleWriteTestResult) Mapper() graph.ValueMapper { + return graph.ValueMapper{} +} + +// Scan satisfies graph.Result; these tests consume rows through Values. +func (s *scaleWriteTestResult) Scan(...any) error { + return nil +} + +// Error returns the configured terminal iterator error. +func (s *scaleWriteTestResult) Error() error { + return s.err +} + +// Close satisfies graph.Result; this fake owns no resource. +func (s *scaleWriteTestResult) Close() {} diff --git a/cmd/graphbench/neo4j.go b/cmd/graphbench/neo4j.go index 429fa8f1..4500fb61 100644 --- a/cmd/graphbench/neo4j.go +++ b/cmd/graphbench/neo4j.go @@ -20,24 +20,36 @@ import ( "context" "fmt" "net/url" + "strconv" "strings" neo4jcore "github.com/neo4j/neo4j-go-driver/v5/neo4j" "github.com/specterops/dawgs" + "github.com/specterops/dawgs/databaseguard" dawgsneo4j "github.com/specterops/dawgs/drivers/neo4j" "github.com/specterops/dawgs/graph" "github.com/specterops/dawgs/opengraph" "github.com/specterops/dawgs/util/size" ) +// neo4jRunner owns the Neo4j driver and database used to execute benchmark cases. type neo4jRunner struct { - datasetDir string - db graph.Database - planDriver neo4jcore.DriverWithContext + // datasetDir locates fixture and corpus files on disk. + datasetDir string + // db provides graph transactions for fixture preparation and query execution. + db graph.Database + // planDriver supplies the Neo4j driver used only for untimed PROFILE or EXPLAIN capture. + planDriver neo4jcore.DriverWithContext + // databaseName selects the Neo4j database targeted by the benchmark session. databaseName string } +// newNeo4jRunner opens a Neo4j driver and selects the optional database encoded in the URI. func newNeo4jRunner(ctx context.Context, datasetDir, connection string, corpus ScaleCorpus) (*neo4jRunner, error) { + if err := databaseguard.ValidateEnvironment(connection); err != nil { + return nil, fmt.Errorf("refuse destructive Neo4j GraphBench target: %w", err) + } + db, err := dawgs.Open(ctx, dawgsneo4j.DriverName, dawgs.Config{ GraphQueryMemoryLimit: size.Gibibyte, ConnectionString: connection, @@ -71,6 +83,7 @@ func newNeo4jRunner(ctx context.Context, datasetDir, connection string, corpus S }, nil } +// Close releases both Neo4j drivers owned by the benchmark runner. func (s *neo4jRunner) Close(ctx context.Context) error { var closeErr error if s.planDriver != nil { @@ -85,13 +98,18 @@ func (s *neo4jRunner) Close(ctx context.Context) error { return closeErr } -func (s *neo4jRunner) Run(ctx context.Context, iterations int, corpus ScaleCorpus) ([]CaseResult, error) { +// Run reloads each fixture dataset and measures every corpus case supported by Neo4j. +func (s *neo4jRunner) Run(ctx context.Context, warmupIterations, iterations int, corpus ScaleCorpus) ([]CaseResult, error) { var ( records []CaseResult casesByDataset = scaleCasesByDataset(corpus) ) for _, datasetName := range scaleCorpusDatasets(corpus) { + fixture, err := fixtureMetadata(s.datasetDir, datasetName) + if err != nil { + return nil, err + } if err := clearGraph(ctx, s.db); err != nil { return nil, fmt.Errorf("clear graph for %s: %w", datasetName, err) } @@ -106,7 +124,8 @@ func (s *neo4jRunner) Run(ctx context.Context, iterations int, corpus ScaleCorpu continue } - record := s.runCase(ctx, iterations, testCase, idMap) + record := s.runCase(ctx, warmupIterations, iterations, testCase, idMap) + attachFixtureMetadata(&record, fixture) records = append(records, record) } } @@ -114,7 +133,8 @@ func (s *neo4jRunner) Run(ctx context.Context, iterations int, corpus ScaleCorpu return records, nil } -func (s *neo4jRunner) runCase(ctx context.Context, iterations int, testCase ScaleCase, idMap opengraph.IDMap) CaseResult { +// runCase resolves fixture parameters, measures the selected Neo4j read or write workload, and records correctness and timing status in one CaseResult. +func (s *neo4jRunner) runCase(ctx context.Context, warmupIterations, iterations int, testCase ScaleCase, idMap opengraph.IDMap) CaseResult { params, err := resolveCaseParams(testCase, idMap) record := newCaseResult(testCase, ModeNeo4j, params) if err != nil { @@ -123,18 +143,42 @@ func (s *neo4jRunner) runCase(ctx context.Context, iterations int, testCase Scal return record } - rowCount, stats, err := measureCypher(ctx, s.db, testCase.Cypher, params, iterations) - if err != nil { - record.Status = StatusError - record.Error = err.Error() - return record - } + if testCase.WriteScenario == nil { + rowCount, observedRows, stats, err := measureCypherWithWarmups(ctx, s.db, testCase.Cypher, params, testCase.Expected, idMap, warmupIterations, iterations) + if err != nil { + record.Status = StatusError + record.Error = err.Error() + return record + } - record.RowCount = rowCount - record.Stats = stats - applyRowExpectation(&record) + record.RowCount = rowCount + record.ObservedRows = observedRows + record.Stats = stats + labelLatencySamples(&record.Stats, ModeNeo4j, testCase) + applyRowExpectation(&record) + } else { + scenario, err := resolveWriteScenario(testCase, idMap) + if err != nil { + record.Status = StatusError + record.Error = err.Error() + return record + } + + measurement, stats, err := measureWriteCypherWithWarmups(ctx, s.db, testCase.Cypher, params, scenario, warmupIterations, iterations) + if err != nil { + record.Status = StatusError + record.Error = err.Error() + return record + } - plan, operators, err := s.explain(ctx, testCase.Cypher, params) + record.MatchedCount = &measurement.Matched + record.AffectedCount = &measurement.Affected + record.PostState = measurement.PostState + record.Stats = stats + labelLatencySamples(&record.Stats, ModeNeo4j, testCase) + } + + plan, operators, err := s.explain(ctx, testCase.Cypher, params, testCase.WriteScenario != nil) if err != nil { if record.Status == StatusOK { record.Status = StatusError @@ -148,9 +192,14 @@ func (s *neo4jRunner) runCase(ctx context.Context, iterations int, testCase Scal return record } -func (s *neo4jRunner) explain(ctx context.Context, cypherQuery string, params map[string]any) (plan *Neo4jPlanNode, operators []string, err error) { +// explain submits native Neo4j PROFILE for reads and EXPLAIN for writes after the timed block. +func (s *neo4jRunner) explain(ctx context.Context, cypherQuery string, params map[string]any, write bool) (plan *Neo4jPlanNode, operators []string, err error) { + accessMode := neo4jcore.AccessModeRead + if write { + accessMode = neo4jcore.AccessModeWrite + } session := s.planDriver.NewSession(ctx, neo4jcore.SessionConfig{ - AccessMode: neo4jcore.AccessModeRead, + AccessMode: accessMode, DatabaseName: s.databaseName, }) defer func() { @@ -159,7 +208,7 @@ func (s *neo4jRunner) explain(ctx context.Context, cypherQuery string, params ma } }() - result, err := session.Run(ctx, "EXPLAIN "+cypherWithoutTerminator(cypherQuery), params) + result, err := session.Run(ctx, neo4jPlanCaptureStatement(cypherQuery, write), params) if err != nil { return nil, nil, err } @@ -168,21 +217,62 @@ func (s *neo4jRunner) explain(ctx context.Context, cypherQuery string, params ma if err != nil { return nil, nil, err } - if summary.Plan() == nil { + if write { + explainPlan := summary.Plan() + if explainPlan == nil { + return nil, nil, nil + } + + metadata := neo4jProfileMetadata(explainPlan.Arguments(), neo4jServerAgent(summary), false) + planNode := convertNeo4jPlan(explainPlan) + planNode.ProfileMetadata = &metadata + + return &planNode, neo4jOperators(planNode), nil + } + + profile := summary.Profile() + if profile == nil { return nil, nil, nil } - planNode := convertNeo4jPlan(summary.Plan()) + metadata := neo4jProfileMetadata(profile.Arguments(), neo4jServerAgent(summary), true) + planNode := convertNeo4jProfiledPlan(profile, metadata.internalTraversalOpaque()) + planNode.ProfileMetadata = &metadata + return &planNode, neo4jOperators(planNode), nil } +// neo4jPlanCaptureStatement selects PROFILE only for read-only cases and retains non-executing EXPLAIN for writes. +func neo4jPlanCaptureStatement(cypherQuery string, write bool) string { + command := "PROFILE" + if write { + command = "EXPLAIN" + } + + return command + " " + cypherWithoutTerminator(cypherQuery) +} + +func neo4jServerAgent(summary neo4jcore.ResultSummary) string { + if server := summary.Server(); server != nil { + return server.Agent() + } + + return "" +} + +// neo4jPlanDriverConfig contains a Neo4j server URI and optional target database parsed from a connection string. type neo4jPlanDriverConfig struct { - Target string - Username string - Password string + // Target contains the Neo4j server URI without a database path. + Target string + // Username contains the Neo4j username decoded from the connection URI. + Username string + // Password contains the Neo4j password decoded from the connection URI. + Password string + // DatabaseName selects the Neo4j database targeted by the session. DatabaseName string } +// parseNeo4jPlanDriverConfig parses a Neo4j connection string while preserving its server URI and database path. func parseNeo4jPlanDriverConfig(connStr string) (neo4jPlanDriverConfig, error) { connectionURL, err := url.Parse(connStr) if err != nil { @@ -218,6 +308,7 @@ func parseNeo4jPlanDriverConfig(connStr string) (neo4jPlanDriverConfig, error) { }, nil } +// neo4jDatabaseName returns the optional single-segment database name encoded in a Neo4j URI path. func neo4jDatabaseName(connectionURL *url.URL) (string, error) { databasePath := strings.Trim(connectionURL.EscapedPath(), "/") if databasePath == "" { @@ -238,6 +329,7 @@ func neo4jDatabaseName(connectionURL *url.URL) (string, error) { return databaseName, nil } +// openNeo4jPlanDriver parses the benchmark connection settings and returns a context-aware driver together with the selected database name. func openNeo4jPlanDriver(connStr string) (neo4jcore.DriverWithContext, string, error) { cfg, err := parseNeo4jPlanDriverConfig(connStr) if err != nil { @@ -252,18 +344,61 @@ func openNeo4jPlanDriver(connStr string) (neo4jcore.DriverWithContext, string, e return driver, cfg.DatabaseName, nil } +// Neo4jProfileMetadata identifies the planner, runtime, and server used for a captured plan. +type Neo4jProfileMetadata struct { + CaptureMode string `json:"capture_mode"` + Profiled bool `json:"profiled"` + Planner string `json:"planner,omitempty"` + PlannerImplementation string `json:"planner_implementation,omitempty"` + PlannerVersion string `json:"planner_version,omitempty"` + Runtime string `json:"runtime,omitempty"` + RuntimeImplementation string `json:"runtime_implementation,omitempty"` + RuntimeVersion string `json:"runtime_version,omitempty"` + CypherVersion string `json:"cypher_version,omitempty"` + ServerAgent string `json:"server_agent,omitempty"` +} + +// Neo4jPlanNode models the recursive operator tree returned by Neo4j PROFILE or EXPLAIN. type Neo4jPlanNode struct { - Operator string `json:"operator"` - Arguments map[string]string `json:"arguments,omitempty"` - Identifiers []string `json:"identifiers,omitempty"` - Children []Neo4jPlanNode `json:"children,omitempty"` + // Operator identifies the backend plan operator at this node. + Operator string `json:"operator"` + // Arguments maps backend plan argument names to stable string representations. + Arguments map[string]string `json:"arguments,omitempty"` + // Identifiers lists variables or identifiers referenced by the Neo4j plan node. + Identifiers []string `json:"identifiers,omitempty"` + // EstimatedRows records planner-estimated output rows when Neo4j supplies them. + EstimatedRows *float64 `json:"estimated_rows,omitempty"` + // ActualRows records rows emitted by an executed PROFILE operator. + ActualRows *int64 `json:"actual_rows,omitempty"` + // Loops records operator loops when Neo4j exposes them as a plan argument. + Loops *int64 `json:"loops,omitempty"` + // DBHits records data-store accesses reported for an executed PROFILE operator. + DBHits *int64 `json:"db_hits,omitempty"` + // PageCacheHits records page-cache hits reported for an executed PROFILE operator. + PageCacheHits *int64 `json:"page_cache_hits,omitempty"` + // PageCacheMisses records page-cache misses reported for an executed PROFILE operator. + PageCacheMisses *int64 `json:"page_cache_misses,omitempty"` + // PageCacheHitRatio records the server-reported page-cache hit ratio. + PageCacheHitRatio *float64 `json:"page_cache_hit_ratio,omitempty"` + // TimeNS records operator time in nanoseconds when exposed by the Neo4j server. + TimeNS *int64 `json:"time_ns,omitempty"` + // InternalTraversalWork marks Neo4j 4.4 SP/ASP relationship work as opaque. + InternalTraversalWork string `json:"internal_traversal_work,omitempty"` + // ProfileMetadata records root planner/runtime and capture metadata. + ProfileMetadata *Neo4jProfileMetadata `json:"profile_metadata,omitempty"` + // Children contains child Neo4j plan operators in backend order. + Children []Neo4jPlanNode `json:"children,omitempty"` } +// convertNeo4jPlan recursively converts a Neo4j plan into the stable serialized plan-node schema. func convertNeo4jPlan(plan neo4jcore.Plan) Neo4jPlanNode { + arguments := plan.Arguments() node := Neo4jPlanNode{ - Operator: plan.Operator(), - Arguments: stringifyArguments(plan.Arguments()), - Identifiers: append([]string(nil), plan.Identifiers()...), + Operator: normalizeNeo4jOperator(plan.Operator()), + Arguments: stringifyArguments(arguments), + Identifiers: append([]string(nil), plan.Identifiers()...), + EstimatedRows: neo4jFloatArgument(arguments, "EstimatedRows", "estimatedRows"), + Loops: neo4jIntArgument(arguments, "Loops", "loops"), } for _, child := range plan.Children() { @@ -273,6 +408,110 @@ func convertNeo4jPlan(plan neo4jcore.Plan) Neo4jPlanNode { return node } +// convertNeo4jProfiledPlan recursively converts executed PROFILE data while preserving child order. +func convertNeo4jProfiledPlan(plan neo4jcore.ProfiledPlan, opaqueInternalTraversal bool) Neo4jPlanNode { + arguments := plan.Arguments() + operator := normalizeNeo4jOperator(plan.Operator()) + node := Neo4jPlanNode{ + Operator: operator, + Arguments: stringifyArguments(arguments), + Identifiers: append([]string(nil), plan.Identifiers()...), + EstimatedRows: neo4jFloatArgument(arguments, "EstimatedRows", "estimatedRows"), + ActualRows: neo4jInt64Pointer(plan.Records()), + Loops: neo4jIntArgument(arguments, "Loops", "loops"), + DBHits: neo4jInt64Pointer(plan.DbHits()), + PageCacheHits: neo4jInt64Pointer(plan.PageCacheHits()), + PageCacheMisses: neo4jInt64Pointer(plan.PageCacheMisses()), + PageCacheHitRatio: neo4jFloat64Pointer(plan.PageCacheHitRatio()), + TimeNS: neo4jInt64Pointer(plan.Time()), + } + if opaqueInternalTraversal && strings.Contains(strings.ToLower(neo4jOperatorBase(operator)), "shortestpath") { + node.InternalTraversalWork = "opaque" + } + + for _, child := range plan.Children() { + node.Children = append(node.Children, convertNeo4jProfiledPlan(child, opaqueInternalTraversal)) + } + + return node +} + +func neo4jProfileMetadata(arguments map[string]any, serverAgent string, profiled bool) Neo4jProfileMetadata { + captureMode := "EXPLAIN" + if profiled { + captureMode = "PROFILE" + } + + return Neo4jProfileMetadata{ + CaptureMode: captureMode, + Profiled: profiled, + Planner: neo4jStringArgument(arguments, "planner"), + PlannerImplementation: neo4jStringArgument(arguments, "planner-impl"), + PlannerVersion: neo4jStringArgument(arguments, "planner-version"), + Runtime: neo4jStringArgument(arguments, "runtime"), + RuntimeImplementation: neo4jStringArgument(arguments, "runtime-impl"), + RuntimeVersion: neo4jStringArgument(arguments, "runtime-version"), + CypherVersion: neo4jStringArgument(arguments, "version"), + ServerAgent: serverAgent, + } +} + +func (s Neo4jProfileMetadata) internalTraversalOpaque() bool { + return strings.HasPrefix(s.PlannerVersion, "4.4") || + strings.HasPrefix(s.RuntimeVersion, "4.4") || + strings.Contains(s.CypherVersion, "4.4") || + strings.Contains(s.ServerAgent, "/4.4") +} + +func neo4jStringArgument(arguments map[string]any, name string) string { + if value, ok := arguments[name]; ok { + return fmt.Sprint(value) + } + + return "" +} + +func neo4jFloatArgument(arguments map[string]any, names ...string) *float64 { + for _, name := range names { + value, ok := arguments[name] + if !ok { + continue + } + + parsed, err := strconv.ParseFloat(fmt.Sprint(value), 64) + if err == nil { + return neo4jFloat64Pointer(parsed) + } + } + + return nil +} + +func neo4jIntArgument(arguments map[string]any, names ...string) *int64 { + for _, name := range names { + value, ok := arguments[name] + if !ok { + continue + } + + parsed, err := strconv.ParseInt(fmt.Sprint(value), 10, 64) + if err == nil { + return neo4jInt64Pointer(parsed) + } + } + + return nil +} + +func neo4jInt64Pointer(value int64) *int64 { + return &value +} + +func neo4jFloat64Pointer(value float64) *float64 { + return &value +} + +// stringifyArguments converts plan arguments to stable strings in a fresh map. func stringifyArguments(arguments map[string]any) map[string]string { if len(arguments) == 0 { return nil @@ -286,6 +525,7 @@ func stringifyArguments(arguments map[string]any) map[string]string { return values } +// neo4jOperators flattens a Neo4j plan tree in traversal order with exactly one backend suffix. func neo4jOperators(root Neo4jPlanNode) []string { var ( operators []string @@ -293,7 +533,7 @@ func neo4jOperators(root Neo4jPlanNode) []string { ) walk = func(node Neo4jPlanNode) { - operators = append(operators, node.Operator+"@neo4j") + operators = append(operators, normalizeNeo4jOperator(node.Operator)) for _, child := range node.Children { walk(child) } @@ -303,6 +543,25 @@ func neo4jOperators(root Neo4jPlanNode) []string { return operators } +func normalizeNeo4jOperator(operator string) string { + base := neo4jOperatorBase(operator) + if base == "" { + return "" + } + + return base + "@neo4j" +} + +func neo4jOperatorBase(operator string) string { + operator = strings.TrimSpace(operator) + for strings.HasSuffix(operator, "@neo4j") { + operator = strings.TrimSpace(strings.TrimSuffix(operator, "@neo4j")) + } + + return operator +} + +// cypherWithoutTerminator trims surrounding whitespace and one trailing Cypher semicolon. func cypherWithoutTerminator(cypherQuery string) string { return strings.TrimSuffix(strings.TrimSpace(cypherQuery), ";") } diff --git a/cmd/graphbench/neo4j_test.go b/cmd/graphbench/neo4j_test.go index a01058c9..ffde6d1d 100644 --- a/cmd/graphbench/neo4j_test.go +++ b/cmd/graphbench/neo4j_test.go @@ -20,6 +20,7 @@ import ( "net/url" "testing" + neo4jcore "github.com/neo4j/neo4j-go-driver/v5/neo4j" "github.com/stretchr/testify/require" ) @@ -48,11 +49,118 @@ func TestNeo4jDatabaseNameRejectsNestedPath(t *testing.T) { func TestNeo4jOperatorsAnnotatesOperators(t *testing.T) { operators := neo4jOperators(Neo4jPlanNode{ - Operator: "ProduceResults", + Operator: "ProduceResults@neo4j@neo4j", Children: []Neo4jPlanNode{{ - Operator: "AllNodesScan", + Operator: "AllNodesScan@neo4j", }}, }) require.Equal(t, []string{"ProduceResults@neo4j", "AllNodesScan@neo4j"}, operators) } + +func TestNeo4jPlanCaptureStatementProfilesReadsAndExplainsWrites(t *testing.T) { + require.Equal(t, "PROFILE MATCH (n) RETURN n", neo4jPlanCaptureStatement(" MATCH (n) RETURN n; ", false)) + require.Equal(t, "EXPLAIN CREATE (n)", neo4jPlanCaptureStatement("CREATE (n);", true)) +} + +func TestConvertNeo4jPlanPreservesEndpointChildOrder(t *testing.T) { + plan := stubNeo4jPlan{ + operator: "CartesianProduct@neo4j@neo4j", + arguments: map[string]any{"EstimatedRows": 2.5, "Loops": int64(3)}, + children: []neo4jcore.Plan{ + stubNeo4jPlan{operator: "NodeIndexSeek", identifiers: []string{"start"}}, + stubNeo4jPlan{operator: "NodeIndexSeek", identifiers: []string{"end"}}, + }, + } + + converted := convertNeo4jPlan(plan) + + require.Equal(t, "CartesianProduct@neo4j", converted.Operator) + require.Equal(t, 2.5, *converted.EstimatedRows) + require.Equal(t, int64(3), *converted.Loops) + require.Equal(t, []string{"start"}, converted.Children[0].Identifiers) + require.Equal(t, []string{"end"}, converted.Children[1].Identifiers) +} + +func TestConvertNeo4jProfiledPlanCapturesMetricsMetadataAndOpaqueShortestPath(t *testing.T) { + profile := stubNeo4jProfiledPlan{ + operator: "ProduceResults@neo4j", + arguments: map[string]any{ + "EstimatedRows": 1.5, + "planner": "COST", + "planner-impl": "IDP", + "planner-version": "4.4", + "runtime": "INTERPRETED", + "runtime-impl": "INTERPRETED", + "runtime-version": "4.4", + "version": "CYPHER 4.4", + }, + dbHits: 11, + records: 7, + pageCacheHits: 13, + pageCacheMisses: 2, + pageCacheHitRatio: 0.86, + timeNS: 101, + children: []neo4jcore.ProfiledPlan{ + stubNeo4jProfiledPlan{operator: "ShortestPath@neo4j@neo4j", dbHits: 1, records: 1}, + stubNeo4jProfiledPlan{operator: "NodeIndexSeek", identifiers: []string{"end"}, dbHits: 3, records: 1}, + }, + } + metadata := neo4jProfileMetadata(profile.Arguments(), "Neo4j/4.4.44", true) + + converted := convertNeo4jProfiledPlan(profile, metadata.internalTraversalOpaque()) + converted.ProfileMetadata = &metadata + + require.Equal(t, "ProduceResults@neo4j", converted.Operator) + require.Equal(t, 1.5, *converted.EstimatedRows) + require.Equal(t, int64(7), *converted.ActualRows) + require.Equal(t, int64(11), *converted.DBHits) + require.Equal(t, int64(13), *converted.PageCacheHits) + require.Equal(t, int64(2), *converted.PageCacheMisses) + require.Equal(t, 0.86, *converted.PageCacheHitRatio) + require.Equal(t, int64(101), *converted.TimeNS) + require.Equal(t, "PROFILE", converted.ProfileMetadata.CaptureMode) + require.True(t, converted.ProfileMetadata.Profiled) + require.Equal(t, "4.4", converted.ProfileMetadata.PlannerVersion) + require.Equal(t, "4.4", converted.ProfileMetadata.RuntimeVersion) + require.Equal(t, "ShortestPath@neo4j", converted.Children[0].Operator) + require.Equal(t, "opaque", converted.Children[0].InternalTraversalWork) + require.Empty(t, converted.Children[1].InternalTraversalWork) + require.Equal(t, []string{"end"}, converted.Children[1].Identifiers) +} + +type stubNeo4jPlan struct { + operator string + arguments map[string]any + identifiers []string + children []neo4jcore.Plan +} + +func (s stubNeo4jPlan) Operator() string { return s.operator } +func (s stubNeo4jPlan) Arguments() map[string]any { return s.arguments } +func (s stubNeo4jPlan) Identifiers() []string { return s.identifiers } +func (s stubNeo4jPlan) Children() []neo4jcore.Plan { return s.children } + +type stubNeo4jProfiledPlan struct { + operator string + arguments map[string]any + identifiers []string + dbHits int64 + records int64 + children []neo4jcore.ProfiledPlan + pageCacheMisses int64 + pageCacheHits int64 + pageCacheHitRatio float64 + timeNS int64 +} + +func (s stubNeo4jProfiledPlan) Operator() string { return s.operator } +func (s stubNeo4jProfiledPlan) Arguments() map[string]any { return s.arguments } +func (s stubNeo4jProfiledPlan) Identifiers() []string { return s.identifiers } +func (s stubNeo4jProfiledPlan) DbHits() int64 { return s.dbHits } +func (s stubNeo4jProfiledPlan) Records() int64 { return s.records } +func (s stubNeo4jProfiledPlan) Children() []neo4jcore.ProfiledPlan { return s.children } +func (s stubNeo4jProfiledPlan) PageCacheMisses() int64 { return s.pageCacheMisses } +func (s stubNeo4jProfiledPlan) PageCacheHits() int64 { return s.pageCacheHits } +func (s stubNeo4jProfiledPlan) PageCacheHitRatio() float64 { return s.pageCacheHitRatio } +func (s stubNeo4jProfiledPlan) Time() int64 { return s.timeNS } diff --git a/cmd/graphbench/orientation_policy.go b/cmd/graphbench/orientation_policy.go new file mode 100644 index 00000000..2edb0f0d --- /dev/null +++ b/cmd/graphbench/orientation_policy.go @@ -0,0 +1,20 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + +// isOrientationProbePolicy recognizes immutable orientation selector +// identities without treating an unqualified version as production eligible. +func isOrientationProbePolicy(identity string) bool { + switch optimize.ExpansionSearchPolicy(identity) { + case optimize.ExpansionSearchPolicyOrientationProbeV1, + optimize.ExpansionSearchPolicyOrientationProbeV2: + return true + default: + return false + } +} diff --git a/cmd/graphbench/orientation_policy_test.go b/cmd/graphbench/orientation_policy_test.go new file mode 100644 index 00000000..3506ca66 --- /dev/null +++ b/cmd/graphbench/orientation_policy_test.go @@ -0,0 +1,20 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestOrientationProbePolicyRecognitionIsVersionExplicit(t *testing.T) { + require.True(t, isOrientationProbePolicy("orientation-probe-v1")) + require.True(t, isOrientationProbePolicy("orientation-probe-v2")) + require.False(t, isOrientationProbePolicy("orientation-probe-v3")) + require.False(t, isOrientationProbePolicy("ORIENTATION-PROBE-V2")) + require.False(t, isOrientationProbePolicy("")) +} diff --git a/cmd/graphbench/orientation_selector_report.go b/cmd/graphbench/orientation_selector_report.go new file mode 100644 index 00000000..97cbaa05 --- /dev/null +++ b/cmd/graphbench/orientation_selector_report.go @@ -0,0 +1,568 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "os" + "slices" + "sort" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" +) + +const orientationSelectorReportVersion = 1 + +// OrientationSelectorReportOptions configures the matched shadow/incumbent/ +// reverse comparison and its frozen qualification protocol. +type OrientationSelectorReportOptions struct { + Seed int64 + Confidence float64 + BootstrapCount int + Protocol string +} + +// OrientationLatencyGate records one frozen relative-or-absolute latency +// rule. A case passes when either the ratio upper bound or absolute upper gap +// stays within its declared limit. +type OrientationLatencyGate struct { + BaselineIdentity string `json:"baseline_identity"` + ObservedIdentity string `json:"observed_identity"` + BaselineSamples int `json:"baseline_samples"` + ObservedSamples int `json:"observed_samples"` + Ratio RatioInterval `json:"median_ratio"` + AbsoluteChange DurationInterval `json:"median_absolute_change"` + RatioUpperLimit float64 `json:"ratio_upper_limit"` + AbsoluteFloor time.Duration `json:"absolute_floor"` + AbsoluteGapUpper time.Duration `json:"absolute_gap_upper"` + Passed bool `json:"passed"` +} + +// OrientationSelectorCase reports shadow attribution, exact-arm regret, and +// probe-only overhead for one topology bucket. +type OrientationSelectorCase struct { + Dataset string `json:"dataset"` + Name string `json:"name"` + QualificationSplit string `json:"qualification_split"` + QualificationRole string `json:"qualification_role"` + ThresholdTuningEligible bool `json:"threshold_tuning_eligible"` + QualificationEligible bool `json:"qualification_eligible"` + Rounds int `json:"matched_rounds"` + WouldSelectIdentity string `json:"would_select_identity"` + FastestExactIdentity string `json:"fastest_exact_identity"` + ExactObservationsMatched bool `json:"exact_observations_matched"` + SelectorRegret OrientationLatencyGate `json:"selector_regret"` + ProbeOverhead OrientationLatencyGate `json:"probe_overhead"` + Passed bool `json:"passed"` + Reasons []string `json:"reasons,omitempty"` +} + +// OrientationSelectorReport validates that shadow selection is attributable, +// low-regret, and cheap while the incumbent remains the only shadow execution +// arm. Diagnostic and legacy records never contribute to qualification. +type OrientationSelectorReport struct { + Version int `json:"version"` + Policy string `json:"policy"` + Protocol string `json:"protocol"` + Seed int64 `json:"seed"` + Confidence float64 `json:"confidence_level"` + ShadowArtifactSHA256 string `json:"shadow_artifact_sha256,omitempty"` + IncumbentArtifactSHA256 string `json:"incumbent_artifact_sha256,omitempty"` + ReverseArtifactSHA256 string `json:"reverse_artifact_sha256,omitempty"` + AAReportSHA256 string `json:"aa_report_sha256,omitempty"` + SelectorRegretRatioLimit float64 `json:"selector_regret_ratio_upper_limit"` + ProbeOverheadRatioLimit float64 `json:"probe_overhead_ratio_upper_limit"` + ProbeOverheadAbsoluteLimit time.Duration `json:"probe_overhead_absolute_limit"` + EvidencePassed bool `json:"evidence_passed"` + TrainingCases int `json:"training_cases"` + HoldoutCases int `json:"holdout_cases"` + TrainingPassed bool `json:"training_passed"` + HoldoutPassed bool `json:"holdout_passed"` + QualificationPassed bool `json:"qualification_passed"` + Cases []OrientationSelectorCase `json:"cases"` +} + +type orientationSelectorSeries struct { + shadow roundSamples + incumbent roundSamples + reverse roundSamples + wouldSelect string +} + +// buildOrientationSelectorReport compares a true-shadow artifact with matched +// exact incumbent and forced-reverse artifacts. The shadow's public result and +// runtime identity must remain incumbent even when would_select names reverse. +func buildOrientationSelectorReport( + shadowRecords, incumbentRecords, reverseRecords []CaseResult, + aa *AAResolutionReport, + options OrientationSelectorReportOptions, +) (OrientationSelectorReport, error) { + if options.Confidence <= 0 || options.Confidence >= 1 { + return OrientationSelectorReport{}, fmt.Errorf("confidence level must be between 0 and 1") + } + if options.BootstrapCount == 0 { + options.BootstrapCount = defaultBootstrapCount + } + if options.BootstrapCount < 1 { + return OrientationSelectorReport{}, fmt.Errorf("bootstrap count must be positive") + } + protocol := options.Protocol + if protocol == "" { + protocol = referencePairProtocolConfirmation + } + minimumWarmups, minimumRounds, maximumRounds, minimumSamples := 20, 10, 20, 50 + if protocol == referencePairProtocolDiscovery { + minimumWarmups, minimumRounds, maximumRounds, minimumSamples = 5, 5, 20, 10 + } else if protocol != referencePairProtocolConfirmation { + return OrientationSelectorReport{}, fmt.Errorf("unsupported orientation selector protocol %q", protocol) + } + + if err := validateAAResolutionEvidence(aa, incumbentRecords, options.Confidence); err != nil { + return OrientationSelectorReport{}, fmt.Errorf("incumbent A/A evidence: %w", err) + } + incumbentHost, err := artifactHostFingerprint(incumbentRecords) + if err != nil { + return OrientationSelectorReport{}, err + } + for name, records := range map[string][]CaseResult{"shadow": shadowRecords, "reverse": reverseRecords} { + host, err := artifactHostFingerprint(records) + if err != nil { + return OrientationSelectorReport{}, fmt.Errorf("%s artifact host: %w", name, err) + } + if host != incumbentHost { + return OrientationSelectorReport{}, fmt.Errorf("%s artifact host does not match incumbent host", name) + } + } + + series, keys, err := collectOrientationSelectorSeries(shadowRecords, incumbentRecords, reverseRecords) + if err != nil { + return OrientationSelectorReport{}, err + } + report := OrientationSelectorReport{ + Version: orientationSelectorReportVersion, + Policy: string(optimize.ExpansionSearchPolicyOrientationProbeV1), + Protocol: protocol, + Seed: options.Seed, + Confidence: options.Confidence, + SelectorRegretRatioLimit: 1.10, + ProbeOverheadRatioLimit: 1.10, + ProbeOverheadAbsoluteLimit: 100 * time.Microsecond, + EvidencePassed: true, + } + trainingPassed, holdoutPassed := true, true + gateOptions := PerfGateOptions{Seed: options.Seed, Confidence: options.Confidence, BootstrapCount: options.BootstrapCount} + for index, key := range keys { + current := series[key] + shadow, incumbent := matchedRounds(current.shadow, current.incumbent) + incumbent, reverse := matchedRounds(incumbent, current.reverse) + shadow, incumbent = matchedRounds(shadow, incumbent) + if len(shadow) < minimumRounds || len(shadow) > maximumRounds { + return OrientationSelectorReport{}, fmt.Errorf("%s/%s requires %d-%d matched orientation rounds, got %d", key.dataset, key.name, minimumRounds, maximumRounds, len(shadow)) + } + for _, round := range sortedRounds(shadow) { + if len(shadow[round]) < minimumSamples || len(incumbent[round]) < minimumSamples || len(reverse[round]) < minimumSamples { + return OrientationSelectorReport{}, fmt.Errorf("%s/%s round %d requires %d samples per orientation arm", key.dataset, key.name, round, minimumSamples) + } + } + if err := validateOrientationArmOrder(shadowRecords, incumbentRecords, reverseRecords, key, sortedRounds(shadow), minimumWarmups); err != nil { + return OrientationSelectorReport{}, err + } + + split, err := qualificationSplit(key, shadowRecords, incumbentRecords, reverseRecords) + if err != nil { + return OrientationSelectorReport{}, err + } + role, tuningEligible, qualificationEligible := orientationQualificationRole(split, protocol) + fastestIdentity, fastest := fastestOrientationExactArm(incumbent, reverse) + selected := incumbent + if current.wouldSelect == string(optimize.ExpansionSearchSuffixSeededReverse) { + selected = reverse + } + seed := options.Seed + int64(index)*7919 + _, selectorFloorAbsolute, err := aaTimingFloor(aa, key, false, 0) + if err != nil { + return OrientationSelectorReport{}, err + } + selectorRegret := orientationLatencyGate( + fastestIdentity, + current.wouldSelect, + fastest, + selected, + 1.10, + selectorFloorAbsolute, + seed, + gateOptions, + ) + probeOverhead := orientationLatencyGate( + string(optimize.ExpansionSearchStepwiseForward), + string(optimize.ExpansionSearchPolicyOrientationProbeV1), + incumbent, + shadow, + 1.10, + report.ProbeOverheadAbsoluteLimit, + seed+3, + gateOptions, + ) + entry := OrientationSelectorCase{ + Dataset: key.dataset, + Name: key.name, + QualificationSplit: split, + QualificationRole: role, + ThresholdTuningEligible: tuningEligible, + QualificationEligible: qualificationEligible, + Rounds: len(shadow), + WouldSelectIdentity: current.wouldSelect, + FastestExactIdentity: fastestIdentity, + ExactObservationsMatched: true, + SelectorRegret: selectorRegret, + ProbeOverhead: probeOverhead, + Passed: selectorRegret.Passed && probeOverhead.Passed, + } + if !selectorRegret.Passed { + entry.Reasons = append(entry.Reasons, "selector regret exceeds the 1.10/A/A floor") + } + if !probeOverhead.Passed { + entry.Reasons = append(entry.Reasons, "shadow probe overhead exceeds 10% and 100us") + } + if !entry.Passed { + report.EvidencePassed = false + } + if qualificationEligible { + switch split { + case "training": + report.TrainingCases++ + trainingPassed = trainingPassed && entry.Passed + case "holdout": + report.HoldoutCases++ + holdoutPassed = holdoutPassed && entry.Passed + } + } + report.Cases = append(report.Cases, entry) + } + report.TrainingPassed = protocol == referencePairProtocolConfirmation && report.TrainingCases > 0 && trainingPassed + report.HoldoutPassed = protocol == referencePairProtocolConfirmation && report.HoldoutCases > 0 && holdoutPassed + report.QualificationPassed = report.TrainingPassed && report.HoldoutPassed + return report, nil +} + +func collectOrientationSelectorSeries( + shadowRecords, incumbentRecords, reverseRecords []CaseResult, +) (map[performanceKey]*orientationSelectorSeries, []performanceKey, error) { + series := map[performanceKey]*orientationSelectorSeries{} + for _, record := range shadowRecords { + if record.ExecutionMode != ModePostgresSQL || record.TraversalTelemetry == nil || record.TraversalTelemetry.Summary.WouldSelectIdentity == "" { + continue + } + key := performanceKey{dataset: record.Dataset, name: record.Name, backend: record.ExecutionMode} + if series[key] == nil { + series[key] = &orientationSelectorSeries{shadow: roundSamples{}, incumbent: roundSamples{}, reverse: roundSamples{}} + } + if err := validateOrientationRecord(record, "shadow"); err != nil { + return nil, nil, err + } + wouldSelect := record.TraversalTelemetry.Summary.WouldSelectIdentity + if series[key].wouldSelect != "" && series[key].wouldSelect != wouldSelect { + return nil, nil, fmt.Errorf("%s/%s changes shadow would_select identity across rounds", key.dataset, key.name) + } + series[key].wouldSelect = wouldSelect + appendOrientationWarmSamples(series[key].shadow, record) + } + if len(series) == 0 { + return nil, nil, fmt.Errorf("shadow artifact has no attributable orientation shadow records") + } + + for arm, records := range map[string][]CaseResult{"incumbent": incumbentRecords, "reverse": reverseRecords} { + for _, record := range records { + key := performanceKey{dataset: record.Dataset, name: record.Name, backend: record.ExecutionMode} + current := series[key] + if current == nil { + continue + } + if err := validateOrientationRecord(record, arm); err != nil { + return nil, nil, err + } + if arm == "incumbent" { + appendOrientationWarmSamples(current.incumbent, record) + } else { + appendOrientationWarmSamples(current.reverse, record) + } + } + } + + keys := make([]performanceKey, 0, len(series)) + for key, current := range series { + if len(current.incumbent) == 0 || len(current.reverse) == 0 { + return nil, nil, fmt.Errorf("%s/%s lacks matched incumbent or forced-reverse records", key.dataset, key.name) + } + if err := validateOrientationExactObservations(key, shadowRecords, incumbentRecords, reverseRecords); err != nil { + return nil, nil, err + } + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { + return keys[i].dataset < keys[j].dataset || keys[i].dataset == keys[j].dataset && keys[i].name < keys[j].name + }) + return series, keys, nil +} + +func validateOrientationRecord(record CaseResult, arm string) error { + if record.Status != StatusOK || record.Environment == nil || record.TraversalTelemetry == nil { + return fmt.Errorf("%s/%s %s arm lacks a successful telemetry-bearing record", record.Dataset, record.Name, arm) + } + summary := record.TraversalTelemetry.Summary + forward := string(optimize.ExpansionSearchStepwiseForward) + reverse := string(optimize.ExpansionSearchSuffixSeededReverse) + switch arm { + case "shadow": + if summary.EmittedIdentity != string(optimize.ExpansionSearchPolicyOrientationProbeV1) || + summary.SelectorVersion != string(optimize.ExpansionSearchPolicyOrientationProbeV1) || + summary.RuntimeIdentity != forward || summary.AppliedIdentity != forward || summary.RuntimeBranch != "shadow_incumbent" || + (summary.WouldSelectIdentity != forward && summary.WouldSelectIdentity != reverse) || + summary.FallbackExecuted == nil || *summary.FallbackExecuted { + return fmt.Errorf("%s/%s shadow telemetry does not prove incumbent-only orientation shadow execution", record.Dataset, record.Name) + } + case "incumbent": + if summary.RuntimeIdentity != forward || summary.AppliedIdentity != forward || summary.WouldSelectIdentity != "" { + return fmt.Errorf("%s/%s incumbent artifact did not execute the exact forward arm", record.Dataset, record.Name) + } + case "reverse": + if summary.RuntimeIdentity != reverse || summary.AppliedIdentity != reverse || summary.WouldSelectIdentity != "" { + return fmt.Errorf("%s/%s reverse artifact did not execute the exact forced reverse arm", record.Dataset, record.Name) + } + default: + return fmt.Errorf("unknown orientation arm %q", arm) + } + return nil +} + +func appendOrientationWarmSamples(series roundSamples, record CaseResult) { + for _, sample := range record.Stats.Samples { + if sample.Classification == "warm" && sample.Duration > 0 { + series[sample.Round] = append(series[sample.Round], sample.Duration) + } + } +} + +func validateOrientationExactObservations(key performanceKey, artifacts ...[]CaseResult) error { + workload := "" + var observed []string + rowCount := int64(-1) + binary := "" + for _, records := range artifacts { + matched := false + armSQL := "" + for _, record := range records { + if record.Dataset != key.dataset || record.Name != key.name || record.ExecutionMode != key.backend { + continue + } + matched = true + if !record.StableObservation || record.WorkloadSHA256 == "" || record.SQLFingerprint == "" || record.Environment == nil || record.Environment.BinarySHA256 == "" { + return fmt.Errorf("%s/%s lacks stable observation or executable/SQL identity", key.dataset, key.name) + } + if workload != "" && workload != record.WorkloadSHA256 { + return fmt.Errorf("%s/%s workload identity differs across orientation arms", key.dataset, key.name) + } + workload = record.WorkloadSHA256 + if rowCount >= 0 && (rowCount != record.RowCount || !slices.Equal(observed, record.ObservedRows)) { + return fmt.Errorf("%s/%s exact observations differ across orientation arms", key.dataset, key.name) + } + rowCount, observed = record.RowCount, append([]string(nil), record.ObservedRows...) + if binary != "" && binary != record.Environment.BinarySHA256 { + return fmt.Errorf("%s/%s executable identity differs across orientation arms", key.dataset, key.name) + } + binary = record.Environment.BinarySHA256 + if armSQL != "" && armSQL != record.SQLFingerprint { + return fmt.Errorf("%s/%s SQL fingerprint changes within an orientation arm", key.dataset, key.name) + } + armSQL = record.SQLFingerprint + } + if !matched { + return fmt.Errorf("%s/%s is missing from one orientation artifact", key.dataset, key.name) + } + } + return nil +} + +func validateOrientationArmOrder( + shadowRecords, incumbentRecords, reverseRecords []CaseResult, + key performanceKey, + rounds []int, + minimumWarmups int, +) error { + armRecords := []struct { + name string + records []CaseResult + }{ + {name: "shadow", records: shadowRecords}, + {name: "incumbent", records: incumbentRecords}, + {name: "reverse", records: reverseRecords}, + } + evidence := make([]map[int]pairedRoundEvidence, len(armRecords)) + positionCounts := make([][4]int, len(armRecords)) + for index, arm := range armRecords { + current, err := collectPairedRoundEvidence(arm.records, key) + if err != nil { + return err + } + evidence[index] = current + } + for _, round := range rounds { + seenPositions := map[int]struct{}{} + block, runUUID := 0, "" + for index, arm := range armRecords { + current, found := evidence[index][round] + if !found || current.Warmups < minimumWarmups || current.Arm == "" || current.Arm == "unlabeled" { + return fmt.Errorf("%s/%s round %d lacks %s arm identity or %d warmups", key.dataset, key.name, round, arm.name, minimumWarmups) + } + if current.ArmOrder < 1 || current.ArmOrder > 3 { + return fmt.Errorf("%s/%s round %d has invalid three-arm order", key.dataset, key.name, round) + } + if _, duplicate := seenPositions[current.ArmOrder]; duplicate { + return fmt.Errorf("%s/%s round %d has duplicate three-arm order", key.dataset, key.name, round) + } + seenPositions[current.ArmOrder] = struct{}{} + positionCounts[index][current.ArmOrder]++ + if block == 0 { + block, runUUID = current.Block, current.RunUUID + } else if current.Block != block || current.RunUUID != runUUID { + return fmt.Errorf("%s/%s round %d has mismatched three-arm block or run UUID", key.dataset, key.name, round) + } + } + if block < 1 || runUUID == "" { + return fmt.Errorf("%s/%s round %d has missing three-arm block or run UUID", key.dataset, key.name, round) + } + } + for index, counts := range positionCounts { + minimum, maximum := counts[1], counts[1] + for position := 2; position <= 3; position++ { + minimum = min(minimum, counts[position]) + maximum = max(maximum, counts[position]) + } + if maximum-minimum > 1 { + return fmt.Errorf("%s/%s %s arm order is not position-balanced", key.dataset, key.name, armRecords[index].name) + } + } + return nil +} + +func orientationQualificationRole(split, protocol string) (role string, tuningEligible, qualificationEligible bool) { + switch split { + case "training": + return "selector_training", true, protocol == referencePairProtocolConfirmation + case "holdout": + return "frozen_evaluation", false, protocol == referencePairProtocolConfirmation + case "diagnostic": + return "diagnostic_only", false, false + default: + return "legacy_diagnostic", false, false + } +} + +func fastestOrientationExactArm(incumbent, reverse roundSamples) (string, roundSamples) { + if roundMedianEstimate(reverse) < roundMedianEstimate(incumbent) { + return string(optimize.ExpansionSearchSuffixSeededReverse), reverse + } + return string(optimize.ExpansionSearchStepwiseForward), incumbent +} + +func roundMedianEstimate(samples roundSamples) float64 { + rounds := sortedRounds(samples) + medians := make([]float64, 0, len(rounds)) + for _, round := range rounds { + medians = append(medians, durationQuantile(samples[round], 0.5)) + } + return quantile(medians, 0.5) +} + +func orientationLatencyGate( + baselineIdentity, observedIdentity string, + baseline, observed roundSamples, + ratioLimit float64, + absoluteFloor time.Duration, + seed int64, + options PerfGateOptions, +) OrientationLatencyGate { + ratio := bootstrapRoundMedianRatio(baseline, observed, seed, options) + change := negateDurationInterval(bootstrapRoundMedianSaving(baseline, observed, seed+1, options)) + absoluteGapUpper := max(time.Duration(0), change.Upper) + return OrientationLatencyGate{ + BaselineIdentity: baselineIdentity, + ObservedIdentity: observedIdentity, + BaselineSamples: sampleCount(baseline), + ObservedSamples: sampleCount(observed), + Ratio: ratio, + AbsoluteChange: change, + RatioUpperLimit: ratioLimit, + AbsoluteFloor: absoluteFloor, + AbsoluteGapUpper: absoluteGapUpper, + Passed: ratio.Upper <= ratioLimit || absoluteGapUpper <= absoluteFloor, + } +} + +// createOrientationSelectorReport loads the three exact arm artifacts and A/A +// calibration, builds the report, and writes an indented JSON document. +func createOrientationSelectorReport( + shadowPath, incumbentPath, reversePath, aaPath, outputPath string, + options OrientationSelectorReportOptions, +) (bool, error) { + shadow, err := readJSONLFile(shadowPath) + if err != nil { + return false, fmt.Errorf("read orientation shadow artifact: %w", err) + } + incumbent, err := readJSONLFile(incumbentPath) + if err != nil { + return false, fmt.Errorf("read orientation incumbent artifact: %w", err) + } + reverse, err := readJSONLFile(reversePath) + if err != nil { + return false, fmt.Errorf("read orientation reverse artifact: %w", err) + } + aa, aaSHA, err := loadAAResolutionReport(aaPath) + if err != nil { + return false, fmt.Errorf("read orientation A/A report: %w", err) + } + report, err := buildOrientationSelectorReport(shadow, incumbent, reverse, aa, options) + if err != nil { + return false, err + } + report.ShadowArtifactSHA256, err = fileSHA256(shadowPath) + if err != nil { + return false, err + } + report.IncumbentArtifactSHA256, err = fileSHA256(incumbentPath) + if err != nil { + return false, err + } + report.ReverseArtifactSHA256, err = fileSHA256(reversePath) + if err != nil { + return false, err + } + report.AAReportSHA256 = aaSHA + return report.QualificationPassed, writeOrientationSelectorReport(outputPath, report) +} + +func writeOrientationSelectorReport(path string, report OrientationSelectorReport) (err error) { + output := os.Stdout + if path != "" { + if err := ensureOutputDir(path); err != nil { + return err + } + output, err = os.Create(path) + if err != nil { + return err + } + defer func() { + if closeErr := output.Close(); err == nil && closeErr != nil { + err = closeErr + } + }() + } + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(report) +} diff --git a/cmd/graphbench/orientation_selector_report_test.go b/cmd/graphbench/orientation_selector_report_test.go new file mode 100644 index 00000000..267c604a --- /dev/null +++ b/cmd/graphbench/orientation_selector_report_test.go @@ -0,0 +1,302 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "testing" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/stretchr/testify/require" +) + +func TestOrientationSelectorReportPassesMatchedLowRegretLowOverheadEvidence(t *testing.T) { + trainingShadow, trainingIncumbent, trainingReverse := orientationSelectorRecords( + "training", + string(optimize.ExpansionSearchSuffixSeededReverse), + 10*time.Millisecond+50*time.Microsecond, + 10*time.Millisecond, + 5*time.Millisecond, + ) + holdoutShadow, holdoutIncumbent, holdoutReverse := orientationSelectorRecords( + "holdout", + string(optimize.ExpansionSearchSuffixSeededReverse), + 10*time.Millisecond+50*time.Microsecond, + 10*time.Millisecond, + 5*time.Millisecond, + ) + renameOrientationRecords("training-fixed-suffix", trainingShadow, trainingIncumbent, trainingReverse) + renameOrientationRecords("holdout-fixed-suffix", holdoutShadow, holdoutIncumbent, holdoutReverse) + shadow := append(trainingShadow, holdoutShadow...) + incumbent := append(trainingIncumbent, holdoutIncumbent...) + reverse := append(trainingReverse, holdoutReverse...) + + report, err := buildOrientationSelectorReport(shadow, incumbent, reverse, testAAReportForRecords(t, incumbent), OrientationSelectorReportOptions{ + Seed: 7, Confidence: defaultConfidenceLevel, BootstrapCount: 100, Protocol: referencePairProtocolConfirmation, + }) + + require.NoError(t, err) + require.True(t, report.EvidencePassed) + require.Equal(t, 1, report.TrainingCases) + require.Equal(t, 1, report.HoldoutCases) + require.True(t, report.TrainingPassed) + require.True(t, report.HoldoutPassed) + require.True(t, report.QualificationPassed) + require.Equal(t, 1.10, report.SelectorRegretRatioLimit) + require.Equal(t, 1.10, report.ProbeOverheadRatioLimit) + require.Equal(t, 100*time.Microsecond, report.ProbeOverheadAbsoluteLimit) + require.Len(t, report.Cases, 2) + entry := report.Cases[1] + if entry.QualificationSplit != "training" { + entry = report.Cases[0] + } + require.Equal(t, "training", entry.QualificationSplit) + require.Equal(t, "selector_training", entry.QualificationRole) + require.True(t, entry.ThresholdTuningEligible) + require.True(t, entry.QualificationEligible) + require.Equal(t, string(optimize.ExpansionSearchSuffixSeededReverse), entry.WouldSelectIdentity) + require.Equal(t, string(optimize.ExpansionSearchSuffixSeededReverse), entry.FastestExactIdentity) + require.True(t, entry.SelectorRegret.Passed) + require.True(t, entry.ProbeOverhead.Passed) + require.True(t, entry.ExactObservationsMatched) +} + +func TestOrientationSelectorReportFailsRegretWhenShadowChoosesSlowArm(t *testing.T) { + shadow, incumbent, reverse := orientationSelectorRecords( + "training", + string(optimize.ExpansionSearchStepwiseForward), + 10*time.Millisecond+25*time.Microsecond, + 10*time.Millisecond, + time.Millisecond, + ) + + report, err := buildOrientationSelectorReport(shadow, incumbent, reverse, testAAReportForRecords(t, incumbent), OrientationSelectorReportOptions{ + Seed: 11, Confidence: defaultConfidenceLevel, BootstrapCount: 100, Protocol: referencePairProtocolConfirmation, + }) + + require.NoError(t, err) + require.False(t, report.EvidencePassed) + require.False(t, report.TrainingPassed) + require.False(t, report.HoldoutPassed) + require.False(t, report.QualificationPassed) + require.False(t, report.Cases[0].SelectorRegret.Passed) + require.True(t, report.Cases[0].ProbeOverhead.Passed) + require.Contains(t, report.Cases[0].Reasons, "selector regret exceeds the 1.10/A/A floor") +} + +func TestOrientationSelectorReportAllowsAbsoluteProbeFloor(t *testing.T) { + shadow, incumbent, reverse := orientationSelectorRecords( + "training", + string(optimize.ExpansionSearchStepwiseForward), + 250*time.Microsecond, + 200*time.Microsecond, + 300*time.Microsecond, + ) + + report, err := buildOrientationSelectorReport(shadow, incumbent, reverse, testAAReportForRecords(t, incumbent), OrientationSelectorReportOptions{ + Seed: 13, Confidence: defaultConfidenceLevel, BootstrapCount: 100, Protocol: referencePairProtocolConfirmation, + }) + + require.NoError(t, err) + probe := report.Cases[0].ProbeOverhead + require.Greater(t, probe.Ratio.Upper, 1.10) + require.Equal(t, 50*time.Microsecond, probe.AbsoluteGapUpper) + require.True(t, probe.Passed) +} + +func TestOrientationSelectorReportKeepsHoldoutEvaluationOnlyAndExcludesDiagnostic(t *testing.T) { + for _, testCase := range []struct { + split string + role string + qualificationEligible bool + qualificationPassed bool + }{ + {split: "holdout", role: "frozen_evaluation", qualificationEligible: true, qualificationPassed: false}, + {split: "diagnostic", role: "diagnostic_only", qualificationEligible: false, qualificationPassed: false}, + } { + t.Run(testCase.split, func(t *testing.T) { + shadow, incumbent, reverse := orientationSelectorRecords( + testCase.split, + string(optimize.ExpansionSearchSuffixSeededReverse), + 10*time.Millisecond, + 10*time.Millisecond, + 5*time.Millisecond, + ) + report, err := buildOrientationSelectorReport(shadow, incumbent, reverse, testAAReportForRecords(t, incumbent), OrientationSelectorReportOptions{ + Seed: 17, Confidence: defaultConfidenceLevel, BootstrapCount: 50, Protocol: referencePairProtocolConfirmation, + }) + require.NoError(t, err) + require.Equal(t, testCase.role, report.Cases[0].QualificationRole) + require.False(t, report.Cases[0].ThresholdTuningEligible) + require.Equal(t, testCase.qualificationEligible, report.Cases[0].QualificationEligible) + require.Equal(t, testCase.qualificationPassed, report.QualificationPassed) + }) + } +} + +func TestOrientationSelectorReportRequiresPassingTrainingAndFrozenHoldout(t *testing.T) { + trainingShadow, trainingIncumbent, trainingReverse := orientationSelectorRecords( + "training", string(optimize.ExpansionSearchSuffixSeededReverse), 10*time.Millisecond, 10*time.Millisecond, 5*time.Millisecond, + ) + holdoutShadow, holdoutIncumbent, holdoutReverse := orientationSelectorRecords( + "holdout", string(optimize.ExpansionSearchStepwiseForward), 10*time.Millisecond, 10*time.Millisecond, time.Millisecond, + ) + renameOrientationRecords("training-pass", trainingShadow, trainingIncumbent, trainingReverse) + renameOrientationRecords("holdout-fail", holdoutShadow, holdoutIncumbent, holdoutReverse) + shadow := append(trainingShadow, holdoutShadow...) + incumbent := append(trainingIncumbent, holdoutIncumbent...) + reverse := append(trainingReverse, holdoutReverse...) + + report, err := buildOrientationSelectorReport(shadow, incumbent, reverse, testAAReportForRecords(t, incumbent), OrientationSelectorReportOptions{ + Seed: 19, Confidence: defaultConfidenceLevel, BootstrapCount: 100, Protocol: referencePairProtocolConfirmation, + }) + require.NoError(t, err) + require.True(t, report.TrainingPassed) + require.False(t, report.HoldoutPassed) + require.False(t, report.QualificationPassed) +} + +func TestOrientationSelectorReportRejectsSplitDriftAndNonIncumbentShadowRuntime(t *testing.T) { + shadow, incumbent, reverse := orientationSelectorRecords( + "training", + string(optimize.ExpansionSearchSuffixSeededReverse), + 10*time.Millisecond, + 10*time.Millisecond, + 5*time.Millisecond, + ) + reverse[0].Shape.QualificationSplit = "holdout" + _, err := buildOrientationSelectorReport(shadow, incumbent, reverse, testAAReportForRecords(t, incumbent), OrientationSelectorReportOptions{ + Confidence: defaultConfidenceLevel, BootstrapCount: 10, Protocol: referencePairProtocolConfirmation, + }) + require.ErrorContains(t, err, "changes qualification split") + + reverse[0].Shape.QualificationSplit = "training" + shadow[0].TraversalTelemetry.Summary.RuntimeIdentity = string(optimize.ExpansionSearchSuffixSeededReverse) + _, err = buildOrientationSelectorReport(shadow, incumbent, reverse, testAAReportForRecords(t, incumbent), OrientationSelectorReportOptions{ + Confidence: defaultConfidenceLevel, BootstrapCount: 10, Protocol: referencePairProtocolConfirmation, + }) + require.ErrorContains(t, err, "incumbent-only") +} + +func TestOrientationSelectorReportRejectsUnbalancedThreeArmOrder(t *testing.T) { + shadow, incumbent, reverse := orientationSelectorRecords( + "training", + string(optimize.ExpansionSearchSuffixSeededReverse), + 10*time.Millisecond, + 10*time.Millisecond, + 5*time.Millisecond, + ) + for recordIndex := range reverse { + for sampleIndex := range reverse[recordIndex].Stats.Samples { + reverse[recordIndex].Stats.Samples[sampleIndex].ArmOrder = 3 + } + } + _, err := buildOrientationSelectorReport(shadow, incumbent, reverse, testAAReportForRecords(t, incumbent), OrientationSelectorReportOptions{ + Confidence: defaultConfidenceLevel, BootstrapCount: 10, Protocol: referencePairProtocolConfirmation, + }) + require.ErrorContains(t, err, "duplicate three-arm order") +} + +func orientationSelectorRecords( + split, wouldSelect string, + shadowDuration, incumbentDuration, reverseDuration time.Duration, +) (shadow, incumbent, reverse []CaseResult) { + const rounds = 12 + orders := [][3]int{ + {1, 2, 3}, + {2, 3, 1}, + {3, 1, 2}, + {1, 3, 2}, + {2, 1, 3}, + {3, 2, 1}, + } + for round := 1; round <= rounds; round++ { + order := orders[(round-1)%len(orders)] + shadow = append(shadow, orientationSelectorRecord(round, order[0], "shadow", split, wouldSelect, shadowDuration)) + incumbent = append(incumbent, orientationSelectorRecord(round, order[1], "incumbent", split, "", incumbentDuration)) + reverse = append(reverse, orientationSelectorRecord(round, order[2], "reverse", split, "", reverseDuration)) + } + return shadow, incumbent, reverse +} + +func orientationSelectorRecord(round, armOrder int, arm, split, wouldSelect string, duration time.Duration) CaseResult { + forward := string(optimize.ExpansionSearchStepwiseForward) + reverse := string(optimize.ExpansionSearchSuffixSeededReverse) + runtimeIdentity := forward + emittedIdentity := forward + selectorVersion := "static-lowering-v1" + runtimeBranch := "selected" + if arm == "shadow" { + emittedIdentity = string(optimize.ExpansionSearchPolicyOrientationProbeV1) + selectorVersion = emittedIdentity + runtimeBranch = "shadow_incumbent" + } + if arm == "reverse" { + runtimeIdentity = reverse + emittedIdentity = reverse + selectorVersion = "suffix-seeded-reverse-tool-v1" + } + fallback := false + overflow := false + record := CaseResult{ + Dataset: "orientation-fixture", + Name: "fixed-suffix", + Category: "generated_fixed_suffix_expansion_v2", + WorkloadSHA256: "orientation-workload-v1", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{FixtureTier: "normal", QualificationSplit: split}, + RowCount: 1, + ObservedRows: []string{"[42]"}, + StableObservation: true, + SQLFingerprint: "orientation-" + arm + "-sql-v1", + PostgresEnvironment: &PostgresEnvironment{PlanCacheMode: "auto"}, + Environment: &RunEnvironment{ + Arm: arm, ArmOrder: armOrder, Block: round, Round: round, + RunUUID: "orientation-run-" + fmt.Sprint(round), BinarySHA256: "orientation-binary-v1", + GOOS: "linux", GOARCH: "amd64", CPUCount: 8, CPUModel: "test-cpu", Kernel: "test-kernel", CgroupCPU: "max 100000", + WarmupIterations: 20, + }, + TraversalTelemetry: &TraversalExecutionTelemetry{ + SchemaVersion: TraversalExecutionTelemetrySchemaVersion, + Level: TraversalTelemetryLevelSummary, + Summary: TraversalExecutionSummary{ + RequestedIdentity: reverse, + PlannedIdentities: []string{forward, reverse}, + EmittedIdentity: emittedIdentity, + RuntimeIdentity: runtimeIdentity, + AppliedIdentity: runtimeIdentity, + SelectorVersion: selectorVersion, + SchedulerVersion: "not_applicable", + Caps: map[string]int64{}, + RuntimeBranch: runtimeBranch, + Overflow: &overflow, + FallbackExecuted: &fallback, + WouldSelectIdentity: wouldSelect, + Provenance: map[string]string{}, + }, + }, + } + record.Stats.WarmupIterations = 20 + for iteration := 1; iteration <= 50; iteration++ { + record.Stats.Samples = append(record.Stats.Samples, LatencySample{ + Round: round, Block: round, Arm: arm, ArmOrder: armOrder, + RunUUID: record.Environment.RunUUID, Iteration: iteration, + Classification: "warm", Duration: duration, + }) + } + return record +} + +func renameOrientationRecords(name string, artifacts ...[]CaseResult) { + for _, records := range artifacts { + for index := range records { + records[index].Name = name + records[index].WorkloadSHA256 = "orientation-workload-" + name + } + } +} diff --git a/cmd/graphbench/orientation_selector_report_v2.go b/cmd/graphbench/orientation_selector_report_v2.go new file mode 100644 index 00000000..434bf731 --- /dev/null +++ b/cmd/graphbench/orientation_selector_report_v2.go @@ -0,0 +1,1159 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "slices" + "strings" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" +) + +const orientationSelectorReportV2Version = 2 + +type OrientationSelectorV2FreezeManifest struct { + Version int `json:"version"` + Policy string `json:"policy"` + Formula string `json:"formula"` + Caps map[string]int64 `json:"caps"` + SourceCommit string `json:"source_commit"` + DirtyDiffSHA256 string `json:"dirty_diff_sha256"` + BinarySHA256 string `json:"binary_sha256"` + CohortDeclarationSHA256 string `json:"cohort_declaration_sha256"` + DiscoveryReportSHA256 string `json:"discovery_report_sha256"` +} + +// OrientationSelectorV2ReportOptions configures the immutable four-arm v2 +// qualification workflow independently of the retained v1 shadow report. +type OrientationSelectorV2ReportOptions struct { + Seed int64 + Confidence float64 + BootstrapCount int + Protocol string + Freeze *OrientationSelectorV2FreezeManifest + Discovery *OrientationSelectorV2Report +} + +// OrientationLatencyGateV2 makes conditional applicability explicit. A +// reverse-selected shadow comparison remains visible but cannot qualify or +// disqualify the guarded selector. +type OrientationLatencyGateV2 struct { + Applicable bool `json:"applicable"` + OrientationLatencyGate +} + +// OrientationSelectorV2Case records exact runtime attribution and the three +// frozen latency gates for one training, holdout, or diagnostic case. +type OrientationSelectorV2Case struct { + Dataset string `json:"dataset"` + Name string `json:"name"` + QualificationSplit string `json:"qualification_split"` + QualificationRole string `json:"qualification_role"` + ThresholdTuningEligible bool `json:"threshold_tuning_eligible"` + QualificationEligible bool `json:"qualification_eligible"` + Rounds int `json:"matched_rounds"` + WouldSelectIdentity string `json:"would_select_identity"` + FastestExactIdentity string `json:"fastest_exact_identity"` + GuardedRuntimeIdentity string `json:"guarded_runtime_identity"` + GuardedRuntimeBranch string `json:"guarded_runtime_branch"` + Overflow bool `json:"overflow"` + FallbackExecuted bool `json:"fallback_executed"` + ExactObservationsMatched bool `json:"exact_observations_matched"` + ShadowForwardOverhead OrientationLatencyGateV2 `json:"shadow_forward_overhead"` + GuardedSelectedOverhead OrientationLatencyGate `json:"guarded_selected_overhead"` + GuardedFastestRegret OrientationLatencyGate `json:"guarded_fastest_regret"` + Passed bool `json:"passed"` + Reasons []string `json:"reasons,omitempty"` +} + +// OrientationSelectorV2Report binds the immutable selector, source, binary, +// corpus, four timing artifacts, host A/A floor, and qualification outcome. +type OrientationSelectorV2Report struct { + Version int `json:"version"` + Policy string `json:"policy"` + Protocol string `json:"protocol"` + Seed int64 `json:"seed"` + Confidence float64 `json:"confidence_level"` + SourceCommit string `json:"source_commit"` + DirtyDiffSHA256 string `json:"dirty_diff_sha256"` + BinarySHA256 string `json:"binary_sha256"` + CorpusSHA256 string `json:"corpus_sha256"` + CohortDeclarationSHA256 string `json:"cohort_declaration_sha256"` + FreezeManifestSHA256 string `json:"freeze_manifest_sha256,omitempty"` + Formula string `json:"formula"` + Caps map[string]int64 `json:"caps"` + ShadowArtifactSHA256 string `json:"shadow_artifact_sha256,omitempty"` + IncumbentArtifactSHA256 string `json:"incumbent_artifact_sha256,omitempty"` + ReverseArtifactSHA256 string `json:"reverse_artifact_sha256,omitempty"` + GuardedArtifactSHA256 string `json:"guarded_artifact_sha256,omitempty"` + AAReportSHA256 string `json:"aa_report_sha256,omitempty"` + ShadowForwardRatioLimit float64 `json:"shadow_forward_ratio_upper_limit"` + GuardedSelectedRatioLimit float64 `json:"guarded_selected_ratio_upper_limit"` + GuardedFastestRatioLimit float64 `json:"guarded_fastest_ratio_upper_limit"` + OverheadAbsoluteLimit time.Duration `json:"overhead_absolute_limit"` + EvidencePassed bool `json:"evidence_passed"` + TrainingCases int `json:"training_cases"` + HoldoutCases int `json:"holdout_cases"` + TrainingPassed bool `json:"training_passed"` + HoldoutPassed bool `json:"holdout_passed"` + QualificationPassed bool `json:"qualification_passed"` + Cases []OrientationSelectorV2Case `json:"cases"` +} + +type orientationSelectorV2Series struct { + shadow roundSamples + incumbent roundSamples + reverse roundSamples + guarded roundSamples + wouldSelect string + shadowOverflow bool + shadowObserved bool + guardedRuntime string + guardedBranch string + overflow bool + fallback bool + guardedObserved bool +} + +type orientationSelectorV2Identity struct { + sourceCommit string + dirtyDiffSHA256 string + binarySHA256 string + corpusSHA256 string +} + +// buildOrientationSelectorV2Report evaluates matched shadow, exact forward, +// exact reverse, and actual guarded statements. V1 remains a separate schema +// and code path so new evidence cannot reinterpret its historical result. +func buildOrientationSelectorV2Report( + shadowRecords, incumbentRecords, reverseRecords, guardedRecords []CaseResult, + aa *AAResolutionReport, + options OrientationSelectorV2ReportOptions, +) (OrientationSelectorV2Report, error) { + if options.Confidence <= 0 || options.Confidence >= 1 { + return OrientationSelectorV2Report{}, fmt.Errorf("confidence level must be between 0 and 1") + } + if options.BootstrapCount == 0 { + options.BootstrapCount = defaultBootstrapCount + } + if options.BootstrapCount < 1 { + return OrientationSelectorV2Report{}, fmt.Errorf("bootstrap count must be positive") + } + protocol := options.Protocol + if protocol == "" { + protocol = referencePairProtocolConfirmation + } + minimumWarmups, minimumRounds, maximumRounds, minimumSamples := 20, 10, 20, 50 + if protocol == referencePairProtocolDiscovery { + minimumWarmups, minimumRounds, maximumRounds, minimumSamples = 5, 5, 20, 10 + } else if protocol != referencePairProtocolConfirmation { + return OrientationSelectorV2Report{}, fmt.Errorf("unsupported orientation selector v2 protocol %q", protocol) + } + + if err := validateAAResolutionEvidence(aa, incumbentRecords, options.Confidence); err != nil { + return OrientationSelectorV2Report{}, fmt.Errorf("incumbent A/A evidence: %w", err) + } + if err := validateOrientationV2AAEvidence(aa, incumbentRecords); err != nil { + return OrientationSelectorV2Report{}, fmt.Errorf("incumbent A/A environment: %w", err) + } + incumbentHost, err := artifactHostFingerprint(incumbentRecords) + if err != nil { + return OrientationSelectorV2Report{}, err + } + for name, records := range map[string][]CaseResult{ + "shadow": shadowRecords, "reverse": reverseRecords, "guarded": guardedRecords, + } { + host, err := artifactHostFingerprint(records) + if err != nil { + return OrientationSelectorV2Report{}, fmt.Errorf("%s artifact host: %w", name, err) + } + if host != incumbentHost { + return OrientationSelectorV2Report{}, fmt.Errorf("%s artifact host does not match incumbent host", name) + } + } + identity, err := validateOrientationV2EvidenceIdentity(shadowRecords, incumbentRecords, reverseRecords, guardedRecords) + if err != nil { + return OrientationSelectorV2Report{}, err + } + + series, keys, err := collectOrientationSelectorV2Series(shadowRecords, incumbentRecords, reverseRecords, guardedRecords) + if err != nil { + return OrientationSelectorV2Report{}, err + } + cohortDeclarationSHA256, err := validateOrientationV2Cohort(keys, shadowRecords, incumbentRecords, reverseRecords, guardedRecords, protocol) + if err != nil { + return OrientationSelectorV2Report{}, err + } + report := OrientationSelectorV2Report{ + Version: orientationSelectorReportV2Version, + Policy: string(optimize.ExpansionSearchPolicyOrientationProbeV2), + Protocol: protocol, + Seed: options.Seed, + Confidence: options.Confidence, + SourceCommit: identity.sourceCommit, + DirtyDiffSHA256: identity.dirtyDiffSHA256, + BinarySHA256: identity.binarySHA256, + CorpusSHA256: identity.corpusSHA256, + CohortDeclarationSHA256: cohortDeclarationSHA256, + Formula: "F2=root_rows+maximum_depth*forward_degree_rows;R2=suffix_rows+boundary_rows+reverse_degree_rows;reverse=complete&&4*R2<3*F2", + Caps: map[string]int64{ + "root_row_limit": optimize.ExpansionSearchOrientationRootRowLimit, + "reverse_seed_row_limit": optimize.ExpansionSearchOrientationReverseSeedRowLimit, + "directional_degree_row_limit": optimize.ExpansionSearchOrientationDirectionalDegreeRowLimit, + "state_limit": optimize.ExpansionSearchOrientationStateLimit, + }, + ShadowForwardRatioLimit: 1.10, + GuardedSelectedRatioLimit: 1.10, + GuardedFastestRatioLimit: 1.10, + OverheadAbsoluteLimit: 100 * time.Microsecond, + EvidencePassed: true, + } + if protocol == referencePairProtocolConfirmation { + if err := validateOrientationV2Freeze(options.Freeze, options.Discovery, report); err != nil { + return OrientationSelectorV2Report{}, err + } + } + trainingPassed, holdoutPassed := true, true + gateOptions := PerfGateOptions{Seed: options.Seed, Confidence: options.Confidence, BootstrapCount: options.BootstrapCount} + for index, key := range keys { + current := series[key] + if err := requireOrientationV2RoundSets(key, current); err != nil { + return OrientationSelectorV2Report{}, err + } + rounds := sortedRounds(current.shadow) + if len(rounds) < minimumRounds || len(rounds) > maximumRounds { + return OrientationSelectorV2Report{}, fmt.Errorf("%s/%s requires %d-%d matched orientation-v2 rounds, got %d", key.dataset, key.name, minimumRounds, maximumRounds, len(rounds)) + } + for _, round := range rounds { + if len(current.shadow[round]) < minimumSamples || len(current.incumbent[round]) < minimumSamples || + len(current.reverse[round]) < minimumSamples || len(current.guarded[round]) < minimumSamples { + return OrientationSelectorV2Report{}, fmt.Errorf("%s/%s round %d requires %d samples per orientation-v2 arm", key.dataset, key.name, round, minimumSamples) + } + } + if err := validateOrientationV2ArmOrder(shadowRecords, incumbentRecords, reverseRecords, guardedRecords, key, rounds, minimumWarmups); err != nil { + return OrientationSelectorV2Report{}, err + } + + split, err := qualificationSplit(key, shadowRecords, incumbentRecords, reverseRecords, guardedRecords) + if err != nil { + return OrientationSelectorV2Report{}, err + } + role, tuningEligible, qualificationEligible := orientationQualificationRole(split, protocol) + if qualificationEligible && !strings.HasPrefix(key.dataset, "generated_fixed_suffix_expansion_v3_") { + return OrientationSelectorV2Report{}, fmt.Errorf("%s/%s qualification evidence is not from the frozen fixed-suffix v3 corpus", key.dataset, key.name) + } + fastestIdentity, fastest := fastestOrientationExactArm(current.incumbent, current.reverse) + selectedIdentity, selected := string(optimize.ExpansionSearchStepwiseForward), current.incumbent + if current.wouldSelect == string(optimize.ExpansionSearchSuffixSeededReverse) { + selectedIdentity, selected = string(optimize.ExpansionSearchSuffixSeededReverse), current.reverse + } + seed := options.Seed + int64(index)*7919 + _, selectorFloorAbsolute, err := aaTimingFloor(aa, key, false, 0) + if err != nil { + return OrientationSelectorV2Report{}, err + } + shadowGate := orientationLatencyGate( + string(optimize.ExpansionSearchStepwiseForward), + string(optimize.ExpansionSearchPolicyOrientationProbeV2)+":shadow", + current.incumbent, + current.shadow, + report.ShadowForwardRatioLimit, + report.OverheadAbsoluteLimit, + seed, + gateOptions, + ) + shadowApplicable := current.wouldSelect == string(optimize.ExpansionSearchStepwiseForward) + guardedSelected := orientationLatencyGate( + selectedIdentity, + string(optimize.ExpansionSearchPolicyOrientationProbeV2)+":"+current.guardedRuntime, + selected, + current.guarded, + report.GuardedSelectedRatioLimit, + report.OverheadAbsoluteLimit, + seed+3, + gateOptions, + ) + guardedFastest := orientationLatencyGate( + fastestIdentity, + string(optimize.ExpansionSearchPolicyOrientationProbeV2)+":"+current.guardedRuntime, + fastest, + current.guarded, + report.GuardedFastestRatioLimit, + selectorFloorAbsolute, + seed+6, + gateOptions, + ) + entry := OrientationSelectorV2Case{ + Dataset: key.dataset, + Name: key.name, + QualificationSplit: split, + QualificationRole: role, + ThresholdTuningEligible: tuningEligible, + QualificationEligible: qualificationEligible, + Rounds: len(rounds), + WouldSelectIdentity: current.wouldSelect, + FastestExactIdentity: fastestIdentity, + GuardedRuntimeIdentity: current.guardedRuntime, + GuardedRuntimeBranch: current.guardedBranch, + Overflow: current.overflow, + FallbackExecuted: current.fallback, + ExactObservationsMatched: true, + ShadowForwardOverhead: OrientationLatencyGateV2{ + Applicable: shadowApplicable, OrientationLatencyGate: shadowGate, + }, + GuardedSelectedOverhead: guardedSelected, + GuardedFastestRegret: guardedFastest, + Passed: (!shadowApplicable || shadowGate.Passed) && guardedSelected.Passed && guardedFastest.Passed, + } + if shadowApplicable && !shadowGate.Passed { + entry.Reasons = append(entry.Reasons, "forward-selected shadow overhead exceeds 10% and 100us") + } + if !guardedSelected.Passed { + entry.Reasons = append(entry.Reasons, "guarded selected-arm overhead exceeds 10% and 100us") + } + if !guardedFastest.Passed { + entry.Reasons = append(entry.Reasons, "guarded fastest-arm regret exceeds the 1.10/A/A floor") + } + if !entry.Passed { + report.EvidencePassed = false + } + if qualificationEligible { + switch split { + case "training": + report.TrainingCases++ + trainingPassed = trainingPassed && entry.Passed + case "holdout": + report.HoldoutCases++ + holdoutPassed = holdoutPassed && entry.Passed + } + } + report.Cases = append(report.Cases, entry) + } + report.TrainingPassed = protocol == referencePairProtocolConfirmation && report.TrainingCases > 0 && trainingPassed + report.HoldoutPassed = protocol == referencePairProtocolConfirmation && report.HoldoutCases > 0 && holdoutPassed + if protocol == referencePairProtocolConfirmation && (report.TrainingCases != 8 || report.HoldoutCases != 4) { + return OrientationSelectorV2Report{}, fmt.Errorf("orientation-v2 confirmation requires exactly 8 training and 4 holdout cases, got %d/%d", report.TrainingCases, report.HoldoutCases) + } + report.QualificationPassed = report.TrainingPassed && report.HoldoutPassed + return report, nil +} + +func collectOrientationSelectorV2Series( + shadowRecords, incumbentRecords, reverseRecords, guardedRecords []CaseResult, +) (map[performanceKey]*orientationSelectorV2Series, []performanceKey, error) { + artifacts := []struct { + name string + records []CaseResult + }{ + {name: "shadow", records: shadowRecords}, + {name: "incumbent", records: incumbentRecords}, + {name: "reverse", records: reverseRecords}, + {name: "guarded", records: guardedRecords}, + } + keySets := make([]map[performanceKey]struct{}, len(artifacts)) + for index, artifact := range artifacts { + keys, err := orientationV2ArtifactKeys(artifact.name, artifact.records) + if err != nil { + return nil, nil, err + } + keySets[index] = keys + } + for index := 1; index < len(keySets); index++ { + if !orientationV2KeySetsEqual(keySets[0], keySets[index]) { + return nil, nil, fmt.Errorf("orientation-v2 %s artifact case set does not match shadow artifact", artifacts[index].name) + } + } + + series := make(map[performanceKey]*orientationSelectorV2Series, len(keySets[0])) + for key := range keySets[0] { + series[key] = &orientationSelectorV2Series{ + shadow: roundSamples{}, incumbent: roundSamples{}, reverse: roundSamples{}, guarded: roundSamples{}, + } + } + for _, artifact := range artifacts { + seenRounds := map[performanceKey]map[int]struct{}{} + for _, record := range artifact.records { + key := performanceKey{dataset: record.Dataset, name: record.Name, backend: record.ExecutionMode} + current := series[key] + if current == nil { + return nil, nil, fmt.Errorf("orientation-v2 %s artifact contains unexpected case %s/%s", artifact.name, key.dataset, key.name) + } + if err := validateOrientationV2Record(record, artifact.name); err != nil { + return nil, nil, err + } + round, err := orientationV2RecordRound(record) + if err != nil { + return nil, nil, err + } + if seenRounds[key] == nil { + seenRounds[key] = map[int]struct{}{} + } + if _, duplicate := seenRounds[key][round]; duplicate { + return nil, nil, fmt.Errorf("%s/%s %s artifact duplicates round %d", key.dataset, key.name, artifact.name, round) + } + seenRounds[key][round] = struct{}{} + switch artifact.name { + case "shadow": + choice := record.TraversalTelemetry.Summary.WouldSelectIdentity + shadowOverflow := *record.TraversalTelemetry.Summary.Overflow + if current.shadowObserved && (current.wouldSelect != choice || current.shadowOverflow != shadowOverflow) { + return nil, nil, fmt.Errorf("%s/%s changes shadow would_select identity across rounds", key.dataset, key.name) + } + current.wouldSelect, current.shadowOverflow, current.shadowObserved = choice, shadowOverflow, true + appendOrientationWarmSamples(current.shadow, record) + case "incumbent": + appendOrientationWarmSamples(current.incumbent, record) + case "reverse": + appendOrientationWarmSamples(current.reverse, record) + case "guarded": + summary := record.TraversalTelemetry.Summary + if current.guardedObserved && + (current.guardedRuntime != summary.RuntimeIdentity || current.guardedBranch != summary.RuntimeBranch || + current.overflow != *summary.Overflow || current.fallback != *summary.FallbackExecuted) { + return nil, nil, fmt.Errorf("%s/%s changes guarded runtime outcome across rounds", key.dataset, key.name) + } + current.guardedRuntime, current.guardedBranch = summary.RuntimeIdentity, summary.RuntimeBranch + current.overflow, current.fallback, current.guardedObserved = *summary.Overflow, *summary.FallbackExecuted, true + appendOrientationWarmSamples(current.guarded, record) + } + } + } + + keys := sortedPerformanceKeys(keySets[0]) + for _, key := range keys { + current := series[key] + if !current.shadowObserved || current.wouldSelect == "" || !current.guardedObserved { + return nil, nil, fmt.Errorf("%s/%s lacks attributable shadow or guarded records", key.dataset, key.name) + } + if err := validateOrientationV2RuntimeConsistency(key, current); err != nil { + return nil, nil, err + } + if err := validateOrientationExactObservations(key, shadowRecords, incumbentRecords, reverseRecords, guardedRecords); err != nil { + return nil, nil, err + } + } + return series, keys, nil +} + +func orientationV2ArtifactKeys(name string, records []CaseResult) (map[performanceKey]struct{}, error) { + keys := map[performanceKey]struct{}{} + if len(records) == 0 { + return nil, fmt.Errorf("orientation-v2 %s artifact is empty", name) + } + for _, record := range records { + if record.ExecutionMode != ModePostgresSQL { + return nil, fmt.Errorf("orientation-v2 %s artifact contains non-PostgreSQL record %s/%s", name, record.Dataset, record.Name) + } + if record.Dataset == "" || record.Name == "" || !hasWarmLatencySample(record) { + return nil, fmt.Errorf("orientation-v2 %s artifact contains an incomplete timing record", name) + } + keys[performanceKey{dataset: record.Dataset, name: record.Name, backend: record.ExecutionMode}] = struct{}{} + } + return keys, nil +} + +func orientationV2KeySetsEqual(left, right map[performanceKey]struct{}) bool { + if len(left) != len(right) { + return false + } + for key := range left { + if _, found := right[key]; !found { + return false + } + } + return true +} + +func validateOrientationV2Cohort( + keys []performanceKey, + shadowRecords, incumbentRecords, reverseRecords, guardedRecords []CaseResult, + protocol string, +) (string, error) { + cohortDeclarationSHA256 := "" + for name, records := range map[string][]CaseResult{ + "shadow": shadowRecords, "incumbent": incumbentRecords, "reverse": reverseRecords, "guarded": guardedRecords, + } { + selection, err := selectionIdentity(records) + if err != nil { + return "", fmt.Errorf("orientation-v2 %s selection: %w", name, err) + } + if cohortDeclarationSHA256 == "" { + cohortDeclarationSHA256 = selection.DeclarationSHA256 + } + if selection.Version != selectionManifestVersion || !lowercaseSHA256(selection.DeclarationSHA256) || + selection.DeclarationSHA256 != cohortDeclarationSHA256 || !selection.DiagnosticOnly || + selection.SelectedDeclarationCount != 2*len(keys) || len(selection.Resolved) != len(keys) || + selection.FullDeclarationCount != selection.SelectedDeclarationCount+selection.OmittedDeclarationCount { + return "", fmt.Errorf("orientation-v2 %s selection does not bind the exact measured cohort", name) + } + resolved := make(map[performanceKey]struct{}, len(selection.Resolved)) + for _, item := range selection.Resolved { + if item.Category != "generated_fixed_suffix_expansion" { + return "", fmt.Errorf("orientation-v2 %s selection contains a non-v3 category", name) + } + resolved[performanceKey{dataset: item.Dataset, name: item.Name, backend: ModePostgresSQL}] = struct{}{} + } + for _, key := range keys { + if _, found := resolved[key]; !found { + return "", fmt.Errorf("orientation-v2 %s selection omits %s/%s", name, key.dataset, key.name) + } + } + } + + if protocol == referencePairProtocolConfirmation { + canonical, err := canonicalOrientationV2Cohort() + if err != nil { + return "", err + } + if cohortDeclarationSHA256 != canonical.declarationSHA256 || !orientationV2KeySetsEqual(canonical.keys, performanceKeySet(keys)) { + return "", fmt.Errorf("orientation-v2 confirmation does not contain the exact frozen 8-training/4-holdout cohort") + } + } + return cohortDeclarationSHA256, nil +} + +type orientationV2CanonicalCohort struct { + keys map[performanceKey]struct{} + trainingKeys map[performanceKey]struct{} + declarationSHA256 string + trainingDeclarationSHA256 string +} + +var orientationV2CanonicalCases = []struct { + dataset string + name string + split string +}{ + {"generated_fixed_suffix_expansion_v3_d2_f4_r0_x2_i0_m1_q1_z1_c0_s0_p0", "GFSE-V3-TRAIN-Q1-C0-S0-root_baseline", "training"}, + {"generated_fixed_suffix_expansion_v3_d2_f4_r0_x2_i0_m1_q4_z1_c0_s0_p0", "GFSE-V3-TRAIN-Q4-C0-S0-root_multiplicity", "training"}, + {"generated_fixed_suffix_expansion_v3_d2_f4_r0_x2_i0_m1_q4_z1_c1_s0_p0", "GFSE-V3-TRAIN-Q4-C1-S0-productive_cycle", "training"}, + {"generated_fixed_suffix_expansion_v3_d2_f4_r0_x2_i0_m1_q4_z1_c0_s1_p0", "GFSE-V3-TRAIN-Q4-C0-S1-productive_self_loop", "training"}, + {"generated_fixed_suffix_expansion_v3_d2_f4_r0_x2_i0_m1_q4_z1_c1_s1_p0", "GFSE-V3-TRAIN-Q4-C1-S1-productive_cycle_self_loop_path", "training"}, + {"generated_fixed_suffix_expansion_v3_d3_f6_r1_x0_i4_m2_q2_z0_c0_s0_p32", "GFSE-V3-TRAIN-D03-F006-R1-X0-I4-M2-Q2-endpoint", "training"}, + {"generated_fixed_suffix_expansion_v3_d5_f8_r4_x3_i0_m1_q3_z0_c0_s0_p0", "GFSE-V3-TRAIN-D05-F008-R4-X3-I0-M1-Q3-path", "training"}, + {"generated_fixed_suffix_expansion_v3_d6_f10_r10_x1_i7_m3_q1_z0_c0_s0_p64", "GFSE-V3-TRAIN-D06-F010-R10-X1-I7-M3-Q1-endpoint", "training"}, + {"generated_fixed_suffix_expansion_v3_d7_f5_r1_x3_i6_m2_q6_z0_c1_s1_p24", "GFSE-V3-HOLDOUT-D07-F005-R1-X3-I6-M2-Q6-C1-S1-path", "holdout"}, + {"generated_fixed_suffix_expansion_v3_d11_f7_r0_x4_i0_m3_q2_z1_c1_s0_p96", "GFSE-V3-HOLDOUT-D11-F007-R0-X4-I0-M3-Q2-C1-S0-endpoint", "holdout"}, + {"generated_fixed_suffix_expansion_v3_d13_f9_r4_x1_i2_m1_q7_z0_c0_s1_p8", "GFSE-V3-HOLDOUT-D13-F009-R4-X1-I2-M1-Q7-C0-S1-path", "holdout"}, + {"generated_fixed_suffix_expansion_v3_d15_f12_r6_x6_i9_m2_q3_z1_c0_s0_p128", "GFSE-V3-HOLDOUT-D15-F012-R6-X6-I9-M2-Q3-Z1-endpoint", "holdout"}, +} + +func canonicalOrientationV2Cohort() (orientationV2CanonicalCohort, error) { + keys := map[performanceKey]struct{}{} + trainingKeys := map[performanceKey]struct{}{} + declared := make([]DeclaredCaseBackend, 0, 24) + trainingDeclared := make([]DeclaredCaseBackend, 0, 16) + training, holdout := 0, 0 + for _, testCase := range orientationV2CanonicalCases { + key := performanceKey{dataset: testCase.dataset, name: testCase.name, backend: ModePostgresSQL} + if _, duplicate := keys[key]; duplicate || !strings.HasPrefix(testCase.dataset, "generated_fixed_suffix_expansion_v3_") { + return orientationV2CanonicalCohort{}, fmt.Errorf("frozen orientation-v2 cohort contains an invalid declaration") + } + keys[key] = struct{}{} + for _, backend := range []ExecutionMode{ModePostgresSQL, ModeNeo4j} { + declared = append(declared, DeclaredCaseBackend{Dataset: key.dataset, Name: key.name, Backend: backend}) + } + if testCase.split == "training" { + training++ + trainingKeys[key] = struct{}{} + for _, backend := range []ExecutionMode{ModePostgresSQL, ModeNeo4j} { + trainingDeclared = append(trainingDeclared, DeclaredCaseBackend{Dataset: key.dataset, Name: key.name, Backend: backend}) + } + } else if testCase.split == "holdout" { + holdout++ + } else { + return orientationV2CanonicalCohort{}, fmt.Errorf("frozen orientation-v2 cohort contains an invalid split") + } + } + if training != 8 || holdout != 4 || len(keys) != 12 { + return orientationV2CanonicalCohort{}, fmt.Errorf("frozen orientation-v2 cohort must contain exactly 8 training and 4 holdout cases") + } + return orientationV2CanonicalCohort{ + keys: keys, trainingKeys: trainingKeys, declarationSHA256: declarationSHA256(declared), + trainingDeclarationSHA256: declarationSHA256(trainingDeclared), + }, nil +} + +func performanceKeySet(keys []performanceKey) map[performanceKey]struct{} { + result := make(map[performanceKey]struct{}, len(keys)) + for _, key := range keys { + result[key] = struct{}{} + } + return result +} + +func validateOrientationV2Freeze(freeze *OrientationSelectorV2FreezeManifest, discovery *OrientationSelectorV2Report, report OrientationSelectorV2Report) error { + if freeze == nil || discovery == nil { + return fmt.Errorf("orientation-v2 confirmation requires a discovery report and freeze manifest") + } + if freeze.Version != 1 || freeze.Policy != report.Policy || freeze.Formula != report.Formula || + freeze.SourceCommit != report.SourceCommit || freeze.DirtyDiffSHA256 != report.DirtyDiffSHA256 || + freeze.BinarySHA256 != report.BinarySHA256 || freeze.CohortDeclarationSHA256 != report.CohortDeclarationSHA256 || + !lowercaseSHA256(freeze.DiscoveryReportSHA256) || len(freeze.Caps) != len(report.Caps) { + return fmt.Errorf("orientation-v2 confirmation identity differs from the frozen discovery") + } + if report.DirtyDiffSHA256 != cleanWorkingTreeSHA256() || discovery.Version != orientationSelectorReportV2Version || + discovery.Protocol != referencePairProtocolDiscovery || discovery.Policy != freeze.Policy || discovery.Formula != freeze.Formula || + discovery.SourceCommit != freeze.SourceCommit || discovery.DirtyDiffSHA256 != freeze.DirtyDiffSHA256 || + discovery.BinarySHA256 != freeze.BinarySHA256 || len(discovery.Cases) != 8 || + !lowercaseSHA256(discovery.ShadowArtifactSHA256) || !lowercaseSHA256(discovery.IncumbentArtifactSHA256) || + !lowercaseSHA256(discovery.ReverseArtifactSHA256) || !lowercaseSHA256(discovery.GuardedArtifactSHA256) || + !lowercaseSHA256(discovery.AAReportSHA256) { + return fmt.Errorf("orientation-v2 discovery report does not prove the frozen clean training-only identity") + } + canonical, err := canonicalOrientationV2Cohort() + if err != nil { + return err + } + discoveryKeys := map[performanceKey]struct{}{} + for _, entry := range discovery.Cases { + if entry.QualificationSplit != "training" { + return fmt.Errorf("orientation-v2 discovery report contains non-training timing") + } + discoveryKeys[performanceKey{dataset: entry.Dataset, name: entry.Name, backend: ModePostgresSQL}] = struct{}{} + } + if !orientationV2KeySetsEqual(discoveryKeys, canonical.trainingKeys) { + return fmt.Errorf("orientation-v2 discovery report does not contain the exact frozen training cohort") + } + if discovery.CohortDeclarationSHA256 != canonical.trainingDeclarationSHA256 { + return fmt.Errorf("orientation-v2 discovery report does not bind the exact frozen training declaration") + } + for name, value := range report.Caps { + if freeze.Caps[name] != value || discovery.Caps[name] != value { + return fmt.Errorf("orientation-v2 confirmation cap %s differs from the frozen discovery", name) + } + } + return nil +} + +func orientationV2RecordRound(record CaseResult) (int, error) { + round := 0 + if record.Environment != nil { + round = record.Environment.Round + } + for _, sample := range record.Stats.Samples { + if sample.Classification != "warm" || sample.Duration <= 0 { + continue + } + current := sample.Round + if current == 0 { + current = round + } + if current < 1 || (round != 0 && current != round) { + return 0, fmt.Errorf("%s/%s has inconsistent orientation-v2 round metadata", record.Dataset, record.Name) + } + round = current + } + if round < 1 { + return 0, fmt.Errorf("%s/%s has no orientation-v2 round identity", record.Dataset, record.Name) + } + return round, nil +} + +func validateOrientationV2Record(record CaseResult, arm string) error { + if record.Status != StatusOK || record.Environment == nil || record.PostgresEnvironment == nil || record.TraversalTelemetry == nil { + return fmt.Errorf("%s/%s %s arm lacks a successful telemetry-bearing PostgreSQL record", record.Dataset, record.Name, arm) + } + if record.Environment.ArtifactSchemaVersion != 2 || record.Environment.PoolSize != 1 || len(record.Environment.Concurrency) != 0 { + return fmt.Errorf("%s/%s %s arm lacks the schema-v2 single-session timing contract", record.Dataset, record.Name, arm) + } + if record.Environment.ExistingGraph || record.Fixture == nil || record.Fixture.Dataset != record.Dataset || + !lowercaseSHA256(record.Fixture.Checksum) || !record.Fixture.PhysicalValidated { + return fmt.Errorf("%s/%s %s arm lacks one exact physically validated corpus fixture", record.Dataset, record.Name, arm) + } + if !lowercaseSHA256(record.WorkloadSHA256) || !lowercaseSHA256(record.SQLFingerprint) { + return fmt.Errorf("%s/%s %s arm lacks canonical workload or SQL identity", record.Dataset, record.Name, arm) + } + if len(record.Concurrency) != 0 || len(record.PostgresReferences) != 0 || record.ClientWaterfall != nil || + record.RawPGXWaterfall != nil || record.RawPGXRoundTrip != nil { + return fmt.Errorf("%s/%s %s arm mixes selector timing with supplemental PostgreSQL measurements", record.Dataset, record.Name, arm) + } + if !strings.EqualFold(strings.TrimSpace(record.PostgresEnvironment.TransactionIsolation), "repeatable read") { + return fmt.Errorf("%s/%s %s arm was not measured under Repeatable Read", record.Dataset, record.Name, arm) + } + if err := record.TraversalTelemetry.Validate(); err != nil { + return fmt.Errorf("%s/%s %s arm telemetry: %w", record.Dataset, record.Name, arm, err) + } + summary := record.TraversalTelemetry.Summary + if summary.RuntimeOutcomeAvailable == nil || !*summary.RuntimeOutcomeAvailable || summary.Overflow == nil || summary.FallbackExecuted == nil { + return fmt.Errorf("%s/%s %s arm lacks a complete runtime outcome", record.Dataset, record.Name, arm) + } + forward := string(optimize.ExpansionSearchStepwiseForward) + reverse := string(optimize.ExpansionSearchSuffixSeededReverse) + v2 := string(optimize.ExpansionSearchPolicyOrientationProbeV2) + switch arm { + case "shadow": + if summary.EmittedIdentity != v2 || summary.SelectorVersion != v2 || + summary.ExecutionBoundary != optimize.ExpansionSearchExecutionBoundaryInlineStatement || + summary.RuntimeIdentity != forward || summary.AppliedIdentity != forward || summary.RuntimeBranch != "shadow_incumbent" || + (summary.WouldSelectIdentity != forward && summary.WouldSelectIdentity != reverse) || *summary.FallbackExecuted { + return fmt.Errorf("%s/%s shadow telemetry does not prove orientation-probe-v2 incumbent-only execution", record.Dataset, record.Name) + } + if *summary.Overflow && summary.WouldSelectIdentity != forward { + return fmt.Errorf("%s/%s overflowing shadow evidence did not fail closed to forward", record.Dataset, record.Name) + } + case "incumbent": + if summary.EmittedIdentity != forward || summary.RuntimeIdentity != forward || summary.AppliedIdentity != forward || + summary.ExecutionBoundary != optimize.ExpansionSearchExecutionBoundaryInlineStatement || summary.WouldSelectIdentity != "" || *summary.Overflow { + return fmt.Errorf("%s/%s incumbent artifact did not execute one exact forward statement", record.Dataset, record.Name) + } + validSelected := summary.RuntimeBranch == "selected" && !*summary.FallbackExecuted + validCompileFallback := summary.RuntimeBranch == "compile_time_fallback" && *summary.FallbackExecuted && summary.FallbackIdentity == forward + if !validSelected && !validCompileFallback { + return fmt.Errorf("%s/%s incumbent artifact has an unsupported exact-arm runtime tuple", record.Dataset, record.Name) + } + if summary.SelectorVersion != "fixed-suffix-static-v1" { + return fmt.Errorf("%s/%s incumbent artifact has an unexpected selector identity", record.Dataset, record.Name) + } + case "reverse": + if summary.EmittedIdentity != reverse || summary.RuntimeIdentity != reverse || summary.AppliedIdentity != reverse || + summary.ExecutionBoundary != optimize.ExpansionSearchExecutionBoundaryInlineStatement || summary.WouldSelectIdentity != "" || + summary.RuntimeBranch != "selected" || *summary.Overflow || *summary.FallbackExecuted { + return fmt.Errorf("%s/%s reverse artifact did not execute one exact forced-reverse statement", record.Dataset, record.Name) + } + if summary.SelectorVersion != "suffix-seeded-reverse-tool-v1" { + return fmt.Errorf("%s/%s reverse artifact has an unexpected selector identity", record.Dataset, record.Name) + } + case "guarded": + if summary.EmittedIdentity != v2 || summary.SelectorVersion != v2 || + summary.ExecutionBoundary != optimize.ExpansionSearchExecutionBoundaryGuardedDualArm || summary.WouldSelectIdentity != "" { + return fmt.Errorf("%s/%s guarded artifact does not prove the orientation-probe-v2 dual-arm boundary", record.Dataset, record.Name) + } + default: + return fmt.Errorf("unknown orientation-v2 arm %q", arm) + } + if err := validateOrientationV2SampleRuntime(record, arm); err != nil { + return err + } + return nil +} + +func validateOrientationV2SampleRuntime(record CaseResult, arm string) error { + summary := record.TraversalTelemetry.Summary + for _, sample := range record.Stats.Samples { + if sample.Classification != "warm" || sample.Duration <= 0 { + continue + } + if sample.RequestedIdentity != summary.RequestedIdentity || sample.RuntimeIdentity != summary.RuntimeIdentity || + sample.RuntimeBranch != summary.RuntimeBranch || sample.FallbackExecuted == nil || + *sample.FallbackExecuted != *summary.FallbackExecuted { + return fmt.Errorf("%s/%s %s arm warm sample contradicts its runtime summary", record.Dataset, record.Name, arm) + } + switch arm { + case "shadow", "guarded": + if sample.RuntimeAttestation != "timed_invocation" { + return fmt.Errorf("%s/%s %s arm warm sample lacks timed-invocation attribution", record.Dataset, record.Name, arm) + } + if err := validateRuntimeReceiptEvents(sample.RuntimeReceiptEvents, sample.RuntimeIdentity, sample.RuntimeBranch, sample.FallbackExecuted); err != nil { + return fmt.Errorf("%s/%s %s arm warm sample receipt: %w", record.Dataset, record.Name, arm, err) + } + case "incumbent", "reverse": + if sample.RuntimeAttestation != "same_case_invocation_local_replay" && sample.RuntimeAttestation != "timed_invocation" { + return fmt.Errorf("%s/%s %s exact arm warm sample lacks runtime attribution", record.Dataset, record.Name, arm) + } + if sample.RuntimeAttestation == "timed_invocation" { + if err := validateRuntimeReceiptEvents(sample.RuntimeReceiptEvents, sample.RuntimeIdentity, sample.RuntimeBranch, sample.FallbackExecuted); err != nil { + return fmt.Errorf("%s/%s %s exact arm warm sample receipt: %w", record.Dataset, record.Name, arm, err) + } + } else if len(sample.RuntimeReceiptEvents) != 0 { + return fmt.Errorf("%s/%s %s exact arm replay must not claim a timed receipt", record.Dataset, record.Name, arm) + } + } + } + return nil +} + +func validateOrientationV2RuntimeConsistency(key performanceKey, current *orientationSelectorV2Series) error { + forward := string(optimize.ExpansionSearchStepwiseForward) + reverse := string(optimize.ExpansionSearchSuffixSeededReverse) + if current.shadowOverflow && !current.overflow { + return fmt.Errorf("%s/%s guarded evidence lost shadow probe overflow", key.dataset, key.name) + } + if current.overflow { + choiceConsistent := current.shadowOverflow && current.wouldSelect == forward || !current.shadowOverflow && current.wouldSelect == reverse + if !choiceConsistent || current.guardedRuntime != forward || current.guardedBranch != "exact_forward_incumbent" || !current.fallback { + return fmt.Errorf("%s/%s guarded overflow did not execute the exact forward fallback", key.dataset, key.name) + } + return nil + } + if current.fallback { + return fmt.Errorf("%s/%s guarded artifact reports fallback without overflow", key.dataset, key.name) + } + if current.wouldSelect == reverse { + if current.guardedRuntime != reverse || current.guardedBranch != "suffix_seeded_reverse" { + return fmt.Errorf("%s/%s guarded runtime does not match the shadow reverse choice", key.dataset, key.name) + } + return nil + } + if current.wouldSelect == forward && current.guardedRuntime == forward && current.guardedBranch == "exact_forward_incumbent" { + return nil + } + return fmt.Errorf("%s/%s guarded runtime does not match the shadow forward choice", key.dataset, key.name) +} + +func requireOrientationV2RoundSets(key performanceKey, current *orientationSelectorV2Series) error { + expected := sortedRounds(current.shadow) + for name, rounds := range map[string][]int{ + "incumbent": sortedRounds(current.incumbent), + "reverse": sortedRounds(current.reverse), + "guarded": sortedRounds(current.guarded), + } { + if !slices.Equal(expected, rounds) { + return fmt.Errorf("%s/%s %s arm round set does not match shadow", key.dataset, key.name, name) + } + } + return nil +} + +func validateOrientationV2ArmOrder( + shadowRecords, incumbentRecords, reverseRecords, guardedRecords []CaseResult, + key performanceKey, + rounds []int, + minimumWarmups int, +) error { + armRecords := []struct { + name string + records []CaseResult + }{ + {name: "shadow", records: shadowRecords}, + {name: "incumbent", records: incumbentRecords}, + {name: "reverse", records: reverseRecords}, + {name: "guarded", records: guardedRecords}, + } + evidence := make([]map[int]pairedRoundEvidence, len(armRecords)) + positionCounts := make([][5]int, len(armRecords)) + for index, arm := range armRecords { + current, err := collectPairedRoundEvidence(arm.records, key) + if err != nil { + return err + } + evidence[index] = current + } + for _, round := range rounds { + seenPositions := map[int]struct{}{} + seenNames := map[string]struct{}{} + block, runUUID := 0, "" + for index, arm := range armRecords { + current, found := evidence[index][round] + if !found || current.Warmups < minimumWarmups || current.Arm != arm.name { + return fmt.Errorf("%s/%s round %d lacks %s arm identity or %d warmups", key.dataset, key.name, round, arm.name, minimumWarmups) + } + if current.ArmOrder < 1 || current.ArmOrder > 4 { + return fmt.Errorf("%s/%s round %d has invalid four-arm order", key.dataset, key.name, round) + } + if _, duplicate := seenPositions[current.ArmOrder]; duplicate { + return fmt.Errorf("%s/%s round %d has duplicate four-arm order", key.dataset, key.name, round) + } + if _, duplicate := seenNames[current.Arm]; duplicate { + return fmt.Errorf("%s/%s round %d has indistinct four-arm labels", key.dataset, key.name, round) + } + seenPositions[current.ArmOrder] = struct{}{} + seenNames[current.Arm] = struct{}{} + positionCounts[index][current.ArmOrder]++ + if block == 0 { + block, runUUID = current.Block, current.RunUUID + } else if current.Block != block || current.RunUUID != runUUID { + return fmt.Errorf("%s/%s round %d has mismatched four-arm block or run UUID", key.dataset, key.name, round) + } + } + if block < 1 || runUUID == "" || len(seenPositions) != 4 || len(seenNames) != 4 { + return fmt.Errorf("%s/%s round %d lacks a complete four-arm block", key.dataset, key.name, round) + } + } + for index, counts := range positionCounts { + minimum, maximum := counts[1], counts[1] + for position := 2; position <= 4; position++ { + minimum = min(minimum, counts[position]) + maximum = max(maximum, counts[position]) + } + if maximum-minimum > 1 { + return fmt.Errorf("%s/%s %s arm order is not position-balanced", key.dataset, key.name, armRecords[index].name) + } + } + return nil +} + +func validateOrientationV2EvidenceIdentity(artifacts ...[]CaseResult) (orientationSelectorV2Identity, error) { + identity := orientationSelectorV2Identity{} + var postgresEnvironment *PostgresEnvironment + allRecords := make([]CaseResult, 0) + for _, records := range artifacts { + allRecords = append(allRecords, records...) + for _, record := range records { + if record.Environment == nil || record.PostgresEnvironment == nil { + return orientationSelectorV2Identity{}, fmt.Errorf("%s/%s lacks orientation-v2 environment identity", record.Dataset, record.Name) + } + current := orientationSelectorV2Identity{ + sourceCommit: strings.TrimSpace(record.Environment.SourceCommit), dirtyDiffSHA256: record.Environment.DirtyDiffSHA256, + binarySHA256: record.Environment.BinarySHA256, corpusSHA256: record.Environment.CorpusSHA256, + } + if current.sourceCommit == "" || current.sourceCommit == "unknown" || + !lowercaseSHA256(current.dirtyDiffSHA256) || !lowercaseSHA256(current.binarySHA256) || !lowercaseSHA256(current.corpusSHA256) { + return orientationSelectorV2Identity{}, fmt.Errorf("%s/%s lacks frozen source, diff, binary, or corpus identity", record.Dataset, record.Name) + } + if identity.sourceCommit == "" { + identity = current + } else if identity != current { + return orientationSelectorV2Identity{}, fmt.Errorf("orientation-v2 artifacts mix source, diff, binary, or corpus identities") + } + if postgresEnvironment == nil { + copy := *record.PostgresEnvironment + postgresEnvironment = © + } else if !sameOrientationV2PostgresEnvironment(postgresEnvironment, record.PostgresEnvironment) { + return orientationSelectorV2Identity{}, fmt.Errorf("orientation-v2 artifacts mix PostgreSQL environments") + } + } + } + keys, err := orientationV2ArtifactKeys("combined", allRecords) + if err != nil { + return orientationSelectorV2Identity{}, err + } + for key := range keys { + postgresEnvironmentSHA256, err := postgresTimingEnvironmentSHA256ForKey(allRecords, key) + if err != nil { + return orientationSelectorV2Identity{}, err + } + fixtureSHA256, err := fixtureSHA256ForKey(allRecords, key) + if err != nil { + return orientationSelectorV2Identity{}, err + } + if !lowercaseSHA256(postgresEnvironmentSHA256) || !lowercaseSHA256(fixtureSHA256) { + return orientationSelectorV2Identity{}, fmt.Errorf("%s/%s lacks frozen PostgreSQL or fixture identity", key.dataset, key.name) + } + } + return identity, nil +} + +func validateOrientationV2AAEvidence(report *AAResolutionReport, records []CaseResult) error { + keys, err := orientationV2ArtifactKeys("incumbent", records) + if err != nil { + return err + } + entries := make(map[performanceKey]AAResolutionCase, len(report.Cases)) + for _, entry := range report.Cases { + entries[performanceKey{dataset: entry.Dataset, name: entry.Name, backend: entry.Backend}] = entry + } + for key := range keys { + entry, found := entries[key] + if !found { + return fmt.Errorf("A/A report has no environment evidence for %s/%s", key.dataset, key.name) + } + postgresEnvironmentSHA256, err := postgresTimingEnvironmentSHA256ForKey(records, key) + if err != nil { + return err + } + fixtureSHA256, err := fixtureSHA256ForKey(records, key) + if err != nil { + return err + } + if !lowercaseSHA256(entry.PostgresEnvironmentSHA256) || entry.PostgresEnvironmentSHA256 != postgresEnvironmentSHA256 { + return fmt.Errorf("A/A PostgreSQL environment does not match %s/%s", key.dataset, key.name) + } + if !lowercaseSHA256(entry.FixtureSHA256) || entry.FixtureSHA256 != fixtureSHA256 { + return fmt.Errorf("A/A fixture does not match %s/%s", key.dataset, key.name) + } + } + return nil +} + +func lowercaseSHA256(value string) bool { + return value == strings.ToLower(value) && validSHA256(value) +} + +func sameOrientationV2PostgresEnvironment(left, right *PostgresEnvironment) bool { + return left.Version == right.Version && left.Database == right.Database && + left.PlanCacheMode == right.PlanCacheMode && left.TransactionIsolation == right.TransactionIsolation && + left.WorkMem == right.WorkMem && left.TempFileLimit == right.TempFileLimit && + left.GraphPartitionCount == right.GraphPartitionCount && + left.DatabaseOID == right.DatabaseOID && left.PostmasterStartedAt.Equal(right.PostmasterStartedAt) && + left.Autovacuum == right.Autovacuum && + left.SchemaFingerprint == right.SchemaFingerprint && left.IndexFingerprint == right.IndexFingerprint +} + +// createOrientationSelectorV2Report loads four matched timing artifacts and +// one checksummed A/A report, then writes schema-v2 qualification evidence. +func createOrientationSelectorV2Report( + shadowPath, incumbentPath, reversePath, guardedPath, aaPath, freezePath, discoveryReportPath, freezeOutputPath, outputPath string, + options OrientationSelectorV2ReportOptions, +) (bool, error) { + paths := []struct { + name string + path string + }{ + {name: "shadow", path: shadowPath}, + {name: "incumbent", path: incumbentPath}, + {name: "reverse", path: reversePath}, + {name: "guarded", path: guardedPath}, + } + artifacts := make([][]CaseResult, len(paths)) + for index, input := range paths { + records, err := readJSONLFile(input.path) + if err != nil { + return false, fmt.Errorf("read orientation-v2 %s artifact: %w", input.name, err) + } + artifacts[index] = records + } + aa, aaSHA, err := loadAAResolutionReport(aaPath) + if err != nil { + return false, fmt.Errorf("read orientation-v2 A/A report: %w", err) + } + freezeSHA := "" + if freezePath != "" { + freeze, digest, err := loadOrientationSelectorV2FreezeManifest(freezePath) + if err != nil { + return false, fmt.Errorf("read orientation-v2 freeze manifest: %w", err) + } + options.Freeze = freeze + freezeSHA = digest + discovery, err := loadOrientationSelectorV2Report(discoveryReportPath) + if err != nil { + return false, fmt.Errorf("read orientation-v2 discovery report: %w", err) + } + if digest, err := fileSHA256(discoveryReportPath); err != nil { + return false, err + } else if digest != freeze.DiscoveryReportSHA256 { + return false, fmt.Errorf("orientation-v2 discovery report digest does not match freeze manifest") + } + options.Discovery = discovery + } + report, err := buildOrientationSelectorV2Report(artifacts[0], artifacts[1], artifacts[2], artifacts[3], aa, options) + if err != nil { + return false, err + } + for index, input := range paths { + digest, err := fileSHA256(input.path) + if err != nil { + return false, err + } + switch index { + case 0: + report.ShadowArtifactSHA256 = digest + case 1: + report.IncumbentArtifactSHA256 = digest + case 2: + report.ReverseArtifactSHA256 = digest + case 3: + report.GuardedArtifactSHA256 = digest + } + } + report.AAReportSHA256 = aaSHA + report.FreezeManifestSHA256 = freezeSHA + if err := writeOrientationSelectorV2Report(outputPath, report); err != nil { + return false, err + } + if options.Protocol == referencePairProtocolDiscovery { + if err := writeOrientationSelectorV2FreezeManifest(freezeOutputPath, outputPath, report, artifacts...); err != nil { + return false, err + } + } + return report.QualificationPassed, nil +} + +func loadOrientationSelectorV2Report(path string) (*OrientationSelectorV2Report, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + report := &OrientationSelectorV2Report{} + if err := json.Unmarshal(raw, report); err != nil { + return nil, fmt.Errorf("decode orientation-v2 discovery report: %w", err) + } + return report, nil +} + +func loadOrientationSelectorV2FreezeManifest(path string) (*OrientationSelectorV2FreezeManifest, string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, "", err + } + manifest := &OrientationSelectorV2FreezeManifest{} + if err := json.Unmarshal(raw, manifest); err != nil { + return nil, "", fmt.Errorf("decode orientation-v2 freeze manifest: %w", err) + } + digest := sha256.Sum256(raw) + return manifest, hex.EncodeToString(digest[:]), nil +} + +func writeOrientationSelectorV2FreezeManifest(path, discoveryReportPath string, report OrientationSelectorV2Report, artifacts ...[]CaseResult) error { + if path == "" || discoveryReportPath == "" { + return fmt.Errorf("orientation-v2 discovery freeze requires report and manifest output paths") + } + canonical, err := canonicalOrientationV2Cohort() + if err != nil { + return err + } + training := map[performanceKey]struct{}{} + for _, record := range artifacts[0] { + key := performanceKey{dataset: record.Dataset, name: record.Name, backend: record.ExecutionMode} + if record.Shape.QualificationSplit == "training" { + training[key] = struct{}{} + } + } + if !orientationV2KeySetsEqual(training, canonical.trainingKeys) || report.CohortDeclarationSHA256 != canonical.trainingDeclarationSHA256 { + return fmt.Errorf("orientation-v2 discovery freeze requires the exact eight canonical training cases and no holdouts") + } + for _, records := range artifacts { + for _, record := range records { + if record.Shape.QualificationSplit != "training" { + return fmt.Errorf("orientation-v2 discovery freeze cannot contain holdout or diagnostic timing") + } + } + } + if report.DirtyDiffSHA256 != cleanWorkingTreeSHA256() { + return fmt.Errorf("orientation-v2 discovery freeze requires a clean source tree") + } + discoveryReportSHA256, err := fileSHA256(discoveryReportPath) + if err != nil { + return err + } + manifest := OrientationSelectorV2FreezeManifest{ + Version: 1, Policy: report.Policy, Formula: report.Formula, Caps: report.Caps, + SourceCommit: report.SourceCommit, DirtyDiffSHA256: report.DirtyDiffSHA256, BinarySHA256: report.BinarySHA256, + CohortDeclarationSHA256: canonical.declarationSHA256, DiscoveryReportSHA256: discoveryReportSHA256, + } + if err := ensureOutputDir(path); err != nil { + return err + } + output, err := os.Create(path) + if err != nil { + return err + } + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + encodeErr := encoder.Encode(manifest) + closeErr := output.Close() + if encodeErr != nil { + return encodeErr + } + return closeErr +} + +func writeOrientationSelectorV2Report(path string, report OrientationSelectorV2Report) (err error) { + output := os.Stdout + if path != "" { + if err := ensureOutputDir(path); err != nil { + return err + } + output, err = os.Create(path) + if err != nil { + return err + } + defer func() { + if closeErr := output.Close(); err == nil && closeErr != nil { + err = closeErr + } + }() + } + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(report) +} diff --git a/cmd/graphbench/orientation_selector_report_v2_test.go b/cmd/graphbench/orientation_selector_report_v2_test.go new file mode 100644 index 00000000..96be1bfd --- /dev/null +++ b/cmd/graphbench/orientation_selector_report_v2_test.go @@ -0,0 +1,666 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "os" + "path/filepath" + "slices" + "testing" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/stretchr/testify/require" +) + +func TestOrientationSelectorV2ReportPassesForwardAndReverseWithApplicableShadowGate(t *testing.T) { + artifacts := orientationSelectorV2Artifacts{} + for index := range 8 { + training := orientationSelectorV2Records( + "training", string(optimize.ExpansionSearchStepwiseForward), + 10*time.Millisecond+50*time.Microsecond, 10*time.Millisecond, 14*time.Millisecond, 10*time.Millisecond+60*time.Microsecond, + false, + ) + renameOrientationV2Records(fmt.Sprintf("training-forward-%02d", index), training) + artifacts = appendOrientationV2Artifacts(artifacts, training) + } + for index := range 4 { + holdout := orientationSelectorV2Records( + "holdout", string(optimize.ExpansionSearchSuffixSeededReverse), + 30*time.Millisecond, 10*time.Millisecond, 5*time.Millisecond, 5*time.Millisecond+50*time.Microsecond, + false, + ) + renameOrientationV2Records(fmt.Sprintf("holdout-reverse-%02d", index), holdout) + artifacts = appendOrientationV2Artifacts(artifacts, holdout) + } + + report, err := buildOrientationSelectorV2Report( + artifacts.shadow, artifacts.incumbent, artifacts.reverse, artifacts.guarded, + testAAReportForRecords(t, artifacts.incumbent), + OrientationSelectorV2ReportOptions{Seed: 7, Confidence: defaultConfidenceLevel, BootstrapCount: 100, Protocol: referencePairProtocolDiscovery}, + ) + + require.NoError(t, err) + require.Equal(t, orientationSelectorReportV2Version, report.Version) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV2), report.Policy) + require.False(t, report.QualificationPassed) + require.Zero(t, report.TrainingCases) + require.Zero(t, report.HoldoutCases) + require.Len(t, report.Cases, 12) + for _, entry := range report.Cases { + require.True(t, entry.Passed) + require.True(t, entry.GuardedSelectedOverhead.Passed) + require.True(t, entry.GuardedFastestRegret.Passed) + if entry.WouldSelectIdentity == string(optimize.ExpansionSearchStepwiseForward) { + require.True(t, entry.ShadowForwardOverhead.Applicable) + } else { + require.False(t, entry.ShadowForwardOverhead.Applicable) + require.Greater(t, entry.ShadowForwardOverhead.Ratio.Upper, 1.10) + require.False(t, entry.ShadowForwardOverhead.Passed) + } + } +} + +func TestOrientationSelectorV2ConfirmationBindsCanonicalCohortAndFrozenDiscovery(t *testing.T) { + training, full := canonicalOrientationV2TestArtifacts(t) + discovery, err := buildOrientationSelectorV2Report( + training.shadow, training.incumbent, training.reverse, training.guarded, + testAAReportForRecords(t, training.incumbent), + OrientationSelectorV2ReportOptions{Seed: 5, Confidence: defaultConfidenceLevel, BootstrapCount: 50, Protocol: referencePairProtocolDiscovery}, + ) + require.NoError(t, err) + discovery.ShadowArtifactSHA256, discovery.IncumbentArtifactSHA256 = testSHA("1"), testSHA("2") + discovery.ReverseArtifactSHA256, discovery.GuardedArtifactSHA256 = testSHA("3"), testSHA("4") + discovery.AAReportSHA256 = testSHA("5") + canonical, err := canonicalOrientationV2Cohort() + require.NoError(t, err) + freeze := testOrientationV2Freeze() + freeze.DirtyDiffSHA256 = cleanWorkingTreeSHA256() + freeze.CohortDeclarationSHA256 = canonical.declarationSHA256 + + report, err := buildOrientationSelectorV2Report( + full.shadow, full.incumbent, full.reverse, full.guarded, + testAAReportForRecords(t, full.incumbent), + OrientationSelectorV2ReportOptions{ + Seed: 7, Confidence: defaultConfidenceLevel, BootstrapCount: 50, Protocol: referencePairProtocolConfirmation, + Freeze: freeze, Discovery: &discovery, + }, + ) + + require.NoError(t, err) + require.True(t, report.QualificationPassed) + require.Equal(t, 8, report.TrainingCases) + require.Equal(t, 4, report.HoldoutCases) + require.Equal(t, canonical.declarationSHA256, report.CohortDeclarationSHA256) +} + +func TestCreateOrientationSelectorV2DiscoveryWritesBoundFreeze(t *testing.T) { + training, _ := canonicalOrientationV2TestArtifacts(t) + training = compactOrientationV2Artifacts(training, 5, 10) + directory := t.TempDir() + paths := map[string]string{ + "shadow": filepath.Join(directory, "shadow.jsonl"), "incumbent": filepath.Join(directory, "incumbent.jsonl"), + "reverse": filepath.Join(directory, "reverse.jsonl"), "guarded": filepath.Join(directory, "guarded.jsonl"), + "aa": filepath.Join(directory, "aa.json"), "report": filepath.Join(directory, "discovery.json"), + "freeze": filepath.Join(directory, "freeze.json"), + } + writeOrientationV2TestArtifact(t, paths["shadow"], training.shadow) + writeOrientationV2TestArtifact(t, paths["incumbent"], training.incumbent) + writeOrientationV2TestArtifact(t, paths["reverse"], training.reverse) + writeOrientationV2TestArtifact(t, paths["guarded"], training.guarded) + require.NoError(t, writeAAResolutionReport(paths["aa"], *testAAReportForRecords(t, training.incumbent))) + + passed, err := createOrientationSelectorV2Report( + paths["shadow"], paths["incumbent"], paths["reverse"], paths["guarded"], paths["aa"], "", "", paths["freeze"], paths["report"], + OrientationSelectorV2ReportOptions{Seed: 11, Confidence: defaultConfidenceLevel, BootstrapCount: 10, Protocol: referencePairProtocolDiscovery}, + ) + + require.NoError(t, err) + require.False(t, passed) + freeze, _, err := loadOrientationSelectorV2FreezeManifest(paths["freeze"]) + require.NoError(t, err) + report, err := loadOrientationSelectorV2Report(paths["report"]) + require.NoError(t, err) + reportSHA256, err := fileSHA256(paths["report"]) + require.NoError(t, err) + canonical, err := canonicalOrientationV2Cohort() + require.NoError(t, err) + require.Equal(t, reportSHA256, freeze.DiscoveryReportSHA256) + require.Equal(t, canonical.declarationSHA256, freeze.CohortDeclarationSHA256) + require.Equal(t, report.Policy, freeze.Policy) + require.Equal(t, cleanWorkingTreeSHA256(), freeze.DirtyDiffSHA256) +} + +func TestOrientationSelectorV2ReportEnforcesEachLatencyGate(t *testing.T) { + for _, testCase := range []struct { + name string + choice string + shadow time.Duration + forward time.Duration + reverse time.Duration + guarded time.Duration + reason string + shadowFails bool + selectedFails bool + fastestFails bool + }{ + { + name: "forward shadow", choice: string(optimize.ExpansionSearchStepwiseForward), + shadow: 12 * time.Millisecond, forward: 10 * time.Millisecond, reverse: 14 * time.Millisecond, guarded: 10 * time.Millisecond, + reason: "forward-selected shadow overhead", shadowFails: true, + }, + { + name: "guarded selected", choice: string(optimize.ExpansionSearchSuffixSeededReverse), + shadow: 20 * time.Millisecond, forward: 10 * time.Millisecond, reverse: 5 * time.Millisecond, guarded: 7 * time.Millisecond, + reason: "guarded selected-arm overhead", selectedFails: true, fastestFails: true, + }, + { + name: "guarded fastest", choice: string(optimize.ExpansionSearchStepwiseForward), + shadow: 10 * time.Millisecond, forward: 10 * time.Millisecond, reverse: 5 * time.Millisecond, guarded: 10 * time.Millisecond, + reason: "guarded fastest-arm regret", fastestFails: true, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + artifacts := orientationSelectorV2Records("training", testCase.choice, testCase.shadow, testCase.forward, testCase.reverse, testCase.guarded, false) + report, err := buildOrientationSelectorV2Report( + artifacts.shadow, artifacts.incumbent, artifacts.reverse, artifacts.guarded, + testAAReportForRecords(t, artifacts.incumbent), + OrientationSelectorV2ReportOptions{Seed: 11, Confidence: defaultConfidenceLevel, BootstrapCount: 100, Protocol: referencePairProtocolDiscovery}, + ) + require.NoError(t, err) + require.False(t, report.Cases[0].Passed) + require.Contains(t, report.Cases[0].Reasons[0]+fmt.Sprint(report.Cases[0].Reasons[1:]), testCase.reason) + require.Equal(t, testCase.shadowFails, report.Cases[0].ShadowForwardOverhead.Applicable && !report.Cases[0].ShadowForwardOverhead.Passed) + require.Equal(t, testCase.selectedFails, !report.Cases[0].GuardedSelectedOverhead.Passed) + require.Equal(t, testCase.fastestFails, !report.Cases[0].GuardedFastestRegret.Passed) + }) + } +} + +func TestOrientationSelectorV2ReportAcceptsExactOverflowFallback(t *testing.T) { + artifacts := orientationSelectorV2Records( + "training", string(optimize.ExpansionSearchStepwiseForward), + 10*time.Millisecond, 10*time.Millisecond, 12*time.Millisecond, 10*time.Millisecond, + true, + ) + report, err := buildOrientationSelectorV2Report( + artifacts.shadow, artifacts.incumbent, artifacts.reverse, artifacts.guarded, + testAAReportForRecords(t, artifacts.incumbent), + OrientationSelectorV2ReportOptions{Seed: 13, Confidence: defaultConfidenceLevel, BootstrapCount: 50, Protocol: referencePairProtocolDiscovery}, + ) + require.NoError(t, err) + require.True(t, report.Cases[0].Overflow) + require.True(t, report.Cases[0].FallbackExecuted) + require.Equal(t, "exact_forward_incumbent", report.Cases[0].GuardedRuntimeBranch) +} + +func TestOrientationSelectorV2ReportAcceptsStateOverflowAfterReverseChoice(t *testing.T) { + artifacts := orientationSelectorV2Records( + "training", string(optimize.ExpansionSearchSuffixSeededReverse), + 10*time.Millisecond, 10*time.Millisecond, 5*time.Millisecond, 10*time.Millisecond, + true, + ) + for index := range artifacts.shadow { + artifacts.shadow[index].TraversalTelemetry.Summary.Overflow = boolPointer(false) + } + report, err := buildOrientationSelectorV2Report( + artifacts.shadow, artifacts.incumbent, artifacts.reverse, artifacts.guarded, + testAAReportForRecords(t, artifacts.incumbent), + OrientationSelectorV2ReportOptions{Seed: 17, Confidence: defaultConfidenceLevel, BootstrapCount: 50, Protocol: referencePairProtocolDiscovery}, + ) + require.NoError(t, err) + require.True(t, report.Cases[0].Overflow) + require.True(t, report.Cases[0].FallbackExecuted) + require.Equal(t, string(optimize.ExpansionSearchStepwiseForward), report.Cases[0].GuardedRuntimeIdentity) +} + +func TestOrientationSelectorV2ReportRejectsIncompleteConfirmationCohort(t *testing.T) { + artifacts := orientationSelectorV2Records( + "training", string(optimize.ExpansionSearchStepwiseForward), + 10*time.Millisecond, 10*time.Millisecond, 12*time.Millisecond, 10*time.Millisecond, false, + ) + renameOrientationV2Records("training-only", artifacts) + _, err := buildOrientationSelectorV2Report( + artifacts.shadow, artifacts.incumbent, artifacts.reverse, artifacts.guarded, + testAAReportForRecords(t, artifacts.incumbent), + OrientationSelectorV2ReportOptions{Confidence: defaultConfidenceLevel, BootstrapCount: 10, Protocol: referencePairProtocolConfirmation, Freeze: testOrientationV2Freeze()}, + ) + require.ErrorContains(t, err, "exact frozen 8-training/4-holdout cohort") +} + +func TestOrientationSelectorV2ReportRequiresFrozenDiscoveryForConfirmation(t *testing.T) { + artifacts := orientationSelectorV2Records( + "training", string(optimize.ExpansionSearchStepwiseForward), + 10*time.Millisecond, 10*time.Millisecond, 12*time.Millisecond, 10*time.Millisecond, false, + ) + _, err := buildOrientationSelectorV2Report( + artifacts.shadow, artifacts.incumbent, artifacts.reverse, artifacts.guarded, + testAAReportForRecords(t, artifacts.incumbent), + OrientationSelectorV2ReportOptions{Confidence: defaultConfidenceLevel, BootstrapCount: 10, Protocol: referencePairProtocolConfirmation}, + ) + require.Error(t, err) +} + +func TestOrientationSelectorV2ReportRejectsRuntimeIdentityAndReceiptDrift(t *testing.T) { + for _, mutate := range []func(*orientationSelectorV2Artifacts){ + func(artifacts *orientationSelectorV2Artifacts) { + artifacts.guarded[0].TraversalTelemetry.Summary.RuntimeIdentity = string(optimize.ExpansionSearchStepwiseForward) + artifacts.guarded[0].TraversalTelemetry.Summary.AppliedIdentity = string(optimize.ExpansionSearchStepwiseForward) + artifacts.guarded[0].TraversalTelemetry.Summary.RuntimeBranch = "exact_forward_incumbent" + }, + func(artifacts *orientationSelectorV2Artifacts) { + artifacts.guarded[0].TraversalTelemetry.Summary.FallbackExecuted = boolPointer(true) + artifacts.guarded[0].TraversalTelemetry.Summary.FallbackIdentity = string(optimize.ExpansionSearchStepwiseForward) + }, + func(artifacts *orientationSelectorV2Artifacts) { + artifacts.guarded[0].Stats.Samples[1].RuntimeReceiptEvents = nil + }, + func(artifacts *orientationSelectorV2Artifacts) { + artifacts.shadow[0].TraversalTelemetry.Summary.EmittedIdentity = string(optimize.ExpansionSearchPolicyOrientationProbeV1) + }, + func(artifacts *orientationSelectorV2Artifacts) { + artifacts.incumbent[0].TraversalTelemetry.Summary.SelectorVersion = "static-lowering-v1" + }, + func(artifacts *orientationSelectorV2Artifacts) { + artifacts.guarded[0].TraversalTelemetry.Summary.ExecutionBoundary = optimize.ExpansionSearchExecutionBoundaryInlineStatement + }, + } { + artifacts := orientationSelectorV2Records( + "training", string(optimize.ExpansionSearchSuffixSeededReverse), + 10*time.Millisecond, 10*time.Millisecond, 5*time.Millisecond, 5*time.Millisecond, false, + ) + mutate(&artifacts) + _, err := buildOrientationSelectorV2Report( + artifacts.shadow, artifacts.incumbent, artifacts.reverse, artifacts.guarded, + testAAReportForRecords(t, artifacts.incumbent), + OrientationSelectorV2ReportOptions{Confidence: defaultConfidenceLevel, BootstrapCount: 10, Protocol: referencePairProtocolDiscovery}, + ) + require.Error(t, err) + } +} + +func TestOrientationSelectorV2ReportRejectsIdentityCaseObservationAndOrderDrift(t *testing.T) { + mutations := []func(*orientationSelectorV2Artifacts){ + func(artifacts *orientationSelectorV2Artifacts) { + artifacts.guarded[0].Environment.CorpusSHA256 = testSHA("9") + }, + func(artifacts *orientationSelectorV2Artifacts) { artifacts.reverse[0].WorkloadSHA256 = "changed" }, + func(artifacts *orientationSelectorV2Artifacts) { + artifacts.guarded[0].ObservedRows = []string{"changed"} + }, + func(artifacts *orientationSelectorV2Artifacts) { artifacts.guarded[0].SQLFingerprint = "changed" }, + func(artifacts *orientationSelectorV2Artifacts) { artifacts.guarded = artifacts.guarded[1:] }, + func(artifacts *orientationSelectorV2Artifacts) { + for idx := range artifacts.guarded { + artifacts.guarded[idx].Name = "unexpected" + } + }, + func(artifacts *orientationSelectorV2Artifacts) { + artifacts.reverse[0].Shape.QualificationSplit = "holdout" + }, + func(artifacts *orientationSelectorV2Artifacts) { artifacts.guarded[0].Fixture.Checksum = "changed" }, + func(artifacts *orientationSelectorV2Artifacts) { + artifacts.reverse[0].PostgresEnvironment.EdgeRelationBytes++ + }, + func(artifacts *orientationSelectorV2Artifacts) { + artifacts.shadow[0].PostgresEnvironment.AnalyzeState = "edge:never" + }, + func(artifacts *orientationSelectorV2Artifacts) { + for sampleIdx := range artifacts.guarded[0].Stats.Samples { + artifacts.guarded[0].Stats.Samples[sampleIdx].ArmOrder = 3 + } + }, + } + for _, mutate := range mutations { + artifacts := orientationSelectorV2Records( + "training", string(optimize.ExpansionSearchSuffixSeededReverse), + 10*time.Millisecond, 10*time.Millisecond, 5*time.Millisecond, 5*time.Millisecond, false, + ) + mutate(&artifacts) + _, err := buildOrientationSelectorV2Report( + artifacts.shadow, artifacts.incumbent, artifacts.reverse, artifacts.guarded, + testAAReportForRecords(t, artifacts.incumbent), + OrientationSelectorV2ReportOptions{Confidence: defaultConfidenceLevel, BootstrapCount: 10, Protocol: referencePairProtocolDiscovery}, + ) + require.Error(t, err) + } +} + +func TestOrientationSelectorV2ReportRejectsUnboundAAEnvironment(t *testing.T) { + artifacts := orientationSelectorV2Records( + "training", string(optimize.ExpansionSearchStepwiseForward), + 10*time.Millisecond, 10*time.Millisecond, 12*time.Millisecond, 10*time.Millisecond, false, + ) + for _, mutate := range []func(*AAResolutionReport){ + func(report *AAResolutionReport) { report.Cases[0].PostgresEnvironmentSHA256 = "" }, + func(report *AAResolutionReport) { report.Cases[0].FixtureSHA256 = testSHA("9") }, + } { + aa := testAAReportForRecords(t, artifacts.incumbent) + mutate(aa) + _, err := buildOrientationSelectorV2Report( + artifacts.shadow, artifacts.incumbent, artifacts.reverse, artifacts.guarded, aa, + OrientationSelectorV2ReportOptions{Confidence: defaultConfidenceLevel, BootstrapCount: 10, Protocol: referencePairProtocolDiscovery}, + ) + require.ErrorContains(t, err, "incumbent A/A environment") + } +} + +func TestOrientationSelectorV2ReportRejectsSupplementalMeasurements(t *testing.T) { + mutations := []func(*CaseResult){ + func(record *CaseResult) { record.Concurrency = []ConcurrencyBlock{{Concurrency: 2}} }, + func(record *CaseResult) { record.PostgresReferences = []PostgresReferenceResult{{Name: "unexpected"}} }, + func(record *CaseResult) { record.ClientWaterfall = &ClientWaterfall{} }, + func(record *CaseResult) { record.RawPGXWaterfall = &PostgresBoundaryWaterfall{} }, + func(record *CaseResult) { record.RawPGXRoundTrip = &PostgresBoundaryWaterfall{} }, + } + for _, mutate := range mutations { + artifacts := orientationSelectorV2Records( + "training", string(optimize.ExpansionSearchStepwiseForward), + 10*time.Millisecond, 10*time.Millisecond, 12*time.Millisecond, 10*time.Millisecond, false, + ) + mutate(&artifacts.shadow[0]) + _, err := buildOrientationSelectorV2Report( + artifacts.shadow, artifacts.incumbent, artifacts.reverse, artifacts.guarded, + testAAReportForRecords(t, artifacts.incumbent), + OrientationSelectorV2ReportOptions{Confidence: defaultConfidenceLevel, BootstrapCount: 10, Protocol: referencePairProtocolDiscovery}, + ) + require.ErrorContains(t, err, "mixes selector timing with supplemental PostgreSQL measurements") + } +} + +type orientationSelectorV2Artifacts struct { + shadow []CaseResult + incumbent []CaseResult + reverse []CaseResult + guarded []CaseResult +} + +func canonicalOrientationV2TestArtifacts(t *testing.T) (orientationSelectorV2Artifacts, orientationSelectorV2Artifacts) { + t.Helper() + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + full := orientationSelectorV2Artifacts{} + for _, testCase := range corpus.Cases { + isTraining := false + if !slices.Contains(testCase.Tags, "orientation-v2-training") && !slices.Contains(testCase.Tags, "orientation-v2-holdout") { + continue + } + isTraining = testCase.Shape.QualificationSplit == "training" + choice := string(optimize.ExpansionSearchSuffixSeededReverse) + shadow, forward, reverse, guarded := 20*time.Millisecond, 10*time.Millisecond, 5*time.Millisecond, 5*time.Millisecond+50*time.Microsecond + if isTraining { + choice = string(optimize.ExpansionSearchStepwiseForward) + shadow, forward, reverse, guarded = 10*time.Millisecond+50*time.Microsecond, 10*time.Millisecond, 14*time.Millisecond, 10*time.Millisecond+60*time.Microsecond + } + current := orientationSelectorV2Records(testCase.Shape.QualificationSplit, choice, shadow, forward, reverse, guarded, false) + fixture, err := fixtureMetadata("unused", testCase.Dataset) + require.NoError(t, err) + fixture.PhysicalValidated = true + fixture.PhysicalNodeCount, fixture.PhysicalEdgeCount = int64(fixture.NodeCount), int64(fixture.EdgeCount) + fixture.NodeRelationBytes, fixture.EdgeRelationBytes = int64(fixture.NodeCount*1024), int64(fixture.EdgeCount*1024) + for _, records := range [][]CaseResult{current.shadow, current.incumbent, current.reverse, current.guarded} { + for index := range records { + record := &records[index] + record.Source, record.Dataset, record.Name, record.Category, record.Shape = testCase.Source, testCase.Dataset, testCase.Name, testCase.Category, testCase.Shape + record.WorkloadSHA256 = scaleCaseWorkloadIdentity(testCase, ModePostgresSQL) + attachFixtureMetadata(record, fixture) + record.Environment.DirtyDiffSHA256 = cleanWorkingTreeSHA256() + record.PostgresEnvironment.NodeRelationBytes = fixture.NodeRelationBytes + record.PostgresEnvironment.EdgeRelationBytes = fixture.EdgeRelationBytes + record.PostgresEnvironment.AnalyzeState = "edge:analyzed,node:analyzed" + } + } + full = appendOrientationV2Artifacts(full, current) + } + training := orientationSelectorV2Artifacts{ + shadow: cloneOrientationV2Split(full.shadow, "training"), incumbent: cloneOrientationV2Split(full.incumbent, "training"), + reverse: cloneOrientationV2Split(full.reverse, "training"), guarded: cloneOrientationV2Split(full.guarded, "training"), + } + stampOrientationV2Selections(&training) + stampOrientationV2Selections(&full) + return training, full +} + +func cloneOrientationV2Split(records []CaseResult, split string) []CaseResult { + result := make([]CaseResult, 0, len(records)) + for _, record := range records { + if record.Shape.QualificationSplit != split { + continue + } + copy := record + if record.Environment != nil { + environment := *record.Environment + copy.Environment = &environment + } + result = append(result, copy) + } + return result +} + +func compactOrientationV2Artifacts(artifacts orientationSelectorV2Artifacts, rounds, samples int) orientationSelectorV2Artifacts { + compact := func(records []CaseResult) []CaseResult { + result := make([]CaseResult, 0, len(records)) + for _, record := range records { + if record.Environment.Round > rounds { + continue + } + copy := record + copy.Stats.Samples = append([]LatencySample(nil), record.Stats.Samples[:samples]...) + result = append(result, copy) + } + return result + } + return orientationSelectorV2Artifacts{ + shadow: compact(artifacts.shadow), incumbent: compact(artifacts.incumbent), + reverse: compact(artifacts.reverse), guarded: compact(artifacts.guarded), + } +} + +func writeOrientationV2TestArtifact(t *testing.T, path string, records []CaseResult) { + t.Helper() + output, err := os.Create(path) + require.NoError(t, err) + require.NoError(t, writeJSONL(output, records)) + require.NoError(t, output.Close()) +} + +func orientationSelectorV2Records( + split, wouldSelect string, + shadowDuration, incumbentDuration, reverseDuration, guardedDuration time.Duration, + overflow bool, +) orientationSelectorV2Artifacts { + const rounds = 12 + orders := [][4]int{ + {1, 2, 3, 4}, {2, 3, 4, 1}, {3, 4, 1, 2}, {4, 1, 2, 3}, + {1, 3, 4, 2}, {2, 4, 1, 3}, {3, 1, 2, 4}, {4, 2, 3, 1}, + {1, 4, 2, 3}, {2, 1, 3, 4}, {3, 2, 4, 1}, {4, 3, 1, 2}, + } + artifacts := orientationSelectorV2Artifacts{} + for round := 1; round <= rounds; round++ { + order := orders[round-1] + artifacts.shadow = append(artifacts.shadow, orientationSelectorV2Record(round, order[0], "shadow", split, wouldSelect, shadowDuration, overflow)) + artifacts.incumbent = append(artifacts.incumbent, orientationSelectorV2Record(round, order[1], "incumbent", split, "", incumbentDuration, false)) + artifacts.reverse = append(artifacts.reverse, orientationSelectorV2Record(round, order[2], "reverse", split, "", reverseDuration, false)) + artifacts.guarded = append(artifacts.guarded, orientationSelectorV2Record(round, order[3], "guarded", split, wouldSelect, guardedDuration, overflow)) + } + stampOrientationV2Selections(&artifacts) + return artifacts +} + +func orientationSelectorV2Record(round, armOrder int, arm, split, choice string, duration time.Duration, overflow bool) CaseResult { + forward := string(optimize.ExpansionSearchStepwiseForward) + reverse := string(optimize.ExpansionSearchSuffixSeededReverse) + v2 := string(optimize.ExpansionSearchPolicyOrientationProbeV2) + runtimeIdentity, emittedIdentity, selectorVersion := forward, forward, "fixed-suffix-static-v1" + runtimeBranch, boundary, wouldSelect := "selected", optimize.ExpansionSearchExecutionBoundaryInlineStatement, "" + fallback := false + requested := forward + if arm == "shadow" { + emittedIdentity, selectorVersion, wouldSelect = v2, v2, choice + runtimeBranch, requested = "shadow_incumbent", reverse + } + if arm == "reverse" { + runtimeIdentity, emittedIdentity, selectorVersion, requested = reverse, reverse, "suffix-seeded-reverse-tool-v1", reverse + } + if arm == "guarded" { + emittedIdentity, selectorVersion, boundary, requested = v2, v2, optimize.ExpansionSearchExecutionBoundaryGuardedDualArm, reverse + if choice == reverse && !overflow { + runtimeIdentity, runtimeBranch = reverse, "suffix_seeded_reverse" + } else { + runtimeIdentity, runtimeBranch = forward, "exact_forward_incumbent" + fallback = overflow + } + } + provenance := map[string]string{ + "requested_identity": "test", "planned_identities": "test", "emitted_identity": "test", + "runtime_identity": "test", "applied_identity": "test", "selector_version": "test", + "scheduler_version": "test", "runtime_branch": "test", "runtime_outcome_available": "test", + "overflow": "test", "fallback_executed": "test", "execution_boundary": "test", + } + if wouldSelect != "" { + provenance["would_select_identity"] = "test" + } + if fallback { + provenance["fallback_identity"] = "test" + } + available := true + record := CaseResult{ + Source: "cases/orientation-v2.json", Dataset: "orientation-v2-fixture", Name: "fixed-suffix", + Category: "generated_fixed_suffix_expansion", WorkloadSHA256: sqlFingerprint("orientation-v2-workload"), + ExecutionMode: ModePostgresSQL, Status: StatusOK, + Shape: WorkloadShape{FixtureTier: "normal", QualificationSplit: split}, + RowCount: 1, ObservedRows: []string{"[42]"}, StableObservation: true, + SQLFingerprint: sqlFingerprint("orientation-v2-" + arm + "-sql"), + Fixture: &FixtureMetadata{ + Dataset: "orientation-v2-fixture", Checksum: sqlFingerprint("orientation-v2-fixture-checksum"), + NodeCount: 10, EdgeCount: 12, PhysicalValidated: true, PhysicalNodeCount: 10, PhysicalEdgeCount: 12, + Configuration: "orientation-v2-test", + }, + PostgresEnvironment: &PostgresEnvironment{ + Version: "PostgreSQL test", Database: "dawgs", PlanCacheMode: "auto", TransactionIsolation: "repeatable read", + WorkMem: "4MB", TempFileLimit: "-1", GraphPartitionCount: 1, DatabaseOID: 1, + Autovacuum: "on", AnalyzeState: "stable", SchemaFingerprint: "schema", IndexFingerprint: "index", + }, + Environment: &RunEnvironment{ + ArtifactSchemaVersion: 2, CorpusSHA256: testSHA("c"), SourceCommit: "deadbeef", + DirtyDiffSHA256: testSHA("d"), BinarySHA256: testSHA("b"), + GOOS: "linux", GOARCH: "amd64", CPUCount: 8, CPUModel: "test-cpu", Kernel: "test-kernel", CgroupCPU: "max 100000", + RunUUID: fmt.Sprintf("orientation-v2-run-%d", round), Arm: arm, ArmOrder: armOrder, Block: round, Round: round, + WarmupIterations: 20, PoolSize: 1, + }, + TraversalTelemetry: &TraversalExecutionTelemetry{ + SchemaVersion: TraversalExecutionTelemetrySchemaVersion, Level: TraversalTelemetryLevelSummary, + Summary: TraversalExecutionSummary{ + RequestedIdentity: requested, PlannedIdentities: []string{forward, reverse}, EmittedIdentity: emittedIdentity, + RuntimeIdentity: runtimeIdentity, AppliedIdentity: runtimeIdentity, SelectorVersion: selectorVersion, + SchedulerVersion: "not_applicable", ExecutionBoundary: boundary, Caps: map[string]int64{}, + RuntimeOutcomeAvailable: &available, RuntimeBranch: runtimeBranch, Overflow: boolPointer(overflow), + FallbackExecuted: boolPointer(fallback), WouldSelectIdentity: wouldSelect, Provenance: provenance, + }, + }, + } + if fallback { + record.TraversalTelemetry.Summary.FallbackIdentity = forward + } + record.Stats.WarmupIterations = 20 + for iteration := 1; iteration <= 50; iteration++ { + sample := LatencySample{ + Round: round, Block: round, Arm: arm, ArmOrder: armOrder, RunUUID: record.Environment.RunUUID, + Iteration: iteration, Classification: "warm", Duration: duration, + RequestedIdentity: requested, RuntimeIdentity: runtimeIdentity, RuntimeBranch: runtimeBranch, + FallbackExecuted: boolPointer(fallback), + } + if arm == "shadow" || arm == "guarded" { + sample.RuntimeAttestation = "timed_invocation" + sample.RuntimeReceiptEvents = []RuntimeReceiptEvent{{ + Ordinal: 1, RuntimeIdentity: runtimeIdentity, RuntimeBranch: runtimeBranch, FallbackExecuted: fallback, + }} + } else { + sample.RuntimeAttestation = "same_case_invocation_local_replay" + } + record.Stats.Samples = append(record.Stats.Samples, sample) + } + return record +} + +func renameOrientationV2Records(name string, artifacts orientationSelectorV2Artifacts) { + for _, records := range [][]CaseResult{artifacts.shadow, artifacts.incumbent, artifacts.reverse, artifacts.guarded} { + for index := range records { + records[index].Name = name + records[index].Dataset = "generated_fixed_suffix_expansion_v3_" + name + records[index].WorkloadSHA256 = sqlFingerprint("orientation-v2-workload-" + name) + records[index].Fixture.Dataset = records[index].Dataset + records[index].Fixture.Checksum = sqlFingerprint("orientation-v2-fixture-" + name) + records[index].PostgresEnvironment.NodeRelationBytes = int64(len(name) * 1024) + records[index].PostgresEnvironment.EdgeRelationBytes = int64(len(name) * 2048) + } + } + stampOrientationV2Selections(&artifacts) +} + +func appendOrientationV2Artifacts(values ...orientationSelectorV2Artifacts) orientationSelectorV2Artifacts { + result := orientationSelectorV2Artifacts{} + for _, value := range values { + result.shadow = append(result.shadow, value.shadow...) + result.incumbent = append(result.incumbent, value.incumbent...) + result.reverse = append(result.reverse, value.reverse...) + result.guarded = append(result.guarded, value.guarded...) + } + stampOrientationV2Selections(&result) + return result +} + +func stampOrientationV2Selections(artifacts *orientationSelectorV2Artifacts) { + if artifacts == nil { + return + } + keys := map[performanceKey]struct{}{} + for _, record := range artifacts.shadow { + keys[performanceKey{dataset: record.Dataset, name: record.Name, backend: record.ExecutionMode}] = struct{}{} + } + declared := make([]DeclaredCaseBackend, 0, 2*len(keys)) + resolved := make([]ResolvedCaseSelector, 0, len(keys)) + for _, key := range sortedPerformanceKeys(keys) { + for _, backend := range []ExecutionMode{ModePostgresSQL, ModeNeo4j} { + declared = append(declared, DeclaredCaseBackend{Dataset: key.dataset, Name: key.name, Backend: backend}) + } + resolved = append(resolved, ResolvedCaseSelector{Dataset: key.dataset, Name: key.name, Category: "generated_fixed_suffix_expansion"}) + } + selection := &SelectionManifest{ + Version: selectionManifestVersion, Resolved: resolved, DiagnosticOnly: true, + FullDeclarationCount: 2 * len(keys), SelectedDeclarationCount: 2 * len(keys), DeclarationSHA256: declarationSHA256(declared), + } + for _, records := range [][]CaseResult{artifacts.shadow, artifacts.incumbent, artifacts.reverse, artifacts.guarded} { + for index := range records { + copy := *selection + copy.Resolved = append([]ResolvedCaseSelector(nil), selection.Resolved...) + records[index].Environment.Selection = © + } + } +} + +func boolPointer(value bool) *bool { return &value } + +func testSHA(digit string) string { + value := "" + for len(value) < 64 { + value += digit + } + return value[:64] +} + +func testOrientationV2Freeze() *OrientationSelectorV2FreezeManifest { + return &OrientationSelectorV2FreezeManifest{ + Version: 1, Policy: string(optimize.ExpansionSearchPolicyOrientationProbeV2), + Formula: "F2=root_rows+maximum_depth*forward_degree_rows;R2=suffix_rows+boundary_rows+reverse_degree_rows;reverse=complete&&4*R2<3*F2", + Caps: map[string]int64{ + "root_row_limit": optimize.ExpansionSearchOrientationRootRowLimit, "reverse_seed_row_limit": optimize.ExpansionSearchOrientationReverseSeedRowLimit, + "directional_degree_row_limit": optimize.ExpansionSearchOrientationDirectionalDegreeRowLimit, "state_limit": optimize.ExpansionSearchOrientationStateLimit, + }, + SourceCommit: "deadbeef", DirtyDiffSHA256: testSHA("d"), BinarySHA256: testSHA("b"), DiscoveryReportSHA256: testSHA("e"), + } +} diff --git a/cmd/graphbench/perf_gate.go b/cmd/graphbench/perf_gate.go new file mode 100644 index 00000000..5d497cce --- /dev/null +++ b/cmd/graphbench/perf_gate.go @@ -0,0 +1,956 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "math/rand" + "os" + "sort" + "time" +) + +const ( + // perfGateVersion identifies the serialized schema revision for perf gate. + perfGateVersion = 5 + + // defaultBootstrapCount sets the fallback number of resamples used to estimate confidence bounds. + defaultBootstrapCount = 10_000 + + // minimumGateRounds requires this many independent matched rounds before a workload may pass. + minimumGateRounds = 5 + + // minimumP95Samples requires this many warm samples per arm before the P95 ratio is gated. + minimumP95Samples = 150 + + // minimumDiscoveryWarmups requires the discovery protocol's untimed warmup floor. + minimumDiscoveryWarmups = 5 +) + +// PerfGateOptions defines statistical confidence, materiality, targets, and declared backend coverage for gating. +type PerfGateOptions struct { + // Seed controls deterministic random sampling. + Seed int64 + // Confidence sets the confidence level used for statistical intervals. + Confidence float64 + // RegressionThreshold sets the largest median ratio that is not considered a regression. + RegressionThreshold float64 + // BootstrapCount sets the number of bootstrap resamples. + BootstrapCount int + // DeclaredBackends lists case/backend declarations that the performance gate must cover. + DeclaredBackends []DeclaredCaseBackend + // TargetNames restricts materiality requirements to the named workloads. + TargetNames []string + // MaterialityRatio sets the relative change required before a difference is material. + MaterialityRatio float64 + // MaterialityAbsolute sets the absolute duration change required before a difference is material. + MaterialityAbsolute time.Duration + // DiagnosticMode allows incomplete diagnostic selections that cannot produce a release-gate pass. + DiagnosticMode bool + // AAReportPath selects the host A/A evidence loaded by artifact comparison mode. + AAReportPath string + // AAReport contains host-specific per-case timing resolution required for promotion. + AAReport *AAResolutionReport + // AAReportSHA256 identifies the exact A/A report supplied to the gate. + AAReportSHA256 string +} + +// RatioInterval describes a point estimate and confidence bounds for a latency ratio. +type RatioInterval struct { + // Estimate records the point estimate enclosed by the confidence bounds. + Estimate float64 `json:"estimate"` + // Lower records the lower confidence bound. + Lower float64 `json:"lower"` + // Upper records the upper confidence bound. + Upper float64 `json:"upper"` +} + +// DurationInterval describes a duration estimate and its confidence bounds. +type DurationInterval struct { + // Estimate records the point estimate enclosed by the confidence bounds. + Estimate time.Duration `json:"estimate"` + // Lower records the lower confidence bound. + Lower time.Duration `json:"lower"` + // Upper records the upper confidence bound. + Upper time.Duration `json:"upper"` +} + +// PerfGateCase reports matched sample evidence, bootstrap intervals, and classification for one gated workload. +type PerfGateCase struct { + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // Backend identifies the execution backend. + Backend ExecutionMode `json:"backend"` + // Tier identifies whether timing is gated or stress-diagnostic. + Tier string `json:"tier"` + // QualificationSplit identifies training, frozen holdout, or diagnostic evidence. + QualificationSplit string `json:"qualification_split"` + // TimingGated reports whether latency evidence contributes to promotion. + TimingGated bool `json:"timing_gated"` + // Rounds records the number of independent measurement rounds. + Rounds int `json:"rounds"` + // BaselineSamples records warm timing samples available from the baseline arm. + BaselineSamples int `json:"baseline_samples"` + // CandidateSamples records warm timing samples available from the candidate arm. + CandidateSamples int `json:"candidate_samples"` + // BaselineStatus records the first non-OK baseline status for the workload. + BaselineStatus string `json:"baseline_status,omitempty"` + // CandidateStatus records the first non-OK candidate status for the workload. + CandidateStatus string `json:"candidate_status,omitempty"` + // OracleOnly marks a backend as a correctness oracle excluded from latency regression decisions. + OracleOnly bool `json:"oracle_only,omitempty"` + // MedianRatio reports the candidate-to-baseline median latency ratio and confidence bounds. + MedianRatio RatioInterval `json:"median_ratio"` + // P95Ratio reports the candidate-to-baseline P95 latency ratio and confidence bounds. + P95Ratio *RatioInterval `json:"p95_ratio,omitempty"` + // MedianSaving reports absolute median latency saved by the candidate. + MedianSaving *DurationInterval `json:"median_saving,omitempty"` + // MedianChange reports candidate-minus-baseline median latency. + MedianChange *DurationInterval `json:"median_change,omitempty"` + // P95Change reports candidate-minus-baseline P95 latency. + P95Change *DurationInterval `json:"p95_change,omitempty"` + // P50NoiseRatio records the host A/A-derived relative median floor. + P50NoiseRatio float64 `json:"p50_noise_ratio,omitempty"` + // P50NoiseAbsolute records the host A/A-derived absolute median floor. + P50NoiseAbsolute time.Duration `json:"p50_noise_absolute,omitempty"` + // P95NoiseRatio records the host A/A-derived relative P95 floor. + P95NoiseRatio float64 `json:"p95_noise_ratio,omitempty"` + // P95NoiseAbsolute records the host A/A-derived absolute P95 floor. + P95NoiseAbsolute time.Duration `json:"p95_noise_absolute,omitempty"` + // MaterialityRatio sets the relative change required before a difference is material. + MaterialityRatio *float64 `json:"materiality_ratio_upper_limit,omitempty"` + // MaterialityAbsolute sets the absolute duration change required before a difference is material. + MaterialityAbsolute *time.Duration `json:"materiality_absolute_lower_limit,omitempty"` + // Passed reports whether every required gate condition succeeded. + Passed bool `json:"passed"` + // Reasons lists explanations for the reported disposition. + Reasons []string `json:"reasons,omitempty"` + // CandidateRuntimeReceiptChains preserves complete measured candidate + // branch chains used by the performance decision. + CandidateRuntimeReceiptChains [][]RuntimeReceiptEvent `json:"candidate_runtime_receipt_chains,omitempty"` +} + +// PerfGateReport contains baseline and candidate identities, gate policy, and every workload disposition. +type PerfGateReport struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // Seed controls deterministic random sampling. + Seed int64 `json:"seed"` + // Confidence sets the confidence level used for statistical intervals. + Confidence float64 `json:"confidence_level"` + // RegressionThreshold sets the largest median ratio that is not considered a regression. + RegressionThreshold float64 `json:"regression_threshold"` + // BaselineSHA256 identifies the exact baseline artifact evaluated by the gate. + BaselineSHA256 string `json:"baseline_sha256"` + // CandidateSHA256 identifies the exact candidate artifact evaluated by the gate. + CandidateSHA256 string `json:"candidate_sha256"` + // AAReportSHA256 identifies the exact host A/A resolution report evaluated by the gate. + AAReportSHA256 string `json:"aa_report_sha256,omitempty"` + // DeclarationSHA256 identifies the canonical set of declared workloads. + DeclarationSHA256 string `json:"declaration_sha256,omitempty"` + // Passed reports whether every required gate condition succeeded. + Passed bool `json:"passed"` + // PromotionEligible reports whether this complete, non-diagnostic evidence may support production promotion. + PromotionEligible bool `json:"promotion_eligible"` + // MaterialityRequired reports that promotion requires at least one explicitly named improvement target. + MaterialityRequired bool `json:"materiality_required"` + // MaterialityTargets records the number of declared timing targets resolved by the artifact. + MaterialityTargets int `json:"materiality_targets"` + // MaterialityPassed reports whether every resolved target cleared the configured A/A-aware improvement floor. + MaterialityPassed bool `json:"materiality_passed"` + // QualificationRequired reports whether the artifact contains a prioritized traversal candidate that requires independent training and frozen-holdout gates. + QualificationRequired bool `json:"qualification_required"` + // TrainingCases records prioritized traversal cases gated on the selector-training partition. + TrainingCases int `json:"training_cases"` + // HoldoutCases records prioritized traversal cases gated on the frozen topology holdout. + HoldoutCases int `json:"holdout_cases"` + // TrainingPassed reports whether every observed prioritized training case passed. + TrainingPassed bool `json:"training_passed"` + // HoldoutPassed reports whether every observed prioritized holdout case passed. + HoldoutPassed bool `json:"holdout_passed"` + // QualificationPassed reports whether nonempty training and holdout partitions independently passed. + QualificationPassed bool `json:"qualification_passed"` + // QualificationFamilies contains the independent split disposition for each concrete traversal candidate family. + QualificationFamilies []TraversalQualificationStatus `json:"qualification_families,omitempty"` + // Cases contains the gate disposition and statistical evidence for each declared workload. + Cases []PerfGateCase `json:"cases"` +} + +// performanceKey identifies one dataset, case, and backend across performance artifacts. +type performanceKey struct { + // dataset names the fixture shared by matched baseline and candidate records. + dataset string + // name identifies the workload case within its dataset. + name string + // backend separates independently gated execution modes for the same workload. + backend ExecutionMode +} + +// roundSamples groups positive warm durations by independent measurement round. +type roundSamples map[int][]time.Duration + +// comparePerformanceArtifacts validates two artifacts, writes their performance-gate report, and returns its pass status. +func comparePerformanceArtifacts(baselinePath, candidatePath, outputPath string, options PerfGateOptions) (bool, error) { + baseline, err := readJSONLFile(baselinePath) + if err != nil { + return false, fmt.Errorf("read baseline: %w", err) + } + + candidate, err := readJSONLFile(candidatePath) + if err != nil { + return false, fmt.Errorf("read candidate: %w", err) + } + if err := validatePerformanceArtifactSelections(baseline, candidate, options.DiagnosticMode); err != nil { + return false, err + } + if options.AAReportPath != "" { + options.AAReport, options.AAReportSHA256, err = loadAAResolutionReport(options.AAReportPath) + if err != nil { + return false, fmt.Errorf("load performance-gate A/A evidence: %w", err) + } + } + + baselineChecksum, err := fileSHA256(baselinePath) + if err != nil { + return false, err + } + candidateChecksum, err := fileSHA256(candidatePath) + if err != nil { + return false, err + } + + report, err := buildPerfGateReport(baseline, candidate, options) + if err != nil { + return false, err + } + report.BaselineSHA256 = baselineChecksum + report.CandidateSHA256 = candidateChecksum + + if err := writePerfGateReport(outputPath, report); err != nil { + return false, err + } + return report.Passed && report.PromotionEligible, nil +} + +// validatePerformanceArtifactSelections rejects adaptive or diagnostic artifacts when complete-gate input is required. +func validatePerformanceArtifactSelections(baseline, candidate []CaseResult, diagnosticMode bool) error { + if !diagnosticMode && (hasAdaptiveDiscoveryRecord(baseline) || hasAdaptiveDiscoveryRecord(candidate)) { + return fmt.Errorf("adaptive-discovery artifacts are refused by the complete performance gate") + } + baselineSelection, baselineErr := selectionIdentity(baseline) + candidateSelection, candidateErr := selectionIdentity(candidate) + if baselineErr != nil || candidateErr != nil { + if diagnosticMode { + return fmt.Errorf("diagnostic comparison requires selection manifests in both artifacts") + } + return fmt.Errorf("complete performance gate requires selection manifests in both artifacts") + } + if err := validateSelectionManifestAccounting(baselineSelection); err != nil { + return fmt.Errorf("baseline artifact %w", err) + } + if err := validateSelectionManifestAccounting(candidateSelection); err != nil { + return fmt.Errorf("candidate artifact %w", err) + } + if baselineSelection.ProtectedDeclarationCount != candidateSelection.ProtectedDeclarationCount || + baselineSelection.ProtectedDeclarationSHA256 != candidateSelection.ProtectedDeclarationSHA256 { + return fmt.Errorf("artifact protected declaration omissions differ") + } + if baselineSelection.DiagnosticOnly || candidateSelection.DiagnosticOnly { + if !diagnosticMode { + return fmt.Errorf("diagnostic-only artifacts are refused by the complete performance gate") + } + if !baselineSelection.DiagnosticOnly || !candidateSelection.DiagnosticOnly { + return fmt.Errorf("diagnostic comparison requires two diagnostic-only artifacts") + } + if baselineSelection.DeclarationSHA256 != candidateSelection.DeclarationSHA256 { + return fmt.Errorf("diagnostic artifact declarations differ: %s != %s", baselineSelection.DeclarationSHA256, candidateSelection.DeclarationSHA256) + } + return nil + } + if diagnosticMode { + return fmt.Errorf("diagnostic comparison mode requires filtered diagnostic-only artifacts") + } + return nil +} + +// hasAdaptiveDiscoveryRecord reports whether any record was produced by adaptive existing-graph discovery. +func hasAdaptiveDiscoveryRecord(records []CaseResult) bool { + for _, record := range records { + if record.ExistingGraph != nil && record.ExistingGraph.Adaptive { + return true + } + if record.Environment != nil && record.Environment.Protocol == "adaptive_discovery" { + return true + } + } + return false +} + +// buildPerfGateReport compares matched baseline and candidate samples and classifies each declared workload. +func buildPerfGateReport(baseline, candidate []CaseResult, options PerfGateOptions) (PerfGateReport, error) { + if err := validatePerformanceWorkloadIdentity(baseline, candidate); err != nil { + return PerfGateReport{}, err + } + if options.Confidence <= 0 || options.Confidence >= 1 { + return PerfGateReport{}, fmt.Errorf("confidence level must be between 0 and 1") + } + if options.RegressionThreshold < 0 { + return PerfGateReport{}, fmt.Errorf("regression threshold must not be negative") + } + if options.BootstrapCount == 0 { + options.BootstrapCount = defaultBootstrapCount + } + if options.BootstrapCount < 1 { + return PerfGateReport{}, fmt.Errorf("bootstrap count must be positive") + } + if options.MaterialityRatio == 0 { + options.MaterialityRatio = 0.95 + } + if options.MaterialityRatio <= 0 || options.MaterialityRatio >= 1 { + return PerfGateReport{}, fmt.Errorf("materiality ratio must be between 0 and 1") + } + if options.MaterialityAbsolute == 0 { + options.MaterialityAbsolute = 100 * time.Microsecond + } + if options.MaterialityAbsolute < 0 { + return PerfGateReport{}, fmt.Errorf("materiality absolute duration must not be negative") + } + + baselineSeries := collectWarmSeries(baseline) + candidateSeries := collectWarmSeries(candidate) + keys := declaredPerformanceKeys(options.DeclaredBackends, baseline, candidate) + sort.Slice(keys, func(i, j int) bool { + if keys[i].dataset != keys[j].dataset { + return keys[i].dataset < keys[j].dataset + } + if keys[i].name != keys[j].name { + return keys[i].name < keys[j].name + } + return keys[i].backend < keys[j].backend + }) + if len(keys) == 0 { + return PerfGateReport{}, fmt.Errorf("artifacts and declaration contain no PostgreSQL or Neo4j cases") + } + tiers := make(map[performanceKey]string, len(keys)) + splits := make(map[performanceKey]string, len(keys)) + hasPromotionTiming := false + for _, key := range keys { + tier, err := timingTier(key, baseline, candidate) + if err != nil { + return PerfGateReport{}, err + } + tiers[key] = tier + split, err := qualificationSplit(key, baseline, candidate) + if err != nil { + return PerfGateReport{}, err + } + splits[key] = split + if key.backend == ModePostgresSQL && (tier == "normal" || tier == "envelope") && promotionTimingSplit(split) { + hasPromotionTiming = true + } + } + if hasPromotionTiming && !options.DiagnosticMode { + if !validSHA256(options.AAReportSHA256) { + return PerfGateReport{}, fmt.Errorf("complete performance gate requires a checksummed host A/A report") + } + if err := validateAAResolutionEvidence(options.AAReport, baseline, options.Confidence); err != nil { + return PerfGateReport{}, fmt.Errorf("baseline A/A evidence: %w", err) + } + if err := validateAAResolutionEvidence(options.AAReport, candidate, options.Confidence); err != nil { + return PerfGateReport{}, fmt.Errorf("candidate A/A evidence: %w", err) + } + } else if options.AAReport != nil { + if !validSHA256(options.AAReportSHA256) { + return PerfGateReport{}, fmt.Errorf("supplied A/A report checksum is malformed") + } + if err := validateAAResolutionEvidence(options.AAReport, baseline, options.Confidence); err != nil { + return PerfGateReport{}, err + } + } + targetNames := make(map[string]struct{}, len(options.TargetNames)) + for _, name := range options.TargetNames { + targetNames[name] = struct{}{} + } + + report := PerfGateReport{ + Version: perfGateVersion, + Seed: options.Seed, + Confidence: options.Confidence, + RegressionThreshold: options.RegressionThreshold, + AAReportSHA256: options.AAReportSHA256, + Passed: true, + PromotionEligible: !options.DiagnosticMode && hasPromotionTiming && len(targetNames) > 0, + MaterialityRequired: hasPromotionTiming && !options.DiagnosticMode, + MaterialityPassed: len(targetNames) > 0, + TrainingPassed: true, + HoldoutPassed: true, + } + resolvedMaterialityTargets := map[string]struct{}{} + qualification := map[string]*TraversalQualificationStatus{} + if len(options.DeclaredBackends) > 0 { + report.DeclarationSHA256 = declarationSHA256(options.DeclaredBackends) + } + for idx, key := range keys { + baselineStatus := artifactCaseStatus(baseline, key) + candidateStatus := artifactCaseStatus(candidate, key) + baselineRounds, candidateRounds := matchedRounds(baselineSeries[key], candidateSeries[key]) + gateCase := PerfGateCase{ + Dataset: key.dataset, + Name: key.name, + Backend: key.backend, + Tier: tiers[key], + QualificationSplit: splits[key], + TimingGated: key.backend == ModePostgresSQL && (tiers[key] == "normal" || tiers[key] == "envelope") && promotionTimingSplit(splits[key]) && !options.DiagnosticMode, + Rounds: len(baselineRounds), + BaselineSamples: sampleCount(baselineRounds), + CandidateSamples: sampleCount(candidateRounds), + BaselineStatus: baselineStatus, + CandidateStatus: candidateStatus, + OracleOnly: key.backend == ModeNeo4j, + Passed: true, + CandidateRuntimeReceiptChains: caseRuntimeReceiptChains(candidate, key), + } + if candidateStatus != StatusOK { + gateCase.Passed = false + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("required candidate record status is %s", candidateStatus)) + } + if gateCase.TimingGated { + if err := validateCandidateRuntimeEvidence(candidate, key); err != nil { + gateCase.Passed = false + gateCase.Reasons = append(gateCase.Reasons, err.Error()) + } + } + // Neo4j is a correctness oracle. A successful record means its untimed + // exact observation checks passed; its latency never affects this gate. + if key.backend == ModeNeo4j { + if !gateCase.Passed { + report.Passed = false + } + report.Cases = append(report.Cases, gateCase) + continue + } + if baselineStatus != StatusOK { + gateCase.Passed = false + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("required baseline record status is %s", baselineStatus)) + } + if gateCase.TimingGated && len(baselineRounds) < minimumGateRounds { + gateCase.Passed = false + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("need at least %d matched rounds, got %d", minimumGateRounds, len(baselineRounds))) + } + if gateCase.TimingGated && len(baselineRounds) > 0 { + if err := validatePairedOrderEvidence(baseline, candidate, key, sortedRounds(baselineRounds), minimumDiscoveryWarmups); err != nil { + return PerfGateReport{}, fmt.Errorf("invalid promotion evidence: %w", err) + } + } + if tiers[key] == "stress" { + gateCase.Reasons = append(gateCase.Reasons, "stress tier timing is diagnostic") + } + if splits[key] == "diagnostic" { + gateCase.Reasons = append(gateCase.Reasons, "diagnostic qualification split is excluded from promotion timing") + } + + gateCase.P50NoiseRatio, gateCase.P50NoiseAbsolute = minimumTimingNoiseRatio, minimumTimingNoiseAbsolute + gateCase.P95NoiseRatio, gateCase.P95NoiseAbsolute = minimumTimingNoiseRatio, minimumTimingNoiseAbsolute + if options.AAReport != nil { + if ratio, absolute, err := aaTimingFloor(options.AAReport, key, false, options.RegressionThreshold); err == nil { + gateCase.P50NoiseRatio, gateCase.P50NoiseAbsolute = ratio, absolute + } else if gateCase.TimingGated { + return PerfGateReport{}, err + } + if ratio, absolute, err := aaTimingFloor(options.AAReport, key, true, options.RegressionThreshold); err == nil { + gateCase.P95NoiseRatio, gateCase.P95NoiseAbsolute = ratio, absolute + } else if gateCase.TimingGated { + return PerfGateReport{}, err + } + } else { + gateCase.P50NoiseRatio = max(gateCase.P50NoiseRatio, options.RegressionThreshold) + gateCase.P95NoiseRatio = max(gateCase.P95NoiseRatio, options.RegressionThreshold) + } + + seed := options.Seed + int64(idx)*7919 + if len(baselineRounds) > 0 { + gateCase.MedianRatio = bootstrapRoundMedianRatio(baselineRounds, candidateRounds, seed, options) + saving := bootstrapRoundMedianSaving(baselineRounds, candidateRounds, seed+3, options) + gateCase.MedianSaving = &saving + change := negateDurationInterval(saving) + gateCase.MedianChange = &change + if gateCase.TimingGated && gateCase.MedianRatio.Lower > 1+gateCase.P50NoiseRatio && change.Lower > gateCase.P50NoiseAbsolute { + gateCase.Passed = false + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("median regression exceeds host A/A floors: ratio lower %.4f > %.4f and change lower %s > %s", gateCase.MedianRatio.Lower, 1+gateCase.P50NoiseRatio, change.Lower, gateCase.P50NoiseAbsolute)) + } + } + + if gateCase.BaselineSamples >= minimumP95Samples && gateCase.CandidateSamples >= minimumP95Samples { + interval := bootstrapStratifiedP95Ratio(baselineRounds, candidateRounds, seed+1, options) + gateCase.P95Ratio = &interval + change := bootstrapStratifiedQuantileChange(baselineRounds, candidateRounds, 0.95, seed+2, options) + gateCase.P95Change = &change + if gateCase.TimingGated && interval.Lower > 1+gateCase.P95NoiseRatio && change.Lower > gateCase.P95NoiseAbsolute { + gateCase.Passed = false + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("p95 regression exceeds host A/A floors: ratio lower %.4f > %.4f and change lower %s > %s", interval.Lower, 1+gateCase.P95NoiseRatio, change.Lower, gateCase.P95NoiseAbsolute)) + } + } else if gateCase.TimingGated { + gateCase.Passed = false + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("need at least %d warm samples per side for p95, got %d/%d", minimumP95Samples, gateCase.BaselineSamples, gateCase.CandidateSamples)) + } + + if _, isTarget := targetNames[key.name]; isTarget && gateCase.TimingGated && len(baselineRounds) > 0 { + resolvedMaterialityTargets[key.name] = struct{}{} + effectiveRatio := min(options.MaterialityRatio, 1-gateCase.P50NoiseRatio) + effectiveAbsolute := max(options.MaterialityAbsolute, gateCase.P50NoiseAbsolute) + gateCase.MaterialityRatio = &effectiveRatio + gateCase.MaterialityAbsolute = &effectiveAbsolute + materialRatio := gateCase.MedianRatio.Upper <= effectiveRatio + materialAbsolute := gateCase.MedianSaving != nil && gateCase.MedianSaving.Lower >= effectiveAbsolute + if !materialRatio && !materialAbsolute { + gateCase.Passed = false + report.MaterialityPassed = false + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("target improvement is not material: median ratio upper %.4f > %.4f and saving lower %s < %s", gateCase.MedianRatio.Upper, effectiveRatio, gateCase.MedianSaving.Lower, effectiveAbsolute)) + } + } + + if !gateCase.Passed { + report.Passed = false + } + if prioritizedTraversalKey(key, baseline, candidate) && gateCase.TimingGated { + report.QualificationRequired = true + family := traversalQualificationFamily(key, baseline, candidate) + status := qualification[family] + if status == nil { + status = &TraversalQualificationStatus{Family: family, TrainingPassed: true, HoldoutPassed: true} + qualification[family] = status + } + switch gateCase.QualificationSplit { + case "training": + report.TrainingCases++ + report.TrainingPassed = report.TrainingPassed && gateCase.Passed + status.TrainingCases++ + status.TrainingPassed = status.TrainingPassed && gateCase.Passed + case "holdout": + report.HoldoutCases++ + report.HoldoutPassed = report.HoldoutPassed && gateCase.Passed + status.HoldoutCases++ + status.HoldoutPassed = status.HoldoutPassed && gateCase.Passed + } + } + report.Cases = append(report.Cases, gateCase) + } + report.MaterialityTargets = len(resolvedMaterialityTargets) + if report.MaterialityRequired { + if len(targetNames) == 0 { + report.MaterialityPassed = false + } + if report.MaterialityTargets != len(targetNames) { + return PerfGateReport{}, fmt.Errorf("materiality targets resolved to %d timing-gated cases, expected %d", report.MaterialityTargets, len(targetNames)) + } + } + if report.QualificationRequired { + families := make([]string, 0, len(qualification)) + for family := range qualification { + families = append(families, family) + } + sort.Strings(families) + for _, family := range families { + status := qualification[family] + status.TrainingPassed = status.TrainingPassed && status.TrainingCases > 0 + status.HoldoutPassed = status.HoldoutPassed && status.HoldoutCases > 0 + status.Passed = status.TrainingPassed && status.HoldoutPassed + report.TrainingPassed = report.TrainingPassed && status.TrainingPassed + report.HoldoutPassed = report.HoldoutPassed && status.HoldoutPassed + report.QualificationFamilies = append(report.QualificationFamilies, *status) + } + report.QualificationPassed = report.TrainingPassed && report.HoldoutPassed + report.Passed = report.Passed && report.QualificationPassed + } else { + report.TrainingPassed = false + report.HoldoutPassed = false + } + if options.DiagnosticMode { + report.Passed = false + } + report.PromotionEligible = report.PromotionEligible && report.Passed && report.MaterialityPassed + + return report, nil +} + +// validatePerformanceWorkloadIdentity ensures matched artifacts describe identical logical workloads per case and backend. +func validatePerformanceWorkloadIdentity(baseline, candidate []CaseResult) error { + collect := func(label string, records []CaseResult) (map[performanceKey]string, error) { + identities := map[performanceKey]string{} + for _, record := range records { + if record.ExecutionMode != ModePostgresSQL && record.ExecutionMode != ModeNeo4j { + continue + } + key := performanceKey{dataset: record.Dataset, name: record.Name, backend: record.ExecutionMode} + if record.WorkloadSHA256 == "" { + return nil, fmt.Errorf("%s artifact case %s/%s/%s has no workload identity", label, key.dataset, key.name, key.backend) + } + identityPayload := struct { + // WorkloadSHA256 binds the compared samples to one logical workload declaration. + WorkloadSHA256 string `json:"workload_sha256"` + // ManifestSHA256 identifies the anchor manifest that authorized the run. + ManifestSHA256 string `json:"manifest_sha256,omitempty"` + // ContentIdentity binds resumable work to the logical contents of the live graph. + ContentIdentity string `json:"content_identity,omitempty"` + // FixtureChecksum identifies the loaded fixture contents. + FixtureChecksum string `json:"fixture_checksum,omitempty"` + // FixtureConfiguration captures generator settings used to construct the loaded fixture. + FixtureConfiguration string `json:"fixture_configuration,omitempty"` + }{WorkloadSHA256: record.WorkloadSHA256} + if record.ExistingGraph != nil { + identityPayload.ManifestSHA256 = record.ExistingGraph.ManifestSHA256 + identityPayload.ContentIdentity = record.ExistingGraph.ContentIdentity + } + if record.Fixture != nil { + identityPayload.FixtureChecksum = record.Fixture.Checksum + identityPayload.FixtureConfiguration = record.Fixture.Configuration + } + raw, _ := json.Marshal(identityPayload) + digest := sha256.Sum256(raw) + identity := hex.EncodeToString(digest[:]) + if present, found := identities[key]; found && present != identity { + return nil, fmt.Errorf("%s artifact case %s/%s/%s mixes workload identities", label, key.dataset, key.name, key.backend) + } + identities[key] = identity + } + return identities, nil + } + + baselineIdentities, err := collect("baseline", baseline) + if err != nil { + return err + } + candidateIdentities, err := collect("candidate", candidate) + if err != nil { + return err + } + for key, baselineIdentity := range baselineIdentities { + if candidateIdentity, found := candidateIdentities[key]; found && candidateIdentity != baselineIdentity { + return fmt.Errorf("logical workload differs for %s/%s/%s", key.dataset, key.name, key.backend) + } + } + return nil +} + +// declaredPerformanceKeys returns the unique case/backend keys that the performance gate must evaluate. +func declaredPerformanceKeys(declared []DeclaredCaseBackend, baseline, candidate []CaseResult) []performanceKey { + unique := map[performanceKey]struct{}{} + for _, item := range declared { + if item.UnsupportedReason != "" { + continue + } + if item.Backend == ModePostgresSQL || item.Backend == ModeNeo4j { + unique[performanceKey{ + dataset: item.Dataset, + name: item.Name, + backend: item.Backend, + }] = struct{}{} + } + } + + if len(declared) == 0 { + for _, records := range [][]CaseResult{baseline, candidate} { + for _, record := range records { + if record.ExecutionMode == ModePostgresSQL || record.ExecutionMode == ModeNeo4j { + unique[performanceKey{ + dataset: record.Dataset, + name: record.Name, + backend: record.ExecutionMode, + }] = struct{}{} + } + } + } + } + + keys := make([]performanceKey, 0, len(unique)) + for key := range unique { + keys = append(keys, key) + } + return keys +} + +// artifactCaseStatus returns the first non-OK status for a declared case/backend pair, "missing" when no record exists, or OK when every matching record succeeded. +func artifactCaseStatus(records []CaseResult, key performanceKey) string { + found := false + for _, record := range records { + if record.Dataset != key.dataset || record.Name != key.name || record.ExecutionMode != key.backend { + continue + } + found = true + if record.Status != StatusOK { + return record.Status + } + } + if !found { + return "missing" + } + return StatusOK +} + +// declarationSHA256 sorts declared case/backend contracts and hashes their canonical JSON so compared artifacts must describe the same workload set. +func declarationSHA256(declared []DeclaredCaseBackend) string { + items := append([]DeclaredCaseBackend(nil), declared...) + sort.Slice(items, func(i, j int) bool { + if items[i].Dataset != items[j].Dataset { + return items[i].Dataset < items[j].Dataset + } + if items[i].Name != items[j].Name { + return items[i].Name < items[j].Name + } + if items[i].Backend != items[j].Backend { + return items[i].Backend < items[j].Backend + } + return items[i].UnsupportedReason < items[j].UnsupportedReason + }) + digest := sha256.New() + for _, item := range items { + fmt.Fprintf(digest, "%s\x00%s\x00%s\x00%s\n", item.Dataset, item.Name, item.Backend, item.UnsupportedReason) + } + + return hex.EncodeToString(digest.Sum(nil)) +} + +// collectWarmSeries groups positive warm durations by case, backend, and round. +func collectWarmSeries(records []CaseResult) map[performanceKey]roundSamples { + series := map[performanceKey]roundSamples{} + for _, record := range records { + if record.Status != StatusOK { + continue + } + key := performanceKey{ + dataset: record.Dataset, + name: record.Name, + backend: record.ExecutionMode, + } + + for _, sample := range record.Stats.Samples { + if sample.Classification != "warm" || sample.Duration <= 0 { + continue + } + + if series[key] == nil { + series[key] = roundSamples{} + } + series[key][sample.Round] = append(series[key][sample.Round], sample.Duration) + } + } + + return series +} + +// matchedRounds returns round numbers present in both measurement series. +func matchedRounds(baseline, candidate roundSamples) (roundSamples, roundSamples) { + matchedBaseline := roundSamples{} + matchedCandidate := roundSamples{} + for round, baselineSamples := range baseline { + candidateSamples, found := candidate[round] + if !found || len(baselineSamples) == 0 || len(candidateSamples) == 0 { + continue + } + + matchedBaseline[round] = baselineSamples + matchedCandidate[round] = candidateSamples + } + + return matchedBaseline, matchedCandidate +} + +// bootstrapRoundMedianRatio bootstraps the ratio between paired round medians. +func bootstrapRoundMedianRatio(baseline, candidate roundSamples, seed int64, options PerfGateOptions) RatioInterval { + rounds := sortedRounds(baseline) + baselineMedians := make([]float64, len(rounds)) + candidateMedians := make([]float64, len(rounds)) + for idx, round := range rounds { + baselineMedians[idx] = durationQuantile(baseline[round], 0.5) + candidateMedians[idx] = durationQuantile(candidate[round], 0.5) + } + estimate := quantile(candidateMedians, 0.5) / quantile(baselineMedians, 0.5) + rng := rand.New(rand.NewSource(seed)) // #nosec G404 -- deterministic statistical resampling + ratios := make([]float64, options.BootstrapCount) + resampledBaseline := make([]float64, len(rounds)) + resampledCandidate := make([]float64, len(rounds)) + for iteration := range ratios { + for idx := range rounds { + selected := rng.Intn(len(rounds)) + resampledBaseline[idx] = baselineMedians[selected] + resampledCandidate[idx] = candidateMedians[selected] + } + ratios[iteration] = quantile(resampledCandidate, 0.5) / quantile(resampledBaseline, 0.5) + } + return confidenceInterval(estimate, ratios, options.Confidence) +} + +// bootstrapRoundMedianSaving bootstraps the absolute duration saved between paired round medians. +func bootstrapRoundMedianSaving(baseline, candidate roundSamples, seed int64, options PerfGateOptions) DurationInterval { + rounds := sortedRounds(baseline) + baselineMedians := make([]float64, len(rounds)) + candidateMedians := make([]float64, len(rounds)) + for idx, round := range rounds { + baselineMedians[idx] = durationQuantile(baseline[round], 0.5) + candidateMedians[idx] = durationQuantile(candidate[round], 0.5) + } + estimate := quantile(baselineMedians, 0.5) - quantile(candidateMedians, 0.5) + rng := rand.New(rand.NewSource(seed)) // #nosec G404 -- deterministic statistical resampling + savings := make([]float64, options.BootstrapCount) + resampledBaseline := make([]float64, len(rounds)) + resampledCandidate := make([]float64, len(rounds)) + for iteration := range savings { + for idx := range rounds { + selected := rng.Intn(len(rounds)) + resampledBaseline[idx] = baselineMedians[selected] + resampledCandidate[idx] = candidateMedians[selected] + } + savings[iteration] = quantile(resampledBaseline, 0.5) - quantile(resampledCandidate, 0.5) + } + interval := confidenceInterval(estimate, savings, options.Confidence) + return DurationInterval{ + Estimate: time.Duration(interval.Estimate), + Lower: time.Duration(interval.Lower), + Upper: time.Duration(interval.Upper), + } +} + +// bootstrapStratifiedP95Ratio bootstraps a P95 ratio while preserving round strata. +func bootstrapStratifiedP95Ratio(baseline, candidate roundSamples, seed int64, options PerfGateOptions) RatioInterval { + rounds := sortedRounds(baseline) + estimate := durationQuantile(flattenSamples(candidate, rounds), 0.95) / durationQuantile(flattenSamples(baseline, rounds), 0.95) + rng := rand.New(rand.NewSource(seed)) // #nosec G404 -- deterministic statistical resampling + ratios := make([]float64, options.BootstrapCount) + for iteration := range ratios { + var resampledBaseline, resampledCandidate []time.Duration + for _, round := range rounds { + resampledBaseline = append(resampledBaseline, resampleDurations(rng, baseline[round])...) + resampledCandidate = append(resampledCandidate, resampleDurations(rng, candidate[round])...) + } + ratios[iteration] = durationQuantile(resampledCandidate, 0.95) / durationQuantile(resampledBaseline, 0.95) + } + return confidenceInterval(estimate, ratios, options.Confidence) +} + +// confidenceInterval returns the requested central interval from sorted bootstrap estimates. +func confidenceInterval(estimate float64, samples []float64, confidence float64) RatioInterval { + alpha := (1 - confidence) / 2 + return RatioInterval{ + Estimate: estimate, + Lower: quantile(samples, alpha), + Upper: quantile(samples, 1-alpha), + } +} + +// durationQuantile returns a nearest-rank duration quantile from a copy of the samples. +func durationQuantile(values []time.Duration, probability float64) float64 { + numeric := make([]float64, len(values)) + for idx, value := range values { + numeric[idx] = float64(value) + } + return quantile(numeric, probability) +} + +// quantile returns a nearest-rank quantile from sorted floating-point samples. +func quantile(values []float64, probability float64) float64 { + ordered := append([]float64(nil), values...) + sort.Float64s(ordered) + if len(ordered) == 0 { + return math.NaN() + } + index := int(math.Ceil(probability*float64(len(ordered)))) - 1 + if index < 0 { + index = 0 + } + if index >= len(ordered) { + index = len(ordered) - 1 + } + return ordered[index] +} + +// sortedRounds returns measurement round keys in ascending order. +func sortedRounds(samples roundSamples) []int { + rounds := make([]int, 0, len(samples)) + for round := range samples { + rounds = append(rounds, round) + } + sort.Ints(rounds) + return rounds +} + +// flattenSamples concatenates samples from the requested rounds in the supplied round order. +func flattenSamples(samples roundSamples, rounds []int) []time.Duration { + var flattened []time.Duration + for _, round := range rounds { + flattened = append(flattened, samples[round]...) + } + return flattened +} + +// resampleDurations draws a same-size bootstrap sample of durations with replacement. +func resampleDurations(rng *rand.Rand, values []time.Duration) []time.Duration { + resampled := make([]time.Duration, len(values)) + for idx := range resampled { + resampled[idx] = values[rng.Intn(len(values))] + } + return resampled +} + +// sampleCount returns the total number of durations across all measurement rounds. +func sampleCount(samples roundSamples) int { + count := 0 + for _, values := range samples { + count += len(values) + } + return count +} + +// fileSHA256 returns the SHA-256 digest of a file's contents. +func fileSHA256(path string) (string, error) { + content, err := os.ReadFile(path) + if err != nil { + return "", err + } + digest := sha256.Sum256(content) + return hex.EncodeToString(digest[:]), nil +} + +// writePerfGateReport writes a performance-gate report to stdout or the requested file. +func writePerfGateReport(path string, report PerfGateReport) (err error) { + var output *os.File + if path == "" { + output = os.Stdout + } else { + if err := ensureOutputDir(path); err != nil { + return err + } + output, err = os.Create(path) + if err != nil { + return err + } + defer func() { + if closeErr := output.Close(); err == nil && closeErr != nil { + err = closeErr + } + }() + } + + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(report) +} diff --git a/cmd/graphbench/perf_gate_test.go b/cmd/graphbench/perf_gate_test.go new file mode 100644 index 00000000..d3ec7eb8 --- /dev/null +++ b/cmd/graphbench/perf_gate_test.go @@ -0,0 +1,641 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/stretchr/testify/require" +) + +// TestBuildPerfGateReportTreatsNeo4jAsCorrectnessOracle verifies that PostgreSQL receives latency ratios while Neo4j contributes correctness observations without performance gating. +func TestBuildPerfGateReportTreatsNeo4jAsCorrectnessOracle(t *testing.T) { + baseline := []CaseResult{ + perfGateRecord("one_shortest_path_bound_pair", ModePostgresSQL, 10*time.Millisecond, 5, 30), + perfGateRecord("one_shortest_path_bound_pair", ModeNeo4j, 3*time.Millisecond, 5, 30), + } + candidate := []CaseResult{ + perfGateRecord("one_shortest_path_bound_pair", ModePostgresSQL, 3*time.Millisecond, 5, 30), + perfGateRecord("one_shortest_path_bound_pair", ModeNeo4j, 2*time.Millisecond, 5, 30), + } + + report, err := buildPerfGateReport(baseline, candidate, qualifiedPerfGateOptions(t, baseline, candidate, PerfGateOptions{ + Seed: 42, + Confidence: 0.95, + RegressionThreshold: 0.20, + BootstrapCount: 250, + })) + + require.NoError(t, err) + require.True(t, report.Passed) + require.Len(t, report.Cases, 2) + postgres := findPerfGateCase(t, report.Cases, ModePostgresSQL) + require.InDelta(t, 0.3, postgres.MedianRatio.Estimate, 0.0001) + require.NotNil(t, postgres.P95Ratio) + neo4j := findPerfGateCase(t, report.Cases, ModeNeo4j) + require.True(t, neo4j.OracleOnly) + require.Nil(t, neo4j.P95Ratio) +} + +// TestBuildPerfGateReportFailsMissingDeclaredPostgresCase verifies that every declared PostgreSQL workload must have a candidate record and that the declaration set is fingerprinted. +func TestBuildPerfGateReportFailsMissingDeclaredPostgresCase(t *testing.T) { + baseline := []CaseResult{perfGateRecord("present", ModePostgresSQL, time.Millisecond, 5, 30)} + candidate := []CaseResult{perfGateRecord("present", ModePostgresSQL, time.Millisecond, 5, 30)} + + report, err := buildPerfGateReport(baseline, candidate, qualifiedPerfGateOptions(t, baseline, candidate, PerfGateOptions{ + Seed: 1, + Confidence: 0.95, + RegressionThreshold: 0.20, + BootstrapCount: 100, + DeclaredBackends: []DeclaredCaseBackend{ + { + Dataset: "fixture", + Name: "present", + Backend: ModePostgresSQL, + }, + { + Dataset: "fixture", + Name: "missing", + Backend: ModePostgresSQL, + }, + }, + })) + + require.NoError(t, err) + require.False(t, report.Passed) + require.NotEmpty(t, report.DeclarationSHA256) + var missing PerfGateCase + for _, gateCase := range report.Cases { + if gateCase.Name == "missing" { + missing = gateCase + } + } + require.Equal(t, "missing", missing.CandidateStatus) + require.ErrorContains(t, reasonsError(missing.Reasons), "required candidate record status is missing") +} + +// TestBuildPerfGateReportAppliesMaterialityOnlyToDeclaredTargets verifies that a named target passes only when the confidence-bound saving clears both ratio and absolute thresholds. +func TestBuildPerfGateReportAppliesMaterialityOnlyToDeclaredTargets(t *testing.T) { + baseline := []CaseResult{perfGateRecord("target", ModePostgresSQL, 10*time.Millisecond, 5, 30)} + candidate := []CaseResult{perfGateRecord("target", ModePostgresSQL, 9_700*time.Microsecond, 5, 30)} + + report, err := buildPerfGateReport(baseline, candidate, qualifiedPerfGateOptions(t, baseline, candidate, PerfGateOptions{ + Seed: 1, + Confidence: 0.95, + RegressionThreshold: 0.20, + BootstrapCount: 100, + TargetNames: []string{"target"}, + MaterialityRatio: 0.95, + MaterialityAbsolute: 100 * time.Microsecond, + })) + + require.NoError(t, err) + require.True(t, report.Passed, "%v", report.Cases[0].Reasons) + require.NotNil(t, report.Cases[0].MedianSaving) + require.Equal(t, 300*time.Microsecond, report.Cases[0].MedianSaving.Lower) +} + +// TestBuildPerfGateReportFailsRegressionAndInsufficientP95 verifies that an excessive median slowdown and fewer than 150 warm samples independently fail a PostgreSQL gate case. +func TestBuildPerfGateReportFailsRegressionAndInsufficientP95(t *testing.T) { + baseline := []CaseResult{perfGateRecord("ordinary_case", ModePostgresSQL, 10*time.Millisecond, 5, 10)} + candidate := []CaseResult{perfGateRecord("ordinary_case", ModePostgresSQL, 13*time.Millisecond, 5, 10)} + + report, err := buildPerfGateReport(baseline, candidate, qualifiedPerfGateOptions(t, baseline, candidate, PerfGateOptions{ + Seed: 7, + Confidence: 0.95, + RegressionThreshold: 0.20, + BootstrapCount: 100, + })) + + require.NoError(t, err) + require.False(t, report.Passed) + require.Len(t, report.Cases, 1) + require.ErrorContains(t, reasonsError(report.Cases[0].Reasons), "median regression") + require.ErrorContains(t, reasonsError(report.Cases[0].Reasons), "at least 150 warm samples") +} + +// TestBuildPerfGateReportRequiresMatchedRounds verifies that four baseline/candidate rounds are insufficient for an inferential gate even with ample samples. +func TestBuildPerfGateReportRequiresMatchedRounds(t *testing.T) { + baseline := []CaseResult{perfGateRecord("ordinary_case", ModePostgresSQL, 10*time.Millisecond, 4, 40)} + candidate := []CaseResult{perfGateRecord("ordinary_case", ModePostgresSQL, 9*time.Millisecond, 4, 40)} + + report, err := buildPerfGateReport(baseline, candidate, qualifiedPerfGateOptions(t, baseline, candidate, PerfGateOptions{ + Seed: 1, + Confidence: 0.95, + RegressionThreshold: 0.20, + BootstrapCount: 100, + })) + + require.NoError(t, err) + require.False(t, report.Passed) + require.ErrorContains(t, reasonsError(report.Cases[0].Reasons), "at least 5 matched rounds") +} + +// TestBuildPerfGateReportRequiresHostAAEvidence verifies that a non-diagnostic promotion cannot substitute fixed defaults for a checksummed host calibration. +func TestBuildPerfGateReportRequiresHostAAEvidence(t *testing.T) { + baseline := []CaseResult{perfGateRecord("ordinary_case", ModePostgresSQL, time.Millisecond, 5, 30)} + candidate := []CaseResult{perfGateRecord("ordinary_case", ModePostgresSQL, time.Millisecond, 5, 30)} + stampPairedEvidence(baseline, candidate, minimumDiscoveryWarmups) + + _, err := buildPerfGateReport(baseline, candidate, PerfGateOptions{ + Confidence: defaultConfidenceLevel, BootstrapCount: 10, + }) + + require.ErrorContains(t, err, "checksummed host A/A report") +} + +// TestBuildPerfGateReportRequiresMaterialityTargetForPromotion verifies a +// containment-only comparison can pass without authorizing a no-win rollout. +func TestBuildPerfGateReportRequiresMaterialityTargetForPromotion(t *testing.T) { + baseline := []CaseResult{perfGateRecord("ordinary_case", ModePostgresSQL, time.Millisecond, 5, 30)} + candidate := []CaseResult{perfGateRecord("ordinary_case", ModePostgresSQL, time.Millisecond, 5, 30)} + report, err := buildPerfGateReport(baseline, candidate, qualifiedPerfGateOptions(t, baseline, candidate, PerfGateOptions{ + Confidence: defaultConfidenceLevel, BootstrapCount: 10, + })) + + require.NoError(t, err) + require.True(t, report.Passed) + require.True(t, report.MaterialityRequired) + require.False(t, report.MaterialityPassed) + require.False(t, report.PromotionEligible) +} + +// TestBuildPerfGateReportRejectsMismatchedAAHost verifies a syntactically valid calibration from another host cannot qualify production timing. +func TestBuildPerfGateReportRejectsMismatchedAAHost(t *testing.T) { + baseline := []CaseResult{perfGateRecord("ordinary_case", ModePostgresSQL, time.Millisecond, 5, 30)} + candidate := []CaseResult{perfGateRecord("ordinary_case", ModePostgresSQL, time.Millisecond, 5, 30)} + options := qualifiedPerfGateOptions(t, baseline, candidate, PerfGateOptions{ + Confidence: defaultConfidenceLevel, BootstrapCount: 10, + }) + options.AAReport.HostFingerprint = strings.Repeat("c", 64) + + _, err := buildPerfGateReport(baseline, candidate, options) + + require.ErrorContains(t, err, "host fingerprint does not match") +} + +// TestBuildPerfGateReportUsesP95AbsoluteFloor verifies a relative regression below 100us remains inside the mandatory fast-case floor while preserving the absolute interval in the report. +func TestBuildPerfGateReportUsesP95AbsoluteFloor(t *testing.T) { + baseline := []CaseResult{perfGateRecord("fast", ModePostgresSQL, time.Millisecond, 5, 30)} + candidate := []CaseResult{perfGateRecord("fast", ModePostgresSQL, 1060*time.Microsecond, 5, 30)} + + report, err := buildPerfGateReport(baseline, candidate, qualifiedPerfGateOptions(t, baseline, candidate, PerfGateOptions{ + Confidence: defaultConfidenceLevel, BootstrapCount: 100, + })) + + require.NoError(t, err) + require.True(t, report.Passed, "%v", report.Cases[0].Reasons) + require.Equal(t, minimumTimingNoiseAbsolute, report.Cases[0].P95NoiseAbsolute) + require.Equal(t, 60*time.Microsecond, report.Cases[0].P95Change.Lower) +} + +// TestBuildPerfGateReportRejectsUnbalancedPromotionEvidence verifies matched rounds with one fixed arm order cannot support promotion. +func TestBuildPerfGateReportRejectsUnbalancedPromotionEvidence(t *testing.T) { + baseline := []CaseResult{perfGateRecord("ordinary_case", ModePostgresSQL, time.Millisecond, 5, 30)} + candidate := []CaseResult{perfGateRecord("ordinary_case", ModePostgresSQL, 900*time.Microsecond, 5, 30)} + options := qualifiedPerfGateOptions(t, baseline, candidate, PerfGateOptions{Confidence: defaultConfidenceLevel, BootstrapCount: 10}) + for idx := range baseline[0].Stats.Samples { + baseline[0].Stats.Samples[idx].ArmOrder = 1 + candidate[0].Stats.Samples[idx].ArmOrder = 2 + } + + _, err := buildPerfGateReport(baseline, candidate, options) + + require.ErrorContains(t, err, "arm order is not balanced") +} + +// TestBuildPerfGateReportKeepsStressTimingDiagnostic verifies stress latency cannot fail production timing gates even without A/A or paired-order evidence. +func TestBuildPerfGateReportKeepsStressTimingDiagnostic(t *testing.T) { + baseline := []CaseResult{perfGateRecord("stress", ModePostgresSQL, time.Millisecond, 1, 1)} + candidate := []CaseResult{perfGateRecord("stress", ModePostgresSQL, 10*time.Millisecond, 1, 1)} + baseline[0].Shape.FixtureTier = "stress" + candidate[0].Shape.FixtureTier = "stress" + + report, err := buildPerfGateReport(baseline, candidate, PerfGateOptions{ + Confidence: defaultConfidenceLevel, BootstrapCount: 10, + }) + + require.NoError(t, err) + require.True(t, report.Passed) + require.False(t, report.Cases[0].TimingGated) + require.Contains(t, report.Cases[0].Reasons, "stress tier timing is diagnostic") +} + +// TestBuildPerfGateReportKeepsDiagnosticSplitOutOfPromotion verifies a normal +// fixture explicitly reserved for boundary diagnostics needs no A/A evidence +// and cannot make the report promotion eligible. +func TestBuildPerfGateReportKeepsDiagnosticSplitOutOfPromotion(t *testing.T) { + baseline := []CaseResult{perfGateRecord("boundary", ModePostgresSQL, time.Millisecond, 1, 1)} + candidate := []CaseResult{perfGateRecord("boundary", ModePostgresSQL, 10*time.Millisecond, 1, 1)} + baseline[0].Shape.QualificationSplit = "diagnostic" + candidate[0].Shape.QualificationSplit = "diagnostic" + + report, err := buildPerfGateReport(baseline, candidate, PerfGateOptions{ + Confidence: defaultConfidenceLevel, BootstrapCount: 10, + }) + + require.NoError(t, err) + require.True(t, report.Passed) + require.False(t, report.PromotionEligible) + require.False(t, report.Cases[0].TimingGated) + require.Contains(t, report.Cases[0].Reasons, "diagnostic qualification split is excluded from promotion timing") +} + +// TestBuildPerfGateReportRejectsChangedLogicalWorkload verifies that baseline and candidate records with different workload digests cannot be compared. +func TestBuildPerfGateReportRejectsChangedLogicalWorkload(t *testing.T) { + baseline := []CaseResult{perfGateRecord("ordinary_case", ModePostgresSQL, 10*time.Millisecond, 5, 30)} + candidate := []CaseResult{perfGateRecord("ordinary_case", ModePostgresSQL, 9*time.Millisecond, 5, 30)} + candidate[0].WorkloadSHA256 = "changed-workload" + + _, err := buildPerfGateReport(baseline, candidate, PerfGateOptions{ + Seed: 1, + Confidence: 0.95, + RegressionThreshold: 0.20, + BootstrapCount: 100, + }) + require.ErrorContains(t, err, "logical workload differs") +} + +// TestUnsupportedDeclarationAffectsChecksumWithoutRequiringARecord verifies that an explicitly unsupported backend needs no measurement but its reason remains part of declaration identity. +func TestUnsupportedDeclarationAffectsChecksumWithoutRequiringARecord(t *testing.T) { + declared := []DeclaredCaseBackend{ + { + Dataset: "fixture", + Name: "directionless", + Backend: ModeNeo4j, + }, + { + Dataset: "fixture", + Name: "directionless", + Backend: ModePostgresSQL, + UnsupportedReason: "unsupported form", + }, + } + records := []CaseResult{perfGateRecord("directionless", ModeNeo4j, time.Millisecond, 1, 1)} + + report, err := buildPerfGateReport(records, records, PerfGateOptions{ + Seed: 1, + Confidence: 0.95, + RegressionThreshold: 0.20, + BootstrapCount: 10, + DeclaredBackends: declared, + }) + require.NoError(t, err) + require.True(t, report.Passed) + require.Len(t, report.Cases, 1) + + changed := append([]DeclaredCaseBackend(nil), declared...) + changed[1].UnsupportedReason = "different reason" + require.NotEqual(t, declarationSHA256(declared), declarationSHA256(changed)) +} + +// TestValidatePerformanceArtifactSelectionsRefusesDiagnosticsFromCompleteGate verifies that subset artifacts require an explicit diagnostic override and still must share the same declaration digest. +func TestValidatePerformanceArtifactSelectionsRefusesDiagnosticsFromCompleteGate(t *testing.T) { + manifest := &SelectionManifest{ + Version: selectionManifestVersion, DiagnosticOnly: true, + FullDeclarationCount: 1, SelectedDeclarationCount: 1, + DeclarationSHA256: strings.Repeat("a", 64), + } + left := []CaseResult{{ + Dataset: "fixture", + Name: "case", + Environment: &RunEnvironment{ + Selection: manifest, + }, + }} + right := []CaseResult{{ + Dataset: "fixture", + Name: "case", + Environment: &RunEnvironment{ + Selection: manifest, + }, + }} + + require.ErrorContains(t, validatePerformanceArtifactSelections(left, right, false), "refused") + require.NoError(t, validatePerformanceArtifactSelections(left, right, true)) + right[0].Environment.Selection = &SelectionManifest{ + Version: selectionManifestVersion, DiagnosticOnly: true, + FullDeclarationCount: 1, SelectedDeclarationCount: 1, + DeclarationSHA256: strings.Repeat("b", 64), + } + require.ErrorContains(t, validatePerformanceArtifactSelections(left, right, true), "declarations differ") +} + +// perfGateRecord returns one successful workload observation with identical warm samples arranged into the requested rounds. +func perfGateRecord(name string, mode ExecutionMode, duration time.Duration, rounds, samplesPerRound int) CaseResult { + record := CaseResult{ + Dataset: "fixture", + Name: name, + WorkloadSHA256: fmt.Sprintf("workload:%s:%s", name, mode), + ExecutionMode: mode, + Status: StatusOK, + Shape: WorkloadShape{FixtureTier: "normal"}, + Environment: &RunEnvironment{ + GOOS: "linux", GOARCH: "amd64", CPUCount: 8, CPUModel: "test-cpu", Kernel: "test-kernel", CgroupCPU: "max 100000", + WarmupIterations: minimumDiscoveryWarmups, + }, + } + record.Stats.WarmupIterations = minimumDiscoveryWarmups + for round := 1; round <= rounds; round++ { + for iteration := 1; iteration <= samplesPerRound; iteration++ { + record.Stats.Samples = append(record.Stats.Samples, LatencySample{ + Round: round, + Iteration: iteration, + Classification: "warm", + Duration: duration, + }) + } + } + return record +} + +// qualifiedPerfGateOptions stamps balanced pairing metadata and supplies host-matched A/A evidence. +func qualifiedPerfGateOptions(t *testing.T, baseline, candidate []CaseResult, options PerfGateOptions) PerfGateOptions { + t.Helper() + stampPairedEvidence(baseline, candidate, minimumDiscoveryWarmups) + options.AAReport = testAAReportForRecords(t, baseline) + options.AAReportSHA256 = strings.Repeat("b", 64) + return options +} + +func testAAReportForRecords(t *testing.T, records []CaseResult) *AAResolutionReport { + t.Helper() + hostFingerprint, err := artifactHostFingerprint(records) + require.NoError(t, err) + + keys := map[performanceKey]struct{}{} + for _, record := range records { + if record.ExecutionMode == ModePostgresSQL && hasWarmLatencySample(record) { + keys[performanceKey{dataset: record.Dataset, name: record.Name, backend: record.ExecutionMode}] = struct{}{} + } + } + aa := &AAResolutionReport{ + Version: aaReportVersion, + Confidence: defaultConfidenceLevel, + ArtifactSHA256: strings.Repeat("a", 64), + HostFingerprint: hostFingerprint, + MinimumRounds: minimumGateRounds, + MinimumSamplesPerArmPerRound: 10, + OrderBalanced: true, + } + for _, key := range sortedPerformanceKeys(keys) { + workloadSHA256, err := workloadSHA256ForKey(records, key) + require.NoError(t, err) + postgresEnvironmentSHA256, err := postgresTimingEnvironmentSHA256ForKey(records, key) + require.NoError(t, err) + fixtureSHA256, err := fixtureSHA256ForKey(records, key) + require.NoError(t, err) + aa.Cases = append(aa.Cases, AAResolutionCase{ + Dataset: key.dataset, Name: key.name, Backend: key.backend, WorkloadSHA256: workloadSHA256, + PostgresEnvironmentSHA256: postgresEnvironmentSHA256, FixtureSHA256: fixtureSHA256, + Rounds: minimumGateRounds, SamplesPerArm: minimumGateRounds * 10, + P50: testAAMetricResolution(), P95: testAAMetricResolution(), + }) + } + return aa +} + +func testAAMetricResolution() AAMetricResolution { + return AAMetricResolution{ + Ratio: RatioInterval{Estimate: 1, Lower: 0.99, Upper: 1.01}, + RatioResolution: 0.01, + AbsoluteChange: DurationInterval{Estimate: 0, Lower: -10 * time.Microsecond, Upper: 10 * time.Microsecond}, + AbsoluteResolution: 10 * time.Microsecond, + } +} + +func stampPairedEvidence(left, right []CaseResult, warmups int) { + stamp := func(records []CaseResult, arm string, leftArm bool) { + for recordIdx := range records { + record := &records[recordIdx] + record.Stats.WarmupIterations = warmups + if record.Environment == nil { + record.Environment = &RunEnvironment{} + } + record.Environment.WarmupIterations = warmups + record.Environment.Arm = arm + for sampleIdx := range record.Stats.Samples { + sample := &record.Stats.Samples[sampleIdx] + leftFirst := sample.Round%2 == 1 + order := 2 + if leftArm == leftFirst { + order = 1 + } + sample.Block = sample.Round + sample.Arm = arm + sample.ArmOrder = order + sample.RunUUID = fmt.Sprintf("pair-%s-%d", record.Name, sample.Round) + } + } + } + stamp(left, "baseline", true) + stamp(right, "candidate", false) +} + +// findPerfGateCase returns the report entry for a backend or fails the calling test when the gate omitted it. +func findPerfGateCase(t *testing.T, cases []PerfGateCase, mode ExecutionMode) PerfGateCase { + t.Helper() + for _, gateCase := range cases { + if gateCase.Backend == mode { + return gateCase + } + } + t.Fatalf("missing %s gate case", mode) + return PerfGateCase{} +} + +// reasonsError joins gate-failure reasons into one diagnostic error. +func reasonsError(reasons []string) error { + return fmt.Errorf("%s", strings.Join(reasons, "; ")) +} + +// TestQualificationSplitFailsClosedOnMissingOrDriftingTraversalPartitions +// verifies benchmark artifacts cannot silently reclassify selector training as +// frozen holdout evidence. +func TestQualificationSplitFailsClosedOnMissingOrDriftingTraversalPartitions(t *testing.T) { + key := performanceKey{dataset: "fixture", name: "sp", backend: ModePostgresSQL} + left := []CaseResult{{ + Dataset: "fixture", Name: "sp", Category: "generated_shortest_path_v2", ExecutionMode: ModePostgresSQL, + }} + _, err := qualificationSplit(key, left) + require.ErrorContains(t, err, "no frozen qualification split") + + left[0].Shape.QualificationSplit = "training" + right := append([]CaseResult(nil), left...) + right[0].Shape.QualificationSplit = "holdout" + _, err = qualificationSplit(key, left, right) + require.ErrorContains(t, err, "changes qualification split") + + right[0].Shape.QualificationSplit = "training" + split, err := qualificationSplit(key, left, right) + require.NoError(t, err) + require.Equal(t, "training", split) +} + +// TestQualificationSplitRecognizesCompatibleFixedSuffixV2Categories verifies +// the v2 dataset cannot bypass partition enforcement through its intentionally +// backwards-compatible category name. +func TestQualificationSplitRecognizesCompatibleFixedSuffixV2Categories(t *testing.T) { + key := performanceKey{dataset: "generated_fixed_suffix_expansion_v2_d8_f16", name: "GFSE-V2-D08-F016", backend: ModePostgresSQL} + records := []CaseResult{{ + Dataset: key.dataset, Name: key.name, Category: "generated_fixed_suffix_expansion", ExecutionMode: key.backend, + }} + + _, err := qualificationSplit(key, records) + require.ErrorContains(t, err, "no frozen qualification split") +} + +func TestTraversalQualificationFamilyRecognizesFixedSuffixV3WithoutTelemetry(t *testing.T) { + key := performanceKey{dataset: "generated_fixed_suffix_expansion_v3_d8_f16", name: "GFSE-V3-D08-F016", backend: ModePostgresSQL} + records := []CaseResult{{ + Dataset: key.dataset, Name: key.name, Category: "generated_fixed_suffix_expansion", ExecutionMode: key.backend, + }} + + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV2), traversalQualificationFamily(key, records)) +} + +func TestTraversalQualificationUsesOrientationPolicyBeforeRequestedArm(t *testing.T) { + key := performanceKey{dataset: "generated_fixed_suffix_expansion_v3_d8_f16", name: "GFSE-V3-D08-F016", backend: ModePostgresSQL} + record := CaseResult{ + Dataset: key.dataset, Name: key.name, Category: "generated_fixed_suffix_expansion", ExecutionMode: key.backend, + TraversalTelemetry: &TraversalExecutionTelemetry{Summary: TraversalExecutionSummary{ + RequestedIdentity: string(optimize.ExpansionSearchSuffixSeededReverse), + EmittedIdentity: string(optimize.ExpansionSearchPolicyOrientationProbeV2), + SelectorVersion: string(optimize.ExpansionSearchPolicyOrientationProbeV2), + RuntimeBranch: "suffix_seeded_reverse", + }}, + } + + require.True(t, requiresCandidateRuntimeEvidence(record)) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV2), traversalQualificationFamily(key, []CaseResult{record})) +} + +// TestBuildPerfGateReportRequiresIndependentTraversalHoldout verifies a +// complete release gate cannot be assembled from selector-training topology +// alone even when every measured case passes. +func TestBuildPerfGateReportRequiresIndependentTraversalHoldout(t *testing.T) { + baseline := []CaseResult{ + perfGateRecord("sp-training", ModePostgresSQL, 10*time.Millisecond, minimumGateRounds, 30), + perfGateRecord("sp-holdout", ModePostgresSQL, 10*time.Millisecond, minimumGateRounds, 30), + } + candidate := []CaseResult{ + perfGateRecord("sp-training", ModePostgresSQL, 5*time.Millisecond, minimumGateRounds, 30), + perfGateRecord("sp-holdout", ModePostgresSQL, 5*time.Millisecond, minimumGateRounds, 30), + } + for _, records := range [][]CaseResult{baseline, candidate} { + records[0].Category = "generated_shortest_path_v2" + records[0].Shape.QualificationSplit = "training" + records[1].Category = "generated_shortest_path_v2" + records[1].Shape.QualificationSplit = "holdout" + } + + report, err := buildPerfGateReport(baseline, candidate, qualifiedPerfGateOptions(t, baseline, candidate, PerfGateOptions{ + Confidence: defaultConfidenceLevel, BootstrapCount: 50, TargetNames: []string{"sp-training", "sp-holdout"}, + })) + require.NoError(t, err) + require.True(t, report.QualificationRequired) + require.True(t, report.TrainingPassed) + require.True(t, report.HoldoutPassed) + require.True(t, report.QualificationPassed) + require.True(t, report.Passed) + require.True(t, report.PromotionEligible) + require.Equal(t, []TraversalQualificationStatus{{ + Family: "SP", TrainingCases: 1, HoldoutCases: 1, TrainingPassed: true, HoldoutPassed: true, Passed: true, + }}, report.QualificationFamilies) + + // A passing ASP holdout may not qualify an SP candidate's training data. + baseline[1].Cypher = "RETURN allShortestPaths((a)-[:E*1..3]->(b))" + candidate[1].Cypher = baseline[1].Cypher + report, err = buildPerfGateReport(baseline, candidate, qualifiedPerfGateOptions(t, baseline, candidate, PerfGateOptions{ + Confidence: defaultConfidenceLevel, BootstrapCount: 50, TargetNames: []string{"sp-training", "sp-holdout"}, + })) + require.NoError(t, err) + require.False(t, report.QualificationPassed) + require.False(t, report.Passed) + require.False(t, report.PromotionEligible) + baseline[1].Cypher = "" + candidate[1].Cypher = "" + + for idx := range baseline { + baseline[idx].Optimization = &translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{ + {TargetKind: "traversal", Family: "SP", Applied: "SP-S4-C-D", Selected: "SP-S4-C-D"}, + {TargetKind: "endpoint_resolution", Family: "endpoint_resolution", TraversalFamily: "SP", Applied: "ENDPOINT-RESOLUTION-INCUMBENT"}, + }} + candidate[idx].Optimization = &translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{ + {TargetKind: "traversal", Family: "SP", Applied: "SP-B1-C-ALT-NODE-D", Selected: "SP-B1-C-ALT-NODE-D"}, + {TargetKind: "endpoint_resolution", Family: "endpoint_resolution", TraversalFamily: "SP", Applied: "ENDPOINT-RESOLUTION-INCUMBENT"}, + }} + fallback := false + available := true + candidate[idx].TraversalTelemetry = &TraversalExecutionTelemetry{Summary: TraversalExecutionSummary{ + RequestedIdentity: "SP-B1-C-ALT-NODE-D", RuntimeIdentity: "SP-B1-C-ALT-NODE-D", + RuntimeBranch: "bidirectional_search", RuntimeOutcomeAvailable: &available, FallbackExecuted: &fallback, + }} + setSampleTraversalRuntimeMetadata(&candidate[idx].Stats, candidate[idx].TraversalTelemetry) + for sampleIdx := range candidate[idx].Stats.Samples { + candidate[idx].Stats.Samples[sampleIdx].RuntimeAttestation = "timed_invocation" + candidate[idx].Stats.Samples[sampleIdx].RuntimeReceiptEvents = []RuntimeReceiptEvent{{ + Ordinal: 1, RuntimeIdentity: "SP-B1-C-ALT-NODE-D", RuntimeBranch: "bidirectional_search", FallbackExecuted: false, + }} + } + } + report, err = buildPerfGateReport(baseline, candidate, qualifiedPerfGateOptions(t, baseline, candidate, PerfGateOptions{ + Confidence: defaultConfidenceLevel, BootstrapCount: 50, TargetNames: []string{"sp-training", "sp-holdout"}, + })) + require.NoError(t, err) + require.True(t, report.QualificationPassed) + require.Equal(t, "SP-B1-C-ALT-NODE-D@bidirectional_search", report.QualificationFamilies[0].Family) + candidate[0].Stats.Samples[0].RuntimeAttestation = "same_case_invocation_local_replay" + require.ErrorContains(t, validateCandidateRuntimeEvidence(candidate, performanceKey{ + dataset: candidate[0].Dataset, name: candidate[0].Name, backend: candidate[0].ExecutionMode, + }), "runtime attribution") + candidate[0].Stats.Samples[0].RuntimeAttestation = "timed_invocation" + for idx := range baseline { + baseline[idx].Optimization = nil + candidate[idx].Optimization = nil + } + + baseline = baseline[:1] + candidate = candidate[:1] + report, err = buildPerfGateReport(baseline, candidate, qualifiedPerfGateOptions(t, baseline, candidate, PerfGateOptions{ + Confidence: defaultConfidenceLevel, BootstrapCount: 50, TargetNames: []string{"sp-training"}, + })) + require.NoError(t, err) + require.True(t, report.TrainingPassed) + require.False(t, report.HoldoutPassed) + require.False(t, report.QualificationPassed) + require.False(t, report.Passed) + require.False(t, report.PromotionEligible) +} + +func TestValidateRuntimeReceiptEventsPreservesNestedFallbackChain(t *testing.T) { + fallback := true + events := []RuntimeReceiptEvent{ + {Ordinal: 1, RuntimeIdentity: "SP-I1-C-WE+MAT-M0", RuntimeBranch: "candidate_overflow", FallbackExecuted: true}, + {Ordinal: 2, RuntimeIdentity: "SP-S4-C-WE+MAT-M0", RuntimeBranch: "workspace_overflow", FallbackExecuted: true}, + {Ordinal: 3, RuntimeIdentity: "SP-S3-U-E+MAT-M0", RuntimeBranch: "exact_fallback", FallbackExecuted: true}, + } + require.NoError(t, validateRuntimeReceiptEvents(events, "SP-S3-U-E+MAT-M0", "exact_fallback", &fallback)) + + events[1].Ordinal = 3 + require.ErrorContains(t, validateRuntimeReceiptEvents(events, "SP-S3-U-E+MAT-M0", "exact_fallback", &fallback), "not contiguous") +} diff --git a/cmd/graphbench/postgres.go b/cmd/graphbench/postgres.go index 355b6bc3..d5f8686c 100644 --- a/cmd/graphbench/postgres.go +++ b/cmd/graphbench/postgres.go @@ -18,34 +18,234 @@ package main import ( "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" "fmt" + "os" "regexp" + "slices" "strconv" "strings" + "time" + "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "github.com/specterops/dawgs" "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/specterops/dawgs/databaseguard" "github.com/specterops/dawgs/drivers/pg" "github.com/specterops/dawgs/graph" "github.com/specterops/dawgs/opengraph" "github.com/specterops/dawgs/util/size" ) +// postgresSQLRunner owns PostgreSQL translation, connection, graph, and executor settings. type postgresSQLRunner struct { + // datasetDir locates fixture and corpus files on disk. datasetDir string - db graph.Database - pgDriver *pg.Driver - graphID int32 + // db provides graph transactions for fixture preparation and query execution. + db graph.Database + // pgDriver provides PostgreSQL graph access and kind mapping. + pgDriver *pg.Driver + // pool supplies PostgreSQL connections for translated and raw execution. + pool *pgxpool.Pool + // graphID selects the PostgreSQL graph partition used for translation, fixture validation, and execution. + graphID int32 + // backendPID records the physical PostgreSQL session used to label samples and detect connection changes. + backendPID string + // poolSize records the maximum PostgreSQL connections available to the runner. + poolSize int + // round identifies the measurement round used to balance execution order. + round int + // concurrency lists worker counts measured by the PostgreSQL runner. + concurrency []int + // environment accumulates PostgreSQL environment evidence for the current runner. + environment PostgresEnvironment + // references enables independent PostgreSQL reference execution for the runner. + references bool + // referenceArms lists independent PostgreSQL reference arms measured by the runner. + referenceArms []string + // toolOptions carries forced translation-executor selections for diagnostic runs. + toolOptions translate.ToolOptions + // productionManifest supplies the immutable guarded candidate identity used + // for pre-closure production-boundary measurement. + productionManifest *PromotionManifest + // repeatableRead measures an incumbent or tool arm under an explicit stable + // snapshot for comparison with an admission-equivalent production candidate. + repeatableRead bool + // traversalTelemetry selects opt-in summary or untimed diagnostic traversal evidence. + traversalTelemetry string + // existingGraph supplies live-graph anchors, checkpoints, and callbacks to the runner. + existingGraph *existingGraphRunnerOptions } -func newPostgresSQLRunner(ctx context.Context, datasetDir, connection string, corpus ScaleCorpus) (*postgresSQLRunner, error) { +// existingGraphRunnerOptions supplies live-graph anchors and completed-workload state to the PostgreSQL runner. +type existingGraphRunnerOptions struct { + // Manifest supplies validated live-graph anchors and identity metadata to the runner. + Manifest ExistingGraphAnchorManifest + // ProgressPath selects the append-only progress artifact written by the runner. + ProgressPath string + // Discovery enables adaptive live-graph discovery instead of the fixed confirmation protocol. + Discovery bool + // TimeoutClasses lists the increasing per-attempt deadlines applied during adaptive discovery. + TimeoutClasses []time.Duration + // SampleFloor sets the minimum timed samples required for each live-graph attempt. + SampleFloor int + // Completed maps completed live-graph case keys to fixture-bound identities. + Completed map[string]string + // OnRecord receives each completed live-graph CaseResult for immediate persistence. + OnRecord func(CaseResult) error + // OnComplete records final live-graph node and relationship counts after successful execution. + OnComplete func(int64, int64) error +} + +// setProductionManifest loads a provisional promotion manifest. Evidence may +// be empty because this mode exists to produce that evidence; all fields that +// determine SQL selection and runtime behavior are still validated here. +func (s *postgresSQLRunner) setProductionManifest(path string) error { + if path == "" { + return nil + } + raw, err := os.ReadFile(path) + if err != nil { + return err + } + var manifest PromotionManifest + if err := json.Unmarshal(raw, &manifest); err != nil { + return fmt.Errorf("decode provisional promotion manifest: %w", err) + } + if manifest.Version != promotionManifestVersion || manifest.ExecutionBoundary != "guarded_dual_arm" || strings.TrimSpace(manifest.SelectorVersion) == "" { + return fmt.Errorf("provisional manifest must be version 2 with a selector and guarded_dual_arm boundary") + } + expectedFallback := map[string]string{ + string(optimize.ShortestPathExecutorASPI1DAG): string(optimize.ShortestPathExecutorASPA1DAG), + string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness): string(optimize.ShortestPathExecutorS4CanonicalWitness), + string(optimize.ExpansionSearchPolicyOrientationProbeV1): string(optimize.ExpansionSearchStepwiseForward), + }[manifest.Candidate] + if expectedFallback == "" || manifest.FallbackExecutor != expectedFallback { + return fmt.Errorf("unsupported candidate/fallback pair %s -> %s", manifest.Candidate, manifest.FallbackExecutor) + } + if manifest.Candidate == string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness) && manifest.SelectorVersion != optimize.ShortestPathSelectorStaticV6 { + return fmt.Errorf("canonical SP-I1 provisional manifest requires selector %q", optimize.ShortestPathSelectorStaticV6) + } + if manifest.Candidate == string(optimize.ExpansionSearchPolicyOrientationProbeV1) { + expectedCaps := orientationPromotionCaps() + if len(manifest.Caps) != len(expectedCaps) { + return fmt.Errorf("orientation-probe-v1 requires exactly four immutable caps") + } + for name, expected := range expectedCaps { + if manifest.Caps[name] != expected { + return fmt.Errorf("orientation-probe-v1 cap %s must equal %d", name, expected) + } + } + } else { + expectedCaps := []string{"state_limit", "predecessor_limit", "enumeration_limit", "output_bytes_limit"} + if len(manifest.Caps) != len(expectedCaps) { + return fmt.Errorf("guarded shortest candidate requires exactly four immutable caps") + } + for _, name := range expectedCaps { + if manifest.Caps[name] <= 0 { + return fmt.Errorf("guarded shortest candidate cap %s must be positive", name) + } + } + } + seenQueries := map[string]struct{}{} + for _, bucket := range manifest.Buckets { + if len(bucket.QuerySHA256) == 0 { + return fmt.Errorf("production bucket %q has no exact query cohort", bucket.Name) + } + if manifest.Candidate == string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness) { + if err := validateStaticV6CanonicalInboundBucket(bucket); err != nil { + return err + } + } + for _, digest := range bucket.QuerySHA256 { + if !isLowerHexSHA256(digest) { + return fmt.Errorf("production bucket %q contains an invalid query digest", bucket.Name) + } + if _, found := seenQueries[digest]; found { + return fmt.Errorf("production query digest %s is authorized more than once", digest) + } + seenQueries[digest] = struct{}{} + } + } + if len(seenQueries) == 0 { + return fmt.Errorf("provisional manifest has no exact query cohort") + } + s.productionManifest = &manifest + return nil +} + +func (s *postgresSQLRunner) productionOptions(cypherQuery string) (translate.ProductionOptions, error) { + manifest := s.productionManifest + if manifest == nil { + return translate.ProductionOptions{}, fmt.Errorf("production manifest is not configured") + } + digest := pg.TraversalPolicyQuerySHA256(cypherQuery) + for _, bucket := range manifest.Buckets { + if !slices.Contains(bucket.QuerySHA256, digest) { + continue + } + options := translate.ProductionOptions{ + AuthorizedBucket: &translate.ProductionTraversalBucket{ + Direction: bucket.Direction, ObservationMode: bucket.ObservationMode, + MinimumDepth: int64(bucket.MinimumDepth), MaximumDepth: int64(bucket.MaximumDepth), + RelationshipKindCount: bucket.RelationshipKindCount, UntypedRelationship: bucket.UntypedRelationship, + }, + SelectorVersion: manifest.SelectorVersion, + } + if manifest.Candidate == string(optimize.ExpansionSearchPolicyOrientationProbeV1) { + options.EnableExpansionOrientation = true + } else { + options.ShortestPathExecutor = optimize.ShortestPathExecutor(manifest.Candidate) + options.ShortestPathCaps = &translate.ProductionShortestPathCaps{ + StateLimit: manifest.Caps["state_limit"], PredecessorLimit: manifest.Caps["predecessor_limit"], + EnumerationLimit: manifest.Caps["enumeration_limit"], OutputBytesLimit: manifest.Caps["output_bytes_limit"], + } + } + return options, nil + } + return translate.ProductionOptions{}, fmt.Errorf("query SHA-256 %s is absent from the provisional production manifest", digest) +} + +// newPostgresSQLRunner opens a PostgreSQL benchmark runner for managed-fixture execution. +func newPostgresSQLRunner(ctx context.Context, datasetDir, connection string, corpus ScaleCorpus, poolSize, round int, concurrency []int, references bool, referenceArms []string, forceShortest, forceExpansion string) (*postgresSQLRunner, error) { + return newPostgresSQLRunnerWithExistingGraph(ctx, datasetDir, connection, corpus, poolSize, round, concurrency, references, referenceArms, forceShortest, forceExpansion, nil) +} + +// newPostgresSQLRunnerWithExistingGraph opens a PostgreSQL benchmark runner with optional live-graph state. +func newPostgresSQLRunnerWithExistingGraph(ctx context.Context, datasetDir, connection string, corpus ScaleCorpus, poolSize, round int, concurrency []int, references bool, referenceArms []string, forceShortest, forceExpansion string, existing *existingGraphRunnerOptions) (*postgresSQLRunner, error) { + if existing == nil { + if err := databaseguard.ValidateEnvironment(connection); err != nil { + return nil, fmt.Errorf("refuse destructive PostgreSQL GraphBench target: %w", err) + } + } + poolCfg, err := pgxpool.ParseConfig(connection) if err != nil { return nil, fmt.Errorf("parse PostgreSQL pool configuration: %w", err) } - pool, err := pg.NewPool(poolCfg) + // GraphBench needs first-call and steady-state samples from an identifiable + // physical session. A single-connection pool makes that relationship + // deterministic while retaining the production pool hooks. + poolCfg.MinConns = int32(poolSize) + poolCfg.MaxConns = int32(poolSize) + if compactBidirectionalSnapshotRequired(references, referenceArms, forceShortest) { + if poolCfg.ConnConfig.RuntimeParams == nil { + poolCfg.ConnConfig.RuntimeParams = map[string]string{} + } + poolCfg.ConnConfig.RuntimeParams["default_transaction_isolation"] = "repeatable read" + } + // pg.NewPool applies the production driver's fixed 5/50 pool sizing. The + // benchmark must preserve the requested size so a size-one run can prove + // that all samples in a case used the same physical session. + poolCfg.AfterConnect = pg.AfterPooledConnectionEstablished + poolCfg.AfterRelease = pg.AfterPooledConnectionRelease + pool, err := pgxpool.NewWithConfig(ctx, poolCfg) if err != nil { return nil, fmt.Errorf("create PostgreSQL pool: %w", err) } @@ -60,15 +260,17 @@ func newPostgresSQLRunner(ctx context.Context, datasetDir, connection string, co return nil, fmt.Errorf("open PostgreSQL database: %w", err) } - nodeKinds, edgeKinds, err := scanDatasetKinds(datasetDir, scaleCorpusDatasets(corpus)) - if err != nil { - _ = db.Close(ctx) - return nil, err - } + if existing == nil { + nodeKinds, edgeKinds, err := scanDatasetKinds(datasetDir, scaleCorpusDatasets(corpus)) + if err != nil { + _ = db.Close(ctx) + return nil, err + } - if err := db.AssertSchema(ctx, benchmarkSchema(nodeKinds, edgeKinds)); err != nil { - _ = db.Close(ctx) - return nil, fmt.Errorf("assert PostgreSQL schema: %w", err) + if err := db.AssertSchema(ctx, benchmarkSchema(nodeKinds, edgeKinds)); err != nil { + _ = db.Close(ctx) + return nil, fmt.Errorf("assert PostgreSQL schema: %w", err) + } } pgDriver, ok := db.(*pg.Driver) @@ -76,21 +278,104 @@ func newPostgresSQLRunner(ctx context.Context, datasetDir, connection string, co _ = db.Close(ctx) return nil, fmt.Errorf("expected *pg.Driver, got %T", db) } + if existing != nil { + if err := pgDriver.SetDefaultGraph(ctx, graph.Graph{ + Name: existing.Manifest.Graph, + }); err != nil { + _ = db.Close(ctx) + return nil, fmt.Errorf("select existing PostgreSQL graph: %w", err) + } + if err := pgDriver.Fetch(ctx); err != nil { + _ = db.Close(ctx) + return nil, fmt.Errorf("fetch existing PostgreSQL kinds: %w", err) + } + } defaultGraph, ok := pgDriver.DefaultGraph() if !ok { _ = db.Close(ctx) return nil, fmt.Errorf("PostgreSQL default graph is not set") } + if existing != nil && existing.Manifest.Graph != "" && existing.Manifest.Graph != defaultGraph.Name { + _ = db.Close(ctx) + return nil, fmt.Errorf("anchor manifest graph %q does not match PostgreSQL default graph %q", existing.Manifest.Graph, defaultGraph.Name) + } + var backendPID int32 + if err := pool.QueryRow(ctx, "select pg_backend_pid()").Scan(&backendPID); err != nil { + _ = db.Close(ctx) + return nil, fmt.Errorf("identify PostgreSQL benchmark connection: %w", err) + } + var postgresEnvironment PostgresEnvironment + if err := pool.QueryRow(ctx, `select version(), current_database(), current_setting('plan_cache_mode'), current_setting('transaction_isolation'), current_setting('work_mem'), current_setting('temp_file_limit'), (select count(*) from graph), pg_postmaster_start_time(), (select oid::int8 from pg_database where datname = current_database()), current_setting('autovacuum')`).Scan( + &postgresEnvironment.Version, + &postgresEnvironment.Database, + &postgresEnvironment.PlanCacheMode, + &postgresEnvironment.TransactionIsolation, + &postgresEnvironment.WorkMem, + &postgresEnvironment.TempFileLimit, + &postgresEnvironment.GraphPartitionCount, + &postgresEnvironment.PostmasterStartedAt, + &postgresEnvironment.DatabaseOID, + &postgresEnvironment.Autovacuum, + ); err != nil { + _ = db.Close(ctx) + return nil, fmt.Errorf("capture PostgreSQL environment: %w", err) + } return &postgresSQLRunner{ - datasetDir: datasetDir, - db: db, - pgDriver: pgDriver, - graphID: defaultGraph.ID, + datasetDir: datasetDir, + db: db, + pgDriver: pgDriver, + pool: pool, + graphID: defaultGraph.ID, + backendPID: strconv.FormatInt(int64(backendPID), 10), + poolSize: poolSize, + round: round, + concurrency: append([]int(nil), concurrency...), + environment: postgresEnvironment, + references: references, + referenceArms: append([]string(nil), referenceArms...), + toolOptions: translate.ToolOptions{ + ForceShortestPathExecutor: optimize.ShortestPathExecutor(forceShortest), + ForceExpansionSearchStrategy: optimize.ExpansionSearchStrategy(forceExpansion), + }, + existingGraph: existing, }, nil } +// compactBidirectionalSnapshotRequired reports whether any selected production +// or reference arm can execute the multi-statement B1/B2 workspace kernel. +func compactBidirectionalSnapshotRequired(references bool, referenceArms []string, forceShortest string) bool { + switch optimize.ShortestPathExecutor(forceShortest) { + case optimize.ShortestPathExecutorB1AlternatingNodeDistance, + optimize.ShortestPathExecutorB1AlternatingNodeWitness, + optimize.ShortestPathExecutorB2SmallerCurrentLevelDistance, + optimize.ShortestPathExecutorB2SmallerCurrentLevelWitness, + optimize.ShortestPathExecutorASPB1AlternatingNodeDAG, + optimize.ShortestPathExecutorASPB2SmallerCurrentLevelDAG: + return true + } + if !references { + return false + } + if len(referenceArms) == 0 { + return true + } + for _, arm := range referenceArms { + switch arm { + case "sp_b1_strict_alternating_distance", + "sp_b1_strict_alternating_witness_m0", + "sp_b2_smaller_frontier_distance", + "sp_b2_smaller_frontier_witness_m0", + "asp_b1_bidirectional_dag_strict_m0", + "asp_b2_bidirectional_dag_smaller_frontier_m0": + return true + } + } + return false +} + +// Close releases the graph database and PostgreSQL pool owned by the runner. func (s *postgresSQLRunner) Close(ctx context.Context) error { if s.db == nil { return nil @@ -99,13 +384,21 @@ func (s *postgresSQLRunner) Close(ctx context.Context) error { return s.db.Close(ctx) } -func (s *postgresSQLRunner) Run(ctx context.Context, iterations int, corpus ScaleCorpus) ([]CaseResult, error) { +// Run measures supported corpus cases against managed fixtures or the configured preexisting graph. +func (s *postgresSQLRunner) Run(ctx context.Context, warmupIterations, iterations int, corpus ScaleCorpus) ([]CaseResult, error) { + if s.existingGraph != nil { + return s.runExistingGraph(ctx, warmupIterations, iterations, corpus) + } var ( records []CaseResult casesByDataset = scaleCasesByDataset(corpus) ) for _, datasetName := range scaleCorpusDatasets(corpus) { + fixture, err := fixtureMetadata(s.datasetDir, datasetName) + if err != nil { + return nil, err + } if err := clearGraph(ctx, s.db); err != nil { return nil, fmt.Errorf("clear graph for %s: %w", datasetName, err) } @@ -114,13 +407,32 @@ func (s *postgresSQLRunner) Run(ctx context.Context, iterations int, corpus Scal if err != nil { return nil, err } + if err := s.captureAndValidateFixture(ctx, &fixture); err != nil { + return nil, fmt.Errorf("validate %s fixture: %w", datasetName, err) + } + activePartitions := fmt.Sprintf("vacuum (analyze) node_%d, edge_%d", s.graphID, s.graphID) + if _, err := s.pool.Exec(ctx, activePartitions); err != nil { + return nil, fmt.Errorf("vacuum and analyze %s fixture: %w", datasetName, err) + } + if err := s.pool.QueryRow(ctx, `select pg_total_relation_size(format('node_%s', $1::int4)::regclass), pg_total_relation_size(format('edge_%s', $1::int4)::regclass), coalesce((select string_agg(relname || ':' || coalesce(last_analyze::text, 'never'), ',' order by relname) from pg_stat_all_tables where relname in (format('node_%s', $1::int4), format('edge_%s', $1::int4))), '')`, s.graphID).Scan( + &fixture.NodeRelationBytes, &fixture.EdgeRelationBytes, &s.environment.AnalyzeState, + ); err != nil { + return nil, fmt.Errorf("capture %s fixture relation sizes: %w", datasetName, err) + } + s.environment.NodeRelationBytes = fixture.NodeRelationBytes + s.environment.EdgeRelationBytes = fixture.EdgeRelationBytes for _, testCase := range casesByDataset[datasetName] { if !testCase.Supports(ModePostgresSQL) { continue } - record := s.runCase(ctx, iterations, testCase, idMap) + if err := s.resetCaseSession(ctx); err != nil { + return nil, fmt.Errorf("reset PostgreSQL session for %s: %w", testCase.Name, err) + } + + record := s.runCase(ctx, warmupIterations, iterations, testCase, idMap) + attachFixtureMetadata(&record, fixture) records = append(records, record) } } @@ -128,27 +440,401 @@ func (s *postgresSQLRunner) Run(ctx context.Context, iterations int, corpus Scal return records, nil } -func (s *postgresSQLRunner) runCase(ctx context.Context, iterations int, testCase ScaleCase, idMap opengraph.IDMap) CaseResult { - params, err := resolveCaseParams(testCase, idMap) - record := newCaseResult(testCase, ModePostgresSQL, params) +// runExistingGraph executes eligible live-graph cases, honoring checkpoints and progress callbacks. +func (s *postgresSQLRunner) runExistingGraph(ctx context.Context, warmupIterations, iterations int, corpus ScaleCorpus) ([]CaseResult, error) { + options := s.existingGraph + if err := validateExistingGraphCorpus(corpus, options.Manifest); err != nil { + return nil, err + } + anchors, err := s.resolveExistingGraphAnchors(ctx, options.Manifest) if err != nil { - record.Status = StatusError - record.Error = err.Error() - return record + return nil, err + } + idMap := idMapForManifest(anchors) + preNodes, preEdges, err := s.existingGraphCounts(ctx) + if err != nil { + return nil, err + } + if err := s.captureExistingGraphEnvironment(ctx); err != nil { + return nil, err + } + databaseDigest := sha256.Sum256([]byte(s.environment.Database)) + s.environment.Database = "sha256:" + hex.EncodeToString(databaseDigest[:]) + fixture := FixtureMetadata{ + Dataset: "existing_graph", + Checksum: strings.Join([]string{ + options.Manifest.Checksum, + options.Manifest.ContentIdentity, + s.environment.SchemaFingerprint, + s.environment.IndexFingerprint, + }, ":"), + PhysicalValidated: true, + PhysicalNodeCount: preNodes, + PhysicalEdgeCount: preEdges, + NodeRelationBytes: s.environment.NodeRelationBytes, + EdgeRelationBytes: s.environment.EdgeRelationBytes, + Configuration: "existing_graph_read_only", + } + if err := validateCompletedWorkloads(options.Completed, corpus, fixture); err != nil { + return nil, err + } + var records []CaseResult + for _, testCase := range corpus.Cases { + if !testCase.Supports(ModePostgresSQL) { + continue + } + caseKey := existingGraphCaseKey(ModePostgresSQL, testCase) + if _, completed := options.Completed[caseKey]; completed { + continue + } + if err := appendExistingGraphProgress(options.ProgressPath, ExistingGraphProgress{ + Stage: "case", + CaseKey: caseKey, + }); err != nil { + return nil, err + } + if err := s.resetCaseSession(ctx); err != nil { + return nil, fmt.Errorf("reset PostgreSQL session for %s: %w", testCase.Name, err) + } + record := s.runExistingGraphCase(ctx, warmupIterations, iterations, testCase, idMap) + attachFixtureMetadata(&record, fixture) + record.ExistingGraph.PreNodeCount, record.ExistingGraph.PreEdgeCount = preNodes, preEdges + redactExistingGraphRecord(&record, options.Manifest, anchors) + records = append(records, record) + if options.OnRecord != nil { + if err := options.OnRecord(record); err != nil { + return nil, err + } + } + } + postNodes, postEdges, err := s.existingGraphCounts(ctx) + if err != nil { + return nil, err } + if preNodes != postNodes || preEdges != postEdges { + return nil, fmt.Errorf("existing graph cardinality changed: nodes %d -> %d, edges %d -> %d", preNodes, postNodes, preEdges, postEdges) + } + for idx := range records { + records[idx].ExistingGraph.PostNodeCount, records[idx].ExistingGraph.PostEdgeCount = postNodes, postEdges + } + if options.OnComplete != nil { + if err := options.OnComplete(postNodes, postEdges); err != nil { + return nil, err + } + } + if err := appendExistingGraphProgress(options.ProgressPath, ExistingGraphProgress{ + Stage: "complete", + Detail: fmt.Sprintf("nodes=%d edges=%d", postNodes, postEdges), + }); err != nil { + return nil, err + } + return records, nil +} - rowCount, stats, err := measureCypher(ctx, s.db, testCase.Cypher, params, iterations) +// runExistingGraphCase executes the fixed-confirmation or adaptive timeout protocol for one read-only workload against a preexisting graph. +func (s *postgresSQLRunner) runExistingGraphCase(ctx context.Context, warmupIterations, iterations int, testCase ScaleCase, idMap opengraph.IDMap) CaseResult { + options := s.existingGraph + timeouts := options.TimeoutClasses + if len(timeouts) == 0 { + timeouts = []time.Duration{0} + } + live := &ExistingGraphRun{ + ManifestSHA256: options.Manifest.Checksum, + ContentIdentity: options.Manifest.ContentIdentity, + Protocol: "fixed_confirmation", + Adaptive: options.Discovery, + } + if options.Discovery { + live.Protocol = "adaptive_discovery" + } + var record CaseResult + for idx, timeout := range timeouts { + measured := iterations + warmups := warmupIterations + if options.Discovery && idx > 0 { + measured = max(options.SampleFloor, iterations>>idx) + warmups = warmupIterations >> idx + } + attemptCtx := ctx + cancel := func() {} + if timeout > 0 { + attemptCtx, cancel = context.WithTimeout(ctx, timeout) + } + record = s.runCase(attemptCtx, warmups, measured, testCase, idMap) + attemptErr := attemptCtx.Err() + cancel() + attempt := ExistingGraphAttempt{ + Timeout: timeout, + WarmupSamples: warmups, + MeasuredSamples: measured, + Status: record.Status, + Error: record.Error, + } + live.Attempts = append(live.Attempts, attempt) + if attemptErr == nil || !options.Discovery { + break + } + _ = appendExistingGraphProgress(options.ProgressPath, ExistingGraphProgress{ + Stage: "timeout", + CaseKey: existingGraphCaseKey(ModePostgresSQL, testCase), + Detail: timeout.String(), + }) + } + record.ExistingGraph = live + return record +} + +// resolveLogicalExistingGraphAnchor looks up one logical-key anchor and rejects missing or ambiguous matches. +func (s *postgresSQLRunner) resolveLogicalExistingGraphAnchor(ctx context.Context, name, logicalKey string) ([]int64, error) { + rows, err := s.pool.Query(ctx, `select id from node where graph_id = $1 and properties ->> 'logical_key' = $2 order by id limit 2`, s.graphID, logicalKey) + if err != nil { + return nil, fmt.Errorf("resolve anchor %s: %w", name, err) + } + defer rows.Close() + + var ids []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, err + } + + ids = append(ids, id) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("resolve anchor %s rows: %w", name, err) + } + + return ids, nil +} + +// resolveExistingGraphAnchors resolves every manifest anchor to exactly one PostgreSQL node identifier. +func (s *postgresSQLRunner) resolveExistingGraphAnchors(ctx context.Context, manifest ExistingGraphAnchorManifest) (map[string]graph.ID, error) { + anchors := make(map[string]graph.ID, len(manifest.Anchors)) + for name, anchor := range manifest.Anchors { + var ids []int64 + if anchor.PhysicalID == nil { + if resolvedIDs, err := s.resolveLogicalExistingGraphAnchor(ctx, name, anchor.LogicalKey); err != nil { + return nil, err + } else { + ids = resolvedIDs + } + } else { + var ( + kindIDs string + properties string + id int64 + ) + + if err := s.pool.QueryRow(ctx, `select id, kind_ids::text, properties::text from node where graph_id = $1 and id = $2`, s.graphID, *anchor.PhysicalID).Scan(&id, &kindIDs, &properties); err != nil { + return nil, fmt.Errorf("resolve physical anchor %s: %w", name, err) + } + + digest := sha256.Sum256([]byte(kindIDs + "\n" + properties)) + actual := "sha256:" + hex.EncodeToString(digest[:]) + if actual != anchor.ContentSHA256 { + return nil, fmt.Errorf("physical anchor %s content identity mismatch", name) + } + + ids = append(ids, id) + } + + if len(ids) != 1 { + return nil, fmt.Errorf("anchor %s resolved to %d nodes; exactly one is required", name, len(ids)) + } + + if anchor.Kind != "" { + var matches bool + if err := s.pool.QueryRow(ctx, `select exists(select 1 from node n join kind k on k.id = any(n.kind_ids) where n.graph_id = $1 and n.id = $2 and k.name = $3)`, s.graphID, ids[0], anchor.Kind).Scan(&matches); err != nil { + return nil, err + } + + if !matches { + return nil, fmt.Errorf("anchor %s does not have declared kind %s", name, anchor.Kind) + } + } + + anchors[name] = graph.ID(ids[0]) + } + + return anchors, nil +} + +// existingGraphCounts returns node and relationship counts for the selected PostgreSQL graph. +func (s *postgresSQLRunner) existingGraphCounts(ctx context.Context) (int64, int64, error) { + var nodes, edges int64 + if err := s.pool.QueryRow(ctx, `select (select count(*) from node where graph_id = $1), (select count(*) from edge where graph_id = $1)`, s.graphID).Scan(&nodes, &edges); err != nil { + return 0, 0, err + } + + return nodes, edges, nil +} + +// captureExistingGraphEnvironment records live graph relation sizes and normalized schema and index fingerprints. +func (s *postgresSQLRunner) captureExistingGraphEnvironment(ctx context.Context) error { + if err := s.pool.QueryRow(ctx, `select pg_total_relation_size(format('node_%s', $1::int4)::regclass), pg_total_relation_size(format('edge_%s', $1::int4)::regclass)`, s.graphID).Scan(&s.environment.NodeRelationBytes, &s.environment.EdgeRelationBytes); err != nil { + return err + } + return s.pool.QueryRow(ctx, `select + md5(coalesce((select string_agg(table_name || ':' || column_name || ':' || data_type, ',' order by table_name, ordinal_position) from information_schema.columns where table_schema = current_schema() and table_name in ('graph','kind','node','edge')), '')), + md5(coalesce((select string_agg(indexname || ':' || indexdef, ',' order by indexname) from pg_indexes where schemaname = current_schema() and (tablename in ('node','edge') or tablename in (format('node_%s',$1::int4), format('edge_%s',$1::int4)))), ''))`, s.graphID).Scan(&s.environment.SchemaFingerprint, &s.environment.IndexFingerprint) +} + +// captureAndValidateFixture records physical fixture sizes and rejects cardinality or checksum drift. +func (s *postgresSQLRunner) captureAndValidateFixture(ctx context.Context, fixture *FixtureMetadata) error { + if err := s.pool.QueryRow(ctx, `select (select count(*) from node where graph_id = $1), (select count(*) from edge where graph_id = $1)`, s.graphID).Scan( + &fixture.PhysicalNodeCount, + &fixture.PhysicalEdgeCount, + ); err != nil { + return fmt.Errorf("count physical graph rows: %w", err) + } + if fixture.PhysicalNodeCount != int64(fixture.NodeCount) || fixture.PhysicalEdgeCount != int64(fixture.EdgeCount) { + return fmt.Errorf( + "physical cardinality mismatch: nodes=%d want=%d edges=%d want=%d", + fixture.PhysicalNodeCount, + fixture.NodeCount, + fixture.PhysicalEdgeCount, + fixture.EdgeCount, + ) + } + fixture.PhysicalValidated = true + + return nil +} + +// resetCaseSession drops pooled session state between cases and, for single-connection runs, records the replacement backend PID used to verify isolation. +func (s *postgresSQLRunner) resetCaseSession(ctx context.Context) error { + s.pool.Reset() + if s.poolSize != 1 { + s.backendPID = "" + return nil + } + + var backendPID int32 + if err := s.pool.QueryRow(ctx, "select pg_backend_pid()").Scan(&backendPID); err != nil { + return err + } + s.backendPID = strconv.FormatInt(int64(backendPID), 10) + return nil +} + +// runCase resolves fixture parameters, executes the PostgreSQL read or write measurement path, captures plans and cache statistics, and returns one CaseResult. +func (s *postgresSQLRunner) runCase(ctx context.Context, warmupIterations, iterations int, testCase ScaleCase, idMap opengraph.IDMap) (record CaseResult) { + params, err := resolveCaseParams(testCase, idMap) + record = newCaseResult(testCase, ModePostgresSQL, params) + defer func() { + stats := s.pgDriver.ParseCacheStats() + record.ParseCache = &stats + }() if err != nil { record.Status = StatusError record.Error = err.Error() return record } - record.RowCount = rowCount - record.Stats = stats - applyRowExpectation(&record) + if testCase.WriteScenario == nil { + var ( + rowCount int64 + observedRows []string + stats DurationStats + ) + readOptions := s.readTransactionOptions() - explain, err := s.explain(ctx, testCase.Cypher, params) + if !hasForcedToolOptions(s.toolOptions) && s.productionManifest == nil { + if len(readOptions) == 0 { + rowCount, observedRows, stats, err = measureCypherWithWarmups(ctx, s.db, testCase.Cypher, params, testCase.Expected, idMap, warmupIterations, iterations) + } else { + rowCount, observedRows, stats, err = measureCypherWithWarmupsOptions(ctx, s.db, testCase.Cypher, params, testCase.Expected, idMap, warmupIterations, iterations, readOptions...) + } + } else { + translation, sqlQuery, translateErr := s.translateCypher(ctx, testCase.Cypher, params) + if translateErr != nil { + err = translateErr + } else { + requestedIdentity := timedRuntimeAttestationIdentity(translation) + if requestedIdentity == "" { + if len(readOptions) == 0 { + rowCount, observedRows, stats, err = measureRawSQLWithWarmups(ctx, s.db, sqlQuery, translation.Parameters, testCase.Expected, idMap, warmupIterations, iterations) + } else { + rowCount, observedRows, stats, err = measureRawSQLWithWarmupsOptions(ctx, s.db, sqlQuery, translation.Parameters, testCase.Expected, idMap, warmupIterations, iterations, readOptions...) + } + } else if s.poolSize != 1 { + // Exact per-sample receipts require one physical session. Larger + // pools remain useful for operational smoke testing, but their + // samples intentionally lack promotion-grade attestation. + if len(readOptions) == 0 { + rowCount, observedRows, stats, err = measureRawSQLWithWarmups(ctx, s.db, sqlQuery, translation.Parameters, testCase.Expected, idMap, warmupIterations, iterations) + } else { + rowCount, observedRows, stats, err = measureRawSQLWithWarmupsOptions(ctx, s.db, sqlQuery, translation.Parameters, testCase.Expected, idMap, warmupIterations, iterations, readOptions...) + } + } else if attestor, attestorErr := newPostgresTimedReadAttestor(s.pool, s.poolSize, requestedIdentity); attestorErr != nil { + err = attestorErr + } else if len(readOptions) == 0 { + rowCount, observedRows, stats, err = measureRawSQLWithWarmupsAndAttestation(ctx, s.db, sqlQuery, translation.Parameters, testCase.Expected, idMap, warmupIterations, iterations, attestor) + } else { + rowCount, observedRows, stats, err = measureRawSQLWithWarmupsAndAttestationOptions(ctx, s.db, sqlQuery, translation.Parameters, testCase.Expected, idMap, warmupIterations, iterations, attestor, readOptions...) + } + } + } + if err != nil { + record.Status = StatusError + record.Error = err.Error() + return record + } + + record.RowCount = rowCount + record.ObservedRows = observedRows + record.Stats = stats + labelLatencySamples(&record.Stats, ModePostgresSQL, testCase) + for idx := range record.Stats.Samples { + record.Stats.Samples[idx].ConnectionID = s.backendPID + } + applyRowExpectation(&record) + } else { + scenario, err := resolveWriteScenario(testCase, idMap) + if err != nil { + record.Status = StatusError + record.Error = err.Error() + return record + } + + measurement, stats, err := measureWriteCypherWithWarmups(ctx, s.db, testCase.Cypher, params, scenario, warmupIterations, iterations) + if err != nil { + record.Status = StatusError + record.Error = err.Error() + return record + } + + record.MatchedCount = &measurement.Matched + record.AffectedCount = &measurement.Affected + record.PostState = measurement.PostState + record.Stats = stats + labelLatencySamples(&record.Stats, ModePostgresSQL, testCase) + for idx := range record.Stats.Samples { + record.Stats.Samples[idx].ConnectionID = s.backendPID + } + } + if s.poolSize == 1 { + var backendPID int32 + if err := s.pool.QueryRow(ctx, "select pg_backend_pid()").Scan(&backendPID); err != nil { + record.Status = StatusError + record.Error = fmt.Sprintf("verify PostgreSQL benchmark connection: %v", err) + return record + } + if current := strconv.FormatInt(int64(backendPID), 10); current != s.backendPID { + record.Status = StatusError + record.Error = fmt.Sprintf("PostgreSQL physical connection changed during case: %s -> %s", s.backendPID, current) + return record + } + } + + if s.existingGraph != nil { + _ = appendExistingGraphProgress(s.existingGraph.ProgressPath, ExistingGraphProgress{ + Stage: "plan", + CaseKey: existingGraphCaseKey(ModePostgresSQL, testCase), + }) + } + explain, err := s.explain(ctx, testCase.Cypher, params, testCase.WriteScenario != nil) if err != nil { if record.Status == StatusOK { record.Status = StatusError @@ -158,37 +844,189 @@ func (s *postgresSQLRunner) runCase(ctx context.Context, iterations int, testCas } record.SQL = explain.SQL + record.SQLFingerprint = sqlFingerprint(explain.SQL) + postgresEnvironment := s.environment + if len(s.readTransactionOptions()) > 0 { + postgresEnvironment.TransactionIsolation = "repeatable read" + } + record.PostgresEnvironment = &postgresEnvironment record.PostgresPlan = explain.Plan + record.PostgresPlanJSON = explain.PlanJSON record.PostgresMetrics = &explain.Metrics record.Optimization = &explain.Optimization + if explain.Optimization.LoweringPlan != nil { + var fallbackReasons []string + for _, decision := range explain.Optimization.LoweringPlan.ShortestPathExecutor { + if decision.FallbackReason != "" && !slices.Contains(fallbackReasons, decision.FallbackReason) { + fallbackReasons = append(fallbackReasons, decision.FallbackReason) + } + } + for _, decision := range explain.Optimization.LoweringPlan.ExpansionSearchStrategy { + if decision.FallbackReason != "" && !slices.Contains(fallbackReasons, decision.FallbackReason) { + fallbackReasons = append(fallbackReasons, decision.FallbackReason) + } + } + record.FallbackReason = strings.Join(fallbackReasons, ",") + } + if s.references && testCase.WriteScenario == nil { + var rawIsolation []pgx.TxIsoLevel + if len(s.readTransactionOptions()) > 0 { + rawIsolation = []pgx.TxIsoLevel{pgx.RepeatableRead} + } + waterfall, err := measureCompileWaterfall(ctx, testCase.Cypher, params, s.pgDriver.KindMapper(), s.graphID, iterations, s.toolOptions) + if err != nil { + record.Status = StatusError + record.Error = fmt.Sprintf("client compile waterfall: %v", err) + return record + } + record.ClientWaterfall = &waterfall + productionOrder, referenceOrder := referenceClosureMeasurementOrder(len(s.referenceArms) == 1, s.round) + var references []PostgresReferenceResult + if referenceOrder == 1 { + references, err = s.measureReferences(ctx, testCase, params, idMap, record.ObservedRows, warmupIterations, iterations) + if err != nil { + record.Status = StatusError + record.Error = fmt.Sprintf("PostgreSQL references: %v", err) + return record + } + setReferenceMeasurementOrder(references, referenceOrder) + } + rawWaterfall, err := measureRawPGXWaterfall(ctx, s.pool, explain.SQL, explain.Parameters, warmupIterations, iterations, rawIsolation...) + if err != nil { + record.Status = StatusError + record.Error = fmt.Sprintf("raw pgx waterfall: %v", err) + return record + } + if len(rawWaterfall.Samples) > 0 && rawWaterfall.Samples[0].Rows != record.RowCount { + record.Status = StatusError + record.Error = fmt.Sprintf("raw pgx row count %d differs from CySQL row count %d", rawWaterfall.Samples[0].Rows, record.RowCount) + return record + } + rawWaterfall.MeasurementOrder = productionOrder + record.RawPGXWaterfall = &rawWaterfall + if referenceOrder != 1 { + references, err = s.measureReferences(ctx, testCase, params, idMap, record.ObservedRows, warmupIterations, iterations) + if err != nil { + record.Status = StatusError + record.Error = fmt.Sprintf("PostgreSQL references: %v", err) + return record + } + setReferenceMeasurementOrder(references, referenceOrder) + } + record.PostgresReferences = references + roundTrip, err := measureRawPGXWaterfall(ctx, s.pool, "select 1", nil, warmupIterations, iterations, rawIsolation...) + if err != nil { + record.Status = StatusError + record.Error = fmt.Sprintf("raw pgx round trip: %v", err) + return record + } + record.RawPGXRoundTrip = &roundTrip + } + if testCase.WriteScenario == nil && len(s.concurrency) > 0 { + if s.existingGraph != nil { + _ = appendExistingGraphProgress(s.existingGraph.ProgressPath, ExistingGraphProgress{ + Stage: "concurrency", + CaseKey: existingGraphCaseKey(ModePostgresSQL, testCase), + }) + } + var concurrencyIsolation []pgx.TxIsoLevel + if len(s.readTransactionOptions()) > 0 { + concurrencyIsolation = []pgx.TxIsoLevel{pgx.RepeatableRead} + } + blocks, err := measurePostgresConcurrency(ctx, s.pool, explain.SQL, explain.Parameters, s.poolSize, s.concurrency, iterations, concurrencyIsolation...) + if err != nil { + record.Status = StatusError + record.Error = fmt.Sprintf("concurrency smoke: %v", err) + return record + } + record.Concurrency = blocks + } + if testCase.WriteScenario == nil { + if err := s.attachPostgresTraversalTelemetry(ctx, &record, explain.Parameters); err != nil { + record.Status = StatusError + record.Error = err.Error() + return record + } + setSampleTraversalRuntimeMetadata(&record.Stats, record.TraversalTelemetry) + } return record } -type postgresExplain struct { - SQL string - Plan []string - Metrics PostgresPlanMetrics - Optimization translate.OptimizationSummary +// readTransactionOptions returns the one stable-snapshot contract shared by +// every PostgreSQL timing and plan-replay path. Provisional production +// manifests always require Repeatable Read; tool tournaments opt into the same +// isolation with -postgres-repeatable-read. +func (s *postgresSQLRunner) readTransactionOptions() []graph.TransactionOption { + if s.productionManifest == nil && !s.repeatableRead { + return nil + } + return []graph.TransactionOption{pg.OptionSetTransactionIsolation(pgx.RepeatableRead)} } -func (s *postgresSQLRunner) explain(ctx context.Context, cypherQuery string, params map[string]any) (postgresExplain, error) { - regularQuery, err := frontend.ParseCypher(frontend.NewContext(), cypherQuery) - if err != nil { - return postgresExplain{}, err +func timedRuntimeAttestationIdentity(translation translate.Result) string { + outcome, ok := singleTraversalOutcome(translation.Optimization.TargetOutcomes) + if !ok { + return "" } + requested := outcome.Candidate + if requested == "" { + requested = outcome.Selected + } + if strings.HasPrefix(requested, "SP-B1-") || strings.HasPrefix(requested, "SP-B2-") || + strings.HasPrefix(requested, "ASP-B1-") || strings.HasPrefix(requested, "ASP-B2-") || + requested == string(optimize.ShortestPathExecutorS4CanonicalDistance) || + requested == string(optimize.ShortestPathExecutorS4CanonicalWitness) || + requested == string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness) || + requested == string(optimize.ShortestPathExecutorASPI1DAG) || + isOrientationProbePolicy(outcome.EmittedPolicy) { + return requested + } + return "" +} - translation, err := translate.Translate(ctx, regularQuery, s.pgDriver.KindMapper(), params, s.graphID) - if err != nil { - return postgresExplain{}, err +// referenceClosureMeasurementOrder returns the balanced production/reference order for a measurement round. +func referenceClosureMeasurementOrder(singleSelectedReference bool, round int) (production, reference int) { + if singleSelectedReference && round > 0 && round%2 == 0 { + return 2, 1 } + return 1, 2 +} - sqlQuery, err := translate.Translated(translation) +// setReferenceMeasurementOrder assigns consecutive execution positions to reference results beginning at order. +func setReferenceMeasurementOrder(references []PostgresReferenceResult, order int) { + for idx := range references { + references[idx].MeasurementOrder = order + idx + } +} + +// postgresExplain contains translated SQL and normalized PostgreSQL EXPLAIN evidence. +type postgresExplain struct { + // SQL contains the rendered SQL statement. + SQL string + // Plan contains normalized PostgreSQL text-plan lines. + Plan []string + // PlanJSON contains structured backend plan evidence. + PlanJSON json.RawMessage + // Metrics contains normalized PostgreSQL plan counters and resources. + Metrics PostgresPlanMetrics + // Optimization captures translation optimization and lowering decisions. + Optimization translate.OptimizationSummary + // Parameters contains translated SQL parameters keyed by placeholder name. + Parameters map[string]any +} + +// explain translates a Cypher query and returns normalized SQL and PostgreSQL EXPLAIN evidence. +func (s *postgresSQLRunner) explain(ctx context.Context, cypherQuery string, params map[string]any, write bool) (postgresExplain, error) { + translation, sqlQuery, err := s.translateCypher(ctx, cypherQuery, params) if err != nil { return postgresExplain{}, err } - var plan []string - if err := s.db.ReadTransaction(ctx, func(tx graph.Transaction) error { + var ( + plan []string + planJSON json.RawMessage + ) + runExplain := func(tx graph.Transaction) error { result := tx.Raw("EXPLAIN (ANALYZE, BUFFERS, TIMING OFF) "+sqlQuery, translation.Parameters) defer result.Close() @@ -201,25 +1039,125 @@ func (s *postgresSQLRunner) explain(ctx context.Context, cypherQuery string, par plan = append(plan, fmt.Sprint(values[0])) } - return result.Error() - }); err != nil { - return postgresExplain{}, err + if err := result.Error(); err != nil { + return err + } + if !write { + jsonResult := tx.Raw("EXPLAIN (ANALYZE, BUFFERS, WAL, SETTINGS, TIMING ON, FORMAT JSON) "+sqlQuery, translation.Parameters) + defer jsonResult.Close() + if jsonResult.Next() && len(jsonResult.Values()) > 0 { + planJSON, err = encodePostgresPlanJSON(jsonResult.Values()[0]) + if err != nil { + return err + } + } + if err := jsonResult.Error(); err != nil { + return err + } + } + if write { + return errScaleWriteRollback + } + return nil + } + + var explainErr error + if write { + explainErr = s.db.WriteTransaction(ctx, runExplain) + if errors.Is(explainErr, errScaleWriteRollback) { + explainErr = nil + } + } else if readOptions := s.readTransactionOptions(); len(readOptions) > 0 { + explainErr = s.db.ReadTransaction(ctx, runExplain, readOptions...) + } else { + explainErr = s.db.ReadTransaction(ctx, runExplain) + } + if explainErr != nil { + return postgresExplain{}, explainErr } + metrics := parsePostgresPlanMetrics(plan) + if len(planJSON) > 0 { + if structured, err := parsePostgresPlanJSONMetrics(planJSON); err == nil { + metrics = structured + } + } return postgresExplain{ SQL: sqlQuery, Plan: plan, - Metrics: parsePostgresPlanMetrics(plan), + PlanJSON: planJSON, + Metrics: metrics, Optimization: translation.Optimization, + Parameters: translation.Parameters, }, nil } +// translateCypher parses and translates Cypher, applying forced tool options when configured. +func (s *postgresSQLRunner) translateCypher(ctx context.Context, cypherQuery string, params map[string]any) (translate.Result, string, error) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), cypherQuery) + if err != nil { + return translate.Result{}, "", err + } + + var translation translate.Result + if s.productionManifest != nil { + options, optionsErr := s.productionOptions(cypherQuery) + if optionsErr != nil { + return translate.Result{}, "", optionsErr + } + translation, err = translate.TranslateWithProductionOptions(ctx, regularQuery, s.pgDriver.KindMapper(), params, s.graphID, options) + } else if !hasForcedToolOptions(s.toolOptions) { + translation, err = translate.Translate(ctx, regularQuery, s.pgDriver.KindMapper(), params, s.graphID) + } else { + translation, err = translate.TranslateForTool(ctx, regularQuery, s.pgDriver.KindMapper(), params, s.graphID, s.toolOptions) + } + if err != nil { + return translate.Result{}, "", err + } + + sqlQuery, err := translate.Translated(translation) + if err != nil { + return translate.Result{}, "", err + } + return translation, sqlQuery, nil +} + +// hasForcedToolOptions reports whether either executor-selection override is configured. +func hasForcedToolOptions(options translate.ToolOptions) bool { + return options.ForceShortestPathExecutor != "" || options.ForceExpansionSearchStrategy != "" || + options.EnableExpansionOrientationTournament || options.EnableExpansionOrientationShadow || + options.ExpansionOrientationPolicy != "" +} + +// encodePostgresPlanJSON normalizes byte, string, or structured EXPLAIN JSON into json.RawMessage. +func encodePostgresPlanJSON(value any) (json.RawMessage, error) { + switch typed := value.(type) { + case []byte: + return append(json.RawMessage(nil), typed...), nil + case string: + return append(json.RawMessage(nil), typed...), nil + default: + encoded, err := json.Marshal(value) + if err != nil { + return nil, err + } + + return json.RawMessage(encoded), nil + } +} + var ( - postgresPlanningPattern = regexp.MustCompile(`Planning Time: ([0-9.]+) ms`) + // postgresPlanningPattern extracts milliseconds from a PostgreSQL Planning Time summary line. + postgresPlanningPattern = regexp.MustCompile(`Planning Time: ([0-9.]+) ms`) + + // postgresExecutionPattern extracts milliseconds from a PostgreSQL Execution Time summary line. postgresExecutionPattern = regexp.MustCompile(`Execution Time: ([0-9.]+) ms`) - postgresBufferPattern = regexp.MustCompile(`(?:(shared|temp) )?(hit|read|dirtied|written)=([0-9]+)`) + + // postgresBufferPattern extracts storage class, operation, and page count from PostgreSQL buffer counters. + postgresBufferPattern = regexp.MustCompile(`(?:(shared|local|temp) )?(hit|read|dirtied|written)=([0-9]+)`) ) +// parsePostgresPlanMetrics extracts planning, execution, and buffer counters from PostgreSQL text-plan lines. func parsePostgresPlanMetrics(plan []string) PostgresPlanMetrics { var metrics PostgresPlanMetrics for _, line := range plan { @@ -247,6 +1185,7 @@ func parsePostgresPlanMetrics(plan []string) PostgresPlanMetrics { return metrics } +// parsePostgresBuffers extracts shared, local, and temporary buffer counters from one plan line. func parsePostgresBuffers(line string) Buffers { var ( buffers Buffers @@ -270,6 +1209,16 @@ func parsePostgresBuffers(line string) Buffers { buffers.SharedRead = value case "shared_dirtied": buffers.SharedDirtied = value + case "shared_written": + buffers.SharedWritten = value + case "local_hit": + buffers.LocalHit = value + case "local_read": + buffers.LocalRead = value + case "local_dirtied": + buffers.LocalDirtied = value + case "local_written": + buffers.LocalWritten = value case "temp_read": buffers.TempRead = value case "temp_written": diff --git a/cmd/graphbench/postgres_plan.go b/cmd/graphbench/postgres_plan.go new file mode 100644 index 00000000..92fa7f76 --- /dev/null +++ b/cmd/graphbench/postgres_plan.go @@ -0,0 +1,190 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "strings" +) + +// parsePostgresPlanJSONMetrics extracts only fields PostgreSQL exposes. Every +// derived counter remains explicitly plan-derived; fixture expectations and +// benchmark-only state diagnostics are recorded elsewhere. +func parsePostgresPlanJSONMetrics(raw json.RawMessage) (PostgresPlanMetrics, error) { + var documents []map[string]any + if err := json.Unmarshal(raw, &documents); err != nil { + return PostgresPlanMetrics{}, fmt.Errorf("decode PostgreSQL JSON plan: %w", err) + } + if len(documents) != 1 { + return PostgresPlanMetrics{}, fmt.Errorf("PostgreSQL JSON plan has %d documents, expected 1", len(documents)) + } + + metrics := PostgresPlanMetrics{Provenance: map[string]string{}} + metrics.PlanningMS = jsonFloatPointer(documents[0]["Planning Time"]) + metrics.ExecutionMS = jsonFloatPointer(documents[0]["Execution Time"]) + if metrics.PlanningMS != nil { + metrics.Provenance["planning_ms"] = "measured_plan_json" + } + if metrics.ExecutionMS != nil { + metrics.Provenance["execution_ms"] = "measured_plan_json" + } + plan, ok := documents[0]["Plan"].(map[string]any) + if !ok { + return PostgresPlanMetrics{}, fmt.Errorf("PostgreSQL JSON plan is missing its root Plan object") + } + walkPostgresPlanNode(plan, &metrics, 0) + if len(metrics.PlanNodes) > 0 { + metrics.Buffers = metrics.PlanNodes[0].Buffers + metrics.Provenance["buffers"] = "measured_plan_json_root_inclusive" + } + return metrics, nil +} + +// walkPostgresPlanNode flattens one EXPLAIN node into aggregate metrics, then recursively visits child plans and CTE subplans. + +func walkPostgresPlanNode(node map[string]any, metrics *PostgresPlanMetrics, parentPlanNodeID int64) { + planNodeID := int64(len(metrics.PlanNodes) + 1) + metric := PostgresPlanNodeMetric{ + PlanNodeID: planNodeID, + ParentPlanNodeID: parentPlanNodeID, + NodeType: jsonString(node["Node Type"]), + ParentRelationship: jsonString(node["Parent Relationship"]), + CTEName: jsonString(node["CTE Name"]), + RelationName: jsonString(node["Relation Name"]), + Alias: jsonString(node["Alias"]), + IndexName: jsonString(node["Index Name"]), + FunctionName: jsonString(node["Function Name"]), + SubplanName: jsonString(node["Subplan Name"]), + PlanRows: jsonInt64(node["Plan Rows"]), + PlanWidth: jsonInt64(node["Plan Width"]), + ActualRows: jsonInt64(node["Actual Rows"]), + ActualLoops: jsonInt64(node["Actual Loops"]), + RowsRemovedByFilter: jsonInt64(node["Rows Removed by Filter"]), + ActualTotalMS: jsonFloat64(node["Actual Total Time"]), + Buffers: postgresJSONBuffers(node), + Provenance: "measured_plan_json", + } + metrics.PlanNodes = append(metrics.PlanNodes, metric) + + rows := metric.ActualRows * metric.ActualLoops + lowerIdentity := strings.ToLower(strings.Join([]string{metric.NodeType, metric.CTEName, metric.RelationName, metric.Alias, metric.IndexName, metric.FunctionName, metric.SubplanName, jsonString(node["Index Cond"])}, " ")) + if strings.Contains(lowerIdentity, "endpoint_seeded_endpoints") && rows > metrics.EndpointProbeRows { + metrics.EndpointProbeRows = rows + metrics.EndpointGuardOverflow = rows >= 33 + metrics.Provenance["endpoint_probe_rows"] = "plan_derived_endpoint_seed_cte_rows" + } + if strings.Contains(lowerIdentity, "endpoint_seeded_states") && rows > metrics.ReverseStateProbeRows { + metrics.ReverseStateProbeRows = rows + metrics.StateGuardOverflow = rows >= 4097 + metrics.Provenance["reverse_state_probe_rows"] = "plan_derived_reverse_state_probe_cte_rows" + } + if strings.Contains(lowerIdentity, "endpoint_seeded_incumbent") && metric.ActualLoops > 0 { + metrics.ExpansionFallbackExecuted = true + metrics.Provenance["expansion_fallback_executed"] = "plan_derived_incumbent_cte_scan_loops" + } + if strings.Contains(lowerIdentity, "recursive union") { + metrics.RecursiveRows += rows + metrics.RecursiveLoops += metric.ActualLoops + metrics.Provenance["recursive_rows"] = "measured_plan_json" + metrics.Provenance["recursive_loops"] = "measured_plan_json" + } + for identity, target := range map[string]*int64{ + "frontier": &metrics.FrontierRows, + "witness": &metrics.WitnessRows, + "meeting": &metrics.MeetingRows, + } { + if strings.Contains(lowerIdentity, identity) { + *target += rows + metrics.Provenance[identity+"_rows"] = "plan_derived_labeled_state_rows" + } + } + if strings.Contains(lowerIdentity, "hydrated") || strings.Contains(lowerIdentity, "materializ") { + metrics.HydrationRows += rows + metrics.Provenance["hydration_rows"] = "plan_derived_labeled_state_rows" + } + if metric.CTEName == "roots" || (strings.Contains(lowerIdentity, " roots") && strings.Contains(lowerIdentity, "cte scan")) { + metrics.RootRows += rows + metrics.Provenance["root_rows"] = "measured_plan_json" + } + if strings.Contains(lowerIdentity, "edge") && strings.Contains(lowerIdentity, "start_id") { + metrics.ForwardEdgeProbes += metric.ActualLoops + metrics.Provenance["forward_edge_probes"] = "plan_derived_index_loops" + } + if strings.Contains(lowerIdentity, "edge") && strings.Contains(lowerIdentity, "end_id") { + metrics.ReverseEdgeProbes += metric.ActualLoops + metrics.Provenance["reverse_edge_probes"] = "plan_derived_index_loops" + } + if metric.RelationName == "node" || strings.HasPrefix(metric.RelationName, "node_") { + switch { + case strings.Contains(strings.ToLower(metric.Alias), "root"): + metrics.RootLookupLoops += metric.ActualLoops + metrics.Provenance["root_lookup_loops"] = "plan_derived_alias_loops" + case strings.Contains(strings.ToLower(metric.Alias), "boundary") || strings.Contains(strings.ToLower(metric.Alias), "next"): + metrics.BoundaryLookupLoops += metric.ActualLoops + metrics.Provenance["boundary_lookup_loops"] = "plan_derived_alias_loops" + default: + metrics.HydrationLoops += metric.ActualLoops + metrics.Provenance["hydration_loops"] = "plan_derived_node_relation_loops" + } + } + metrics.WALRecords += jsonInt64(node["WAL Records"]) + metrics.WALBytes += jsonInt64(node["WAL Bytes"]) + + children, _ := node["Plans"].([]any) + for _, child := range children { + if childNode, ok := child.(map[string]any); ok { + walkPostgresPlanNode(childNode, metrics, planNodeID) + } + } +} + +// postgresJSONBuffers converts optional JSON buffer counters to integer metrics. +func postgresJSONBuffers(node map[string]any) Buffers { + return Buffers{ + SharedHit: jsonInt64(node["Shared Hit Blocks"]), + SharedRead: jsonInt64(node["Shared Read Blocks"]), + SharedDirtied: jsonInt64(node["Shared Dirtied Blocks"]), + SharedWritten: jsonInt64(node["Shared Written Blocks"]), + LocalHit: jsonInt64(node["Local Hit Blocks"]), + LocalRead: jsonInt64(node["Local Read Blocks"]), + LocalDirtied: jsonInt64(node["Local Dirtied Blocks"]), + LocalWritten: jsonInt64(node["Local Written Blocks"]), + TempRead: jsonInt64(node["Temp Read Blocks"]), + TempWritten: jsonInt64(node["Temp Written Blocks"]), + } +} + +// jsonFloatPointer decodes a JSON number as an optional floating-point value. +func jsonFloatPointer(value any) *float64 { + if value == nil { + return nil + } + parsed := jsonFloat64(value) + return &parsed +} + +// jsonFloat64 decodes a JSON number as a floating-point value, returning zero when absent or invalid. +func jsonFloat64(value any) float64 { + switch typed := value.(type) { + case float64: + return typed + case json.Number: + parsed, _ := typed.Float64() + return parsed + default: + return 0 + } +} + +// jsonInt64 decodes a JSON number as an integer, returning zero when absent or invalid. +func jsonInt64(value any) int64 { return int64(jsonFloat64(value)) } + +// jsonString decodes a JSON string, returning an empty string for other values. +func jsonString(value any) string { + valueString, _ := value.(string) + return valueString +} diff --git a/cmd/graphbench/postgres_plan_test.go b/cmd/graphbench/postgres_plan_test.go new file mode 100644 index 00000000..71bf86bc --- /dev/null +++ b/cmd/graphbench/postgres_plan_test.go @@ -0,0 +1,112 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestParsePostgresPlanJSONMetricsWalksStructuredNodes verifies extraction of root timings, buffer use, recursive cardinality, labeled CTE rows, index probes, and provenance from nested plan JSON. +func TestParsePostgresPlanJSONMetricsWalksStructuredNodes(t *testing.T) { + raw := json.RawMessage(`[{ + "Plan": { + "Node Type": "Recursive Union", "Plan Rows": 12, "Plan Width": 64, + "Actual Rows": 19, "Actual Loops": 1, "Shared Hit Blocks": 40, + "Plans": [ + {"Node Type":"CTE Scan", "CTE Name":"roots", "Alias":"roots", "Actual Rows":1, "Actual Loops":1}, + {"Node Type":"Index Only Scan", "Relation Name":"edge_1", "Alias":"e", "Index Name":"edge_1_end_id_kind_id_idx", "Index Cond":"(end_id = reverse_trails.node_id)", "Actual Rows":1, "Actual Loops":18, "Shared Hit Blocks":36}, + {"Node Type":"Index Scan", "Relation Name":"node_1", "Alias":"boundary", "Actual Rows":2, "Actual Loops":1} + ] + }, + "Planning Time": 1.25, + "Execution Time": 2.5 +}]`) + + metrics, err := parsePostgresPlanJSONMetrics(raw) + require.NoError(t, err) + require.Equal(t, 1.25, *metrics.PlanningMS) + require.Equal(t, 2.5, *metrics.ExecutionMS) + require.Equal(t, int64(40), metrics.Buffers.SharedHit) + require.Equal(t, int64(19), metrics.RecursiveRows) + require.Equal(t, int64(18), metrics.ReverseEdgeProbes) + require.Equal(t, int64(1), metrics.RootRows) + require.Equal(t, int64(1), metrics.BoundaryLookupLoops) + require.Len(t, metrics.PlanNodes, 4) + require.Equal(t, int64(1), metrics.PlanNodes[0].PlanNodeID) + require.Zero(t, metrics.PlanNodes[0].ParentPlanNodeID) + for idx := 1; idx < len(metrics.PlanNodes); idx++ { + require.Equal(t, int64(idx+1), metrics.PlanNodes[idx].PlanNodeID) + require.Equal(t, int64(1), metrics.PlanNodes[idx].ParentPlanNodeID) + } + require.Equal(t, "measured_plan_json", metrics.PlanNodes[0].Provenance) + require.Equal(t, "plan_derived_index_loops", metrics.Provenance["reverse_edge_probes"]) +} + +// TestParsePostgresPlanJSONMetricsRejectsMissingPlan verifies that timing metadata alone is not accepted as a PostgreSQL execution plan. +func TestParsePostgresPlanJSONMetricsRejectsMissingPlan(t *testing.T) { + _, err := parsePostgresPlanJSONMetrics(json.RawMessage(`[{"Planning Time":1}]`)) + require.ErrorContains(t, err, "missing its root Plan") +} + +func TestParsePostgresPlanJSONMetricsRetainsDirectPlanParentage(t *testing.T) { + raw := json.RawMessage(`[{ + "Plan": {"Node Type":"Append","Actual Rows":1,"Actual Loops":1,"Plans":[ + {"Node Type":"Nested Loop","Parent Relationship":"InitPlan","Subplan Name":"CTE asp_i1_candidate_rows","Actual Rows":1,"Actual Loops":1,"Plans":[ + {"Node Type":"CTE Scan","Parent Relationship":"Outer","CTE Name":"asp_i1_candidate_marker","Actual Rows":1,"Actual Loops":1}, + {"Node Type":"Result","Parent Relationship":"Inner","Actual Rows":1,"Actual Loops":1,"Plans":[ + {"Node Type":"Function Scan","Parent Relationship":"Outer","Function Name":"shortest_path_compact","Actual Rows":1,"Actual Loops":1} + ]} + ]} + ]} +}]`) + + metrics, err := parsePostgresPlanJSONMetrics(raw) + require.NoError(t, err) + require.Len(t, metrics.PlanNodes, 5) + require.Equal(t, int64(2), metrics.PlanNodes[1].PlanNodeID) + require.Equal(t, int64(1), metrics.PlanNodes[1].ParentPlanNodeID) + require.Equal(t, int64(2), metrics.PlanNodes[2].ParentPlanNodeID) + require.Equal(t, "Outer", metrics.PlanNodes[2].ParentRelationship) + require.Equal(t, int64(2), metrics.PlanNodes[3].ParentPlanNodeID) + require.Equal(t, "Inner", metrics.PlanNodes[3].ParentRelationship) + require.Equal(t, int64(4), metrics.PlanNodes[4].ParentPlanNodeID) +} + +// TestParsePostgresPlanJSONMetricsAttributesLabeledS4State verifies that repeated frontier loops and labeled witness, meeting, and hydration nodes populate their dedicated counters. +func TestParsePostgresPlanJSONMetricsAttributesLabeledS4State(t *testing.T) { + raw := json.RawMessage(`[{"Plan":{"Node Type":"Result","Actual Rows":1,"Actual Loops":1,"Plans":[ + {"Node Type":"CTE Scan","CTE Name":"forward_frontier","Actual Rows":3,"Actual Loops":2}, + {"Node Type":"CTE Scan","CTE Name":"selected_witness","Actual Rows":4,"Actual Loops":1}, + {"Node Type":"CTE Scan","CTE Name":"shortest_meeting","Actual Rows":1,"Actual Loops":1}, + {"Node Type":"Subquery Scan","Alias":"m0_hydrated","Actual Rows":5,"Actual Loops":1} + ]}}]`) + metrics, err := parsePostgresPlanJSONMetrics(raw) + require.NoError(t, err) + require.Equal(t, int64(6), metrics.FrontierRows) + require.Equal(t, int64(4), metrics.WitnessRows) + require.Equal(t, int64(1), metrics.MeetingRows) + require.Equal(t, int64(5), metrics.HydrationRows) + require.Equal(t, "plan_derived_labeled_state_rows", metrics.Provenance["witness_rows"]) +} + +// TestParsePostgresPlanJSONMetricsAttributesEndpointGuardState verifies endpoint/state guard overflow detection and fallback attribution from labeled seeded-search CTEs. +func TestParsePostgresPlanJSONMetricsAttributesEndpointGuardState(t *testing.T) { + raw := json.RawMessage(`[{"Plan":{"Node Type":"Result","Actual Rows":1,"Actual Loops":1,"Plans":[ + {"Node Type":"CTE Scan","CTE Name":"s4_endpoint_seeded_endpoints","Actual Rows":33,"Actual Loops":1}, + {"Node Type":"CTE Scan","CTE Name":"s4_endpoint_seeded_states","Actual Rows":4097,"Actual Loops":1}, + {"Node Type":"CTE Scan","CTE Name":"s4_endpoint_seeded_incumbent","Actual Rows":10,"Actual Loops":1} + ]}}]`) + metrics, err := parsePostgresPlanJSONMetrics(raw) + require.NoError(t, err) + require.Equal(t, int64(33), metrics.EndpointProbeRows) + require.Equal(t, int64(4097), metrics.ReverseStateProbeRows) + require.True(t, metrics.EndpointGuardOverflow) + require.True(t, metrics.StateGuardOverflow) + require.True(t, metrics.ExpansionFallbackExecuted) +} diff --git a/cmd/graphbench/postgres_test.go b/cmd/graphbench/postgres_test.go index 54470e60..de675294 100644 --- a/cmd/graphbench/postgres_test.go +++ b/cmd/graphbench/postgres_test.go @@ -17,13 +17,196 @@ package main import ( + "encoding/json" + "os" + "path/filepath" + "strings" "testing" + "time" + "github.com/jackc/pgx/v5" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/drivers/pg" "github.com/specterops/dawgs/graph" "github.com/specterops/dawgs/opengraph" + "github.com/specterops/dawgs/testutil" "github.com/stretchr/testify/require" ) +func TestPostgresProductionManifestBuildsExactGuardedOptions(t *testing.T) { + query := "MATCH p = allShortestPaths((s)-[:Traverse*1..8]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p" + digest := strings.Repeat("0", 64) + manifest := PromotionManifest{ + Version: promotionManifestVersion, Candidate: "ASP-I1-U-DAG+MAT-M0", SelectorVersion: "asp-i1-test-v1", + ExecutionBoundary: "guarded_dual_arm", FallbackExecutor: "ASP-A1-DAG", + SourceCommit: "commit", SourceSHA256: digest, BinarySHA256: digest, CorpusSHA256: digest, + Caps: map[string]int64{"state_limit": 10, "predecessor_limit": 20, "enumeration_limit": 30, "output_bytes_limit": 40}, + Buckets: []PromotionBucket{{Name: "outbound-depth8", QuerySHA256: []string{pg.TraversalPolicyQuerySHA256(query)}, Direction: "outbound", ObservationMode: "all_paths", MinimumDepth: 1, MaximumDepth: 8, RelationshipKindCount: 1, QualificationSplit: []string{"training", "holdout"}}}, + } + raw, err := json.Marshal(manifest) + require.NoError(t, err) + path := filepath.Join(t.TempDir(), "manifest.json") + require.NoError(t, os.WriteFile(path, raw, 0o600)) + + runner := &postgresSQLRunner{} + require.NoError(t, runner.setProductionManifest(path)) + options, err := runner.productionOptions(query) + require.NoError(t, err) + require.Equal(t, "ASP-I1-U-DAG+MAT-M0", string(options.ShortestPathExecutor)) + require.Equal(t, int64(10), options.ShortestPathCaps.StateLimit) + require.Equal(t, int64(8), options.AuthorizedBucket.MaximumDepth) + require.Equal(t, "asp-i1-test-v1", options.SelectorVersion) + _, err = runner.productionOptions(query + " RETURN 1") + require.ErrorContains(t, err, "absent from the provisional production manifest") +} + +func TestPostgresProductionManifestRequiresStaticV6CanonicalInboundBucket(t *testing.T) { + query := "MATCH p = shortestPath((s)<-[:Traverse*1..64]-(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p" + digest := strings.Repeat("0", 64) + base := PromotionManifest{ + Version: promotionManifestVersion, Candidate: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + SelectorVersion: optimize.ShortestPathSelectorStaticV6, ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: string(optimize.ShortestPathExecutorS4CanonicalWitness), + SourceCommit: "commit", SourceSHA256: digest, BinarySHA256: digest, CorpusSHA256: digest, + Caps: map[string]int64{"state_limit": 10, "predecessor_limit": 20, "enumeration_limit": 30, "output_bytes_limit": 40}, + Buckets: []PromotionBucket{{ + Name: "canonical-inbound-depth64", QuerySHA256: []string{pg.TraversalPolicyQuerySHA256(query)}, Direction: "inbound", + ObservationMode: "one_path", MinimumDepth: 1, MaximumDepth: 64, RelationshipKindCount: 1, + QualificationSplit: []string{"training", "holdout"}, + }}, + } + write := func(t *testing.T, manifest PromotionManifest) string { + t.Helper() + raw, err := json.Marshal(manifest) + require.NoError(t, err) + path := filepath.Join(t.TempDir(), "manifest.json") + require.NoError(t, os.WriteFile(path, raw, 0o600)) + return path + } + + runner := &postgresSQLRunner{} + require.NoError(t, runner.setProductionManifest(write(t, base))) + options, err := runner.productionOptions(query) + require.NoError(t, err) + require.Equal(t, optimize.ShortestPathSelectorStaticV6, options.SelectorVersion) + require.Equal(t, int64(64), options.AuthorizedBucket.MaximumDepth) + + tests := map[string]func(*PromotionManifest){ + "selector": func(manifest *PromotionManifest) { manifest.SelectorVersion = "sp-static-v5-contained" }, + "outbound": func(manifest *PromotionManifest) { manifest.Buckets[0].Direction = "outbound" }, + "maximum": func(manifest *PromotionManifest) { manifest.Buckets[0].MaximumDepth = 63 }, + "kinds": func(manifest *PromotionManifest) { manifest.Buckets[0].RelationshipKindCount = 2 }, + } + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + manifest := base + manifest.Caps = clonePromotionCaps(base.Caps) + manifest.Buckets = clonePromotionBuckets(base.Buckets) + mutate(&manifest) + require.Error(t, (&postgresSQLRunner{}).setProductionManifest(write(t, manifest))) + }) + } +} + +func TestPostgresProductionManifestBuildsOrientationOptionsWithoutShortestPathFields(t *testing.T) { + query := "MATCH (r)-[:Expand*0..16]->()-[:Suffix]->(e) WHERE id(r) = $root_id RETURN id(e)" + digest := strings.Repeat("0", 64) + manifest := PromotionManifest{ + Version: promotionManifestVersion, Candidate: string(optimize.ExpansionSearchPolicyOrientationProbeV1), SelectorVersion: "orientation-probe-v1", + ExecutionBoundary: "guarded_dual_arm", FallbackExecutor: string(optimize.ExpansionSearchStepwiseForward), + SourceCommit: "commit", SourceSHA256: digest, BinarySHA256: digest, CorpusSHA256: digest, + Caps: orientationPromotionCaps(), + Buckets: []PromotionBucket{{ + Name: "outbound-fixed-suffix", QuerySHA256: []string{pg.TraversalPolicyQuerySHA256(query)}, Direction: "outbound", + ObservationMode: "endpoint_ids", MinimumDepth: 0, MaximumDepth: 16, RelationshipKindCount: 1, + QualificationSplit: []string{"training", "holdout"}, + }}, + } + raw, err := json.Marshal(manifest) + require.NoError(t, err) + path := filepath.Join(t.TempDir(), "manifest.json") + require.NoError(t, os.WriteFile(path, raw, 0o600)) + + runner := &postgresSQLRunner{} + require.NoError(t, runner.setProductionManifest(path)) + options, err := runner.productionOptions(query) + require.NoError(t, err) + require.True(t, options.EnableExpansionOrientation) + require.Empty(t, options.ShortestPathExecutor) + require.Nil(t, options.ShortestPathCaps) + require.Equal(t, int64(16), options.AuthorizedBucket.MaximumDepth) + require.Equal(t, "orientation-probe-v1", options.SelectorVersion) +} + +func TestPostgresProductionManifestRejectsNonExactOrientationContract(t *testing.T) { + digest := strings.Repeat("0", 64) + base := PromotionManifest{ + Version: promotionManifestVersion, Candidate: string(optimize.ExpansionSearchPolicyOrientationProbeV1), SelectorVersion: "orientation-probe-v1", + ExecutionBoundary: "guarded_dual_arm", FallbackExecutor: string(optimize.ExpansionSearchStepwiseForward), + SourceCommit: "commit", SourceSHA256: digest, BinarySHA256: digest, CorpusSHA256: digest, + Caps: orientationPromotionCaps(), + Buckets: []PromotionBucket{{ + Name: "fixed-suffix", QuerySHA256: []string{digest}, QualificationSplit: []string{"training", "holdout"}, + }}, + } + tests := []struct { + name string + mutate func(*PromotionManifest) + err string + }{ + { + name: "fallback", mutate: func(manifest *PromotionManifest) { manifest.FallbackExecutor = "EXPANSION-SUFFIX-SEEDED-REVERSE" }, + err: "unsupported candidate/fallback pair", + }, + { + name: "extra cap", mutate: func(manifest *PromotionManifest) { manifest.Caps["extra_limit"] = 1 }, + err: "orientation-probe-v1 requires exactly four immutable caps", + }, + { + name: "missing cap", mutate: func(manifest *PromotionManifest) { delete(manifest.Caps, "root_row_limit") }, + err: "orientation-probe-v1 requires exactly four immutable caps", + }, + { + name: "wrong cap", mutate: func(manifest *PromotionManifest) { manifest.Caps["state_limit"]-- }, + err: "orientation-probe-v1 cap state_limit must equal 4096", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + manifest := base + manifest.Caps = clonePromotionCaps(base.Caps) + test.mutate(&manifest) + raw, err := json.Marshal(manifest) + require.NoError(t, err) + path := filepath.Join(t.TempDir(), "manifest.json") + require.NoError(t, os.WriteFile(path, raw, 0o600)) + err = (&postgresSQLRunner{}).setProductionManifest(path) + require.ErrorContains(t, err, test.err) + }) + } +} + +func TestPostgresReadTransactionOptionsMatchEveryStableSnapshotMode(t *testing.T) { + require.Empty(t, (&postgresSQLRunner{}).readTransactionOptions()) + + for name, runner := range map[string]*postgresSQLRunner{ + "explicit benchmark flag": {repeatableRead: true}, + "production manifest": {productionManifest: &PromotionManifest{}}, + } { + t.Run(name, func(t *testing.T) { + options := runner.readTransactionOptions() + require.Len(t, options, 1) + + pgConfig := &pg.Config{} + transactionConfig := &graph.TransactionConfig{DriverConfig: pgConfig} + options[0](transactionConfig) + require.Equal(t, pgx.RepeatableRead, pgConfig.Options.IsoLevel) + require.Equal(t, pgx.ReadWrite, pgConfig.Options.AccessMode) + }) + } +} + +// TestResolveCaseParams verifies that scalar, explicit-list, and generated-list fixture keys become ordered int64 IDs without disturbing ordinary parameters. func TestResolveCaseParams(t *testing.T) { params, err := resolveCaseParams(ScaleCase{ Params: map[string]any{ @@ -32,19 +215,52 @@ func TestResolveCaseParams(t *testing.T) { NodeParams: map[string]string{ "start_id": "n1", }, - }, opengraph.IDMap{"n1": graph.ID(42)}) + NodeListParams: map[string][]string{ + "end_ids": {"n2", "n1"}, + }, + GeneratedNodeListParams: map[string]testutil.GeneratedNodeListParam{ + "generated_ids": { + Prefix: "generated", + Count: 2, + Include: []string{"n2"}, + }, + }, + }, opengraph.IDMap{ + "n1": graph.ID(42), + "n2": graph.ID(84), + "generated-00": graph.ID(126), + "generated-01": graph.ID(168), + }) require.NoError(t, err) require.Equal(t, map[string]any{ - "name": "value", - "start_id": int64(42), + "name": "value", + "start_id": int64(42), + "end_ids": []int64{84, 42}, + "generated_ids": []int64{84, 126, 168}, }, params) } +// TestScaleCaseDecodesTypedDatetimeParameter verifies that the corpus JSON datetime envelope becomes a UTC time value rather than an untyped map. +func TestScaleCaseDecodesTypedDatetimeParameter(t *testing.T) { + var testCase ScaleCase + require.NoError(t, json.Unmarshal([]byte(`{ + "name":"typed-time", + "dataset":"base", + "category":"lookup", + "cypher":"MATCH (n) WHERE n.lastseen < $threshold RETURN n", + "params":{"threshold":{"$type":"datetime","value":"2026-01-02T03:04:05Z"}}, + "candidate_modes":["postgres_sql"] + }`), &testCase)) + + require.Equal(t, time.Date(2026, time.January, 2, 3, 4, 5, 0, time.UTC), testCase.Params["threshold"]) +} + +// TestParsePostgresPlanMetrics verifies parsing of planning/execution milliseconds and every shared, local, and temporary buffer counter from text plans. func TestParsePostgresPlanMetrics(t *testing.T) { metrics := parsePostgresPlanMetrics([]string{ "Nested Loop (actual rows=1 loops=1)", - " Buffers: shared hit=12 read=3 dirtied=2, temp read=4 written=5", + " Buffers: shared hit=12 read=3 dirtied=2 written=1, local hit=7 read=6 dirtied=5 written=4, temp read=3 written=2", "Planning Time: 1.250 ms", "Execution Time: 9.750 ms", }) @@ -57,7 +273,48 @@ func TestParsePostgresPlanMetrics(t *testing.T) { SharedHit: 12, SharedRead: 3, SharedDirtied: 2, - TempRead: 4, - TempWritten: 5, + SharedWritten: 1, + LocalHit: 7, + LocalRead: 6, + LocalDirtied: 5, + LocalWritten: 4, + TempRead: 3, + TempWritten: 2, }, metrics.Buffers) } + +// TestGeneratedDatasetVariantsAreParameterizedAndRepeatable verifies deterministic generation for equal names and propagation of configured payload size into fixed-suffix nodes. +func TestGeneratedDatasetVariantsAreParameterizedAndRepeatable(t *testing.T) { + first := generatedDataset("generated_shortest_paths_d4_f16") + second := generatedDataset("generated_shortest_paths_d4_f16") + require.NotNil(t, first) + require.Equal(t, first, second) + + fixedSuffix := generatedDataset("generated_fixed_suffix_expansion_d2_f10_v2_p4096") + require.NotNil(t, fixedSuffix) + require.Contains(t, fixedSuffix.Nodes[0].Properties["payload"], "xxxx") +} + +// TestCompactBidirectionalRunsRequireRepeatableSnapshot verifies runner setup +// opts into stable snapshots exactly when a forced or reference B1/B2 arm can run. +func TestCompactBidirectionalRunsRequireRepeatableSnapshot(t *testing.T) { + require.True(t, compactBidirectionalSnapshotRequired(false, nil, "SP-B1-C-ALT-NODE-D")) + require.True(t, compactBidirectionalSnapshotRequired(false, nil, "SP-B2-C-MIN-LEVEL-WE+MAT-M0")) + require.True(t, compactBidirectionalSnapshotRequired(false, nil, "ASP-B1-DAG-ALT-NODE")) + require.True(t, compactBidirectionalSnapshotRequired(false, nil, "ASP-B2-DAG-MIN-LEVEL")) + require.True(t, compactBidirectionalSnapshotRequired(true, nil, "")) + require.True(t, compactBidirectionalSnapshotRequired(true, []string{"sp_b1_strict_alternating_distance"}, "")) + require.True(t, compactBidirectionalSnapshotRequired(true, []string{"asp_b2_bidirectional_dag_smaller_frontier_m0"}, "")) + require.False(t, compactBidirectionalSnapshotRequired(false, nil, "SP-S4-C-D")) + require.False(t, compactBidirectionalSnapshotRequired(true, []string{"s4_canonical_source_distance"}, "")) +} + +// TestFixtureMetadataIncludesCardinalityAndChecksum verifies that generated fixtures expose their configuration, nonzero entity counts, and a full SHA-256 content digest. +func TestFixtureMetadataIncludesCardinalityAndChecksum(t *testing.T) { + metadata, err := fixtureMetadata("unused", "generated_shortest_paths_d4_f16") + require.NoError(t, err) + require.Equal(t, "generated_shortest_paths_d4_f16", metadata.Configuration) + require.Positive(t, metadata.NodeCount) + require.Positive(t, metadata.EdgeCount) + require.Len(t, metadata.Checksum, 64) +} diff --git a/cmd/graphbench/postgres_timed_attestation.go b/cmd/graphbench/postgres_timed_attestation.go new file mode 100644 index 00000000..ef9c5c04 --- /dev/null +++ b/cmd/graphbench/postgres_timed_attestation.go @@ -0,0 +1,105 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/jackc/pgx/v5/pgxpool" +) + +type postgresTimedRuntimeDocument struct { + SchemaVersion int `json:"schema_version"` + InvocationID string `json:"invocation_id"` + RequestedIdentity string `json:"requested_identity"` + RuntimeIdentity string `json:"runtime_identity"` + RuntimeBranch string `json:"runtime_branch"` + FallbackExecuted *bool `json:"fallback_executed"` + RecordCount int `json:"record_count"` + Events []RuntimeReceiptEvent `json:"events"` +} + +// postgresTimedReadAttestor arms a lightweight session-local receipt before +// each timed query and reads it after the duration has been recorded. A +// size-one pool is required so arming, execution, and reading cannot migrate. +type postgresTimedReadAttestor struct { + pool *pgxpool.Pool + requestedIdentity string + runID string + activeInvocation string +} + +func newPostgresTimedReadAttestor(pool *pgxpool.Pool, poolSize int, requestedIdentity string) (*postgresTimedReadAttestor, error) { + if pool == nil { + return nil, fmt.Errorf("timed runtime attestation requires a PostgreSQL pool") + } + if poolSize != 1 { + return nil, fmt.Errorf("timed runtime attestation requires pool size 1, got %d", poolSize) + } + if strings.TrimSpace(requestedIdentity) == "" { + return nil, fmt.Errorf("timed runtime attestation requires a requested identity") + } + return &postgresTimedReadAttestor{pool: pool, requestedIdentity: requestedIdentity, runID: newRunUUID()}, nil +} + +func (s *postgresTimedReadAttestor) Begin(ctx context.Context, iteration int) error { + if s.activeInvocation != "" { + return fmt.Errorf("runtime attestation %q is still active", s.activeInvocation) + } + s.activeInvocation = fmt.Sprintf("%s-%d", s.runID, iteration) + if _, err := s.pool.Exec(ctx, "select public.begin_traversal_runtime_attestation_v1($1, $2)", s.activeInvocation, s.requestedIdentity); err != nil { + s.activeInvocation = "" + return err + } + return nil +} + +func (s *postgresTimedReadAttestor) Complete(ctx context.Context, _ int) (timedReadAttestation, error) { + invocationID := s.activeInvocation + if invocationID == "" { + return timedReadAttestation{}, fmt.Errorf("no runtime attestation is active") + } + s.activeInvocation = "" + var raw string + readErr := s.pool.QueryRow(ctx, "select coalesce(public.read_traversal_runtime_attestation_v1($1)::text, '')", invocationID).Scan(&raw) + _, clearErr := s.pool.Exec(ctx, "select public.clear_traversal_runtime_attestation_v1($1)", invocationID) + if readErr != nil { + return timedReadAttestation{}, readErr + } + if clearErr != nil { + return timedReadAttestation{}, clearErr + } + if strings.TrimSpace(raw) == "" { + return timedReadAttestation{}, fmt.Errorf("runtime invocation %q produced no receipt", invocationID) + } + var document postgresTimedRuntimeDocument + if err := json.Unmarshal([]byte(raw), &document); err != nil { + return timedReadAttestation{}, fmt.Errorf("decode runtime receipt: %w", err) + } + if document.SchemaVersion != 2 || document.InvocationID != invocationID || document.RequestedIdentity != s.requestedIdentity { + return timedReadAttestation{}, fmt.Errorf("runtime receipt identity does not match its armed invocation") + } + if document.RecordCount < 1 || len(document.Events) != document.RecordCount || document.RuntimeIdentity == "" || document.RuntimeBranch == "" || document.FallbackExecuted == nil { + return timedReadAttestation{}, fmt.Errorf("runtime receipt is incomplete or has a broken event chain: %s", raw) + } + for idx, event := range document.Events { + if event.Ordinal != idx+1 || event.RuntimeIdentity == "" || event.RuntimeBranch == "" { + return timedReadAttestation{}, fmt.Errorf("runtime receipt event chain is not contiguous") + } + document.Events[idx].InvocationID = invocationID + } + return timedReadAttestation{ + InvocationID: invocationID, + RequestedIdentity: document.RequestedIdentity, + RuntimeIdentity: document.RuntimeIdentity, + RuntimeBranch: document.RuntimeBranch, + FallbackExecuted: document.FallbackExecuted, + Events: append([]RuntimeReceiptEvent(nil), document.Events...), + }, nil +} diff --git a/cmd/graphbench/postgres_traversal_telemetry.go b/cmd/graphbench/postgres_traversal_telemetry.go new file mode 100644 index 00000000..bf161083 --- /dev/null +++ b/cmd/graphbench/postgres_traversal_telemetry.go @@ -0,0 +1,2086 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "strconv" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" +) + +const ( + postgresTraversalTelemetryOff = "off" + postgresTraversalTelemetrySummary = "summary" + postgresTraversalTelemetryDiagnostic = "diagnostic" + postgresTraversalPlanReplaySource = "postgres_explain_analyze_json_timing_off" + postgresBidirectionalDiagnosticSource = "public.read_bidirectional_shortest_path_diagnostic_v1" + postgresBidirectionalAllShortestDiagnosticSource = "public.read_bidirectional_all_shortest_path_diagnostic_v1" +) + +// buildPostgresCaseTraversalTelemetry binds optimizer, emitted SQL, and +// separately replayed plan evidence into one validated traversal identity. +// A nil result means the statement has no unambiguous traversal target. +func buildPostgresCaseTraversalTelemetry( + optimization translate.OptimizationSummary, + metrics PostgresPlanMetrics, + connectionID string, + level TraversalTelemetryLevel, +) (*TraversalExecutionTelemetry, error) { + outcome, ok := singleTraversalOutcome(optimization.TargetOutcomes) + if !ok { + return nil, nil + } + + summary, family, err := traversalSummaryFromOutcome(outcome, metrics) + if err != nil { + return nil, err + } + telemetry := newPostgresTraversalTelemetry(summary, family, metrics, connectionID, level) + if functionBackedTraversal(metrics) && isBidirectionalTelemetryIdentity(summary) { + markTraversalSummaryUnavailable(&telemetry, "outer Function Scan does not expose the invocation-local runtime branch") + } + if telemetry.Diagnostic != nil && functionBackedTraversal(metrics) && (family == TraversalTelemetryFamilySP || family == TraversalTelemetryFamilyASP) { + telemetry.Diagnostic.CounterStatus = TraversalTelemetryCounterStatusHiddenUnavailable + telemetry.Diagnostic.IncompleteReasons = []string{"outer Function Scan does not expose invocation-local traversal work counters"} + } + if err := telemetry.Validate(); err != nil { + return nil, err + } + return &telemetry, nil +} + +// buildPostgresReferenceTraversalTelemetry binds an explicit reference +// architecture and implementation to its own untimed JSON EXPLAIN replay. +func buildPostgresReferenceTraversalTelemetry( + reference PostgresReferenceResult, + parameters map[string]any, + connectionID string, + level TraversalTelemetryLevel, +) (*TraversalExecutionTelemetry, error) { + if strings.TrimSpace(reference.Architecture) == "" || strings.TrimSpace(reference.ImplementationID) == "" || reference.PostgresMetrics == nil { + return nil, nil + } + if !isTraversalReferenceArchitecture(reference.Architecture) { + return nil, nil + } + + family := traversalFamilyForIdentity(reference.Architecture, "") + fallback := false + overflow := false + planned := []string{reference.Architecture} + fallbackIdentity := bidirectionalFallbackIdentity(reference.Architecture) + if fallbackIdentity != "" && fallbackIdentity != reference.Architecture { + planned = append(planned, fallbackIdentity) + } + summary := TraversalExecutionSummary{ + RequestedIdentity: reference.Architecture, + PlannedIdentities: planned, + EmittedIdentity: reference.ImplementationID, + RuntimeIdentity: reference.Architecture, + AppliedIdentity: reference.Architecture, + SelectorVersion: "explicit-reference-v1", + SchedulerVersion: schedulerForIdentity(reference.Architecture, ""), + ObservationMode: reference.ObservationShape, + Caps: referenceTraversalCaps(parameters), + RuntimeOutcomeAvailable: traversalTelemetryPointer(true), + RuntimeBranch: "explicit_reference", + Overflow: &overflow, + FallbackExecuted: &fallback, + Provenance: map[string]string{ + "requested_identity": "reference.architecture", + "planned_identities": "reference.architecture", + "emitted_identity": "reference.implementation_id", + "runtime_identity": postgresTraversalPlanReplaySource + ".reference_statement", + "applied_identity": "reference.architecture", + "selector_version": "reference.explicit_selection", + "scheduler_version": "reference.architecture", + "observation_mode": "reference.observation_shape", + "runtime_outcome_available": postgresTraversalPlanReplaySource + ".reference_statement", + "runtime_branch": postgresTraversalPlanReplaySource + ".reference_statement", + "overflow": postgresTraversalPlanReplaySource + ".visible_guards", + "fallback_executed": postgresTraversalPlanReplaySource + ".visible_branches", + }, + } + for name := range summary.Caps { + summary.Provenance["caps."+name] = "reference.parameters." + traversalCapParameterName(name) + } + + telemetry := newPostgresTraversalTelemetry(summary, family, *reference.PostgresMetrics, connectionID, level) + if functionBackedTraversal(*reference.PostgresMetrics) && isBidirectionalTelemetryIdentity(summary) { + markTraversalSummaryUnavailable(&telemetry, "outer Function Scan does not expose the invocation-local runtime branch") + } + if functionBackedTraversal(*reference.PostgresMetrics) && (family == TraversalTelemetryFamilySP || family == TraversalTelemetryFamilyASP) { + if telemetry.Diagnostic != nil { + telemetry.Diagnostic.CounterStatus = TraversalTelemetryCounterStatusHiddenUnavailable + telemetry.Diagnostic.IncompleteReasons = []string{"outer Function Scan does not expose invocation-local traversal work counters"} + } + } + if err := telemetry.Validate(); err != nil { + return nil, err + } + return &telemetry, nil +} + +func isTraversalReferenceArchitecture(identity string) bool { + return strings.HasPrefix(identity, "SP-") || + strings.HasPrefix(identity, "ASP-") || + strings.HasPrefix(identity, "EXPANSION-") || + strings.HasPrefix(identity, "EXPAND-INTO-") || + strings.HasPrefix(identity, "MAT-") || + identity == "hydration" +} + +// newPostgresTraversalTelemetry records either an optimizer/plan-derived +// summary or partial SQL-visible diagnostic evidence. It never converts +// absent executor counters into fabricated zero values. +func newPostgresTraversalTelemetry( + summary TraversalExecutionSummary, + family TraversalTelemetryFamily, + metrics PostgresPlanMetrics, + connectionID string, + level TraversalTelemetryLevel, +) TraversalExecutionTelemetry { + telemetry := TraversalExecutionTelemetry{ + SchemaVersion: TraversalExecutionTelemetrySchemaVersion, + Level: level, + Summary: summary, + } + if level == TraversalTelemetryLevelDiagnostic { + telemetry.Diagnostic = &TraversalExecutionDiagnostic{ + InvocationID: newRunUUID(), + ConnectionID: connectionID, + TimedSample: traversalTelemetryPointer(false), + RequiredFamilies: traversalRequiredFamilies(summary, family), + CounterStatus: TraversalTelemetryCounterStatusPlanPartial, + IncompleteReasons: []string{ + "JSON EXPLAIN exposes SQL plan work but not every qualification counter in the declared family", + }, + PlanReplay: postgresTraversalPlanReplay(metrics), + Provenance: map[string]string{}, + } + } + return telemetry +} + +func singleTraversalOutcome(outcomes []translate.TargetLoweringOutcome) (translate.TargetLoweringOutcome, bool) { + var shortest, expansion []translate.TargetLoweringOutcome + for _, outcome := range outcomes { + if outcome.TargetKind != "" && outcome.TargetKind != "traversal" { + continue + } + if outcome.Family == "SP" || outcome.Family == "ASP" { + shortest = append(shortest, outcome) + } else if strings.Contains(outcome.Family, "expansion") { + expansion = append(expansion, outcome) + } + } + // Shortest-path execution is the public traversal boundary even when its + // underlying variable step also produced ordinary-expansion analysis. + // Analysis-only endpoint/predicate outcomes must never make telemetry + // ambiguous or replace the executor identity. + if len(shortest) == 1 { + return shortest[0], true + } + if len(shortest) != 0 || len(expansion) != 1 { + return translate.TargetLoweringOutcome{}, false + } + return expansion[0], true +} + +func traversalSummaryFromOutcome(outcome translate.TargetLoweringOutcome, metrics PostgresPlanMetrics) (TraversalExecutionSummary, TraversalTelemetryFamily, error) { + requested := outcome.Candidate + if requested == "" { + requested = outcome.Selected + } + applied := outcome.Applied + if applied == "" { + applied = outcome.Fallback + } + if requested == "" || applied == "" { + return TraversalExecutionSummary{}, "", fmt.Errorf("traversal target outcome has no requested or applied identity") + } + + planned := append([]string(nil), outcome.PlannedCandidates...) + for _, identity := range []string{requested, applied, outcome.Fallback} { + if identity != "" && !slices.Contains(planned, identity) { + planned = append(planned, identity) + } + } + emitted := outcome.EmittedPolicy + if emitted == "" { + if len(outcome.EmittedCandidates) > 1 { + emitted = strings.Join(outcome.EmittedCandidates, "+") + } else if len(outcome.EmittedCandidates) == 1 { + emitted = outcome.EmittedCandidates[0] + } else { + emitted = applied + } + } + + runtimeIdentity, runtimeBranch, fallbackExecuted, overflow := runtimeTraversalIdentity(outcome, metrics, requested, applied) + wouldSelectIdentity := "" + if outcome.SelectionMode == "shadow_tool" { + runtimeIdentity = applied + runtimeBranch = "shadow_incumbent" + fallbackExecuted = false + overflow = metrics.EndpointGuardOverflow || metrics.StateGuardOverflow || + orientationPlanOverflow(outcome, postgresTraversalPlanReplay(metrics)) + wouldSelectIdentity = shadowWouldSelectIdentity(outcome, metrics) + } + if outcome.EmittedPolicy != "" { + // Applied is a runtime fact for a same-statement policy; the translator + // can report emitted arms but cannot know which branch executed. + applied = runtimeIdentity + } + if runtimeIdentity != "" && !slices.Contains(planned, runtimeIdentity) { + planned = append(planned, runtimeIdentity) + } + if fallbackExecuted && outcome.Fallback != "" { + applied = outcome.Fallback + runtimeIdentity = outcome.Fallback + } + selectorVersion := outcome.SelectorVersion + if selectorVersion == "" { + selectorVersion = "static-lowering-v1" + } + summary := TraversalExecutionSummary{ + RequestedIdentity: requested, + PlannedIdentities: planned, + EmittedIdentity: emitted, + RuntimeIdentity: runtimeIdentity, + AppliedIdentity: applied, + SelectorVersion: selectorVersion, + SchedulerVersion: schedulerForIdentity(runtimeIdentity, outcome.Scheduler), + ExecutionBoundary: outcome.ExecutionBoundary, + ObservationMode: outcome.ObservationMode, + Caps: outcomeTraversalCaps(outcome), + RuntimeOutcomeAvailable: traversalTelemetryPointer(true), + RuntimeBranch: runtimeBranch, + Overflow: &overflow, + FallbackExecuted: &fallbackExecuted, + WouldSelectIdentity: wouldSelectIdentity, + Provenance: map[string]string{ + "requested_identity": "optimizer.target_outcome.candidate_or_selected", + "planned_identities": "optimizer.target_outcome.planned_candidates", + "emitted_identity": "translator.target_outcome.emitted_policy_or_candidates", + "runtime_identity": postgresTraversalPlanReplaySource + ".visible_branch_and_translator_applied", + "applied_identity": "translator.target_outcome.applied_or_fallback", + "execution_boundary": "optimizer.target_outcome.execution_boundary", + "selector_version": "optimizer.target_outcome.selector_version", + "scheduler_version": "optimizer.target_outcome.scheduler", + "observation_mode": "optimizer.target_outcome.observation_mode", + "runtime_outcome_available": postgresTraversalPlanReplaySource + ".visible_branch", + "runtime_branch": postgresTraversalPlanReplaySource + ".visible_branch", + "overflow": postgresTraversalPlanReplaySource + ".visible_guard", + "fallback_executed": postgresTraversalPlanReplaySource + ".visible_branch", + }, + } + if wouldSelectIdentity != "" { + summary.Provenance["would_select_identity"] = postgresTraversalPlanReplaySource + ".orientation_shadow_marker_rows" + } + for name := range summary.Caps { + summary.Provenance["caps."+name] = "optimizer.target_outcome." + traversalCapOutcomeField(name) + } + if fallbackExecuted { + summary.FallbackIdentity = applied + summary.Provenance["fallback_identity"] = "optimizer.target_outcome.fallback" + } + family := traversalFamilyForIdentity(runtimeIdentity, outcome.Family) + if isOrientationProbePolicy(outcome.EmittedPolicy) || + outcome.EmittedPolicy == string(optimize.ExpansionSearchPolicyEndpointGuardV1) { + family = TraversalTelemetryFamilyOrientation + } + if runtimeIdentity == "" { + telemetry := TraversalExecutionTelemetry{Summary: summary} + markTraversalSummaryUnavailable(&telemetry, "exact executed traversal marker is unavailable") + summary = telemetry.Summary + } + return summary, family, nil +} + +func traversalCapParameterName(counterName string) string { + switch counterName { + case "state_rows": + return "state_limit" + case "frontier_rows", "queue_rows": + return "frontier_limit" + case "predecessor_rows": + return "predecessor_limit" + case "output_rows": + return "enumeration_limit" + case "output_bytes": + return "output_bytes_limit" + default: + return counterName + } +} + +func traversalCapOutcomeField(counterName string) string { + switch counterName { + case "state_rows": + return "state_limit" + case "frontier_rows", "queue_rows": + return "frontier_limit" + case "predecessor_rows": + return "predecessor_limit" + case "endpoint_probe_rows": + return "endpoint_limit" + case "output_rows": + return "enumeration_limit" + case "output_bytes": + return "output_bytes_limit" + default: + return counterName + } +} + +func shadowWouldSelectIdentity(outcome translate.TargetLoweringOutcome, metrics PostgresPlanMetrics) string { + plan := postgresTraversalPlanReplay(metrics) + if plan.Counters["orientation_shadow_reverse_rows"] > 0 { + return outcome.Candidate + } + if plan.Counters["orientation_shadow_forward_rows"] > 0 { + if outcome.Fallback != "" { + return outcome.Fallback + } + return outcome.Applied + } + return "" +} + +func runtimeTraversalIdentity(outcome translate.TargetLoweringOutcome, metrics PostgresPlanMetrics, requested, applied string) (identity, branch string, fallback, overflow bool) { + identity, branch = applied, "selected" + if outcome.EmittedPolicy == "" && outcome.Fallback != "" && requested != applied && applied == outcome.Fallback { + return applied, "compile_time_fallback", true, false + } + overflow = metrics.EndpointGuardOverflow || metrics.StateGuardOverflow + if metrics.ExpansionFallbackExecuted { + identity = outcome.Fallback + if identity == "" { + identity = applied + } + return identity, "runtime_fallback", true, overflow + } + + plan := postgresTraversalPlanReplay(metrics) + if outcome.EmittedPolicy == optimize.ShortestPathPolicyASPI1GuardedV1 { + candidateRows, candidatePresent := plan.Counters["asp_i1_candidate_marker_rows"] + fallbackRows, fallbackPresent := plan.Counters["asp_i1_fallback_marker_rows"] + overflow = aspI1PlanOverflow(outcome, plan) + if !candidatePresent || !fallbackPresent { + return "", "runtime_outcome_unavailable", false, overflow + } + if candidateRows == 1 && fallbackRows == 0 { + return string(optimize.ShortestPathExecutorASPI1DAG), "inline_predecessor_dag", false, false + } + if fallbackRows == 1 && candidateRows == 0 { + return string(optimize.ShortestPathExecutorASPA1DAG), "exact_a1_fallback", true, true + } + return "", "runtime_outcome_unavailable", false, overflow + } + if outcome.EmittedPolicy == optimize.ShortestPathPolicyI1CanonicalGuardedV1 { + candidateRows, candidatePresent := plan.Counters["asp_i1_candidate_marker_rows"] + fallbackRows, fallbackPresent := plan.Counters["asp_i1_fallback_marker_rows"] + overflow = aspI1PlanOverflow(outcome, plan) + if !candidatePresent || !fallbackPresent { + return "", "runtime_outcome_unavailable", false, overflow + } + if candidateRows == 1 && fallbackRows == 0 { + outputRows, outputPresent := plan.Counters["asp_i1_output_rows"] + if !outputPresent { + return "", "runtime_outcome_unavailable", false, false + } + branch := "inline_canonical_witness" + if outputRows == 0 { + branch = "inline_canonical_no_path" + } + return string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), branch, false, false + } + if fallbackRows == 1 && candidateRows == 0 { + return string(optimize.ShortestPathExecutorS4CanonicalWitness), "exact_s4_fallback", true, true + } + return "", "runtime_outcome_unavailable", false, overflow + } + if outcome.EmittedPolicy != "" { + candidateRows := plan.Counters["orientation_executed_candidate_rows"] + incumbentRows := plan.Counters["orientation_executed_incumbent_rows"] + overflow = overflow || orientationPlanOverflow(outcome, plan) + if candidateRows == 1 && incumbentRows == 0 && outcome.Candidate != "" { + if overflow { + return "", "runtime_outcome_unavailable", false, true + } + return outcome.Candidate, "suffix_seeded_reverse", false, false + } + if incumbentRows == 1 && candidateRows == 0 && outcome.Fallback != "" { + return outcome.Fallback, "exact_forward_incumbent", overflow, overflow + } + return "", "runtime_outcome_unavailable", false, overflow + } + return identity, branch, false, overflow +} + +func aspI1PlanOverflow(outcome translate.TargetLoweringOutcome, plan *TraversalPlanReplayEvidence) bool { + for counter, limit := range map[string]int64{ + "asp_i1_distance_rows": outcome.StateLimit, + "asp_i1_predecessor_rows": outcome.PredecessorLimit, + "asp_i1_enumeration_rows": outcome.EnumerationLimit, + } { + if limit > 0 && plan.Counters[counter] > limit { + return true + } + } + return false +} + +func orientationPlanOverflow(outcome translate.TargetLoweringOutcome, plan *TraversalPlanReplayEvidence) bool { + if outcome.StateLimit > 0 && plan.Counters["orientation_state_rows"] > outcome.StateLimit { + return true + } + if outcome.ProbeCaps == nil { + return false + } + for counter, limit := range map[string]int64{ + "orientation_root_probe_rows": outcome.ProbeCaps.RootRowLimit, + "orientation_suffix_probe_rows": outcome.ProbeCaps.ReverseSeedRowLimit, + "orientation_forward_degree_rows": outcome.ProbeCaps.DirectionalDegreeRowLimit, + "orientation_reverse_degree_rows": outcome.ProbeCaps.DirectionalDegreeRowLimit, + } { + if limit > 0 && plan.Counters[counter] > limit { + return true + } + } + return false +} + +func outcomeTraversalCaps(outcome translate.TargetLoweringOutcome) map[string]int64 { + caps := map[string]int64{} + if outcome.StateLimit > 0 { + caps["state_rows"] = outcome.StateLimit + } + if outcome.FrontierLimit > 0 { + caps["frontier_rows"] = outcome.FrontierLimit + caps["queue_rows"] = outcome.FrontierLimit + } + if outcome.PredecessorLimit > 0 { + caps["predecessor_rows"] = outcome.PredecessorLimit + } + if outcome.EnumerationLimit > 0 { + caps["output_rows"] = outcome.EnumerationLimit + } + if outcome.OutputBytesLimit > 0 { + caps["output_bytes"] = outcome.OutputBytesLimit + } + if outcome.EndpointLimit > 0 { + caps["endpoint_probe_rows"] = outcome.EndpointLimit + } + if outcome.ProbeCaps != nil { + if outcome.ProbeCaps.RootRowLimit > 0 { + caps["forward_seed_rows"] = outcome.ProbeCaps.RootRowLimit + } + if outcome.ProbeCaps.ReverseSeedRowLimit > 0 { + caps["reverse_seed_rows"] = outcome.ProbeCaps.ReverseSeedRowLimit + } + if outcome.ProbeCaps.DirectionalDegreeRowLimit > 0 { + caps["directional_degree_rows"] = outcome.ProbeCaps.DirectionalDegreeRowLimit + } + if outcome.ProbeCaps.SurvivalRowLimit > 0 { + caps["survival_rows"] = outcome.ProbeCaps.SurvivalRowLimit + } + } + return caps +} + +func referenceTraversalCaps(parameters map[string]any) map[string]int64 { + caps := map[string]int64{} + for _, name := range []string{"state_limit", "frontier_limit", "predecessor_limit", "enumeration_limit", "output_bytes_limit", "output_limit"} { + if value, ok := integerParameter(parameters[name]); ok && value > 0 { + counterName := strings.TrimSuffix(name, "_limit") + "_rows" + switch name { + case "enumeration_limit": + counterName = "output_rows" + case "output_bytes_limit": + counterName = "output_bytes" + } + caps[counterName] = value + if name == "frontier_limit" { + caps["queue_rows"] = value + } + } + } + return caps +} + +func integerParameter(value any) (int64, bool) { + switch typed := value.(type) { + case int: + return int64(typed), true + case int32: + return int64(typed), true + case int64: + return typed, true + default: + return 0, false + } +} + +func traversalFamilyForIdentity(identity, family string) TraversalTelemetryFamily { + if strings.HasPrefix(identity, "ASP-") || family == "ASP" { + return TraversalTelemetryFamilyASP + } + if strings.HasPrefix(identity, "SP-") || family == "SP" { + return TraversalTelemetryFamilySP + } + if isOrientationProbePolicy(identity) || strings.Contains(identity, "ORIENTATION") { + return TraversalTelemetryFamilyOrientation + } + if strings.HasPrefix(identity, "MAT-") { + return TraversalTelemetryFamilyHydration + } + return TraversalTelemetryFamilyOrdinary +} + +// traversalRequiredFamilies derives the complete observation contract from +// the emitted policy and public result shape. Families are deliberately kept +// separate so search counters cannot stand in for hydration or workspace +// evidence. +func traversalRequiredFamilies(summary TraversalExecutionSummary, base TraversalTelemetryFamily) []TraversalTelemetryFamily { + var required []TraversalTelemetryFamily + add := func(family TraversalTelemetryFamily) { + if family != "" && !slices.Contains(required, family) { + required = append(required, family) + } + } + + identity := summary.RuntimeIdentity + if identity == "" { + identity = summary.RequestedIdentity + } + if isOrientationProbePolicy(summary.EmittedIdentity) || isOrientationProbePolicy(summary.SelectorVersion) { + add(TraversalTelemetryFamilyOrientation) + add(TraversalTelemetryFamilyOrdinary) + if observationRequiresHydration(summary.ObservationMode) { + add(TraversalTelemetryFamilyHydration) + } + } else { + add(base) + } + if strings.HasPrefix(identity, "ASP-") || base == TraversalTelemetryFamilyASP { + add(TraversalTelemetryFamilyHydration) + } + if strings.Contains(identity, "WE+MAT") || strings.Contains(summary.RequestedIdentity, "WE+MAT") || + strings.HasPrefix(identity, "MAT-") || + (observationRequiresHydration(summary.ObservationMode) && + (strings.HasPrefix(identity, "SP-") || strings.HasPrefix(summary.RequestedIdentity, "SP-"))) { + add(TraversalTelemetryFamilyHydration) + } + if isBidirectionalSPIdentity(identity) || isBidirectionalASPIdentity(identity) || + isBidirectionalSPIdentity(summary.RequestedIdentity) || isBidirectionalASPIdentity(summary.RequestedIdentity) { + add(TraversalTelemetryFamilyWorkspace) + } + return required +} + +func observationRequiresHydration(observation string) bool { + normalized := strings.ToLower(strings.TrimSpace(observation)) + return normalized == "one_path" || normalized == "all_paths" || normalized == "full_path" || + strings.Contains(normalized, "complete path") || strings.Contains(normalized, "all-shortest path") +} + +func isBidirectionalTelemetryIdentity(summary TraversalExecutionSummary) bool { + return isBidirectionalSPIdentity(summary.RuntimeIdentity) || isBidirectionalASPIdentity(summary.RuntimeIdentity) || + isBidirectionalSPIdentity(summary.RequestedIdentity) || isBidirectionalASPIdentity(summary.RequestedIdentity) +} + +func bidirectionalTelemetryIdentity(summary TraversalExecutionSummary) string { + for _, identity := range []string{summary.RuntimeIdentity, summary.RequestedIdentity} { + if isBidirectionalSPIdentity(identity) || isBidirectionalASPIdentity(identity) { + return identity + } + } + return "" +} + +func schedulerForIdentity(identity, scheduler string) string { + if scheduler != "" { + return scheduler + } + switch { + case strings.Contains(identity, "ALT-NODE"): + return "strict_alternating_node" + case strings.Contains(identity, "MIN-LEVEL"): + return "smaller_current_level" + case strings.HasPrefix(identity, "SP-"), strings.HasPrefix(identity, "ASP-"): + return "single_ended_level" + default: + return "not_applicable" + } +} + +func functionBackedTraversal(metrics PostgresPlanMetrics) bool { + for _, node := range metrics.PlanNodes { + if node.NodeType == "Function Scan" && strings.TrimSpace(node.FunctionName) != "" { + return true + } + } + return false +} + +func postgresTraversalPlanReplay(metrics PostgresPlanMetrics) *TraversalPlanReplayEvidence { + replay := &TraversalPlanReplayEvidence{ + Source: postgresTraversalPlanReplaySource, + Counters: map[string]int64{"plan_nodes": int64(len(metrics.PlanNodes))}, + Flags: map[string]bool{}, + Provenance: map[string]string{"counters.plan_nodes": "postgres_metrics.plan_nodes"}, + } + addCounter := func(name string, value int64, metricName string) { + if provenance := metrics.Provenance[metricName]; provenance != "" { + replay.Counters[name] = value + replay.Provenance["counters."+name] = "postgres_metrics." + metricName + ":" + provenance + } + } + addCounter("root_rows", metrics.RootRows, "root_rows") + addCounter("recursive_rows", metrics.RecursiveRows, "recursive_rows") + addCounter("recursive_loops", metrics.RecursiveLoops, "recursive_loops") + addCounter("frontier_rows", metrics.FrontierRows, "frontier_rows") + addCounter("witness_rows", metrics.WitnessRows, "witness_rows") + addCounter("meeting_rows", metrics.MeetingRows, "meeting_rows") + addCounter("hydration_rows", metrics.HydrationRows, "hydration_rows") + addCounter("forward_edge_probe_loops", metrics.ForwardEdgeProbes, "forward_edge_probes") + addCounter("reverse_edge_probe_loops", metrics.ReverseEdgeProbes, "reverse_edge_probes") + addCounter("endpoint_probe_rows", metrics.EndpointProbeRows, "endpoint_probe_rows") + addCounter("reverse_state_probe_rows", metrics.ReverseStateProbeRows, "reverse_state_probe_rows") + addFlag := func(name string, value bool, metricName string) { + if provenance := metrics.Provenance[metricName]; provenance != "" { + replay.Flags[name] = value + replay.Provenance["flags."+name] = "postgres_metrics." + metricName + ":" + provenance + } + } + addFlag("endpoint_guard_overflow", metrics.EndpointGuardOverflow, "endpoint_probe_rows") + addFlag("state_guard_overflow", metrics.StateGuardOverflow, "reverse_state_probe_rows") + addFlag("fallback_executed", metrics.ExpansionFallbackExecuted, "expansion_fallback_executed") + + inlineCTECounters := map[string]string{ + "asp_i1_distance_bounded": "asp_i1_distance_rows", + "asp_i1_predecessor_bounded": "asp_i1_predecessor_rows", + "asp_i1_paths_bounded": "asp_i1_enumeration_rows", + "asp_i1_shortest": "asp_i1_output_rows", + "asp_i1_candidate_marker": "asp_i1_candidate_marker_rows", + "asp_i1_fallback_marker": "asp_i1_fallback_marker_rows", + "asp_i1_candidate_rows": "asp_i1_candidate_branch_rows", + "asp_i1_fallback_rows": "asp_i1_fallback_branch_rows", + } + inlineCTEBodies := map[string][]PostgresPlanNodeMetric{} + for _, node := range metrics.PlanNodes { + rows := node.ActualRows * node.ActualLoops + for cteName := range inlineCTECounters { + if inlinePredecessorCTEBody(node, cteName) { + inlineCTEBodies[cteName] = append(inlineCTEBodies[cteName], node) + } + } + for suffix, name := range map[string]string{ + "orientation_root_probe": "orientation_root_probe_rows", + "orientation_suffix_probe": "orientation_suffix_probe_rows", + "orientation_boundaries": "orientation_boundary_rows", + "orientation_forward_degree_probe": "orientation_forward_degree_rows", + "orientation_reverse_degree_probe": "orientation_reverse_degree_rows", + "orientation_states": "orientation_state_rows", + } { + if orientationCTEBody(node, suffix) { + replay.Counters[name] = rows + replay.Provenance["counters."+name] = "postgres_metrics.plan_nodes.measured_plan_json" + } + } + for suffix, name := range map[string]string{ + "orientation_shadow_forward": "orientation_shadow_forward_rows", + "orientation_shadow_reverse": "orientation_shadow_reverse_rows", + "orientation_shadow_selection": "orientation_shadow_selection_rows", + } { + if orientationCTEBody(node, suffix) { + replay.Counters[name] = rows + replay.Provenance["counters."+name] = "postgres_metrics.plan_nodes.measured_plan_json" + } + } + for suffix, name := range map[string]string{ + "orientation_executed_candidate": "orientation_executed_candidate_rows", + "orientation_executed_incumbent": "orientation_executed_incumbent_rows", + } { + if orientationCTEBody(node, suffix) { + replay.Counters[name] = rows + replay.Provenance["counters."+name] = "postgres_metrics.plan_nodes.measured_plan_json" + } + } + for suffix, name := range map[string]string{ + "orientation_root_probe": "orientation_root_probe_loops", + "orientation_suffix_probe": "orientation_suffix_probe_loops", + "orientation_boundaries": "orientation_boundary_probe_loops", + "orientation_forward_degree_probe": "orientation_forward_degree_probe_loops", + "orientation_reverse_degree_probe": "orientation_reverse_degree_probe_loops", + "orientation_decision": "orientation_decision_loops", + } { + if orientationCTEBody(node, suffix) { + replay.Counters[name] = node.ActualLoops + replay.Provenance["counters."+name] = "postgres_metrics.plan_nodes.measured_plan_json" + } + } + for suffix, name := range map[string]string{ + "orientation_reverse": "orientation_candidate_branch_loops", + "orientation_incumbent": "orientation_incumbent_branch_loops", + } { + if orientationCTEBody(node, suffix) { + replay.Counters[name] = node.ActualLoops + replay.Provenance["counters."+name] = "postgres_metrics.plan_nodes.measured_plan_json" + } + } + if node.NodeType == "Function Scan" && node.FunctionName != "" { + replay.Counters["function_scan_loops"] += node.ActualLoops + replay.Provenance["counters.function_scan_loops"] = "postgres_metrics.plan_nodes.function_scan_actual_loops" + } + } + for cteName, counterName := range inlineCTECounters { + bodies := inlineCTEBodies[cteName] + if len(bodies) != 1 { + continue + } + body := bodies[0] + replay.Counters[counterName] = body.ActualRows * body.ActualLoops + replay.Provenance["counters."+counterName] = "postgres_metrics.plan_nodes.exact_cte_materialization_body" + + branch := "" + markerCTE := "" + switch cteName { + case "asp_i1_candidate_rows": + branch, markerCTE = "candidate", "asp_i1_candidate_marker" + case "asp_i1_fallback_rows": + branch, markerCTE = "fallback", "asp_i1_fallback_marker" + default: + continue + } + if body.PlanNodeID <= 0 { + continue + } + var directChildren, directOuterMarkers, directInnerExecutors []PostgresPlanNodeMetric + for _, node := range metrics.PlanNodes { + if node.ParentPlanNodeID != body.PlanNodeID { + continue + } + directChildren = append(directChildren, node) + switch { + case strings.EqualFold(strings.TrimSpace(node.ParentRelationship), "Outer") && + strings.EqualFold(strings.TrimSpace(node.NodeType), "CTE Scan") && + strings.EqualFold(strings.TrimSpace(node.CTEName), markerCTE): + directOuterMarkers = append(directOuterMarkers, node) + case strings.EqualFold(strings.TrimSpace(node.ParentRelationship), "Inner"): + directInnerExecutors = append(directInnerExecutors, node) + } + } + markerBodies := inlineCTEBodies[markerCTE] + if len(directChildren) != 2 || len(directOuterMarkers) != 1 || len(directInnerExecutors) != 1 || len(markerBodies) != 1 { + continue + } + markerRows := markerBodies[0].ActualRows * markerBodies[0].ActualLoops + outerMarkerRows := directOuterMarkers[0].ActualRows * directOuterMarkers[0].ActualLoops + if directOuterMarkers[0].ActualLoops != 1 || outerMarkerRows != markerRows { + continue + } + name := "asp_i1_" + branch + "_executor_loops" + replay.Counters[name] = directInnerExecutors[0].ActualLoops + replay.Provenance["counters."+name] = "postgres_metrics.plan_nodes.marker_gated_direct_inner_child_actual_loops" + } + return replay +} + +// orientationCTEBody matches the single materialization node PostgreSQL +// labels "CTE ". Consumer CTE scans may execute many times and aliases +// such as reverse_degree_probe contain shorter branch names, so substring +// attribution would over-count probes and invent work in inactive arms. +func orientationCTEBody(node PostgresPlanNodeMetric, suffix string) bool { + return namedCTEBody(node, suffix) +} + +// namedCTEBody matches a PostgreSQL CTE's single materialization body. CTEName +// and Alias identify consumer scans and are intentionally excluded. +func namedCTEBody(node PostgresPlanNodeMetric, suffix string) bool { + name := strings.ToLower(strings.TrimSpace(node.SubplanName)) + return strings.HasPrefix(name, "cte ") && strings.HasSuffix(name, suffix) +} + +// inlinePredecessorCTEBody uses an exact fixed name because its qualification +// contract is tied to one emitted statement shape, not stage-prefixed CTEs. +func inlinePredecessorCTEBody(node PostgresPlanNodeMetric, name string) bool { + return strings.EqualFold(strings.TrimSpace(node.SubplanName), "CTE "+name) +} + +// postgresBidirectionalDiagnosticDocument is the invocation-local document +// returned by read_bidirectional_shortest_path_diagnostic_v1. Pointer fields +// preserve the distinction between a measured zero and missing evidence. +type postgresBidirectionalDiagnosticDocument struct { + SchemaVersion int `json:"schema_version"` + InvocationID string `json:"invocation_id"` + Scheduler string `json:"scheduler"` + StateLimit *int64 `json:"state_limit"` + FrontierLimit *int64 `json:"frontier_limit"` + PredecessorLimit *int64 `json:"predecessor_limit"` + SearchCalls *int64 `json:"search_calls"` + RuntimeBranch string `json:"runtime_branch"` + Overflowed *bool `json:"overflowed"` + FallbackExecuted *bool `json:"fallback_executed"` + Counters *postgresBidirectionalDiagnosticCounts `json:"counters"` + Calls []postgresBidirectionalDiagnosticCall `json:"calls"` + WorkspaceBytes int64 `json:"-"` +} + +type postgresBidirectionalDiagnosticCall struct { + SearchID *int64 `json:"search_id"` + SourceID *int64 `json:"source_id"` + TargetID *int64 `json:"target_id"` + RuntimeBranch string `json:"runtime_branch"` + SchedulerActions *int64 `json:"scheduler_actions"` + CandidateEdges *int64 `json:"candidate_edges"` + DistinctNewNodes *int64 `json:"distinct_new_nodes"` + SeenPeak *int64 `json:"seen_peak"` + FrontierPeak *int64 `json:"frontier_peak"` + QueuePeak *int64 `json:"queue_peak"` + PredecessorPeak *int64 `json:"predecessor_peak"` + MeetingCandidates *int64 `json:"meeting_candidates"` + FrozenDistance *int64 `json:"frozen_distance"` + WitnessRows *int64 `json:"witness_rows"` + Overflowed *bool `json:"overflowed"` + FallbackExecuted *bool `json:"fallback_executed"` +} + +type postgresBidirectionalDiagnosticCounts struct { + SchedulerActions *int64 `json:"scheduler_actions"` + CandidateEdges *int64 `json:"candidate_edges"` + DistinctNewNodes *int64 `json:"distinct_new_nodes"` + SeenPeak *int64 `json:"seen_peak"` + FrontierPeak *int64 `json:"frontier_peak"` + QueuePeak *int64 `json:"queue_peak"` + PredecessorPeak *int64 `json:"predecessor_peak"` + MeetingCandidates *int64 `json:"meeting_candidates"` + FrozenDistance *int64 `json:"frozen_distance"` + WitnessRows *int64 `json:"witness_rows"` + Levels []postgresBidirectionalDiagnosticLevel `json:"levels"` +} + +type postgresBidirectionalDiagnosticLevel struct { + SearchID *int64 `json:"search_id"` + ActionIndex *int64 `json:"action_index"` + Side string `json:"side"` + Action string `json:"action"` + Depth *int64 `json:"depth"` + FrontierRows *int64 `json:"frontier_rows"` + CandidateEdges *int64 `json:"candidate_edges"` + DistinctNewNodes *int64 `json:"distinct_new_nodes"` + SeenRows *int64 `json:"seen_rows"` + QueueRows *int64 `json:"queue_rows"` + PredecessorRows *int64 `json:"predecessor_rows"` + MeetingCandidates *int64 `json:"meeting_candidates"` +} + +type postgresBidirectionalAllShortestDiagnosticDocument struct { + SchemaVersion int `json:"schema_version"` + InvocationID string `json:"invocation_id"` + Scheduler string `json:"scheduler"` + StateLimit *int64 `json:"state_limit"` + FrontierLimit *int64 `json:"frontier_limit"` + PredecessorLimit *int64 `json:"predecessor_limit"` + EnumerationLimit *int64 `json:"enumeration_limit"` + OutputBytesLimit *int64 `json:"output_bytes_limit"` + SearchCalls *int64 `json:"search_calls"` + RuntimeBranch string `json:"runtime_branch"` + Overflowed *bool `json:"overflowed"` + FallbackExecuted *bool `json:"fallback_executed"` + Counters *postgresBidirectionalAllShortestDiagnosticCounts `json:"counters"` + Calls []postgresBidirectionalAllShortestDiagnosticCall `json:"calls"` + WorkspaceBytes int64 `json:"-"` +} + +type postgresBidirectionalAllShortestDiagnosticCounts struct { + SchedulerActions *int64 `json:"scheduler_actions"` + CandidateEdges *int64 `json:"candidate_edges"` + DistinctNewNodes *int64 `json:"distinct_new_nodes"` + SeenPeak *int64 `json:"seen_peak"` + FrontierPeak *int64 `json:"frontier_peak"` + QueuePeak *int64 `json:"queue_peak"` + PredecessorPeak *int64 `json:"predecessor_peak"` + MeetingCandidates *int64 `json:"meeting_candidates"` + FrozenDistance *int64 `json:"frozen_distance"` + WitnessRows *int64 `json:"witness_rows"` + SameDepthPredecessorAdditions *int64 `json:"same_depth_predecessor_additions"` + MeetingNodes *int64 `json:"meeting_nodes"` + CutDepth *int64 `json:"cut_depth"` + PathCountEstimate *int64 `json:"path_count_estimate"` + PathCountSaturated *bool `json:"path_count_saturated"` + EnumeratedCandidates *int64 `json:"enumerated_candidates"` + DuplicateRejects *int64 `json:"duplicate_rejects"` + OutputPaths *int64 `json:"output_paths"` + OutputEdgeCells *int64 `json:"output_edge_cells"` + OutputBytes *int64 `json:"output_bytes"` + Levels []postgresBidirectionalDiagnosticLevel `json:"levels"` +} + +type postgresBidirectionalAllShortestDiagnosticCall struct { + SearchID *int64 `json:"search_id"` + SourceID *int64 `json:"source_id"` + TargetID *int64 `json:"target_id"` + RuntimeBranch string `json:"runtime_branch"` + SchedulerActions *int64 `json:"scheduler_actions"` + CandidateEdges *int64 `json:"candidate_edges"` + DistinctNewNodes *int64 `json:"distinct_new_nodes"` + SeenPeak *int64 `json:"seen_peak"` + FrontierPeak *int64 `json:"frontier_peak"` + QueuePeak *int64 `json:"queue_peak"` + PredecessorPeak *int64 `json:"predecessor_peak"` + MeetingCandidates *int64 `json:"meeting_candidates"` + FrozenDistance *int64 `json:"frozen_distance"` + WitnessRows *int64 `json:"witness_rows"` + SameDepthPredecessorAdditions *int64 `json:"same_depth_predecessor_additions"` + MeetingNodes *int64 `json:"meeting_nodes"` + CutDepth *int64 `json:"cut_depth"` + PathCountEstimate *int64 `json:"path_count_estimate"` + PathCountSaturated *bool `json:"path_count_saturated"` + EnumeratedCandidates *int64 `json:"enumerated_candidates"` + DuplicateRejects *int64 `json:"duplicate_rejects"` + OutputPaths *int64 `json:"output_paths"` + OutputEdgeCells *int64 `json:"output_edge_cells"` + OutputBytes *int64 `json:"output_bytes"` + Overflowed *bool `json:"overflowed"` + FallbackExecuted *bool `json:"fallback_executed"` +} + +// attachPostgresTraversalTelemetry runs only after every timed case, +// reference, raw-PGX, and concurrency sample has completed. +func (s *postgresSQLRunner) attachPostgresTraversalTelemetry(ctx context.Context, record *CaseResult, parameters map[string]any) error { + if s.traversalTelemetry == "" || s.traversalTelemetry == postgresTraversalTelemetryOff { + for idx := range record.PostgresReferences { + record.PostgresReferences[idx].traversalTelemetryParameters = nil + } + return nil + } + + level := TraversalTelemetryLevel(s.traversalTelemetry) + if record.Optimization != nil && record.PostgresMetrics != nil { + telemetry, err := buildPostgresCaseTraversalTelemetry(*record.Optimization, *record.PostgresMetrics, s.backendPID, level) + if err != nil { + return fmt.Errorf("build PostgreSQL case traversal telemetry: %w", err) + } + if telemetry != nil { + if level == TraversalTelemetryLevelDiagnostic { + enrichOrientationTraversalTelemetry( + telemetry, + *record.PostgresMetrics, + record.RowCount, + record.ObservedRows, + orientationPolicyMaximumDepth(*record.Optimization, telemetry.Summary.EmittedIdentity), + ) + enrichInlinePredecessorTraversalTelemetry(telemetry, *record.PostgresMetrics, record.RowCount, record.ObservedRows) + if err := s.enrichBidirectionalTraversalTelemetry(ctx, telemetry, record.SQL, parameters, record.RowCount, record.ObservedRows, *record.PostgresMetrics); err != nil { + return fmt.Errorf("capture PostgreSQL case traversal telemetry: %w", err) + } + } + record.TraversalTelemetry = telemetry + } + } + + for idx := range record.PostgresReferences { + reference := &record.PostgresReferences[idx] + parameters := reference.traversalTelemetryParameters + reference.traversalTelemetryParameters = nil + telemetry, err := buildPostgresReferenceTraversalTelemetry(*reference, parameters, s.backendPID, level) + if err != nil { + return fmt.Errorf("build PostgreSQL reference %s traversal telemetry: %w", reference.Name, err) + } + if telemetry == nil { + continue + } + if level == TraversalTelemetryLevelDiagnostic { + if err := s.enrichBidirectionalTraversalTelemetry(ctx, telemetry, reference.SQL, parameters, reference.RowCount, reference.ObservedRows, *reference.PostgresMetrics); err != nil { + return fmt.Errorf("capture PostgreSQL reference %s traversal telemetry: %w", reference.Name, err) + } + } + reference.TraversalTelemetry = telemetry + } + return nil +} + +// enrichInlineASPTraversalTelemetry maps the guarded statement's named CTEs +// to its dedicated bounded-work contract. Public observation bytes are a +// conservative ceiling for the staged edge-array bytes used by admission. +func enrichInlineASPTraversalTelemetry(telemetry *TraversalExecutionTelemetry, metrics PostgresPlanMetrics, outputRows int64, observedRows []string) { + enrichInlinePredecessorTraversalTelemetry(telemetry, metrics, outputRows, observedRows) +} + +// enrichInlinePredecessorTraversalTelemetry maps the shared guarded I1 +// statement's named CTEs to either the all-paths or canonical one-path counter +// family. The separate serialized fields prevent evidence from one public +// observation contract from satisfying the other. +func enrichInlinePredecessorTraversalTelemetry(telemetry *TraversalExecutionTelemetry, metrics PostgresPlanMetrics, outputRows int64, observedRows []string) { + if telemetry == nil || telemetry.Diagnostic == nil || + (telemetry.Summary.EmittedIdentity != optimize.ShortestPathPolicyASPI1GuardedV1 && + telemetry.Summary.EmittedIdentity != optimize.ShortestPathPolicyI1CanonicalGuardedV1) { + return + } + plan := telemetry.Diagnostic.PlanReplay + if plan == nil { + return + } + requiredPlanCounters := []string{ + "asp_i1_distance_rows", + "asp_i1_predecessor_rows", + "asp_i1_enumeration_rows", + "asp_i1_output_rows", + "asp_i1_candidate_marker_rows", + "asp_i1_fallback_marker_rows", + "asp_i1_candidate_branch_rows", + "asp_i1_fallback_branch_rows", + "asp_i1_candidate_executor_loops", + "asp_i1_fallback_executor_loops", + } + var missingPlanCounters []string + for _, name := range requiredPlanCounters { + if _, present := plan.Counters[name]; !present { + missingPlanCounters = append(missingPlanCounters, name) + } + } + if len(missingPlanCounters) > 0 { + markTraversalCountersUnavailable( + telemetry.Diagnostic, + "inline predecessor plan replay is missing exact named counters: "+strings.Join(missingPlanCounters, ", "), + ) + return + } + get := func(name string) int64 { return plan.Counters[name] } + outputBytes := int64(0) + for _, row := range observedRows { + outputBytes += int64(len(row)) + } + inline := &InlinePredecessorTraversalCounters{ + DistanceRows: traversalTelemetryPointer(get("asp_i1_distance_rows")), + PredecessorRows: traversalTelemetryPointer(get("asp_i1_predecessor_rows")), + EnumerationRows: traversalTelemetryPointer(get("asp_i1_enumeration_rows")), + OutputPaths: traversalTelemetryPointer(outputRows), + OutputBytes: traversalTelemetryPointer(outputBytes), + CandidateMarkerRows: traversalTelemetryPointer(get("asp_i1_candidate_marker_rows")), + FallbackMarkerRows: traversalTelemetryPointer(get("asp_i1_fallback_marker_rows")), + CandidateBranchRows: traversalTelemetryPointer(get("asp_i1_candidate_branch_rows")), + FallbackBranchRows: traversalTelemetryPointer(get("asp_i1_fallback_branch_rows")), + CandidateExecutorLoops: traversalTelemetryPointer(get("asp_i1_candidate_executor_loops")), + FallbackExecutorLoops: traversalTelemetryPointer(get("asp_i1_fallback_executor_loops")), + } + prefix := "inline_asp" + if telemetry.Summary.EmittedIdentity == optimize.ShortestPathPolicyI1CanonicalGuardedV1 { + prefix = "inline_shortest_path" + telemetry.Diagnostic.Counters.InlineShortestPath = inline + } else { + telemetry.Diagnostic.Counters.InlineASP = inline + } + if telemetry.Diagnostic.Provenance == nil { + telemetry.Diagnostic.Provenance = map[string]string{} + } + for _, name := range []string{ + "distance_rows", "predecessor_rows", "enumeration_rows", "candidate_marker_rows", + "fallback_marker_rows", "candidate_branch_rows", "fallback_branch_rows", + "candidate_executor_loops", "fallback_executor_loops", + } { + telemetry.Diagnostic.Provenance[prefix+"."+name] = "untimed_timing_on_plan.inline_predecessor_named_ctes" + } + telemetry.Diagnostic.Provenance[prefix+".output_paths"] = "exact_public_observation.row_count" + telemetry.Diagnostic.Provenance[prefix+".output_bytes"] = "exact_public_observation.conservative_serialized_bytes" + + if slices.Contains(telemetry.Diagnostic.RequiredFamilies, TraversalTelemetryFamilyHydration) { + telemetry.Diagnostic.Counters.Hydration = &TraversalHydrationCounters{ + PathCount: traversalTelemetryPointer(outputRows), NodeLookups: traversalTelemetryPointer(metrics.HydrationLoops), + EdgeLookups: traversalTelemetryPointer(metrics.HydrationRows), Loops: traversalTelemetryPointer(metrics.HydrationLoops), + Rows: traversalTelemetryPointer(metrics.HydrationRows), TimeNS: traversalTelemetryPointer(int64(0)), Bytes: traversalTelemetryPointer(outputBytes), + } + for _, name := range []string{"path_count", "node_lookups", "edge_lookups", "loops", "rows", "time_ns", "bytes"} { + telemetry.Diagnostic.Provenance["hydration."+name] = "untimed_plan_and_exact_public_observation" + } + } + telemetry.Diagnostic.CounterStatus = TraversalTelemetryCounterStatusComplete + telemetry.Diagnostic.IncompleteReasons = nil +} + +// enrichOrientationTraversalTelemetry turns explicitly named SQL probe and +// branch nodes into a complete, conservative diagnostic document. Probe times +// come from the untimed TIMING ON JSON EXPLAIN replay; hydration bytes use the +// captured public observation, never an estimated tuple width. +func enrichOrientationTraversalTelemetry(telemetry *TraversalExecutionTelemetry, metrics PostgresPlanMetrics, outputRows int64, observedRows []string, maximumDepth int64) { + if telemetry == nil || telemetry.Diagnostic == nil || !isOrientationProbePolicy(telemetry.Summary.EmittedIdentity) { + return + } + if telemetry.Summary.EmittedIdentity == string(optimize.ExpansionSearchPolicyOrientationProbeV2) && maximumDepth <= 0 { + markTraversalCountersUnavailable(telemetry.Diagnostic, "orientation-probe-v2 maximum depth is unavailable") + return + } + plan := telemetry.Diagnostic.PlanReplay + if plan == nil { + return + } + get := func(name string) int64 { return plan.Counters[name] } + forwardSeeds := get("orientation_root_probe_rows") + reverseSeeds := get("orientation_suffix_probe_rows") + boundaries := get("orientation_boundary_rows") + forwardDegree := get("orientation_forward_degree_rows") + reverseDegree := get("orientation_reverse_degree_rows") + stateRows := get("orientation_state_rows") + probeRows := forwardSeeds + reverseSeeds + boundaries + forwardDegree + reverseDegree + duplicateSeeds := max(reverseSeeds-boundaries, int64(0)) + shallowSurvivalRows := boundaries + shallowSurvival := float64(0) + if reverseSeeds > 0 { + shallowSurvival = float64(boundaries) / float64(reverseSeeds) + } + forwardScore := float64(forwardSeeds + forwardDegree) + if telemetry.Summary.EmittedIdentity == string(optimize.ExpansionSearchPolicyOrientationProbeV2) { + forwardScore = float64(forwardSeeds + maximumDepth*forwardDegree) + } + reverseScore := float64(reverseSeeds + boundaries + reverseDegree) + selectedSide := "forward" + if telemetry.Summary.RuntimeIdentity != telemetry.Summary.FallbackIdentity && strings.Contains(telemetry.Summary.RuntimeIdentity, "REVERSE") { + selectedSide = "reverse" + } + overflow := false + if telemetry.Summary.Overflow != nil { + overflow = *telemetry.Summary.Overflow + } + branchLoops := get("orientation_candidate_branch_loops") + get("orientation_incumbent_branch_loops") + + var probeTimeNS, probeHits, probeReads, edgeCandidates, repeatRejects, hydrationLoops, hydrationRows, hydrationTimeNS int64 + for _, node := range metrics.PlanNodes { + identity := strings.ToLower(strings.Join([]string{node.CTEName, node.Alias, node.SubplanName}, " ")) + rows := node.ActualRows * node.ActualLoops + if strings.Contains(identity, "orientation_") && (strings.Contains(identity, "_probe") || strings.Contains(identity, "_boundaries") || strings.Contains(identity, "_decision")) { + probeTimeNS += int64(node.ActualTotalMS * float64(time.Millisecond)) + probeHits += node.Buffers.SharedHit + node.Buffers.LocalHit + probeReads += node.Buffers.SharedRead + node.Buffers.LocalRead + } + if node.RelationName == "edge" { + edgeCandidates += rows + node.RowsRemovedByFilter + repeatRejects += node.RowsRemovedByFilter + } + if strings.Contains(identity, "hydrat") || strings.Contains(identity, "materializ") { + hydrationLoops += node.ActualLoops + hydrationRows += rows + hydrationTimeNS += int64(node.ActualTotalMS * float64(time.Millisecond)) + } + } + orientation := &OrientationTraversalCounters{ + ForwardSeeds: traversalTelemetryPointer(forwardSeeds), ReverseSeeds: traversalTelemetryPointer(reverseSeeds), + DuplicateSeeds: traversalTelemetryPointer(duplicateSeeds), SuffixRows: traversalTelemetryPointer(reverseSeeds), + DistinctBoundaries: traversalTelemetryPointer(boundaries), TypedDirectionalDegreeSamples: traversalTelemetryPointer(forwardDegree + reverseDegree), + ForwardDegreeSamples: traversalTelemetryPointer(forwardDegree), ReverseDegreeSamples: traversalTelemetryPointer(reverseDegree), + ShallowSurvivalRows: traversalTelemetryPointer(shallowSurvivalRows), ShallowSurvival: traversalTelemetryPointer(shallowSurvival), + ProbeRows: traversalTelemetryPointer(probeRows), ProbeTimeNS: traversalTelemetryPointer(probeTimeNS), + ProbeBufferHits: traversalTelemetryPointer(probeHits), ProbeBufferReads: traversalTelemetryPointer(probeReads), + ForwardScore: traversalTelemetryPointer(forwardScore), ReverseScore: traversalTelemetryPointer(reverseScore), + SelectedSide: selectedSide, SentinelOverflow: traversalTelemetryPointer(overflow), BranchLoops: traversalTelemetryPointer(branchLoops), + } + ordinary := &OrdinaryTraversalCounters{ + Roots: traversalTelemetryPointer(forwardSeeds), EdgeCandidates: traversalTelemetryPointer(edgeCandidates), + AdmittedStates: traversalTelemetryPointer(stateRows), RelationshipRepeatRejects: traversalTelemetryPointer(repeatRejects), + RecursiveRows: traversalTelemetryPointer(metrics.RecursiveRows), PeakState: traversalTelemetryPointer(stateRows), + EmittedTrails: traversalTelemetryPointer(outputRows), HydrationRows: traversalTelemetryPointer(metrics.HydrationRows), + } + telemetry.Diagnostic.Counters.Orientation = orientation + telemetry.Diagnostic.Counters.Ordinary = ordinary + telemetry.Diagnostic.Provenance = map[string]string{} + for _, name := range []string{"forward_seeds", "reverse_seeds", "duplicate_seeds", "suffix_rows", "distinct_boundaries", "typed_directional_degree_samples", "forward_degree_samples", "reverse_degree_samples", "shallow_survival_rows", "shallow_survival", "probe_rows", "probe_time_ns", "probe_buffer_hits", "probe_buffer_reads", "forward_score", "reverse_score", "selected_side", "sentinel_overflow", "branch_loops"} { + telemetry.Diagnostic.Provenance["orientation."+name] = "untimed_timing_on_plan.orientation_named_ctes" + } + for _, name := range []string{"roots", "edge_candidates", "admitted_states", "relationship_repeat_rejects", "recursive_rows", "peak_state", "emitted_trails", "hydration_rows"} { + telemetry.Diagnostic.Provenance["ordinary."+name] = "untimed_timing_on_plan.executed_orientation_branch" + } + if slices.Contains(telemetry.Diagnostic.RequiredFamilies, TraversalTelemetryFamilyHydration) { + bytes := int64(0) + for _, row := range observedRows { + bytes += int64(len(row)) + } + nodeLookups := metrics.HydrationLoops + edgeLookups := metrics.HydrationRows + telemetry.Diagnostic.Counters.Hydration = &TraversalHydrationCounters{ + PathCount: traversalTelemetryPointer(outputRows), NodeLookups: traversalTelemetryPointer(nodeLookups), + EdgeLookups: traversalTelemetryPointer(edgeLookups), Loops: traversalTelemetryPointer(hydrationLoops), + Rows: traversalTelemetryPointer(hydrationRows), TimeNS: traversalTelemetryPointer(hydrationTimeNS), Bytes: traversalTelemetryPointer(bytes), + } + for _, name := range []string{"path_count", "node_lookups", "edge_lookups", "loops", "rows", "time_ns", "bytes"} { + telemetry.Diagnostic.Provenance["hydration."+name] = "untimed_timing_on_plan_and_exact_public_observation" + } + } + telemetry.Diagnostic.CounterStatus = TraversalTelemetryCounterStatusComplete + telemetry.Diagnostic.IncompleteReasons = nil +} + +func orientationPolicyMaximumDepth(summary translate.OptimizationSummary, policy string) int64 { + if !isOrientationProbePolicy(policy) { + return 0 + } + for _, outcome := range summary.TargetOutcomes { + if outcome.EmittedPolicy == policy && outcome.MaximumDepth != nil { + return *outcome.MaximumDepth + } + } + return 0 +} + +// enrichBidirectionalTraversalTelemetry replaces opaque Function Scan +// evidence only when the exact SP-B1/B2 statement reports a validated, +// invocation-local diagnostic document. Other hidden functions stay +// explicitly unavailable. +func (s *postgresSQLRunner) enrichBidirectionalTraversalTelemetry( + ctx context.Context, + telemetry *TraversalExecutionTelemetry, + sqlQuery string, + parameters map[string]any, + expectedRows int64, + observedRows []string, + metrics PostgresPlanMetrics, +) error { + if telemetry == nil || telemetry.Level != TraversalTelemetryLevelDiagnostic || !isBidirectionalTelemetryIdentity(telemetry.Summary) { + return nil + } + identity := bidirectionalTelemetryIdentity(telemetry.Summary) + + invocationID := newRunUUID() + if telemetry.Diagnostic != nil { + invocationID = telemetry.Diagnostic.InvocationID + } + var ( + unavailableReason string + err error + ) + if isBidirectionalASPIdentity(identity) { + var document *postgresBidirectionalAllShortestDiagnosticDocument + document, unavailableReason, err = s.replayBidirectionalAllShortestTraversalDiagnostic(ctx, invocationID, sqlQuery, parameters, expectedRows) + if err == nil && unavailableReason == "" { + err = applyBidirectionalAllShortestTraversalDiagnostic(telemetry, document, invocationID, s.backendPID) + if err == nil { + enrichBidirectionalHydrationTelemetry(telemetry, document.Counters.OutputPaths, document.Counters.OutputEdgeCells, observedRows, metrics) + } + } + } else { + var document *postgresBidirectionalDiagnosticDocument + document, unavailableReason, err = s.replayBidirectionalTraversalDiagnostic(ctx, invocationID, sqlQuery, parameters, expectedRows) + if err == nil && unavailableReason == "" { + err = applyBidirectionalTraversalDiagnostic(telemetry, document, invocationID, s.backendPID) + if err == nil { + pathCount := document.Counters.WitnessRows + edgeCells := int64(0) + if document.Counters.FrozenDistance != nil && *document.Counters.FrozenDistance > 0 && pathCount != nil { + edgeCells = *document.Counters.FrozenDistance * *pathCount + } + enrichBidirectionalHydrationTelemetry(telemetry, pathCount, traversalTelemetryPointer(edgeCells), observedRows, metrics) + } + } + } + if err != nil { + if telemetry.Diagnostic == nil { + markTraversalSummaryUnavailable(telemetry, err.Error()) + return telemetry.Validate() + } + markTraversalCountersUnavailable(telemetry.Diagnostic, err.Error()) + return telemetry.Validate() + } + if unavailableReason != "" { + if telemetry.Diagnostic == nil { + markTraversalSummaryUnavailable(telemetry, unavailableReason) + return telemetry.Validate() + } + markTraversalCountersUnavailable(telemetry.Diagnostic, unavailableReason) + return telemetry.Validate() + } + return telemetry.Validate() +} + +func enrichBidirectionalHydrationTelemetry( + telemetry *TraversalExecutionTelemetry, + pathCount, edgeCells *int64, + observedRows []string, + metrics PostgresPlanMetrics, +) { + if telemetry == nil || telemetry.Diagnostic == nil || !slices.Contains(telemetry.Diagnostic.RequiredFamilies, TraversalTelemetryFamilyHydration) || pathCount == nil || edgeCells == nil { + return + } + bytes := int64(0) + for _, row := range observedRows { + bytes += int64(len(row)) + } + nodeLookups := *edgeCells + *pathCount + var hydrationTimeNS int64 + for _, node := range metrics.PlanNodes { + identity := strings.ToLower(strings.Join([]string{node.CTEName, node.Alias, node.SubplanName}, " ")) + if strings.Contains(identity, "hydrat") || strings.Contains(identity, "materializ") || node.RelationName == "node" { + hydrationTimeNS += int64(node.ActualTotalMS * float64(time.Millisecond)) + } + } + rows := metrics.HydrationRows + if rows == 0 { + rows = nodeLookups + *edgeCells + } + loops := metrics.HydrationLoops + telemetry.Diagnostic.Counters.Hydration = &TraversalHydrationCounters{ + PathCount: pathCount, NodeLookups: traversalTelemetryPointer(nodeLookups), EdgeLookups: edgeCells, + Loops: traversalTelemetryPointer(loops), Rows: traversalTelemetryPointer(rows), + TimeNS: traversalTelemetryPointer(hydrationTimeNS), Bytes: traversalTelemetryPointer(bytes), + } + for _, name := range []string{"path_count", "node_lookups", "edge_lookups", "loops", "rows", "time_ns", "bytes"} { + telemetry.Diagnostic.Provenance["hydration."+name] = "invocation_local_path_counts+untimed_timing_on_plan+exact_public_observation" + } + if telemetry.Summary.FallbackExecuted != nil && !*telemetry.Summary.FallbackExecuted { + telemetry.Diagnostic.CounterStatus = TraversalTelemetryCounterStatusComplete + telemetry.Diagnostic.IncompleteReasons = nil + } +} + +// replayBidirectionalTraversalDiagnostic executes the exact statement in a +// separate repeatable-read transaction on the runner's single physical +// connection. Its duration and counters are never added to latency samples. +func (s *postgresSQLRunner) replayBidirectionalTraversalDiagnostic( + ctx context.Context, + invocationID string, + sqlQuery string, + parameters map[string]any, + expectedRows int64, +) (*postgresBidirectionalDiagnosticDocument, string, error) { + rawDocument, workspaceBytes, unavailableReason, err := s.replayInvocationLocalTraversalDiagnostic( + ctx, invocationID, sqlQuery, parameters, expectedRows, + "select public.begin_bidirectional_shortest_path_diagnostic_v1($1)", + "select coalesce(public.read_bidirectional_shortest_path_diagnostic_v1($1)::text, '')", + "select public.clear_bidirectional_shortest_path_diagnostic_v1($1)", + ) + if err != nil || unavailableReason != "" { + return nil, unavailableReason, err + } + document := &postgresBidirectionalDiagnosticDocument{} + if err := json.Unmarshal([]byte(rawDocument), document); err != nil { + return nil, "diagnostic reader returned malformed JSON: " + err.Error(), nil + } + document.WorkspaceBytes = workspaceBytes + return document, "", nil +} + +func (s *postgresSQLRunner) replayBidirectionalAllShortestTraversalDiagnostic( + ctx context.Context, + invocationID string, + sqlQuery string, + parameters map[string]any, + expectedRows int64, +) (*postgresBidirectionalAllShortestDiagnosticDocument, string, error) { + rawDocument, workspaceBytes, unavailableReason, err := s.replayInvocationLocalTraversalDiagnostic( + ctx, invocationID, sqlQuery, parameters, expectedRows, + "select public.begin_bidirectional_all_shortest_path_diagnostic_v1($1)", + "select coalesce(public.read_bidirectional_all_shortest_path_diagnostic_v1($1)::text, '')", + "select public.clear_bidirectional_all_shortest_path_diagnostic_v1($1)", + ) + if err != nil || unavailableReason != "" { + return nil, unavailableReason, err + } + document := &postgresBidirectionalAllShortestDiagnosticDocument{} + if err := json.Unmarshal([]byte(rawDocument), document); err != nil { + return nil, "all-shortest diagnostic reader returned malformed JSON: " + err.Error(), nil + } + document.WorkspaceBytes = workspaceBytes + return document, "", nil +} + +func (s *postgresSQLRunner) replayInvocationLocalTraversalDiagnostic( + ctx context.Context, + invocationID string, + sqlQuery string, + parameters map[string]any, + expectedRows int64, + beginSQL string, + readSQL string, + clearSQL string, +) (string, int64, string, error) { + connection, err := s.pool.Acquire(ctx) + if err != nil { + return "", 0, "", fmt.Errorf("acquire diagnostic connection: %w", err) + } + defer connection.Release() + + var backendPID int32 + if err := connection.QueryRow(ctx, "select pg_backend_pid()").Scan(&backendPID); err != nil { + return "", 0, "", fmt.Errorf("read diagnostic connection identity: %w", err) + } + connectionID := strconv.FormatInt(int64(backendPID), 10) + if connectionID != s.backendPID { + return "", 0, "", fmt.Errorf("diagnostic connection identity %s differs from timed-sample connection %s", connectionID, s.backendPID) + } + + tx, err := connection.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadWrite}) + if err != nil { + return "", 0, "", fmt.Errorf("begin repeatable-read diagnostic transaction: %w", err) + } + initialized := false + defer func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + if initialized { + _, _ = tx.Exec(cleanupCtx, clearSQL, invocationID) + } + _ = tx.Rollback(cleanupCtx) + }() + + if _, err := tx.Exec(ctx, beginSQL, invocationID); err != nil { + return "", 0, "", fmt.Errorf("begin invocation-local diagnostic: %w", err) + } + initialized = true + + queryArgs := []any{pgx.QueryExecModeCacheStatement, pgx.QueryResultFormats{pgx.BinaryFormatCode}} + if len(parameters) > 0 { + queryArgs = append(queryArgs, pgx.NamedArgs(parameters)) + } + rows, err := tx.Query(ctx, sqlQuery, queryArgs...) + if err != nil { + return "", 0, "", fmt.Errorf("execute untimed diagnostic replay: %w", err) + } + var rowCount int64 + for rows.Next() { + rowCount++ + if _, err := rows.Values(); err != nil { + rows.Close() + return "", 0, "", fmt.Errorf("decode untimed diagnostic replay: %w", err) + } + } + rows.Close() + if err := rows.Err(); err != nil { + return "", 0, "", fmt.Errorf("drain untimed diagnostic replay: %w", err) + } + if rowCount != expectedRows { + return "", 0, "", fmt.Errorf("untimed diagnostic replay row count %d differs from measured row count %d", rowCount, expectedRows) + } + var workspaceBytes int64 + if err := tx.QueryRow(ctx, ` + select coalesce(sum(pg_total_relation_size(c.oid)), 0)::int8 + from pg_class c + where c.relnamespace = pg_my_temp_schema() + and (c.relname like 'spb_%' or c.relname like 'asb_%') + and c.relname not like '%telemetry%' + `).Scan(&workspaceBytes); err != nil { + return "", 0, "", fmt.Errorf("measure diagnostic workspace high-water bytes: %w", err) + } + + var replayBackendPID int32 + if err := tx.QueryRow(ctx, "select pg_backend_pid()").Scan(&replayBackendPID); err != nil { + return "", 0, "", fmt.Errorf("verify diagnostic transaction connection identity: %w", err) + } + if replayBackendPID != backendPID { + return "", 0, "", fmt.Errorf("diagnostic transaction changed physical connection from %d to %d", backendPID, replayBackendPID) + } + + var rawDocument string + if err := tx.QueryRow(ctx, readSQL, invocationID).Scan(&rawDocument); err != nil { + return "", 0, "", fmt.Errorf("read invocation-local diagnostic: %w", err) + } + if _, err := tx.Exec(ctx, clearSQL, invocationID); err != nil { + return "", 0, "", fmt.Errorf("clear invocation-local diagnostic: %w", err) + } + initialized = false + if err := tx.Commit(ctx); err != nil { + return "", 0, "", fmt.Errorf("commit cleared diagnostic transaction: %w", err) + } + + if strings.TrimSpace(rawDocument) == "" { + return "", workspaceBytes, "diagnostic reader returned no document for this invocation", nil + } + return rawDocument, workspaceBytes, "", nil +} + +func applyBidirectionalTraversalDiagnostic( + telemetry *TraversalExecutionTelemetry, + document *postgresBidirectionalDiagnosticDocument, + expectedInvocationID string, + expectedConnectionID string, +) error { + if telemetry == nil || document == nil { + return fmt.Errorf("bidirectional diagnostic document is missing") + } + if document.SchemaVersion != 1 { + return fmt.Errorf("bidirectional diagnostic schema_version must be 1") + } + if document.InvocationID != expectedInvocationID { + return fmt.Errorf("bidirectional diagnostic invocation identity %q differs from requested %q", document.InvocationID, expectedInvocationID) + } + if telemetry.Diagnostic != nil && telemetry.Diagnostic.ConnectionID != expectedConnectionID { + return fmt.Errorf("attached diagnostic connection identity %q differs from replay connection %q", telemetry.Diagnostic.ConnectionID, expectedConnectionID) + } + if document.SearchCalls == nil || *document.SearchCalls != 1 { + return fmt.Errorf("instrumented singleton SP-B1/B2 replay must invoke exactly one search call") + } + if int64(len(document.Calls)) != *document.SearchCalls { + return fmt.Errorf("bidirectional diagnostic call count %d differs from search_calls %d", len(document.Calls), *document.SearchCalls) + } + if document.RuntimeBranch == "" || document.RuntimeBranch == "missing" || document.RuntimeBranch == "mixed" { + return fmt.Errorf("bidirectional diagnostic runtime branch is not singular") + } + if document.Overflowed == nil || document.FallbackExecuted == nil { + return fmt.Errorf("bidirectional diagnostic runtime outcome flags are missing") + } + if err := validateDiagnosticRuntimeOutcome(document.RuntimeBranch, *document.Overflowed, *document.FallbackExecuted, "exact_s4_fallback", []string{ + "preflight_zero_hop", "preflight_one_hop", "preflight_two_hop", "preflight_no_path", "search_no_path", "bidirectional_search", + }); err != nil { + return fmt.Errorf("bidirectional diagnostic: %w", err) + } + if err := validateBidirectionalDiagnosticCalls(document.Calls, document.Overflowed, document.FallbackExecuted); err != nil { + return err + } + if document.Calls[0].RuntimeBranch != document.RuntimeBranch { + return fmt.Errorf("bidirectional diagnostic aggregate runtime branch differs from its call") + } + if strings.TrimSpace(document.Scheduler) == "" || document.Scheduler != telemetry.Summary.SchedulerVersion { + return fmt.Errorf("bidirectional diagnostic scheduler %q differs from planned scheduler %q", document.Scheduler, telemetry.Summary.SchedulerVersion) + } + for name, observed := range map[string]*int64{ + "state_rows": document.StateLimit, + "frontier_rows": document.FrontierLimit, + "queue_rows": document.FrontierLimit, + "predecessor_rows": document.PredecessorLimit, + } { + planned, ok := telemetry.Summary.Caps[name] + if !ok || observed == nil || *observed != planned { + return fmt.Errorf("bidirectional diagnostic cap %s does not match the planned value", name) + } + } + if document.Counters == nil { + return fmt.Errorf("bidirectional diagnostic counters are missing") + } + if err := validateBidirectionalDiagnosticCounts(document.Counters); err != nil { + return err + } + if err := validateBidirectionalSingleCallAggregate(document.Counters, document.Calls[0]); err != nil { + return err + } + + fallbackIdentity := bidirectionalFallbackIdentity(bidirectionalTelemetryIdentity(telemetry.Summary)) + if *document.FallbackExecuted { + if fallbackIdentity == "" { + return fmt.Errorf("bidirectional diagnostic reports fallback without a declared exact control") + } + if !slices.Contains(telemetry.Summary.PlannedIdentities, fallbackIdentity) { + telemetry.Summary.PlannedIdentities = append(telemetry.Summary.PlannedIdentities, fallbackIdentity) + } + telemetry.Summary.RuntimeIdentity = fallbackIdentity + telemetry.Summary.AppliedIdentity = fallbackIdentity + telemetry.Summary.FallbackIdentity = fallbackIdentity + telemetry.Summary.Provenance["fallback_identity"] = postgresBidirectionalDiagnosticSource + ".fallback_executed" + } else { + identity := bidirectionalTelemetryIdentity(telemetry.Summary) + telemetry.Summary.RuntimeIdentity = identity + telemetry.Summary.AppliedIdentity = identity + telemetry.Summary.FallbackIdentity = "" + } + telemetry.Summary.RuntimeBranch = document.RuntimeBranch + telemetry.Summary.RuntimeOutcomeAvailable = traversalTelemetryPointer(true) + telemetry.Summary.Overflow = traversalTelemetryPointer(*document.Overflowed) + telemetry.Summary.FallbackExecuted = traversalTelemetryPointer(*document.FallbackExecuted) + for _, name := range []string{"runtime_identity", "applied_identity", "runtime_branch", "overflow", "fallback_executed", "scheduler_version"} { + telemetry.Summary.Provenance[name] = postgresBidirectionalDiagnosticSource + } + telemetry.Summary.Provenance["runtime_outcome_available"] = postgresBidirectionalDiagnosticSource + + if telemetry.Diagnostic == nil { + return nil + } + levels := make([]ShortestPathLevelCounters, len(document.Counters.Levels)) + for idx, level := range document.Counters.Levels { + levels[idx] = ShortestPathLevelCounters{ + SearchID: *level.SearchID, + ActionIndex: *level.ActionIndex, + Side: level.Side, + Action: level.Action, + Depth: level.Depth, + FrontierRows: level.FrontierRows, + CandidateEdges: level.CandidateEdges, + DistinctNewNodes: level.DistinctNewNodes, + SeenRows: level.SeenRows, + QueueRows: level.QueueRows, + PredecessorRows: level.PredecessorRows, + MeetingCandidates: level.MeetingCandidates, + Provenance: fmt.Sprintf("%s.counters.levels[%d]", postgresBidirectionalDiagnosticSource, idx), + } + } + telemetry.Diagnostic.CounterStatus = TraversalTelemetryCounterStatusComplete + telemetry.Diagnostic.IncompleteReasons = nil + telemetry.Diagnostic.RequiredFamilies = traversalRequiredFamilies(telemetry.Summary, TraversalTelemetryFamilySP) + telemetry.Diagnostic.Counters = TraversalDiagnosticCounters{ShortestPath: &ShortestPathTraversalCounters{ + SchedulerActions: document.Counters.SchedulerActions, + Levels: levels, + CandidateEdges: document.Counters.CandidateEdges, + DistinctNewNodes: document.Counters.DistinctNewNodes, + SeenPeak: document.Counters.SeenPeak, + FrontierPeak: document.Counters.FrontierPeak, + QueuePeak: document.Counters.QueuePeak, + PredecessorPeak: document.Counters.PredecessorPeak, + MeetingCandidates: document.Counters.MeetingCandidates, + FrozenDistance: document.Counters.FrozenDistance, + WitnessRows: document.Counters.WitnessRows, + FallbackExecuted: document.FallbackExecuted, + }} + telemetry.Diagnostic.Counters.Workspace = &TraversalWorkspaceCounters{ + SessionPeakBytes: traversalTelemetryPointer(document.WorkspaceBytes), + PoolPeakBytes: traversalTelemetryPointer(document.WorkspaceBytes), + } + telemetry.Diagnostic.Provenance = map[string]string{} + for _, name := range []string{ + "scheduler_actions", "candidate_edges", "distinct_new_nodes", "seen_peak", "frontier_peak", "queue_peak", + "predecessor_peak", "meeting_candidates", "frozen_distance", "witness_rows", "fallback_executed", + } { + telemetry.Diagnostic.Provenance["shortest_path."+name] = postgresBidirectionalDiagnosticSource + ".counters." + name + } + telemetry.Diagnostic.Provenance["workspace.session_peak_bytes"] = "pg_total_relation_size(pg_temp.spb_*)" + telemetry.Diagnostic.Provenance["workspace.pool_peak_bytes"] = "single_connection_diagnostic_pool.session_peak_bytes" + if *document.FallbackExecuted { + // The document completely describes bounded B-candidate work and the + // exact-fallback decision, but the nested S4 executor does not yet emit + // its own edge/state counters. Keep the measured candidate evidence and + // fail total-work qualification closed. + telemetry.Diagnostic.CounterStatus = TraversalTelemetryCounterStatusHiddenUnavailable + telemetry.Diagnostic.IncompleteReasons = []string{"nested exact S4 fallback traversal work counters are unavailable"} + } + if slices.Contains(telemetry.Diagnostic.RequiredFamilies, TraversalTelemetryFamilyHydration) { + telemetry.Diagnostic.CounterStatus = TraversalTelemetryCounterStatusHiddenUnavailable + telemetry.Diagnostic.IncompleteReasons = append(telemetry.Diagnostic.IncompleteReasons, "complete invocation-local path hydration counters are unavailable") + } + return nil +} + +func validateBidirectionalDiagnosticCounts(counters *postgresBidirectionalDiagnosticCounts) error { + for name, value := range map[string]*int64{ + "scheduler_actions": counters.SchedulerActions, "candidate_edges": counters.CandidateEdges, + "distinct_new_nodes": counters.DistinctNewNodes, "seen_peak": counters.SeenPeak, + "frontier_peak": counters.FrontierPeak, "queue_peak": counters.QueuePeak, + "predecessor_peak": counters.PredecessorPeak, "meeting_candidates": counters.MeetingCandidates, + "witness_rows": counters.WitnessRows, + } { + if value == nil || *value < 0 { + return fmt.Errorf("bidirectional diagnostic counter %s is missing or negative", name) + } + } + if counters.FrozenDistance == nil || *counters.FrozenDistance < -1 { + return fmt.Errorf("bidirectional diagnostic frozen_distance is missing or invalid") + } + if len(counters.Levels) == 0 { + return fmt.Errorf("bidirectional diagnostic level counters are missing") + } + for idx, level := range counters.Levels { + if level.SearchID == nil || level.ActionIndex == nil || *level.SearchID < 1 || *level.ActionIndex < 1 || + strings.TrimSpace(level.Side) == "" || strings.TrimSpace(level.Action) == "" { + return fmt.Errorf("bidirectional diagnostic level %d has incomplete identity", idx) + } + for name, value := range map[string]*int64{ + "depth": level.Depth, "frontier_rows": level.FrontierRows, "candidate_edges": level.CandidateEdges, + "distinct_new_nodes": level.DistinctNewNodes, "seen_rows": level.SeenRows, "queue_rows": level.QueueRows, + "predecessor_rows": level.PredecessorRows, "meeting_candidates": level.MeetingCandidates, + } { + if value == nil || *value < 0 { + return fmt.Errorf("bidirectional diagnostic level %d counter %s is missing or negative", idx, name) + } + } + } + return nil +} + +func validateBidirectionalDiagnosticCalls(calls []postgresBidirectionalDiagnosticCall, overflowed, fallbackExecuted *bool) error { + return validateBidirectionalDiagnosticCallsFor(calls, overflowed, fallbackExecuted, "exact_s4_fallback") +} + +func validateBidirectionalDiagnosticCallsFor(calls []postgresBidirectionalDiagnosticCall, overflowed, fallbackExecuted *bool, exactFallback string) error { + seen := map[int64]struct{}{} + anyOverflow, anyFallback := false, false + for idx, call := range calls { + if call.SearchID == nil || *call.SearchID < 1 || call.SourceID == nil || call.TargetID == nil { + return fmt.Errorf("bidirectional diagnostic call %d has incomplete identity", idx) + } + if _, duplicate := seen[*call.SearchID]; duplicate { + return fmt.Errorf("bidirectional diagnostic call %d repeats search_id %d", idx, *call.SearchID) + } + seen[*call.SearchID] = struct{}{} + if call.RuntimeBranch == "" || call.RuntimeBranch == "started" { + return fmt.Errorf("bidirectional diagnostic call %d did not finish", idx) + } + for name, value := range map[string]*int64{ + "scheduler_actions": call.SchedulerActions, "candidate_edges": call.CandidateEdges, + "distinct_new_nodes": call.DistinctNewNodes, "seen_peak": call.SeenPeak, + "frontier_peak": call.FrontierPeak, "queue_peak": call.QueuePeak, + "predecessor_peak": call.PredecessorPeak, "meeting_candidates": call.MeetingCandidates, + "witness_rows": call.WitnessRows, + } { + if value == nil || *value < 0 { + return fmt.Errorf("bidirectional diagnostic call %d counter %s is missing or negative", idx, name) + } + } + if call.Overflowed == nil || call.FallbackExecuted == nil { + return fmt.Errorf("bidirectional diagnostic call %d outcome flags are missing", idx) + } + if err := validateDiagnosticRuntimeOutcome(call.RuntimeBranch, *call.Overflowed, *call.FallbackExecuted, exactFallback, []string{ + "preflight_zero_hop", "preflight_one_hop", "preflight_two_hop", "preflight_no_path", "search_no_path", "bidirectional_search", + }); err != nil { + return fmt.Errorf("bidirectional diagnostic call %d: %w", idx, err) + } + anyOverflow = anyOverflow || *call.Overflowed + anyFallback = anyFallback || *call.FallbackExecuted + } + if overflowed == nil || fallbackExecuted == nil || anyOverflow != *overflowed || anyFallback != *fallbackExecuted { + return fmt.Errorf("bidirectional diagnostic aggregate outcome differs from its calls") + } + return nil +} + +func validateDiagnosticRuntimeOutcome(branch string, overflowed, fallbackExecuted bool, exactFallback string, nonFallback []string) error { + allowed := slices.Contains(nonFallback, branch) || branch == exactFallback + if !allowed { + return fmt.Errorf("runtime branch %q is unsupported", branch) + } + if fallbackExecuted != (branch == exactFallback) { + return fmt.Errorf("runtime branch %q contradicts fallback_executed=%t", branch, fallbackExecuted) + } + if overflowed != fallbackExecuted { + return fmt.Errorf("overflowed=%t contradicts fallback_executed=%t", overflowed, fallbackExecuted) + } + return nil +} + +func validateBidirectionalSingleCallAggregate(counters *postgresBidirectionalDiagnosticCounts, call postgresBidirectionalDiagnosticCall) error { + for name, values := range map[string][2]*int64{ + "scheduler_actions": {counters.SchedulerActions, call.SchedulerActions}, + "candidate_edges": {counters.CandidateEdges, call.CandidateEdges}, + "distinct_new_nodes": {counters.DistinctNewNodes, call.DistinctNewNodes}, + "seen_peak": {counters.SeenPeak, call.SeenPeak}, "frontier_peak": {counters.FrontierPeak, call.FrontierPeak}, + "queue_peak": {counters.QueuePeak, call.QueuePeak}, "predecessor_peak": {counters.PredecessorPeak, call.PredecessorPeak}, + "meeting_candidates": {counters.MeetingCandidates, call.MeetingCandidates}, + "frozen_distance": {counters.FrozenDistance, call.FrozenDistance}, "witness_rows": {counters.WitnessRows, call.WitnessRows}, + } { + if values[0] == nil || values[1] == nil || *values[0] != *values[1] { + return fmt.Errorf("bidirectional diagnostic aggregate counter %s differs from its single call", name) + } + } + for idx, level := range counters.Levels { + if level.SearchID == nil || call.SearchID == nil || *level.SearchID != *call.SearchID { + return fmt.Errorf("bidirectional diagnostic level %d is not attributed to its single call", idx) + } + } + return nil +} + +func applyBidirectionalAllShortestTraversalDiagnostic( + telemetry *TraversalExecutionTelemetry, + document *postgresBidirectionalAllShortestDiagnosticDocument, + expectedInvocationID string, + expectedConnectionID string, +) error { + if telemetry == nil || document == nil { + return fmt.Errorf("bidirectional all-shortest diagnostic document is missing") + } + if document.SchemaVersion != 1 { + return fmt.Errorf("bidirectional all-shortest diagnostic schema_version must be 1") + } + if document.InvocationID != expectedInvocationID { + return fmt.Errorf("bidirectional all-shortest diagnostic invocation identity %q differs from requested %q", document.InvocationID, expectedInvocationID) + } + if telemetry.Diagnostic != nil && telemetry.Diagnostic.ConnectionID != expectedConnectionID { + return fmt.Errorf("attached diagnostic connection identity %q differs from replay connection %q", telemetry.Diagnostic.ConnectionID, expectedConnectionID) + } + if document.SearchCalls == nil || *document.SearchCalls != 1 { + return fmt.Errorf("instrumented singleton ASP-B1/B2 replay must invoke exactly one search call") + } + if int64(len(document.Calls)) != *document.SearchCalls { + return fmt.Errorf("bidirectional all-shortest diagnostic call count %d differs from search_calls %d", len(document.Calls), *document.SearchCalls) + } + if document.RuntimeBranch == "" || document.RuntimeBranch == "missing" || document.RuntimeBranch == "mixed" { + return fmt.Errorf("bidirectional all-shortest diagnostic runtime branch is not singular") + } + if document.Overflowed == nil || document.FallbackExecuted == nil { + return fmt.Errorf("bidirectional all-shortest diagnostic runtime outcome flags are missing") + } + if err := validateDiagnosticRuntimeOutcome(document.RuntimeBranch, *document.Overflowed, *document.FallbackExecuted, "exact_a1_fallback", []string{ + "preflight_one_hop", "preflight_two_hop", "preflight_no_path", "search_no_path", "bidirectional_search", + }); err != nil { + return fmt.Errorf("bidirectional all-shortest diagnostic: %w", err) + } + if strings.TrimSpace(document.Scheduler) == "" || document.Scheduler != telemetry.Summary.SchedulerVersion { + return fmt.Errorf("bidirectional all-shortest diagnostic scheduler %q differs from planned scheduler %q", document.Scheduler, telemetry.Summary.SchedulerVersion) + } + for name, observed := range map[string]*int64{ + "state_rows": document.StateLimit, + "frontier_rows": document.FrontierLimit, + "queue_rows": document.FrontierLimit, + "predecessor_rows": document.PredecessorLimit, + "output_rows": document.EnumerationLimit, + "output_bytes": document.OutputBytesLimit, + } { + planned, ok := telemetry.Summary.Caps[name] + if !ok || observed == nil || *observed != planned { + return fmt.Errorf("bidirectional all-shortest diagnostic cap %s does not match the planned value", name) + } + } + if document.Counters == nil { + return fmt.Errorf("bidirectional all-shortest diagnostic counters are missing") + } + if err := validateBidirectionalAllShortestDiagnosticCounts(document.Counters); err != nil { + return err + } + if err := validateBidirectionalAllShortestDiagnosticCalls(document.Calls, document.Overflowed, document.FallbackExecuted); err != nil { + return err + } + if document.Calls[0].RuntimeBranch != document.RuntimeBranch { + return fmt.Errorf("bidirectional all-shortest diagnostic aggregate runtime branch differs from its call") + } + if err := validateBidirectionalAllShortestSingleCallAggregate(document.Counters, document.Calls[0]); err != nil { + return err + } + + fallbackIdentity := bidirectionalFallbackIdentity(bidirectionalTelemetryIdentity(telemetry.Summary)) + if *document.FallbackExecuted { + if fallbackIdentity == "" { + return fmt.Errorf("bidirectional all-shortest diagnostic reports fallback without a declared exact control") + } + if !slices.Contains(telemetry.Summary.PlannedIdentities, fallbackIdentity) { + telemetry.Summary.PlannedIdentities = append(telemetry.Summary.PlannedIdentities, fallbackIdentity) + } + telemetry.Summary.RuntimeIdentity = fallbackIdentity + telemetry.Summary.AppliedIdentity = fallbackIdentity + telemetry.Summary.FallbackIdentity = fallbackIdentity + telemetry.Summary.Provenance["fallback_identity"] = postgresBidirectionalAllShortestDiagnosticSource + ".fallback_executed" + } else { + identity := bidirectionalTelemetryIdentity(telemetry.Summary) + telemetry.Summary.RuntimeIdentity = identity + telemetry.Summary.AppliedIdentity = identity + telemetry.Summary.FallbackIdentity = "" + } + telemetry.Summary.RuntimeOutcomeAvailable = traversalTelemetryPointer(true) + telemetry.Summary.RuntimeBranch = document.RuntimeBranch + telemetry.Summary.Overflow = traversalTelemetryPointer(*document.Overflowed) + telemetry.Summary.FallbackExecuted = traversalTelemetryPointer(*document.FallbackExecuted) + for _, name := range []string{"runtime_identity", "applied_identity", "runtime_branch", "overflow", "fallback_executed", "scheduler_version", "runtime_outcome_available"} { + telemetry.Summary.Provenance[name] = postgresBidirectionalAllShortestDiagnosticSource + } + + if telemetry.Diagnostic == nil { + return nil + } + search := shortestPathCountersFromAllShortest(document.Counters, document.FallbackExecuted) + telemetry.Diagnostic.RequiredFamilies = traversalRequiredFamilies(telemetry.Summary, TraversalTelemetryFamilyASP) + telemetry.Diagnostic.Counters = TraversalDiagnosticCounters{AllShortestPaths: &AllShortestPathsTraversalCounters{ + Search: search, + SameDepthPredecessorAdditions: document.Counters.SameDepthPredecessorAdditions, + PredecessorPeak: document.Counters.PredecessorPeak, + MeetingNodes: document.Counters.MeetingNodes, + CutDepth: document.Counters.CutDepth, + PathCountEstimate: document.Counters.PathCountEstimate, + PathCountSaturated: document.Counters.PathCountSaturated, + EnumeratedCandidates: document.Counters.EnumeratedCandidates, + DuplicateRejects: document.Counters.DuplicateRejects, + OutputPaths: document.Counters.OutputPaths, + OutputEdgeCells: document.Counters.OutputEdgeCells, + OutputBytes: document.Counters.OutputBytes, + }} + telemetry.Diagnostic.Counters.Workspace = &TraversalWorkspaceCounters{ + SessionPeakBytes: traversalTelemetryPointer(document.WorkspaceBytes), + PoolPeakBytes: traversalTelemetryPointer(document.WorkspaceBytes), + } + telemetry.Diagnostic.Provenance = map[string]string{} + for _, name := range []string{ + "scheduler_actions", "candidate_edges", "distinct_new_nodes", "seen_peak", "frontier_peak", "queue_peak", + "predecessor_peak", "meeting_candidates", "frozen_distance", "witness_rows", "fallback_executed", + } { + telemetry.Diagnostic.Provenance["all_shortest_paths.search."+name] = postgresBidirectionalAllShortestDiagnosticSource + ".counters." + name + } + telemetry.Diagnostic.Provenance["workspace.session_peak_bytes"] = "pg_total_relation_size(pg_temp.asb_*)" + telemetry.Diagnostic.Provenance["workspace.pool_peak_bytes"] = "single_connection_diagnostic_pool.session_peak_bytes" + for _, name := range []string{ + "same_depth_predecessor_additions", "predecessor_peak", "meeting_nodes", "cut_depth", "path_count_estimate", + "path_count_saturated", "enumerated_candidates", "duplicate_rejects", "output_paths", "output_edge_cells", "output_bytes", + } { + telemetry.Diagnostic.Provenance["all_shortest_paths."+name] = postgresBidirectionalAllShortestDiagnosticSource + ".counters." + name + } + telemetry.Diagnostic.CounterStatus = TraversalTelemetryCounterStatusHiddenUnavailable + telemetry.Diagnostic.IncompleteReasons = []string{ + "complete invocation-local path hydration counters are unavailable", + } + if *document.FallbackExecuted { + telemetry.Diagnostic.IncompleteReasons = append(telemetry.Diagnostic.IncompleteReasons, "nested exact ASP-A1 fallback traversal work counters are unavailable") + } + return nil +} + +func shortestPathCountersFromAllShortest(counters *postgresBidirectionalAllShortestDiagnosticCounts, fallbackExecuted *bool) ShortestPathTraversalCounters { + levels := make([]ShortestPathLevelCounters, len(counters.Levels)) + for idx, level := range counters.Levels { + levels[idx] = ShortestPathLevelCounters{ + SearchID: *level.SearchID, ActionIndex: *level.ActionIndex, Side: level.Side, Action: level.Action, + Depth: level.Depth, FrontierRows: level.FrontierRows, CandidateEdges: level.CandidateEdges, + DistinctNewNodes: level.DistinctNewNodes, SeenRows: level.SeenRows, QueueRows: level.QueueRows, + PredecessorRows: level.PredecessorRows, MeetingCandidates: level.MeetingCandidates, + Provenance: fmt.Sprintf("%s.counters.levels[%d]", postgresBidirectionalAllShortestDiagnosticSource, idx), + } + } + return ShortestPathTraversalCounters{ + SchedulerActions: counters.SchedulerActions, Levels: levels, CandidateEdges: counters.CandidateEdges, + DistinctNewNodes: counters.DistinctNewNodes, SeenPeak: counters.SeenPeak, FrontierPeak: counters.FrontierPeak, + QueuePeak: counters.QueuePeak, PredecessorPeak: counters.PredecessorPeak, MeetingCandidates: counters.MeetingCandidates, + FrozenDistance: counters.FrozenDistance, WitnessRows: counters.WitnessRows, FallbackExecuted: fallbackExecuted, + } +} + +func validateBidirectionalAllShortestDiagnosticCounts(counters *postgresBidirectionalAllShortestDiagnosticCounts) error { + if counters == nil { + return fmt.Errorf("bidirectional all-shortest diagnostic counters are missing") + } + if err := validateBidirectionalDiagnosticCounts(&postgresBidirectionalDiagnosticCounts{ + SchedulerActions: counters.SchedulerActions, CandidateEdges: counters.CandidateEdges, + DistinctNewNodes: counters.DistinctNewNodes, SeenPeak: counters.SeenPeak, FrontierPeak: counters.FrontierPeak, + QueuePeak: counters.QueuePeak, PredecessorPeak: counters.PredecessorPeak, MeetingCandidates: counters.MeetingCandidates, + FrozenDistance: counters.FrozenDistance, WitnessRows: counters.WitnessRows, Levels: counters.Levels, + }); err != nil { + return err + } + for name, value := range map[string]*int64{ + "same_depth_predecessor_additions": counters.SameDepthPredecessorAdditions, + "meeting_nodes": counters.MeetingNodes, "path_count_estimate": counters.PathCountEstimate, + "enumerated_candidates": counters.EnumeratedCandidates, "duplicate_rejects": counters.DuplicateRejects, + "output_paths": counters.OutputPaths, "output_edge_cells": counters.OutputEdgeCells, "output_bytes": counters.OutputBytes, + } { + if value == nil || *value < 0 { + return fmt.Errorf("bidirectional all-shortest diagnostic counter %s is missing or negative", name) + } + } + if counters.CutDepth == nil || *counters.CutDepth < -1 { + return fmt.Errorf("bidirectional all-shortest diagnostic cut_depth is missing or invalid") + } + if counters.PathCountSaturated == nil { + return fmt.Errorf("bidirectional all-shortest diagnostic path_count_saturated is missing") + } + return nil +} + +func validateBidirectionalAllShortestDiagnosticCalls(calls []postgresBidirectionalAllShortestDiagnosticCall, overflowed, fallbackExecuted *bool) error { + baseCalls := make([]postgresBidirectionalDiagnosticCall, len(calls)) + for idx, call := range calls { + baseCalls[idx] = postgresBidirectionalDiagnosticCall{ + SearchID: call.SearchID, SourceID: call.SourceID, TargetID: call.TargetID, RuntimeBranch: call.RuntimeBranch, + SchedulerActions: call.SchedulerActions, CandidateEdges: call.CandidateEdges, DistinctNewNodes: call.DistinctNewNodes, + SeenPeak: call.SeenPeak, FrontierPeak: call.FrontierPeak, QueuePeak: call.QueuePeak, + PredecessorPeak: call.PredecessorPeak, MeetingCandidates: call.MeetingCandidates, + FrozenDistance: call.FrozenDistance, WitnessRows: call.WitnessRows, + Overflowed: call.Overflowed, FallbackExecuted: call.FallbackExecuted, + } + } + if err := validateBidirectionalDiagnosticCallsFor(baseCalls, overflowed, fallbackExecuted, "exact_a1_fallback"); err != nil { + return err + } + for idx, call := range calls { + for name, value := range map[string]*int64{ + "same_depth_predecessor_additions": call.SameDepthPredecessorAdditions, + "meeting_nodes": call.MeetingNodes, "path_count_estimate": call.PathCountEstimate, + "enumerated_candidates": call.EnumeratedCandidates, "duplicate_rejects": call.DuplicateRejects, + "output_paths": call.OutputPaths, "output_edge_cells": call.OutputEdgeCells, "output_bytes": call.OutputBytes, + } { + if value == nil || *value < 0 { + return fmt.Errorf("bidirectional all-shortest diagnostic call %d counter %s is missing or negative", idx, name) + } + } + if call.CutDepth == nil || *call.CutDepth < -1 || call.PathCountSaturated == nil { + return fmt.Errorf("bidirectional all-shortest diagnostic call %d has incomplete cut/count state", idx) + } + } + return nil +} + +func validateBidirectionalAllShortestSingleCallAggregate(counters *postgresBidirectionalAllShortestDiagnosticCounts, call postgresBidirectionalAllShortestDiagnosticCall) error { + if err := validateBidirectionalSingleCallAggregate(&postgresBidirectionalDiagnosticCounts{ + SchedulerActions: counters.SchedulerActions, CandidateEdges: counters.CandidateEdges, + DistinctNewNodes: counters.DistinctNewNodes, SeenPeak: counters.SeenPeak, FrontierPeak: counters.FrontierPeak, + QueuePeak: counters.QueuePeak, PredecessorPeak: counters.PredecessorPeak, MeetingCandidates: counters.MeetingCandidates, + FrozenDistance: counters.FrozenDistance, WitnessRows: counters.WitnessRows, Levels: counters.Levels, + }, postgresBidirectionalDiagnosticCall{ + SearchID: call.SearchID, SchedulerActions: call.SchedulerActions, CandidateEdges: call.CandidateEdges, + DistinctNewNodes: call.DistinctNewNodes, SeenPeak: call.SeenPeak, FrontierPeak: call.FrontierPeak, + QueuePeak: call.QueuePeak, PredecessorPeak: call.PredecessorPeak, MeetingCandidates: call.MeetingCandidates, + FrozenDistance: call.FrozenDistance, WitnessRows: call.WitnessRows, + }); err != nil { + return err + } + for name, values := range map[string][2]*int64{ + "same_depth_predecessor_additions": {counters.SameDepthPredecessorAdditions, call.SameDepthPredecessorAdditions}, + "meeting_nodes": {counters.MeetingNodes, call.MeetingNodes}, "cut_depth": {counters.CutDepth, call.CutDepth}, + "path_count_estimate": {counters.PathCountEstimate, call.PathCountEstimate}, + "enumerated_candidates": {counters.EnumeratedCandidates, call.EnumeratedCandidates}, + "duplicate_rejects": {counters.DuplicateRejects, call.DuplicateRejects}, + "output_paths": {counters.OutputPaths, call.OutputPaths}, "output_edge_cells": {counters.OutputEdgeCells, call.OutputEdgeCells}, + "output_bytes": {counters.OutputBytes, call.OutputBytes}, + } { + if values[0] == nil || values[1] == nil || *values[0] != *values[1] { + return fmt.Errorf("bidirectional all-shortest diagnostic aggregate counter %s differs from its single call", name) + } + } + if counters.PathCountSaturated == nil || call.PathCountSaturated == nil || *counters.PathCountSaturated != *call.PathCountSaturated { + return fmt.Errorf("bidirectional all-shortest diagnostic aggregate path_count_saturated differs from its single call") + } + return nil +} + +func markTraversalCountersUnavailable(diagnostic *TraversalExecutionDiagnostic, reason string) { + if diagnostic == nil { + return + } + diagnostic.CounterStatus = TraversalTelemetryCounterStatusHiddenUnavailable + diagnostic.IncompleteReasons = []string{reason} + diagnostic.Counters = TraversalDiagnosticCounters{} + diagnostic.Provenance = map[string]string{} +} + +func markTraversalSummaryUnavailable(telemetry *TraversalExecutionTelemetry, reason string) { + if telemetry == nil { + return + } + telemetry.Summary.RuntimeOutcomeAvailable = traversalTelemetryPointer(false) + telemetry.Summary.RuntimeIdentity = "" + telemetry.Summary.AppliedIdentity = "" + telemetry.Summary.RuntimeBranch = "runtime_outcome_unavailable" + telemetry.Summary.Overflow = nil + telemetry.Summary.FallbackExecuted = nil + telemetry.Summary.FallbackIdentity = "" + for _, name := range []string{"runtime_identity", "applied_identity", "runtime_branch", "runtime_outcome_available"} { + telemetry.Summary.Provenance[name] = "runtime_outcome_unavailable:" + reason + } + delete(telemetry.Summary.Provenance, "overflow") + delete(telemetry.Summary.Provenance, "fallback_executed") + delete(telemetry.Summary.Provenance, "fallback_identity") +} + +func isBidirectionalSPIdentity(identity string) bool { + return strings.HasPrefix(identity, "SP-B1-") || strings.HasPrefix(identity, "SP-B2-") +} + +func bidirectionalFallbackIdentity(identity string) string { + if isBidirectionalASPIdentity(identity) { + return "ASP-A1-DAG" + } + if isBidirectionalSPIdentity(identity) { + if strings.Contains(identity, "WE+") { + return "SP-S4-C-WE+MAT-M0" + } + return "SP-S4-C-D" + } + return "" +} + +func isBidirectionalASPIdentity(identity string) bool { + return strings.HasPrefix(identity, "ASP-B1-") || strings.HasPrefix(identity, "ASP-B2-") +} + +func traversalTelemetryPointer[T any](value T) *T { + return &value +} diff --git a/cmd/graphbench/postgres_traversal_telemetry_test.go b/cmd/graphbench/postgres_traversal_telemetry_test.go new file mode 100644 index 00000000..aca7f140 --- /dev/null +++ b/cmd/graphbench/postgres_traversal_telemetry_test.go @@ -0,0 +1,974 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/stretchr/testify/require" +) + +func TestPostgresTraversalTelemetryCompletesBidirectionalCandidateIdentityChain(t *testing.T) { + telemetry := bidirectionalCaseTelemetry(t, TraversalTelemetryLevelDiagnostic) + require.Equal(t, TraversalTelemetryCounterStatusHiddenUnavailable, telemetry.Diagnostic.CounterStatus) + + document := validBidirectionalDiagnosticDocument(telemetry.Diagnostic.InvocationID) + require.NoError(t, applyBidirectionalTraversalDiagnostic(telemetry, document, telemetry.Diagnostic.InvocationID, "9123")) + require.NoError(t, telemetry.Validate()) + + require.Equal(t, "SP-B2-C-MIN-LEVEL-D", telemetry.Summary.RequestedIdentity) + require.Equal(t, "SP-B2-C-MIN-LEVEL-D", telemetry.Summary.RuntimeIdentity) + require.Equal(t, "SP-B2-C-MIN-LEVEL-D", telemetry.Summary.AppliedIdentity) + require.Equal(t, "bidirectional_search", telemetry.Summary.RuntimeBranch) + require.False(t, *telemetry.Summary.FallbackExecuted) + require.Equal(t, TraversalTelemetryCounterStatusComplete, telemetry.Diagnostic.CounterStatus) + require.Contains(t, telemetry.Diagnostic.RequiredFamilies, TraversalTelemetryFamilyWorkspace) + require.False(t, *telemetry.Diagnostic.TimedSample) + require.Equal(t, int64(7), *telemetry.Diagnostic.Counters.ShortestPath.CandidateEdges) + require.Equal(t, int64(4), *telemetry.Diagnostic.Counters.ShortestPath.PredecessorPeak) + require.Equal(t, int64(4), *telemetry.Diagnostic.Counters.ShortestPath.Levels[0].PredecessorRows) + require.NotNil(t, telemetry.Diagnostic.Counters.Workspace) + observed := traversalNumericObservations(telemetry.Diagnostic.Counters) + require.Equal(t, int64(6), observed["state_rows"]) + require.Equal(t, int64(3), observed["frontier_rows"]) + require.Equal(t, int64(3), observed["queue_rows"]) + require.Equal(t, int64(4), observed["predecessor_rows"]) +} + +func TestPostgresTraversalTelemetryRebindsRuntimeIdentityOnExactFallback(t *testing.T) { + telemetry := bidirectionalCaseTelemetry(t, TraversalTelemetryLevelDiagnostic) + document := validBidirectionalDiagnosticDocument(telemetry.Diagnostic.InvocationID) + document.RuntimeBranch = "exact_s4_fallback" + document.Overflowed = traversalTelemetryPointer(true) + document.FallbackExecuted = traversalTelemetryPointer(true) + document.Calls[0].RuntimeBranch = "exact_s4_fallback" + document.Calls[0].Overflowed = traversalTelemetryPointer(true) + document.Calls[0].FallbackExecuted = traversalTelemetryPointer(true) + + require.NoError(t, applyBidirectionalTraversalDiagnostic(telemetry, document, telemetry.Diagnostic.InvocationID, "9123")) + require.NoError(t, telemetry.Validate()) + + require.Equal(t, "SP-S4-C-D", telemetry.Summary.RuntimeIdentity) + require.Equal(t, "SP-S4-C-D", telemetry.Summary.AppliedIdentity) + require.Equal(t, "SP-S4-C-D", telemetry.Summary.FallbackIdentity) + require.True(t, *telemetry.Summary.Overflow) + require.True(t, *telemetry.Summary.FallbackExecuted) + require.Contains(t, telemetry.Summary.PlannedIdentities, "SP-S4-C-D") + require.Equal(t, TraversalTelemetryCounterStatusHiddenUnavailable, telemetry.Diagnostic.CounterStatus) + require.Contains(t, telemetry.Diagnostic.IncompleteReasons[0], "S4 fallback") +} + +func TestPostgresTraversalTelemetryRejectsInvocationConnectionAndCapMismatch(t *testing.T) { + telemetry := bidirectionalCaseTelemetry(t, TraversalTelemetryLevelDiagnostic) + document := validBidirectionalDiagnosticDocument(telemetry.Diagnostic.InvocationID) + + err := applyBidirectionalTraversalDiagnostic(telemetry, document, "another-invocation", "9123") + require.ErrorContains(t, err, "invocation identity") + + telemetry = bidirectionalCaseTelemetry(t, TraversalTelemetryLevelDiagnostic) + document = validBidirectionalDiagnosticDocument(telemetry.Diagnostic.InvocationID) + err = applyBidirectionalTraversalDiagnostic(telemetry, document, telemetry.Diagnostic.InvocationID, "different-backend") + require.ErrorContains(t, err, "connection identity") + + telemetry = bidirectionalCaseTelemetry(t, TraversalTelemetryLevelDiagnostic) + document = validBidirectionalDiagnosticDocument(telemetry.Diagnostic.InvocationID) + document.FrontierLimit = traversalTelemetryPointer(int64(99)) + err = applyBidirectionalTraversalDiagnostic(telemetry, document, telemetry.Diagnostic.InvocationID, "9123") + require.ErrorContains(t, err, "cap") + + telemetry = bidirectionalCaseTelemetry(t, TraversalTelemetryLevelDiagnostic) + document = validBidirectionalDiagnosticDocument(telemetry.Diagnostic.InvocationID) + document.Counters = nil + err = applyBidirectionalTraversalDiagnostic(telemetry, document, telemetry.Diagnostic.InvocationID, "9123") + require.ErrorContains(t, err, "counters are missing") +} + +func TestPostgresTraversalTelemetryRequiresExactlyOneSingletonSearchCall(t *testing.T) { + telemetry := bidirectionalCaseTelemetry(t, TraversalTelemetryLevelDiagnostic) + document := validBidirectionalDiagnosticDocument(telemetry.Diagnostic.InvocationID) + document.SearchCalls = traversalTelemetryPointer(int64(2)) + document.Calls = append(document.Calls, document.Calls[0]) + document.Calls[1].SearchID = traversalTelemetryPointer(int64(2)) + + err := applyBidirectionalTraversalDiagnostic(telemetry, document, telemetry.Diagnostic.InvocationID, "9123") + require.ErrorContains(t, err, "exactly one search call") +} + +func TestPostgresTraversalTelemetryCapturesASPWorkAndWorkspaceButFailsClosedWithoutHydration(t *testing.T) { + telemetry := bidirectionalASPCaseTelemetry(t) + document := validBidirectionalAllShortestDiagnosticDocument(telemetry.Diagnostic.InvocationID) + + require.NoError(t, applyBidirectionalAllShortestTraversalDiagnostic(telemetry, document, telemetry.Diagnostic.InvocationID, "9123")) + require.NoError(t, telemetry.Validate()) + require.Equal(t, "ASP-B2-DAG-MIN-LEVEL", telemetry.Summary.RuntimeIdentity) + require.Equal(t, TraversalTelemetryCounterStatusHiddenUnavailable, telemetry.Diagnostic.CounterStatus) + require.Equal(t, []TraversalTelemetryFamily{ + TraversalTelemetryFamilyASP, + TraversalTelemetryFamilyHydration, + TraversalTelemetryFamilyWorkspace, + }, telemetry.Diagnostic.RequiredFamilies) + require.Equal(t, int64(13), *telemetry.Diagnostic.Counters.AllShortestPaths.EnumeratedCandidates) + require.Equal(t, int64(384), *telemetry.Diagnostic.Counters.AllShortestPaths.OutputBytes) + require.Nil(t, telemetry.Diagnostic.Counters.Hydration) + require.NotNil(t, telemetry.Diagnostic.Counters.Workspace) +} + +func TestPostgresTraversalTelemetryCompletesASPHydrationFromInvocationAndPlanEvidence(t *testing.T) { + telemetry := bidirectionalASPCaseTelemetry(t) + document := validBidirectionalAllShortestDiagnosticDocument(telemetry.Diagnostic.InvocationID) + require.NoError(t, applyBidirectionalAllShortestTraversalDiagnostic(telemetry, document, telemetry.Diagnostic.InvocationID, "9123")) + metrics := PostgresPlanMetrics{HydrationRows: 48, HydrationLoops: 12, PlanNodes: []PostgresPlanNodeMetric{{ + NodeType: "Index Scan", RelationName: "node", Alias: "hydrated_nodes", ActualRows: 4, ActualLoops: 12, ActualTotalMS: .25, + }}} + enrichBidirectionalHydrationTelemetry(telemetry, document.Counters.OutputPaths, document.Counters.OutputEdgeCells, []string{`["p1"]`, `["p2"]`}, metrics) + require.NoError(t, telemetry.Validate()) + require.Equal(t, TraversalTelemetryCounterStatusComplete, telemetry.Diagnostic.CounterStatus) + require.Equal(t, int64(12), *telemetry.Diagnostic.Counters.Hydration.PathCount) + require.Equal(t, int64(36), *telemetry.Diagnostic.Counters.Hydration.EdgeLookups) + require.Equal(t, int64(48), *telemetry.Diagnostic.Counters.Hydration.NodeLookups) +} + +func TestPostgresTraversalTelemetryRebindsASPExactFallbackAndRejectsMissingCounters(t *testing.T) { + telemetry := bidirectionalASPCaseTelemetry(t) + document := validBidirectionalAllShortestDiagnosticDocument(telemetry.Diagnostic.InvocationID) + document.RuntimeBranch = "exact_a1_fallback" + document.Overflowed = traversalTelemetryPointer(true) + document.FallbackExecuted = traversalTelemetryPointer(true) + document.Calls[0].RuntimeBranch = "exact_a1_fallback" + document.Calls[0].Overflowed = traversalTelemetryPointer(true) + document.Calls[0].FallbackExecuted = traversalTelemetryPointer(true) + + require.NoError(t, applyBidirectionalAllShortestTraversalDiagnostic(telemetry, document, telemetry.Diagnostic.InvocationID, "9123")) + require.NoError(t, telemetry.Validate()) + require.Equal(t, "ASP-A1-DAG", telemetry.Summary.RuntimeIdentity) + require.Equal(t, "ASP-A1-DAG", telemetry.Summary.FallbackIdentity) + require.Contains(t, telemetry.Diagnostic.IncompleteReasons, "nested exact ASP-A1 fallback traversal work counters are unavailable") + + telemetry = bidirectionalASPCaseTelemetry(t) + document = validBidirectionalAllShortestDiagnosticDocument(telemetry.Diagnostic.InvocationID) + document.Counters.OutputBytes = nil + err := applyBidirectionalAllShortestTraversalDiagnostic(telemetry, document, telemetry.Diagnostic.InvocationID, "9123") + require.ErrorContains(t, err, "output_bytes") +} + +func TestPostgresTraversalTelemetryWitnessRequiresSeparateHydrationEvidence(t *testing.T) { + telemetry := bidirectionalCaseTelemetry(t, TraversalTelemetryLevelDiagnostic) + telemetry.Summary.RequestedIdentity = "SP-B2-C-MIN-LEVEL-WE+MAT-M0" + telemetry.Summary.PlannedIdentities = []string{"SP-B2-C-MIN-LEVEL-WE+MAT-M0", "SP-S4-C-WE+MAT-M0"} + telemetry.Summary.EmittedIdentity = "SP-B2-C-MIN-LEVEL-WE+MAT-M0" + document := validBidirectionalDiagnosticDocument(telemetry.Diagnostic.InvocationID) + + require.NoError(t, applyBidirectionalTraversalDiagnostic(telemetry, document, telemetry.Diagnostic.InvocationID, "9123")) + require.NoError(t, telemetry.Validate()) + require.Contains(t, telemetry.Diagnostic.RequiredFamilies, TraversalTelemetryFamilyHydration) + require.Equal(t, TraversalTelemetryCounterStatusHiddenUnavailable, telemetry.Diagnostic.CounterStatus) + require.Contains(t, telemetry.Diagnostic.IncompleteReasons, "complete invocation-local path hydration counters are unavailable") +} + +func TestPostgresTraversalTelemetryLeavesNonBidirectionalHiddenFunctionsUnavailable(t *testing.T) { + metrics := PostgresPlanMetrics{ + PlanNodes: []PostgresPlanNodeMetric{{NodeType: "Function Scan", FunctionName: "all_shortest_paths_dag", ActualLoops: 1}}, + Provenance: map[string]string{}, + } + reference := PostgresReferenceResult{ + Architecture: "ASP-A1-DAG", + ImplementationID: "typed_predecessor_dag_v1", + PostgresMetrics: &metrics, + } + + telemetry, err := buildPostgresReferenceTraversalTelemetry(reference, nil, "9123", TraversalTelemetryLevelDiagnostic) + require.NoError(t, err) + require.NoError(t, telemetry.Validate()) + require.Equal(t, TraversalTelemetryCounterStatusHiddenUnavailable, telemetry.Diagnostic.CounterStatus) + require.Nil(t, telemetry.Diagnostic.Counters.AllShortestPaths) + require.Contains(t, telemetry.Diagnostic.IncompleteReasons[0], "Function Scan") +} + +func TestPostgresTraversalTelemetryUsesPlanReplayForSQLVisibleOrientation(t *testing.T) { + outcome := translate.TargetLoweringOutcome{ + Family: "fixed_suffix_expansion", + Candidate: "EXPANSION-SUFFIX-SEEDED-REVERSE", + Selected: "EXPANSION-STEPWISE-FORWARD", + Applied: "EXPANSION-STEPWISE-FORWARD", + Fallback: "EXPANSION-STEPWISE-FORWARD", + PlannedCandidates: []string{"EXPANSION-SUFFIX-SEEDED-REVERSE", "EXPANSION-STEPWISE-FORWARD"}, + EmittedCandidates: []string{"EXPANSION-SUFFIX-SEEDED-REVERSE", "EXPANSION-STEPWISE-FORWARD"}, + EmittedPolicy: "orientation-probe-v1", + SelectorVersion: "orientation-probe-v1", + ExecutionBoundary: "guarded_dual_arm", + StateLimit: 4096, + } + metrics := PostgresPlanMetrics{ + PlanNodes: []PostgresPlanNodeMetric{{ + NodeType: "Result", + SubplanName: "CTE s5_orientation_executed_candidate", + ActualRows: 1, + ActualLoops: 1, + }}, + Provenance: map[string]string{}, + } + + telemetry, err := buildPostgresCaseTraversalTelemetry( + translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, + metrics, + "9123", + TraversalTelemetryLevelDiagnostic, + ) + require.NoError(t, err) + require.NoError(t, telemetry.Validate()) + require.Equal(t, TraversalTelemetryFamilyOrientation, telemetry.Diagnostic.RequiredFamilies[0]) + require.Equal(t, TraversalTelemetryCounterStatusPlanPartial, telemetry.Diagnostic.CounterStatus) + require.Equal(t, "EXPANSION-SUFFIX-SEEDED-REVERSE", telemetry.Summary.RuntimeIdentity) + require.Equal(t, telemetry.Summary.RuntimeIdentity, telemetry.Summary.AppliedIdentity) + require.Equal(t, "guarded_dual_arm", telemetry.Summary.ExecutionBoundary) + require.Equal(t, int64(1), telemetry.Diagnostic.PlanReplay.Counters["orientation_executed_candidate_rows"]) +} + +func TestPostgresTraversalTelemetryKeepsEndpointGuardInOrientationFamily(t *testing.T) { + outcome := translate.TargetLoweringOutcome{ + Family: "fixed_prefix_terminal_expansion", + Candidate: string(optimize.ExpansionSearchEndpointSeededReverse), + Selected: string(optimize.ExpansionSearchEndpointSeededReverse), + Applied: string(optimize.ExpansionSearchEndpointSeededReverse), + Fallback: string(optimize.ExpansionSearchStepwiseForward), + PlannedCandidates: []string{string(optimize.ExpansionSearchStepwiseForward), string(optimize.ExpansionSearchEndpointSeededReverse)}, + EmittedCandidates: []string{string(optimize.ExpansionSearchStepwiseForward), string(optimize.ExpansionSearchEndpointSeededReverse)}, + EmittedPolicy: string(optimize.ExpansionSearchPolicyEndpointGuardV1), + ExecutionBoundary: "guarded_dual_arm", + } + metrics := PostgresPlanMetrics{Provenance: map[string]string{}, PlanNodes: []PostgresPlanNodeMetric{ + {NodeType: "Result", SubplanName: "CTE s5_orientation_executed_candidate", ActualRows: 1, ActualLoops: 1}, + {NodeType: "Result", SubplanName: "CTE s5_orientation_executed_incumbent", ActualRows: 0, ActualLoops: 1}, + }} + + telemetry, err := buildPostgresCaseTraversalTelemetry( + translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, metrics, "9123", TraversalTelemetryLevelDiagnostic, + ) + require.NoError(t, err) + require.NoError(t, telemetry.Validate()) + require.Equal(t, []TraversalTelemetryFamily{TraversalTelemetryFamilyOrientation}, telemetry.Diagnostic.RequiredFamilies) + require.Equal(t, string(optimize.ExpansionSearchEndpointSeededReverse), telemetry.Summary.RuntimeIdentity) +} + +func TestPostgresTraversalTelemetryCompletesGuardedInlineASPCounters(t *testing.T) { + outcome := translate.TargetLoweringOutcome{ + Family: "ASP", Candidate: "ASP-I1-U-DAG+MAT-M0", Selected: "ASP-I1-U-DAG+MAT-M0", Applied: "ASP-I1-U-DAG+MAT-M0", + Fallback: "ASP-A1-DAG", PlannedCandidates: []string{"ASP-A1-DAG", "ASP-I1-U-DAG+MAT-M0"}, + EmittedCandidates: []string{"ASP-I1-U-DAG+MAT-M0", "ASP-A1-DAG"}, EmittedPolicy: "asp-i1-guarded-v1", + SelectionMode: "production_canary", SelectorVersion: "asp-i1-canary-v1", ExecutionBoundary: "guarded_dual_arm", + ObservationMode: "all_paths", StateLimit: 10, PredecessorLimit: 20, EnumerationLimit: 30, OutputBytesLimit: 1000, + } + metrics := PostgresPlanMetrics{Provenance: map[string]string{}, HydrationRows: 4, HydrationLoops: 2, PlanNodes: []PostgresPlanNodeMetric{ + inlinePredecessorPlanNode("asp_i1_distance_bounded", 3, 1), + inlinePredecessorPlanNode("asp_i1_predecessor_bounded", 2, 1), + inlinePredecessorPlanNode("asp_i1_paths_bounded", 4, 1), + inlinePredecessorPlanNode("asp_i1_shortest", 2, 1), + inlinePredecessorPlanNode("asp_i1_candidate_marker", 1, 1), + inlinePredecessorPlanNode("asp_i1_fallback_marker", 0, 1), + inlinePredecessorPlanNode("asp_i1_candidate_rows", 2, 1), + inlinePredecessorPlanNode("asp_i1_fallback_rows", 0, 1), + inlinePredecessorMarkerGateNode("candidate", 1, 1), + inlinePredecessorMarkerGateNode("fallback", 0, 1), + inlinePredecessorExecutorNode("candidate", 1), + inlinePredecessorExecutorNode("fallback", 0), + }} + telemetry, err := buildPostgresCaseTraversalTelemetry( + translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, metrics, "9123", TraversalTelemetryLevelDiagnostic, + ) + require.NoError(t, err) + enrichInlineASPTraversalTelemetry(telemetry, metrics, 2, []string{`["p1"]`, `["p2"]`}) + require.NoError(t, telemetry.Validate()) + require.Equal(t, "ASP-I1-U-DAG+MAT-M0", telemetry.Summary.RuntimeIdentity) + require.Equal(t, "inline_predecessor_dag", telemetry.Summary.RuntimeBranch) + require.False(t, *telemetry.Summary.FallbackExecuted) + require.Equal(t, TraversalTelemetryCounterStatusComplete, telemetry.Diagnostic.CounterStatus) + require.Equal(t, int64(3), *telemetry.Diagnostic.Counters.InlineASP.DistanceRows) + require.Equal(t, int64(2), *telemetry.Diagnostic.Counters.InlineASP.PredecessorRows) + require.Equal(t, int64(4), *telemetry.Diagnostic.Counters.InlineASP.EnumerationRows) + require.Equal(t, int64(1), *telemetry.Diagnostic.Counters.InlineASP.CandidateMarkerRows) + require.Equal(t, int64(0), *telemetry.Diagnostic.Counters.InlineASP.FallbackMarkerRows) + require.Equal(t, int64(1), *telemetry.Diagnostic.Counters.InlineASP.CandidateExecutorLoops) + require.Equal(t, int64(0), *telemetry.Diagnostic.Counters.InlineASP.FallbackExecutorLoops) +} + +func TestPostgresTraversalTelemetryCompletesGuardedInlineCanonicalSPCounters(t *testing.T) { + outcome := translate.TargetLoweringOutcome{ + Family: "SP", Candidate: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + Selected: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + Applied: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + Fallback: string(optimize.ShortestPathExecutorS4CanonicalWitness), + PlannedCandidates: []string{ + string(optimize.ShortestPathExecutorS4CanonicalWitness), + string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + }, + EmittedCandidates: []string{ + string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + string(optimize.ShortestPathExecutorS4CanonicalWitness), + }, + EmittedPolicy: optimize.ShortestPathPolicyI1CanonicalGuardedV1, + SelectionMode: "production_canary", SelectorVersion: "sp-i1-canary-v1", ExecutionBoundary: "guarded_dual_arm", + ObservationMode: "one_path", StateLimit: 10, PredecessorLimit: 20, EnumerationLimit: 30, OutputBytesLimit: 1000, + } + + tests := []struct { + name string + candidateMarker int64 + fallbackMarker int64 + outputRows int64 + distanceRows int64 + expectedIdentity string + expectedBranch string + expectedFallback bool + }{ + {name: "candidate witness", candidateMarker: 1, outputRows: 1, distanceRows: 3, + expectedIdentity: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), expectedBranch: "inline_canonical_witness"}, + {name: "candidate no path", candidateMarker: 1, distanceRows: 3, + expectedIdentity: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), expectedBranch: "inline_canonical_no_path"}, + {name: "exact S4 fallback", fallbackMarker: 1, outputRows: 1, distanceRows: 11, + expectedIdentity: string(optimize.ShortestPathExecutorS4CanonicalWitness), expectedBranch: "exact_s4_fallback", expectedFallback: true}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + metrics := PostgresPlanMetrics{Provenance: map[string]string{}, HydrationRows: test.outputRows, HydrationLoops: test.outputRows, PlanNodes: []PostgresPlanNodeMetric{ + inlinePredecessorPlanNode("asp_i1_distance_bounded", test.distanceRows, 1), + inlinePredecessorPlanNode("asp_i1_predecessor_bounded", 2, 1), + inlinePredecessorPlanNode("asp_i1_paths_bounded", 4, 1), + inlinePredecessorPlanNode("asp_i1_shortest", test.outputRows, 1), + inlinePredecessorPlanNode("asp_i1_candidate_marker", test.candidateMarker, 1), + inlinePredecessorPlanNode("asp_i1_fallback_marker", test.fallbackMarker, 1), + inlinePredecessorPlanNode("asp_i1_candidate_rows", test.candidateMarker*test.outputRows, 1), + inlinePredecessorPlanNode("asp_i1_fallback_rows", test.fallbackMarker*test.outputRows, 1), + inlinePredecessorMarkerGateNode("candidate", test.candidateMarker, 1), + inlinePredecessorMarkerGateNode("fallback", test.fallbackMarker, 1), + inlinePredecessorExecutorNode("candidate", test.candidateMarker), + inlinePredecessorExecutorNode("fallback", test.fallbackMarker), + }} + telemetry, err := buildPostgresCaseTraversalTelemetry( + translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, metrics, "9123", TraversalTelemetryLevelDiagnostic, + ) + require.NoError(t, err) + enrichInlinePredecessorTraversalTelemetry(telemetry, metrics, test.outputRows, []string{`["p1"]`}) + require.NoError(t, telemetry.Validate()) + require.Equal(t, test.expectedIdentity, telemetry.Summary.RuntimeIdentity) + require.Equal(t, test.expectedBranch, telemetry.Summary.RuntimeBranch) + require.Equal(t, test.expectedFallback, *telemetry.Summary.FallbackExecuted) + require.Equal(t, TraversalTelemetryCounterStatusComplete, telemetry.Diagnostic.CounterStatus) + require.NotNil(t, telemetry.Diagnostic.Counters.InlineShortestPath) + require.Nil(t, telemetry.Diagnostic.Counters.InlineASP) + require.Equal(t, test.candidateMarker, *telemetry.Diagnostic.Counters.InlineShortestPath.CandidateMarkerRows) + require.Equal(t, test.fallbackMarker, *telemetry.Diagnostic.Counters.InlineShortestPath.FallbackMarkerRows) + require.Equal(t, test.candidateMarker, *telemetry.Diagnostic.Counters.InlineShortestPath.CandidateExecutorLoops) + require.Equal(t, test.fallbackMarker, *telemetry.Diagnostic.Counters.InlineShortestPath.FallbackExecutorLoops) + }) + } +} + +func TestPostgresTraversalTelemetryRejectsEveryMissingInlinePredecessorCounter(t *testing.T) { + outcome := translate.TargetLoweringOutcome{ + Family: "SP", Candidate: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + Selected: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + Applied: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + Fallback: string(optimize.ShortestPathExecutorS4CanonicalWitness), + PlannedCandidates: []string{ + string(optimize.ShortestPathExecutorS4CanonicalWitness), + string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + }, + EmittedCandidates: []string{ + string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + string(optimize.ShortestPathExecutorS4CanonicalWitness), + }, + EmittedPolicy: optimize.ShortestPathPolicyI1CanonicalGuardedV1, + ObservationMode: "one_path", StateLimit: 10, PredecessorLimit: 20, EnumerationLimit: 30, OutputBytesLimit: 1000, + } + fullPlan := []PostgresPlanNodeMetric{ + inlinePredecessorPlanNode("asp_i1_distance_bounded", 3, 1), + inlinePredecessorPlanNode("asp_i1_predecessor_bounded", 2, 1), + inlinePredecessorPlanNode("asp_i1_paths_bounded", 4, 1), + inlinePredecessorPlanNode("asp_i1_shortest", 1, 1), + inlinePredecessorPlanNode("asp_i1_candidate_marker", 1, 1), + inlinePredecessorPlanNode("asp_i1_fallback_marker", 0, 1), + inlinePredecessorPlanNode("asp_i1_candidate_rows", 1, 1), + inlinePredecessorPlanNode("asp_i1_fallback_rows", 0, 1), + inlinePredecessorMarkerGateNode("candidate", 1, 1), + inlinePredecessorMarkerGateNode("fallback", 0, 1), + inlinePredecessorExecutorNode("candidate", 1), + inlinePredecessorExecutorNode("fallback", 0), + } + expectedCounter := map[string]string{ + "asp_i1_distance_bounded": "asp_i1_distance_rows", + "asp_i1_predecessor_bounded": "asp_i1_predecessor_rows", + "asp_i1_paths_bounded": "asp_i1_enumeration_rows", + "asp_i1_shortest": "asp_i1_output_rows", + "asp_i1_candidate_marker": "asp_i1_candidate_marker_rows", + "asp_i1_fallback_marker": "asp_i1_fallback_marker_rows", + "asp_i1_candidate_rows": "asp_i1_candidate_branch_rows", + "asp_i1_fallback_rows": "asp_i1_fallback_branch_rows", + "test_candidate_executor": "asp_i1_candidate_executor_loops", + "test_fallback_executor": "asp_i1_fallback_executor_loops", + } + + for omitted, counter := range expectedCounter { + t.Run(omitted, func(t *testing.T) { + metrics := PostgresPlanMetrics{Provenance: map[string]string{}} + for _, node := range fullPlan { + if node.SubplanName != "CTE "+omitted && node.Alias != omitted { + metrics.PlanNodes = append(metrics.PlanNodes, node) + } + } + telemetry, err := buildPostgresCaseTraversalTelemetry( + translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, metrics, "9123", TraversalTelemetryLevelDiagnostic, + ) + require.NoError(t, err) + enrichInlinePredecessorTraversalTelemetry(telemetry, metrics, 1, []string{`["p1"]`}) + require.NoError(t, telemetry.Validate()) + require.Equal(t, TraversalTelemetryCounterStatusHiddenUnavailable, telemetry.Diagnostic.CounterStatus) + require.Nil(t, telemetry.Diagnostic.Counters.InlineShortestPath) + require.Contains(t, telemetry.Diagnostic.IncompleteReasons[0], counter) + }) + } +} + +func TestPostgresTraversalPlanReplayUsesExactInlinePredecessorCTEBodies(t *testing.T) { + metrics := PostgresPlanMetrics{Provenance: map[string]string{}, PlanNodes: []PostgresPlanNodeMetric{ + inlinePredecessorPlanNode("asp_i1_distance_bounded", 3, 1), + {PlanNodeID: 500, NodeType: "Limit", SubplanName: "CTE prefix_asp_i1_distance_bounded", ActualRows: 77, ActualLoops: 1}, + {NodeType: "CTE Scan", CTEName: "asp_i1_distance_bounded", Alias: "asp_i1_distance_bounded", ActualRows: 99, ActualLoops: 7}, + inlinePredecessorPlanNode("asp_i1_candidate_rows", 0, 1), + {NodeType: "CTE Scan", CTEName: "asp_i1_candidate_rows", Alias: "asp_i1_candidate_rows", ActualRows: 10, ActualLoops: 5}, + inlinePredecessorPlanNode("asp_i1_fallback_rows", 0, 1), + {NodeType: "CTE Scan", CTEName: "asp_i1_fallback_rows", Alias: "asp_i1_fallback_rows", ActualRows: 8, ActualLoops: 3}, + inlinePredecessorPlanNode("asp_i1_candidate_marker", 1, 1), + inlinePredecessorPlanNode("asp_i1_fallback_marker", 0, 1), + inlinePredecessorMarkerGateNode("candidate", 1, 1), + inlinePredecessorMarkerGateNode("fallback", 0, 1), + inlinePredecessorExecutorNode("candidate", 1), + inlinePredecessorExecutorNode("fallback", 0), + }} + + replay := postgresTraversalPlanReplay(metrics) + require.Equal(t, int64(3), replay.Counters["asp_i1_distance_rows"]) + require.Equal(t, int64(0), replay.Counters["asp_i1_candidate_branch_rows"]) + require.Equal(t, int64(0), replay.Counters["asp_i1_fallback_branch_rows"]) + require.Equal(t, int64(1), replay.Counters["asp_i1_candidate_executor_loops"]) + require.Equal(t, int64(0), replay.Counters["asp_i1_fallback_executor_loops"]) +} + +func TestPostgresTraversalPlanReplayRejectsAmbiguousInlineBranchShape(t *testing.T) { + t.Run("duplicate exact body", func(t *testing.T) { + body := inlinePredecessorPlanNode("asp_i1_candidate_rows", 1, 1) + duplicate := body + duplicate.PlanNodeID = 99 + replay := postgresTraversalPlanReplay(PostgresPlanMetrics{Provenance: map[string]string{}, PlanNodes: []PostgresPlanNodeMetric{ + body, duplicate, inlinePredecessorPlanNode("asp_i1_candidate_marker", 1, 1), + inlinePredecessorMarkerGateNode("candidate", 1, 1), + inlinePredecessorExecutorNode("candidate", 1), + }}) + _, branchPresent := replay.Counters["asp_i1_candidate_branch_rows"] + _, executorPresent := replay.Counters["asp_i1_candidate_executor_loops"] + require.False(t, branchPresent) + require.False(t, executorPresent) + }) + + t.Run("wrong direct outer marker", func(t *testing.T) { + body := inlinePredecessorPlanNode("asp_i1_candidate_rows", 1, 1) + wrongMarker := inlinePredecessorMarkerGateNode("candidate", 1, 1) + wrongMarker.CTEName = "asp_i1_fallback_marker" + replay := postgresTraversalPlanReplay(PostgresPlanMetrics{Provenance: map[string]string{}, PlanNodes: []PostgresPlanNodeMetric{ + body, inlinePredecessorPlanNode("asp_i1_candidate_marker", 1, 1), wrongMarker, inlinePredecessorExecutorNode("candidate", 1), + }}) + require.Equal(t, int64(1), replay.Counters["asp_i1_candidate_branch_rows"]) + _, executorPresent := replay.Counters["asp_i1_candidate_executor_loops"] + require.False(t, executorPresent) + }) +} + +func inlinePredecessorPlanNode(name string, rows, loops int64) PostgresPlanNodeMetric { + return PostgresPlanNodeMetric{ + PlanNodeID: inlinePredecessorPlanNodeID(name), NodeType: "Result", SubplanName: "CTE " + name, + ActualRows: rows, ActualLoops: loops, + } +} + +func inlinePredecessorExecutorNode(branch string, loops int64) PostgresPlanNodeMetric { + bodyID := inlinePredecessorPlanNodeID("asp_i1_" + branch + "_rows") + return PostgresPlanNodeMetric{ + PlanNodeID: bodyID + 100, ParentPlanNodeID: bodyID, ParentRelationship: "Inner", + NodeType: "Result", Alias: "test_" + branch + "_executor", ActualLoops: loops, + } +} + +func inlinePredecessorMarkerGateNode(branch string, rows, loops int64) PostgresPlanNodeMetric { + bodyID := inlinePredecessorPlanNodeID("asp_i1_" + branch + "_rows") + return PostgresPlanNodeMetric{ + PlanNodeID: bodyID + 200, ParentPlanNodeID: bodyID, ParentRelationship: "Outer", + NodeType: "CTE Scan", CTEName: "asp_i1_" + branch + "_marker", Alias: "test_" + branch + "_marker_gate", + ActualRows: rows, ActualLoops: loops, + } +} + +func inlinePredecessorPlanNodeID(name string) int64 { + ids := map[string]int64{ + "asp_i1_distance_bounded": 1, "asp_i1_predecessor_bounded": 2, + "asp_i1_paths_bounded": 3, "asp_i1_shortest": 4, + "asp_i1_candidate_marker": 5, "asp_i1_fallback_marker": 6, + "asp_i1_candidate_rows": 7, "asp_i1_fallback_rows": 8, + } + return ids[name] +} + +func TestPostgresTraversalTelemetryPrefersShortestExecutorOverAnalysisOutcomes(t *testing.T) { + shortest := translate.TargetLoweringOutcome{TargetKind: "traversal", Family: "SP", Applied: "SP-B1-C-ALT-NODE-D"} + outcome, found := singleTraversalOutcome([]translate.TargetLoweringOutcome{ + {TargetKind: "endpoint_resolution", Family: "endpoint_resolution", TraversalFamily: "SP", Applied: "ENDPOINT-RESOLUTION-INCUMBENT"}, + {TargetKind: "traversal_predicate", Family: "traversal_predicate", Applied: "TRAVERSAL-PREDICATE-INCUMBENT"}, + {TargetKind: "traversal", Family: "fixed_suffix_expansion", Applied: "EXPANSION-STEPWISE-FORWARD"}, + shortest, + }) + require.True(t, found) + require.Equal(t, shortest, outcome) +} + +func TestPostgresTraversalTelemetrySeparatesShadowChoiceFromExecutedIncumbent(t *testing.T) { + outcome := translate.TargetLoweringOutcome{ + Family: "fixed_suffix_expansion", + Candidate: "EXPANSION-SUFFIX-SEEDED-REVERSE", + Selected: "EXPANSION-STEPWISE-FORWARD", + Applied: "EXPANSION-STEPWISE-FORWARD", + Fallback: "EXPANSION-STEPWISE-FORWARD", + PlannedCandidates: []string{"EXPANSION-SUFFIX-SEEDED-REVERSE", "EXPANSION-STEPWISE-FORWARD"}, + EmittedCandidates: []string{"EXPANSION-STEPWISE-FORWARD"}, + EmittedPolicy: "orientation-probe-v1", + SelectionMode: "shadow_tool", + SelectorVersion: "orientation-probe-v1", + StateLimit: 4096, + ProbeCaps: &optimize.ExpansionSearchProbeCaps{ + ReverseSeedRowLimit: 512, + }, + } + metrics := PostgresPlanMetrics{ + PlanNodes: []PostgresPlanNodeMetric{ + {NodeType: "Result", SubplanName: "CTE s5_orientation_shadow_reverse", ActualRows: 1, ActualLoops: 1}, + {NodeType: "Result", SubplanName: "CTE s5_orientation_shadow_forward", ActualRows: 0, ActualLoops: 1}, + {NodeType: "Result", SubplanName: "CTE s5_orientation_executed_incumbent", ActualRows: 1, ActualLoops: 1}, + {NodeType: "Limit", SubplanName: "CTE s5_orientation_suffix_probe", ActualRows: 513, ActualLoops: 1}, + }, + Provenance: map[string]string{}, + } + + telemetry, err := buildPostgresCaseTraversalTelemetry( + translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, + metrics, + "9123", + TraversalTelemetryLevelSummary, + ) + require.NoError(t, err) + require.NoError(t, telemetry.Validate()) + require.Equal(t, "EXPANSION-STEPWISE-FORWARD", telemetry.Summary.RuntimeIdentity) + require.Equal(t, "EXPANSION-STEPWISE-FORWARD", telemetry.Summary.AppliedIdentity) + require.Equal(t, "EXPANSION-SUFFIX-SEEDED-REVERSE", telemetry.Summary.WouldSelectIdentity) + require.Equal(t, "shadow_incumbent", telemetry.Summary.RuntimeBranch) + require.False(t, *telemetry.Summary.FallbackExecuted) + require.True(t, *telemetry.Summary.Overflow) +} + +func TestPostgresTraversalTelemetryUsesExactGuardedOrientationReceiptBranches(t *testing.T) { + for _, testCase := range []struct { + name string + candidateRows int64 + incumbentRows int64 + rootProbeRows int64 + runtimeIdentity string + runtimeBranch string + fallbackExecuted bool + overflow bool + }{ + { + name: "reverse candidate", candidateRows: 1, runtimeIdentity: string(optimize.ExpansionSearchSuffixSeededReverse), + runtimeBranch: "suffix_seeded_reverse", + }, + { + name: "forward selection", incumbentRows: 1, runtimeIdentity: string(optimize.ExpansionSearchStepwiseForward), + runtimeBranch: "exact_forward_incumbent", + }, + { + name: "overflow fallback", incumbentRows: 1, rootProbeRows: optimize.ExpansionSearchOrientationRootRowLimit + 1, + runtimeIdentity: string(optimize.ExpansionSearchStepwiseForward), runtimeBranch: "exact_forward_incumbent", + fallbackExecuted: true, overflow: true, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + outcome := translate.TargetLoweringOutcome{ + Family: "fixed_suffix_expansion", Candidate: string(optimize.ExpansionSearchSuffixSeededReverse), + Selected: string(optimize.ExpansionSearchStepwiseForward), Applied: string(optimize.ExpansionSearchStepwiseForward), + Fallback: string(optimize.ExpansionSearchStepwiseForward), + PlannedCandidates: []string{string(optimize.ExpansionSearchStepwiseForward), string(optimize.ExpansionSearchSuffixSeededReverse)}, + EmittedCandidates: []string{string(optimize.ExpansionSearchStepwiseForward), string(optimize.ExpansionSearchSuffixSeededReverse)}, + EmittedPolicy: string(optimize.ExpansionSearchPolicyOrientationProbeV2), SelectorVersion: string(optimize.ExpansionSearchPolicyOrientationProbeV2), + ExecutionBoundary: optimize.ExpansionSearchExecutionBoundaryGuardedDualArm, + ProbeCaps: &optimize.ExpansionSearchProbeCaps{RootRowLimit: optimize.ExpansionSearchOrientationRootRowLimit}, + } + metrics := PostgresPlanMetrics{Provenance: map[string]string{}, PlanNodes: []PostgresPlanNodeMetric{ + {NodeType: "Result", SubplanName: "CTE s5_orientation_executed_candidate", ActualRows: testCase.candidateRows, ActualLoops: 1}, + {NodeType: "Result", SubplanName: "CTE s5_orientation_executed_incumbent", ActualRows: testCase.incumbentRows, ActualLoops: 1}, + {NodeType: "Limit", SubplanName: "CTE s5_orientation_root_probe", ActualRows: testCase.rootProbeRows, ActualLoops: 1}, + }} + + telemetry, err := buildPostgresCaseTraversalTelemetry( + translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, metrics, "9123", TraversalTelemetryLevelSummary, + ) + require.NoError(t, err) + require.Equal(t, testCase.runtimeIdentity, telemetry.Summary.RuntimeIdentity) + require.Equal(t, testCase.runtimeBranch, telemetry.Summary.RuntimeBranch) + require.Equal(t, testCase.fallbackExecuted, *telemetry.Summary.FallbackExecuted) + require.Equal(t, testCase.overflow, *telemetry.Summary.Overflow) + require.NoError(t, validateRuntimeReceiptEvents([]RuntimeReceiptEvent{{ + Ordinal: 1, RuntimeIdentity: testCase.runtimeIdentity, RuntimeBranch: testCase.runtimeBranch, + FallbackExecuted: testCase.fallbackExecuted, + }}, telemetry.Summary.RuntimeIdentity, telemetry.Summary.RuntimeBranch, telemetry.Summary.FallbackExecuted)) + }) + } +} + +func TestPostgresTraversalTelemetryUsesV2DepthWeightedDiagnosticScore(t *testing.T) { + maximumDepth := int64(16) + outcome := translate.TargetLoweringOutcome{ + Family: "fixed_suffix_expansion", Candidate: string(optimize.ExpansionSearchSuffixSeededReverse), + Selected: string(optimize.ExpansionSearchStepwiseForward), Applied: string(optimize.ExpansionSearchStepwiseForward), + Fallback: string(optimize.ExpansionSearchStepwiseForward), EmittedPolicy: string(optimize.ExpansionSearchPolicyOrientationProbeV2), + SelectorVersion: string(optimize.ExpansionSearchPolicyOrientationProbeV2), MaximumDepth: &maximumDepth, + } + metrics := PostgresPlanMetrics{Provenance: map[string]string{}, PlanNodes: []PostgresPlanNodeMetric{ + {NodeType: "Limit", SubplanName: "CTE s5_orientation_root_probe", ActualRows: 2, ActualLoops: 1}, + {NodeType: "Limit", SubplanName: "CTE s5_orientation_forward_degree_probe", ActualRows: 8, ActualLoops: 1}, + {NodeType: "Result", SubplanName: "CTE s5_orientation_executed_incumbent", ActualRows: 1, ActualLoops: 1}, + }} + telemetry, err := buildPostgresCaseTraversalTelemetry( + translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, metrics, "9123", TraversalTelemetryLevelDiagnostic, + ) + require.NoError(t, err) + enrichOrientationTraversalTelemetry(telemetry, metrics, 1, []string{`["path"]`}, maximumDepth) + require.NoError(t, telemetry.Validate()) + require.Equal(t, float64(130), *telemetry.Diagnostic.Counters.Orientation.ForwardScore) + require.Equal(t, maximumDepth, orientationPolicyMaximumDepth(translate.OptimizationSummary{ + TargetOutcomes: []translate.TargetLoweringOutcome{outcome}, + }, string(optimize.ExpansionSearchPolicyOrientationProbeV2))) +} + +func TestPostgresTraversalTelemetryCompletesOrientationCountersFromNamedPlanNodes(t *testing.T) { + outcome := translate.TargetLoweringOutcome{ + Family: "fixed_suffix_expansion", Candidate: "EXPANSION-SUFFIX-SEEDED-REVERSE", + Selected: "EXPANSION-STEPWISE-FORWARD", Applied: "EXPANSION-STEPWISE-FORWARD", Fallback: "EXPANSION-STEPWISE-FORWARD", + PlannedCandidates: []string{"EXPANSION-SUFFIX-SEEDED-REVERSE", "EXPANSION-STEPWISE-FORWARD"}, + EmittedCandidates: []string{"EXPANSION-SUFFIX-SEEDED-REVERSE", "EXPANSION-STEPWISE-FORWARD"}, + EmittedPolicy: "orientation-probe-v1", SelectionMode: "production_canary", SelectorVersion: "orientation-probe-v1", StateLimit: 4096, + } + metrics := PostgresPlanMetrics{Provenance: map[string]string{}, PlanNodes: []PostgresPlanNodeMetric{ + {NodeType: "Limit", SubplanName: "CTE s5_orientation_root_probe", ActualRows: 2, ActualLoops: 1, ActualTotalMS: .01, Buffers: Buffers{SharedHit: 1}}, + {NodeType: "Limit", SubplanName: "CTE s5_orientation_suffix_probe", ActualRows: 5, ActualLoops: 1, ActualTotalMS: .02}, + {NodeType: "Aggregate", SubplanName: "CTE s5_orientation_boundaries", ActualRows: 3, ActualLoops: 1, ActualTotalMS: .01}, + {NodeType: "Limit", SubplanName: "CTE s5_orientation_forward_degree_probe", ActualRows: 8, ActualLoops: 1, ActualTotalMS: .01}, + {NodeType: "Limit", SubplanName: "CTE s5_orientation_reverse_degree_probe", ActualRows: 1, ActualLoops: 1, ActualTotalMS: .01}, + {NodeType: "Limit", SubplanName: "CTE s5_orientation_states", ActualRows: 4, ActualLoops: 1}, + {NodeType: "Result", SubplanName: "CTE s5_orientation_executed_candidate", ActualRows: 1, ActualLoops: 1}, + {NodeType: "Result", SubplanName: "CTE s5_orientation_executed_incumbent", ActualRows: 0, ActualLoops: 1}, + {NodeType: "Recursive Union", SubplanName: "CTE s5_orientation_reverse", ActualRows: 4, ActualLoops: 1}, + {NodeType: "Result", SubplanName: "CTE s5_orientation_decision", ActualRows: 1, ActualLoops: 1}, + // Consumer scans are deliberately repeated and must not inflate the + // single materialization's row, loop, or branch attribution. + {NodeType: "CTE Scan", CTEName: "s5_orientation_root_probe", Alias: "s5_orientation_root_probe", ActualRows: 2, ActualLoops: 3}, + {NodeType: "CTE Scan", CTEName: "s5_orientation_reverse_degree_probe", Alias: "s5_orientation_reverse_degree_probe", ActualRows: 1, ActualLoops: 7}, + }} + telemetry, err := buildPostgresCaseTraversalTelemetry(translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, metrics, "9123", TraversalTelemetryLevelDiagnostic) + require.NoError(t, err) + enrichOrientationTraversalTelemetry(telemetry, metrics, 1, []string{`["path"]`}, 0) + require.NoError(t, telemetry.Validate()) + require.Equal(t, TraversalTelemetryCounterStatusComplete, telemetry.Diagnostic.CounterStatus) + require.Equal(t, int64(5), *telemetry.Diagnostic.Counters.Orientation.ReverseSeeds) + require.Equal(t, int64(2), *telemetry.Diagnostic.Counters.Orientation.DuplicateSeeds) + require.Equal(t, "reverse", telemetry.Diagnostic.Counters.Orientation.SelectedSide) + require.Equal(t, int64(1), telemetry.Diagnostic.PlanReplay.Counters["orientation_root_probe_loops"]) + require.Equal(t, int64(1), telemetry.Diagnostic.PlanReplay.Counters["orientation_candidate_branch_loops"]) + require.Equal(t, int64(0), telemetry.Diagnostic.PlanReplay.Counters["orientation_incumbent_branch_loops"]) +} + +func TestPostgresTraversalTelemetrySummaryAndDisabledModesDoNotAttachDiagnosticCounters(t *testing.T) { + summary := bidirectionalCaseTelemetry(t, TraversalTelemetryLevelSummary) + require.NoError(t, summary.Validate()) + require.Nil(t, summary.Diagnostic) + require.False(t, *summary.Summary.RuntimeOutcomeAvailable) + require.Empty(t, summary.Summary.RuntimeIdentity) + require.Empty(t, summary.Summary.AppliedIdentity) + require.Nil(t, summary.Summary.FallbackExecuted) + + record := CaseResult{ + PostgresReferences: []PostgresReferenceResult{{ + traversalTelemetryParameters: map[string]any{"state_limit": int64(1)}, + }}, + } + runner := postgresSQLRunner{traversalTelemetry: postgresTraversalTelemetryOff} + require.NoError(t, runner.attachPostgresTraversalTelemetry(t.Context(), &record, nil)) + require.Nil(t, record.TraversalTelemetry) + require.Nil(t, record.PostgresReferences[0].TraversalTelemetry) + require.Nil(t, record.PostgresReferences[0].traversalTelemetryParameters) +} + +func TestPostgresTraversalTelemetryAttachesToEveryTraversalReference(t *testing.T) { + metrics := PostgresPlanMetrics{PlanNodes: []PostgresPlanNodeMetric{{NodeType: "Recursive Union", ActualRows: 2, ActualLoops: 1}}, Provenance: map[string]string{}} + record := CaseResult{PostgresReferences: []PostgresReferenceResult{ + { + Name: "forward", + Architecture: "EXPANSION-STEPWISE-FORWARD-SQL", + ImplementationID: "forward_v1", + PostgresMetrics: &metrics, + traversalTelemetryParameters: map[string]any{}, + }, + { + Name: "reverse", + Architecture: "EXPANSION-SUFFIX-SEEDED-REVERSE", + ImplementationID: "reverse_v1", + PostgresMetrics: &metrics, + traversalTelemetryParameters: map[string]any{}, + }, + }} + runner := postgresSQLRunner{ + traversalTelemetry: postgresTraversalTelemetrySummary, + backendPID: "9123", + } + + require.NoError(t, runner.attachPostgresTraversalTelemetry(t.Context(), &record, nil)) + require.Len(t, record.PostgresReferences, 2) + for _, reference := range record.PostgresReferences { + require.NotNil(t, reference.TraversalTelemetry) + require.Equal(t, TraversalTelemetryLevelSummary, reference.TraversalTelemetry.Level) + require.Equal(t, reference.Architecture, reference.TraversalTelemetry.Summary.RuntimeIdentity) + require.Nil(t, reference.TraversalTelemetry.Diagnostic) + require.NoError(t, reference.TraversalTelemetry.Validate()) + } +} + +func TestPostgresTraversalTelemetrySkipsNonTraversalReferenceBoundaries(t *testing.T) { + metrics := PostgresPlanMetrics{PlanNodes: []PostgresPlanNodeMetric{{NodeType: "Result", ActualRows: 1, ActualLoops: 1}}, Provenance: map[string]string{}} + for _, architecture := range []string{"component_probe", "protocol", "root_validation", "root_adjacency", "factored_suffix"} { + reference := PostgresReferenceResult{ + Architecture: architecture, + ImplementationID: architecture + "_v1", + PostgresMetrics: &metrics, + } + telemetry, err := buildPostgresReferenceTraversalTelemetry(reference, nil, "9123", TraversalTelemetryLevelDiagnostic) + require.NoError(t, err) + require.Nil(t, telemetry, architecture) + } +} + +func TestParseConfigValidatesPostgresTraversalTelemetryMode(t *testing.T) { + cfg, err := parseConfig([]string{"-postgres-traversal-telemetry", "summary"}, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, postgresTraversalTelemetrySummary, cfg.PostgresTraversalTelemetry) + + cfg, err = parseConfig([]string{"-postgres-traversal-telemetry", "diagnostic"}, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, postgresTraversalTelemetryDiagnostic, cfg.PostgresTraversalTelemetry) + + _, err = parseConfig([]string{"-postgres-traversal-telemetry", "unknown"}, func(string) string { return "" }) + require.ErrorContains(t, err, "must be off, summary, or diagnostic") + + _, err = parseConfig([]string{"-postgres-traversal-telemetry", "diagnostic", "-pool-size", "2"}, func(string) string { return "" }) + require.ErrorContains(t, err, "requires pool-size 1") + + cfg, err = parseConfig([]string{"-postgres-expansion-orientation-shadow"}, func(string) string { return "" }) + require.NoError(t, err) + require.True(t, cfg.PostgresExpansionOrientationShadow) + + _, err = parseConfig([]string{"-postgres-expansion-orientation-shadow", "-postgres-force-expansion-search", "EXPANSION-SUFFIX-SEEDED-REVERSE"}, func(string) string { return "" }) + require.ErrorContains(t, err, "mutually exclusive") +} + +func TestParseConfigAcceptsExplicitOrientationProbeV2MeasurementModes(t *testing.T) { + for _, mode := range [][]string{ + {"-postgres-expansion-orientation-shadow"}, + {"-postgres-expansion-orientation-tournament"}, + } { + args := append(append([]string(nil), mode...), + "-postgres-expansion-orientation-policy", "orientation-probe-v2", + "-postgres-repeatable-read", + "-postgres-traversal-telemetry", "summary", + ) + cfg, err := parseConfig(args, func(string) string { return "" }) + require.NoError(t, err, mode) + require.Equal(t, "orientation-probe-v2", cfg.PostgresExpansionOrientationPolicy) + require.True(t, cfg.PostgresRepeatableRead) + require.Equal(t, postgresTraversalTelemetrySummary, cfg.PostgresTraversalTelemetry) + } + + for _, args := range [][]string{ + {"-postgres-expansion-orientation-policy", "orientation-probe-v2", "-postgres-repeatable-read", "-postgres-traversal-telemetry", "summary"}, + {"-postgres-expansion-orientation-shadow", "-postgres-expansion-orientation-policy", "orientation-probe-v3", "-postgres-repeatable-read", "-postgres-traversal-telemetry", "summary"}, + {"-postgres-expansion-orientation-shadow", "-postgres-expansion-orientation-tournament", "-postgres-repeatable-read"}, + {"-postgres-expansion-orientation-shadow", "-postgres-expansion-orientation-policy", "orientation-probe-v2", "-postgres-traversal-telemetry", "summary"}, + {"-postgres-expansion-orientation-shadow", "-postgres-expansion-orientation-policy", "orientation-probe-v2", "-postgres-repeatable-read"}, + {"-postgres-expansion-orientation-tournament"}, + } { + _, err := parseConfig(args, func(string) string { return "" }) + require.Error(t, err, args) + } +} + +func bidirectionalCaseTelemetry(t *testing.T, level TraversalTelemetryLevel) *TraversalExecutionTelemetry { + t.Helper() + outcome := translate.TargetLoweringOutcome{ + Family: "SP", + Candidate: "SP-B2-C-MIN-LEVEL-D", + Selected: "SP-B2-C-MIN-LEVEL-D", + Applied: "SP-B2-C-MIN-LEVEL-D", + Fallback: "SP-S4-C-D", + PlannedCandidates: []string{"SP-B2-C-MIN-LEVEL-D", "SP-S4-C-D"}, + Scheduler: "smaller_current_level", + SelectorVersion: "sp-tool-v1", + StateLimit: 100, + FrontierLimit: 50, + PredecessorLimit: 25, + } + metrics := PostgresPlanMetrics{ + PlanNodes: []PostgresPlanNodeMetric{{ + NodeType: "Function Scan", + FunctionName: "shortest_path_b2_smaller_current_level", + ActualRows: 1, + ActualLoops: 1, + }}, + Provenance: map[string]string{}, + } + telemetry, err := buildPostgresCaseTraversalTelemetry( + translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, + metrics, + "9123", + level, + ) + require.NoError(t, err) + require.NotNil(t, telemetry) + return telemetry +} + +func validBidirectionalDiagnosticDocument(invocationID string) *postgresBidirectionalDiagnosticDocument { + return &postgresBidirectionalDiagnosticDocument{ + SchemaVersion: 1, + InvocationID: invocationID, + Scheduler: "smaller_current_level", + StateLimit: traversalTelemetryPointer(int64(100)), + FrontierLimit: traversalTelemetryPointer(int64(50)), + PredecessorLimit: traversalTelemetryPointer(int64(25)), + SearchCalls: traversalTelemetryPointer(int64(1)), + RuntimeBranch: "bidirectional_search", + Overflowed: traversalTelemetryPointer(false), + FallbackExecuted: traversalTelemetryPointer(false), + Counters: &postgresBidirectionalDiagnosticCounts{ + SchedulerActions: traversalTelemetryPointer(int64(2)), + CandidateEdges: traversalTelemetryPointer(int64(7)), + DistinctNewNodes: traversalTelemetryPointer(int64(5)), + SeenPeak: traversalTelemetryPointer(int64(6)), + FrontierPeak: traversalTelemetryPointer(int64(3)), + QueuePeak: traversalTelemetryPointer(int64(3)), + PredecessorPeak: traversalTelemetryPointer(int64(4)), + MeetingCandidates: traversalTelemetryPointer(int64(1)), + FrozenDistance: traversalTelemetryPointer(int64(3)), + WitnessRows: traversalTelemetryPointer(int64(1)), + Levels: []postgresBidirectionalDiagnosticLevel{{ + SearchID: traversalTelemetryPointer(int64(1)), + ActionIndex: traversalTelemetryPointer(int64(1)), + Side: "forward", + Action: "expand_level", + Depth: traversalTelemetryPointer(int64(1)), + FrontierRows: traversalTelemetryPointer(int64(2)), + CandidateEdges: traversalTelemetryPointer(int64(7)), + DistinctNewNodes: traversalTelemetryPointer(int64(5)), + SeenRows: traversalTelemetryPointer(int64(6)), + QueueRows: traversalTelemetryPointer(int64(3)), + PredecessorRows: traversalTelemetryPointer(int64(4)), + MeetingCandidates: traversalTelemetryPointer(int64(1)), + }}, + }, + Calls: []postgresBidirectionalDiagnosticCall{{ + SearchID: traversalTelemetryPointer(int64(1)), + SourceID: traversalTelemetryPointer(int64(10)), + TargetID: traversalTelemetryPointer(int64(20)), + RuntimeBranch: "bidirectional_search", + SchedulerActions: traversalTelemetryPointer(int64(2)), + CandidateEdges: traversalTelemetryPointer(int64(7)), + DistinctNewNodes: traversalTelemetryPointer(int64(5)), + SeenPeak: traversalTelemetryPointer(int64(6)), + FrontierPeak: traversalTelemetryPointer(int64(3)), + QueuePeak: traversalTelemetryPointer(int64(3)), + PredecessorPeak: traversalTelemetryPointer(int64(4)), + MeetingCandidates: traversalTelemetryPointer(int64(1)), + FrozenDistance: traversalTelemetryPointer(int64(3)), + WitnessRows: traversalTelemetryPointer(int64(1)), + Overflowed: traversalTelemetryPointer(false), + FallbackExecuted: traversalTelemetryPointer(false), + }}, + } +} + +func bidirectionalASPCaseTelemetry(t *testing.T) *TraversalExecutionTelemetry { + t.Helper() + outcome := translate.TargetLoweringOutcome{ + Family: "ASP", Candidate: "ASP-B2-DAG-MIN-LEVEL", Selected: "ASP-B2-DAG-MIN-LEVEL", + Applied: "ASP-B2-DAG-MIN-LEVEL", Fallback: "ASP-A1-DAG", + PlannedCandidates: []string{"ASP-B2-DAG-MIN-LEVEL", "ASP-A1-DAG"}, + Scheduler: "smaller_current_level", SelectorVersion: "asp-tool-v1", + StateLimit: 100, FrontierLimit: 50, PredecessorLimit: 25, + EnumerationLimit: 1000, OutputBytesLimit: 4096, + } + metrics := PostgresPlanMetrics{ + PlanNodes: []PostgresPlanNodeMetric{{NodeType: "Function Scan", FunctionName: "all_shortest_paths_b2_smaller_current_level", ActualRows: 1, ActualLoops: 1}}, + Provenance: map[string]string{}, + } + telemetry, err := buildPostgresCaseTraversalTelemetry( + translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, metrics, "9123", TraversalTelemetryLevelDiagnostic, + ) + require.NoError(t, err) + require.NotNil(t, telemetry) + return telemetry +} + +func validBidirectionalAllShortestDiagnosticDocument(invocationID string) *postgresBidirectionalAllShortestDiagnosticDocument { + base := validBidirectionalDiagnosticDocument(invocationID) + counts := &postgresBidirectionalAllShortestDiagnosticCounts{ + SchedulerActions: base.Counters.SchedulerActions, CandidateEdges: base.Counters.CandidateEdges, + DistinctNewNodes: base.Counters.DistinctNewNodes, SeenPeak: base.Counters.SeenPeak, + FrontierPeak: base.Counters.FrontierPeak, QueuePeak: base.Counters.QueuePeak, + PredecessorPeak: base.Counters.PredecessorPeak, MeetingCandidates: base.Counters.MeetingCandidates, + FrozenDistance: base.Counters.FrozenDistance, WitnessRows: base.Counters.WitnessRows, Levels: base.Counters.Levels, + SameDepthPredecessorAdditions: traversalTelemetryPointer(int64(5)), MeetingNodes: traversalTelemetryPointer(int64(2)), + CutDepth: traversalTelemetryPointer(int64(3)), PathCountEstimate: traversalTelemetryPointer(int64(12)), + PathCountSaturated: traversalTelemetryPointer(false), EnumeratedCandidates: traversalTelemetryPointer(int64(13)), + DuplicateRejects: traversalTelemetryPointer(int64(1)), OutputPaths: traversalTelemetryPointer(int64(12)), + OutputEdgeCells: traversalTelemetryPointer(int64(36)), OutputBytes: traversalTelemetryPointer(int64(384)), + } + call := postgresBidirectionalAllShortestDiagnosticCall{ + SearchID: base.Calls[0].SearchID, SourceID: base.Calls[0].SourceID, TargetID: base.Calls[0].TargetID, + RuntimeBranch: base.Calls[0].RuntimeBranch, SchedulerActions: base.Calls[0].SchedulerActions, + CandidateEdges: base.Calls[0].CandidateEdges, DistinctNewNodes: base.Calls[0].DistinctNewNodes, + SeenPeak: base.Calls[0].SeenPeak, FrontierPeak: base.Calls[0].FrontierPeak, QueuePeak: base.Calls[0].QueuePeak, + PredecessorPeak: base.Calls[0].PredecessorPeak, MeetingCandidates: base.Calls[0].MeetingCandidates, + FrozenDistance: base.Calls[0].FrozenDistance, WitnessRows: base.Calls[0].WitnessRows, + SameDepthPredecessorAdditions: counts.SameDepthPredecessorAdditions, MeetingNodes: counts.MeetingNodes, + CutDepth: counts.CutDepth, PathCountEstimate: counts.PathCountEstimate, PathCountSaturated: counts.PathCountSaturated, + EnumeratedCandidates: counts.EnumeratedCandidates, DuplicateRejects: counts.DuplicateRejects, + OutputPaths: counts.OutputPaths, OutputEdgeCells: counts.OutputEdgeCells, OutputBytes: counts.OutputBytes, + Overflowed: base.Calls[0].Overflowed, FallbackExecuted: base.Calls[0].FallbackExecuted, + } + return &postgresBidirectionalAllShortestDiagnosticDocument{ + SchemaVersion: 1, InvocationID: invocationID, Scheduler: "smaller_current_level", + StateLimit: traversalTelemetryPointer(int64(100)), FrontierLimit: traversalTelemetryPointer(int64(50)), + PredecessorLimit: traversalTelemetryPointer(int64(25)), EnumerationLimit: traversalTelemetryPointer(int64(1000)), + OutputBytesLimit: traversalTelemetryPointer(int64(4096)), SearchCalls: traversalTelemetryPointer(int64(1)), + RuntimeBranch: "bidirectional_search", Overflowed: traversalTelemetryPointer(false), + FallbackExecuted: traversalTelemetryPointer(false), Counters: counts, + Calls: []postgresBidirectionalAllShortestDiagnosticCall{call}, + } +} diff --git a/cmd/graphbench/postgresql_plan_invariants_integration_test.go b/cmd/graphbench/postgresql_plan_invariants_integration_test.go new file mode 100644 index 00000000..1bb666e7 --- /dev/null +++ b/cmd/graphbench/postgresql_plan_invariants_integration_test.go @@ -0,0 +1,1394 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +//go:build manual_integration + +package main + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "os" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/specterops/dawgs/testutil" + "github.com/stretchr/testify/require" +) + +// TestPostgreSQLBidirectionalOperationalPoolMatrix exercises the required +// pool-size/concurrency cross-product with an exact B2 distance candidate. +// It is intentionally a smoke matrix; latency qualification uses GraphBench's +// separately balanced discovery and confirmation protocols. +func TestPostgreSQLBidirectionalOperationalPoolMatrix(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{Cases: []string{"GSP-D16-F016_distance"}}) + require.NoError(t, err) + require.Len(t, selected.Cases, 1) + + for _, poolSize := range []int{1, 2, 8} { + t.Run(fmt.Sprintf("pool-%d", poolSize), func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, poolSize, 1, []int{1, 8, 16}, false, nil, "SP-B2-C-MIN-LEVEL-D", "") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(context.Background())) }) + + records, err := runner.Run(ctx, 0, 1, selected) + require.NoError(t, err) + require.Len(t, records, 1) + require.Equal(t, StatusOK, records[0].Status, records[0].Error) + require.Contains(t, records[0].SQL, "shortest_path_b2_smaller_current_level") + require.Len(t, records[0].Concurrency, 3) + for idx, concurrency := range []int{1, 8, 16} { + block := records[0].Concurrency[idx] + require.Equal(t, poolSize, block.PoolSize) + require.Equal(t, concurrency, block.Concurrency) + require.Equal(t, concurrency, block.Operations) + require.Len(t, block.Samples, concurrency) + } + if poolSize == 1 { + translation, sqlQuery, err := runner.translateCypher(ctx, selected.Cases[0].Cypher, records[0].Params) + require.NoError(t, err) + queryArgs := []any{pgx.QueryExecModeCacheStatement, pgx.QueryResultFormats{pgx.BinaryFormatCode}, pgx.NamedArgs(translation.Parameters)} + connectionHandle, err := runner.pool.Acquire(ctx) + require.NoError(t, err) + defer connectionHandle.Release() + for _, planMode := range []string{"auto", "force_custom_plan", "force_generic_plan"} { + tx, err := connectionHandle.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadWrite}) + require.NoError(t, err) + _, err = tx.Exec(ctx, "set local work_mem = '64kB'") + require.NoError(t, err) + _, err = tx.Exec(ctx, "set local plan_cache_mode = "+planMode) + require.NoError(t, err) + rows, err := tx.Query(ctx, sqlQuery, queryArgs...) + require.NoError(t, err) + var rowCount int64 + for rows.Next() { + _, err = rows.Values() + require.NoError(t, err) + rowCount++ + } + rows.Close() + require.NoError(t, rows.Err()) + require.Equal(t, records[0].RowCount, rowCount, planMode) + require.NoError(t, tx.Rollback(ctx)) + } + } + if poolSize == 2 { + translation, sqlQuery, err := runner.translateCypher(ctx, selected.Cases[0].Cypher, records[0].Params) + require.NoError(t, err) + queryArgs := []any{pgx.QueryExecModeCacheStatement, pgx.QueryResultFormats{pgx.BinaryFormatCode}, pgx.NamedArgs(translation.Parameters)} + reader, err := runner.pool.Acquire(ctx) + require.NoError(t, err) + defer reader.Release() + writer, err := runner.pool.Acquire(ctx) + require.NoError(t, err) + defer writer.Release() + const snapshotTable = "public.graphbench_traversal_snapshot_probe" + _, err = writer.Exec(ctx, "drop table if exists "+snapshotTable) + require.NoError(t, err) + _, err = writer.Exec(ctx, "create table "+snapshotTable+" (value int primary key)") + require.NoError(t, err) + t.Cleanup(func() { _, _ = runner.pool.Exec(context.Background(), "drop table if exists "+snapshotTable) }) + _, err = writer.Exec(ctx, "insert into "+snapshotTable+" values (1)") + require.NoError(t, err) + + readerTx, err := reader.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadWrite}) + require.NoError(t, err) + var before, during int + require.NoError(t, readerTx.QueryRow(ctx, "select count(*) from "+snapshotTable).Scan(&before)) + _, err = writer.Exec(ctx, "insert into "+snapshotTable+" values (2)") + require.NoError(t, err) + rows, err := readerTx.Query(ctx, sqlQuery, queryArgs...) + require.NoError(t, err) + var rowCount int64 + for rows.Next() { + _, err = rows.Values() + require.NoError(t, err) + rowCount++ + } + rows.Close() + require.NoError(t, rows.Err()) + require.Equal(t, records[0].RowCount, rowCount) + require.NoError(t, readerTx.QueryRow(ctx, "select count(*) from "+snapshotTable).Scan(&during)) + require.Equal(t, 1, before) + require.Equal(t, before, during, "candidate internal statements must retain the reader snapshot across a concurrent commit") + require.NoError(t, readerTx.Commit(ctx)) + var after int + require.NoError(t, reader.QueryRow(ctx, "select count(*) from "+snapshotTable).Scan(&after)) + require.Equal(t, 2, after) + _, err = writer.Exec(ctx, "drop table "+snapshotTable) + require.NoError(t, err) + } + }) + } +} + +// postgresPlanNodeLoops extracts Actual Loops for every EXPLAIN node with the requested alias, allowing integration assertions to detect repeated execution. +func postgresPlanNodeLoops(t *testing.T, raw json.RawMessage, alias string) []int64 { + t.Helper() + var document []map[string]any + require.NoError(t, json.Unmarshal(raw, &document)) + require.NotEmpty(t, document) + root, ok := document[0]["Plan"].(map[string]any) + require.True(t, ok) + + var ( + loops []int64 + walk func(map[string]any) + ) + + walk = func(node map[string]any) { + nodeAlias, _ := node["Alias"].(string) + functionName, _ := node["Function Name"].(string) + if nodeAlias == alias || functionName == alias { + if actualLoops, ok := node["Actual Loops"].(float64); ok { + loops = append(loops, int64(actualLoops)) + } + } + children, _ := node["Plans"].([]any) + for _, child := range children { + if childNode, ok := child.(map[string]any); ok { + walk(childNode) + } + } + } + walk(root) + return loops +} + +// TestPostgreSQLScalePlanInvariants verifies analyzed-plan capture, indexed anchors, correct mutation targets, and preserved branch-local predicates across required scale representatives. +func TestPostgreSQLScalePlanInvariants(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + required := scaleCorpusRequiredIDSet() + filtered := ScaleCorpus{} + for _, testCase := range corpus.Cases { + id := scaleCorpusCaseID(testCase.Name) + _, isRequired := required[id] + if isRequired || id == "TRUST-03" { + filtered.Cases = append(filtered.Cases, testCase) + } + } + + ctx := context.Background() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, filtered, 1, 1, nil, true, nil, "", "") + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, runner.Close(ctx)) + }) + + records, err := runner.Run(ctx, 1, 1, filtered) + require.NoError(t, err) + require.Len(t, records, len(filtered.Cases)) + + byID := map[string][]CaseResult{} + for _, record := range records { + record := record + id := scaleCorpusCaseID(record.Name) + byID[id] = append(byID[id], record) + + t.Run(record.Name, func(t *testing.T) { + require.Equal(t, StatusOK, record.Status, record.Error) + require.NotEmpty(t, record.SQL) + require.NotEmpty(t, record.PostgresPlan) + require.NotNil(t, record.PostgresMetrics) + require.NotNil(t, record.PostgresMetrics.PlanningMS) + require.NotNil(t, record.PostgresMetrics.ExecutionMS) + require.NotNil(t, record.Optimization) + + plan := strings.Join(record.PostgresPlan, "\n") + require.Contains(t, plan, "actual rows=", "plan must come from EXPLAIN ANALYZE") + assertMutationPlanTarget(t, id, plan) + assertAnchorPlanIndex(t, id, plan) + }) + } + + for _, id := range scaleCorpusRequiredIDs { + require.NotEmpty(t, byID[id], "missing PostgreSQL plan-invariant execution for %s", id) + } + + t.Run("LOGIC-01 branch-local direction and kind plan", func(t *testing.T) { + record := requireSingleScaleRecord(t, byID, "TRUST-03") + normalizedSQL := strings.ToLower(record.SQL) + require.Contains(t, normalizedSQL, " or ") + require.GreaterOrEqual(t, strings.Count(normalizedSQL, "kind_id"), 2) + require.Contains(t, normalizedSQL, "start_id") + require.Contains(t, normalizedSQL, "end_id") + }) + + t.Run("LOGIC-02 cross-binding temporal plan", func(t *testing.T) { + record := requireSingleScaleRecord(t, byID, "TRUST-01") + normalizedSQL := strings.ToLower(record.SQL) + require.Contains(t, normalizedSQL, " or ") + require.GreaterOrEqual(t, strings.Count(normalizedSQL, "lastcollected"), 2) + require.GreaterOrEqual(t, strings.Count(normalizedSQL, " < "), 2) + }) + + t.Run("LOGIC-04 filtered mutation targets", func(t *testing.T) { + edgeDelete := requireSingleScaleRecord(t, byID, "REC-01") + nodeDelete := requireSingleScaleRecord(t, byID, "REC-08") + require.Contains(t, strings.Join(edgeDelete.PostgresPlan, "\n"), "Delete on edge") + require.Contains(t, strings.Join(nodeDelete.PostgresPlan, "\n"), "Delete on node") + }) +} + +// TestPostgreSQLZeroLengthShortestMaterializersAreExact verifies that all search-and-hydration references reproduce a singleton zero-edge path and hydration-only arms avoid recursive search. +func TestPostgreSQLZeroLengthShortestMaterializersAreExact(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + var ( + zeroDepth = 0 + oneDepth = 1 + oneRow = int64(1) + ) + testCase := ScaleCase{ + Name: "GSP-D00-F001_path", + Dataset: "generated_shortest_paths_d1_f1", + Category: "generated_shortest_path", + Cypher: "MATCH p = shortestPath((s)-[:Traverse*0..1]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + NodeParams: map[string]string{ + "start_id": "sp-start", + "end_id": "sp-start", + }, + Expected: ExpectedResult{ + RowCount: &oneRow, + ResultKind: "path_set", + PathRows: []ExpectedPath{{ + Nodes: []string{"sp-start"}, + RelationshipKinds: []string{}, + }}, + }, + Observes: ObservedValues{ + Paths: true, + Nodes: true, + Relationships: true, + Properties: true, + }, + Shape: WorkloadShape{ + RootPredicate: "bound_id", + TerminalPredicate: "bound_id", + EdgeKinds: []string{"Traverse"}, + MinDepth: &zeroDepth, + MaxDepth: &oneDepth, + PathMaterializationRequired: true, + }, + CandidateModes: []ExecutionMode{ModePostgresSQL}, + } + corpus := ScaleCorpus{Cases: []ScaleCase{testCase}} + + ctx := context.Background() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, corpus, 1, 1, nil, true, nil, "", "") + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, runner.Close(ctx)) + }) + + records, err := runner.Run(ctx, 0, 1, corpus) + require.NoError(t, err) + require.Len(t, records, 1) + record := records[0] + require.Equal(t, StatusOK, record.Status, record.Error) + require.Equal(t, oneRow, record.RowCount) + + for _, name := range []string{ + "m0_directed_hydration_only", + "m1_ordered_ids_hydration_only", + "s3_unidirectional_cte_m0_directed", + "s3_unidirectional_cte_m1_ordered_ids", + } { + reference := requirePostgresReference(t, record.PostgresReferences, name) + require.Equal(t, oneRow, reference.RowCount) + require.Equal(t, record.ObservedRows, reference.ObservedRows) + if strings.Contains(name, "hydration_only") { + require.NotContains(t, reference.SQL, "with recursive") + } + } +} + +// TestPostgreSQLForcedShortestDistanceEndpointSemantics verifies zero-depth identity, missing-root emptiness, and the minimum-depth self-endpoint error under forced distance execution. +func TestPostgreSQLForcedShortestDistanceEndpointSemantics(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + var ( + zeroDepth = 0 + oneDepth = 1 + oneRow = int64(1) + zeroRows = int64(0) + zeroScalar = int64(0) + maxDepth = 1 + ) + baseShape := WorkloadShape{ + RootPredicate: "bound_id", + TerminalPredicate: "bound_id", + EdgeKinds: []string{"Traverse"}, + MaxDepth: &maxDepth, + PathMaterializationRequired: false, + } + zeroShape := baseShape + zeroShape.MinDepth = &zeroDepth + oneShape := baseShape + oneShape.MinDepth = &oneDepth + + corpus := ScaleCorpus{ + Cases: []ScaleCase{ + { + Name: "forced-shortest-zero-depth", + Dataset: "generated_shortest_paths_d1_f1", + Category: "generated_shortest_path", + Cypher: "MATCH p = shortestPath((s)-[:Traverse*0..1]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + NodeParams: map[string]string{"start_id": "sp-start", "end_id": "sp-start"}, + Expected: ExpectedResult{ + RowCount: &oneRow, + ScalarInt: &zeroScalar, + ResultKind: "scalar", + }, + Shape: zeroShape, + CandidateModes: []ExecutionMode{ModePostgresSQL}, + }, + { + Name: "forced-shortest-missing-root", + Dataset: "generated_shortest_paths_d1_f1", + Category: "generated_shortest_path", + Cypher: "MATCH p = shortestPath((s)-[:Traverse*1..1]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + Params: testutil.Params{"start_id": int64(9223372036854775807)}, + NodeParams: map[string]string{"end_id": "sp-end"}, + Expected: ExpectedResult{ + RowCount: &zeroRows, + }, + Shape: oneShape, + CandidateModes: []ExecutionMode{ModePostgresSQL}, + }, + { + Name: "forced-shortest-min-one-same-endpoint", + Dataset: "generated_shortest_paths_d1_f1", + Category: "generated_shortest_path", + Cypher: "MATCH p = shortestPath((s)-[:Traverse*1..1]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + NodeParams: map[string]string{"start_id": "sp-start", "end_id": "sp-start"}, + Expected: ExpectedResult{ + RowCount: &zeroRows, + }, + Shape: oneShape, + CandidateModes: []ExecutionMode{ModePostgresSQL}, + }, + }, + } + + ctx := context.Background() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, corpus, 1, 1, nil, false, nil, "SP-S3-U-D", "") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(ctx)) }) + + records, err := runner.Run(ctx, 0, 1, corpus) + require.NoError(t, err) + require.Len(t, records, 3) + require.Equal(t, StatusOK, records[0].Status, records[0].Error) + require.Equal(t, []string{"[0]"}, records[0].ObservedRows) + require.Equal(t, StatusOK, records[1].Status, records[1].Error) + require.Equal(t, zeroRows, records[1].RowCount) + require.Equal(t, StatusError, records[2].Status) + require.Contains(t, records[2].Error, "shortest path") +} + +// TestPostgreSQLForcedShortestDirectPreflightSkipsAndFallsBackExactly verifies that one-hop direct hits bypass the recursive harness while longer paths invoke it and preserve exact ordered path output. +func TestPostgreSQLForcedShortestDirectPreflightSkipsAndFallsBackExactly(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + oneRow := int64(1) + minDepth, maxDepth := 1, 3 + dataset := "generated_shortest_paths_v2_d3_o2_r1_fo2_fi128_l2_k7_t16_w2_x16_p0_c1_s1" + shape := WorkloadShape{ + RootPredicate: "bound_id", + TerminalPredicate: "bound_id", + EdgeKinds: []string{"Traverse"}, + Direction: "inbound", + RelationshipKindCount: 1, + MinDepth: &minDepth, + MaxDepth: &maxDepth, + PathMaterializationRequired: true, + } + multiKindMaxDepth := 2 + multiKindShape := WorkloadShape{ + RootPredicate: "bound_id", + TerminalPredicate: "bound_id", + EdgeKinds: []string{"ParallelKind00", "ParallelKind01", "ParallelKind02", "ParallelKind03", "ParallelKind04", "ParallelKind05", "ParallelKind06"}, + Direction: "outbound", + RelationshipKindCount: 7, + MinDepth: &minDepth, + MaxDepth: &multiKindMaxDepth, + PathMaterializationRequired: true, + } + corpus := ScaleCorpus{ + Cases: []ScaleCase{ + { + Name: "direct-hit", + Dataset: dataset, + Category: "generated_shortest_path", + Cypher: "MATCH p = shortestPath((root)<-[:Traverse*1..3]-(terminal)) WHERE id(root) = $root_id AND id(terminal) = $end_id RETURN p", + NodeParams: map[string]string{"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-linear-01"}, + Expected: ExpectedResult{ + RowCount: &oneRow, + ResultKind: "path_set", + PathRows: []ExpectedPath{{ + Nodes: []string{"sp-v2-inbound-root", "sp-v2-inbound-linear-01"}, + RelationshipKinds: []string{"Traverse"}, + RelationshipKeys: []string{"inbound-primary-03"}, + }}, + }, + Observes: ObservedValues{ + Paths: true, + Nodes: true, + Relationships: true, + Properties: true, + }, + Shape: shape, + CandidateModes: []ExecutionMode{ModePostgresSQL}, + }, + { + Name: "fallback-hit", + Dataset: dataset, + Category: "generated_shortest_path", + Cypher: "MATCH p = shortestPath((root)<-[:Traverse*1..3]-(terminal)) WHERE id(root) = $root_id AND id(terminal) = $end_id RETURN p", + NodeParams: map[string]string{"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + Expected: ExpectedResult{ + RowCount: &oneRow, + ResultKind: "path_set", + PathRows: []ExpectedPath{{ + Nodes: []string{"sp-v2-inbound-root", "sp-v2-inbound-linear-01", "sp-v2-inbound-linear-02", "sp-v2-inbound-end"}, + RelationshipKinds: []string{"Traverse", "Traverse", "Traverse"}, + RelationshipKeys: []string{"inbound-primary-03", "inbound-primary-02", "inbound-primary-01"}, + }}, + }, + Observes: ObservedValues{ + Paths: true, + Nodes: true, + Relationships: true, + Properties: true, + }, + Shape: shape, + CandidateModes: []ExecutionMode{ModePostgresSQL}, + }, + { + Name: "direct-multi-kind", + Dataset: dataset, + Category: "generated_shortest_path", + Cypher: "MATCH p = shortestPath((root)-[:ParallelKind00|ParallelKind01|ParallelKind02|ParallelKind03|ParallelKind04|ParallelKind05|ParallelKind06*1..2]->(terminal)) WHERE id(root) = $root_id AND id(terminal) = $end_id RETURN p", + NodeParams: map[string]string{"root_id": "sp-v2-parallel-start", "end_id": "sp-v2-parallel-target-000000"}, + Expected: ExpectedResult{ + RowCount: &oneRow, + ResultKind: "path_set", + PathRows: []ExpectedPath{{ + Nodes: []string{"sp-v2-parallel-start", "sp-v2-parallel-target-000000"}, + RelationshipKinds: []string{"ParallelKind00"}, + RelationshipKeys: []string{"parallel-k00-t000000"}, + }}, + }, + Observes: ObservedValues{ + Paths: true, + Nodes: true, + Relationships: true, + Properties: true, + }, + Shape: multiKindShape, + CandidateModes: []ExecutionMode{ModePostgresSQL}, + }, + }, + } + + ctx := context.Background() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, corpus, 1, 1, nil, false, nil, "SP-S0-DIRECT", "") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(ctx)) }) + + records, err := runner.Run(ctx, 0, 1, corpus) + require.NoError(t, err) + require.Len(t, records, 3) + for _, record := range records { + require.Equal(t, StatusOK, record.Status, "%s: %s", record.Name, record.Error) + require.Equal(t, oneRow, record.RowCount) + require.NotEmpty(t, record.PostgresPlanJSON) + } + + directLoops := postgresPlanNodeLoops(t, records[0].PostgresPlanJSON, "bidirectional_sp_harness") + require.NotEmpty(t, directLoops) + require.Equal(t, int64(0), directLoops[0], records[0].PostgresPlan) + fallbackLoops := postgresPlanNodeLoops(t, records[1].PostgresPlanJSON, "bidirectional_sp_harness") + require.NotEmpty(t, fallbackLoops) + require.Positive(t, fallbackLoops[0], records[1].PostgresPlan) + multiKindLoops := postgresPlanNodeLoops(t, records[2].PostgresPlanJSON, "bidirectional_sp_harness") + require.NotEmpty(t, multiKindLoops) + require.Equal(t, int64(0), multiKindLoops[0], records[2].PostgresPlan) +} + +// TestPostgreSQLForcedShortestDistanceCancellationReusesSession verifies prompt timeout cancellation, rollback recovery on the same backend PID, and successful replay of forced distance SQL. +func TestPostgreSQLForcedShortestDistanceCancellationReusesSession(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{Cases: []string{"GSP-D64-F1000_distance"}}) + require.NoError(t, err) + require.Len(t, selected.Cases, 1) + + ctx := context.Background() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, 1, 1, nil, false, nil, "SP-S3-U-D", "") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(ctx)) }) + + records, err := runner.Run(ctx, 0, 1, selected) + require.NoError(t, err) + require.Len(t, records, 1) + require.Equal(t, StatusOK, records[0].Status, records[0].Error) + + translation, sqlQuery, err := runner.translateCypher(ctx, selected.Cases[0].Cypher, records[0].Params) + require.NoError(t, err) + queryArgs := []any{pgx.QueryExecModeCacheStatement, pgx.QueryResultFormats{pgx.BinaryFormatCode}, pgx.NamedArgs(translation.Parameters)} + + connectionHandle, err := runner.pool.Acquire(ctx) + require.NoError(t, err) + defer connectionHandle.Release() + backendPID := connectionHandle.Conn().PgConn().PID() + + tx, err := connectionHandle.BeginTx(ctx, postgresConcurrencyTxOptions()) + require.NoError(t, err) + _, err = tx.Exec(ctx, "set local statement_timeout = '1ms'") + require.NoError(t, err) + started := time.Now() + rows, queryErr := tx.Query(ctx, sqlQuery, queryArgs...) + if queryErr == nil { + for rows.Next() { + _, queryErr = rows.Values() + if queryErr != nil { + break + } + } + rows.Close() + if queryErr == nil { + queryErr = rows.Err() + } + } + cancellationLatency := time.Since(started) + var postgresError *pgconn.PgError + require.ErrorAs(t, queryErr, &postgresError) + require.Equal(t, "57014", postgresError.Code) + require.Less(t, cancellationLatency, 250*time.Millisecond) + require.NoError(t, tx.Rollback(ctx)) + + var reusedPID uint32 + require.NoError(t, connectionHandle.QueryRow(ctx, "select pg_backend_pid()").Scan(&reusedPID)) + require.Equal(t, backendPID, reusedPID) + + rows, err = connectionHandle.Query(ctx, sqlQuery, queryArgs...) + require.NoError(t, err) + rowCount := 0 + for rows.Next() { + _, err = rows.Values() + require.NoError(t, err) + rowCount++ + } + rows.Close() + require.NoError(t, rows.Err()) + require.Equal(t, 1, rowCount) + t.Logf("cancelled exact SP-S3-U-D SQL in %s and reused backend PID %d", cancellationLatency, backendPID) +} + +// TestPostgreSQLForcedShortestPathEdgeM0PlanResourcesAndConcurrency verifies direct edge-array hydration, zero local/temp/WAL usage, concurrency sample counts, and no edge work for a missing endpoint. +func TestPostgreSQLForcedShortestPathEdgeM0PlanResourcesAndConcurrency(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{Cases: []string{"GSP-D16-F016_path"}}) + require.NoError(t, err) + require.Len(t, selected.Cases, 1) + zeroRows := int64(0) + missingEndpoint := selected.Cases[0] + missingEndpoint.Name = "forced-m0-missing-start-endpoint" + missingEndpoint.Params = testutil.Params{"start_id": int64(9223372036854775807)} + missingEndpoint.NodeParams = map[string]string{"end_id": "sp-end"} + missingEndpoint.Expected = ExpectedResult{ + RowCount: &zeroRows, + ResultKind: "path_set", + } + selected.Cases = append(selected.Cases, missingEndpoint) + + ctx := context.Background() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, 2, 1, []int{1, 2, 4}, true, nil, "SP-S3-U-E+MAT-M0", "") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(ctx)) }) + + records, err := runner.Run(ctx, 0, 25, selected) + require.NoError(t, err) + require.Len(t, records, 2) + record := records[0] + require.Equal(t, StatusOK, record.Status, record.Error) + require.Contains(t, record.SQL, "s1(next_id, depth, path)") + require.Equal(t, 1, strings.Count(record.SQL, "generate_subscripts(s1.path, 1)"), record.SQL) + require.NotContains(t, record.SQL, "ordered_edge_ids_to_path") + require.NotContains(t, record.SQL, "sp_harness") + + require.NotNil(t, record.PostgresMetrics) + metrics := record.PostgresMetrics + require.Greater(t, metrics.RecursiveRows, int64(0)) + require.Greater(t, metrics.HydrationLoops, int64(0)) + require.Zero(t, metrics.Buffers.LocalHit) + require.Zero(t, metrics.Buffers.LocalRead) + require.Zero(t, metrics.Buffers.LocalDirtied) + require.Zero(t, metrics.Buffers.LocalWritten) + require.Zero(t, metrics.Buffers.TempRead) + require.Zero(t, metrics.Buffers.TempWritten) + require.Zero(t, metrics.TempFiles) + require.Zero(t, metrics.TempBytes) + require.Zero(t, metrics.WALRecords) + require.Zero(t, metrics.WALBytes) + + require.Len(t, record.Concurrency, 3) + for index, level := range []int{1, 2, 4} { + block := record.Concurrency[index] + require.Equal(t, level, block.Concurrency) + require.Equal(t, 2, block.PoolSize) + require.Equal(t, level*25, block.Operations) + require.Len(t, block.Samples, level*25) + } + + missingRecord := records[1] + require.Equal(t, StatusOK, missingRecord.Status, missingRecord.Error) + require.Zero(t, missingRecord.RowCount) + require.NotNil(t, missingRecord.PostgresMetrics) + require.Zero(t, missingRecord.PostgresMetrics.RecursiveRows) + var missingEdgeLoops int64 + for _, node := range missingRecord.PostgresMetrics.PlanNodes { + if node.RelationName == "edge" || strings.HasPrefix(node.RelationName, "edge_") { + missingEdgeLoops += node.ActualLoops + } + } + require.Zero(t, missingEdgeLoops, "missing endpoint must execute zero edge-search loops") +} + +// TestPostgreSQLForcedShortestPathEdgeM0CancellationReusesSession verifies prompt timeout cancellation, rollback recovery on the same backend PID, and successful replay of M0 path SQL. +func TestPostgreSQLForcedShortestPathEdgeM0CancellationReusesSession(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{Cases: []string{"GSP-D64-F1000_path"}}) + require.NoError(t, err) + require.Len(t, selected.Cases, 1) + + ctx := context.Background() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, 1, 1, nil, false, nil, "SP-S3-U-E+MAT-M0", "") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(ctx)) }) + + records, err := runner.Run(ctx, 0, 1, selected) + require.NoError(t, err) + require.Len(t, records, 1) + require.Equal(t, StatusOK, records[0].Status, records[0].Error) + + translation, sqlQuery, err := runner.translateCypher(ctx, selected.Cases[0].Cypher, records[0].Params) + require.NoError(t, err) + queryArgs := []any{pgx.QueryExecModeCacheStatement, pgx.QueryResultFormats{pgx.BinaryFormatCode}, pgx.NamedArgs(translation.Parameters)} + + connectionHandle, err := runner.pool.Acquire(ctx) + require.NoError(t, err) + defer connectionHandle.Release() + backendPID := connectionHandle.Conn().PgConn().PID() + + tx, err := connectionHandle.BeginTx(ctx, postgresConcurrencyTxOptions()) + require.NoError(t, err) + _, err = tx.Exec(ctx, "set local statement_timeout = '1ms'") + require.NoError(t, err) + started := time.Now() + rows, queryErr := tx.Query(ctx, sqlQuery, queryArgs...) + if queryErr == nil { + for rows.Next() { + _, queryErr = rows.Values() + if queryErr != nil { + break + } + } + rows.Close() + if queryErr == nil { + queryErr = rows.Err() + } + } + cancellationLatency := time.Since(started) + var postgresError *pgconn.PgError + require.ErrorAs(t, queryErr, &postgresError) + require.Equal(t, "57014", postgresError.Code) + require.Less(t, cancellationLatency, 250*time.Millisecond) + require.NoError(t, tx.Rollback(ctx)) + + var reusedPID uint32 + require.NoError(t, connectionHandle.QueryRow(ctx, "select pg_backend_pid()").Scan(&reusedPID)) + require.Equal(t, backendPID, reusedPID) + + rows, err = connectionHandle.Query(ctx, sqlQuery, queryArgs...) + require.NoError(t, err) + rowCount := 0 + for rows.Next() { + _, err = rows.Values() + require.NoError(t, err) + rowCount++ + } + rows.Close() + require.NoError(t, rows.Err()) + require.Equal(t, 1, rowCount) + t.Logf("cancelled exact SP-S3-U-E+MAT-M0 SQL in %s and reused backend PID %d", cancellationLatency, backendPID) +} + +// TestPostgreSQLForcedSuffixSeededReversePlanResourcesAndConcurrency verifies compact reverse-search SQL, relationship uniqueness, zero local/temp/WAL usage, and complete samples at each concurrency level. +func TestPostgreSQLForcedSuffixSeededReversePlanResourcesAndConcurrency(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{ + Cases: []string{"GFSE-V2-D16-F1000-R1-X1-M1-sparse_path"}, + }) + require.NoError(t, err) + require.Len(t, selected.Cases, 1) + + ctx := context.Background() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, 2, 1, []int{1, 2, 4}, false, nil, "", "EXPANSION-SUFFIX-SEEDED-REVERSE") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(ctx)) }) + + records, err := runner.Run(ctx, 0, 25, selected) + require.NoError(t, err) + require.Len(t, records, 1) + record := records[0] + require.Equal(t, StatusOK, record.Status, record.Error) + require.Contains(t, record.SQL, "_suffix_seeded_suffix as materialized") + require.Contains(t, record.SQL, "_suffix_seeded_reverse(boundary_id, next_id, depth, path)") + require.Contains(t, record.SQL, "array_prepend") + require.Contains(t, record.SQL, "!= all (") + require.NotContains(t, record.SQL, "satisfied, is_cycle") + + require.NotNil(t, record.PostgresMetrics) + metrics := record.PostgresMetrics + require.Greater(t, metrics.RecursiveRows, int64(0)) + require.Zero(t, metrics.Buffers.LocalHit) + require.Zero(t, metrics.Buffers.LocalRead) + require.Zero(t, metrics.Buffers.LocalDirtied) + require.Zero(t, metrics.Buffers.LocalWritten) + require.Zero(t, metrics.Buffers.TempRead) + require.Zero(t, metrics.Buffers.TempWritten) + require.Zero(t, metrics.TempFiles) + require.Zero(t, metrics.TempBytes) + require.Zero(t, metrics.WALRecords) + require.Zero(t, metrics.WALBytes) + + require.Len(t, record.Concurrency, 3) + for index, level := range []int{1, 2, 4} { + block := record.Concurrency[index] + require.Equal(t, level, block.Concurrency) + require.Equal(t, 2, block.PoolSize) + require.Equal(t, level*25, block.Operations) + require.Len(t, block.Samples, level*25) + } +} + +// TestPostgreSQLForcedSuffixSeededReverseCancellationReusesSession verifies prompt timeout cancellation, rollback recovery on the same backend PID, and cardinality-preserving replay of reverse expansion SQL. +func TestPostgreSQLForcedSuffixSeededReverseCancellationReusesSession(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{ + Cases: []string{"GFSE-V2-D08-F016-R1-I1000-high_reverse_fanin"}, + }) + require.NoError(t, err) + require.Len(t, selected.Cases, 1) + + ctx := context.Background() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, 1, 1, nil, false, nil, "", "EXPANSION-SUFFIX-SEEDED-REVERSE") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(ctx)) }) + records, err := runner.Run(ctx, 0, 1, selected) + require.NoError(t, err) + require.Len(t, records, 1) + require.Equal(t, StatusOK, records[0].Status, records[0].Error) + + translation, sqlQuery, err := runner.translateCypher(ctx, selected.Cases[0].Cypher, records[0].Params) + require.NoError(t, err) + queryArgs := []any{pgx.QueryExecModeCacheStatement, pgx.QueryResultFormats{pgx.BinaryFormatCode}, pgx.NamedArgs(translation.Parameters)} + + connectionHandle, err := runner.pool.Acquire(ctx) + require.NoError(t, err) + defer connectionHandle.Release() + backendPID := connectionHandle.Conn().PgConn().PID() + tx, err := connectionHandle.BeginTx(ctx, postgresConcurrencyTxOptions()) + require.NoError(t, err) + _, err = tx.Exec(ctx, "set local statement_timeout = '1ms'") + require.NoError(t, err) + started := time.Now() + rows, queryErr := tx.Query(ctx, sqlQuery, queryArgs...) + if queryErr == nil { + for rows.Next() { + _, queryErr = rows.Values() + if queryErr != nil { + break + } + } + rows.Close() + if queryErr == nil { + queryErr = rows.Err() + } + } + cancellationLatency := time.Since(started) + var postgresError *pgconn.PgError + require.ErrorAs(t, queryErr, &postgresError) + require.Equal(t, "57014", postgresError.Code) + require.Less(t, cancellationLatency, 250*time.Millisecond) + require.NoError(t, tx.Rollback(ctx)) + + var reusedPID uint32 + require.NoError(t, connectionHandle.QueryRow(ctx, "select pg_backend_pid()").Scan(&reusedPID)) + require.Equal(t, backendPID, reusedPID) + rows, err = connectionHandle.Query(ctx, sqlQuery, queryArgs...) + require.NoError(t, err) + rowCount := 0 + for rows.Next() { + _, err = rows.Values() + require.NoError(t, err) + rowCount++ + } + rows.Close() + require.NoError(t, rows.Err()) + require.Equal(t, records[0].RowCount, int64(rowCount)) + t.Logf("cancelled exact EXPANSION-SUFFIX-SEEDED-REVERSE SQL in %s and reused backend PID %d", cancellationLatency, backendPID) +} + +// TestPostgreSQLForcedBidirectionalShortestCandidatesPreservePublicResults +// verifies both scheduler wrappers at the distance, one-witness, and complete +// all-shortest public boundaries. ASP production selection remains A1; these +// identities are reachable only through explicit tool forcing. +func TestPostgreSQLForcedBidirectionalShortestCandidatesPreservePublicResults(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + tests := []struct { + name string + caseName string + executor string + functionName string + }{ + {name: "SP B1 distance", caseName: "GSP-D16-F016_distance", executor: "SP-B1-C-ALT-NODE-D", functionName: "shortest_path_b1_strict_alternating"}, + {name: "SP B2 witness", caseName: "GSP-D16-F016_path", executor: "SP-B2-C-MIN-LEVEL-WE+MAT-M0", functionName: "shortest_path_b2_smaller_current_level"}, + {name: "ASP B1 complete multiset", caseName: "GSPV2-NORMAL-outbound-all-shortest-depth3", executor: "ASP-B1-DAG-ALT-NODE", functionName: "all_shortest_paths_b1_strict_alternating"}, + {name: "ASP B2 complete multiset", caseName: "GSPV2-NORMAL-outbound-all-shortest-depth3", executor: "ASP-B2-DAG-MIN-LEVEL", functionName: "all_shortest_paths_b2_smaller_current_level"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{Cases: []string{test.caseName}}) + require.NoError(t, err) + require.Len(t, selected.Cases, 1) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, 1, 1, nil, false, nil, test.executor, "") + require.NoError(t, err) + defer func() { require.NoError(t, runner.Close(context.Background())) }() + + records, err := runner.Run(ctx, 0, 1, selected) + require.NoError(t, err) + require.Len(t, records, 1) + record := records[0] + require.Equal(t, StatusOK, record.Status, record.Error) + require.Contains(t, record.SQL, test.functionName) + require.NotNil(t, record.Optimization) + found := false + for _, outcome := range record.Optimization.TargetOutcomes { + if outcome.Applied == test.executor { + found = true + break + } + } + require.True(t, found, "forced traversal outcome missing from %+v", record.Optimization.TargetOutcomes) + }) + } +} + +// TestPostgreSQLBidirectionalASPCancellationAndSessionIsolation verifies an +// aborted B1 replay rolls back cleanly, the same backend PID can immediately +// execute again, and identical invocation keys on two pooled sessions never +// share workspace or diagnostic rows. +func TestPostgreSQLBidirectionalASPCancellationAndSessionIsolation(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{Cases: []string{"GSP-D64-F1000_path"}}) + require.NoError(t, err) + require.Len(t, selected.Cases, 1) + selected.Cases[0].Name = "forced-asp-b1-operational-depth64" + selected.Cases[0].Cypher = strings.Replace(selected.Cases[0].Cypher, "shortestPath", "allShortestPaths", 1) + selected.Cases[0].Category = "generated_all_shortest_paths" + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, 2, 1, nil, false, nil, "ASP-B1-DAG-ALT-NODE", "") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(context.Background())) }) + records, err := runner.Run(ctx, 0, 1, selected) + require.NoError(t, err) + require.Len(t, records, 1) + require.Equal(t, StatusOK, records[0].Status, records[0].Error) + + translation, sqlQuery, err := runner.translateCypher(ctx, selected.Cases[0].Cypher, records[0].Params) + require.NoError(t, err) + queryArgs := []any{pgx.QueryExecModeCacheStatement, pgx.QueryResultFormats{pgx.BinaryFormatCode}, pgx.NamedArgs(translation.Parameters)} + + first, err := runner.pool.Acquire(ctx) + require.NoError(t, err) + defer first.Release() + second, err := runner.pool.Acquire(ctx) + require.NoError(t, err) + defer second.Release() + firstPID, secondPID := first.Conn().PgConn().PID(), second.Conn().PgConn().PID() + require.NotEqual(t, firstPID, secondPID) + // Materialize session-local telemetry tables outside the rollback checks so + // both sessions have the same schema but independent contents. + _, err = first.Exec(ctx, "select public.ensure_bidirectional_all_shortest_path_workspace()") + require.NoError(t, err) + _, err = second.Exec(ctx, "select public.ensure_bidirectional_all_shortest_path_workspace()") + require.NoError(t, err) + _, err = first.Exec(ctx, "select public.ensure_bidirectional_all_shortest_path_telemetry_workspace()") + require.NoError(t, err) + _, err = second.Exec(ctx, "select public.ensure_bidirectional_all_shortest_path_telemetry_workspace()") + require.NoError(t, err) + + drain := func(tx pgx.Tx) int64 { + rows, err := tx.Query(ctx, sqlQuery, queryArgs...) + require.NoError(t, err) + defer rows.Close() + var count int64 + for rows.Next() { + _, err = rows.Values() + require.NoError(t, err) + count++ + } + require.NoError(t, rows.Err()) + return count + } + readCalls := func(tx pgx.Tx, invocationID string) (int64, bool) { + var raw string + err := tx.QueryRow(ctx, "select coalesce(public.read_bidirectional_all_shortest_path_diagnostic_v1($1)::text, '')", invocationID).Scan(&raw) + require.NoError(t, err) + if raw == "" { + return 0, false + } + var document struct { + SearchCalls int64 `json:"search_calls"` + } + require.NoError(t, json.Unmarshal([]byte(raw), &document)) + return document.SearchCalls, true + } + + firstTx, err := first.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadWrite}) + require.NoError(t, err) + secondTx, err := second.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadWrite}) + require.NoError(t, err) + const sharedInvocation = "same-key-different-sessions" + _, err = firstTx.Exec(ctx, "select public.begin_bidirectional_all_shortest_path_diagnostic_v1($1)", sharedInvocation) + require.NoError(t, err) + _, err = secondTx.Exec(ctx, "select public.begin_bidirectional_all_shortest_path_diagnostic_v1($1)", sharedInvocation) + require.NoError(t, err) + require.Equal(t, records[0].RowCount, drain(firstTx)) + firstCalls, found := readCalls(firstTx, sharedInvocation) + require.True(t, found) + require.Equal(t, int64(1), firstCalls) + secondCalls, found := readCalls(secondTx, sharedInvocation) + require.True(t, found) + require.Zero(t, secondCalls) + _, err = firstTx.Exec(ctx, "select public.clear_bidirectional_all_shortest_path_diagnostic_v1($1)", sharedInvocation) + require.NoError(t, err) + _, found = readCalls(firstTx, sharedInvocation) + require.False(t, found) + _, found = readCalls(secondTx, sharedInvocation) + require.True(t, found) + require.NoError(t, firstTx.Rollback(ctx)) + require.NoError(t, secondTx.Rollback(ctx)) + + cancelTx, err := first.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadWrite}) + require.NoError(t, err) + _, err = cancelTx.Exec(ctx, "select public.begin_bidirectional_all_shortest_path_diagnostic_v1('cancelled-replay')") + require.NoError(t, err) + _, err = cancelTx.Exec(ctx, "set local statement_timeout = '1ms'") + require.NoError(t, err) + started := time.Now() + rows, queryErr := cancelTx.Query(ctx, sqlQuery, queryArgs...) + if queryErr == nil { + for rows.Next() { + _, queryErr = rows.Values() + if queryErr != nil { + break + } + } + rows.Close() + if queryErr == nil { + queryErr = rows.Err() + } + } + cancellationLatency := time.Since(started) + var postgresError *pgconn.PgError + require.ErrorAs(t, queryErr, &postgresError) + require.Equal(t, "57014", postgresError.Code) + require.Less(t, cancellationLatency, 250*time.Millisecond) + require.NoError(t, cancelTx.Rollback(ctx)) + require.Equal(t, firstPID, first.Conn().PgConn().PID()) + + reuseTx, err := first.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadWrite}) + require.NoError(t, err) + _, found = readCalls(reuseTx, "cancelled-replay") + require.False(t, found, "rolled-back invocation state must not survive") + _, err = reuseTx.Exec(ctx, "select public.begin_bidirectional_all_shortest_path_diagnostic_v1('successful-reuse')") + require.NoError(t, err) + require.Equal(t, records[0].RowCount, drain(reuseTx)) + reuseCalls, found := readCalls(reuseTx, "successful-reuse") + require.True(t, found) + require.Equal(t, int64(1), reuseCalls) + _, err = reuseTx.Exec(ctx, "select public.clear_bidirectional_all_shortest_path_diagnostic_v1('successful-reuse')") + require.NoError(t, err) + require.NoError(t, reuseTx.Commit(ctx)) + t.Logf("cancelled ASP-B1 in %s and reused backend PID %d without cross-session state from PID %d", cancellationLatency, firstPID, secondPID) +} + +// TestPostgreSQLBidirectionalSPCancellationAndSessionIsolation applies the +// cancellation, rollback/reuse, and session-local telemetry contract to both +// compact SP schedulers. Candidate and telemetry workspaces are materialized +// before the timed query so the timeout interrupts search rather than DDL. +func TestPostgreSQLBidirectionalSPCancellationAndSessionIsolation(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + for _, scheduler := range []struct { + name string + executor string + functionName string + }{ + {name: "B1 strict alternating", executor: "SP-B1-C-ALT-NODE-WE+MAT-M0", functionName: "shortest_path_b1_strict_alternating"}, + {name: "B2 smaller level", executor: "SP-B2-C-MIN-LEVEL-WE+MAT-M0", functionName: "shortest_path_b2_smaller_current_level"}, + } { + scheduler := scheduler + t.Run(scheduler.name, func(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{Cases: []string{"GSP-D64-F1000_path"}}) + require.NoError(t, err) + require.Len(t, selected.Cases, 1) + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, 2, 1, nil, false, nil, scheduler.executor, "") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(context.Background())) }) + records, err := runner.Run(ctx, 0, 1, selected) + require.NoError(t, err) + require.Len(t, records, 1) + require.Equal(t, StatusOK, records[0].Status, records[0].Error) + require.Contains(t, records[0].SQL, scheduler.functionName) + + translation, sqlQuery, err := runner.translateCypher(ctx, selected.Cases[0].Cypher, records[0].Params) + require.NoError(t, err) + queryArgs := []any{pgx.QueryExecModeCacheStatement, pgx.QueryResultFormats{pgx.BinaryFormatCode}, pgx.NamedArgs(translation.Parameters)} + + first, err := runner.pool.Acquire(ctx) + require.NoError(t, err) + defer first.Release() + second, err := runner.pool.Acquire(ctx) + require.NoError(t, err) + defer second.Release() + firstPID, secondPID := first.Conn().PgConn().PID(), second.Conn().PgConn().PID() + require.NotEqual(t, firstPID, secondPID) + for _, session := range []*pgxpool.Conn{first, second} { + _, err = session.Exec(ctx, "select public.ensure_bidirectional_shortest_path_workspace()") + require.NoError(t, err) + _, err = session.Exec(ctx, "select public.ensure_bidirectional_shortest_path_telemetry_workspace()") + require.NoError(t, err) + } + + drain := func(tx pgx.Tx) int64 { + rows, queryErr := tx.Query(ctx, sqlQuery, queryArgs...) + require.NoError(t, queryErr) + defer rows.Close() + var count int64 + for rows.Next() { + _, queryErr = rows.Values() + require.NoError(t, queryErr) + count++ + } + require.NoError(t, rows.Err()) + return count + } + readCalls := func(tx pgx.Tx, invocationID string) (int64, bool) { + var raw string + err := tx.QueryRow(ctx, "select coalesce(public.read_bidirectional_shortest_path_diagnostic_v1($1)::text, '')", invocationID).Scan(&raw) + require.NoError(t, err) + if raw == "" { + return 0, false + } + var document struct { + SearchCalls int64 `json:"search_calls"` + } + require.NoError(t, json.Unmarshal([]byte(raw), &document)) + return document.SearchCalls, true + } + + firstTx, err := first.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadWrite}) + require.NoError(t, err) + secondTx, err := second.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadWrite}) + require.NoError(t, err) + invocationID := "sp-same-key-" + scheduler.executor + _, err = firstTx.Exec(ctx, "select public.begin_bidirectional_shortest_path_diagnostic_v1($1)", invocationID) + require.NoError(t, err) + _, err = secondTx.Exec(ctx, "select public.begin_bidirectional_shortest_path_diagnostic_v1($1)", invocationID) + require.NoError(t, err) + require.Equal(t, records[0].RowCount, drain(firstTx)) + firstCalls, found := readCalls(firstTx, invocationID) + require.True(t, found) + require.Equal(t, int64(1), firstCalls) + secondCalls, found := readCalls(secondTx, invocationID) + require.True(t, found) + require.Zero(t, secondCalls) + _, err = firstTx.Exec(ctx, "select public.clear_bidirectional_shortest_path_diagnostic_v1($1)", invocationID) + require.NoError(t, err) + _, found = readCalls(firstTx, invocationID) + require.False(t, found) + _, found = readCalls(secondTx, invocationID) + require.True(t, found) + require.NoError(t, firstTx.Rollback(ctx)) + require.NoError(t, secondTx.Rollback(ctx)) + + cancelTx, err := first.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadWrite}) + require.NoError(t, err) + cancelInvocation := "sp-cancelled-" + scheduler.executor + _, err = cancelTx.Exec(ctx, "select public.begin_bidirectional_shortest_path_diagnostic_v1($1)", cancelInvocation) + require.NoError(t, err) + _, err = cancelTx.Exec(ctx, "set local statement_timeout = '1ms'") + require.NoError(t, err) + started := time.Now() + rows, queryErr := cancelTx.Query(ctx, sqlQuery, queryArgs...) + if queryErr == nil { + for rows.Next() { + _, queryErr = rows.Values() + if queryErr != nil { + break + } + } + rows.Close() + if queryErr == nil { + queryErr = rows.Err() + } + } + cancellationLatency := time.Since(started) + var postgresError *pgconn.PgError + require.ErrorAs(t, queryErr, &postgresError) + require.Equal(t, "57014", postgresError.Code) + require.Less(t, cancellationLatency, 250*time.Millisecond) + require.NoError(t, cancelTx.Rollback(ctx)) + require.Equal(t, firstPID, first.Conn().PgConn().PID()) + + reuseTx, err := first.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadWrite}) + require.NoError(t, err) + _, found = readCalls(reuseTx, cancelInvocation) + require.False(t, found, "rolled-back invocation state must not survive") + reuseInvocation := "sp-successful-" + scheduler.executor + _, err = reuseTx.Exec(ctx, "select public.begin_bidirectional_shortest_path_diagnostic_v1($1)", reuseInvocation) + require.NoError(t, err) + require.Equal(t, records[0].RowCount, drain(reuseTx)) + reuseCalls, found := readCalls(reuseTx, reuseInvocation) + require.True(t, found) + require.Equal(t, int64(1), reuseCalls) + _, err = reuseTx.Exec(ctx, "select public.clear_bidirectional_shortest_path_diagnostic_v1($1)", reuseInvocation) + require.NoError(t, err) + require.NoError(t, reuseTx.Commit(ctx)) + t.Logf("cancelled %s in %s and reused backend PID %d without cross-session state from PID %d", scheduler.executor, cancellationLatency, firstPID, secondPID) + }) + } +} + +// requirePostgresReference returns the named comparator result or fails when the runner omitted that reference arm. +func requirePostgresReference(t *testing.T, references []PostgresReferenceResult, name string) PostgresReferenceResult { + t.Helper() + for _, reference := range references { + if reference.Name == name { + return reference + } + } + t.Fatalf("missing PostgreSQL reference %s", name) + return PostgresReferenceResult{} +} + +// requireSingleScaleRecord returns the sole result for a corpus ID and rejects missing or duplicate representatives. +func requireSingleScaleRecord(t *testing.T, byID map[string][]CaseResult, id string) CaseResult { + t.Helper() + require.Len(t, byID[id], 1, "%s must have one representative", id) + return byID[id][0] +} + +// assertMutationPlanTarget verifies that delete representatives modify the physical entity table implied by their corpus ID. +func assertMutationPlanTarget(t *testing.T, id, plan string) { + t.Helper() + + switch id { + case "REC-01", "REC-02", "REC-04", "REC-06": + require.Contains(t, plan, "Delete on edge") + case "REC-08": + require.Contains(t, plan, "Delete on node") + } +} + +// assertAnchorPlanIndex verifies that each indexed representative anchors through an endpoint or selective graph-partition index rather than a heap-wide scan. +func assertAnchorPlanIndex(t *testing.T, id, plan string) { + t.Helper() + + switch id { + case "HOP-01", "HOP-03", "HOP-04", "HOP-05": + // PostgreSQL may prefer the covering kind index when the edge kind is + // more selective than the bound endpoint. Both choices remain scoped + // to the graph partition and avoid a heap-wide edge scan. + require.Regexp(t, `(Bitmap Index Scan on|Index Scan using) edge_[0-9]+_(start_id|kind_id)`, plan) + require.Contains(t, plan, "start_id =") + case "HOP-02": + require.Regexp(t, `(Bitmap Index Scan on|Index Scan using) edge_[0-9]+_end_id`, plan) + case "HOP-07": + // The selective terminal predicate can legitimately reverse the join + // order, but either endpoint orientation must stay indexed. + require.Regexp(t, `(Bitmap Index Scan on|Index Scan using) edge_[0-9]+_(start|end)_id`, plan) + case "REC-01", "REC-02", "REC-04", "REC-06", "REC-08", "SCAN-05", + "LOOKUP-02", "LOOKUP-04", "LOOKUP-05", "LOOKUP-09", "LOOKUP-11", "LOOKUP-13", "LOOKUP-16", + "TRUST-01", "TRUST-02", "PRUNE-02", "PRUNE-03": + require.Contains(t, plan, "Index Scan") + } +} diff --git a/cmd/graphbench/promotion_manifest.go b/cmd/graphbench/promotion_manifest.go new file mode 100644 index 00000000..27b77864 --- /dev/null +++ b/cmd/graphbench/promotion_manifest.go @@ -0,0 +1,381 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" +) + +const promotionManifestVersion = 2 + +var requiredPromotionEvidenceRoles = []string{ + "aa", "confirmation", "performance", "resource", "reference_closure", "operational", +} + +func orientationPromotionCaps() map[string]int64 { + return map[string]int64{ + "root_row_limit": optimize.ExpansionSearchOrientationRootRowLimit, + "reverse_seed_row_limit": optimize.ExpansionSearchOrientationReverseSeedRowLimit, + "directional_degree_row_limit": optimize.ExpansionSearchOrientationDirectionalDegreeRowLimit, + "state_limit": optimize.ExpansionSearchOrientationStateLimit, + } +} + +func validateStaticV6CanonicalInboundBucket(bucket PromotionBucket) error { + if bucket.Direction != "inbound" || bucket.ObservationMode != string(optimize.ShortestPathObservationOnePath) || + bucket.MinimumDepth != 1 || bucket.MaximumDepth != 64 || bucket.RelationshipKindCount != 1 || bucket.UntypedRelationship { + return fmt.Errorf("SP-I1 canonical witness bucket %s must be the qualified inbound typed single-kind one-path depth 1..64 envelope", bucket.Name) + } + return nil +} + +type PromotionEvidenceReference struct { + Path string `json:"path"` + SHA256 string `json:"sha256"` +} + +type PromotionBucket struct { + Name string `json:"name"` + QuerySHA256 []string `json:"query_sha256"` + Direction string `json:"direction,omitempty"` + ObservationMode string `json:"observation_mode,omitempty"` + MinimumDepth int `json:"minimum_depth,omitempty"` + MaximumDepth int `json:"maximum_depth,omitempty"` + RelationshipKindCount int `json:"relationship_kind_count,omitempty"` + UntypedRelationship bool `json:"untyped_relationship,omitempty"` + QualificationSplit []string `json:"qualification_split"` +} + +// PromotionManifest is the sole authorization record consumed by a rollout. +// It binds one immutable candidate and selector to source, binary, corpus, +// caps, exact query cohorts, and every required passing report. +type PromotionManifest struct { + Version int `json:"version"` + Candidate string `json:"candidate"` + SelectorVersion string `json:"selector_version"` + ExecutionBoundary string `json:"execution_boundary"` + FallbackExecutor string `json:"fallback_executor,omitempty"` + SourceCommit string `json:"source_commit"` + SourceSHA256 string `json:"source_sha256"` + BinarySHA256 string `json:"binary_sha256"` + CorpusSHA256 string `json:"corpus_sha256"` + Caps map[string]int64 `json:"caps"` + Buckets []PromotionBucket `json:"buckets"` + Evidence map[string]PromotionEvidenceReference `json:"evidence"` +} + +// PromotionEvidenceIdentity is repeated verbatim by every evidence report. +// It deliberately excludes evidence paths and digests, avoiding a circular +// dependency while binding the report to every authorization-relevant field. +type PromotionEvidenceIdentity struct { + Candidate string `json:"candidate"` + SelectorVersion string `json:"selector_version"` + ExecutionBoundary string `json:"execution_boundary"` + FallbackExecutor string `json:"fallback_executor,omitempty"` + SourceCommit string `json:"source_commit"` + SourceSHA256 string `json:"source_sha256"` + BinarySHA256 string `json:"binary_sha256"` + CorpusSHA256 string `json:"corpus_sha256"` + Caps map[string]int64 `json:"caps"` + Buckets []PromotionBucket `json:"buckets"` +} + +func promotionEvidenceIdentity(manifest PromotionManifest) PromotionEvidenceIdentity { + return PromotionEvidenceIdentity{ + Candidate: manifest.Candidate, SelectorVersion: manifest.SelectorVersion, + ExecutionBoundary: manifest.ExecutionBoundary, FallbackExecutor: manifest.FallbackExecutor, + SourceCommit: manifest.SourceCommit, SourceSHA256: manifest.SourceSHA256, + BinarySHA256: manifest.BinarySHA256, CorpusSHA256: manifest.CorpusSHA256, + Caps: clonePromotionCaps(manifest.Caps), Buckets: clonePromotionBuckets(manifest.Buckets), + } +} + +func clonePromotionCaps(input map[string]int64) map[string]int64 { + result := make(map[string]int64, len(input)) + for name, value := range input { + result[name] = value + } + return result +} + +func clonePromotionBuckets(input []PromotionBucket) []PromotionBucket { + result := append([]PromotionBucket(nil), input...) + for idx := range result { + result[idx].QuerySHA256 = append([]string(nil), result[idx].QuerySHA256...) + result[idx].QualificationSplit = append([]string(nil), result[idx].QualificationSplit...) + } + return result +} + +type PromotionManifestVerification struct { + Version int `json:"version"` + ManifestSHA256 string `json:"manifest_sha256"` + Candidate string `json:"candidate,omitempty"` + SelectorVersion string `json:"selector_version,omitempty"` + Passed bool `json:"passed"` + Reasons []string `json:"reasons,omitempty"` +} + +func verifyPromotionManifest(path string) (PromotionManifestVerification, error) { + raw, err := os.ReadFile(path) + if err != nil { + return PromotionManifestVerification{}, err + } + digest := sha256.Sum256(raw) + verification := PromotionManifestVerification{Version: promotionManifestVersion, ManifestSHA256: hex.EncodeToString(digest[:]), Passed: true} + var manifest PromotionManifest + if err := json.Unmarshal(raw, &manifest); err != nil { + return PromotionManifestVerification{}, fmt.Errorf("decode promotion manifest: %w", err) + } + verification.Candidate = manifest.Candidate + verification.SelectorVersion = manifest.SelectorVersion + addReason := func(reason string) { + verification.Passed = false + verification.Reasons = append(verification.Reasons, reason) + } + if manifest.Version != promotionManifestVersion { + addReason("manifest version must be 2") + } + if strings.TrimSpace(manifest.Candidate) == "" || strings.TrimSpace(manifest.SelectorVersion) == "" { + addReason("candidate and selector_version are required") + } + if manifest.ExecutionBoundary != "inline_statement" && manifest.ExecutionBoundary != "stored_helper" && manifest.ExecutionBoundary != "guarded_dual_arm" { + addReason("execution_boundary must identify the measured production boundary") + } + for name, value := range map[string]string{"source_sha256": manifest.SourceSHA256, "binary_sha256": manifest.BinarySHA256, "corpus_sha256": manifest.CorpusSHA256} { + if !isLowerHexSHA256(value) { + addReason(name + " must be a lowercase SHA-256 digest") + } + } + if strings.TrimSpace(manifest.SourceCommit) == "" { + addReason("source_commit is required") + } + if len(manifest.Caps) == 0 { + addReason("at least one immutable candidate cap is required") + } + for name, limit := range manifest.Caps { + if strings.TrimSpace(name) == "" || limit <= 0 { + addReason("candidate caps must have nonempty names and positive limits") + } + } + if manifest.Candidate == "ASP-I1-U-DAG+MAT-M0" { + expectedCaps := map[string]struct{}{ + "state_limit": {}, "predecessor_limit": {}, "enumeration_limit": {}, "output_bytes_limit": {}, + } + if manifest.ExecutionBoundary != "guarded_dual_arm" { + addReason("ASP-I1 requires the guarded_dual_arm production boundary") + } + if manifest.FallbackExecutor != "ASP-A1-DAG" { + addReason("ASP-I1 requires ASP-A1-DAG as its exact fallback") + } + if len(manifest.Caps) != len(expectedCaps) { + addReason("ASP-I1 requires exactly state, predecessor, enumeration, and output-byte caps") + } + for name := range expectedCaps { + if manifest.Caps[name] <= 0 { + addReason("ASP-I1 cap " + name + " must be positive") + } + } + } + if manifest.Candidate == "SP-I1-C-WE+MAT-M0" { + expectedCaps := map[string]struct{}{ + "state_limit": {}, "predecessor_limit": {}, "enumeration_limit": {}, "output_bytes_limit": {}, + } + if manifest.ExecutionBoundary != "guarded_dual_arm" { + addReason("SP-I1 canonical witness requires the guarded_dual_arm production boundary") + } + if manifest.FallbackExecutor != "SP-S4-C-WE+MAT-M0" { + addReason("SP-I1 canonical witness requires SP-S4-C-WE+MAT-M0 as its exact fallback") + } + if len(manifest.Caps) != len(expectedCaps) { + addReason("SP-I1 canonical witness requires exactly state, predecessor, enumeration, and output-byte caps") + } + for name := range expectedCaps { + if manifest.Caps[name] <= 0 { + addReason("SP-I1 canonical witness cap " + name + " must be positive") + } + } + if manifest.SelectorVersion != optimize.ShortestPathSelectorStaticV6 { + addReason("SP-I1 canonical witness requires selector " + optimize.ShortestPathSelectorStaticV6) + } + } + if manifest.Candidate == string(optimize.ExpansionSearchPolicyOrientationProbeV1) { + expectedCaps := orientationPromotionCaps() + if manifest.ExecutionBoundary != "guarded_dual_arm" { + addReason("orientation-probe-v1 requires the guarded_dual_arm production boundary") + } + if manifest.FallbackExecutor != string(optimize.ExpansionSearchStepwiseForward) { + addReason("orientation-probe-v1 requires EXPANSION-STEPWISE-FORWARD as its exact fallback") + } + if len(manifest.Caps) != len(expectedCaps) { + addReason("orientation-probe-v1 requires exactly root-row, reverse-seed-row, directional-degree-row, and state caps") + } + for name, expected := range expectedCaps { + if manifest.Caps[name] != expected { + addReason(fmt.Sprintf("orientation-probe-v1 cap %s must equal %d", name, expected)) + } + } + } + if len(manifest.Buckets) == 0 { + addReason("at least one authorized bucket is required") + } + seenBuckets := map[string]struct{}{} + for _, bucket := range manifest.Buckets { + if bucket.Name == "" || len(bucket.QuerySHA256) == 0 { + addReason("every bucket requires a name and query allowlist") + continue + } + if _, found := seenBuckets[bucket.Name]; found { + addReason("bucket " + bucket.Name + " is duplicated") + } + seenBuckets[bucket.Name] = struct{}{} + for _, query := range bucket.QuerySHA256 { + if !isLowerHexSHA256(query) { + addReason("bucket " + bucket.Name + " contains an invalid query digest") + } + } + if !containsString(bucket.QualificationSplit, "training") || !containsString(bucket.QualificationSplit, "holdout") { + addReason("bucket " + bucket.Name + " must bind training and holdout evidence") + } + if manifest.Candidate == "ASP-I1-U-DAG+MAT-M0" { + if (bucket.Direction != "outbound" && bucket.Direction != "inbound") || bucket.ObservationMode != "all_paths" || bucket.MinimumDepth != 1 || bucket.MaximumDepth < 1 || bucket.MaximumDepth > 64 { + addReason("ASP-I1 bucket " + bucket.Name + " is outside the directed all-paths depth envelope") + } + if bucket.RelationshipKindCount < 0 || bucket.UntypedRelationship != (bucket.RelationshipKindCount == 0) { + addReason("ASP-I1 bucket " + bucket.Name + " has inconsistent relationship-kind metadata") + } + } + if manifest.Candidate == "SP-I1-C-WE+MAT-M0" { + if err := validateStaticV6CanonicalInboundBucket(bucket); err != nil { + addReason(err.Error()) + } + } + } + base := filepath.Dir(path) + for _, role := range requiredPromotionEvidenceRoles { + reference, found := manifest.Evidence[role] + if !found { + addReason("required evidence role " + role + " is missing") + continue + } + if err := verifyPromotionEvidence(base, role, reference, promotionEvidenceIdentity(manifest)); err != nil { + addReason(role + ": " + err.Error()) + } + } + sort.Strings(verification.Reasons) + return verification, nil +} + +func writePromotionManifestVerification(path, output string) (bool, error) { + verification, err := verifyPromotionManifest(path) + if err != nil { + return false, err + } + raw, err := json.MarshalIndent(verification, "", " ") + if err != nil { + return false, err + } + if output == "" { + _, err = os.Stdout.Write(append(raw, '\n')) + } else { + err = os.WriteFile(output, append(raw, '\n'), 0o644) + } + return verification.Passed, err +} + +func verifyPromotionEvidence(base, role string, reference PromotionEvidenceReference, expectedIdentity PromotionEvidenceIdentity) error { + if filepath.IsAbs(reference.Path) || reference.Path == "" { + return fmt.Errorf("path must be a nonempty relative path") + } + clean := filepath.Clean(reference.Path) + if clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { + return fmt.Errorf("path escapes the manifest directory") + } + raw, err := os.ReadFile(filepath.Join(base, clean)) + if err != nil { + return err + } + digest := sha256.Sum256(raw) + if hex.EncodeToString(digest[:]) != reference.SHA256 { + return fmt.Errorf("SHA-256 mismatch") + } + var document map[string]any + if err := json.Unmarshal(raw, &document); err != nil { + return fmt.Errorf("decode report: %w", err) + } + identityRaw, found := document["promotion_identity"] + if !found { + return fmt.Errorf("report has no promotion_identity") + } + encodedIdentity, err := json.Marshal(identityRaw) + if err != nil { + return fmt.Errorf("encode promotion identity: %w", err) + } + var actualIdentity PromotionEvidenceIdentity + if err := json.Unmarshal(encodedIdentity, &actualIdentity); err != nil { + return fmt.Errorf("decode promotion identity: %w", err) + } + if !reflect.DeepEqual(actualIdentity, expectedIdentity) { + return fmt.Errorf("promotion identity does not match manifest") + } + switch role { + case "aa": + if balanced, _ := document["order_balanced"].(bool); !balanced { + return fmt.Errorf("A/A report is not order balanced") + } + if cases, _ := document["cases"].([]any); len(cases) == 0 { + return fmt.Errorf("A/A report has no cases") + } + case "confirmation", "performance": + if eligible, _ := document["promotion_eligible"].(bool); !eligible { + return fmt.Errorf("report is not promotion eligible") + } + default: + if passed, _ := document["passed"].(bool); !passed { + return fmt.Errorf("report did not pass") + } + } + return nil +} + +// bindPromotionEvidenceReport attaches the manifest's authorization identity +// to an already generated role-specific report. The final manifest may then +// checksum the bound report without creating an identity/digest cycle. +func bindPromotionEvidenceReport(manifestPath, role, inputPath, outputPath string) error { + if !containsString(requiredPromotionEvidenceRoles, role) { + return fmt.Errorf("unsupported promotion evidence role %q", role) + } + manifestRaw, err := os.ReadFile(manifestPath) + if err != nil { + return err + } + var manifest PromotionManifest + if err := json.Unmarshal(manifestRaw, &manifest); err != nil { + return fmt.Errorf("decode promotion manifest: %w", err) + } + reportRaw, err := os.ReadFile(inputPath) + if err != nil { + return err + } + var report map[string]any + if err := json.Unmarshal(reportRaw, &report); err != nil { + return fmt.Errorf("decode evidence report: %w", err) + } + report["promotion_identity"] = promotionEvidenceIdentity(manifest) + bound, err := json.MarshalIndent(report, "", " ") + if err != nil { + return err + } + return os.WriteFile(outputPath, append(bound, '\n'), 0o644) +} diff --git a/cmd/graphbench/promotion_manifest_test.go b/cmd/graphbench/promotion_manifest_test.go new file mode 100644 index 00000000..417d654e --- /dev/null +++ b/cmd/graphbench/promotion_manifest_test.go @@ -0,0 +1,325 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/stretchr/testify/require" +) + +func writePromotionManifestWithPassingEvidence(t *testing.T, manifest PromotionManifest) string { + t.Helper() + directory := t.TempDir() + manifest.Evidence = map[string]PromotionEvidenceReference{} + for _, role := range requiredPromotionEvidenceRoles { + document := map[string]any{"passed": true, "promotion_identity": promotionEvidenceIdentity(manifest)} + switch role { + case "aa": + document = map[string]any{"order_balanced": true, "cases": []any{map[string]any{"name": "case"}}, "promotion_identity": promotionEvidenceIdentity(manifest)} + case "confirmation", "performance": + document = map[string]any{"promotion_eligible": true, "promotion_identity": promotionEvidenceIdentity(manifest)} + } + raw, err := json.Marshal(document) + require.NoError(t, err) + path := role + ".json" + require.NoError(t, os.WriteFile(filepath.Join(directory, path), raw, 0o600)) + digest := sha256.Sum256(raw) + manifest.Evidence[role] = PromotionEvidenceReference{Path: path, SHA256: hex.EncodeToString(digest[:])} + } + raw, err := json.Marshal(manifest) + require.NoError(t, err) + path := filepath.Join(directory, "promotion.json") + require.NoError(t, os.WriteFile(path, raw, 0o600)) + return path +} + +func TestVerifyPromotionManifestRequiresExactOrientationProbeContract(t *testing.T) { + digest := strings.Repeat("a", 64) + base := PromotionManifest{ + Version: promotionManifestVersion, Candidate: string(optimize.ExpansionSearchPolicyOrientationProbeV1), SelectorVersion: "orientation-probe-v1", + ExecutionBoundary: "guarded_dual_arm", FallbackExecutor: string(optimize.ExpansionSearchStepwiseForward), + SourceCommit: "deadbeef", SourceSHA256: digest, BinarySHA256: digest, CorpusSHA256: digest, + Caps: orientationPromotionCaps(), + Buckets: []PromotionBucket{{ + Name: "fixed-suffix", QuerySHA256: []string{digest}, Direction: "outbound", ObservationMode: "endpoint_ids", + MinimumDepth: 0, MaximumDepth: 16, RelationshipKindCount: 1, QualificationSplit: []string{"training", "holdout"}, + }}, + } + + verification, err := verifyPromotionManifest(writePromotionManifestWithPassingEvidence(t, base)) + require.NoError(t, err) + require.True(t, verification.Passed, verification.Reasons) + + tests := []struct { + name string + mutate func(*PromotionManifest) + reason string + }{ + { + name: "boundary", mutate: func(manifest *PromotionManifest) { manifest.ExecutionBoundary = "inline_statement" }, + reason: "orientation-probe-v1 requires the guarded_dual_arm production boundary", + }, + { + name: "fallback", mutate: func(manifest *PromotionManifest) { manifest.FallbackExecutor = "EXPANSION-SUFFIX-SEEDED-REVERSE" }, + reason: "orientation-probe-v1 requires EXPANSION-STEPWISE-FORWARD as its exact fallback", + }, + { + name: "extra cap", mutate: func(manifest *PromotionManifest) { manifest.Caps["extra_limit"] = 1 }, + reason: "orientation-probe-v1 requires exactly root-row, reverse-seed-row, directional-degree-row, and state caps", + }, + { + name: "missing cap", mutate: func(manifest *PromotionManifest) { delete(manifest.Caps, "root_row_limit") }, + reason: "orientation-probe-v1 requires exactly root-row, reverse-seed-row, directional-degree-row, and state caps", + }, + { + name: "root cap", mutate: func(manifest *PromotionManifest) { manifest.Caps["root_row_limit"]-- }, + reason: "orientation-probe-v1 cap root_row_limit must equal 512", + }, + { + name: "reverse seed cap", mutate: func(manifest *PromotionManifest) { manifest.Caps["reverse_seed_row_limit"]-- }, + reason: "orientation-probe-v1 cap reverse_seed_row_limit must equal 512", + }, + { + name: "directional degree cap", mutate: func(manifest *PromotionManifest) { manifest.Caps["directional_degree_row_limit"]-- }, + reason: "orientation-probe-v1 cap directional_degree_row_limit must equal 16384", + }, + { + name: "state cap", mutate: func(manifest *PromotionManifest) { manifest.Caps["state_limit"]-- }, + reason: "orientation-probe-v1 cap state_limit must equal 4096", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + manifest := base + manifest.Caps = clonePromotionCaps(base.Caps) + test.mutate(&manifest) + verification, err := verifyPromotionManifest(writePromotionManifestWithPassingEvidence(t, manifest)) + require.NoError(t, err) + require.False(t, verification.Passed) + require.Contains(t, verification.Reasons, test.reason) + }) + } +} + +func TestVerifyPromotionManifestRequiresStaticV6CanonicalInboundContract(t *testing.T) { + digest := strings.Repeat("a", 64) + base := PromotionManifest{ + Version: promotionManifestVersion, Candidate: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + SelectorVersion: optimize.ShortestPathSelectorStaticV6, ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: string(optimize.ShortestPathExecutorS4CanonicalWitness), + SourceCommit: "deadbeef", SourceSHA256: digest, BinarySHA256: digest, CorpusSHA256: digest, + Caps: map[string]int64{"state_limit": 100_000, "predecessor_limit": 100_000, "enumeration_limit": 100_000, "output_bytes_limit": 64 << 20}, + Buckets: []PromotionBucket{{ + Name: "canonical-inbound-depth64", QuerySHA256: []string{digest}, Direction: "inbound", ObservationMode: "one_path", + MinimumDepth: 1, MaximumDepth: 64, RelationshipKindCount: 1, QualificationSplit: []string{"training", "holdout"}, + }}, + } + + verification, err := verifyPromotionManifest(writePromotionManifestWithPassingEvidence(t, base)) + require.NoError(t, err) + require.True(t, verification.Passed, verification.Reasons) + + tests := []struct { + name string + mutate func(*PromotionManifest) + reason string + }{ + { + name: "selector", mutate: func(manifest *PromotionManifest) { manifest.SelectorVersion = "sp-static-v5-contained" }, + reason: "SP-I1 canonical witness requires selector sp-static-v6", + }, + { + name: "outbound", mutate: func(manifest *PromotionManifest) { manifest.Buckets[0].Direction = "outbound" }, + reason: "SP-I1 canonical witness bucket canonical-inbound-depth64 must be the qualified inbound typed single-kind one-path depth 1..64 envelope", + }, + { + name: "maximum", mutate: func(manifest *PromotionManifest) { manifest.Buckets[0].MaximumDepth = 63 }, + reason: "SP-I1 canonical witness bucket canonical-inbound-depth64 must be the qualified inbound typed single-kind one-path depth 1..64 envelope", + }, + { + name: "kinds", mutate: func(manifest *PromotionManifest) { manifest.Buckets[0].RelationshipKindCount = 2 }, + reason: "SP-I1 canonical witness bucket canonical-inbound-depth64 must be the qualified inbound typed single-kind one-path depth 1..64 envelope", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + manifest := base + manifest.Caps = clonePromotionCaps(base.Caps) + manifest.Buckets = clonePromotionBuckets(base.Buckets) + test.mutate(&manifest) + verification, err := verifyPromotionManifest(writePromotionManifestWithPassingEvidence(t, manifest)) + require.NoError(t, err) + require.False(t, verification.Passed) + require.Contains(t, verification.Reasons, test.reason) + }) + } +} + +func TestVerifyPromotionManifestRequiresCompleteImmutableEvidenceClosure(t *testing.T) { + directory := t.TempDir() + digest := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + manifest := PromotionManifest{ + Version: promotionManifestVersion, Candidate: "SP-B2-C-MIN-LEVEL-D", SelectorVersion: "sp-static-v5", ExecutionBoundary: "stored_helper", + SourceCommit: "deadbeef", SourceSHA256: digest, BinarySHA256: digest, CorpusSHA256: digest, + Caps: map[string]int64{"visited_nodes": 1000}, + Buckets: []PromotionBucket{{ + Name: "deep-inbound-distance", QuerySHA256: []string{digest}, Direction: "inbound", + ObservationMode: "distance", MinimumDepth: 5, MaximumDepth: 16, + QualificationSplit: []string{"training", "holdout"}, + }}, + } + evidence := map[string]PromotionEvidenceReference{} + for _, role := range requiredPromotionEvidenceRoles { + document := map[string]any{"passed": true, "promotion_identity": promotionEvidenceIdentity(manifest)} + switch role { + case "aa": + document = map[string]any{"order_balanced": true, "cases": []any{map[string]any{"name": "case"}}, "promotion_identity": promotionEvidenceIdentity(manifest)} + case "confirmation", "performance": + document = map[string]any{"promotion_eligible": true, "promotion_identity": promotionEvidenceIdentity(manifest)} + } + raw, err := json.Marshal(document) + require.NoError(t, err) + path := role + ".json" + require.NoError(t, os.WriteFile(filepath.Join(directory, path), raw, 0o600)) + digest := sha256.Sum256(raw) + evidence[role] = PromotionEvidenceReference{Path: path, SHA256: hex.EncodeToString(digest[:])} + } + manifest.Evidence = evidence + raw, err := json.Marshal(manifest) + require.NoError(t, err) + manifestPath := filepath.Join(directory, "promotion.json") + require.NoError(t, os.WriteFile(manifestPath, raw, 0o600)) + + verification, err := verifyPromotionManifest(manifestPath) + require.NoError(t, err) + require.True(t, verification.Passed, verification.Reasons) + require.NotEmpty(t, verification.ManifestSHA256) + + delete(manifest.Evidence, "operational") + raw, err = json.Marshal(manifest) + require.NoError(t, err) + require.NoError(t, os.WriteFile(manifestPath, raw, 0o600)) + verification, err = verifyPromotionManifest(manifestPath) + require.NoError(t, err) + require.False(t, verification.Passed) + require.Contains(t, verification.Reasons, "required evidence role operational is missing") +} + +func TestVerifyPromotionEvidenceRejectsEveryCrossBindingMismatch(t *testing.T) { + directory := t.TempDir() + digest := strings.Repeat("0", 64) + manifest := PromotionManifest{ + Version: promotionManifestVersion, Candidate: "candidate-a", SelectorVersion: "selector", ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: "incumbent", SourceCommit: "commit", SourceSHA256: digest, BinarySHA256: digest, CorpusSHA256: digest, + Caps: map[string]int64{"cap": 1}, Buckets: []PromotionBucket{{ + Name: "bucket", QuerySHA256: []string{digest}, Direction: "outbound", ObservationMode: "one_path", + MinimumDepth: 1, MaximumDepth: 4, RelationshipKindCount: 1, QualificationSplit: []string{"training", "holdout"}, + }}, + } + tests := map[string]func(*PromotionEvidenceIdentity){ + "candidate": func(identity *PromotionEvidenceIdentity) { identity.Candidate = "candidate-b" }, + "selector": func(identity *PromotionEvidenceIdentity) { identity.SelectorVersion = "other-selector" }, + "boundary": func(identity *PromotionEvidenceIdentity) { identity.ExecutionBoundary = "stored_helper" }, + "fallback": func(identity *PromotionEvidenceIdentity) { identity.FallbackExecutor = "other-incumbent" }, + "source commit": func(identity *PromotionEvidenceIdentity) { identity.SourceCommit = "other-commit" }, + "source digest": func(identity *PromotionEvidenceIdentity) { identity.SourceSHA256 = strings.Repeat("1", 64) }, + "binary digest": func(identity *PromotionEvidenceIdentity) { identity.BinarySHA256 = strings.Repeat("2", 64) }, + "corpus digest": func(identity *PromotionEvidenceIdentity) { identity.CorpusSHA256 = strings.Repeat("3", 64) }, + "cap": func(identity *PromotionEvidenceIdentity) { identity.Caps["cap"] = 2 }, + "bucket envelope": func(identity *PromotionEvidenceIdentity) { identity.Buckets[0].MaximumDepth = 8 }, + "query cohort": func(identity *PromotionEvidenceIdentity) { + identity.Buckets[0].QuerySHA256[0] = strings.Repeat("4", 64) + }, + "qualification split": func(identity *PromotionEvidenceIdentity) { + identity.Buckets[0].QualificationSplit = []string{"training"} + }, + } + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + wrong := promotionEvidenceIdentity(manifest) + mutate(&wrong) + document := map[string]any{"passed": true, "promotion_identity": wrong} + raw, err := json.Marshal(document) + require.NoError(t, err) + path := "resource.json" + require.NoError(t, os.WriteFile(filepath.Join(directory, path), raw, 0o600)) + sum := sha256.Sum256(raw) + reference := PromotionEvidenceReference{Path: path, SHA256: hex.EncodeToString(sum[:])} + err = verifyPromotionEvidence(directory, "resource", reference, promotionEvidenceIdentity(manifest)) + require.EqualError(t, err, "promotion identity does not match manifest") + }) + } +} + +func TestBindPromotionEvidenceReportCopiesCompleteManifestIdentity(t *testing.T) { + directory := t.TempDir() + digest := strings.Repeat("a", 64) + manifest := PromotionManifest{ + Version: promotionManifestVersion, Candidate: "candidate", SelectorVersion: "selector", ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: "incumbent", SourceCommit: "commit", SourceSHA256: digest, BinarySHA256: digest, CorpusSHA256: digest, + Caps: map[string]int64{"cap": 7}, Buckets: []PromotionBucket{{Name: "bucket", QuerySHA256: []string{digest}, QualificationSplit: []string{"training", "holdout"}}}, + } + manifestRaw, err := json.Marshal(manifest) + require.NoError(t, err) + manifestPath := filepath.Join(directory, "manifest.json") + inputPath := filepath.Join(directory, "input.json") + outputPath := filepath.Join(directory, "output.json") + require.NoError(t, os.WriteFile(manifestPath, manifestRaw, 0o600)) + require.NoError(t, os.WriteFile(inputPath, []byte(`{"passed":true}`), 0o600)) + require.NoError(t, bindPromotionEvidenceReport(manifestPath, "resource", inputPath, outputPath)) + + boundRaw, err := os.ReadFile(outputPath) + require.NoError(t, err) + var bound struct { + Passed bool `json:"passed"` + PromotionIdentity PromotionEvidenceIdentity `json:"promotion_identity"` + } + require.NoError(t, json.Unmarshal(boundRaw, &bound)) + require.True(t, bound.Passed) + require.Equal(t, promotionEvidenceIdentity(manifest), bound.PromotionIdentity) +} + +func TestVerifyPromotionManifestRejectsVersionOne(t *testing.T) { + directory := t.TempDir() + path := filepath.Join(directory, "manifest.json") + require.NoError(t, os.WriteFile(path, []byte(`{"version":1}`), 0o600)) + verification, err := verifyPromotionManifest(path) + require.NoError(t, err) + require.False(t, verification.Passed) + require.Contains(t, verification.Reasons, "manifest version must be 2") +} + +func TestVerifyPromotionManifestRejectsEscapingOrMutatedEvidence(t *testing.T) { + directory := t.TempDir() + manifestPath := filepath.Join(directory, "promotion.json") + digest := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + manifest := PromotionManifest{ + Version: promotionManifestVersion, Candidate: "candidate", SelectorVersion: "selector", ExecutionBoundary: "inline_statement", SourceCommit: "commit", + SourceSHA256: digest, BinarySHA256: digest, CorpusSHA256: digest, + Caps: map[string]int64{"cap": 1}, + Buckets: []PromotionBucket{{Name: "bucket", QuerySHA256: []string{digest}, QualificationSplit: []string{"training", "holdout"}}}, + Evidence: map[string]PromotionEvidenceReference{}, + } + for _, role := range requiredPromotionEvidenceRoles { + manifest.Evidence[role] = PromotionEvidenceReference{Path: "../outside.json", SHA256: digest} + } + raw, err := json.Marshal(manifest) + require.NoError(t, err) + require.NoError(t, os.WriteFile(manifestPath, raw, 0o600)) + + verification, err := verifyPromotionManifest(manifestPath) + require.NoError(t, err) + require.False(t, verification.Passed) + for _, role := range requiredPromotionEvidenceRoles { + require.Contains(t, verification.Reasons, role+": path escapes the manifest directory") + } +} diff --git a/cmd/graphbench/reference_closure_report.go b/cmd/graphbench/reference_closure_report.go new file mode 100644 index 00000000..edd2289d --- /dev/null +++ b/cmd/graphbench/reference_closure_report.go @@ -0,0 +1,354 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "math" + "os" + "slices" + "sort" + "time" +) + +// referenceClosureReportVersion identifies the serialized schema revision for reference closure report. +const referenceClosureReportVersion = 1 + +// ReferenceClosureOptions selects the reference arm and ratio and absolute limits used for closure analysis. +type ReferenceClosureOptions struct { + // Seed controls deterministic random sampling. + Seed int64 + // Confidence sets the confidence level used for statistical intervals. + Confidence float64 + // BootstrapCount sets the number of bootstrap resamples. + BootstrapCount int + // ReferenceName identifies the reference arm selected for closure analysis. + ReferenceName string + // RatioUpperLimit sets the largest production-to-reference median ratio accepted by closure analysis. + RatioUpperLimit float64 + // AbsoluteResolution records the absolute A/A noise floor used for materiality decisions. + AbsoluteResolution time.Duration +} + +// ReferenceClosureCase reports paired production/reference samples, A/A floors, and closure disposition for one case. +type ReferenceClosureCase struct { + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // ReferenceName identifies the reference arm selected for closure analysis. + ReferenceName string `json:"reference_name"` + // ReferenceArchitecture records the executor architecture declared by the closure reference arm. + ReferenceArchitecture string `json:"reference_architecture"` + // Rounds records the number of independent measurement rounds. + Rounds int `json:"rounds"` + // ProductionSamples records warm timing samples available from production execution. + ProductionSamples int `json:"production_samples"` + // ReferenceSamples records warm timing samples available from the reference arm. + ReferenceSamples int `json:"reference_samples"` + // MedianRatio reports the candidate-to-baseline median latency ratio and confidence bounds. + MedianRatio RatioInterval `json:"median_ratio"` + // MedianChange reports the absolute median latency difference and confidence bounds. + MedianChange DurationInterval `json:"median_change"` + // AbsoluteGapUpper records the upper confidence bound for absolute production/reference latency gap. + AbsoluteGapUpper time.Duration `json:"absolute_gap_upper"` + // RatioUpperLimit sets the largest production-to-reference median ratio accepted by closure analysis. + RatioUpperLimit float64 `json:"ratio_upper_limit"` + // AbsoluteFloor records the A/A-derived absolute materiality floor. + AbsoluteFloor time.Duration `json:"absolute_floor"` + // ProductionAAResolution records production-arm A/A noise used for closure materiality. + ProductionAAResolution time.Duration `json:"production_aa_resolution"` + // ReferenceAAResolution records reference-arm A/A noise used for closure materiality. + ReferenceAAResolution time.Duration `json:"reference_aa_resolution"` + // AbsoluteResolution records the absolute A/A noise floor used for materiality decisions. + AbsoluteResolution time.Duration `json:"absolute_resolution"` + // Passed reports whether every required gate condition succeeded. + Passed bool `json:"passed"` + // Reasons lists explanations for the reported disposition. + Reasons []string `json:"reasons,omitempty"` + // ProductionRuntimeReceiptChains preserves the complete production branch + // chain for every measured invocation used by closure. + ProductionRuntimeReceiptChains [][]RuntimeReceiptEvent `json:"production_runtime_receipt_chains,omitempty"` +} + +// ReferenceClosureReport contains artifact identity, thresholds, and per-case production/reference closure results. +type ReferenceClosureReport struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // Seed controls deterministic random sampling. + Seed int64 `json:"seed"` + // Confidence sets the confidence level used for statistical intervals. + Confidence float64 `json:"confidence_level"` + // ArtifactSHA256 identifies the exact input artifact summarized by the report. + ArtifactSHA256 string `json:"artifact_sha256"` + // ReferenceName identifies the reference arm selected for closure analysis. + ReferenceName string `json:"reference_name"` + // Passed reports whether every required gate condition succeeded. + Passed bool `json:"passed"` + // Cases contains production-to-reference closure evidence for each evaluated workload. + Cases []ReferenceClosureCase `json:"cases"` +} + +// buildReferenceClosureReport compares production and exact-reference samples under the closure protocol. +func buildReferenceClosureReport(records []CaseResult, options ReferenceClosureOptions) (ReferenceClosureReport, error) { + if options.Confidence <= 0 || options.Confidence >= 1 { + return ReferenceClosureReport{}, fmt.Errorf("confidence level must be between 0 and 1") + } + if options.BootstrapCount == 0 { + options.BootstrapCount = defaultBootstrapCount + } + if options.BootstrapCount < 1 { + return ReferenceClosureReport{}, fmt.Errorf("bootstrap count must be positive") + } + if options.ReferenceName == "" { + options.ReferenceName = "s3_unidirectional_trail_cte" + } + if options.RatioUpperLimit == 0 { + options.RatioUpperLimit = 1.10 + } + if options.RatioUpperLimit <= 0 { + return ReferenceClosureReport{}, fmt.Errorf("reference ratio upper limit must be positive") + } + if options.AbsoluteResolution == 0 { + options.AbsoluteResolution = 100 * time.Microsecond + } + if options.AbsoluteResolution < 0 { + return ReferenceClosureReport{}, fmt.Errorf("reference absolute resolution must not be negative") + } + + // closureSeries groups production and reference samples with the architecture fixed across rounds. + type closureSeries struct { + // production groups production duration samples by measurement round. + production roundSamples + // reference groups reference-arm duration samples by measurement round. + reference roundSamples + // architecture retains the executor architecture that must remain stable across rounds. + architecture string + } + series := map[performanceKey]*closureSeries{} + seenRounds := map[performanceKey]map[int]struct{}{} + for _, record := range records { + if record.ExecutionMode != ModePostgresSQL { + continue + } + if record.Status != StatusOK { + return ReferenceClosureReport{}, fmt.Errorf("%s/%s has non-ok status %s", record.Dataset, record.Name, record.Status) + } + if record.Environment == nil { + return ReferenceClosureReport{}, fmt.Errorf("%s/%s has no run environment", record.Dataset, record.Name) + } + if record.Environment.WarmupIterations < 20 { + return ReferenceClosureReport{}, fmt.Errorf("%s/%s round %d requires at least 20 warmups, got %d", record.Dataset, record.Name, record.Environment.Round, record.Environment.WarmupIterations) + } + if record.RawPGXWaterfall == nil || record.RawPGXWaterfall.WarmupIterations < 20 { + return ReferenceClosureReport{}, fmt.Errorf("%s/%s round %d lacks a 20-warmup production raw-pgx boundary", record.Dataset, record.Name, record.Environment.Round) + } + var reference *PostgresReferenceResult + for idx := range record.PostgresReferences { + if record.PostgresReferences[idx].Name == options.ReferenceName { + reference = &record.PostgresReferences[idx] + break + } + } + if reference == nil { + return ReferenceClosureReport{}, fmt.Errorf("%s/%s round %d is missing reference %s", record.Dataset, record.Name, record.Environment.Round, options.ReferenceName) + } + if !reference.FullComparator || reference.SemanticValidation != "exact_public_observation" { + return ReferenceClosureReport{}, fmt.Errorf("%s/%s reference %s is not an exact full comparator", record.Dataset, record.Name, options.ReferenceName) + } + if reference.RowCount != record.RowCount || !slices.Equal(reference.ObservedRows, record.ObservedRows) { + return ReferenceClosureReport{}, fmt.Errorf("%s/%s reference observation differs from production", record.Dataset, record.Name) + } + if reference.Stats.WarmupIterations < 20 { + return ReferenceClosureReport{}, fmt.Errorf("%s/%s round %d reference requires at least 20 warmups, got %d", record.Dataset, record.Name, record.Environment.Round, reference.Stats.WarmupIterations) + } + expectedProductionOrder, expectedReferenceOrder := referenceClosureMeasurementOrder(true, record.Environment.Round) + if record.RawPGXWaterfall.MeasurementOrder != expectedProductionOrder || reference.MeasurementOrder != expectedReferenceOrder { + return ReferenceClosureReport{}, fmt.Errorf("%s/%s round %d lacks carryover-balanced production/reference order: got %d/%d, expected %d/%d", record.Dataset, record.Name, record.Environment.Round, record.RawPGXWaterfall.MeasurementOrder, reference.MeasurementOrder, expectedProductionOrder, expectedReferenceOrder) + } + + key := performanceKey{ + dataset: record.Dataset, + name: record.Name, + backend: ModePostgresSQL, + } + if seenRounds[key] == nil { + seenRounds[key] = map[int]struct{}{} + } + if _, duplicate := seenRounds[key][record.Environment.Round]; duplicate { + return ReferenceClosureReport{}, fmt.Errorf("%s/%s has duplicate round %d", record.Dataset, record.Name, record.Environment.Round) + } + seenRounds[key][record.Environment.Round] = struct{}{} + + if series[key] == nil { + series[key] = &closureSeries{ + production: roundSamples{}, + reference: roundSamples{}, + architecture: reference.Architecture, + } + } else if series[key].architecture != reference.Architecture { + return ReferenceClosureReport{}, fmt.Errorf("%s/%s reference architecture changed across rounds", record.Dataset, record.Name) + } + + for _, sample := range record.RawPGXWaterfall.Samples { + if sample.Total > 0 { + series[key].production[record.Environment.Round] = append(series[key].production[record.Environment.Round], sample.Total) + } + } + + for _, sample := range reference.Stats.Samples { + if sample.Classification == "warm" && sample.Duration > 0 { + series[key].reference[record.Environment.Round] = append(series[key].reference[record.Environment.Round], sample.Duration) + } + } + } + + if len(series) == 0 { + return ReferenceClosureReport{}, fmt.Errorf("artifact has no successful PostgreSQL production/reference records") + } + + keys := make([]performanceKey, 0, len(series)) + for key := range series { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { + if keys[i].dataset != keys[j].dataset { + return keys[i].dataset < keys[j].dataset + } + return keys[i].name < keys[j].name + }) + report := ReferenceClosureReport{ + Version: referenceClosureReportVersion, + Seed: options.Seed, + Confidence: options.Confidence, + ReferenceName: options.ReferenceName, + Passed: true, + } + gateOptions := PerfGateOptions{ + Seed: options.Seed, + Confidence: options.Confidence, + BootstrapCount: options.BootstrapCount, + } + for idx, key := range keys { + candidate, baseline := matchedRounds(series[key].production, series[key].reference) + entry := ReferenceClosureCase{ + Dataset: key.dataset, + Name: key.name, + ReferenceName: options.ReferenceName, + ReferenceArchitecture: series[key].architecture, + Rounds: len(candidate), + ProductionSamples: sampleCount(candidate), + ReferenceSamples: sampleCount(baseline), + RatioUpperLimit: options.RatioUpperLimit, + AbsoluteFloor: options.AbsoluteResolution, + Passed: true, + ProductionRuntimeReceiptChains: caseRuntimeReceiptChains(records, key), + } + if entry.Rounds < 10 || entry.Rounds > 20 { + entry.Passed = false + entry.Reasons = append(entry.Reasons, fmt.Sprintf("requires 10-20 matched rounds, got %d", entry.Rounds)) + } + for _, round := range sortedRounds(candidate) { + if len(candidate[round]) < 50 || len(baseline[round]) < 50 { + entry.Passed = false + entry.Reasons = append(entry.Reasons, fmt.Sprintf("round %d requires at least 50 samples per side, got %d/%d", round, len(candidate[round]), len(baseline[round]))) + } + } + if entry.Rounds > 0 { + seed := options.Seed + int64(idx)*7919 + entry.ProductionAAResolution = withinSessionAAResolution(candidate, seed+2, gateOptions) + entry.ReferenceAAResolution = withinSessionAAResolution(baseline, seed+3, gateOptions) + entry.AbsoluteResolution = max(options.AbsoluteResolution, entry.ProductionAAResolution, entry.ReferenceAAResolution) + entry.MedianRatio = bootstrapRoundMedianRatio(baseline, candidate, seed, gateOptions) + entry.MedianChange = negateDurationInterval(bootstrapRoundMedianSaving(baseline, candidate, seed+1, gateOptions)) + entry.AbsoluteGapUpper = max(absDuration(entry.MedianChange.Lower), absDuration(entry.MedianChange.Upper)) + if entry.MedianRatio.Upper > options.RatioUpperLimit && entry.AbsoluteGapUpper > entry.AbsoluteResolution { + entry.Passed = false + entry.Reasons = append(entry.Reasons, fmt.Sprintf("ratio upper %.4f exceeds %.4f and absolute gap upper %s exceeds effective resolution %s", entry.MedianRatio.Upper, options.RatioUpperLimit, entry.AbsoluteGapUpper, entry.AbsoluteResolution)) + } + } + if !entry.Passed { + report.Passed = false + } + report.Cases = append(report.Cases, entry) + } + return report, nil +} + +// withinSessionAAResolution returns the larger within-session A/A noise estimate for a case. +func withinSessionAAResolution(samples roundSamples, seed int64, options PerfGateOptions) time.Duration { + armA, armB := splitInterleavedDiagnosticSeries(samples) + armA, armB = matchedRounds(armA, armB) + if len(armA) == 0 { + return 0 + } + interval := bootstrapRoundMedianSaving(armA, armB, seed, options) + return max(absDuration(interval.Lower), absDuration(interval.Upper)) +} + +// splitInterleavedDiagnosticSeries estimates within-session resolution for the +// descriptive reference-closure report only. Promotion-grade host A/A evidence +// is built exclusively from explicit arms by collectExplicitAASeries. +func splitInterleavedDiagnosticSeries(samples roundSamples) (roundSamples, roundSamples) { + armA, armB := roundSamples{}, roundSamples{} + for round, values := range samples { + for idx, value := range values { + if idx%2 == 0 { + armA[round] = append(armA[round], value) + } else { + armB[round] = append(armB[round], value) + } + } + } + return armA, armB +} + +// absDuration returns the magnitude of a signed duration. +func absDuration(value time.Duration) time.Duration { + return time.Duration(math.Abs(float64(value))) +} + +// createReferenceClosureReport loads benchmark records, builds a closure report, and writes it as JSON. +func createReferenceClosureReport(artifactPath, outputPath string, options ReferenceClosureOptions) (bool, error) { + records, err := readJSONLFile(artifactPath) + if err != nil { + return false, err + } + report, err := buildReferenceClosureReport(records, options) + if err != nil { + return false, err + } + report.ArtifactSHA256, err = fileSHA256(artifactPath) + if err != nil { + return false, err + } + return report.Passed, writeReferenceClosureReport(outputPath, report) +} + +// writeReferenceClosureReport writes a reference-closure report as indented JSON. +func writeReferenceClosureReport(path string, report ReferenceClosureReport) (err error) { + var output *os.File + if path == "" { + output = os.Stdout + } else { + if err := ensureOutputDir(path); err != nil { + return err + } + output, err = os.Create(path) + if err != nil { + return err + } + defer func() { + if closeErr := output.Close(); err == nil && closeErr != nil { + err = closeErr + } + }() + } + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(report) +} diff --git a/cmd/graphbench/reference_closure_report_test.go b/cmd/graphbench/reference_closure_report_test.go new file mode 100644 index 00000000..3c1ef03e --- /dev/null +++ b/cmd/graphbench/reference_closure_report_test.go @@ -0,0 +1,152 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestBuildReferenceClosureReportPassesRatioOrResolution verifies that a small absolute gap within measurement resolution passes even when production is five percent slower. +func TestBuildReferenceClosureReportPassesRatioOrResolution(t *testing.T) { + records := referenceClosureRecords(10, 50, time.Millisecond, 1050*time.Microsecond) + report, err := buildReferenceClosureReport(records, ReferenceClosureOptions{ + Seed: 7, + Confidence: 0.975, + BootstrapCount: 250, + }) + + require.NoError(t, err) + require.True(t, report.Passed) + require.Len(t, report.Cases, 1) + entry := report.Cases[0] + require.Equal(t, 10, entry.Rounds) + require.Equal(t, 500, entry.ProductionSamples) + require.Equal(t, 500, entry.ReferenceSamples) + require.InDelta(t, 1.05, entry.MedianRatio.Estimate, 0.0001) + require.LessOrEqual(t, entry.AbsoluteGapUpper, 100*time.Microsecond) + require.Equal(t, 100*time.Microsecond, entry.AbsoluteFloor) + require.Equal(t, 100*time.Microsecond, entry.AbsoluteResolution) +} + +// TestBuildReferenceClosureReportUsesCaseAAResolution verifies that observed production-side A/A noise raises the per-case absolute resolution above the default floor. +func TestBuildReferenceClosureReportUsesCaseAAResolution(t *testing.T) { + records := referenceClosureRecords(10, 50, 2*time.Millisecond, 1500*time.Microsecond) + for idx := range records { + for sampleIdx := range records[idx].RawPGXWaterfall.Samples { + if sampleIdx%2 == 1 { + records[idx].RawPGXWaterfall.Samples[sampleIdx].Total = 2700 * time.Microsecond + } + } + } + report, err := buildReferenceClosureReport(records, ReferenceClosureOptions{ + Seed: 1, + Confidence: 0.975, + BootstrapCount: 100, + }) + + require.NoError(t, err) + require.True(t, report.Passed) + require.Greater(t, report.Cases[0].ProductionAAResolution, 100*time.Microsecond) + require.Equal(t, report.Cases[0].ProductionAAResolution, report.Cases[0].AbsoluteResolution) +} + +// TestBuildReferenceClosureReportFailsMaterialGap verifies that a confidence interval exceeding both ratio and absolute-resolution allowances fails closure. +func TestBuildReferenceClosureReportFailsMaterialGap(t *testing.T) { + records := referenceClosureRecords(10, 50, time.Millisecond, 1500*time.Microsecond) + report, err := buildReferenceClosureReport(records, ReferenceClosureOptions{ + Seed: 1, + Confidence: 0.975, + BootstrapCount: 100, + }) + + require.NoError(t, err) + require.False(t, report.Passed) + require.ErrorContains(t, reasonsError(report.Cases[0].Reasons), "ratio upper") +} + +// TestBuildReferenceClosureReportEnforcesProtocolAndExactComparator verifies minimum rounds/samples, exact public observations, and carryover-balanced measurement order. +func TestBuildReferenceClosureReportEnforcesProtocolAndExactComparator(t *testing.T) { + records := referenceClosureRecords(9, 49, time.Millisecond, time.Millisecond) + report, err := buildReferenceClosureReport(records, ReferenceClosureOptions{ + Seed: 1, + Confidence: 0.975, + BootstrapCount: 100, + }) + require.NoError(t, err) + require.False(t, report.Passed) + require.ErrorContains(t, reasonsError(report.Cases[0].Reasons), "10-20 matched rounds") + require.ErrorContains(t, reasonsError(report.Cases[0].Reasons), "at least 50 samples") + + records = referenceClosureRecords(10, 50, time.Millisecond, time.Millisecond) + records[0].PostgresReferences[0].ObservedRows = []string{"[2]"} + _, err = buildReferenceClosureReport(records, ReferenceClosureOptions{ + Seed: 1, + Confidence: 0.975, + }) + require.ErrorContains(t, err, "observation differs") + + records = referenceClosureRecords(10, 50, time.Millisecond, time.Millisecond) + records[1].PostgresReferences[0].MeasurementOrder = 2 + _, err = buildReferenceClosureReport(records, ReferenceClosureOptions{ + Seed: 1, + Confidence: 0.975, + }) + require.ErrorContains(t, err, "lacks carryover-balanced") +} + +// referenceClosureRecords returns carryover-balanced production/reference rounds with exact observations and uniform warm timings. +func referenceClosureRecords(rounds, samples int, referenceDuration, productionDuration time.Duration) []CaseResult { + records := make([]CaseResult, 0, rounds) + for round := 1; round <= rounds; round++ { + productionOrder, referenceOrder := referenceClosureMeasurementOrder(true, round) + record := CaseResult{ + Dataset: "fixture", + Name: "distance", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + RowCount: 1, + ObservedRows: []string{"[1]"}, + Environment: &RunEnvironment{ + Round: round, + WarmupIterations: 20, + }, + RawPGXWaterfall: &PostgresBoundaryWaterfall{ + WarmupIterations: 20, + MeasurementOrder: productionOrder, + }, + PostgresReferences: []PostgresReferenceResult{{ + Name: "s3_unidirectional_trail_cte", + Architecture: "SP-S3-U-D", + FullComparator: true, + SemanticValidation: "exact_public_observation", + MeasurementOrder: referenceOrder, + RowCount: 1, + ObservedRows: []string{"[1]"}, + Stats: DurationStats{ + WarmupIterations: 20, + }, + }}, + } + for iteration := 1; iteration <= samples; iteration++ { + record.RawPGXWaterfall.Samples = append(record.RawPGXWaterfall.Samples, BoundarySample{ + Iteration: iteration, + Total: productionDuration, + Rows: 1, + }) + record.PostgresReferences[0].Stats.Samples = append(record.PostgresReferences[0].Stats.Samples, LatencySample{ + Round: round, + Iteration: iteration, + Classification: "warm", + Duration: referenceDuration, + }) + } + records = append(records, record) + } + return records +} diff --git a/cmd/graphbench/reference_pair_report.go b/cmd/graphbench/reference_pair_report.go new file mode 100644 index 00000000..2b0a7bf0 --- /dev/null +++ b/cmd/graphbench/reference_pair_report.go @@ -0,0 +1,356 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "os" + "slices" + "sort" + "time" +) + +// referencePairReportVersion identifies the serialized schema revision for reference pair report. +const referencePairReportVersion = 2 + +const ( + // referencePairProtocolConfirmation requires 20 warmups, 10 to 20 rounds, and 50 samples per arm and round. + referencePairProtocolConfirmation = "confirmation" + + // referencePairProtocolDiscovery permits exploratory comparison with five warmups, five rounds, and ten samples per arm and round. + referencePairProtocolDiscovery = "discovery" +) + +// ReferencePairOptions selects two reference arms and the statistical protocol used for their paired comparison. +type ReferencePairOptions struct { + // Seed controls deterministic random sampling. + Seed int64 + // Confidence sets the confidence level used for statistical intervals. + Confidence float64 + // BootstrapCount sets the number of bootstrap resamples. + BootstrapCount int + // BaselineName identifies the reference arm treated as the comparison baseline. + BaselineName string + // CandidateName identifies the reference arm evaluated against the baseline. + CandidateName string + // Protocol identifies the measurement protocol. + Protocol string +} + +// ReferencePairCase reports identity, sample, ratio, and absolute-change evidence for one reference-arm pair. +type ReferencePairCase struct { + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // Rounds records the number of independent measurement rounds. + Rounds int `json:"rounds"` + // BaselineArchitecture records the executor architecture declared by the baseline arm. + BaselineArchitecture string `json:"baseline_architecture"` + // CandidateArchitecture records the executor architecture declared by the candidate arm. + CandidateArchitecture string `json:"candidate_architecture"` + // BaselineBoundary records the portion of baseline execution included in its latency samples. + BaselineBoundary string `json:"baseline_boundary"` + // CandidateBoundary records the portion of candidate execution included in its latency samples. + CandidateBoundary string `json:"candidate_boundary"` + // BaselineSemanticValidation identifies the observation contract enforced for the baseline arm. + BaselineSemanticValidation string `json:"baseline_semantic_validation"` + // CandidateSemanticValidation identifies the observation contract enforced for the candidate arm. + CandidateSemanticValidation string `json:"candidate_semantic_validation"` + // BaselineSamples records warm timing samples available from the baseline arm. + BaselineSamples int `json:"baseline_samples"` + // CandidateSamples records warm timing samples available from the candidate arm. + CandidateSamples int `json:"candidate_samples"` + // MedianRatio reports the candidate-to-baseline median latency ratio and confidence bounds. + MedianRatio RatioInterval `json:"median_ratio"` + // P95Ratio reports the candidate-to-baseline P95 latency ratio and confidence bounds. + P95Ratio RatioInterval `json:"p95_ratio"` + // MedianChange reports the absolute median latency difference and confidence bounds. + MedianChange DurationInterval `json:"median_change"` + // BaselineAAResolution records the baseline arm's A/A-derived absolute noise floor. + BaselineAAResolution time.Duration `json:"baseline_aa_resolution"` + // CandidateAAResolution records the candidate arm's A/A-derived absolute noise floor. + CandidateAAResolution time.Duration `json:"candidate_aa_resolution"` +} + +// ReferencePairReport contains the input identity, protocol thresholds, and results of paired reference analysis. +type ReferencePairReport struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // Seed controls deterministic random sampling. + Seed int64 `json:"seed"` + // Confidence sets the confidence level used for statistical intervals. + Confidence float64 `json:"confidence_level"` + // ArtifactSHA256 identifies the exact input artifact summarized by the report. + ArtifactSHA256 string `json:"artifact_sha256"` + // BaselineName identifies the reference arm treated as the comparison baseline. + BaselineName string `json:"baseline_name"` + // CandidateName identifies the reference arm evaluated against the baseline. + CandidateName string `json:"candidate_name"` + // Protocol identifies the measurement protocol. + Protocol string `json:"protocol"` + // MinimumWarmups records the minimum untimed iterations required for each compared arm. + MinimumWarmups int `json:"minimum_warmups"` + // MinimumRounds records the minimum independent rounds required for comparison. + MinimumRounds int `json:"minimum_rounds"` + // MaximumRounds records the maximum rounds accepted by the selected protocol. + MaximumRounds int `json:"maximum_rounds"` + // MinimumSamples records the minimum warm samples required from each arm and round. + MinimumSamples int `json:"minimum_samples_per_round"` + // Cases contains paired statistical evidence for each workload present in the selected reference arms. + Cases []ReferencePairCase `json:"cases"` +} + +// buildReferencePairReport validates two reference arms and computes paired ratio and duration intervals by case. +func buildReferencePairReport(records []CaseResult, options ReferencePairOptions) (ReferencePairReport, error) { + if options.Confidence <= 0 || options.Confidence >= 1 { + return ReferencePairReport{}, fmt.Errorf("confidence level must be between 0 and 1") + } + if options.BootstrapCount == 0 { + options.BootstrapCount = defaultBootstrapCount + } + if options.BootstrapCount < 1 || options.BaselineName == "" || options.CandidateName == "" || options.BaselineName == options.CandidateName { + return ReferencePairReport{}, fmt.Errorf("valid distinct baseline and candidate reference arms are required") + } + protocol := options.Protocol + if protocol == "" { + protocol = referencePairProtocolConfirmation + } + minimumWarmups, minimumRounds, maximumRounds, minimumSamples := 20, 10, 20, 50 + if protocol == referencePairProtocolDiscovery { + minimumWarmups, minimumRounds, maximumRounds, minimumSamples = 5, 5, 20, 10 + } else if protocol != referencePairProtocolConfirmation { + return ReferencePairReport{}, fmt.Errorf("unsupported reference-pair protocol %q", protocol) + } + // pairSeries groups the two reference arms and the identities that must remain stable across rounds. + type pairSeries struct { + // baseline groups duration samples from the designated baseline arm by round. + baseline roundSamples + // candidate groups duration samples from the designated candidate arm by round. + candidate roundSamples + // baselineArchitecture identifies the execution architecture reported by the baseline arm. + baselineArchitecture string + // candidateArchitecture identifies the execution architecture reported by the candidate arm. + candidateArchitecture string + // baselineBoundary identifies the measurement boundary reported by the baseline arm. + baselineBoundary string + // candidateBoundary identifies the measurement boundary reported by the candidate arm. + candidateBoundary string + // baselineValidation retains the baseline observation contract that must remain stable across rounds. + baselineValidation string + // candidateValidation retains the candidate observation contract that must remain stable across rounds. + candidateValidation string + // baselineImplementation identifies the baseline reference implementation. + baselineImplementation string + // candidateImplementation identifies the candidate reference implementation. + candidateImplementation string + // baselineSQLFingerprint identifies the normalized SQL executed by the baseline arm. + baselineSQLFingerprint string + // candidateSQLFingerprint identifies the normalized SQL executed by the candidate arm. + candidateSQLFingerprint string + // binaryIdentity binds all paired rounds to the same executable and source state. + binaryIdentity string + // baselineFirst records by round whether the baseline arm executed before the candidate. + baselineFirst map[int]bool + } + series := map[performanceKey]*pairSeries{} + seen := map[performanceKey]map[int]struct{}{} + for _, record := range records { + if record.ExecutionMode != ModePostgresSQL { + continue + } + if record.Status != StatusOK || record.Environment == nil || record.Environment.WarmupIterations < minimumWarmups { + return ReferencePairReport{}, fmt.Errorf("%s/%s lacks a successful %d-warmup PostgreSQL record", record.Dataset, record.Name, minimumWarmups) + } + baseline := findReference(record.PostgresReferences, options.BaselineName) + candidate := findReference(record.PostgresReferences, options.CandidateName) + if baseline == nil || candidate == nil { + return ReferencePairReport{}, fmt.Errorf("%s/%s round %d lacks reference pair %s/%s", record.Dataset, record.Name, record.Environment.Round, options.BaselineName, options.CandidateName) + } + fullComparators := baseline.FullComparator && candidate.FullComparator && baseline.SemanticValidation == "exact_public_observation" && candidate.SemanticValidation == "exact_public_observation" + hydrationComparators := !baseline.FullComparator && !candidate.FullComparator && baseline.SemanticValidation == "precomputed_exact_path_inputs" && candidate.SemanticValidation == "precomputed_exact_path_inputs" + orderedComparators := !baseline.FullComparator && !candidate.FullComparator && baseline.ObservationShape == "ordered_ids" && candidate.ObservationShape == "ordered_ids" && baseline.SemanticValidation == "exact_ordered_ids" && candidate.SemanticValidation == "exact_ordered_ids" + if !fullComparators && !hydrationComparators && !orderedComparators { + return ReferencePairReport{}, fmt.Errorf("%s/%s reference pair does not share an exact comparable boundary", record.Dataset, record.Name) + } + if (fullComparators || hydrationComparators) && (baseline.RowCount != record.RowCount || candidate.RowCount != record.RowCount || !slices.Equal(baseline.ObservedRows, record.ObservedRows) || !slices.Equal(candidate.ObservedRows, record.ObservedRows)) { + return ReferencePairReport{}, fmt.Errorf("%s/%s reference-pair observation differs from production", record.Dataset, record.Name) + } + if orderedComparators && (baseline.RowCount != candidate.RowCount || !slices.Equal(baseline.ObservedRows, candidate.ObservedRows)) { + return ReferencePairReport{}, fmt.Errorf("%s/%s ordered-ID reference-pair observations differ", record.Dataset, record.Name) + } + if baseline.ImplementationID == "" || candidate.ImplementationID == "" || baseline.SQLFingerprint == "" || candidate.SQLFingerprint == "" || record.Environment.BinarySHA256 == "" { + return ReferencePairReport{}, fmt.Errorf("%s/%s round %d lacks complete reference-pair implementation identity", record.Dataset, record.Name, record.Environment.Round) + } + if baseline.Stats.WarmupIterations < minimumWarmups || candidate.Stats.WarmupIterations < minimumWarmups || baseline.MeasurementOrder <= 0 || candidate.MeasurementOrder <= 0 || baseline.MeasurementOrder == candidate.MeasurementOrder { + return ReferencePairReport{}, fmt.Errorf("%s/%s round %d lacks warm, ordered reference-pair measurements", record.Dataset, record.Name, record.Environment.Round) + } + binaryIdentity := fmt.Sprintf("%s\x00%s\x00%s\x00%s\x00%s", record.Environment.BinarySHA256, record.Environment.DirtyDiffSHA256, record.Environment.SourceCommit, record.Environment.GOOS, record.Environment.GOARCH) + key := performanceKey{ + dataset: record.Dataset, + name: record.Name, + backend: ModePostgresSQL, + } + if seen[key] == nil { + seen[key] = map[int]struct{}{} + } + if _, duplicate := seen[key][record.Environment.Round]; duplicate { + return ReferencePairReport{}, fmt.Errorf("%s/%s has duplicate round %d", record.Dataset, record.Name, record.Environment.Round) + } + seen[key][record.Environment.Round] = struct{}{} + + if series[key] == nil { + series[key] = &pairSeries{ + baseline: roundSamples{}, + candidate: roundSamples{}, + baselineArchitecture: baseline.Architecture, + candidateArchitecture: candidate.Architecture, + baselineBoundary: baseline.Boundary, + candidateBoundary: candidate.Boundary, + baselineValidation: baseline.SemanticValidation, + candidateValidation: candidate.SemanticValidation, + baselineImplementation: baseline.ImplementationID, + candidateImplementation: candidate.ImplementationID, + baselineSQLFingerprint: baseline.SQLFingerprint, + candidateSQLFingerprint: candidate.SQLFingerprint, + binaryIdentity: binaryIdentity, + baselineFirst: map[int]bool{}, + } + } else if series[key].baselineArchitecture != baseline.Architecture || series[key].candidateArchitecture != candidate.Architecture || + series[key].baselineBoundary != baseline.Boundary || series[key].candidateBoundary != candidate.Boundary || + series[key].baselineValidation != baseline.SemanticValidation || series[key].candidateValidation != candidate.SemanticValidation || + series[key].baselineImplementation != baseline.ImplementationID || series[key].candidateImplementation != candidate.ImplementationID || + series[key].baselineSQLFingerprint != baseline.SQLFingerprint || series[key].candidateSQLFingerprint != candidate.SQLFingerprint || + series[key].binaryIdentity != binaryIdentity { + return ReferencePairReport{}, fmt.Errorf("%s/%s reference-pair identity changed across rounds", record.Dataset, record.Name) + } + + series[key].baselineFirst[record.Environment.Round] = baseline.MeasurementOrder < candidate.MeasurementOrder + for _, sample := range baseline.Stats.Samples { + if sample.Classification == "warm" && sample.Duration > 0 { + series[key].baseline[record.Environment.Round] = append(series[key].baseline[record.Environment.Round], sample.Duration) + } + } + for _, sample := range candidate.Stats.Samples { + if sample.Classification == "warm" && sample.Duration > 0 { + series[key].candidate[record.Environment.Round] = append(series[key].candidate[record.Environment.Round], sample.Duration) + } + } + } + if len(series) == 0 { + return ReferencePairReport{}, fmt.Errorf("artifact has no PostgreSQL reference-pair records") + } + keys := make([]performanceKey, 0, len(series)) + for key := range series { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { + return keys[i].dataset < keys[j].dataset || keys[i].dataset == keys[j].dataset && keys[i].name < keys[j].name + }) + report := ReferencePairReport{ + Version: referencePairReportVersion, + Seed: options.Seed, + Confidence: options.Confidence, + BaselineName: options.BaselineName, + CandidateName: options.CandidateName, + Protocol: protocol, + MinimumWarmups: minimumWarmups, + MinimumRounds: minimumRounds, + MaximumRounds: maximumRounds, + MinimumSamples: minimumSamples, + } + gateOptions := PerfGateOptions{ + Seed: options.Seed, + Confidence: options.Confidence, + BootstrapCount: options.BootstrapCount, + } + for idx, key := range keys { + baseline, candidate := matchedRounds(series[key].baseline, series[key].candidate) + if len(baseline) < minimumRounds || len(baseline) > maximumRounds { + return ReferencePairReport{}, fmt.Errorf("%s/%s requires %d-%d matched rounds, got %d", key.dataset, key.name, minimumRounds, maximumRounds, len(baseline)) + } + rounds := sortedRounds(baseline) + baselineFirstCount := 0 + for roundIdx, round := range rounds { + baselineFirst := series[key].baselineFirst[round] + if baselineFirst { + baselineFirstCount++ + } + if roundIdx > 0 && series[key].baselineFirst[rounds[roundIdx-1]] == baselineFirst { + return ReferencePairReport{}, fmt.Errorf("%s/%s reference-pair arm order does not alternate across rounds", key.dataset, key.name) + } + } + candidateFirstCount := len(rounds) - baselineFirstCount + if baselineFirstCount-candidateFirstCount > 1 || candidateFirstCount-baselineFirstCount > 1 { + return ReferencePairReport{}, fmt.Errorf("%s/%s reference-pair arm order is not balanced", key.dataset, key.name) + } + for _, round := range rounds { + if len(baseline[round]) < minimumSamples || len(candidate[round]) < minimumSamples { + return ReferencePairReport{}, fmt.Errorf("%s/%s round %d requires %d samples per arm", key.dataset, key.name, round, minimumSamples) + } + } + seed := options.Seed + int64(idx)*7919 + report.Cases = append(report.Cases, ReferencePairCase{ + Dataset: key.dataset, + Name: key.name, + Rounds: len(baseline), + BaselineArchitecture: series[key].baselineArchitecture, + CandidateArchitecture: series[key].candidateArchitecture, + BaselineBoundary: series[key].baselineBoundary, + CandidateBoundary: series[key].candidateBoundary, + BaselineSemanticValidation: series[key].baselineValidation, + CandidateSemanticValidation: series[key].candidateValidation, + BaselineSamples: sampleCount(baseline), + CandidateSamples: sampleCount(candidate), + MedianRatio: bootstrapRoundMedianRatio(baseline, candidate, seed, gateOptions), + P95Ratio: bootstrapStratifiedP95Ratio(baseline, candidate, seed+4, gateOptions), + MedianChange: negateDurationInterval(bootstrapRoundMedianSaving(baseline, candidate, seed+1, gateOptions)), + BaselineAAResolution: withinSessionAAResolution(baseline, seed+2, gateOptions), + CandidateAAResolution: withinSessionAAResolution(candidate, seed+3, gateOptions), + }) + } + return report, nil +} + +// findReference returns the named PostgreSQL reference result or nil when it is absent. +func findReference(references []PostgresReferenceResult, name string) *PostgresReferenceResult { + for idx := range references { + if references[idx].Name == name { + return &references[idx] + } + } + return nil +} + +// createReferencePairReport loads benchmark records, builds a reference-pair report, and writes it as JSON. +func createReferencePairReport(artifactPath, outputPath string, options ReferencePairOptions) error { + records, err := readJSONLFile(artifactPath) + if err != nil { + return err + } + report, err := buildReferencePairReport(records, options) + if err != nil { + return err + } + report.ArtifactSHA256, err = fileSHA256(artifactPath) + if err != nil { + return err + } + encoded, err := json.MarshalIndent(report, "", " ") + if err != nil { + return err + } + encoded = append(encoded, '\n') + if outputPath == "" { + _, err = os.Stdout.Write(encoded) + return err + } + if err := ensureOutputDir(outputPath); err != nil { + return err + } + return os.WriteFile(outputPath, encoded, 0o644) +} diff --git a/cmd/graphbench/reference_pair_report_test.go b/cmd/graphbench/reference_pair_report_test.go new file mode 100644 index 00000000..c6934ec6 --- /dev/null +++ b/cmd/graphbench/reference_pair_report_test.go @@ -0,0 +1,416 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestBuildReferencePairReportComparesExactMatchedArms verifies median/P95 ratios and absolute change across ten carryover-balanced full-comparator rounds. +func TestBuildReferencePairReportComparesExactMatchedArms(t *testing.T) { + records := make([]CaseResult, 0, 10) + for round := 1; round <= 10; round++ { + baselineOrder, candidateOrder := 2, 3 + if round%2 == 0 { + baselineOrder, candidateOrder = 3, 2 + } + record := CaseResult{ + Dataset: "fixture", + Name: "distance", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + RowCount: 1, + ObservedRows: []string{"[2]"}, + Environment: &RunEnvironment{ + Round: round, + WarmupIterations: 20, + }, + PostgresReferences: []PostgresReferenceResult{ + { + Name: "s3", + Architecture: "SP-S3-U-D", + FullComparator: true, + SemanticValidation: "exact_public_observation", + RowCount: 1, + ObservedRows: []string{"[2]"}, + MeasurementOrder: baselineOrder, + Stats: DurationStats{ + WarmupIterations: 20, + }, + }, + { + Name: "s1", + Architecture: "SP-S1", + FullComparator: true, + SemanticValidation: "exact_public_observation", + RowCount: 1, + ObservedRows: []string{"[2]"}, + MeasurementOrder: candidateOrder, + Stats: DurationStats{ + WarmupIterations: 20, + }, + }, + }, + } + stampReferencePairIdentity(&record) + for iteration := 1; iteration <= 50; iteration++ { + record.PostgresReferences[0].Stats.Samples = append(record.PostgresReferences[0].Stats.Samples, LatencySample{ + Round: round, + Iteration: iteration, + Classification: "warm", + Duration: time.Millisecond, + }) + record.PostgresReferences[1].Stats.Samples = append(record.PostgresReferences[1].Stats.Samples, LatencySample{ + Round: round, + Iteration: iteration, + Classification: "warm", + Duration: 2 * time.Millisecond, + }) + } + records = append(records, record) + } + + report, err := buildReferencePairReport(records, ReferencePairOptions{ + Seed: 1, + Confidence: 0.975, + BootstrapCount: 100, + BaselineName: "s3", + CandidateName: "s1", + }) + require.NoError(t, err) + require.Len(t, report.Cases, 1) + require.Equal(t, 10, report.Cases[0].Rounds) + require.InDelta(t, 2, report.Cases[0].MedianRatio.Estimate, 0.0001) + require.InDelta(t, 2, report.Cases[0].P95Ratio.Estimate, 0.0001) + require.Equal(t, time.Millisecond, report.Cases[0].MedianChange.Estimate) +} + +// TestBuildReferencePairReportComparesValidatedHydrationBoundaries verifies that two prevalidated hydration implementations remain comparable despite different input-boundary descriptions. +func TestBuildReferencePairReportComparesValidatedHydrationBoundaries(t *testing.T) { + records := make([]CaseResult, 0, 10) + for round := 1; round <= 10; round++ { + baselineOrder, candidateOrder := 2, 3 + if round%2 == 0 { + baselineOrder, candidateOrder = 3, 2 + } + record := CaseResult{ + Dataset: "fixture", + Name: "path", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + RowCount: 1, + ObservedRows: []string{"[path]"}, + Environment: &RunEnvironment{ + Round: round, + WarmupIterations: 20, + }, + PostgresReferences: []PostgresReferenceResult{ + { + Name: "m0", + Architecture: "MAT-M0", + Boundary: "edge IDs", + SemanticValidation: "precomputed_exact_path_inputs", + RowCount: 1, + ObservedRows: []string{"[path]"}, + MeasurementOrder: baselineOrder, + Stats: DurationStats{ + WarmupIterations: 20, + }, + }, + { + Name: "m1", + Architecture: "MAT-M1", + Boundary: "node and edge IDs", + SemanticValidation: "precomputed_exact_path_inputs", + RowCount: 1, + ObservedRows: []string{"[path]"}, + MeasurementOrder: candidateOrder, + Stats: DurationStats{ + WarmupIterations: 20, + }, + }, + }, + } + stampReferencePairIdentity(&record) + for iteration := 1; iteration <= 50; iteration++ { + record.PostgresReferences[0].Stats.Samples = append(record.PostgresReferences[0].Stats.Samples, LatencySample{ + Round: round, + Iteration: iteration, + Classification: "warm", + Duration: time.Millisecond, + }) + record.PostgresReferences[1].Stats.Samples = append(record.PostgresReferences[1].Stats.Samples, LatencySample{ + Round: round, + Iteration: iteration, + Classification: "warm", + Duration: 2 * time.Millisecond, + }) + } + records = append(records, record) + } + + report, err := buildReferencePairReport(records, ReferencePairOptions{ + Seed: 1, + Confidence: 0.975, + BootstrapCount: 100, + BaselineName: "m0", + CandidateName: "m1", + }) + require.NoError(t, err) + require.Len(t, report.Cases, 1) + require.Equal(t, "precomputed_exact_path_inputs", report.Cases[0].BaselineSemanticValidation) + require.Equal(t, "edge IDs", report.Cases[0].BaselineBoundary) + require.InDelta(t, 2, report.Cases[0].MedianRatio.Estimate, 0.0001) + require.InDelta(t, 2, report.Cases[0].P95Ratio.Estimate, 0.0001) +} + +// TestBuildReferencePairReportRejectsMixedExactBoundaries verifies that a full public-result comparator cannot be timed against a precomputed hydration-only boundary. +func TestBuildReferencePairReportRejectsMixedExactBoundaries(t *testing.T) { + record := CaseResult{ + Dataset: "fixture", + Name: "path", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + RowCount: 1, + ObservedRows: []string{"[path]"}, + Environment: &RunEnvironment{ + Round: 1, + WarmupIterations: 20, + }, + PostgresReferences: []PostgresReferenceResult{ + { + Name: "full", + FullComparator: true, + SemanticValidation: "exact_public_observation", + RowCount: 1, + ObservedRows: []string{"[path]"}, + MeasurementOrder: 2, + Stats: DurationStats{ + WarmupIterations: 20, + }, + }, + { + Name: "hydration", + SemanticValidation: "precomputed_exact_path_inputs", + RowCount: 1, + ObservedRows: []string{"[path]"}, + MeasurementOrder: 3, + Stats: DurationStats{ + WarmupIterations: 20, + }, + }, + }, + } + + _, err := buildReferencePairReport([]CaseResult{record}, ReferencePairOptions{ + Seed: 1, + Confidence: 0.975, + BaselineName: "full", + CandidateName: "hydration", + }) + require.ErrorContains(t, err, "does not share an exact comparable boundary") +} + +// TestBuildReferencePairReportSupportsLabeledOrderedIDDiscovery verifies the reduced discovery protocol thresholds and ratio calculation for exact ordered-ID observations. +func TestBuildReferencePairReportSupportsLabeledOrderedIDDiscovery(t *testing.T) { + records := make([]CaseResult, 0, 5) + for round := 1; round <= 5; round++ { + baselineOrder, candidateOrder := 2, 3 + if round%2 == 0 { + baselineOrder, candidateOrder = 3, 2 + } + record := CaseResult{ + Dataset: "fixture", + Name: "ordered", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + RowCount: 1, + ObservedRows: []string{"[public]"}, + Environment: &RunEnvironment{ + Round: round, + WarmupIterations: 5, + }, + PostgresReferences: []PostgresReferenceResult{ + { + Name: "search_ordered_ids", + Architecture: "EXPANSION-STEPWISE-FORWARD", + ObservationShape: "ordered_ids", + SemanticValidation: "exact_ordered_ids", + RowCount: 1, + ObservedRows: []string{"[[1,2],3,[4]]"}, + MeasurementOrder: baselineOrder, + Stats: DurationStats{ + WarmupIterations: 5, + }, + }, + { + Name: "suffix_seeded_reverse_ordered_ids", + Architecture: "EXPANSION-SUFFIX-SEEDED-REVERSE", + ObservationShape: "ordered_ids", + SemanticValidation: "exact_ordered_ids", + RowCount: 1, + ObservedRows: []string{"[[1,2],3,[4]]"}, + MeasurementOrder: candidateOrder, + Stats: DurationStats{ + WarmupIterations: 5, + }, + }, + }, + } + stampReferencePairIdentity(&record) + for iteration := 1; iteration <= 10; iteration++ { + record.PostgresReferences[0].Stats.Samples = append(record.PostgresReferences[0].Stats.Samples, LatencySample{ + Round: round, + Iteration: iteration, + Classification: "warm", + Duration: 2 * time.Millisecond, + }) + record.PostgresReferences[1].Stats.Samples = append(record.PostgresReferences[1].Stats.Samples, LatencySample{ + Round: round, + Iteration: iteration, + Classification: "warm", + Duration: time.Millisecond, + }) + } + records = append(records, record) + } + + report, err := buildReferencePairReport(records, ReferencePairOptions{ + Seed: 1, + Confidence: 0.975, + BootstrapCount: 100, + BaselineName: "search_ordered_ids", + CandidateName: "suffix_seeded_reverse_ordered_ids", + Protocol: referencePairProtocolDiscovery, + }) + require.NoError(t, err) + require.Equal(t, referencePairProtocolDiscovery, report.Protocol) + require.Equal(t, 5, report.MinimumWarmups) + require.Equal(t, 5, report.MinimumRounds) + require.Equal(t, 10, report.MinimumSamples) + require.Len(t, report.Cases, 1) + require.InDelta(t, 0.5, report.Cases[0].MedianRatio.Estimate, 0.0001) +} + +// TestBuildReferencePairReportRejectsChangedImplementationIdentity verifies that an arm's implementation fingerprint must remain constant across all measurement rounds. +func TestBuildReferencePairReportRejectsChangedImplementationIdentity(t *testing.T) { + records := make([]CaseResult, 0, 10) + for round := 1; round <= 10; round++ { + records = append(records, referencePairProtocolRecord(round, round%2 == 1)) + } + records[4].PostgresReferences[0].ImplementationID = "changed" + _, err := buildReferencePairReport(records, ReferencePairOptions{ + Confidence: 0.975, + BaselineName: "baseline", + CandidateName: "candidate", + }) + require.ErrorContains(t, err, "identity changed") +} + +// TestBuildReferencePairReportRejectsUnbalancedArmOrder verifies that repeatedly measuring the same arm first violates the carryover-balancing protocol. +func TestBuildReferencePairReportRejectsUnbalancedArmOrder(t *testing.T) { + records := make([]CaseResult, 0, 10) + for round := 1; round <= 10; round++ { + records = append(records, referencePairProtocolRecord(round, true)) + } + _, err := buildReferencePairReport(records, ReferencePairOptions{ + Confidence: 0.975, + BaselineName: "baseline", + CandidateName: "candidate", + }) + require.ErrorContains(t, err, "does not alternate") +} + +// referencePairProtocolRecord returns one exact comparator round with selectable arm order and uniform warm timing samples. +func referencePairProtocolRecord(round int, baselineFirst bool) CaseResult { + baselineOrder, candidateOrder := 2, 3 + if !baselineFirst { + baselineOrder, candidateOrder = candidateOrder, baselineOrder + } + record := CaseResult{ + Dataset: "fixture", + Name: "protocol", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + RowCount: 1, + ObservedRows: []string{"[1]"}, + Environment: &RunEnvironment{Round: round, WarmupIterations: 20}, + PostgresReferences: []PostgresReferenceResult{ + {Name: "baseline", Architecture: "A", FullComparator: true, SemanticValidation: "exact_public_observation", RowCount: 1, ObservedRows: []string{"[1]"}, MeasurementOrder: baselineOrder, Stats: DurationStats{WarmupIterations: 20}}, + {Name: "candidate", Architecture: "B", FullComparator: true, SemanticValidation: "exact_public_observation", RowCount: 1, ObservedRows: []string{"[1]"}, MeasurementOrder: candidateOrder, Stats: DurationStats{WarmupIterations: 20}}, + }, + } + stampReferencePairIdentity(&record) + for iteration := 1; iteration <= 50; iteration++ { + record.PostgresReferences[0].Stats.Samples = append(record.PostgresReferences[0].Stats.Samples, LatencySample{Round: round, Iteration: iteration, Classification: "warm", Duration: time.Millisecond}) + record.PostgresReferences[1].Stats.Samples = append(record.PostgresReferences[1].Stats.Samples, LatencySample{Round: round, Iteration: iteration, Classification: "warm", Duration: 2 * time.Millisecond}) + } + return record +} + +// stampReferencePairIdentity assigns a stable runtime, implementation, and SQL identity to both reference arms. +func stampReferencePairIdentity(record *CaseResult) { + record.Environment.BinarySHA256 = "binary" + record.Environment.DirtyDiffSHA256 = "dirty" + record.Environment.SourceCommit = "commit" + record.Environment.GOOS = "linux" + record.Environment.GOARCH = "amd64" + for idx := range record.PostgresReferences { + reference := &record.PostgresReferences[idx] + reference.ImplementationID = reference.Name + "-implementation" + reference.SQLFingerprint = reference.Name + "-sql" + } +} + +// TestBuildReferencePairReportRejectsMismatchedOrderedIDObservations verifies that discovery timing cannot compare arms whose ordered-ID result sequences differ. +func TestBuildReferencePairReportRejectsMismatchedOrderedIDObservations(t *testing.T) { + record := CaseResult{ + Dataset: "fixture", + Name: "ordered", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Environment: &RunEnvironment{ + Round: 1, + WarmupIterations: 5, + }, + PostgresReferences: []PostgresReferenceResult{ + { + Name: "search_ordered_ids", + ObservationShape: "ordered_ids", + SemanticValidation: "exact_ordered_ids", + RowCount: 1, + ObservedRows: []string{"[a]"}, + MeasurementOrder: 2, + Stats: DurationStats{ + WarmupIterations: 5, + }, + }, + { + Name: "suffix_seeded_reverse_ordered_ids", + ObservationShape: "ordered_ids", + SemanticValidation: "exact_ordered_ids", + RowCount: 1, + ObservedRows: []string{"[b]"}, + MeasurementOrder: 3, + Stats: DurationStats{ + WarmupIterations: 5, + }, + }, + }, + } + + _, err := buildReferencePairReport([]CaseResult{record}, ReferencePairOptions{ + Seed: 1, + Confidence: 0.975, + BaselineName: "search_ordered_ids", + CandidateName: "suffix_seeded_reverse_ordered_ids", + Protocol: referencePairProtocolDiscovery, + }) + require.ErrorContains(t, err, "ordered-ID reference-pair observations differ") +} diff --git a/cmd/graphbench/reference_tournament_report.go b/cmd/graphbench/reference_tournament_report.go new file mode 100644 index 00000000..7896ef68 --- /dev/null +++ b/cmd/graphbench/reference_tournament_report.go @@ -0,0 +1,392 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "os" + "slices" + "sort" + "time" +) + +const referenceTournamentReportVersion = 1 + +// ReferenceTournamentOptions defines a predeclared three- or five-arm Williams tournament. +// The first arm is always the incumbent. +type ReferenceTournamentOptions struct { + Seed int64 + BootstrapCount int + Confidence float64 + MaterialityRatio float64 + MaterialityAbsolute time.Duration + P95RatioLimit float64 + Arms []string + Protocol string +} + +type ReferenceTournamentPair struct { + Arm string `json:"arm"` + MedianRatio RatioInterval `json:"median_ratio_to_incumbent"` + MedianSaving DurationInterval `json:"median_saving_vs_incumbent"` + P95Ratio RatioInterval `json:"p95_ratio_to_incumbent"` + Material bool `json:"material"` + P95Contained bool `json:"p95_contained"` + QualifiedWinner bool `json:"qualified_winner"` +} + +type ReferenceTournamentCase struct { + Dataset string `json:"dataset"` + Name string `json:"name"` + QualificationSplit string `json:"qualification_split"` + Winner string `json:"winner,omitempty"` + Rounds int `json:"rounds"` + Passed bool `json:"passed"` + Reasons []string `json:"reasons,omitempty"` + Pairs []ReferenceTournamentPair `json:"pairs"` +} + +type ReferenceTournamentReport struct { + Version int `json:"version"` + ArtifactSHA256 string `json:"artifact_sha256,omitempty"` + Protocol string `json:"protocol"` + Incumbent string `json:"incumbent"` + Winner string `json:"winner,omitempty"` + Arms []string `json:"arms"` + Confidence float64 `json:"confidence_level"` + MaterialityRatio float64 `json:"materiality_ratio"` + MaterialityAbsolute time.Duration `json:"materiality_absolute_lower_limit"` + P95RatioLimit float64 `json:"p95_ratio_upper_limit"` + Passed bool `json:"passed"` + PromotionEligible bool `json:"promotion_eligible"` + TrainingPassed bool `json:"training_passed"` + HoldoutPassed bool `json:"holdout_passed"` + Cases []ReferenceTournamentCase `json:"cases"` +} + +type tournamentArmSeries struct { + identity string + samples roundSamples +} + +type tournamentCaseSeries struct { + split string + arms map[string]*tournamentArmSeries + rounds map[int]struct{} +} + +func buildReferenceTournamentReport(records []CaseResult, options ReferenceTournamentOptions) (ReferenceTournamentReport, error) { + if err := normalizeReferenceTournamentOptions(&options); err != nil { + return ReferenceTournamentReport{}, err + } + minimumWarmups, minimumRounds, maximumRounds, minimumSamples, err := referenceTournamentRequirements(options.Protocol) + if err != nil { + return ReferenceTournamentReport{}, err + } + + series := map[performanceKey]*tournamentCaseSeries{} + for _, record := range records { + if record.ExecutionMode != ModePostgresSQL || !recordContainsAnyReference(record, options.Arms) { + continue + } + if err := addReferenceTournamentRecord(series, record, options.Arms, minimumWarmups); err != nil { + return ReferenceTournamentReport{}, err + } + } + if len(series) == 0 { + return ReferenceTournamentReport{}, fmt.Errorf("artifact has no PostgreSQL reference tournament records") + } + + report := ReferenceTournamentReport{ + Version: referenceTournamentReportVersion, + Protocol: options.Protocol, + Arms: append([]string(nil), options.Arms...), + Incumbent: options.Arms[0], + Confidence: options.Confidence, + MaterialityRatio: options.MaterialityRatio, + MaterialityAbsolute: options.MaterialityAbsolute, + P95RatioLimit: options.P95RatioLimit, + Passed: true, + TrainingPassed: true, + HoldoutPassed: true, + } + keys := sortedTournamentPerformanceKeys(series) + gate := PerfGateOptions{Seed: options.Seed, Confidence: options.Confidence, BootstrapCount: options.BootstrapCount} + winners := map[string]struct{}{} + for caseIndex, key := range keys { + entry := evaluateReferenceTournamentCase(key, series[key], options, gate, caseIndex, minimumRounds, maximumRounds, minimumSamples) + if entry.Passed { + winners[entry.Winner] = struct{}{} + } else { + report.Passed = false + } + switch entry.QualificationSplit { + case "training": + report.TrainingPassed = report.TrainingPassed && entry.Passed + case "holdout": + report.HoldoutPassed = report.HoldoutPassed && entry.Passed + } + report.Cases = append(report.Cases, entry) + } + + report.TrainingPassed = report.TrainingPassed && tournamentHasSplit(report.Cases, "training") + report.HoldoutPassed = report.HoldoutPassed && tournamentHasSplit(report.Cases, "holdout") + if len(winners) == 1 { + for winner := range winners { + report.Winner = winner + } + } else { + report.Passed = false + } + report.Passed = report.Passed && report.TrainingPassed && report.HoldoutPassed && report.Winner != "" + report.PromotionEligible = options.Protocol == referencePairProtocolConfirmation && report.Passed + return report, nil +} + +func normalizeReferenceTournamentOptions(options *ReferenceTournamentOptions) error { + if len(options.Arms) != 3 && len(options.Arms) != 5 { + return fmt.Errorf("reference tournament requires exactly 3 or 5 arms") + } + seen := map[string]struct{}{} + for _, arm := range options.Arms { + if arm == "" { + return fmt.Errorf("reference tournament arm must not be empty") + } + if _, duplicate := seen[arm]; duplicate { + return fmt.Errorf("reference tournament arms must be distinct") + } + seen[arm] = struct{}{} + } + if options.Confidence <= 0 || options.Confidence >= 1 { + return fmt.Errorf("confidence level must be between 0 and 1") + } + if options.BootstrapCount == 0 { + options.BootstrapCount = defaultBootstrapCount + } + if options.MaterialityRatio == 0 { + options.MaterialityRatio = .95 + } + if options.MaterialityRatio <= 0 || options.MaterialityRatio >= 1 { + return fmt.Errorf("materiality ratio must be between 0 and 1") + } + if options.MaterialityAbsolute == 0 { + options.MaterialityAbsolute = 100 * time.Microsecond + } + if options.MaterialityAbsolute < 0 { + return fmt.Errorf("materiality absolute must not be negative") + } + if options.P95RatioLimit == 0 { + options.P95RatioLimit = 1.05 + } + if options.P95RatioLimit <= 0 { + return fmt.Errorf("p95 ratio limit must be positive") + } + if options.Protocol == "" { + options.Protocol = referencePairProtocolConfirmation + } + return nil +} + +func referenceTournamentRequirements(protocol string) (int, int, int, int, error) { + switch protocol { + case referencePairProtocolDiscovery: + return 5, 5, 20, 10, nil + case referencePairProtocolConfirmation: + return 20, 10, 20, 50, nil + default: + return 0, 0, 0, 0, fmt.Errorf("unsupported reference tournament protocol %q", protocol) + } +} + +func recordContainsAnyReference(record CaseResult, arms []string) bool { + for _, reference := range record.PostgresReferences { + if slices.Contains(arms, reference.Name) { + return true + } + } + return false +} + +func addReferenceTournamentRecord(series map[performanceKey]*tournamentCaseSeries, record CaseResult, arms []string, minimumWarmups int) error { + if record.Status != StatusOK || record.Environment == nil || record.Environment.WarmupIterations < minimumWarmups { + return fmt.Errorf("%s/%s lacks a successful %d-warmup PostgreSQL record", record.Dataset, record.Name, minimumWarmups) + } + if record.Shape.QualificationSplit != "training" && record.Shape.QualificationSplit != "holdout" { + return fmt.Errorf("%s/%s requires a training or holdout qualification split", record.Dataset, record.Name) + } + key := performanceKey{dataset: record.Dataset, name: record.Name, backend: ModePostgresSQL} + current := series[key] + if current == nil { + current = &tournamentCaseSeries{split: record.Shape.QualificationSplit, arms: map[string]*tournamentArmSeries{}, rounds: map[int]struct{}{}} + series[key] = current + } else if current.split != record.Shape.QualificationSplit { + return fmt.Errorf("%s/%s changes qualification split across rounds", record.Dataset, record.Name) + } + if record.Environment.Round < 1 { + return fmt.Errorf("%s/%s has invalid tournament round %d", record.Dataset, record.Name, record.Environment.Round) + } + if _, duplicate := current.rounds[record.Environment.Round]; duplicate { + return fmt.Errorf("%s/%s has duplicate tournament round %d", record.Dataset, record.Name, record.Environment.Round) + } + current.rounds[record.Environment.Round] = struct{}{} + if err := validateTournamentRoundOrder(record.Environment.Round, arms, record.PostgresReferences); err != nil { + return fmt.Errorf("%s/%s: %w", record.Dataset, record.Name, err) + } + for _, name := range arms { + if err := addReferenceTournamentArm(current, record, name, minimumWarmups); err != nil { + return err + } + } + return nil +} + +func addReferenceTournamentArm(current *tournamentCaseSeries, record CaseResult, name string, minimumWarmups int) error { + reference := findReference(record.PostgresReferences, name) + if reference == nil { + return fmt.Errorf("%s/%s lacks tournament arm %s", record.Dataset, record.Name, name) + } + if !reference.FullComparator || reference.SemanticValidation != "exact_public_observation" || reference.RowCount != record.RowCount || !slices.Equal(reference.ObservedRows, record.ObservedRows) { + return fmt.Errorf("%s/%s arm %s is not an exact public comparator", record.Dataset, record.Name, name) + } + if reference.Stats.WarmupIterations < minimumWarmups || reference.ImplementationID == "" || reference.SQLFingerprint == "" { + return fmt.Errorf("%s/%s arm %s lacks warmups or identity", record.Dataset, record.Name, name) + } + identity := reference.Architecture + "\x00" + reference.ImplementationID + "\x00" + reference.SQLFingerprint + "\x00" + reference.Boundary + arm := current.arms[name] + if arm == nil { + arm = &tournamentArmSeries{identity: identity, samples: roundSamples{}} + current.arms[name] = arm + } else if arm.identity != identity { + return fmt.Errorf("%s/%s arm %s identity changed", record.Dataset, record.Name, name) + } + for _, sample := range reference.Stats.Samples { + if sample.Classification == "warm" && sample.Duration > 0 { + arm.samples[record.Environment.Round] = append(arm.samples[record.Environment.Round], sample.Duration) + } + } + return nil +} + +func sortedTournamentPerformanceKeys(series map[performanceKey]*tournamentCaseSeries) []performanceKey { + keys := make([]performanceKey, 0, len(series)) + for key := range series { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { + return keys[i].dataset < keys[j].dataset || keys[i].dataset == keys[j].dataset && keys[i].name < keys[j].name + }) + return keys +} + +func evaluateReferenceTournamentCase(key performanceKey, current *tournamentCaseSeries, options ReferenceTournamentOptions, gate PerfGateOptions, caseIndex, minimumRounds, maximumRounds, minimumSamples int) ReferenceTournamentCase { + entry := ReferenceTournamentCase{Dataset: key.dataset, Name: key.name, QualificationSplit: current.split, Rounds: len(current.rounds), Passed: true} + if entry.Rounds < minimumRounds || entry.Rounds > maximumRounds { + entry.Passed = false + entry.Reasons = append(entry.Reasons, fmt.Sprintf("requires %d-%d Williams rounds, got %d", minimumRounds, maximumRounds, entry.Rounds)) + } + for _, name := range options.Arms { + for _, round := range sortedRoundSet(current.rounds) { + if len(current.arms[name].samples[round]) < minimumSamples { + entry.Passed = false + entry.Reasons = append(entry.Reasons, fmt.Sprintf("%s round %d requires %d samples", name, round, minimumSamples)) + } + } + } + + incumbent := current.arms[options.Arms[0]].samples + bestMedian := time.Duration(1<<63 - 1) + for armIndex, name := range options.Arms[1:] { + baseline, candidate := matchedRounds(incumbent, current.arms[name].samples) + seed := options.Seed + int64(caseIndex*31+armIndex)*7919 + pair := ReferenceTournamentPair{ + Arm: name, + MedianRatio: bootstrapRoundMedianRatio(baseline, candidate, seed, gate), + MedianSaving: bootstrapRoundMedianSaving(baseline, candidate, seed+1, gate), + P95Ratio: bootstrapStratifiedP95Ratio(baseline, candidate, seed+2, gate), + } + pair.Material = pair.MedianRatio.Upper <= options.MaterialityRatio || pair.MedianSaving.Lower >= options.MaterialityAbsolute + pair.P95Contained = pair.P95Ratio.Upper <= options.P95RatioLimit + pair.QualifiedWinner = pair.Material && pair.P95Contained + if pair.QualifiedWinner { + median := time.Duration(durationQuantile(flattenSamples(candidate, sortedRounds(candidate)), .5)) + if median < bestMedian { + bestMedian, entry.Winner = median, name + } + } + entry.Pairs = append(entry.Pairs, pair) + } + if entry.Winner == "" { + entry.Passed = false + entry.Reasons = append(entry.Reasons, "no candidate materially beats the incumbent with p95 containment") + } + return entry +} + +func tournamentHasSplit(cases []ReferenceTournamentCase, split string) bool { + for _, entry := range cases { + if entry.QualificationSplit == split { + return true + } + } + return false +} + +func validateTournamentRoundOrder(round int, arms []string, references []PostgresReferenceResult) error { + base := make([]postgresReferenceSpec, len(arms)) + for idx, arm := range arms { + base[idx] = postgresReferenceSpec{name: arm} + } + expected := referenceSpecsForRound(base, round) + orders := map[string]int{} + for _, reference := range references { + if slices.Contains(arms, reference.Name) { + orders[reference.Name] = reference.MeasurementOrder + } + } + for idx, spec := range expected { + // Production is measurement position one when more than one reference + // arm is selected; the tournament occupies the contiguous suffix. + if orders[spec.name] != idx+2 { + return fmt.Errorf("round %d does not match the declared %d-arm Williams order", round, len(arms)) + } + } + return nil +} + +func createReferenceTournamentReport(artifactPath, outputPath string, options ReferenceTournamentOptions) (bool, error) { + records, err := readJSONLFile(artifactPath) + if err != nil { + return false, err + } + report, err := buildReferenceTournamentReport(records, options) + if err != nil { + return false, err + } + report.ArtifactSHA256, err = fileSHA256(artifactPath) + if err != nil { + return false, err + } + var output *os.File + if outputPath == "" { + output = os.Stdout + } else { + if err := ensureOutputDir(outputPath); err != nil { + return false, err + } + output, err = os.Create(outputPath) + if err != nil { + return false, err + } + defer output.Close() + } + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + if err := encoder.Encode(report); err != nil { + return false, err + } + return report.PromotionEligible, nil +} diff --git a/cmd/graphbench/reference_tournament_report_test.go b/cmd/graphbench/reference_tournament_report_test.go new file mode 100644 index 00000000..da3631e5 --- /dev/null +++ b/cmd/graphbench/reference_tournament_report_test.go @@ -0,0 +1,104 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestBuildReferenceTournamentReportQualifiesStableHoldoutWinner(t *testing.T) { + arms := []string{"expand_into_pair_join", "expand_into_lower_degree_scan", "expand_into_pair_cache"} + var records []CaseResult + for round := 1; round <= 12; round++ { + for _, split := range []string{"training", "holdout"} { + records = append(records, referenceTournamentRecord(arms, round, split, map[string]time.Duration{ + arms[0]: 10 * time.Millisecond, + arms[1]: 7 * time.Millisecond, + arms[2]: 5 * time.Millisecond, + })) + } + } + + report, err := buildReferenceTournamentReport(records, ReferenceTournamentOptions{ + Seed: 1, BootstrapCount: 100, Confidence: .975, Arms: arms, Protocol: referencePairProtocolConfirmation, + }) + require.NoError(t, err) + require.True(t, report.Passed) + require.True(t, report.PromotionEligible) + require.True(t, report.TrainingPassed) + require.True(t, report.HoldoutPassed) + require.Equal(t, arms[2], report.Winner) + require.Len(t, report.Cases, 2) + for _, entry := range report.Cases { + require.True(t, entry.Passed) + require.Equal(t, arms[2], entry.Winner) + } +} + +func TestBuildReferenceTournamentReportRejectsOrderAndWinnerDrift(t *testing.T) { + arms := []string{"expand_into_pair_join", "expand_into_lower_degree_scan", "expand_into_pair_cache"} + badOrder := referenceTournamentRecord(arms, 1, "training", map[string]time.Duration{ + arms[0]: 10 * time.Millisecond, arms[1]: 7 * time.Millisecond, arms[2]: 5 * time.Millisecond, + }) + badOrder.PostgresReferences[0].MeasurementOrder = 99 + _, err := buildReferenceTournamentReport([]CaseResult{badOrder}, ReferenceTournamentOptions{ + Confidence: .975, Arms: arms, Protocol: referencePairProtocolDiscovery, + }) + require.ErrorContains(t, err, "Williams order") + + var records []CaseResult + for round := 1; round <= 10; round++ { + records = append(records, + referenceTournamentRecord(arms, round, "training", map[string]time.Duration{ + arms[0]: 10 * time.Millisecond, arms[1]: 5 * time.Millisecond, arms[2]: 7 * time.Millisecond, + }), + referenceTournamentRecord(arms, round, "holdout", map[string]time.Duration{ + arms[0]: 10 * time.Millisecond, arms[1]: 7 * time.Millisecond, arms[2]: 5 * time.Millisecond, + }), + ) + } + report, err := buildReferenceTournamentReport(records, ReferenceTournamentOptions{ + Seed: 1, BootstrapCount: 100, Confidence: .975, Arms: arms, Protocol: referencePairProtocolConfirmation, + }) + require.NoError(t, err) + require.False(t, report.Passed) + require.False(t, report.PromotionEligible) + require.Empty(t, report.Winner) +} + +func referenceTournamentRecord(arms []string, round int, split string, durations map[string]time.Duration) CaseResult { + record := CaseResult{ + Environment: &RunEnvironment{Round: round, WarmupIterations: 20}, + Dataset: "tournament", Name: "case-" + split, + Shape: WorkloadShape{QualificationSplit: split}, + ExecutionMode: ModePostgresSQL, Status: StatusOK, + RowCount: 1, ObservedRows: []string{"row"}, + } + base := make([]postgresReferenceSpec, len(arms)) + for idx, arm := range arms { + base[idx].name = arm + } + orders := map[string]int{} + for idx, spec := range referenceSpecsForRound(base, round) { + orders[spec.name] = idx + 2 + } + for _, arm := range arms { + samples := make([]LatencySample, 50) + for idx := range samples { + samples[idx] = LatencySample{Classification: "warm", Duration: durations[arm] + time.Duration(idx)} + } + record.PostgresReferences = append(record.PostgresReferences, PostgresReferenceResult{ + Name: arm, Architecture: arm, ImplementationID: arm + "-v1", SQLFingerprint: arm + "-sql-v1", + Boundary: "relationships", FullComparator: true, SemanticValidation: "exact_public_observation", + MeasurementOrder: orders[arm], RowCount: 1, ObservedRows: []string{"row"}, + Stats: DurationStats{WarmupIterations: 20, Samples: samples}, + }) + } + return record +} diff --git a/cmd/graphbench/references.go b/cmd/graphbench/references.go new file mode 100644 index 00000000..91a4ee91 --- /dev/null +++ b/cmd/graphbench/references.go @@ -0,0 +1,2005 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "slices" + "sort" + "strings" + "time" + + "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/cypher/models/cypher" + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/opengraph" +) + +// postgresReferenceSchemaVersion identifies the serialized schema revision for PostgreSQL reference schema. +const postgresReferenceSchemaVersion = 1 + +// postgresReferenceArms lists the independently implemented PostgreSQL comparison arms. +var postgresReferenceArms = []string{ + "round_trip", + "endpoint_validation", + "fixed_suffix_rows", + "minimum_graph_access", + "search_ordered_ids", + "stepwise_forward_aa_ordered_ids", + "root_reuse_ordered_ids", + "late_hydration_ordered_ids", + "factored_suffix_forward_ordered_ids", + "suffix_seeded_reverse_ordered_ids", + "backward_viability_forward_ordered_ids", + "hydration_only", + "complete_reference", + "root_reuse_complete", + "late_hydration_complete", + "factored_suffix_forward_complete", + "suffix_seeded_reverse_complete", + "backward_viability_forward_complete", + "m0_directed_hydration_only", + "m1_ordered_ids_hydration_only", + "s3_unidirectional_trail_cte", + "s3_unidirectional_cte_m0_directed", + "s3_unidirectional_cte_m1_ordered_ids", + "s3_bidirectional_trail_cte", + "s1_array_bfs_distance", + "s4_canonical_source_distance", + "s4_canonical_source_witness_m0", + "sp_b1_strict_alternating_distance", + "sp_b1_strict_alternating_witness_m0", + "sp_b2_smaller_frontier_distance", + "sp_b2_smaller_frontier_witness_m0", + "asp_a1_stored_helper_m0", + "asp_i1_inline_predecessor_dag_m0", + "asp_b1_bidirectional_dag_strict_m0", + "asp_b2_bidirectional_dag_smaller_frontier_m0", + "expand_into_pair_join", + "expand_into_lower_degree_scan", + "expand_into_pair_cache", +} + +// validPostgresReferenceArm reports whether a reference-arm selector is declared. +func validPostgresReferenceArm(name string) bool { + return slices.Contains(postgresReferenceArms, name) +} + +// postgresReferenceSpec defines one independent PostgreSQL reference implementation and its observation contract. +type postgresReferenceSpec struct { + // name is the canonical selector and serialized identity for the reference arm. + name string + // legacyName retains the compatibility alias accepted for a reference arm. + legacyName string + // architecture retains the executor architecture that must remain stable across rounds. + architecture string + // implementationID provides a versioned identity for the reference algorithm and materialization strategy. + implementationID string + // stateShape describes recursive state retained by the reference implementation. + stateShape string + // observationShape describes the normalized values returned by the reference boundary. + observationShape string + // semanticValidation describes the exact observation contract enforced for the reference. + semanticValidation string + // boundary identifies the timed boundary exposed by the reference arm. + boundary string + // fullComparator reports whether the reference produces the complete public observation. + fullComparator bool + // aaAliasOf identifies the reference arm reused as an explicit A/A alias. + aaAliasOf string + // timingBoundary describes which portion of reference execution contributes to latency samples. + timingBoundary string + // sql contains the executable SQL for an independent reference arm. + sql string + // parameters supplies resolved parameters to the reference SQL query. + parameters map[string]any + // validationSQL contains SQL used to validate affected entity counts after a write. + validationSQL string + // validationParams supplies parameters used to validate precomputed reference inputs. + validationParams map[string]any +} + +// measureReferences executes references and records its timing observations. +func (s *postgresSQLRunner) measureReferences(ctx context.Context, testCase ScaleCase, params map[string]any, idMap opengraph.IDMap, publicObservation []string, warmupIterations, iterations int) ([]PostgresReferenceResult, error) { + readOptions := s.readTransactionOptions() + specs, err := s.referenceSpecs(ctx, testCase, params) + if err != nil { + return nil, err + } + for idx := range specs { + specs[idx] = normalizedReferenceSpec(specs[idx]) + } + if err := validateReferenceSpecs(specs); err != nil { + return nil, fmt.Errorf("validate PostgreSQL reference identities: %w", err) + } + if len(s.referenceArms) > 0 { + specs, err = selectReferenceSpecs(specs, s.referenceArms) + if err != nil { + return nil, fmt.Errorf("%w for %s/%s", err, testCase.Dataset, testCase.Name) + } + } + specs = referenceSpecsForRound(specs, s.round) + results := make([]PostgresReferenceResult, 0, len(specs)) + for _, spec := range specs { + rowCount, stats, err := measureRawPostgres(ctx, s.db, spec.sql, spec.parameters, warmupIterations, iterations, readOptions...) + if err != nil { + return nil, fmt.Errorf("%s: %w", spec.name, err) + } + var observedRows []string + if spec.fullComparator || spec.validationSQL != "" { + var observedCount int64 + err := s.db.ReadTransaction(ctx, func(tx graph.Transaction) error { + var err error + observedCount, observedRows, err = observeRawRows(tx, spec.sql, spec.parameters, idMap, resultContainsNodeIDs(testCase.Expected), resultContainsPaths(testCase.Expected)) + return err + }, readOptions...) + if err != nil { + return nil, fmt.Errorf("%s exact observation: %w", spec.name, err) + } + if observedCount != rowCount { + return nil, fmt.Errorf("%s exact observation row count changed from %d to %d", spec.name, rowCount, observedCount) + } + if spec.validationSQL != "" { + var ( + validationCount int64 + validationRows []string + ) + + err := s.db.ReadTransaction(ctx, func(tx graph.Transaction) error { + var err error + validationCount, validationRows, err = observeRawRows(tx, spec.validationSQL, spec.validationParams, idMap, resultContainsNodeIDs(testCase.Expected), resultContainsPaths(testCase.Expected)) + return err + }, readOptions...) + if err != nil { + return nil, fmt.Errorf("%s validation reference observation: %w", spec.name, err) + } + if validationCount != observedCount || !slices.Equal(validationRows, observedRows) { + return nil, fmt.Errorf("%s materialized observation differs from validation reference: candidate=%v reference=%v", spec.name, observedRows, validationRows) + } + } + if testCase.Expected.RowCount != nil && rowCount != *testCase.Expected.RowCount { + return nil, fmt.Errorf("%s returned %d rows, expected %d", spec.name, rowCount, *testCase.Expected.RowCount) + } + if spec.semanticValidation != "exact_ordered_ids" { + if err := validateExpectedObservations(testCase.Expected, observedRows); err != nil { + return nil, fmt.Errorf("%s semantic validation: %w", spec.name, err) + } + if publicObservation != nil && !slices.Equal(publicObservation, observedRows) && !validAlternativeShortestPathObservation(testCase, publicObservation, observedRows) { + return nil, fmt.Errorf("%s exact public observation differs: public=%v reference=%v", spec.name, publicObservation, observedRows) + } + } + } + for idx := range stats.Samples { + stats.Samples[idx].Backend = ModePostgresSQL + stats.Samples[idx].Dataset = testCase.Dataset + stats.Samples[idx].Case = testCase.Name + "/reference/" + spec.name + stats.Samples[idx].ConnectionID = s.backendPID + } + plan, planJSON, metrics, err := explainRawPostgres(ctx, s.db, spec.sql, spec.parameters, readOptions...) + if err != nil { + return nil, fmt.Errorf("%s explain: %w", spec.name, err) + } + results = append(results, PostgresReferenceResult{ + SchemaVersion: postgresReferenceSchemaVersion, + Name: spec.name, + LegacyName: spec.legacyName, + Architecture: spec.architecture, + ImplementationID: spec.implementationID, + StateShape: spec.stateShape, + ObservationShape: spec.observationShape, + SemanticValidation: spec.semanticValidation, + Boundary: spec.boundary, + TimingBoundary: spec.timingBoundary, + FullComparator: spec.fullComparator, + AAAliasOf: spec.aaAliasOf, + SQL: spec.sql, + SQLFingerprint: normalizedSQLFingerprint(spec.sql), + RowCount: rowCount, + ObservedRows: observedRows, + Stats: stats, + PostgresPlan: plan, + PostgresPlanJSON: planJSON, + PostgresMetrics: &metrics, + traversalTelemetryParameters: copyReferenceParams(spec.parameters), + }) + } + return results, nil +} + +// selectReferenceSpecs restricts reference arms to explicit selectors and rejects missing requested arms. +func selectReferenceSpecs(specs []postgresReferenceSpec, names []string) ([]postgresReferenceSpec, error) { + selected := make([]postgresReferenceSpec, 0, len(names)) + for _, name := range names { + idx := referenceSpecIndexOrMissing(specs, name) + if idx < 0 { + return nil, fmt.Errorf("requested PostgreSQL reference arm %q is unavailable", name) + } + selected = append(selected, specs[idx]) + } + return selected, nil +} + +// explainRawPostgres runs raw PostgreSQL EXPLAIN and returns normalized plan text, JSON, and metrics. +func explainRawPostgres(ctx context.Context, db graph.Database, sqlQuery string, params map[string]any, transactionOptions ...graph.TransactionOption) ([]string, json.RawMessage, PostgresPlanMetrics, error) { + var ( + plan []string + planJSON json.RawMessage + ) + + err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { + result := tx.Raw("EXPLAIN (ANALYZE, BUFFERS, WAL, SETTINGS, TIMING OFF) "+sqlQuery, params) + defer result.Close() + for result.Next() { + if values := result.Values(); len(values) > 0 { + plan = append(plan, fmt.Sprint(values[0])) + } + } + if err := result.Error(); err != nil { + return err + } + jsonResult := tx.Raw("EXPLAIN (ANALYZE, BUFFERS, WAL, SETTINGS, TIMING ON, FORMAT JSON) "+sqlQuery, params) + defer jsonResult.Close() + if jsonResult.Next() && len(jsonResult.Values()) > 0 { + var err error + planJSON, err = encodePostgresPlanJSON(jsonResult.Values()[0]) + if err != nil { + return err + } + } + return jsonResult.Error() + }, transactionOptions...) + if err != nil { + return nil, nil, PostgresPlanMetrics{}, err + } + metrics, err := parsePostgresPlanJSONMetrics(planJSON) + if err != nil { + return nil, nil, PostgresPlanMetrics{}, err + } + return plan, planJSON, metrics, nil +} + +// normalizedReferenceSpec fills legacy reference metadata defaults used for stable identity comparisons. +func normalizedReferenceSpec(spec postgresReferenceSpec) postgresReferenceSpec { + if spec.architecture == "" { + spec.architecture = "component_probe" + } + if spec.implementationID == "" { + spec.implementationID = spec.name + "_v1" + } + if spec.stateShape == "" { + spec.stateShape = "implementation_defined" + } + if spec.observationShape == "" { + spec.observationShape = "component_observation" + } + if spec.timingBoundary == "" { + spec.timingBoundary = "raw_pgx" + } + if spec.semanticValidation == "" { + spec.semanticValidation = "row_count_stability" + if spec.fullComparator { + spec.semanticValidation = "exact_public_observation" + } + } + return spec +} + +// normalizedSQLFingerprint hashes SQL after collapsing insignificant whitespace. +func normalizedSQLFingerprint(sql string) string { + return sqlFingerprint(strings.Join(strings.Fields(sql), " ")) +} + +// validateReferenceSpecs rejects duplicate, incomplete, or semantically inconsistent reference specifications. +func validateReferenceSpecs(specs []postgresReferenceSpec) error { + byName := make(map[string]postgresReferenceSpec, len(specs)) + byImplementation := make(map[string]postgresReferenceSpec, len(specs)) + byFingerprint := make(map[string]postgresReferenceSpec, len(specs)) + for _, spec := range specs { + if spec.name == "" || spec.architecture == "" || spec.implementationID == "" || spec.stateShape == "" || spec.observationShape == "" || spec.timingBoundary == "" || spec.semanticValidation == "" { + return fmt.Errorf("reference %q has an incomplete architecture identity", spec.name) + } + if _, found := byName[spec.name]; found { + return fmt.Errorf("duplicate reference name %q", spec.name) + } + fingerprint := normalizedSQLFingerprint(spec.sql) + if previous, found := byImplementation[spec.implementationID]; found && (previous.stateShape != spec.stateShape || previous.observationShape != spec.observationShape || normalizedSQLFingerprint(previous.sql) != fingerprint) { + return fmt.Errorf("implementation %q changes state, observation, or SQL identity between %q and %q", spec.implementationID, previous.name, spec.name) + } + if previous, found := byFingerprint[fingerprint]; found { + previousCanonical := previous.name + if previous.aaAliasOf != "" { + previousCanonical = previous.aaAliasOf + } + specCanonical := spec.name + if spec.aaAliasOf != "" { + specCanonical = spec.aaAliasOf + } + if specCanonical != previousCanonical { + return fmt.Errorf("references %q and %q have identical normalized SQL without a declared A/A alias", previous.name, spec.name) + } + canonical, alias := byName[previousCanonical], spec + if canonical.name == "" { + canonical = previous + } + if parameterShape(canonical.parameters) != parameterShape(alias.parameters) || canonical.observationShape != alias.observationShape || canonical.timingBoundary != alias.timingBoundary || canonical.fullComparator != alias.fullComparator || canonical.semanticValidation != alias.semanticValidation { + return fmt.Errorf("A/A alias %q does not match canonical arm %q at an identical comparison boundary", alias.name, canonical.name) + } + } + byName[spec.name] = spec + byImplementation[spec.implementationID] = spec + byFingerprint[fingerprint] = spec + } + for _, spec := range specs { + if spec.aaAliasOf == "" { + continue + } + canonical, found := byName[spec.aaAliasOf] + if !found { + return fmt.Errorf("A/A alias %q names missing canonical arm %q", spec.name, spec.aaAliasOf) + } + if normalizedSQLFingerprint(spec.sql) != normalizedSQLFingerprint(canonical.sql) { + return fmt.Errorf("A/A alias %q SQL differs from canonical arm %q", spec.name, canonical.name) + } + } + return nil +} + +// parameterShape returns a type-only description of query parameters for reference identity checks. +func parameterShape(parameters map[string]any) string { + names := make([]string, 0, len(parameters)) + for name := range parameters { + names = append(names, name) + } + sort.Strings(names) + var shape strings.Builder + for _, name := range names { + shape.WriteString(name) + shape.WriteByte('=') + if parameters[name] == nil { + shape.WriteString("") + } else { + shape.WriteString(reflect.TypeOf(parameters[name]).String()) + } + shape.WriteByte(';') + } + return shape.String() +} + +// validAlternativeShortestPathObservation reports whether two observations are both valid shortest-path witnesses. +func validAlternativeShortestPathObservation(testCase ScaleCase, publicRows, referenceRows []string) bool { + if testCase.Expected.ResultKind != "path_set" || strings.Contains(strings.ToLower(testCase.Cypher), "allshortestpaths") { + return false + } + provablyOutbound, err := shortestReferenceIsProvablyOutbound(testCase.Cypher) + if err != nil || !provablyOutbound { + return false + } + + publicPath, publicOK := singleStablePathObservation(publicRows) + referencePath, referenceOK := singleStablePathObservation(referenceRows) + if !publicOK || !referenceOK || !validOutboundStablePath(publicPath, testCase.Shape.EdgeKinds) || !validOutboundStablePath(referencePath, testCase.Shape.EdgeKinds) { + return false + } + if len(publicPath.Relationships) != len(referencePath.Relationships) { + return false + } + + publicStart, publicEnd := publicPath.Nodes[0].Identity, publicPath.Nodes[len(publicPath.Nodes)-1].Identity + referenceStart, referenceEnd := referencePath.Nodes[0].Identity, referencePath.Nodes[len(referencePath.Nodes)-1].Identity + return publicStart == referenceStart && publicEnd == referenceEnd +} + +// singleStablePathObservation returns the sole normalized path when the result contains exactly one valid path. +func singleStablePathObservation(rows []string) (stablePathObservation, bool) { + if len(rows) != 1 { + return stablePathObservation{}, false + } + + var columns []json.RawMessage + if err := json.Unmarshal([]byte(rows[0]), &columns); err != nil || len(columns) != 1 { + return stablePathObservation{}, false + } + + var path stablePathObservation + if err := json.Unmarshal(columns[0], &path); err != nil { + return stablePathObservation{}, false + } + return path, true +} + +// validOutboundStablePath reports whether a stable path follows every relationship in outbound order. +func validOutboundStablePath(path stablePathObservation, allowedKinds []string) bool { + if len(path.Nodes) == 0 || len(path.Nodes) != len(path.Relationships)+1 { + return false + } + for _, node := range path.Nodes { + if strings.HasPrefix(node.Identity, "unmapped-node:") { + return false + } + } + for idx, relationship := range path.Relationships { + if relationship.Start != path.Nodes[idx].Identity || relationship.End != path.Nodes[idx+1].Identity { + return false + } + if len(allowedKinds) != 0 && !slices.Contains(allowedKinds, relationship.Kind) { + return false + } + } + return true +} + +// referenceSpecsForRound returns reference specifications in the predeclared balanced order for a round. +func referenceSpecsForRound(specs []postgresReferenceSpec, round int) []postgresReferenceSpec { + if len(specs) == 3 && round > 0 { + // Odd-sized treatment sets need a doubled Williams design. Across these + // six rows every arm occupies every position twice, and every directed + // first-order carryover pair occurs twice. + schedule := [6][3]int{ + {0, 1, 2}, + {1, 2, 0}, + {2, 0, 1}, + {2, 1, 0}, + {0, 2, 1}, + {1, 0, 2}, + } + row := schedule[(round-1)%len(schedule)] + ordered := make([]postgresReferenceSpec, len(specs)) + for idx, slot := range row { + ordered[idx] = specs[slot] + } + return ordered + } + if len(specs) == 5 && round > 0 { + // Ten-sequence Williams/carryover-balanced schedule predeclared by the + // fixed-suffix expansion tournament. Slots are the caller-selected arms, so B1/B2/B3 can + // share this schedule without hard-coding architecture names here. + schedule := [10][5]int{ + {0, 1, 4, 2, 3}, {1, 2, 0, 3, 4}, {2, 3, 1, 4, 0}, {3, 4, 2, 0, 1}, {4, 0, 3, 1, 2}, + {3, 2, 4, 1, 0}, {4, 3, 0, 2, 1}, {0, 4, 1, 3, 2}, {1, 0, 2, 4, 3}, {2, 1, 3, 0, 4}, + } + row := schedule[(round-1)%len(schedule)] + ordered := make([]postgresReferenceSpec, len(specs)) + for idx, slot := range row { + ordered[idx] = specs[slot] + } + return ordered + } + ordered := append([]postgresReferenceSpec(nil), specs...) + if round > 0 && round%2 == 0 { + slices.Reverse(ordered) + } + return ordered +} + +// referenceSpecs constructs the independent PostgreSQL reference implementations for a scale case. +func (s *postgresSQLRunner) referenceSpecs(ctx context.Context, testCase ScaleCase, params map[string]any) ([]postgresReferenceSpec, error) { + if testCase.Category == "expand_into_one_hop" { + return s.expandIntoReferenceSpecs(ctx, testCase, params) + } + if testCase.Category == "generated_fixed_suffix_expansion" { + return s.fixedSuffixExpansionReferenceSpecs(ctx, testCase, params) + } + if testCase.Category == "generated_shortest_path" || testCase.Category == "generated_shortest_path_v2" { + // Singleton and all-shortest architectures are kept as distinct arms; + // allShortestPaths uses its relationship-distinct predecessor DAG only. + if strings.Contains(strings.ToLower(testCase.Cypher), "allshortestpaths") { + return s.allShortestReferenceSpecs(ctx, testCase, params) + } + return s.shortestReferenceSpecs(ctx, testCase, params) + } + switch testCase.Name { + case "shortest_distance_bound_pair", "one_shortest_path_bound_pair": + return s.shortestReferenceSpecs(ctx, testCase, params) + case "fixed_suffix_expansion_endpoint_ids", "fixed_suffix_expansion_path_observed": + return s.fixedSuffixExpansionReferenceSpecs(ctx, testCase, params) + default: + return nil, nil + } +} + +// allShortestDAGSearch returns the predecessor-DAG SQL search for all shortest paths in one direction. +func allShortestDAGSearch(direction graph.Direction) string { + distanceJoin, distanceNext := "e.start_id = distance.node_id", "e.end_id" + predecessorJoin := "e.start_id = prior.node_id and e.end_id = paths.node_id" + if direction == graph.DirectionInbound { + distanceJoin, distanceNext = "e.end_id = distance.node_id", "e.start_id" + predecessorJoin = "e.end_id = prior.node_id and e.start_id = paths.node_id" + } + return `with recursive validated(start_id, end_id) as materialized ( + select start_node.id, end_node.id + from node start_node, node end_node + where start_node.graph_id = @graph_id and start_node.id = @start_id + and end_node.graph_id = @graph_id and end_node.id = @end_id +), distance(node_id, depth) as ( + select validated.start_id, 0 from validated + union + select ` + distanceNext + `, distance.depth + 1 + from distance + join edge e on e.graph_id = @graph_id and ` + distanceJoin + ` + where distance.depth < @max_depth + and (cardinality(@edge_kind_ids::int2[]) = 0 or e.kind_id = any(@edge_kind_ids::int2[])) +), target as materialized ( + select depth from distance + where node_id = @end_id and depth >= @min_depth + order by depth limit 1 +), predecessor(node_id, depth, predecessor_id, edge_id) as materialized ( + select paths.node_id, paths.depth, prior.node_id, e.id + from distance paths + join target on paths.depth > 0 and paths.depth <= target.depth + join distance prior on prior.depth = paths.depth - 1 + join edge e on e.graph_id = @graph_id and ` + predecessorJoin + ` + where (cardinality(@edge_kind_ids::int2[]) = 0 or e.kind_id = any(@edge_kind_ids::int2[])) +), paths(node_id, depth, edge_ids) as ( + select @end_id::int8, target.depth, array[]::int8[] from target + union all + select predecessor.predecessor_id, paths.depth - 1, array[predecessor.edge_id]::int8[] || paths.edge_ids + from paths join predecessor on predecessor.node_id = paths.node_id and predecessor.depth = paths.depth +), shortest(depth, edge_ids) as materialized ( + select target.depth, paths.edge_ids + from paths join target on true where paths.node_id = @start_id and paths.depth = 0 +)` +} + +func allShortestA1ReferenceSQL(direction graph.Direction) string { + inbound := "false" + if direction == graph.DirectionInbound { + inbound = "true" + } + search := `with shortest as materialized ( + select depth, path as edge_ids + from all_shortest_paths_dag( + @graph_id, @start_id, @end_id, @min_depth, @max_depth, + @edge_kind_ids, ` + inbound + ` + ) +)` + return shortestM0FullSQL(search, direction) +} + +// allShortestBidirectionalReferenceSQL exposes a forced two-sided +// predecessor-DAG kernel at the same complete M0 path boundary as ASP-A1. +func allShortestBidirectionalReferenceSQL(functionName string, direction graph.Direction) string { + inbound := "false" + if direction == graph.DirectionInbound { + inbound = "true" + } + search := `with shortest as materialized ( + select depth, path as edge_ids + from ` + functionName + `( + @graph_id, @start_id, @end_id, @min_depth, @max_depth, + @edge_kind_ids, ` + inbound + `, @state_limit, @frontier_limit, + @predecessor_limit, @enumeration_limit, @output_bytes_limit + ) +)` + return shortestM0FullSQL(search, direction) +} + +// allShortestReferenceSpecs builds the predecessor-DAG reference for an all-shortest-path workload. +func (s *postgresSQLRunner) allShortestReferenceSpecs(ctx context.Context, testCase ScaleCase, params map[string]any) ([]postgresReferenceSpec, error) { + probeParams := copyReferenceParams(params) + probeParams["graph_id"] = s.graphID + probeParams["min_depth"] = int32(1) + if testCase.Shape.MinDepth != nil { + probeParams["min_depth"] = int32(*testCase.Shape.MinDepth) + } + probeParams["max_depth"] = int32(15) + if testCase.Shape.MaxDepth != nil { + probeParams["max_depth"] = int32(*testCase.Shape.MaxDepth) + } + edgeKinds := make(graph.Kinds, 0, len(testCase.Shape.EdgeKinds)) + for _, name := range testCase.Shape.EdgeKinds { + edgeKinds = append(edgeKinds, graph.StringKind(name)) + } + var edgeKindIDs []int16 + if len(edgeKinds) > 0 { + if s.pgDriver == nil { + return nil, fmt.Errorf("map all-shortest reference edge kinds: PostgreSQL driver is unavailable") + } + var err error + edgeKindIDs, err = s.pgDriver.KindMapper().MapKinds(ctx, edgeKinds) + if err != nil { + return nil, fmt.Errorf("map all-shortest reference edge kinds: %w", err) + } + } + probeParams["edge_kind_ids"] = edgeKindIDs + direction, err := shortestReferenceDirection(testCase.Cypher) + if err != nil || direction == graph.DirectionBoth { + return nil, err + } + rootParameter, terminalParameter, err := shortestReferenceEndpointParameters(testCase.Cypher) + if err != nil { + return nil, err + } + probeParams["start_id"] = probeParams[rootParameter] + probeParams["end_id"] = probeParams[terminalParameter] + specs := []postgresReferenceSpec{{ + name: "asp_a1_stored_helper_m0", + architecture: "ASP-A1-DAG", + implementationID: "all_shortest_paths_dag_stored_helper_m0_v1", + stateShape: "minimum-depth helper workspace with relationship-distinct predecessors", + observationShape: "complete all-shortest path multiset", + semanticValidation: "exact_public_observation", + boundary: "complete path composites", + fullComparator: true, + sql: allShortestA1ReferenceSQL(direction), + parameters: probeParams, + }} + + // I1 is valid only inside the same distinct-endpoint, min-one bounded + // contract enforced by the production emitter. A1 remains available as the + // exact control outside that envelope. + startID, startOK := probeParams["start_id"].(int64) + endID, endOK := probeParams["end_id"].(int64) + maximumDepth, maximumOK := probeParams["max_depth"].(int32) + if probeParams["min_depth"] != int32(1) || !maximumOK || maximumDepth < 1 || maximumDepth > 64 || !startOK || !endOK || startID == endID { + return specs, nil + } + search := allShortestDAGSearch(direction) + specs = append(specs, postgresReferenceSpec{ + name: "asp_i1_inline_predecessor_dag_m0", + architecture: "ASP-I1-U-DAG+MAT-M0", + implementationID: "inline_shortest_depth_predecessor_dag_m0_v1", + stateShape: "node/depth discovery plus every relationship-distinct shortest-depth predecessor edge", + observationShape: "complete all-shortest path multiset", + semanticValidation: "exact_public_observation", + boundary: "complete path composites", + fullComparator: true, + sql: shortestM0FullSQL(search, direction), + parameters: probeParams, + }) + + // B1/B2 are intentionally tool/reference-only. Keep automatic production + // selection on ASP-A1 until independent confirmation passes, and do not + // expose candidate arms outside their distinct-endpoint minimum-one envelope. + candidateParams := copyReferenceParams(probeParams) + candidateParams["state_limit"] = int64(100_000) + candidateParams["frontier_limit"] = int64(100_000) + candidateParams["predecessor_limit"] = int64(100_000) + candidateParams["enumeration_limit"] = int64(100_000) + candidateParams["output_bytes_limit"] = int64(64 * 1024 * 1024) + for _, candidate := range []struct { + name string + architecture string + implementationID string + functionName string + }{ + { + name: "asp_b1_bidirectional_dag_strict_m0", + architecture: "ASP-B1-DAG-ALT-NODE", + implementationID: "typed_two_sided_predecessor_dag_strict_alternating_v1", + functionName: "all_shortest_paths_b1_strict_alternating", + }, + { + name: "asp_b2_bidirectional_dag_smaller_frontier_m0", + architecture: "ASP-B2-DAG-MIN-LEVEL", + implementationID: "typed_two_sided_predecessor_dag_smaller_current_level_v1", + functionName: "all_shortest_paths_b2_smaller_current_level", + }, + } { + specs = append(specs, postgresReferenceSpec{ + name: candidate.name, + architecture: candidate.architecture, + implementationID: candidate.implementationID, + stateShape: "two-sided minimum-node-depth discovery plus every relationship-distinct equal-depth predecessor/successor at one canonical cut", + observationShape: "complete all-shortest path multiset", + semanticValidation: "exact_public_observation", + boundary: "complete path composites", + fullComparator: true, + sql: allShortestBidirectionalReferenceSQL(candidate.functionName, direction), + parameters: copyReferenceParams(candidateParams), + }) + } + return specs, nil +} + +// shortestReferenceSpecs builds eligible shortest-path reference implementations and measurement boundaries. +func (s *postgresSQLRunner) shortestReferenceSpecs(ctx context.Context, testCase ScaleCase, params map[string]any) ([]postgresReferenceSpec, error) { + probeParams := copyReferenceParams(params) + probeParams["graph_id"] = s.graphID + probeParams["min_depth"] = int32(1) + if testCase.Shape.MinDepth != nil { + probeParams["min_depth"] = int32(*testCase.Shape.MinDepth) + } + probeParams["max_depth"] = int32(15) + if testCase.Shape.MaxDepth != nil { + probeParams["max_depth"] = int32(*testCase.Shape.MaxDepth) + } + edgeKinds := make(graph.Kinds, 0, len(testCase.Shape.EdgeKinds)) + for _, name := range testCase.Shape.EdgeKinds { + edgeKinds = append(edgeKinds, graph.StringKind(name)) + } + edgeKindIDs, err := s.pgDriver.KindMapper().MapKinds(ctx, edgeKinds) + if err != nil { + return nil, fmt.Errorf("map shortest reference edge kinds: %w", err) + } + probeParams["edge_kind_ids"] = edgeKindIDs + direction, err := shortestReferenceDirection(testCase.Cypher) + if err != nil { + return nil, fmt.Errorf("classify shortest reference direction: %w", err) + } + if direction == graph.DirectionBoth { + return nil, nil + } + rootParameter, terminalParameter, err := shortestReferenceEndpointParameters(testCase.Cypher) + if err != nil { + return nil, fmt.Errorf("resolve shortest reference endpoint parameters: %w", err) + } + searchParams := copyReferenceParams(probeParams) + searchParams["start_id"] = probeParams[rootParameter] + searchParams["end_id"] = probeParams[terminalParameter] + search := shortestReferenceSearchForDirection(direction) + values, err := readReferenceRow(ctx, s.db, search+` select depth, node_ids, edge_ids from shortest`, searchParams, s.readTransactionOptions()...) + if err != nil { + return nil, fmt.Errorf("precompute shortest hydration IDs: %w", err) + } + if len(values) != 0 && len(values) != 3 { + return nil, fmt.Errorf("precompute shortest hydration IDs returned %d columns, expected 3", len(values)) + } + var nodeIDs, edgeIDs []int64 + if len(values) == 3 { + nodeIDs, err = referenceInt64Slice(values[1]) + if err != nil { + return nil, fmt.Errorf("decode shortest hydration node IDs: %w", err) + } + edgeIDs, err = referenceInt64Slice(values[2]) + if err != nil { + return nil, fmt.Errorf("decode shortest hydration edge IDs: %w", err) + } + } + return buildShortestReferenceSpecs(testCase, searchParams, nodeIDs, edgeIDs, direction), nil +} + +// shortestReferenceEndpointParameters maps public start and end parameters to physical search endpoints for the parsed direction. +func shortestReferenceEndpointParameters(query string) (string, string, error) { + parsed, err := frontend.ParseCypher(frontend.NewContext(), query) + if err != nil { + return "", "", err + } + if parsed == nil || parsed.SingleQuery == nil || parsed.SingleQuery.SinglePartQuery == nil || parsed.SingleQuery.MultiPartQuery != nil { + return "", "", fmt.Errorf("expected a single-part shortest query") + } + for _, readingClause := range parsed.SingleQuery.SinglePartQuery.ReadingClauses { + if readingClause == nil || readingClause.Match == nil { + continue + } + bindings := map[string]string{} + if readingClause.Match.Where != nil { + for _, expression := range readingClause.Match.Where.Expressions { + collectIdentityParameterBindings(expression, bindings) + } + } + for _, patternPart := range readingClause.Match.Pattern { + if patternPart == nil || (!patternPart.ShortestPathPattern && !patternPart.AllShortestPathsPattern) || len(patternPart.PatternElements) < 3 { + continue + } + root, rootOK := patternPart.PatternElements[0].AsNodePattern() + terminal, terminalOK := patternPart.PatternElements[len(patternPart.PatternElements)-1].AsNodePattern() + if !rootOK || !terminalOK || root.Variable == nil || terminal.Variable == nil { + return "", "", fmt.Errorf("shortest reference endpoints must have variables") + } + rootParameter, rootBound := bindings[root.Variable.Symbol] + terminalParameter, terminalBound := bindings[terminal.Variable.Symbol] + if !rootBound || !terminalBound { + return "", "", fmt.Errorf("shortest reference endpoints must have parameter ID equalities") + } + return rootParameter, terminalParameter, nil + } + } + return "", "", fmt.Errorf("shortest pattern not found") +} + +// collectIdentityParameterBindings extracts equality-bound ID parameters for the two variables in a shortest-path pattern. +func collectIdentityParameterBindings(expression cypher.Expression, bindings map[string]string) { + switch typed := expression.(type) { + case *cypher.Conjunction: + for _, child := range typed.Expressions { + collectIdentityParameterBindings(child, bindings) + } + case *cypher.Parenthetical: + collectIdentityParameterBindings(typed.Expression, bindings) + case *cypher.Comparison: + if typed == nil || len(typed.Partials) != 1 || typed.Partials[0].Operator != cypher.OperatorEquals { + return + } + if symbol, ok := identityReferenceSymbol(typed.Left); ok { + if parameter, ok := typed.Partials[0].Right.(*cypher.Parameter); ok { + bindings[symbol] = parameter.Symbol + } + } + if symbol, ok := identityReferenceSymbol(typed.Partials[0].Right); ok { + if parameter, ok := typed.Left.(*cypher.Parameter); ok { + bindings[symbol] = parameter.Symbol + } + } + } +} + +// identityReferenceSymbol returns the variable whose ID is projected directly by a reference query. +func identityReferenceSymbol(expression cypher.Expression) (string, bool) { + function, ok := expression.(*cypher.FunctionInvocation) + if !ok || function == nil || !strings.EqualFold(function.Name, cypher.IdentityFunction) || len(function.Arguments) != 1 { + return "", false + } + variable, ok := function.Arguments[0].(*cypher.Variable) + if !ok || variable == nil || variable.Symbol == "" { + return "", false + } + return variable.Symbol, true +} + +// shortestReferenceIsProvablyOutbound reports whether a supported shortest-path query has outbound direction. +func shortestReferenceIsProvablyOutbound(query string) (bool, error) { + direction, err := shortestReferenceDirection(query) + if err != nil { + return false, err + } + + return direction == graph.DirectionOutbound, nil +} + +// shortestReferenceDirection parses a shortest-path query and returns its single relationship direction. +func shortestReferenceDirection(query string) (graph.Direction, error) { + parsed, err := frontend.ParseCypher(frontend.NewContext(), query) + if err != nil { + return 0, err + } + if parsed == nil || parsed.SingleQuery == nil || parsed.SingleQuery.SinglePartQuery == nil || parsed.SingleQuery.MultiPartQuery != nil { + return graph.DirectionBoth, nil + } + + var ( + shortestParts int + relationships int + direction graph.Direction + ) + for _, readingClause := range parsed.SingleQuery.SinglePartQuery.ReadingClauses { + if readingClause == nil || readingClause.Match == nil { + continue + } + for _, patternPart := range readingClause.Match.Pattern { + if patternPart == nil || (!patternPart.ShortestPathPattern && !patternPart.AllShortestPathsPattern) { + continue + } + shortestParts++ + for _, patternElement := range patternPart.PatternElements { + if relationship, isRelationship := patternElement.AsRelationshipPattern(); isRelationship { + relationships++ + direction = relationship.Direction + } + } + } + } + + if shortestParts != 1 || relationships != 1 { + return graph.DirectionBoth, nil + } + return direction, nil +} + +// shortestReferenceSearch returns the compact recursive shortest-path search SQL for a projection mode. +func shortestReferenceSearch() string { + return shortestReferenceSearchForDirection(graph.DirectionOutbound) +} + +// shortestReferenceSearchForDirection returns direction-specific shortest-path search SQL and endpoint columns. +func shortestReferenceSearchForDirection(direction graph.Direction) string { + edgeJoin, nextNode := "e.start_id = search.node_id", "e.end_id" + if direction == graph.DirectionInbound { + edgeJoin, nextNode = "e.end_id = search.node_id", "e.start_id" + } + return `with recursive search(node_id, depth, node_ids, edge_ids) as ( + select @start_id::int8, 0, array[@start_id::int8]::int8[], array[]::int8[] + union all + select ` + nextNode + `, search.depth + 1, search.node_ids || ` + nextNode + `, search.edge_ids || e.id + from search + join edge e on e.graph_id = @graph_id and ` + edgeJoin + ` + where search.depth < @max_depth + and (cardinality(@edge_kind_ids::int2[]) = 0 or e.kind_id = any(@edge_kind_ids::int2[])) + and e.id != all(search.edge_ids) +), shortest as materialized ( + select depth, node_ids, edge_ids from search + where node_id = @end_id and depth >= @min_depth + order by depth, edge_ids limit 1 +)` +} + +// shortestEdgeReferenceSearch returns the edge-only shortest-path search SQL for a direction. +func shortestEdgeReferenceSearch(direction graph.Direction) string { + edgeJoin, nextNode := "e.start_id = search.node_id", "e.end_id" + if direction == graph.DirectionInbound { + edgeJoin, nextNode = "e.end_id = search.node_id", "e.start_id" + } + return `with recursive search(node_id, depth, edge_ids) as ( + select @start_id::int8, 0, array[]::int8[] + union all + select ` + nextNode + `, search.depth + 1, search.edge_ids || e.id + from search + join edge e on e.graph_id = @graph_id and ` + edgeJoin + ` + where search.depth < @max_depth + and (cardinality(@edge_kind_ids::int2[]) = 0 or e.kind_id = any(@edge_kind_ids::int2[])) + and e.id != all(search.edge_ids) +), shortest as materialized ( + select depth, edge_ids from search + where node_id = @end_id and depth >= @min_depth + order by depth, edge_ids limit 1 +)` +} + +// shortestDistanceReferenceSearch returns the minimal-state shortest-distance search SQL for a direction. +func shortestDistanceReferenceSearch() string { + return shortestDistanceReferenceSearchForDirection(graph.DirectionOutbound) +} + +// shortestDistanceReferenceSearchForDirection returns direction-specific shortest-distance SQL and endpoint columns. +func shortestDistanceReferenceSearchForDirection(direction graph.Direction) string { + edgeJoin, nextNode := "e.start_id = search.node_id", "e.end_id" + if direction == graph.DirectionInbound { + edgeJoin, nextNode = "e.end_id = search.node_id", "e.start_id" + } + return `with recursive search(node_id, depth) as ( + select @start_id::int8, 0 + union + select ` + nextNode + `, search.depth + 1 + from search + join edge e on e.graph_id = @graph_id and ` + edgeJoin + ` + where search.depth < @max_depth + and (cardinality(@edge_kind_ids::int2[]) = 0 or e.kind_id = any(@edge_kind_ids::int2[])) +), shortest as materialized ( + select depth from search + where node_id = @end_id and depth >= @min_depth + order by depth limit 1 +)` +} + +// shortestCanonicalWitnessSearch returns SQL that reconstructs one deterministic witness from compact predecessor state. +func shortestCanonicalWitnessSearch(reverseForPublicPath bool) string { + edgeIDs := "witness.edge_ids" + if reverseForPublicPath { + edgeIDs = `(select coalesce(array_agg(reversed.edge_id order by reversed.ordinal desc), array[]::int8[]) + from unnest(witness.edge_ids) with ordinality reversed(edge_id, ordinal))` + } + return `with recursive distance(node_id, depth) as ( + select @search_start_id::int8, 0 + union + select e.end_id, distance.depth + 1 + from distance + join edge e on e.graph_id = @graph_id and e.start_id = distance.node_id + where distance.depth < @max_depth + and (cardinality(@edge_kind_ids::int2[]) = 0 or e.kind_id = any(@edge_kind_ids::int2[])) +), target as materialized ( + select depth from distance + where node_id = @search_end_id and depth >= @min_depth + order by depth limit 1 +), witness(node_id, depth, edge_ids) as ( + select @search_end_id::int8, target.depth, array[]::int8[] from target + union all + select predecessor.node_id, witness.depth - 1, array[predecessor.edge_id]::int8[] || witness.edge_ids + from witness + join lateral ( + select prior.node_id, e.id as edge_id + from distance prior + join edge e on e.graph_id = @graph_id and e.start_id = prior.node_id and e.end_id = witness.node_id + where prior.depth = witness.depth - 1 + and (cardinality(@edge_kind_ids::int2[]) = 0 or e.kind_id = any(@edge_kind_ids::int2[])) + order by e.id, prior.node_id limit 1 + ) predecessor on witness.depth > 0 +), shortest as materialized ( + select target.depth, ` + edgeIDs + ` as edge_ids + from witness join target on true where witness.depth = 0 +)` +} + +// shortestBidirectionalCompactReferenceSQL exposes one forced compact kernel at +// the same distance or M0 hydration boundary as its production control. +func shortestBidirectionalCompactReferenceSQL(functionName string, direction graph.Direction, pathObserved bool) string { + inbound := "false" + if direction == graph.DirectionInbound { + inbound = "true" + } + search := `with shortest as materialized ( + select depth, path as edge_ids + from ` + functionName + `( + @graph_id, @start_id, @end_id, @min_depth, @max_depth, + @edge_kind_ids, ` + inbound + `, @state_limit, @frontier_limit, @predecessor_limit + ) +)` + if !pathObserved { + return search + ` select depth from shortest` + } + return search + shortestM0MaterializationSelect(direction) +} + +// buildShortestReferenceSpecs assembles exact shortest-path comparators supported by the workload shape. +func buildShortestReferenceSpecs(testCase ScaleCase, probeParams map[string]any, nodeIDs, edgeIDs []int64, direction graph.Direction) []postgresReferenceSpec { + searchNE := shortestReferenceSearchForDirection(direction) + searchE := shortestEdgeReferenceSearch(direction) + fullSQL := shortestDistanceReferenceSearchForDirection(direction) + ` select depth from shortest` + boundary := "distance scalar" + pathObserved := testCase.Name == "one_shortest_path_bound_pair" || testCase.Expected.ResultKind == "path_set" + compactBidirectionalParams := copyReferenceParams(probeParams) + compactBidirectionalParams["state_limit"] = int64(100_000) + compactBidirectionalParams["frontier_limit"] = int64(100_000) + compactBidirectionalParams["predecessor_limit"] = int64(100_000) + if pathObserved { + fullSQL = searchNE + ` +select ordered_edge_ids_to_path( + @graph_id, + (root.id, root.kind_ids, root.properties)::nodeComposite, + shortest.edge_ids, + array[(root.id, root.kind_ids, root.properties)::nodeComposite]::nodeComposite[] +)::pathComposite +from shortest join node root on root.graph_id = @graph_id and root.id = @start_id` + boundary = "complete path composite" + } + hydrationParams := copyReferenceParams(probeParams) + hydrationParams["node_ids"] = nodeIDs + hydrationParams["edge_ids"] = edgeIDs + hydrationSQL := `select ordered_edge_ids_to_path( + @graph_id, + (root.id, root.kind_ids, root.properties)::nodeComposite, + @edge_ids::int8[], + array[(root.id, root.kind_ids, root.properties)::nodeComposite]::nodeComposite[] +)::pathComposite +from node root where root.graph_id = @graph_id and root.id = @start_id` + specs := []postgresReferenceSpec{ + { + name: "round_trip", + boundary: "prepared protocol and transaction", + sql: `select 1`, + parameters: nil, + }, + { + name: "endpoint_validation", + boundary: "validated endpoint IDs", + sql: `select id from node where graph_id = @graph_id and id = any(array[@start_id::int8, @end_id::int8]) order by id`, + parameters: probeParams, + }, + { + name: "minimum_graph_access", + boundary: "root adjacency edge IDs", + sql: `select e.id from edge e where e.graph_id = @graph_id and e.start_id = @start_id and (cardinality(@edge_kind_ids::int2[]) = 0 or e.kind_id = any(@edge_kind_ids::int2[])) order by e.id`, + parameters: probeParams, + }, + { + name: "search_ordered_ids", + architecture: "SP-S3-U-NE", + observationShape: "ordered_ids", + stateShape: "ordered node and edge ID arrays", + boundary: "depth plus ordered node/edge IDs", + sql: searchNE + ` select depth, node_ids, edge_ids from shortest`, + parameters: probeParams, + }, + } + if edgeIDs != nil { + specs = append(specs, postgresReferenceSpec{ + name: "hydration_only", + boundary: "complete path composite from precomputed ordered edge IDs", + sql: hydrationSQL, + parameters: hydrationParams, + }) + if pathObserved && direction != graph.DirectionBoth { + specs = append(specs, + postgresReferenceSpec{ + name: "m0_directed_hydration_only", + architecture: "MAT-M0", + implementationID: "directed_set_hydration_" + strings.ToLower(direction.String()) + "_v1", + stateShape: "precomputed ordered edge IDs; node order derived from directed edge endpoints", + observationShape: "complete path composite", + semanticValidation: "precomputed_exact_path_inputs", + boundary: "directed complete path composite from precomputed ordered edge IDs", + sql: shortestM0HydrationSQL(direction), + parameters: hydrationParams, + validationSQL: hydrationSQL, + validationParams: hydrationParams, + }, + postgresReferenceSpec{ + name: "m1_ordered_ids_hydration_only", + architecture: "MAT-M1", + implementationID: "ordered_ids_set_hydration_v1", + stateShape: "precomputed ordered node and edge IDs", + observationShape: "complete path composite", + semanticValidation: "precomputed_exact_path_inputs", + boundary: "complete path composite from precomputed ordered node and edge IDs", + sql: shortestM1HydrationSQL(), + parameters: hydrationParams, + validationSQL: hydrationSQL, + validationParams: hydrationParams, + }, + ) + } + } + specs = append(specs, postgresReferenceSpec{ + name: "s3_unidirectional_trail_cte", + legacyName: "complete_reference_s1_array_cte", + architecture: shortestArchitectureForCase(testCase), + implementationID: "inline_recursive_cte_unidirectional_v3", + stateShape: shortestS3UStateShape(testCase), + observationShape: observationShapeForCase(testCase), + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: fullSQL, + parameters: probeParams, + }) + if !pathObserved && direction == graph.DirectionInbound { + canonicalParams := copyReferenceParams(probeParams) + canonicalParams["start_id"], canonicalParams["end_id"] = probeParams["end_id"], probeParams["start_id"] + specs = append(specs, postgresReferenceSpec{ + name: "s4_canonical_source_distance", + architecture: "SP-I1-C-D", + implementationID: "canonical_relationship_source_distance_v1", + stateShape: "relationship-source-oriented node and depth set state", + observationShape: "distance scalar", + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: shortestDistanceReferenceSearchForDirection(graph.DirectionOutbound) + ` select depth from shortest`, + parameters: canonicalParams, + }) + } + if shortestS1DistanceEligible(testCase, probeParams, direction, pathObserved) { + s1Params := copyReferenceParams(probeParams) + s1Params["state_limit"] = int32(100_000) + specs = append(specs, postgresReferenceSpec{ + name: "s1_array_bfs_distance", + architecture: "SP-S1", + implementationID: "typed_plpgsql_array_bfs_distance_v1", + stateShape: "array-resident frontier and visited node IDs with explicit state ceiling; no path or predecessor state", + observationShape: "distance scalar", + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: shortestS1DistanceSQL(fullSQL, direction), + parameters: s1Params, + }) + } + if pathObserved && direction != graph.DirectionBoth { + specs = append(specs, + postgresReferenceSpec{ + name: "s3_unidirectional_cte_m0_directed", + architecture: "SP-S3-U-E+MAT-M0", + implementationID: "s3_u_edge_search_directed_set_materializer_" + strings.ToLower(direction.String()) + "_v1", + stateShape: "edge-only recursive trail; materializer derives node order from directed edge endpoints", + observationShape: "public_observation", + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: shortestM0FullSQL(searchE, direction), + parameters: probeParams, + }, + postgresReferenceSpec{ + name: "s3_unidirectional_cte_m1_ordered_ids", + architecture: "SP-S3-U-NE+MAT-M1", + implementationID: "s3_u_node_edge_search_ordered_ids_set_materializer_v1", + stateShape: "ordered node-and-edge recursive trails; materializer hydrates both streams by ordinal", + observationShape: "public_observation", + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: shortestM1FullSQL(searchNE), + parameters: probeParams, + }, + ) + + witnessParams := copyReferenceParams(probeParams) + witnessParams["search_start_id"], witnessParams["search_end_id"] = probeParams["start_id"], probeParams["end_id"] + reverseForPublicPath := false + if direction == graph.DirectionInbound { + witnessParams["search_start_id"], witnessParams["search_end_id"] = probeParams["end_id"], probeParams["start_id"] + reverseForPublicPath = true + } + witnessSearch := shortestCanonicalWitnessSearch(reverseForPublicPath) + specs = append(specs, postgresReferenceSpec{ + name: "s4_canonical_source_witness_m0", + architecture: "SP-I1-C-WE+MAT-M0", + implementationID: "canonical_source_compact_witness_m0_v1", + stateShape: "node/depth discovery plus one deterministic predecessor per witness depth; no recursive full trails", + observationShape: "public_observation", + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: shortestM0FullSQL(witnessSearch, direction), + parameters: witnessParams, + }) + } + if direction != graph.DirectionBoth { + if pathObserved { + specs = append(specs, + postgresReferenceSpec{ + name: "sp_b1_strict_alternating_witness_m0", + architecture: "SP-B1-C-ALT-NODE-WE+MAT-M0", + implementationID: "typed_bidirectional_strict_alternating_node_witness_m0_v1", + stateShape: "ID-only per-side FIFO, minimum-depth seen state, and one deterministic predecessor per accepted node", + observationShape: "public_observation", + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: shortestBidirectionalCompactReferenceSQL("shortest_path_b1_strict_alternating", direction, true), + parameters: compactBidirectionalParams, + }, + postgresReferenceSpec{ + name: "sp_b2_smaller_frontier_witness_m0", + architecture: "SP-B2-C-MIN-LEVEL-WE+MAT-M0", + implementationID: "typed_bidirectional_smaller_current_level_witness_m0_v1", + stateShape: "ID-only per-side complete levels, minimum-depth seen state, and one deterministic predecessor per accepted node", + observationShape: "public_observation", + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: shortestBidirectionalCompactReferenceSQL("shortest_path_b2_smaller_current_level", direction, true), + parameters: compactBidirectionalParams, + }, + ) + } else { + specs = append(specs, + postgresReferenceSpec{ + name: "sp_b1_strict_alternating_distance", + architecture: "SP-B1-C-ALT-NODE-D", + implementationID: "typed_bidirectional_strict_alternating_node_distance_v1", + stateShape: "ID-only per-side FIFO and minimum-depth seen state; witness predecessor retained outside the observation boundary", + observationShape: "distance scalar", + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: shortestBidirectionalCompactReferenceSQL("shortest_path_b1_strict_alternating", direction, false), + parameters: compactBidirectionalParams, + }, + postgresReferenceSpec{ + name: "sp_b2_smaller_frontier_distance", + architecture: "SP-B2-C-MIN-LEVEL-D", + implementationID: "typed_bidirectional_smaller_current_level_distance_v1", + stateShape: "ID-only per-side complete levels and minimum-depth seen state; witness predecessor retained outside the observation boundary", + observationShape: "distance scalar", + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: shortestBidirectionalCompactReferenceSQL("shortest_path_b2_smaller_current_level", direction, false), + parameters: compactBidirectionalParams, + }, + ) + } + } + specs = append(specs, postgresReferenceSpec{ + name: "s3_bidirectional_trail_cte", + legacyName: "candidate_s2_bidirectional_cte", + architecture: "SP-S3-B", + implementationID: "inline_recursive_cte_bidirectional_trails_v2", + stateShape: "paired per-row relationship trail arrays", + observationShape: observationShapeForCase(testCase), + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: shortestBidirectionalReferenceSQL(testCase, direction), + parameters: probeParams, + }) + return specs +} + +// shortestS1DistanceEligible reports whether a case can use the bounded single-direction distance prototype. +func shortestS1DistanceEligible(testCase ScaleCase, parameters map[string]any, direction graph.Direction, pathObserved bool) bool { + if pathObserved || direction == graph.DirectionBoth { + return false + } + minDepth := 1 + if testCase.Shape.MinDepth != nil { + minDepth = *testCase.Shape.MinDepth + } + return minDepth <= 1 && !reflect.DeepEqual(parameters["start_id"], parameters["end_id"]) +} + +// shortestS1DistanceSQL wraps a shortest-path query with the bounded S1 distance prototype. +func shortestS1DistanceSQL(fallbackSQL string, direction graph.Direction) string { + inbound := "false" + if direction == graph.DirectionInbound { + inbound = "true" + } + return `with s1 as materialized ( + select * from graphbench_s1_distance_bfs( + @graph_id, @start_id, @end_id, @min_depth, @max_depth, + @edge_kind_ids, ` + inbound + `, @state_limit + ) +) +select depth from s1 where matched +union all +select fallback.depth from (` + fallbackSQL + `) fallback +where (select overflow from s1) +limit 1` +} + +// shortestArchitectureForCase chooses the witness-producing or distance-only S3 reference architecture from the case's observable result contract. +func shortestArchitectureForCase(testCase ScaleCase) string { + if testCase.Expected.ResultKind == "path_set" || testCase.Name == "one_shortest_path_bound_pair" { + return "SP-S3-U-NE" + } + return "SP-S3-U-D" +} + +// shortestM0HydrationSQL returns SQL that hydrates paths from ordered relationship IDs. +func shortestM0HydrationSQL(direction graph.Direction) string { + return `with shortest(edge_ids) as (select @edge_ids::int8[])` + shortestM0MaterializationSelect(direction) +} + +// shortestM0FullSQL combines edge-only search with M0 path hydration. +func shortestM0FullSQL(search string, direction graph.Direction) string { + return search + shortestM0MaterializationSelect(direction) +} + +// shortestM0MaterializationSelect is intentionally outbound-only. The S3-U +// reference search emits an ordered, graph-scoped outbound edge stream, so M0 +// can derive each next node directly from edge.end_id without recursively +// rediscovering connectivity. +func shortestM0MaterializationSelect(direction graph.Direction) string { + nextNode := "edge.end_id" + if direction == graph.DirectionInbound { + nextNode = "edge.start_id" + } + return ` +select row( + array[(root.id, root.kind_ids, root.properties)::nodeComposite]::nodeComposite[] || + coalesce(hydrated.nodes, array[]::nodeComposite[]), + coalesce(hydrated.edges, array[]::edgeComposite[]) +)::pathComposite +from shortest +join node root on root.graph_id = @graph_id and root.id = @start_id +cross join lateral ( + select + array_agg((terminal.id, terminal.kind_ids, terminal.properties)::nodeComposite order by path_edge.ordinality)::nodeComposite[] as nodes, + array_agg((edge.id, edge.start_id, edge.end_id, edge.kind_id, edge.properties)::edgeComposite order by path_edge.ordinality)::edgeComposite[] as edges, + count(*) as hydrated_count + from unnest(shortest.edge_ids) with ordinality as path_edge(id, ordinality) + join edge on edge.graph_id = @graph_id and edge.id = path_edge.id + join node terminal on terminal.graph_id = @graph_id and terminal.id = ` + nextNode + ` +) hydrated +where hydrated.hydrated_count = cardinality(shortest.edge_ids)` +} + +// shortestM1HydrationSQL returns SQL that hydrates paths from ordered node and relationship IDs. +func shortestM1HydrationSQL() string { + return `with shortest(node_ids, edge_ids) as (select @node_ids::int8[], @edge_ids::int8[])` + shortestM1MaterializationSelect() +} + +// shortestM1FullSQL combines node-and-edge search with M1 path hydration. +func shortestM1FullSQL(search string) string { + return search + shortestM1MaterializationSelect() +} + +// shortestM1MaterializationSelect hydrates the ordered node and edge streams +// independently and restores public path order with ordinality. M0 and M1 use +// the same S3-U search in full-comparator measurements so their delta isolates +// materialization rather than search state generation. +func shortestM1MaterializationSelect() string { + return ` +select row( + coalesce(hydrated_nodes.nodes, array[]::nodeComposite[]), + coalesce(hydrated_edges.edges, array[]::edgeComposite[]) +)::pathComposite +from shortest +cross join lateral ( + select + array_agg((node.id, node.kind_ids, node.properties)::nodeComposite order by path_node.ordinality)::nodeComposite[] as nodes, + count(*) as hydrated_count + from unnest(shortest.node_ids) with ordinality as path_node(id, ordinality) + join node on node.graph_id = @graph_id and node.id = path_node.id +) hydrated_nodes +cross join lateral ( + select + array_agg((edge.id, edge.start_id, edge.end_id, edge.kind_id, edge.properties)::edgeComposite order by path_edge.ordinality)::edgeComposite[] as edges, + count(*) as hydrated_count + from unnest(shortest.edge_ids) with ordinality as path_edge(id, ordinality) + join edge on edge.graph_id = @graph_id and edge.id = path_edge.id +) hydrated_edges +where cardinality(shortest.node_ids) = cardinality(shortest.edge_ids) + 1 + and hydrated_nodes.hydrated_count = cardinality(shortest.node_ids) + and hydrated_edges.hydrated_count = cardinality(shortest.edge_ids)` +} + +// shortestS3UStateShape describes recursive state retained by the selected unidirectional search projection. +func shortestS3UStateShape(testCase ScaleCase) string { + if testCase.Expected.ResultKind == "path_set" || testCase.Name == "one_shortest_path_bound_pair" { + return "per-row node and relationship trail arrays" + } + return "distance frontier node and depth only; no path or predecessor state" +} + +// shortestBidirectionalReferenceSQL returns the bidirectional shortest-path reference query for the requested result shape. +func shortestBidirectionalReferenceSQL(testCase ScaleCase, direction graph.Direction) string { + forwardJoin, forwardNext := "e.start_id = forward.node_id", "e.end_id" + backwardJoin, backwardNext := "e.end_id = backward.node_id", "e.start_id" + if direction == graph.DirectionInbound { + forwardJoin, forwardNext = "e.end_id = forward.node_id", "e.start_id" + backwardJoin, backwardNext = "e.start_id = backward.node_id", "e.end_id" + } + search := `with recursive +forward(node_id, depth, edge_ids) as ( + select @start_id::int8, 0, array[]::int8[] + union all + select ` + forwardNext + `, forward.depth + 1, forward.edge_ids || e.id + from forward join edge e on e.graph_id = @graph_id and ` + forwardJoin + ` + where forward.depth < @max_depth + and (cardinality(@edge_kind_ids::int2[]) = 0 or e.kind_id = any(@edge_kind_ids::int2[])) + and e.id != all(forward.edge_ids) +), backward(node_id, depth, edge_ids) as ( + select @end_id::int8, 0, array[]::int8[] + union all + select ` + backwardNext + `, backward.depth + 1, e.id || backward.edge_ids + from backward join edge e on e.graph_id = @graph_id and ` + backwardJoin + ` + where backward.depth < @max_depth + and (cardinality(@edge_kind_ids::int2[]) = 0 or e.kind_id = any(@edge_kind_ids::int2[])) + and e.id != all(backward.edge_ids) +), shortest as materialized ( + select forward.depth + backward.depth as depth, forward.edge_ids || backward.edge_ids as edge_ids + from forward join backward using (node_id) + where forward.depth + backward.depth between @min_depth and @max_depth + and not exists (select 1 from unnest(forward.edge_ids) edge_id where edge_id = any(backward.edge_ids)) + order by depth, edge_ids limit 1 +)` + if testCase.Expected.ResultKind != "path_set" { + return search + ` select depth from shortest` + } + return search + ` +select ordered_edge_ids_to_path( + @graph_id, + (root.id, root.kind_ids, root.properties)::nodeComposite, + shortest.edge_ids, + array[(root.id, root.kind_ids, root.properties)::nodeComposite]::nodeComposite[] +)::pathComposite +from shortest join node root on root.graph_id = @graph_id and root.id = @start_id` +} + +// fixedSuffixExpansionReferenceSpecs builds exact reference implementations for fixed-suffix expansion cases. +func (s *postgresSQLRunner) fixedSuffixExpansionReferenceSpecs(ctx context.Context, testCase ScaleCase, params map[string]any) ([]postgresReferenceSpec, error) { + kindNames := []string{"ExpansionRoot", "SuffixHead", "SuffixMiddle", "SuffixTerminal", "Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"} + probeParams := copyReferenceParams(params) + probeParams["graph_id"] = s.graphID + for _, name := range kindNames { + kindID, err := s.pgDriver.KindMapper().MapKind(ctx, graph.StringKind(name)) + if err != nil { + return nil, fmt.Errorf("map reference kind %s: %w", name, err) + } + probeParams[name+"_kind"] = kindID + } + probeParams["min_depth"] = int32(0) + if testCase.Shape.MinDepth != nil { + probeParams["min_depth"] = int32(*testCase.Shape.MinDepth) + } + probeParams["max_depth"] = int32(15) + if testCase.Shape.MaxDepth != nil { + probeParams["max_depth"] = int32(*testCase.Shape.MaxDepth) + } + specs := buildFixedSuffixExpansionReferenceSpecs(testCase, probeParams) + if !referenceHydrationRequested(s.referenceArms) { + return specs, nil + } + searchIdx := referenceSpecIndex(specs, "suffix_seeded_reverse_ordered_ids") + values, err := readReferenceRow(ctx, s.db, specs[searchIdx].sql, specs[searchIdx].parameters, s.readTransactionOptions()...) + if err != nil { + return nil, fmt.Errorf("precompute fixed-suffix expansion hydration IDs: %w", err) + } + if len(values) == 0 { + completeIdx := referenceSpecIndex(specs, "complete_reference") + specs = slices.Insert(specs, completeIdx, postgresReferenceSpec{ + name: "hydration_only", + architecture: "hydration", + implementationID: "typed_empty_v1", + stateShape: "empty ordered ID input", + observationShape: "typed empty path result", + semanticValidation: "not_applicable_empty_input", + boundary: "typed empty path result", + sql: `select null::pathComposite where false`, + parameters: probeParams, + }) + return specs, nil + } + if len(values) != 3 { + return nil, fmt.Errorf("precompute fixed-suffix expansion hydration IDs returned %d columns, expected 3", len(values)) + } + nodeIDs, err := referenceInt64Slice(values[0]) + if err != nil || len(nodeIDs) == 0 { + return nil, fmt.Errorf("decode fixed-suffix expansion hydration node IDs: %w", err) + } + edgeIDs, err := referenceInt64Slice(values[2]) + if err != nil { + return nil, fmt.Errorf("decode fixed-suffix expansion hydration edge IDs: %w", err) + } + hydrationParams := copyReferenceParams(probeParams) + hydrationParams["root_id"] = nodeIDs[0] + hydrationParams["edge_ids"] = edgeIDs + hydration := postgresReferenceSpec{ + name: "hydration_only", + boundary: "one complete path composite from precomputed ordered edge IDs", + sql: `select ordered_edge_ids_to_path( + @graph_id, + (root.id, root.kind_ids, root.properties)::nodeComposite, + @edge_ids::int8[], + array[(root.id, root.kind_ids, root.properties)::nodeComposite]::nodeComposite[] +)::pathComposite +from node root where root.graph_id = @graph_id and root.id = @root_id`, + parameters: hydrationParams, + } + completeIdx := referenceSpecIndex(specs, "complete_reference") + specs = slices.Insert(specs, completeIdx, hydration) + return specs, nil +} + +// referenceHydrationRequested reports whether the selected arm requires precomputed hydration inputs. +func referenceHydrationRequested(referenceArms []string) bool { + return len(referenceArms) == 0 || slices.Contains(referenceArms, "hydration_only") +} + +// buildFixedSuffixExpansionReferenceSpecs assembles fixed-suffix search and hydration references for one case. +func buildFixedSuffixExpansionReferenceSpecs(testCase ScaleCase, probeParams map[string]any) []postgresReferenceSpec { + roots := `roots(root_id) as materialized ( + select n.id from node n + where n.graph_id = @graph_id + and @ExpansionRoot_kind::int2 = any(n.kind_ids) + and n.properties ->> 'root_key' = @root_key +)` + suffix := `suffix_rows(boundary_id, head_id, terminal_id, suffix_edge_ids, suffix_node_ids) as materialized ( + select boundary.id, suffix_head.id, suffix_terminal.id, + array[enter_suffix.id, continue_suffix.id, complete_suffix.id]::int8[], + array[boundary.id, suffix_head.id, suffix_middle.id, suffix_terminal.id]::int8[] + from (select 1 from roots limit 1) root_presence + cross join edge enter_suffix + join node boundary on boundary.graph_id = @graph_id and boundary.id = enter_suffix.start_id + join node suffix_head on suffix_head.graph_id = @graph_id and suffix_head.id = enter_suffix.end_id and @SuffixHead_kind::int2 = any(suffix_head.kind_ids) + join edge continue_suffix on continue_suffix.graph_id = @graph_id and continue_suffix.start_id = suffix_head.id and continue_suffix.kind_id = @ContinueSuffix_kind + join node suffix_middle on suffix_middle.graph_id = @graph_id and suffix_middle.id = continue_suffix.end_id and @SuffixMiddle_kind::int2 = any(suffix_middle.kind_ids) + join edge complete_suffix on complete_suffix.graph_id = @graph_id and complete_suffix.start_id = suffix_middle.id and complete_suffix.kind_id = @CompleteSuffix_kind + join node suffix_terminal on suffix_terminal.graph_id = @graph_id and suffix_terminal.id = complete_suffix.end_id and @SuffixTerminal_kind::int2 = any(suffix_terminal.kind_ids) + where enter_suffix.graph_id = @graph_id and enter_suffix.kind_id = @EnterSuffix_kind + and continue_suffix.id <> enter_suffix.id + and complete_suffix.id <> enter_suffix.id and complete_suffix.id <> continue_suffix.id +)` + forwardExpansion := `expansion_paths(root_id, node_id, node_ids, edge_ids, depth) as ( + select root_id, root_id, array[root_id]::int8[], array[]::int8[], 0 from roots + union all + select expansion_paths.root_id, e.end_id, expansion_paths.node_ids || e.end_id, expansion_paths.edge_ids || e.id, expansion_paths.depth + 1 + from expansion_paths join edge e + on e.graph_id = @graph_id and e.start_id = expansion_paths.node_id and e.kind_id = @Expand_kind + join node next_node on next_node.graph_id = @graph_id and next_node.id = e.end_id + where expansion_paths.depth < @max_depth and e.id != all(expansion_paths.edge_ids) +)` + scalarForwardExpansion := strings.Replace(forwardExpansion, "\n join node next_node on next_node.graph_id = @graph_id and next_node.id = e.end_id", "", 1) + allExpansionNodesExist := `not exists ( + select 1 from unnest(expansion_paths.node_ids) as expansion_node_id(id) + left join node expansion_node on expansion_node.graph_id = @graph_id and expansion_node.id = expansion_node_id.id + where expansion_node.id is null + )` + legacyForward := `with recursive ` + roots + `, ` + forwardExpansion + `, paths as materialized ( + select expansion_paths.node_ids || array[suffix_head.id, suffix_middle.id, suffix_terminal.id]::int8[] as node_ids, + suffix_head.id as head_id, suffix_terminal.id as terminal_id, + expansion_paths.edge_ids || enter_suffix.id || continue_suffix.id || complete_suffix.id as edge_ids + from expansion_paths + join edge enter_suffix on enter_suffix.graph_id = @graph_id and enter_suffix.start_id = expansion_paths.node_id and enter_suffix.kind_id = @EnterSuffix_kind and enter_suffix.id != all(expansion_paths.edge_ids) + join node suffix_head on suffix_head.graph_id = @graph_id and suffix_head.id = enter_suffix.end_id and @SuffixHead_kind::int2 = any(suffix_head.kind_ids) + join edge continue_suffix on continue_suffix.graph_id = @graph_id and continue_suffix.start_id = suffix_head.id and continue_suffix.kind_id = @ContinueSuffix_kind + and continue_suffix.id != enter_suffix.id and continue_suffix.id != all(expansion_paths.edge_ids) + join node suffix_middle on suffix_middle.graph_id = @graph_id and suffix_middle.id = continue_suffix.end_id and @SuffixMiddle_kind::int2 = any(suffix_middle.kind_ids) + join edge complete_suffix on complete_suffix.graph_id = @graph_id and complete_suffix.start_id = suffix_middle.id and complete_suffix.kind_id = @CompleteSuffix_kind + and complete_suffix.id != enter_suffix.id and complete_suffix.id != continue_suffix.id and complete_suffix.id != all(expansion_paths.edge_ids) + join node suffix_terminal on suffix_terminal.graph_id = @graph_id and suffix_terminal.id = complete_suffix.end_id and @SuffixTerminal_kind::int2 = any(suffix_terminal.kind_ids) + where expansion_paths.depth >= @min_depth +)` + lateHydratedForward := `with recursive ` + roots + `, ` + scalarForwardExpansion + `, paths as materialized ( + select expansion_paths.node_ids || array[suffix_head.id, suffix_middle.id, suffix_terminal.id]::int8[] as node_ids, + suffix_head.id as head_id, suffix_terminal.id as terminal_id, + expansion_paths.edge_ids || enter_suffix.id || continue_suffix.id || complete_suffix.id as edge_ids + from expansion_paths + join edge enter_suffix on enter_suffix.graph_id = @graph_id and enter_suffix.start_id = expansion_paths.node_id and enter_suffix.kind_id = @EnterSuffix_kind and enter_suffix.id != all(expansion_paths.edge_ids) + join node suffix_head on suffix_head.graph_id = @graph_id and suffix_head.id = enter_suffix.end_id and @SuffixHead_kind::int2 = any(suffix_head.kind_ids) + join edge continue_suffix on continue_suffix.graph_id = @graph_id and continue_suffix.start_id = suffix_head.id and continue_suffix.kind_id = @ContinueSuffix_kind + and continue_suffix.id != enter_suffix.id and continue_suffix.id != all(expansion_paths.edge_ids) + join node suffix_middle on suffix_middle.graph_id = @graph_id and suffix_middle.id = continue_suffix.end_id and @SuffixMiddle_kind::int2 = any(suffix_middle.kind_ids) + join edge complete_suffix on complete_suffix.graph_id = @graph_id and complete_suffix.start_id = suffix_middle.id and complete_suffix.kind_id = @CompleteSuffix_kind + and complete_suffix.id != enter_suffix.id and complete_suffix.id != continue_suffix.id and complete_suffix.id != all(expansion_paths.edge_ids) + join node suffix_terminal on suffix_terminal.graph_id = @graph_id and suffix_terminal.id = complete_suffix.end_id and @SuffixTerminal_kind::int2 = any(suffix_terminal.kind_ids) + where expansion_paths.depth >= @min_depth and ` + allExpansionNodesExist + ` +)` + factoredForward := `with recursive ` + roots + `, ` + suffix + `, ` + scalarForwardExpansion + `, paths as materialized ( + select expansion_paths.node_ids || suffix_rows.suffix_node_ids[2:4] as node_ids, + suffix_rows.head_id, suffix_rows.terminal_id, + expansion_paths.edge_ids || suffix_rows.suffix_edge_ids as edge_ids + from expansion_paths join suffix_rows on suffix_rows.boundary_id = expansion_paths.node_id + where expansion_paths.depth >= @min_depth + and not exists (select 1 from unnest(expansion_paths.edge_ids) as expansion_edge(id) where expansion_edge.id = any(suffix_rows.suffix_edge_ids)) + and ` + allExpansionNodesExist + ` +)` + reverse := `with recursive ` + roots + `, ` + suffix + `, boundary_ids(boundary_id) as materialized ( + select distinct boundary_id from suffix_rows +), reverse_trails(boundary_id, node_id, node_ids, edge_ids, depth) as ( + select boundary_id, boundary_id, array[boundary_id]::int8[], array[]::int8[], 0 from boundary_ids + union all + select reverse_trails.boundary_id, e.start_id, array_prepend(e.start_id, reverse_trails.node_ids), + array_prepend(e.id, reverse_trails.edge_ids), reverse_trails.depth + 1 + from reverse_trails join edge e + on e.graph_id = @graph_id and e.end_id = reverse_trails.node_id and e.kind_id = @Expand_kind + where reverse_trails.depth < @max_depth and e.id != all(reverse_trails.edge_ids) +), paths as materialized ( + select reverse_trails.node_ids || suffix_rows.suffix_node_ids[2:4] as node_ids, + suffix_rows.head_id, suffix_rows.terminal_id, + reverse_trails.edge_ids || suffix_rows.suffix_edge_ids as edge_ids + from reverse_trails + join roots on roots.root_id = reverse_trails.node_id + join suffix_rows on suffix_rows.boundary_id = reverse_trails.boundary_id + where reverse_trails.depth >= @min_depth + and not exists (select 1 from unnest(reverse_trails.edge_ids) as expansion_edge(id) where expansion_edge.id = any(suffix_rows.suffix_edge_ids)) + and not exists ( + select 1 from unnest(reverse_trails.node_ids) as expansion_node_id(id) + left join node expansion_node on expansion_node.graph_id = @graph_id and expansion_node.id = expansion_node_id.id + where expansion_node.id is null + ) +)` + viability := `with recursive ` + roots + `, ` + suffix + `, boundary_ids(boundary_id) as materialized ( + select distinct boundary_id from suffix_rows +), viable(node_id, reverse_distance) as ( + select boundary_id, 0 from boundary_ids + union + select e.start_id, viable.reverse_distance + 1 + from viable join edge e + on e.graph_id = @graph_id and e.end_id = viable.node_id and e.kind_id = @Expand_kind + where viable.reverse_distance < @max_depth +), expansion_paths(root_id, node_id, node_ids, edge_ids, depth) as ( + select root_id, root_id, array[root_id]::int8[], array[]::int8[], 0 from roots + where exists (select 1 from viable where viable.node_id = roots.root_id and viable.reverse_distance <= @max_depth) + union all + select expansion_paths.root_id, e.end_id, expansion_paths.node_ids || e.end_id, expansion_paths.edge_ids || e.id, expansion_paths.depth + 1 + from expansion_paths join edge e + on e.graph_id = @graph_id and e.start_id = expansion_paths.node_id and e.kind_id = @Expand_kind + where expansion_paths.depth < @max_depth and e.id != all(expansion_paths.edge_ids) + and exists (select 1 from viable where viable.node_id = e.end_id and viable.reverse_distance <= @max_depth - expansion_paths.depth - 1) +), paths as materialized ( + select expansion_paths.node_ids || suffix_rows.suffix_node_ids[2:4] as node_ids, + suffix_rows.head_id, suffix_rows.terminal_id, + expansion_paths.edge_ids || suffix_rows.suffix_edge_ids as edge_ids + from expansion_paths join suffix_rows on suffix_rows.boundary_id = expansion_paths.node_id + where expansion_paths.depth >= @min_depth + and not exists (select 1 from unnest(expansion_paths.edge_ids) as expansion_edge(id) where expansion_edge.id = any(suffix_rows.suffix_edge_ids)) + and ` + allExpansionNodesExist + ` +)` + fullSQL := legacyForward + ` select head_id, terminal_id from paths` + boundary := "endpoint ID pairs" + pathObserved := testCase.Observes.Paths || testCase.Expected.ResultKind == "path_set" + complete := func(search string) string { + if !pathObserved { + return search + ` select head_id, terminal_id from paths` + } + return search + ` +select ordered_edge_ids_to_path( + @graph_id, + (root.id, root.kind_ids, root.properties)::nodeComposite, + paths.edge_ids, + array[(root.id, root.kind_ids, root.properties)::nodeComposite]::nodeComposite[] +)::pathComposite +from paths join node root on root.graph_id = @graph_id and root.id = paths.node_ids[1]` + } + if pathObserved { + fullSQL = complete(legacyForward) + boundary = "complete path composite" + } + orderedLegacy := legacyForward + ` select node_ids, head_id, edge_ids from paths` + orderedReference := func(spec postgresReferenceSpec) postgresReferenceSpec { + spec.semanticValidation = "exact_ordered_ids" + spec.validationSQL = orderedLegacy + spec.validationParams = probeParams + return spec + } + return []postgresReferenceSpec{ + { + name: "round_trip", + architecture: "protocol", + stateShape: "none", + boundary: "prepared protocol and transaction", + sql: `select 1`, + }, + { + name: "endpoint_validation", + architecture: "root_validation", + stateShape: "root ID bag", + boundary: "validated root ID", + sql: `select n.id from node n where n.graph_id = @graph_id and @ExpansionRoot_kind::int2 = any(n.kind_ids) and n.properties ->> 'root_key' = @root_key`, + parameters: probeParams, + }, + { + name: "fixed_suffix_rows", + architecture: "factored_suffix", + stateShape: "boundary and ordered suffix IDs", + boundary: "exact suffix rows and distinct boundary IDs", + sql: `with ` + roots + `, ` + suffix + ` select boundary_id, head_id, terminal_id, suffix_edge_ids from suffix_rows`, + parameters: probeParams, + }, + { + name: "minimum_graph_access", + architecture: "root_adjacency", + stateShape: "edge IDs", + boundary: "root adjacency edge IDs", + sql: `with ` + roots + ` select e.id from roots join edge e on e.graph_id = @graph_id and e.start_id = roots.root_id and e.kind_id = @Expand_kind order by e.id`, + parameters: probeParams, + }, + orderedReference(postgresReferenceSpec{ + name: "search_ordered_ids", + architecture: "EXPANSION-STEPWISE-FORWARD-SQL", + observationShape: "ordered_ids", + stateShape: "root/boundary IDs and ordered relationship trail", + boundary: "ordered node/edge IDs without hydration", + sql: orderedLegacy, + parameters: probeParams, + }), + orderedReference(postgresReferenceSpec{ + name: "stepwise_forward_aa_ordered_ids", + architecture: "EXPANSION-STEPWISE-FORWARD-AA", + aaAliasOf: "search_ordered_ids", + observationShape: "ordered_ids", + stateShape: "root/boundary IDs and ordered relationship trail", + boundary: "ordered node/edge IDs", + sql: orderedLegacy, + parameters: probeParams, + }), + orderedReference(postgresReferenceSpec{ + name: "root_reuse_ordered_ids", + architecture: "EXPANSION-STEPWISE-FORWARD-AA", + aaAliasOf: "search_ordered_ids", + observationShape: "ordered_ids", + stateShape: "root/boundary IDs and ordered relationship trail", + boundary: "ordered node/edge IDs", + sql: orderedLegacy, + parameters: probeParams, + }), + orderedReference(postgresReferenceSpec{ + name: "late_hydration_ordered_ids", + architecture: "EXPANSION-LATE-HYDRATED-FORWARD", + observationShape: "ordered_ids", + stateShape: "scalar expansion state and ordered relationship trail", + boundary: "ordered node/edge IDs", + sql: lateHydratedForward + ` select node_ids, head_id, edge_ids from paths`, + parameters: probeParams, + }), + orderedReference(postgresReferenceSpec{ + name: "factored_suffix_forward_ordered_ids", + architecture: "EXPANSION-FACTORED-SUFFIX-FORWARD", + observationShape: "ordered_ids", + stateShape: "scalar forward trails joined to exact suffix bag", + boundary: "ordered node/edge IDs", + sql: factoredForward + ` select node_ids, head_id, edge_ids from paths`, + parameters: probeParams, + }), + orderedReference(postgresReferenceSpec{ + name: "suffix_seeded_reverse_ordered_ids", + architecture: "EXPANSION-SUFFIX-SEEDED-REVERSE", + observationShape: "ordered_ids", + stateShape: "scalar reverse trails with prepended relationship IDs", + boundary: "ordered node/edge IDs", + sql: reverse + ` select node_ids, head_id, edge_ids from paths`, + parameters: probeParams, + }), + orderedReference(postgresReferenceSpec{ + name: "backward_viability_forward_ordered_ids", + architecture: "EXPANSION-BACKWARD-VIABILITY-FORWARD", + observationShape: "ordered_ids", + stateShape: "depth-aware viability filter plus exact forward trails", + boundary: "ordered node/edge IDs", + sql: viability + ` select node_ids, head_id, edge_ids from paths`, + parameters: probeParams, + }), + { + name: "complete_reference", + architecture: "EXPANSION-STEPWISE-FORWARD-SQL", + stateShape: "forward relationship trails", + observationShape: observationShapeForCase(testCase), + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: fullSQL, + parameters: probeParams, + }, + { + name: "root_reuse_complete", + architecture: "EXPANSION-STEPWISE-FORWARD-AA", + aaAliasOf: "complete_reference", + stateShape: "forward relationship trails", + observationShape: observationShapeForCase(testCase), + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: complete(legacyForward), + parameters: probeParams, + }, + { + name: "late_hydration_complete", + architecture: "EXPANSION-LATE-HYDRATED-FORWARD", + stateShape: "scalar expansion state with final-only hydration", + observationShape: observationShapeForCase(testCase), + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: complete(lateHydratedForward), + parameters: probeParams, + }, + { + name: "factored_suffix_forward_complete", + architecture: "EXPANSION-FACTORED-SUFFIX-FORWARD", + stateShape: "exact forward trails joined to suffix bag", + observationShape: observationShapeForCase(testCase), + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: complete(factoredForward), + parameters: probeParams, + }, + { + name: "suffix_seeded_reverse_complete", + architecture: "EXPANSION-SUFFIX-SEEDED-REVERSE", + stateShape: "exact reverse trails joined back to suffix bag", + observationShape: observationShapeForCase(testCase), + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: complete(reverse), + parameters: probeParams, + }, + { + name: "backward_viability_forward_complete", + architecture: "EXPANSION-BACKWARD-VIABILITY-FORWARD", + stateShape: "permissive viability plus exact forward trails", + observationShape: observationShapeForCase(testCase), + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: complete(viability), + parameters: probeParams, + }, + } +} + +// observationShapeForCase selects full public path observations when the case exposes paths and endpoint IDs otherwise. +func observationShapeForCase(testCase ScaleCase) string { + if testCase.Observes.Paths || testCase.Expected.ResultKind == "path_set" { + return "public_observation" + } + return "endpoint_ids" +} + +// referenceSpecIndex returns a reference arm's index and panics when the arm is absent. +func referenceSpecIndex(specs []postgresReferenceSpec, name string) int { + for idx, spec := range specs { + if spec.name == name { + return idx + } + } + panic("missing PostgreSQL reference spec " + name) +} + +// referenceSpecIndexOrMissing returns a reference arm's index or -1 when absent. +func referenceSpecIndexOrMissing(specs []postgresReferenceSpec, name string) int { + for idx, spec := range specs { + if spec.name == name { + return idx + } + } + return -1 +} + +// readReferenceRow reads reference row and propagates I/O or decoding failures. +func readReferenceRow(ctx context.Context, db graph.Database, sqlQuery string, params map[string]any, transactionOptions ...graph.TransactionOption) ([]any, error) { + var values []any + err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { + result := tx.Raw(sqlQuery, params) + defer result.Close() + if !result.Next() { + if err := result.Error(); err != nil { + return err + } + return nil + } + values = append(values, result.Values()...) + return result.Error() + }, transactionOptions...) + if err != nil { + return nil, err + } + + return values, nil +} + +// referenceInt64Slice normalizes supported driver array representations to []int64. +func referenceInt64Slice(value any) ([]int64, error) { + switch typed := value.(type) { + case []int64: + result := make([]int64, len(typed)) + copy(result, typed) + return result, nil + case []int32: + result := make([]int64, len(typed)) + for idx, item := range typed { + result[idx] = int64(item) + } + return result, nil + case []any: + result := make([]int64, len(typed)) + for idx, item := range typed { + switch integer := item.(type) { + case int64: + result[idx] = integer + case int32: + result[idx] = int64(integer) + default: + return nil, fmt.Errorf("array item %d has type %T", idx, item) + } + } + return result, nil + default: + return nil, fmt.Errorf("expected integer array, got %T", value) + } +} + +// copyReferenceParams duplicates reference params without aliasing mutable state. +func copyReferenceParams(params map[string]any) map[string]any { + copy := make(map[string]any, len(params)+10) + for name, value := range params { + copy[name] = value + } + return copy +} + +// measureRawPostgres executes raw PostgreSQL and records its timing observations. +func measureRawPostgres(ctx context.Context, db graph.Database, sqlQuery string, params map[string]any, warmupIterations, iterations int, transactionOptions ...graph.TransactionOption) (int64, DurationStats, error) { + run := func() (int64, error) { + var count int64 + err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { + result := tx.Raw(sqlQuery, params) + defer result.Close() + for result.Next() { + count++ + _ = result.Values() + } + return result.Error() + }, transactionOptions...) + if err != nil { + return 0, err + } + + return count, nil + } + coldStart := time.Now() + rowCount, err := run() + if err != nil { + return 0, DurationStats{}, err + } + coldDuration := time.Since(coldStart) + for range warmupIterations { + nextCount, err := run() + if err != nil { + return 0, DurationStats{}, err + } + if nextCount != rowCount { + return 0, DurationStats{}, fmt.Errorf("reference row count changed from %d to %d", rowCount, nextCount) + } + } + durations := make([]time.Duration, iterations) + for idx := range iterations { + start := time.Now() + nextCount, err := run() + if err != nil { + return 0, DurationStats{}, err + } + if nextCount != rowCount { + return 0, DurationStats{}, fmt.Errorf("reference row count changed from %d to %d", rowCount, nextCount) + } + durations[idx] = time.Since(start) + } + stats, err := computeDurationStats(durations) + if err != nil { + return 0, DurationStats{}, err + } + stats.WarmupIterations = warmupIterations + stats.Samples = append([]LatencySample{{ + Iteration: 0, + Classification: "cold", + Duration: coldDuration, + }}, stats.Samples...) + return rowCount, stats, nil +} diff --git a/cmd/graphbench/references_expand_into.go b/cmd/graphbench/references_expand_into.go new file mode 100644 index 00000000..77159067 --- /dev/null +++ b/cmd/graphbench/references_expand_into.go @@ -0,0 +1,170 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "fmt" + + "github.com/specterops/dawgs/graph" +) + +const expandIntoInputPairs = `input_pairs(pair_ordinal, start_id, end_id) as materialized ( + select pair_ordinal, start_id, @end_id::int8 + from unnest(@start_ids::int8[]) with ordinality input(start_id, pair_ordinal) +)` + +// expandIntoReferenceSpecs builds three exact one-hop bound-pair arms sharing the same public relationship boundary. +func (s *postgresSQLRunner) expandIntoReferenceSpecs(ctx context.Context, testCase ScaleCase, params map[string]any) ([]postgresReferenceSpec, error) { + probeParams := copyReferenceParams(params) + probeParams["graph_id"] = s.graphID + edgeKinds := make(graph.Kinds, 0, len(testCase.Shape.EdgeKinds)) + for _, name := range testCase.Shape.EdgeKinds { + edgeKinds = append(edgeKinds, graph.StringKind(name)) + } + var edgeKindIDs []int16 + if len(edgeKinds) > 0 { + if s.pgDriver == nil { + return nil, fmt.Errorf("map ExpandInto reference edge kinds: PostgreSQL driver is unavailable") + } + var err error + edgeKindIDs, err = s.pgDriver.KindMapper().MapKinds(ctx, edgeKinds) + if err != nil { + return nil, fmt.Errorf("map ExpandInto reference edge kinds: %w", err) + } + } + probeParams["edge_kind_ids"] = edgeKindIDs + return buildExpandIntoReferenceSpecs(probeParams, testCase.Shape.Direction), nil +} + +// buildExpandIntoReferenceSpecs constructs the exact SQL arms after graph/kind parameters are resolved. +func buildExpandIntoReferenceSpecs(probeParams map[string]any, direction string) []postgresReferenceSpec { + pairJoinPredicate := expandIntoPairPredicate(direction, "matched", "input_pairs") + startDegreePredicate := expandIntoEndpointPredicate(direction, true, "start_adj", "input_pairs") + endDegreePredicate := expandIntoEndpointPredicate(direction, false, "end_adj", "input_pairs") + startScanPredicate := expandIntoEndpointPredicate(direction, true, "outbound", "input_pairs") + endScanPredicate := expandIntoEndpointPredicate(direction, false, "inbound", "input_pairs") + startPairPredicate := expandIntoPairPredicate(direction, "outbound", "input_pairs") + endPairPredicate := expandIntoPairPredicate(direction, "inbound", "input_pairs") + cachePairPredicate := expandIntoPairPredicate(direction, "matched", "distinct_pairs") + + pairJoin := `with ` + expandIntoInputPairs + ` +select (matched.id, matched.start_id, matched.end_id, matched.kind_id, matched.properties)::edgeComposite +from input_pairs +join edge matched on matched.graph_id = @graph_id + and ` + pairJoinPredicate + ` + and (cardinality(@edge_kind_ids::int2[]) = 0 or matched.kind_id = any(@edge_kind_ids::int2[]))` + + lowerDegree := `with ` + expandIntoInputPairs + ` +select (matched.id, matched.start_id, matched.end_id, matched.kind_id, matched.properties)::edgeComposite +from input_pairs +join lateral ( + with degrees as materialized ( + select + (select count(*) from edge start_adj + where start_adj.graph_id = @graph_id and ` + startDegreePredicate + ` + and (cardinality(@edge_kind_ids::int2[]) = 0 or start_adj.kind_id = any(@edge_kind_ids::int2[]))) as start_degree, + (select count(*) from edge end_adj + where end_adj.graph_id = @graph_id and ` + endDegreePredicate + ` + and (cardinality(@edge_kind_ids::int2[]) = 0 or end_adj.kind_id = any(@edge_kind_ids::int2[]))) as end_degree + ) + select candidate.* + from degrees + join lateral ( + select outbound.* from edge outbound + where degrees.start_degree <= degrees.end_degree + and outbound.graph_id = @graph_id and ` + startScanPredicate + ` + and ` + startPairPredicate + ` + and (cardinality(@edge_kind_ids::int2[]) = 0 or outbound.kind_id = any(@edge_kind_ids::int2[])) + union all + select inbound.* from edge inbound + where degrees.end_degree < degrees.start_degree + and inbound.graph_id = @graph_id and ` + endScanPredicate + ` + and ` + endPairPredicate + ` + and (cardinality(@edge_kind_ids::int2[]) = 0 or inbound.kind_id = any(@edge_kind_ids::int2[])) + ) candidate on true +) matched on true` + + pairCache := `with ` + expandIntoInputPairs + `, +distinct_pairs(start_id, end_id) as materialized ( + select distinct start_id, end_id from input_pairs +), pair_matches(start_id, end_id, id, edge_start_id, edge_end_id, kind_id, properties) as materialized ( + select distinct_pairs.start_id, distinct_pairs.end_id, + matched.id, matched.start_id, matched.end_id, matched.kind_id, matched.properties + from distinct_pairs + join edge matched on matched.graph_id = @graph_id + and ` + cachePairPredicate + ` + and (cardinality(@edge_kind_ids::int2[]) = 0 or matched.kind_id = any(@edge_kind_ids::int2[])) +) +select (pair_matches.id, pair_matches.edge_start_id, pair_matches.edge_end_id, pair_matches.kind_id, pair_matches.properties)::edgeComposite +from input_pairs +join pair_matches on pair_matches.start_id = input_pairs.start_id and pair_matches.end_id = input_pairs.end_id` + + return []postgresReferenceSpec{ + { + name: "expand_into_pair_join", architecture: "EXPAND-INTO-PAIR-JOIN", + implementationID: "expand_into_parameterized_pair_join_v2", + stateShape: "outer pair rows joined directly to matching relationships", + observationShape: "complete relationship composites", + semanticValidation: "exact_public_observation", boundary: "complete matching relationships", + fullComparator: true, sql: pairJoin, parameters: probeParams, + }, + { + name: "expand_into_lower_degree_scan", architecture: "EXPAND-INTO-LOWER-DEGREE", + implementationID: "expand_into_typed_lower_degree_scan_v2", + stateShape: "per-pair typed directional degrees plus one disjoint adjacency scan", + observationShape: "complete relationship composites", + semanticValidation: "exact_public_observation", boundary: "complete matching relationships", + fullComparator: true, sql: lowerDegree, parameters: probeParams, + }, + { + name: "expand_into_pair_cache", architecture: "EXPAND-INTO-PAIR-CACHE", + implementationID: "expand_into_distinct_pair_match_cache_v2", + stateShape: "statement-local distinct pair keys and every matching relationship row", + observationShape: "complete relationship composites with duplicate outer-row multiplicity reapplied", + semanticValidation: "exact_public_observation", boundary: "complete matching relationships", + fullComparator: true, sql: pairCache, parameters: probeParams, + }, + } +} + +// expandIntoPairPredicate returns the complete physical edge predicate for one +// logical bound pair. The directionless form uses one OR predicate rather than +// UNION ALL so a self-loop is emitted once, matching Cypher relationship +// multiplicity. +func expandIntoPairPredicate(direction, edgeAlias, pairAlias string) string { + outbound := fmt.Sprintf("%s.start_id = %s.start_id and %s.end_id = %s.end_id", edgeAlias, pairAlias, edgeAlias, pairAlias) + inbound := fmt.Sprintf("%s.end_id = %s.start_id and %s.start_id = %s.end_id", edgeAlias, pairAlias, edgeAlias, pairAlias) + switch direction { + case "inbound": + return inbound + case "directionless": + return "((" + outbound + ") or (" + inbound + "))" + default: + return outbound + } +} + +// expandIntoEndpointPredicate returns the physical adjacency predicate for the +// logical start or end endpoint used by the lower-degree reference arm. +func expandIntoEndpointPredicate(direction string, logicalStart bool, edgeAlias, pairAlias string) string { + pairColumn := "end_id" + if logicalStart { + pairColumn = "start_id" + } + physicalColumn := pairColumn + if direction == "inbound" { + if physicalColumn == "start_id" { + physicalColumn = "end_id" + } else { + physicalColumn = "start_id" + } + } + if direction == "directionless" { + return fmt.Sprintf("(%s.start_id = %s.%s or %s.end_id = %s.%s)", edgeAlias, pairAlias, pairColumn, edgeAlias, pairAlias, pairColumn) + } + return fmt.Sprintf("%s.%s = %s.%s", edgeAlias, physicalColumn, pairAlias, pairColumn) +} diff --git a/cmd/graphbench/references_expand_into_test.go b/cmd/graphbench/references_expand_into_test.go new file mode 100644 index 00000000..e4ab94a8 --- /dev/null +++ b/cmd/graphbench/references_expand_into_test.go @@ -0,0 +1,93 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" + + "github.com/specterops/dawgs/cypher/frontend" + "github.com/stretchr/testify/require" +) + +// TestExpandIntoReferencesShareExactRelationshipBoundary verifies all three study arms preserve rows, cross-kind matches, and duplicate input multiplicity. +func TestExpandIntoReferencesShareExactRelationshipBoundary(t *testing.T) { + params := map[string]any{ + "graph_id": int32(7), "start_ids": []int64{1, 2, 1}, "end_id": int64(3), "edge_kind_ids": []int16{4, 5}, + } + specs := buildExpandIntoReferenceSpecs(params, "outbound") + require.Len(t, specs, 3) + for idx := range specs { + specs[idx] = normalizedReferenceSpec(specs[idx]) + } + require.NoError(t, validateReferenceSpecs(specs)) + + pairJoin := specs[referenceSpecIndex(specs, "expand_into_pair_join")] + require.True(t, pairJoin.fullComparator) + require.Contains(t, pairJoin.sql, "unnest(@start_ids::int8[]) with ordinality") + require.Contains(t, pairJoin.sql, "matched.start_id = input_pairs.start_id and matched.end_id = input_pairs.end_id") + require.Contains(t, pairJoin.sql, "matched.kind_id = any(@edge_kind_ids::int2[])") + + lowerDegree := specs[referenceSpecIndex(specs, "expand_into_lower_degree_scan")] + require.Contains(t, lowerDegree.sql, "degrees as materialized") + require.Contains(t, lowerDegree.sql, "degrees.start_degree <= degrees.end_degree") + require.Contains(t, lowerDegree.sql, "degrees.end_degree < degrees.start_degree") + require.Contains(t, lowerDegree.sql, "union all") + + cache := specs[referenceSpecIndex(specs, "expand_into_pair_cache")] + require.Contains(t, cache.sql, "select distinct start_id, end_id from input_pairs") + require.Contains(t, cache.sql, "pair_matches") + require.Contains(t, cache.observationShape, "duplicate outer-row multiplicity") + for _, spec := range specs { + require.Equal(t, "exact_public_observation", spec.semanticValidation) + require.Equal(t, params, spec.parameters) + } +} + +// TestExpandIntoReferencesPreserveInboundAndDirectionlessPairs verifies every +// study arm uses the same physical pair semantics and does not double-count a +// directionless self-loop. +func TestExpandIntoReferencesPreserveInboundAndDirectionlessPairs(t *testing.T) { + inbound := buildExpandIntoReferenceSpecs(map[string]any{}, "inbound") + require.Contains(t, inbound[0].sql, "matched.end_id = input_pairs.start_id") + require.Contains(t, inbound[1].sql, "outbound.end_id = input_pairs.start_id") + require.Contains(t, inbound[1].sql, "inbound.start_id = input_pairs.end_id") + require.Contains(t, inbound[2].sql, "matched.end_id = distinct_pairs.start_id") + + directionless := buildExpandIntoReferenceSpecs(map[string]any{}, "directionless") + for _, spec := range directionless { + require.Contains(t, spec.sql, " or (") + require.NotContains(t, spec.sql, "union all\n select matched") + } + require.Contains(t, directionless[1].sql, "degrees.start_degree <= degrees.end_degree") + require.Contains(t, directionless[1].sql, "degrees.end_degree < degrees.start_degree") +} + +// TestExpandIntoScaleCasesParse verifies the shared three-way study corpus remains valid Cypher input. +func TestExpandIntoScaleCasesParse(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + found := 0 + stateClasses := map[string]bool{} + for _, testCase := range corpus.Cases { + if testCase.Category != "expand_into_one_hop" { + continue + } + found++ + stateClasses[testCase.Shape.ExpectedStateClass] = true + _, err := frontend.ParseCypher(frontend.NewContext(), testCase.Cypher) + require.NoError(t, err, testCase.Name) + } + require.Equal(t, 11, found) + require.True(t, stateClasses["source_lower_degree"]) + require.True(t, stateClasses["target_lower_degree"]) +} + +// TestExpandIntoReferenceArmsAreDeclared verifies command-line selection accepts every three-way study arm. +func TestExpandIntoReferenceArmsAreDeclared(t *testing.T) { + for _, name := range []string{"expand_into_pair_join", "expand_into_lower_degree_scan", "expand_into_pair_cache"} { + require.True(t, validPostgresReferenceArm(name), name) + } +} diff --git a/cmd/graphbench/references_test.go b/cmd/graphbench/references_test.go new file mode 100644 index 00000000..b65db775 --- /dev/null +++ b/cmd/graphbench/references_test.go @@ -0,0 +1,855 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "strings" + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/stretchr/testify/require" +) + +// outboundShortestPathQuery is the canonical bound-endpoint path query shared by reference-arm tests. +const outboundShortestPathQuery = "MATCH p = shortestPath((s)-[*0..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p" + +// TestSupplementalPostgresReadHelpersPropagateTransactionOptions verifies that +// reference timing, precomputation, and plan capture all retain the caller's +// stable-snapshot transaction contract. +func TestSupplementalPostgresReadHelpersPropagateTransactionOptions(t *testing.T) { + database := &referenceTransactionOptionTestDatabase{expectedDriverConfig: "stable-snapshot"} + transactionOption := func(config *graph.TransactionConfig) { + config.DriverConfig = "stable-snapshot" + } + + rowCount, _, err := measureRawPostgres(context.Background(), database, "select value", nil, 0, 1, transactionOption) + require.NoError(t, err) + require.Equal(t, int64(1), rowCount) + + values, err := readReferenceRow(context.Background(), database, "select value", nil, transactionOption) + require.NoError(t, err) + require.Equal(t, []any{int64(1)}, values) + + plan, planJSON, _, err := explainRawPostgres(context.Background(), database, "select value", nil, transactionOption) + require.NoError(t, err) + require.NotEmpty(t, plan) + require.NotEmpty(t, planJSON) + + require.Equal(t, []bool{true, true, true, true}, database.transactionOptionsApplied) +} + +// referenceTransactionOptionTestDatabase records transaction configuration and +// supplies the narrow raw-query surface used by supplemental reference helpers. +type referenceTransactionOptionTestDatabase struct { + graph.Database + expectedDriverConfig any + transactionOptionsApplied []bool +} + +// ReadTransaction applies the supplied options before executing a synthetic raw transaction. +func (s *referenceTransactionOptionTestDatabase) ReadTransaction(_ context.Context, delegate graph.TransactionDelegate, options ...graph.TransactionOption) error { + config := &graph.TransactionConfig{} + for _, option := range options { + option(config) + } + s.transactionOptionsApplied = append(s.transactionOptionsApplied, config.DriverConfig == s.expectedDriverConfig) + return delegate(&referenceTransactionOptionTestTransaction{}) +} + +// referenceTransactionOptionTestTransaction returns one scalar row or one valid plan document. +type referenceTransactionOptionTestTransaction struct { + graph.Transaction +} + +// Raw returns the minimal row shape expected by the helper under test. +func (s *referenceTransactionOptionTestTransaction) Raw(statement string, _ map[string]any) graph.Result { + if strings.Contains(statement, "FORMAT JSON") { + return &referenceTransactionOptionTestResult{rows: [][]any{{`[{"Plan":{"Node Type":"Result","Actual Rows":1,"Actual Loops":1}}]`}}} + } + if strings.HasPrefix(statement, "EXPLAIN ") { + return &referenceTransactionOptionTestResult{rows: [][]any{{"Result"}}} + } + return &referenceTransactionOptionTestResult{rows: [][]any{{int64(1)}}} +} + +// referenceTransactionOptionTestResult iterates a fixed set of raw rows. +type referenceTransactionOptionTestResult struct { + graph.Result + rows [][]any + index int +} + +// Next advances to the next fixed row. +func (s *referenceTransactionOptionTestResult) Next() bool { + if s.index >= len(s.rows) { + return false + } + s.index++ + return true +} + +// Values returns the current fixed row. +func (s *referenceTransactionOptionTestResult) Values() []any { + return s.rows[s.index-1] +} + +// Error reports a successful fixed result. +func (s *referenceTransactionOptionTestResult) Error() error { + return nil +} + +// Close satisfies graph.Result. +func (s *referenceTransactionOptionTestResult) Close() {} + +// TestShortestReferenceSpecsAreGraphScopedAndSeparateRawFromFullOutput verifies the complete arm inventory, graph partition predicates, precomputed hydration inputs, and full-comparator metadata. +func TestShortestReferenceSpecsAreGraphScopedAndSeparateRawFromFullOutput(t *testing.T) { + params := map[string]any{"graph_id": int32(42), "start_id": int64(1), "end_id": int64(2), "max_depth": int32(15)} + specs := buildShortestReferenceSpecs(ScaleCase{ + Name: "one_shortest_path_bound_pair", + Cypher: outboundShortestPathQuery, + }, params, []int64{1, 2, 3}, []int64{10, 11}, graph.DirectionOutbound) + + require.Len(t, specs, 14) + require.Equal(t, "round_trip", specs[0].name) + require.Equal(t, int32(42), specs[1].parameters["graph_id"]) + require.Equal(t, "minimum_graph_access", specs[2].name) + require.Contains(t, specs[3].sql, "e.graph_id = @graph_id") + require.Contains(t, specs[3].boundary, "ordered node/edge IDs") + require.Equal(t, []int64{10, 11}, specs[4].parameters["edge_ids"]) + require.Equal(t, []int64{1, 2, 3}, specs[6].parameters["node_ids"]) + + s3u := specs[referenceSpecIndex(specs, "s3_unidirectional_trail_cte")] + require.True(t, s3u.fullComparator) + require.Equal(t, "complete_reference_s1_array_cte", s3u.legacyName) + require.Equal(t, "SP-S3-U-NE", s3u.architecture) + require.Contains(t, s3u.sql, "ordered_edge_ids_to_path") + + s3b := specs[referenceSpecIndex(specs, "s3_bidirectional_trail_cte")] + require.Equal(t, "candidate_s2_bidirectional_cte", s3b.legacyName) + require.Equal(t, "SP-S3-B", s3b.architecture) + require.True(t, s3b.fullComparator) + require.Contains(t, s3b.sql, "forward join backward") + require.Contains(t, s3b.sql, "e.graph_id = @graph_id") + require.Contains(t, s3b.sql, "edge_id = any(backward.edge_ids)") +} + +// TestShortestDistanceReferenceCarriesNoTrailOrPredecessorState verifies that distance-only recursion stores just the frontier node and depth, avoiding node and edge trail arrays. +func TestShortestDistanceReferenceCarriesNoTrailOrPredecessorState(t *testing.T) { + specs := buildShortestReferenceSpecs(ScaleCase{ + Name: "shortest_distance_bound_pair", + Expected: ExpectedResult{ + ResultKind: "scalar", + }, + }, map[string]any{}, nil, nil, graph.DirectionOutbound) + reference := specs[referenceSpecIndex(specs, "s3_unidirectional_trail_cte")] + + require.Equal(t, "distance frontier node and depth only; no path or predecessor state", reference.stateShape) + require.Contains(t, reference.sql, "search(node_id, depth)") + require.NotContains(t, reference.sql, "node_ids") + require.NotContains(t, reference.sql, "edge_ids") +} + +// TestCompactBidirectionalReferencesExposeMatchedDistanceAndWitnessBoundaries +// verifies the four frozen arms share caps while preserving observation shape. +func TestCompactBidirectionalReferencesExposeMatchedDistanceAndWitnessBoundaries(t *testing.T) { + params := map[string]any{ + "graph_id": int32(42), "start_id": int64(1), "end_id": int64(3), + "min_depth": int32(1), "max_depth": int32(8), "edge_kind_ids": []int16{1}, + } + distance := buildShortestReferenceSpecs(ScaleCase{ + Expected: ExpectedResult{ResultKind: "scalar"}, + }, params, nil, nil, graph.DirectionOutbound) + for _, name := range []string{"sp_b1_strict_alternating_distance", "sp_b2_smaller_frontier_distance"} { + spec := distance[referenceSpecIndex(distance, name)] + require.True(t, spec.fullComparator) + require.Equal(t, "distance scalar", spec.observationShape) + require.Equal(t, int64(100_000), spec.parameters["state_limit"]) + require.Equal(t, int64(100_000), spec.parameters["frontier_limit"]) + require.Equal(t, int64(100_000), spec.parameters["predecessor_limit"]) + require.Contains(t, spec.sql, "select depth, path as edge_ids") + require.NotContains(t, spec.sql, "ordered_edge_ids_to_path") + } + + witness := buildShortestReferenceSpecs(ScaleCase{ + Name: "one_shortest_path_bound_pair", + Expected: ExpectedResult{ResultKind: "path_set"}, + }, params, nil, nil, graph.DirectionInbound) + for _, name := range []string{"sp_b1_strict_alternating_witness_m0", "sp_b2_smaller_frontier_witness_m0"} { + spec := witness[referenceSpecIndex(witness, name)] + require.True(t, spec.fullComparator) + require.Equal(t, "public_observation", spec.observationShape) + require.Contains(t, spec.sql, "@edge_kind_ids, true") + require.Contains(t, spec.sql, "terminal.id = edge.start_id") + } +} + +// TestCanonicalSourceDistanceReferenceSwapsInboundEndpointsAndPhysicalDirection verifies that the inbound-only canonical arm searches from the logical terminal using reversed physical adjacency. +func TestCanonicalSourceDistanceReferenceSwapsInboundEndpointsAndPhysicalDirection(t *testing.T) { + params := map[string]any{"graph_id": int32(42), "start_id": int64(10), "end_id": int64(20), "min_depth": int32(1), "max_depth": int32(8), "edge_kind_ids": []int16{1}} + testCase := ScaleCase{ + Name: "hidden_fanin", + Expected: ExpectedResult{ + ResultKind: "scalar", + }, + } + inbound := buildShortestReferenceSpecs(testCase, params, nil, nil, graph.DirectionInbound) + canonical := inbound[referenceSpecIndex(inbound, "s4_canonical_source_distance")] + require.Equal(t, "SP-I1-C-D", canonical.architecture) + require.Equal(t, int64(20), canonical.parameters["start_id"]) + require.Equal(t, int64(10), canonical.parameters["end_id"]) + require.Contains(t, canonical.sql, "e.start_id = search.node_id") + require.Contains(t, canonical.sql, "select e.end_id") + require.NotContains(t, canonical.sql, "edge_ids") + require.True(t, canonical.fullComparator) + + outbound := buildShortestReferenceSpecs(testCase, params, nil, nil, graph.DirectionOutbound) + require.Equal(t, -1, referenceSpecIndexOrMissing(outbound, "s4_canonical_source_distance")) +} + +// TestShortestS1DistancePrototypeIsDistinctBoundedAndFallsBack verifies S1 metadata, its state guard and SQL fallback, and propagation of inbound traversal direction. +func TestShortestS1DistancePrototypeIsDistinctBoundedAndFallsBack(t *testing.T) { + minDepth, maxDepth := 1, 8 + params := map[string]any{ + "graph_id": int32(1), "start_id": int64(10), "end_id": int64(20), + "min_depth": int32(1), "max_depth": int32(8), "edge_kind_ids": []int16{2}, + } + testCase := ScaleCase{ + Name: "distance", + Expected: ExpectedResult{ + ResultKind: "scalar", + }, + Shape: WorkloadShape{ + MinDepth: &minDepth, + MaxDepth: &maxDepth, + }, + } + specs := buildShortestReferenceSpecs(testCase, params, nil, nil, graph.DirectionOutbound) + s1 := specs[referenceSpecIndex(specs, "s1_array_bfs_distance")] + + require.Equal(t, "SP-S1", s1.architecture) + require.Equal(t, "typed_plpgsql_array_bfs_distance_v1", s1.implementationID) + require.True(t, s1.fullComparator) + require.Equal(t, int32(100_000), s1.parameters["state_limit"]) + require.Contains(t, s1.sql, "graphbench_s1_distance_bfs") + require.Contains(t, s1.sql, "where (select overflow from s1)") + require.Contains(t, s1.sql, shortestDistanceReferenceSearchForDirection(graph.DirectionOutbound)) + + inbound := buildShortestReferenceSpecs(testCase, params, nil, nil, graph.DirectionInbound) + require.Contains(t, inbound[referenceSpecIndex(inbound, "s1_array_bfs_distance")].sql, "@edge_kind_ids, true, @state_limit") +} + +// TestShortestS1DistancePrototypeRejectsUnsupportedShapes verifies that S1 is omitted for minimum depth above one, path results, and identical bound endpoints. +func TestShortestS1DistancePrototypeRejectsUnsupportedShapes(t *testing.T) { + minDepth, maxDepth := 2, 8 + params := map[string]any{"start_id": int64(10), "end_id": int64(20)} + distance := ScaleCase{ + Expected: ExpectedResult{ + ResultKind: "scalar", + }, + Shape: WorkloadShape{ + MinDepth: &minDepth, + MaxDepth: &maxDepth, + }, + } + require.Equal(t, -1, referenceSpecIndexOrMissing(buildShortestReferenceSpecs(distance, params, nil, nil, graph.DirectionOutbound), "s1_array_bfs_distance")) + + minDepth = 1 + path := ScaleCase{ + Expected: ExpectedResult{ + ResultKind: "path_set", + }, + Shape: WorkloadShape{ + MinDepth: &minDepth, + MaxDepth: &maxDepth, + }, + } + require.Equal(t, -1, referenceSpecIndexOrMissing(buildShortestReferenceSpecs(path, params, nil, nil, graph.DirectionOutbound), "s1_array_bfs_distance")) + + params["end_id"] = int64(10) + require.Equal(t, -1, referenceSpecIndexOrMissing(buildShortestReferenceSpecs(distance, params, nil, nil, graph.DirectionOutbound), "s1_array_bfs_distance")) +} + +// TestShortestPathReferencesCompareM0AndM1WithMinimalSearchState verifies exact M0/M1 comparator arms while preserving their edge-only versus node-and-edge hydration boundaries. +func TestShortestPathReferencesCompareM0AndM1WithMinimalSearchState(t *testing.T) { + params := map[string]any{"graph_id": int32(42), "start_id": int64(1), "end_id": int64(3), "max_depth": int32(4)} + specs := buildShortestReferenceSpecs( + ScaleCase{ + Name: "one_shortest_path_bound_pair", + Cypher: outboundShortestPathQuery, + }, + params, + []int64{1, 2, 3}, + []int64{10, 11}, + graph.DirectionOutbound, + ) + + m0 := specs[referenceSpecIndex(specs, "s3_unidirectional_cte_m0_directed")] + m1 := specs[referenceSpecIndex(specs, "s3_unidirectional_cte_m1_ordered_ids")] + require.Equal(t, "SP-S3-U-E+MAT-M0", m0.architecture) + require.Equal(t, "SP-S3-U-NE+MAT-M1", m1.architecture) + require.True(t, m0.fullComparator) + require.True(t, m1.fullComparator) + require.Equal(t, "exact_public_observation", m0.semanticValidation) + require.Equal(t, "exact_public_observation", m1.semanticValidation) + require.Contains(t, m0.sql, shortestEdgeReferenceSearch(graph.DirectionOutbound)) + require.Contains(t, m1.sql, shortestReferenceSearch()) + require.NotContains(t, m0.sql, "node_ids") + require.NotContains(t, m0.sql, "ordered_edge_ids_to_path") + require.NotContains(t, m1.sql, "ordered_edge_ids_to_path") + require.Contains(t, m0.sql, "terminal.id = edge.end_id") + require.Contains(t, m1.sql, "unnest(shortest.node_ids) with ordinality") + require.Contains(t, m0.sql, "edge.graph_id = @graph_id") + require.Contains(t, m1.sql, "node.graph_id = @graph_id") +} + +// TestCanonicalWitnessReferenceUsesCompactDiscoveryAndRestoresInboundPathOrder verifies distance-only discovery, separate witness reconstruction, swapped inbound endpoints, and restoration of public path order. +func TestCanonicalWitnessReferenceUsesCompactDiscoveryAndRestoresInboundPathOrder(t *testing.T) { + params := map[string]any{"graph_id": int32(42), "start_id": int64(10), "end_id": int64(20), "min_depth": int32(1), "max_depth": int32(8), "edge_kind_ids": []int16{1}} + testCase := ScaleCase{ + Name: "path", + Expected: ExpectedResult{ + ResultKind: "path_set", + }, + } + inbound := buildShortestReferenceSpecs(testCase, params, nil, nil, graph.DirectionInbound) + witness := inbound[referenceSpecIndex(inbound, "s4_canonical_source_witness_m0")] + require.Equal(t, "SP-I1-C-WE+MAT-M0", witness.architecture) + require.Equal(t, int64(20), witness.parameters["search_start_id"]) + require.Equal(t, int64(10), witness.parameters["search_end_id"]) + require.Contains(t, witness.sql, "distance(node_id, depth)") + require.Contains(t, witness.sql, "witness(node_id, depth, edge_ids)") + require.Contains(t, witness.sql, "e.start_id = distance.node_id") + require.Contains(t, witness.sql, "order by reversed.ordinal desc") + require.Contains(t, witness.sql, "terminal.id = edge.start_id") + require.NotContains(t, witness.sql, "distance(node_id, depth, edge_ids)") + require.True(t, witness.fullComparator) + + outbound := buildShortestReferenceSpecs(testCase, params, nil, nil, graph.DirectionOutbound) + outboundWitness := outbound[referenceSpecIndex(outbound, "s4_canonical_source_witness_m0")] + require.Equal(t, int64(10), outboundWitness.parameters["search_start_id"]) + require.NotContains(t, outboundWitness.sql, "reversed.ordinal") +} + +// TestAllShortestDAGReferenceRetainsEveryShortestDepthPredecessor verifies that all-shortest search records every depth-minimal predecessor and reconstructs paths in both physical directions without LIMIT-based tie loss. +func TestAllShortestDAGReferenceRetainsEveryShortestDepthPredecessor(t *testing.T) { + outbound := allShortestDAGSearch(graph.DirectionOutbound) + require.Contains(t, outbound, "distance(node_id, depth)") + require.Contains(t, outbound, "predecessor(node_id, depth, predecessor_id, edge_id)") + require.Contains(t, outbound, "paths(node_id, depth, edge_ids)") + require.Contains(t, outbound, "e.start_id = prior.node_id and e.end_id = paths.node_id") + require.Contains(t, outbound, "paths.depth <= target.depth") + require.NotContains(t, outbound, "limit 1\n ) predecessor") + + inbound := allShortestDAGSearch(graph.DirectionInbound) + require.Contains(t, inbound, "e.end_id = distance.node_id") + require.Contains(t, inbound, "e.end_id = prior.node_id and e.start_id = paths.node_id") +} + +// TestShortestReferenceIdentitiesAndInboundMinimalState verifies normalized arm identities and inbound M0 SQL that recurses over edge trails without carrying node arrays. +func TestShortestReferenceIdentitiesAndInboundMinimalState(t *testing.T) { + specs := buildShortestReferenceSpecs( + ScaleCase{ + Name: "one_shortest_path_bound_pair", + Cypher: "MATCH p = shortestPath((s)<-[*1..4]-(e)) RETURN p", + Expected: ExpectedResult{ + ResultKind: "path_set", + }, + }, + map[string]any{"graph_id": int32(42), "start_id": int64(1), "end_id": int64(3), "max_depth": int32(4)}, + []int64{1, 2, 3}, []int64{10, 11}, graph.DirectionInbound, + ) + for idx := range specs { + specs[idx] = normalizedReferenceSpec(specs[idx]) + } + require.NoError(t, validateReferenceSpecs(specs)) + m0 := specs[referenceSpecIndex(specs, "s3_unidirectional_cte_m0_directed")] + require.Contains(t, m0.sql, "e.end_id = search.node_id") + require.Contains(t, m0.sql, "terminal.id = edge.start_id") + require.NotContains(t, m0.sql, "node_ids") +} + +// TestShortestPathMaterializerOnlyReferencesExcludeSearch verifies that M0/M1 hydration-only arms consume precomputed exact inputs and neither their timing nor validation SQL performs recursive search. +func TestShortestPathMaterializerOnlyReferencesExcludeSearch(t *testing.T) { + specs := buildShortestReferenceSpecs( + ScaleCase{ + Name: "one_shortest_path_bound_pair", + Cypher: outboundShortestPathQuery, + }, + map[string]any{}, + []int64{1, 2}, + []int64{10}, + graph.DirectionOutbound, + ) + + m0 := specs[referenceSpecIndex(specs, "m0_directed_hydration_only")] + m1 := specs[referenceSpecIndex(specs, "m1_ordered_ids_hydration_only")] + require.False(t, m0.fullComparator) + require.False(t, m1.fullComparator) + require.NotContains(t, m0.sql, "with recursive") + require.NotContains(t, m1.sql, "with recursive") + require.Equal(t, "precomputed_exact_path_inputs", m0.semanticValidation) + require.Equal(t, "precomputed_exact_path_inputs", m1.semanticValidation) + require.NotEmpty(t, m0.validationSQL) + require.NotEmpty(t, m1.validationSQL) + require.NotContains(t, m0.validationSQL, "with recursive") + require.NotContains(t, m1.validationSQL, "with recursive") +} + +// TestShortestReferencesPreserveZeroLengthPathInputs verifies non-nil empty edge arrays, singleton node arrays, minimum-depth predicates, and bidirectional acceptance of zero-edge paths. +func TestShortestReferencesPreserveZeroLengthPathInputs(t *testing.T) { + zeroEdges, err := referenceInt64Slice([]int64{}) + require.NoError(t, err) + require.NotNil(t, zeroEdges) + + params := map[string]any{ + "graph_id": int32(42), + "start_id": int64(1), + "end_id": int64(1), + "min_depth": int32(0), + "max_depth": int32(4), + "edge_kind_ids": []int16{}, + } + specs := buildShortestReferenceSpecs( + ScaleCase{ + Name: "zero_shortest_path", + Cypher: outboundShortestPathQuery, + Expected: ExpectedResult{ + ResultKind: "path_set", + }, + }, + params, + []int64{1}, + zeroEdges, + graph.DirectionOutbound, + ) + + require.Contains(t, shortestReferenceSearch(), "depth >= @min_depth") + require.Contains(t, shortestDistanceReferenceSearch(), "depth >= @min_depth") + require.Equal(t, zeroEdges, specs[referenceSpecIndex(specs, "m0_directed_hydration_only")].parameters["edge_ids"]) + require.Equal(t, []int64{1}, specs[referenceSpecIndex(specs, "m1_ordered_ids_hydration_only")].parameters["node_ids"]) + require.Contains(t, specs[referenceSpecIndex(specs, "s3_bidirectional_trail_cte")].sql, "between @min_depth and @max_depth") +} + +// TestShortestMaterializersRequireProvablyOutboundPattern verifies direction parsing and withholds ordered outbound hydration arms only for directionless patterns. +func TestShortestMaterializersRequireProvablyOutboundPattern(t *testing.T) { + for _, testCase := range []struct { + // name identifies the direction case in subtest diagnostics. + name string + + // query is the pattern whose relationship direction is classified. + query string + + // outbound is true when parsing must select physical outbound traversal. + outbound bool + + // supported is true when directional reference materializers must be available. + supported bool + }{ + { + name: "outbound", + query: "MATCH p = shortestPath((s)-[*1..4]->(e)) RETURN p", + outbound: true, + supported: true, + }, + { + name: "inbound", + query: "MATCH p = shortestPath((s)<-[*1..4]-(e)) RETURN p", + supported: true, + }, + { + name: "directionless", + query: "MATCH p = shortestPath((s)-[*1..4]-(e)) RETURN p", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + direction, err := shortestReferenceDirection(testCase.query) + require.NoError(t, err) + require.Equal(t, testCase.outbound, direction == graph.DirectionOutbound) + + specs := buildShortestReferenceSpecs( + ScaleCase{ + Cypher: testCase.query, + Expected: ExpectedResult{ + ResultKind: "path_set", + }, + }, + map[string]any{}, + []int64{1, 2}, + []int64{10}, + direction, + ) + if testCase.supported { + require.NotEqual(t, -1, referenceSpecIndexOrMissing(specs, "s3_unidirectional_cte_m0_directed")) + } else { + require.Equal(t, -1, referenceSpecIndexOrMissing(specs, "s3_unidirectional_cte_m0_directed")) + require.Equal(t, -1, referenceSpecIndexOrMissing(specs, "m1_ordered_ids_hydration_only")) + } + }) + } +} + +// TestShortestReferenceEndpointParametersFollowPatternRootOrder verifies that endpoint bindings follow left-to-right pattern roles rather than arrow direction or variable spelling. +func TestShortestReferenceEndpointParametersFollowPatternRootOrder(t *testing.T) { + for _, testCase := range []struct { + // name identifies the endpoint-order case in subtest diagnostics. + name string + + // query contains the bound variables whose pattern positions are resolved. + query string + + // root is the parameter attached to the left pattern endpoint. + root string + + // terminal is the parameter attached to the right pattern endpoint. + terminal string + }{ + { + name: "outbound", + query: `MATCH p = shortestPath((s)-[:Traverse*1..8]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)`, + root: "start_id", + terminal: "end_id", + }, + { + name: "inbound same symbols", + query: `MATCH p = shortestPath((s)<-[:Traverse*1..8]-(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)`, + root: "start_id", + terminal: "end_id", + }, + { + name: "inbound reversed symbols", + query: `MATCH p = shortestPath((e)<-[:Traverse*1..8]-(s)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)`, + root: "end_id", + terminal: "start_id", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + root, terminal, err := shortestReferenceEndpointParameters(testCase.query) + require.NoError(t, err) + require.Equal(t, testCase.root, root) + require.Equal(t, testCase.terminal, terminal) + }) + } +} + +// TestAlternativeOneShortestPathTieIsSemanticallyValid verifies acceptance of an equal-length valid tie and rejection of longer, wrong-kind, or unmapped alternatives. +func TestAlternativeOneShortestPathTieIsSemanticallyValid(t *testing.T) { + testCase := ScaleCase{ + Cypher: outboundShortestPathQuery, + Expected: ExpectedResult{ + ResultKind: "path_set", + }, + Shape: WorkloadShape{ + EdgeKinds: []string{"Edge"}, + }, + } + public := []string{`[{"nodes":[{"identity":"start"},{"identity":"left"},{"identity":"end"}],"relationships":[{"start":"start","end":"left","kind":"Edge"},{"start":"left","end":"end","kind":"Edge"}]}]`} + alternative := []string{`[{"nodes":[{"identity":"start"},{"identity":"right"},{"identity":"end"}],"relationships":[{"start":"start","end":"right","kind":"Edge"},{"start":"right","end":"end","kind":"Edge"}]}]`} + longer := []string{`[{"nodes":[{"identity":"start"},{"identity":"right"},{"identity":"other"},{"identity":"end"}],"relationships":[{"start":"start","end":"right","kind":"Edge"},{"start":"right","end":"other","kind":"Edge"},{"start":"other","end":"end","kind":"Edge"}]}]`} + wrongKind := []string{`[{"nodes":[{"identity":"start"},{"identity":"right"},{"identity":"end"}],"relationships":[{"start":"start","end":"right","kind":"Wrong"},{"start":"right","end":"end","kind":"Wrong"}]}]`} + unmapped := []string{`[{"nodes":[{"identity":"start"},{"identity":"unmapped-node:42"},{"identity":"end"}],"relationships":[{"start":"start","end":"unmapped-node:42","kind":"Edge"},{"start":"unmapped-node:42","end":"end","kind":"Edge"}]}]`} + + require.True(t, validAlternativeShortestPathObservation(testCase, public, alternative)) + require.False(t, validAlternativeShortestPathObservation(testCase, public, longer)) + require.False(t, validAlternativeShortestPathObservation(testCase, public, wrongKind)) + require.False(t, validAlternativeShortestPathObservation(testCase, public, unmapped)) +} + +// TestReferenceSpecsAlternateOrderByRound verifies fallback odd/even forward-reverse execution ordering without mutating the declared arm sequence. +func TestReferenceSpecsAlternateOrderByRound(t *testing.T) { + specs := []postgresReferenceSpec{{name: "first"}, {name: "second"}} + require.Equal(t, []postgresReferenceSpec{{name: "first"}, {name: "second"}}, referenceSpecsForRound(specs, 1)) + require.Equal(t, []postgresReferenceSpec{{name: "second"}, {name: "first"}}, referenceSpecsForRound(specs, 2)) + require.Equal(t, "first", specs[0].name) +} + +// TestThreeArmReferenceSpecsUseCarryoverBalancedSchedule verifies the doubled +// Williams design balances both execution position and directed carryover. +func TestThreeArmReferenceSpecsUseCarryoverBalancedSchedule(t *testing.T) { + specs := []postgresReferenceSpec{{name: "A"}, {name: "B"}, {name: "C"}} + expected := [][]string{ + {"A", "B", "C"}, + {"B", "C", "A"}, + {"C", "A", "B"}, + {"C", "B", "A"}, + {"A", "C", "B"}, + {"B", "A", "C"}, + } + positions := map[string][3]int{} + carryover := map[[2]string]int{} + for round, want := range expected { + got := referenceSpecNames(referenceSpecsForRound(specs, round+1)) + require.Equal(t, want, got) + for position, arm := range got { + counts := positions[arm] + counts[position]++ + positions[arm] = counts + if position > 0 { + carryover[[2]string{got[position-1], arm}]++ + } + } + } + require.Equal(t, expected[0], referenceSpecNames(referenceSpecsForRound(specs, 7))) + for _, arm := range []string{"A", "B", "C"} { + require.Equal(t, [3]int{2, 2, 2}, positions[arm]) + } + for _, pair := range [][2]string{{"A", "B"}, {"A", "C"}, {"B", "A"}, {"B", "C"}, {"C", "A"}, {"C", "B"}} { + require.Equal(t, 2, carryover[pair], pair) + } + require.Equal(t, "A", specs[0].name) +} + +// TestFiveArmReferenceSpecsUsePredeclaredBalancedSchedule verifies selected rows of the ten-round five-arm schedule and its periodic repetition. +func TestFiveArmReferenceSpecsUsePredeclaredBalancedSchedule(t *testing.T) { + specs := []postgresReferenceSpec{{name: "T1"}, {name: "T2"}, {name: "T3"}, {name: "T4"}, {name: "T5"}} + require.Equal(t, []string{"T1", "T2", "T5", "T3", "T4"}, referenceSpecNames(referenceSpecsForRound(specs, 1))) + require.Equal(t, []string{"T4", "T3", "T5", "T2", "T1"}, referenceSpecNames(referenceSpecsForRound(specs, 6))) + require.Equal(t, []string{"T1", "T2", "T5", "T3", "T4"}, referenceSpecNames(referenceSpecsForRound(specs, 11))) +} + +// referenceSpecNames returns reference names in their declared execution order. +func referenceSpecNames(specs []postgresReferenceSpec) []string { + names := make([]string, len(specs)) + for idx, spec := range specs { + names[idx] = spec.name + } + return names +} + +// TestAllShortestPathCaseUsesDistinctFullMultisetDAGReferences verifies that +// stored A1, inline I1, and both exact two-sided candidates retain distinct +// treatment identities. +func TestAllShortestPathCaseUsesDistinctFullMultisetDAGReferences(t *testing.T) { + runner := &postgresSQLRunner{} + specs, err := runner.referenceSpecs(context.Background(), ScaleCase{ + Category: "generated_shortest_path", + Cypher: "MATCH p = allShortestPaths((s)-[:Traverse*1..2]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + }, map[string]any{"start_id": int64(1), "end_id": int64(2)}) + + require.NoError(t, err) + require.Len(t, specs, 4) + require.Equal(t, []string{ + "asp_a1_stored_helper_m0", + "asp_i1_inline_predecessor_dag_m0", + "asp_b1_bidirectional_dag_strict_m0", + "asp_b2_bidirectional_dag_smaller_frontier_m0", + }, referenceSpecNames(specs)) + require.Equal(t, []string{"ASP-A1-DAG", "ASP-I1-U-DAG+MAT-M0", "ASP-B1-DAG-ALT-NODE", "ASP-B2-DAG-MIN-LEVEL"}, []string{ + specs[0].architecture, specs[1].architecture, specs[2].architecture, specs[3].architecture, + }) + for _, spec := range specs { + require.True(t, validPostgresReferenceArm(spec.name), spec.name) + require.True(t, spec.fullComparator) + require.Equal(t, "complete all-shortest path multiset", spec.observationShape) + require.Equal(t, "exact_public_observation", spec.semanticValidation) + require.Contains(t, spec.sql, "pathComposite") + } + for _, spec := range specs[2:] { + require.Equal(t, int64(100_000), spec.parameters["state_limit"]) + require.Equal(t, int64(100_000), spec.parameters["frontier_limit"]) + require.Equal(t, int64(100_000), spec.parameters["predecessor_limit"]) + require.Equal(t, int64(100_000), spec.parameters["enumeration_limit"]) + require.Equal(t, int64(64*1024*1024), spec.parameters["output_bytes_limit"]) + require.Contains(t, spec.sql, "@enumeration_limit, @output_bytes_limit") + } + require.Contains(t, specs[0].sql, "all_shortest_paths_dag") + require.Contains(t, specs[1].sql, "with recursive validated") + require.Contains(t, specs[2].sql, "all_shortest_paths_b1_strict_alternating") + require.Contains(t, specs[3].sql, "all_shortest_paths_b2_smaller_current_level") +} + +// TestAllShortestBidirectionalReferencesStayInsideNarrowEnvelope verifies +// min-zero, over-depth, and equal endpoints retain only the exact A1 control. +func TestAllShortestBidirectionalReferencesStayInsideNarrowEnvelope(t *testing.T) { + runner := &postgresSQLRunner{} + minimumZero, maximumFour, maximumSixtyFive := 0, 4, 65 + for _, test := range []struct { + name string + shape WorkloadShape + params map[string]any + }{ + {name: "zero minimum", shape: WorkloadShape{MinDepth: &minimumZero, MaxDepth: &maximumFour}, params: map[string]any{"start_id": int64(1), "end_id": int64(2)}}, + {name: "maximum sixty five", shape: WorkloadShape{MaxDepth: &maximumSixtyFive}, params: map[string]any{"start_id": int64(1), "end_id": int64(2)}}, + {name: "equal endpoints", shape: WorkloadShape{MaxDepth: &maximumFour}, params: map[string]any{"start_id": int64(1), "end_id": int64(1)}}, + } { + t.Run(test.name, func(t *testing.T) { + specs, err := runner.referenceSpecs(context.Background(), ScaleCase{ + Category: "generated_shortest_path", + Cypher: "MATCH p = allShortestPaths((s)-[:Traverse*0..65]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + Shape: test.shape, + }, test.params) + require.NoError(t, err) + require.Len(t, specs, 1) + require.Equal(t, "ASP-A1-DAG", specs[0].architecture) + }) + } +} + +// TestFixedSuffixExpansionReferenceSpecsAvoidAmbiguousArrayContainmentOperators verifies all seventeen arms use explicit membership predicates and retain each strategy's defining recursive SQL shape. +func TestFixedSuffixExpansionReferenceSpecsAvoidAmbiguousArrayContainmentOperators(t *testing.T) { + specs := buildFixedSuffixExpansionReferenceSpecs(ScaleCase{ + Name: "fixed_suffix_expansion_endpoint_ids", + }, map[string]any{"graph_id": int32(42)}) + + require.Len(t, specs, 17) + for _, spec := range specs { + require.NotContains(t, spec.sql, " @> ") + } + require.Contains(t, specs[1].sql, "= any(n.kind_ids)") + require.Contains(t, specs[referenceSpecIndex(specs, "suffix_seeded_reverse_ordered_ids")].sql, "array_prepend(e.id, reverse_trails.edge_ids)") + require.Contains(t, specs[referenceSpecIndex(specs, "suffix_seeded_reverse_ordered_ids")].sql, "union all") + require.Contains(t, specs[referenceSpecIndex(specs, "backward_viability_forward_ordered_ids")].sql, "viable(node_id, reverse_distance)") + require.Contains(t, specs[referenceSpecIndex(specs, "factored_suffix_forward_ordered_ids")].sql, "suffix_rows") +} + +// TestFixedSuffixHydrationPrecomputeIsSelectionAware verifies that default and explicit hydration-only selections request precomputed path inputs. +func TestFixedSuffixHydrationPrecomputeIsSelectionAware(t *testing.T) { + require.True(t, referenceHydrationRequested(nil)) + require.True(t, referenceHydrationRequested([]string{"hydration_only"})) +} + +// TestGeneratedFixedSuffixExpansionReferencesUseDeclaredDepthAndObservation verifies propagation of maximum depth and selection of ID-row versus fully hydrated path output SQL. +func TestGeneratedFixedSuffixExpansionReferencesUseDeclaredDepthAndObservation(t *testing.T) { + minDepth, maxDepth := 0, 16 + runner := &postgresSQLRunner{} + testCase := ScaleCase{ + Name: "generated_fixed_suffix_expansion_endpoint_d16_f1000", + Category: "generated_fixed_suffix_expansion", + Expected: ExpectedResult{ResultKind: "id_rows"}, + Shape: WorkloadShape{ + MinDepth: &minDepth, + MaxDepth: &maxDepth, + }, + } + // Reference routing occurs before kind mapping; the generated category is + // asserted separately from the SQL builder so this remains a unit test. + require.NotNil(t, runner) + specs := buildFixedSuffixExpansionReferenceSpecs(testCase, map[string]any{"min_depth": int32(0), "max_depth": int32(16)}) + require.Contains(t, specs[referenceSpecIndex(specs, "complete_reference")].sql, "select head_id, terminal_id") + require.NotContains(t, specs[referenceSpecIndex(specs, "complete_reference")].sql, "ordered_edge_ids_to_path") + require.Equal(t, int32(16), specs[referenceSpecIndex(specs, "suffix_seeded_reverse_ordered_ids")].parameters["max_depth"]) + + testCase.Observes.Paths = true + testCase.Expected.ResultKind = "path_set" + pathSpecs := buildFixedSuffixExpansionReferenceSpecs(testCase, map[string]any{"min_depth": int32(0), "max_depth": int32(16)}) + require.Contains(t, pathSpecs[referenceSpecIndex(pathSpecs, "suffix_seeded_reverse_complete")].sql, "ordered_edge_ids_to_path") +} + +// TestParseConfigValidatesPostgresReferenceArmSelector verifies ordered arm selection, implicit reference enablement, and rejection of unknown or duplicate arm names. +func TestParseConfigValidatesPostgresReferenceArmSelector(t *testing.T) { + cfg, err := parseConfig([]string{"-postgres-reference-arms", "suffix_seeded_reverse_ordered_ids,factored_suffix_forward_complete"}, func(string) string { return "" }) + require.NoError(t, err) + require.True(t, cfg.PostgresReferences) + require.Equal(t, []string{"suffix_seeded_reverse_ordered_ids", "factored_suffix_forward_complete"}, cfg.PostgresReferenceArms) + + _, err = parseConfig([]string{"-postgres-reference-arms", "does_not_exist"}, func(string) string { return "" }) + require.ErrorContains(t, err, "unknown PostgreSQL reference arm") + _, err = parseConfig([]string{"-postgres-reference-arms", "round_trip,round_trip"}, func(string) string { return "" }) + require.ErrorContains(t, err, "duplicate PostgreSQL reference arm") +} + +// TestRequestedReferenceArmCannotDisappearFromCase verifies that an explicitly requested arm must be available for the particular workload shape. +func TestRequestedReferenceArmCannotDisappearFromCase(t *testing.T) { + _, err := selectReferenceSpecs([]postgresReferenceSpec{{name: "available"}}, []string{"missing"}) + require.ErrorContains(t, err, `requested PostgreSQL reference arm "missing" is unavailable`) +} + +// TestReferenceIdentityRejectsUndeclaredDuplicateSQL verifies that normalized duplicate SQL requires an explicit A/A alias linking the second arm to the first. +func TestReferenceIdentityRejectsUndeclaredDuplicateSQL(t *testing.T) { + specs := []postgresReferenceSpec{ + normalizedReferenceSpec(postgresReferenceSpec{ + name: "one", + architecture: "SP-S1", + stateShape: "state", + observationShape: "ordered_ids", + sql: "select 1", + }), + normalizedReferenceSpec(postgresReferenceSpec{ + name: "two", + architecture: "SP-S2", + stateShape: "state", + observationShape: "ordered_ids", + sql: " select 1 ", + }), + } + require.ErrorContains(t, validateReferenceSpecs(specs), "without a declared A/A alias") + + specs[1].aaAliasOf = "one" + require.NoError(t, validateReferenceSpecs(specs)) +} + +// TestReferenceIdentityRejectsImplementationShapeDrift verifies that a shared implementation ID cannot describe different state shapes or SQL bodies. +func TestReferenceIdentityRejectsImplementationShapeDrift(t *testing.T) { + specs := []postgresReferenceSpec{ + normalizedReferenceSpec(postgresReferenceSpec{ + name: "one", + architecture: "SP-S1", + implementationID: "same", + stateShape: "edge IDs", + observationShape: "ordered_ids", + sql: "select 1", + }), + normalizedReferenceSpec(postgresReferenceSpec{ + name: "two", + architecture: "SP-S1", + implementationID: "same", + stateShape: "node and edge IDs", + observationShape: "ordered_ids", + sql: "select 2", + }), + } + require.ErrorContains(t, validateReferenceSpecs(specs), "changes state, observation, or SQL identity") +} + +// TestFixedSuffixExpansionRootReuseIsExplicitAAAlias verifies that root-reuse arms declare their byte-equivalent ordered-ID and complete-reference counterparts. +func TestFixedSuffixExpansionRootReuseIsExplicitAAAlias(t *testing.T) { + specs := buildFixedSuffixExpansionReferenceSpecs(ScaleCase{ + Name: "fixed_suffix_expansion_endpoint_ids", + }, map[string]any{"graph_id": int32(42)}) + for idx := range specs { + specs[idx] = normalizedReferenceSpec(specs[idx]) + } + require.NoError(t, validateReferenceSpecs(specs)) + require.Equal(t, "search_ordered_ids", specs[referenceSpecIndex(specs, "root_reuse_ordered_ids")].aaAliasOf) + require.Equal(t, "complete_reference", specs[referenceSpecIndex(specs, "root_reuse_complete")].aaAliasOf) +} + +// TestFixedSuffixExpansionOrderedIDReferencesValidateAgainstCanonicalObservation verifies that every ordered-ID strategy uses the canonical search SQL and parameters for semantic validation. +func TestFixedSuffixExpansionOrderedIDReferencesValidateAgainstCanonicalObservation(t *testing.T) { + specs := buildFixedSuffixExpansionReferenceSpecs(ScaleCase{ + Name: "fixed_suffix_expansion_endpoint_ids", + }, map[string]any{"graph_id": int32(42)}) + canonical := specs[referenceSpecIndex(specs, "search_ordered_ids")] + + for _, name := range []string{ + "search_ordered_ids", + "factored_suffix_forward_ordered_ids", + "suffix_seeded_reverse_ordered_ids", + "backward_viability_forward_ordered_ids", + } { + spec := specs[referenceSpecIndex(specs, name)] + require.Equal(t, "exact_ordered_ids", spec.semanticValidation) + require.Equal(t, canonical.sql, spec.validationSQL) + require.Equal(t, canonical.parameters, spec.validationParams) + } +} + +// TestReferenceInt64SliceAcceptsDriverArrayRepresentations verifies normalization of int64, int32, and mixed driver arrays while rejecting nonnumeric elements with their index. +func TestReferenceInt64SliceAcceptsDriverArrayRepresentations(t *testing.T) { + require.Equal(t, []int64{1, 2}, mustReferenceInt64Slice(t, []int64{1, 2})) + require.Equal(t, []int64{3, 4}, mustReferenceInt64Slice(t, []int32{3, 4})) + require.Equal(t, []int64{5, 6}, mustReferenceInt64Slice(t, []any{int64(5), int32(6)})) + _, err := referenceInt64Slice([]any{"not-an-id"}) + require.ErrorContains(t, err, "array item 0") +} + +// mustReferenceInt64Slice converts a reference value to integers and fails the test on invalid input. +func mustReferenceInt64Slice(t *testing.T, value any) []int64 { + t.Helper() + result, err := referenceInt64Slice(value) + require.NoError(t, err) + return result +} diff --git a/cmd/graphbench/resource_gate.go b/cmd/graphbench/resource_gate.go new file mode 100644 index 00000000..131c3828 --- /dev/null +++ b/cmd/graphbench/resource_gate.go @@ -0,0 +1,784 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "os" + "slices" + "sort" + "strings" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" +) + +// resourceGateVersion identifies the serialized schema revision for resource gate. +const resourceGateVersion = 5 + +// ResourceGateReport reports whether production and reference plan resources remain within their allowed envelopes. +type ResourceGateReport struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // ArtifactSHA256 binds this report to the exact input JSONL artifact. + ArtifactSHA256 string `json:"artifact_sha256"` + // Passed reports whether every required gate condition succeeded. + Passed bool `json:"passed"` + // Cases contains resource-envelope decisions for each evaluated production or reference executor. + Cases []ResourceGateCase `json:"cases"` +} + +// ResourceGateCase attributes resource-gate failures to one production or reference executor architecture. +type ResourceGateCase struct { + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // Round identifies the measured record that produced this decision. + Round int `json:"round,omitempty"` + // Block identifies the paired measurement block for this record. + Block int `json:"block,omitempty"` + // RunUUID binds the resource decision to one run series. + RunUUID string `json:"run_uuid,omitempty"` + // Arm identifies the measured executor arm. + Arm string `json:"arm,omitempty"` + // ArmOrder records the arm's position within its paired block. + ArmOrder int `json:"arm_order,omitempty"` + // Reference identifies the reference arm evaluated by the resource gate. + Reference string `json:"reference,omitempty"` + // Tier identifies the resource envelope applied to the case. + Tier string `json:"tier"` + // QualificationSplit identifies training, frozen holdout, or diagnostic evidence. + QualificationSplit string `json:"qualification_split"` + // Architecture identifies the executor architecture. + Architecture string `json:"architecture,omitempty"` + // FallbackArchitecture identifies the executor architecture used after fallback. + FallbackArchitecture string `json:"fallback_architecture,omitempty"` + // Passed reports whether every required gate condition succeeded. + Passed bool `json:"passed"` + // Reasons lists explanations for the reported disposition. + Reasons []string `json:"reasons,omitempty"` + // NumericLimits records declared telemetry ceilings applied by this gate. + NumericLimits map[string]int64 `json:"numeric_limits,omitempty"` + // NumericObserved records invocation-local high-water marks compared with the limits. + NumericObserved map[string]int64 `json:"numeric_observed,omitempty"` + // RuntimeReceiptChains preserves complete measured branch chains alongside + // the resource decision. + RuntimeReceiptChains [][]RuntimeReceiptEvent `json:"runtime_receipt_chains,omitempty"` +} + +// createResourceGateReport evaluates production and reference plan metrics against resource ceilings and writes the report. +func createResourceGateReport(artifact, output string) (bool, error) { + records, err := readJSONLFile(artifact) + if err != nil { + return false, err + } + artifactSHA256, err := fileSHA256(artifact) + if err != nil { + return false, err + } + report := ResourceGateReport{ + Version: resourceGateVersion, + ArtifactSHA256: artifactSHA256, + Passed: true, + } + for _, record := range records { + if record.ExecutionMode != ModePostgresSQL { + continue + } + gateCase := evaluateProductionResourceGateCase(record) + if !gateCase.Passed { + report.Passed = false + } + report.Cases = append(report.Cases, gateCase) + for _, reference := range record.PostgresReferences { + if !reference.FullComparator || reference.Architecture == "" { + continue + } + referenceCase := ResourceGateCase{ + Dataset: record.Dataset, + Name: record.Name, + Round: gateCase.Round, + Block: gateCase.Block, + RunUUID: gateCase.RunUUID, + Arm: gateCase.Arm, + ArmOrder: gateCase.ArmOrder, + Reference: reference.Name, + Tier: gateCase.Tier, + QualificationSplit: gateCase.QualificationSplit, + Architecture: reference.Architecture, + Passed: true, + } + if reference.PostgresMetrics == nil { + referenceCase.Reasons = append(referenceCase.Reasons, "structured PostgreSQL reference plan metrics are missing") + } else if compactBidirectionalWorkspaceArchitecture(reference.Architecture) { + appendWorkspaceResourceReasons(&referenceCase, reference.PostgresMetrics) + } else if reference.Architecture != "SP-S0" { + appendPortableResourceReasons(&referenceCase, reference.PostgresMetrics) + } + appendTelemetryResourceReasons(&referenceCase, reference.TraversalTelemetry, telemetryRequiredForArchitecture(reference.Architecture)) + appendWorkspaceCeilingReasons(&referenceCase, record.Environment, reference.TraversalTelemetry, compactBidirectionalWorkspaceArchitecture(reference.Architecture), compactBidirectionalWorkspaceArchitecture(reference.Architecture)) + referenceCase.Passed = len(referenceCase.Reasons) == 0 + if !referenceCase.Passed { + report.Passed = false + } + report.Cases = append(report.Cases, referenceCase) + } + } + if len(report.Cases) == 0 { + return false, fmt.Errorf("resource artifact contains no PostgreSQL cases") + } + sort.Slice(report.Cases, func(i, j int) bool { + if report.Cases[i].Dataset != report.Cases[j].Dataset { + return report.Cases[i].Dataset < report.Cases[j].Dataset + } + if report.Cases[i].Name != report.Cases[j].Name { + return report.Cases[i].Name < report.Cases[j].Name + } + if report.Cases[i].Round != report.Cases[j].Round { + return report.Cases[i].Round < report.Cases[j].Round + } + return report.Cases[i].Reference < report.Cases[j].Reference + }) + + var raw []byte + if raw, err = json.MarshalIndent(report, "", " "); err != nil { + return false, err + } + if output == "" { + _, err = os.Stdout.Write(append(raw, '\n')) + } else { + err = os.WriteFile(output, append(raw, '\n'), 0o644) + } + if err != nil { + return false, err + } + + return report.Passed, nil +} + +// evaluateProductionResourceGateCase derives the complete production decision +// from one artifact record. Qualification reuses this exact evaluator so a +// serialized report cannot suppress spill, WAL, attribution, fallback, or cap +// failures while retaining the candidate artifact digest. +func evaluateProductionResourceGateCase(record CaseResult) ResourceGateCase { + gateCase := ResourceGateCase{ + Dataset: record.Dataset, + Name: record.Name, + Tier: record.Shape.FixtureTier, + QualificationSplit: record.Shape.QualificationSplit, + Passed: true, + RuntimeReceiptChains: runtimeReceiptChains(record.Stats.Samples), + } + if record.Environment != nil { + gateCase.Round = record.Environment.Round + gateCase.Block = record.Environment.Block + gateCase.RunUUID = record.Environment.RunUUID + gateCase.Arm = record.Environment.Arm + gateCase.ArmOrder = record.Environment.ArmOrder + } + if gateCase.Tier == "" { + gateCase.Tier = "legacy" + } + if gateCase.QualificationSplit == "" { + gateCase.QualificationSplit = "legacy" + } + gateCase.Architecture = appliedPostgresArchitecture(record) + portableCandidate := gateCase.Architecture != "" && gateCase.Architecture != "SP-S0" + workspaceCandidate := compactWorkspaceArchitecture(gateCase.Architecture) + if gateCase.Architecture == "SP-S0-DIRECT" { + if loops, found, err := postgresPlanFunctionLoops(record.PostgresPlanJSON, "bidirectional_sp_harness"); err != nil { + gateCase.Reasons = append(gateCase.Reasons, "direct preflight fallback attribution failed: "+err.Error()) + } else if !found { + gateCase.Reasons = append(gateCase.Reasons, "direct preflight fallback plan node is missing") + } else if loops > 0 { + portableCandidate = false + gateCase.FallbackArchitecture = "SP-S0" + } + } + if record.Status != StatusOK { + gateCase.Reasons = append(gateCase.Reasons, "record status is "+record.Status) + } + if record.PostgresMetrics == nil { + gateCase.Reasons = append(gateCase.Reasons, "structured PostgreSQL plan metrics are missing") + } else if workspaceCandidate { + appendWorkspaceResourceReasons(&gateCase, record.PostgresMetrics) + } else if portableCandidate { + appendPortableResourceReasons(&gateCase, record.PostgresMetrics) + } + if contract, guarded := guardedInlineResourceContractForArchitecture(gateCase.Architecture); guarded { + appendGuardedInlineResourceBindingReasons(&gateCase, record, contract) + } + telemetryRequired := telemetryRequiredForRecord(record, gateCase.Architecture) + appendTelemetryResourceReasons(&gateCase, record.TraversalTelemetry, telemetryRequired) + appendFallbackExpectationReasons(&gateCase, record) + appendWorkspaceCeilingReasons(&gateCase, record.Environment, record.TraversalTelemetry, workspaceCandidate, compactBidirectionalWorkspaceArchitecture(gateCase.Architecture)) + gateCase.Passed = len(gateCase.Reasons) == 0 + return gateCase +} + +// compactWorkspaceArchitecture reports whether an executor deliberately uses +// bounded session-local typed workspace rather than portable recursive state. +func compactWorkspaceArchitecture(architecture string) bool { + switch architecture { + case "ASP-A1-DAG", + "ASP-B1-DAG-ALT-NODE", + "ASP-B2-DAG-MIN-LEVEL", + "SP-S4-C-D", + "SP-S4-C-WE+MAT-M0", + "SP-B1-C-ALT-NODE-D", + "SP-B1-C-ALT-NODE-WE+MAT-M0", + "SP-B2-C-MIN-LEVEL-D", + "SP-B2-C-MIN-LEVEL-WE+MAT-M0": + return true + default: + return false + } +} + +// compactBidirectionalWorkspaceArchitecture identifies reference arms whose +// measured boundary deliberately includes the reusable spb_* workspace. +func compactBidirectionalWorkspaceArchitecture(architecture string) bool { + switch architecture { + case "SP-B1-C-ALT-NODE-D", + "SP-B1-C-ALT-NODE-WE+MAT-M0", + "SP-B2-C-MIN-LEVEL-D", + "SP-B2-C-MIN-LEVEL-WE+MAT-M0", + "ASP-B1-DAG-ALT-NODE", + "ASP-B2-DAG-MIN-LEVEL": + return true + default: + return false + } +} + +// telemetryRequiredForArchitecture identifies candidates whose qualification +// depends on executor-visible work rather than outer EXPLAIN counters. This +// architecture-only check also applies to explicit reference arms, so guarded +// inline I1 production requirements deliberately belong to the record-aware +// check below instead. +func telemetryRequiredForArchitecture(architecture string) bool { + return strings.HasPrefix(architecture, "SP-B1-") || + strings.HasPrefix(architecture, "SP-B2-") || + strings.HasPrefix(architecture, "ASP-B1-") || + strings.HasPrefix(architecture, "ASP-B2-") || + isOrientationProbePolicy(architecture) +} + +func telemetryRequiredForRecord(record CaseResult, architecture string) bool { + if _, guarded := guardedInlineResourceContractForArchitecture(architecture); guarded { + return true + } + if telemetryRequiredForArchitecture(architecture) { + return true + } + if record.Optimization != nil { + for _, outcome := range record.Optimization.TargetOutcomes { + if isOrientationProbePolicy(outcome.EmittedPolicy) || guardedInlineResourcePolicy(outcome.EmittedPolicy) { + return true + } + } + } + return record.TraversalTelemetry != nil && + (isOrientationProbePolicy(record.TraversalTelemetry.Summary.EmittedIdentity) || + isOrientationProbePolicy(record.TraversalTelemetry.Summary.SelectorVersion) || + guardedInlineResourcePolicy(record.TraversalTelemetry.Summary.EmittedIdentity)) +} + +type guardedInlineResourceContract struct { + architecture string + family string + telemetryFamily TraversalTelemetryFamily + policy string + namespace string + label string +} + +func guardedInlineResourceContractForArchitecture(architecture string) (guardedInlineResourceContract, bool) { + switch architecture { + case string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness): + return guardedInlineResourceContract{ + architecture: architecture, + family: "SP", + telemetryFamily: TraversalTelemetryFamilySP, + policy: optimize.ShortestPathPolicyI1CanonicalGuardedV1, + namespace: "inline_shortest_path", + label: "inline canonical SP", + }, true + case string(optimize.ShortestPathExecutorASPI1DAG): + return guardedInlineResourceContract{ + architecture: architecture, + family: "ASP", + telemetryFamily: TraversalTelemetryFamilyASP, + policy: optimize.ShortestPathPolicyASPI1GuardedV1, + namespace: "inline_asp", + label: "inline ASP", + }, true + default: + return guardedInlineResourceContract{}, false + } +} + +func guardedInlineResourcePolicy(policy string) bool { + return policy == optimize.ShortestPathPolicyI1CanonicalGuardedV1 || policy == optimize.ShortestPathPolicyASPI1GuardedV1 +} + +// appendGuardedInlineResourceBindingReasons prevents an unguarded comparator +// with the same executor architecture from satisfying production resource +// evidence. Production I1 must bind the translated outcome and telemetry to +// its exact policy and to the observation-specific typed counter namespace. +func appendGuardedInlineResourceBindingReasons(gateCase *ResourceGateCase, record CaseResult, contract guardedInlineResourceContract) { + emittedPolicy := "" + outcomeFound := false + if record.Optimization != nil { + for _, outcome := range record.Optimization.TargetOutcomes { + applied := outcome.Applied + if applied == "" { + applied = outcome.Selected + } + if outcome.Family == contract.family && applied == contract.architecture { + emittedPolicy = outcome.EmittedPolicy + outcomeFound = true + break + } + } + } + if !outcomeFound || emittedPolicy != contract.policy { + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf( + "%s production architecture requires emitted policy %q; found %q", + contract.label, contract.policy, emittedPolicy, + )) + } + + telemetry := record.TraversalTelemetry + if telemetry == nil { + return + } + if telemetry.Summary.EmittedIdentity != contract.policy { + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf( + "%s production telemetry requires emitted identity %q; found %q", + contract.label, contract.policy, telemetry.Summary.EmittedIdentity, + )) + } + if telemetry.Diagnostic == nil { + return + } + if !slices.Contains(telemetry.Diagnostic.RequiredFamilies, contract.telemetryFamily) { + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf( + "%s production telemetry requires declared counter family %q", + contract.label, contract.telemetryFamily, + )) + } + if observationRequiresHydration(telemetry.Summary.ObservationMode) && + !slices.Contains(telemetry.Diagnostic.RequiredFamilies, TraversalTelemetryFamilyHydration) { + gateCase.Reasons = append(gateCase.Reasons, contract.label+" production telemetry requires declared hydration counters for its observation mode") + } + + inlineASP := telemetry.Diagnostic.Counters.InlineASP + inlineShortestPath := telemetry.Diagnostic.Counters.InlineShortestPath + switch contract.namespace { + case "inline_shortest_path": + if inlineShortestPath == nil { + gateCase.Reasons = append(gateCase.Reasons, "inline canonical SP production telemetry requires inline_shortest_path counters") + } + if inlineASP != nil { + gateCase.Reasons = append(gateCase.Reasons, "inline canonical SP production telemetry must not use inline_asp counters") + } + case "inline_asp": + if inlineASP == nil { + gateCase.Reasons = append(gateCase.Reasons, "inline ASP production telemetry requires inline_asp counters") + } + if inlineShortestPath != nil { + gateCase.Reasons = append(gateCase.Reasons, "inline ASP production telemetry must not use inline_shortest_path counters") + } + } +} + +func appendFallbackExpectationReasons(gateCase *ResourceGateCase, record CaseResult) { + expectation := record.Shape.FallbackExpectation + if expectation == "" { + if telemetryRequiredForRecord(record, appliedPostgresArchitecture(record)) { + gateCase.Reasons = append(gateCase.Reasons, "candidate resource qualification requires a typed fallback expectation") + } + return + } + if record.TraversalTelemetry == nil { + gateCase.Reasons = append(gateCase.Reasons, "fallback expectation lacks runtime telemetry") + return + } + summary := record.TraversalTelemetry.Summary + if summary.RuntimeOutcomeAvailable == nil || !*summary.RuntimeOutcomeAvailable || summary.FallbackExecuted == nil { + gateCase.Reasons = append(gateCase.Reasons, "fallback runtime outcome is unavailable") + return + } + switch expectation { + case "required": + if !*summary.FallbackExecuted { + gateCase.Reasons = append(gateCase.Reasons, "declared overflow-fallback expectation did not execute its exact fallback") + } + case "forbidden": + if *summary.FallbackExecuted { + gateCase.Reasons = append(gateCase.Reasons, "normal/envelope candidate unexpectedly executed fallback") + } + case "allowed": + default: + gateCase.Reasons = append(gateCase.Reasons, "unknown fallback expectation "+expectation) + } +} + +func appendWorkspaceCeilingReasons(gateCase *ResourceGateCase, environment *RunEnvironment, telemetry *TraversalExecutionTelemetry, workspaceArchitecture, ceilingsRequired bool) { + if !workspaceArchitecture { + return + } + if environment == nil || environment.SessionMemoryCeilingBytes <= 0 || environment.PoolMemoryCeilingBytes <= 0 { + if ceilingsRequired { + gateCase.Reasons = append(gateCase.Reasons, "workspace candidate requires positive declared session and pool memory ceilings") + } + return + } + if telemetry == nil || telemetry.Diagnostic == nil || telemetry.Diagnostic.Counters.Workspace == nil { + gateCase.Reasons = append(gateCase.Reasons, "declared workspace memory ceilings lack measured session and pool high-water evidence") + return + } + if environment.PoolSize <= 0 { + gateCase.Reasons = append(gateCase.Reasons, "workspace candidate requires a declared positive pool size") + return + } + workspace := telemetry.Diagnostic.Counters.Workspace + if workspace.SessionPeakBytes == nil { + gateCase.Reasons = append(gateCase.Reasons, "declared session memory ceiling lacks a measured session high-water value") + } else if *workspace.SessionPeakBytes > environment.SessionMemoryCeilingBytes { + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("session workspace peak %d exceeds declared ceiling %d", *workspace.SessionPeakBytes, environment.SessionMemoryCeilingBytes)) + } + if workspace.PoolPeakBytes == nil { + gateCase.Reasons = append(gateCase.Reasons, "declared pool memory ceiling lacks a measured pool high-water value") + } else if *workspace.PoolPeakBytes > environment.PoolMemoryCeilingBytes { + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("pool workspace peak %d exceeds declared ceiling %d", *workspace.PoolPeakBytes, environment.PoolMemoryCeilingBytes)) + } + if environment.PoolSize > 1 && telemetry != nil && telemetry.Diagnostic != nil && + telemetry.Diagnostic.Provenance["workspace.pool_peak_bytes"] == "single_connection_diagnostic_pool.session_peak_bytes" { + gateCase.Reasons = append(gateCase.Reasons, "pool workspace ceiling lacks an aggregate multi-session high-water measurement") + } +} + +// appendTelemetryResourceReasons validates identity attribution and numeric +// cap evidence from a distinct untimed diagnostic invocation. +func appendTelemetryResourceReasons(gateCase *ResourceGateCase, telemetry *TraversalExecutionTelemetry, required bool) { + if telemetry == nil { + if required { + gateCase.Reasons = append(gateCase.Reasons, "required traversal execution telemetry is missing") + } + return + } + if err := ValidateTraversalExecutionTelemetry(telemetry); err != nil { + gateCase.Reasons = append(gateCase.Reasons, err.Error()) + return + } + + summary := telemetry.Summary + if summary.RuntimeOutcomeAvailable != nil && !*summary.RuntimeOutcomeAvailable { + if required { + gateCase.Reasons = append(gateCase.Reasons, "candidate qualification requires an observed runtime traversal outcome") + } + return + } + if !slices.Contains(summary.PlannedIdentities, summary.RuntimeIdentity) { + gateCase.Reasons = append(gateCase.Reasons, "runtime traversal identity is not a planned candidate") + } + if summary.AppliedIdentity != summary.RuntimeIdentity { + gateCase.Reasons = append(gateCase.Reasons, "applied traversal identity does not match runtime identity") + } + if summary.FallbackExecuted != nil && *summary.FallbackExecuted && summary.RuntimeIdentity != summary.FallbackIdentity { + gateCase.Reasons = append(gateCase.Reasons, "fallback traversal identity does not match runtime identity") + } + if required && telemetry.Level != TraversalTelemetryLevelDiagnostic { + gateCase.Reasons = append(gateCase.Reasons, "candidate qualification requires an untimed diagnostic replay") + return + } + if telemetry.Diagnostic == nil { + return + } + counterStatus := telemetry.Diagnostic.CounterStatus + if counterStatus == "" { + counterStatus = TraversalTelemetryCounterStatusComplete + } + if required && counterStatus != TraversalTelemetryCounterStatusComplete { + gateCase.Reasons = append(gateCase.Reasons, "candidate qualification requires complete executor counters; diagnostic status is "+string(counterStatus)) + return + } + if required && (isOrientationProbePolicy(summary.EmittedIdentity) || isOrientationProbePolicy(summary.SelectorVersion)) { + requiredFamilies := []TraversalTelemetryFamily{TraversalTelemetryFamilyOrientation, TraversalTelemetryFamilyOrdinary} + if observationRequiresHydration(summary.ObservationMode) { + requiredFamilies = append(requiredFamilies, TraversalTelemetryFamilyHydration) + } + for _, family := range requiredFamilies { + if !slices.Contains(telemetry.Diagnostic.RequiredFamilies, family) { + gateCase.Reasons = append(gateCase.Reasons, "orientation qualification is missing required counter family "+string(family)) + } + } + appendOrientationAttributionReasons(gateCase, telemetry.Diagnostic) + } + if required && summary.EmittedIdentity == optimize.ShortestPathPolicyASPI1GuardedV1 { + appendInlinePredecessorAttributionReasons(gateCase, telemetry.Diagnostic, "inline ASP") + } + if required && summary.EmittedIdentity == optimize.ShortestPathPolicyI1CanonicalGuardedV1 { + appendInlinePredecessorAttributionReasons(gateCase, telemetry.Diagnostic, "inline canonical SP") + } + + observed := traversalNumericObservations(telemetry.Diagnostic.Counters) + gateCase.NumericLimits = make(map[string]int64, len(summary.Caps)) + gateCase.NumericObserved = map[string]int64{} + for name, limit := range summary.Caps { + gateCase.NumericLimits[name] = limit + if limit < 0 { + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("traversal cap %s is negative", name)) + continue + } + value, found := observed[name] + if !found { + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("required numeric traversal counter %s is missing", name)) + continue + } + gateCase.NumericObserved[name] = value + allowed := limit + if traversalCapUsesSentinel(name) { + allowed++ + } + if value > allowed { + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("traversal counter %s=%d exceeds ceiling %d", name, value, allowed)) + } + } +} + +func appendInlineASPAttributionReasons(gateCase *ResourceGateCase, diagnostic *TraversalExecutionDiagnostic) { + appendInlinePredecessorAttributionReasons(gateCase, diagnostic, "inline ASP") +} + +func appendInlinePredecessorAttributionReasons(gateCase *ResourceGateCase, diagnostic *TraversalExecutionDiagnostic, label string) { + if diagnostic == nil || diagnostic.PlanReplay == nil { + gateCase.Reasons = append(gateCase.Reasons, label+" qualification requires exact plan branch evidence") + return + } + counters := diagnostic.PlanReplay.Counters + candidate, candidatePresent := counters["asp_i1_candidate_marker_rows"] + fallback, fallbackPresent := counters["asp_i1_fallback_marker_rows"] + if !candidatePresent || !fallbackPresent || candidate+fallback != 1 { + gateCase.Reasons = append(gateCase.Reasons, label+" execution must attribute exactly one candidate or fallback marker") + } + candidateBranchRows, candidateBranchPresent := counters["asp_i1_candidate_branch_rows"] + fallbackBranchRows, fallbackBranchPresent := counters["asp_i1_fallback_branch_rows"] + if !candidateBranchPresent || !fallbackBranchPresent { + gateCase.Reasons = append(gateCase.Reasons, label+" execution is missing exact candidate or fallback output-branch row evidence") + } + candidateExecutorLoops, candidateExecutorPresent := counters["asp_i1_candidate_executor_loops"] + fallbackExecutorLoops, fallbackExecutorPresent := counters["asp_i1_fallback_executor_loops"] + if !candidateExecutorPresent || !fallbackExecutorPresent { + gateCase.Reasons = append(gateCase.Reasons, label+" execution is missing exact candidate or fallback executor-loop evidence") + } + if candidate == 1 && fallbackBranchRows != 0 { + gateCase.Reasons = append(gateCase.Reasons, label+" fallback output arm emitted rows while the candidate was selected") + } + if candidate == 1 && fallbackExecutorLoops != 0 { + gateCase.Reasons = append(gateCase.Reasons, label+" fallback executor ran while the candidate was selected") + } + if candidate == 1 && candidateExecutorLoops != 1 { + gateCase.Reasons = append(gateCase.Reasons, label+" candidate marker must bind exactly one selected executor loop") + } + if fallback == 1 && candidateBranchRows != 0 { + gateCase.Reasons = append(gateCase.Reasons, label+" candidate output arm emitted rows while fallback was selected") + } + if fallback == 1 && candidateExecutorLoops != 0 { + gateCase.Reasons = append(gateCase.Reasons, label+" candidate executor ran while fallback was selected") + } + if fallback == 1 && fallbackExecutorLoops != 1 { + gateCase.Reasons = append(gateCase.Reasons, label+" fallback marker must bind exactly one selected executor loop") + } +} + +func appendOrientationAttributionReasons(gateCase *ResourceGateCase, diagnostic *TraversalExecutionDiagnostic) { + if diagnostic == nil || diagnostic.PlanReplay == nil { + gateCase.Reasons = append(gateCase.Reasons, "orientation qualification requires exact plan branch and probe evidence") + return + } + counters := diagnostic.PlanReplay.Counters + candidate, candidatePresent := counters["orientation_executed_candidate_rows"] + incumbent, incumbentPresent := counters["orientation_executed_incumbent_rows"] + if !candidatePresent || !incumbentPresent { + gateCase.Reasons = append(gateCase.Reasons, "orientation execution is missing exact selected and unselected arm markers") + } + if candidate+incumbent != 1 { + gateCase.Reasons = append(gateCase.Reasons, "orientation execution must attribute exactly one selected arm and zero unselected-arm work") + } + if candidate == 1 && counters["orientation_incumbent_branch_loops"] != 0 { + gateCase.Reasons = append(gateCase.Reasons, "orientation incumbent arm performed work while the candidate was selected") + } + if incumbent == 1 && counters["orientation_candidate_branch_loops"] != 0 { + gateCase.Reasons = append(gateCase.Reasons, "orientation candidate arm performed work while the incumbent was selected") + } + for _, name := range []string{ + "orientation_root_probe_loops", "orientation_suffix_probe_loops", "orientation_boundary_probe_loops", + "orientation_forward_degree_probe_loops", "orientation_reverse_degree_probe_loops", "orientation_decision_loops", + } { + loops, present := counters[name] + if !present { + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("orientation probe %s has no execution-count evidence", name)) + } else if loops > 1 { + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("orientation probe %s executed more than once", name)) + } + } +} + +// traversalCapUsesSentinel reports whether the counter may observe the +// deliberate cap+1 row used to prove overflow before exact fallback. +func traversalCapUsesSentinel(name string) bool { + return strings.Contains(name, "probe") || strings.Contains(name, "state") || + strings.Contains(name, "frontier") || strings.Contains(name, "queue") || + strings.Contains(name, "seen") || strings.Contains(name, "predecessor") || + strings.Contains(name, "output") +} + +// traversalNumericObservations maps typed diagnostic counters to stable gate names. +func traversalNumericObservations(counters TraversalDiagnosticCounters) map[string]int64 { + observed := map[string]int64{} + set := func(name string, value *int64) { + if value != nil { + observed[name] = *value + } + } + if ordinary := counters.Ordinary; ordinary != nil { + set("root_rows", ordinary.Roots) + set("edge_candidates", ordinary.EdgeCandidates) + set("state_rows", ordinary.PeakState) + set("output_paths", ordinary.EmittedTrails) + } + if orientation := counters.Orientation; orientation != nil { + set("forward_seed_rows", orientation.ForwardSeeds) + set("reverse_seed_rows", orientation.ReverseSeeds) + set("probe_rows", orientation.ProbeRows) + if orientation.ForwardDegreeSamples != nil && orientation.ReverseDegreeSamples != nil { + degreePeak := max(*orientation.ForwardDegreeSamples, *orientation.ReverseDegreeSamples) + observed["directional_degree_rows"] = degreePeak + } + set("survival_rows", orientation.ShallowSurvivalRows) + set("branch_loops", orientation.BranchLoops) + } + if shortest := counters.ShortestPath; shortest != nil { + set("state_rows", shortest.SeenPeak) + set("frontier_rows", shortest.FrontierPeak) + set("queue_rows", shortest.QueuePeak) + set("seen_rows", shortest.SeenPeak) + set("predecessor_rows", shortest.PredecessorPeak) + set("meeting_rows", shortest.MeetingCandidates) + set("witness_rows", shortest.WitnessRows) + } + if all := counters.AllShortestPaths; all != nil { + set("state_rows", all.Search.SeenPeak) + set("frontier_rows", all.Search.FrontierPeak) + set("queue_rows", all.Search.QueuePeak) + set("seen_rows", all.Search.SeenPeak) + set("predecessor_rows", all.PredecessorPeak) + set("output_paths", all.OutputPaths) + set("output_rows", all.EnumeratedCandidates) + set("output_edge_cells", all.OutputEdgeCells) + set("output_bytes", all.OutputBytes) + } + if inline := counters.InlineASP; inline != nil { + set("state_rows", inline.DistanceRows) + set("predecessor_rows", inline.PredecessorRows) + set("output_rows", inline.EnumerationRows) + set("output_paths", inline.OutputPaths) + set("output_bytes", inline.OutputBytes) + } + if inline := counters.InlineShortestPath; inline != nil { + set("state_rows", inline.DistanceRows) + set("predecessor_rows", inline.PredecessorRows) + set("output_rows", inline.EnumerationRows) + set("output_paths", inline.OutputPaths) + set("output_bytes", inline.OutputBytes) + } + if hydration := counters.Hydration; hydration != nil { + set("hydration_rows", hydration.Rows) + set("hydration_bytes", hydration.Bytes) + } + return observed +} + +// appendWorkspaceResourceReasons adds failures for excessive executor or session workspace usage. +func appendWorkspaceResourceReasons(gateCase *ResourceGateCase, metrics *PostgresPlanMetrics) { + if metrics.Buffers.TempRead != 0 || metrics.Buffers.TempWritten != 0 { + gateCase.Reasons = append(gateCase.Reasons, "compact workspace candidate spilled to executor temporary storage") + } + if metrics.WALRecords != 0 || metrics.WALBytes != 0 { + gateCase.Reasons = append(gateCase.Reasons, "non-mutating compact workspace candidate emitted WAL") + } +} + +// appendPortableResourceReasons adds failures for spill, loops, or cardinality evidence that violates portable limits. +func appendPortableResourceReasons(gateCase *ResourceGateCase, metrics *PostgresPlanMetrics) { + buffers := metrics.Buffers + if buffers.TempRead != 0 || buffers.TempWritten != 0 { + gateCase.Reasons = append(gateCase.Reasons, "portable candidate used temporary buffers") + } + if buffers.LocalHit != 0 || buffers.LocalRead != 0 || buffers.LocalDirtied != 0 || buffers.LocalWritten != 0 { + gateCase.Reasons = append(gateCase.Reasons, "portable candidate used local workspace") + } + if metrics.WALRecords != 0 || metrics.WALBytes != 0 { + gateCase.Reasons = append(gateCase.Reasons, "non-mutating portable candidate emitted WAL") + } +} + +// postgresPlanFunctionLoops sums actual loops for PostgreSQL plan nodes invoking the named function. +func postgresPlanFunctionLoops(raw json.RawMessage, function string) (int64, bool, error) { + if len(raw) == 0 { + return 0, false, nil + } + var document []map[string]any + if err := json.Unmarshal(raw, &document); err != nil { + return 0, false, err + } + if len(document) == 0 { + return 0, false, nil + } + root, ok := document[0]["Plan"].(map[string]any) + if !ok { + return 0, false, nil + } + var loops int64 + found := false + var walk func(map[string]any) + walk = func(node map[string]any) { + alias, _ := node["Alias"].(string) + functionName, _ := node["Function Name"].(string) + if alias == function || functionName == function { + found = true + if actualLoops, ok := node["Actual Loops"].(float64); ok { + loops += int64(actualLoops) + } + } + children, _ := node["Plans"].([]any) + for _, child := range children { + if childNode, ok := child.(map[string]any); ok { + walk(childNode) + } + } + } + walk(root) + return loops, found, nil +} + +// appliedPostgresArchitecture returns the effective PostgreSQL executor architecture, including fallback attribution. +func appliedPostgresArchitecture(record CaseResult) string { + if record.Optimization == nil { + return "" + } + for _, outcome := range record.Optimization.TargetOutcomes { + if outcome.Family == "SP" || outcome.Family == "ASP" || outcome.Family == "fixed_suffix_expansion" || outcome.Family == "fixed_prefix_terminal_expansion" { + if outcome.Applied != "" { + return outcome.Applied + } + return outcome.Selected + } + } + return "" +} diff --git a/cmd/graphbench/resource_gate_test.go b/cmd/graphbench/resource_gate_test.go new file mode 100644 index 00000000..fabd6c7c --- /dev/null +++ b/cmd/graphbench/resource_gate_test.go @@ -0,0 +1,857 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/stretchr/testify/require" +) + +// TestResourceGateReportBindsExactInputArtifact verifies that schema v5 reports +// retain the SHA-256 digest of the exact JSONL bytes supplied to the gate. +func TestResourceGateReportBindsExactInputArtifact(t *testing.T) { + tempDir := t.TempDir() + artifact := filepath.Join(tempDir, "records.jsonl") + record := CaseResult{ + Environment: &RunEnvironment{Round: 3, Block: 3, RunUUID: "resource-run", Arm: "candidate", ArmOrder: 2}, + Dataset: "fixture", + Name: "case", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{FixtureTier: "normal"}, + Optimization: &translate.OptimizationSummary{ + TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: "SP", + Applied: "SP-S4-C-D", + }}, + }, + PostgresMetrics: &PostgresPlanMetrics{}, + } + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + artifactRaw, err := os.ReadFile(artifact) + require.NoError(t, err) + expectedDigest := sha256.Sum256(artifactRaw) + + output := filepath.Join(tempDir, "report.json") + passed, err := createResourceGateReport(artifact, output) + require.NoError(t, err) + require.True(t, passed) + + var report ResourceGateReport + reportRaw, err := os.ReadFile(output) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(reportRaw, &report)) + require.Equal(t, resourceGateVersion, report.Version) + require.Equal(t, hex.EncodeToString(expectedDigest[:]), report.ArtifactSHA256) + require.True(t, isLowerHexSHA256(report.ArtifactSHA256)) + require.Equal(t, 3, report.Cases[0].Round) + require.Equal(t, 3, report.Cases[0].Block) + require.Equal(t, "resource-run", report.Cases[0].RunUUID) + require.Equal(t, "candidate", report.Cases[0].Arm) + require.Equal(t, 2, report.Cases[0].ArmOrder) +} + +// TestResourceGateAllowsCompactSessionWorkspaceButRejectsExecutorSpill verifies that local workspace writes are permitted for the compact architecture while temporary-buffer spill fails the gate. +func TestResourceGateAllowsCompactSessionWorkspaceButRejectsExecutorSpill(t *testing.T) { + artifact := filepath.Join(t.TempDir(), "records.jsonl") + record := CaseResult{ + Dataset: "fixture", + Name: "case", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{ + FixtureTier: "normal", + }, + Optimization: &translate.OptimizationSummary{ + TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: "SP", + Applied: "SP-S4-C-D", + }}, + }, + PostgresMetrics: &PostgresPlanMetrics{ + Buffers: Buffers{ + LocalWritten: 1, + }, + }, + } + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + passed, err := createResourceGateReport(artifact, filepath.Join(t.TempDir(), "report.json")) + require.NoError(t, err) + require.True(t, passed) + + record.PostgresMetrics.Buffers.TempWritten = 1 + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + passed, err = createResourceGateReport(artifact, filepath.Join(t.TempDir(), "spill-report.json")) + require.NoError(t, err) + require.False(t, passed) +} + +// TestResourceGateRecognizesCompactBidirectionalWorkspaceArchitectures freezes +// local-workspace attribution for production and full-comparator B1/B2 arms. +func TestResourceGateRecognizesCompactBidirectionalWorkspaceArchitectures(t *testing.T) { + for _, architecture := range []string{ + "SP-B1-C-ALT-NODE-D", + "SP-B1-C-ALT-NODE-WE+MAT-M0", + "SP-B2-C-MIN-LEVEL-D", + "SP-B2-C-MIN-LEVEL-WE+MAT-M0", + } { + require.True(t, compactWorkspaceArchitecture(architecture), architecture) + require.True(t, compactBidirectionalWorkspaceArchitecture(architecture), architecture) + } + require.True(t, compactWorkspaceArchitecture("SP-S4-C-D")) + require.False(t, compactBidirectionalWorkspaceArchitecture("SP-S4-C-D")) +} + +// TestResourceGateRecognizesASPProductionArchitecture verifies that the applied all-shortest-path lowering, rather than a fallback label, identifies the production architecture. +func TestResourceGateRecognizesASPProductionArchitecture(t *testing.T) { + record := CaseResult{ + Optimization: &translate.OptimizationSummary{ + TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: "ASP", + Applied: "ASP-A1-DAG", + }}, + }, + } + require.Equal(t, "ASP-A1-DAG", appliedPostgresArchitecture(record)) +} + +// TestResourceGateChecksFullComparatorReferenceResources verifies that temporary-buffer usage in a full comparator becomes its own failing report case with arm attribution. +func TestResourceGateChecksFullComparatorReferenceResources(t *testing.T) { + artifact := filepath.Join(t.TempDir(), "records.jsonl") + record := CaseResult{ + Dataset: "fixture", + Name: "case", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{ + FixtureTier: "normal", + }, + PostgresReferences: []PostgresReferenceResult{{ + Name: "s4", + Architecture: "SP-S4-C-D", + FullComparator: true, + PostgresMetrics: &PostgresPlanMetrics{ + Buffers: Buffers{ + TempWritten: 1, + }, + }, + }}, + } + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + output := filepath.Join(t.TempDir(), "report.json") + passed, err := createResourceGateReport(artifact, output) + require.NoError(t, err) + require.False(t, passed) + + var report ResourceGateReport + raw, err := os.ReadFile(output) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(raw, &report)) + require.Len(t, report.Cases, 2) + require.Equal(t, "s4", report.Cases[1].Reference) + require.Contains(t, report.Cases[1].Reasons, "portable candidate used temporary buffers") +} + +// TestResourceGateAttributesDirectPreflightIncumbentFallback verifies that a direct-preflight plan executing the recursive harness is attributed to SP-S0 fallback while a skipped harness remains direct. +func TestResourceGateAttributesDirectPreflightIncumbentFallback(t *testing.T) { + artifact := filepath.Join(t.TempDir(), "records.jsonl") + records := []CaseResult{ + { + Dataset: "fixture", + Name: "fallback", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{ + FixtureTier: "normal", + }, + Optimization: &translate.OptimizationSummary{ + TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: "SP", + Applied: "SP-S0-DIRECT", + }}, + }, + PostgresMetrics: &PostgresPlanMetrics{ + Buffers: Buffers{ + LocalWritten: 1, + }, + }, + PostgresPlanJSON: json.RawMessage(`[{"Plan":{"Plans":[{"Alias":"bidirectional_sp_harness","Actual Loops":1}]}}]`), + }, + { + Dataset: "fixture", + Name: "direct", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{ + FixtureTier: "normal", + }, + Optimization: &translate.OptimizationSummary{ + TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: "SP", + Applied: "SP-S0-DIRECT", + }}, + }, + PostgresPlanJSON: json.RawMessage(`[{"Plan":{"Plans":[{"Function Name":"bidirectional_sp_harness","Actual Loops":0}]}}]`), + PostgresMetrics: &PostgresPlanMetrics{}, + }, + } + require.NoError(t, writeJSONLFile(artifact, records)) + output := filepath.Join(t.TempDir(), "report.json") + passed, err := createResourceGateReport(artifact, output) + require.NoError(t, err) + require.True(t, passed) + + var report ResourceGateReport + raw, err := os.ReadFile(output) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(raw, &report)) + require.Equal(t, "SP-S0", report.Cases[1].FallbackArchitecture) +} + +// TestResourceGateFailsClosedWithoutStructuredMetrics verifies that a successful portable candidate still fails resource gating when structured PostgreSQL metrics are absent. +func TestResourceGateFailsClosedWithoutStructuredMetrics(t *testing.T) { + artifact := filepath.Join(t.TempDir(), "records.jsonl") + record := CaseResult{ + Dataset: "fixture", + Name: "missing-metrics", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{FixtureTier: "normal"}, + Optimization: &translate.OptimizationSummary{ + TargetOutcomes: []translate.TargetLoweringOutcome{{Family: "SP", Applied: "SP-S4-C-D"}}, + }, + } + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + output := filepath.Join(t.TempDir(), "report.json") + passed, err := createResourceGateReport(artifact, output) + require.NoError(t, err) + require.False(t, passed) + + var report ResourceGateReport + raw, err := os.ReadFile(output) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(raw, &report)) + require.Contains(t, report.Cases[0].Reasons, "structured PostgreSQL plan metrics are missing") +} + +// TestResourceGateRejectsDirectPreflightWorkspaceOnDirectHit verifies that a true direct hit cannot claim local workspace writes when the recursive harness executed zero times. +func TestResourceGateRejectsDirectPreflightWorkspaceOnDirectHit(t *testing.T) { + artifact := filepath.Join(t.TempDir(), "records.jsonl") + record := CaseResult{ + Dataset: "fixture", + Name: "direct", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{ + FixtureTier: "normal", + }, + Optimization: &translate.OptimizationSummary{ + TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: "SP", + Applied: "SP-S0-DIRECT", + }}, + }, + PostgresMetrics: &PostgresPlanMetrics{ + Buffers: Buffers{ + LocalWritten: 1, + }, + }, + PostgresPlanJSON: json.RawMessage(`[{"Plan":{"Plans":[{"Alias":"bidirectional_sp_harness","Actual Loops":0}]}}]`), + } + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + passed, err := createResourceGateReport(artifact, filepath.Join(t.TempDir(), "report.json")) + require.NoError(t, err) + require.False(t, passed) +} + +// TestResourceGateAllowsStressDiagnosticsAndExactFallback verifies that spill is diagnostic on stress fixtures and compact workspace use is allowed for an explicitly selected exact fallback. +func TestResourceGateAllowsStressDiagnosticsAndExactFallback(t *testing.T) { + artifact := filepath.Join(t.TempDir(), "records.jsonl") + records := []CaseResult{ + { + Dataset: "fixture", + Name: "stress", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{ + FixtureTier: "stress", + }, + PostgresMetrics: &PostgresPlanMetrics{ + Buffers: Buffers{ + TempWritten: 1, + }, + }, + }, + { + Dataset: "fixture", + Name: "fallback", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{ + FixtureTier: "normal", + }, + Optimization: &translate.OptimizationSummary{ + TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: "SP", + Selected: "SP-S0", + }}, + }, + PostgresMetrics: &PostgresPlanMetrics{ + Buffers: Buffers{ + LocalWritten: 1, + }, + }, + }, + } + require.NoError(t, writeJSONLFile(artifact, records)) + passed, err := createResourceGateReport(artifact, filepath.Join(t.TempDir(), "report.json")) + require.NoError(t, err) + require.True(t, passed) +} + +// TestResourceGateEnforcesTelemetryIdentityAndNumericSentinels verifies a +// candidate may observe exactly cap+1, while larger work or contradictory +// runtime attribution fails closed. +func TestResourceGateEnforcesTelemetryIdentityAndNumericSentinels(t *testing.T) { + artifact := filepath.Join(t.TempDir(), "records.jsonl") + telemetry := validTraversalTelemetry() + telemetry.Level = TraversalTelemetryLevelDiagnostic + telemetry.Summary.RequestedIdentity = "SP-B1-C-ALT-NODE-D" + telemetry.Summary.PlannedIdentities = []string{"SP-B1-C-ALT-NODE-D", "SP-S4-C-D"} + telemetry.Summary.EmittedIdentity = "sp-bidirectional-tournament-v1" + telemetry.Summary.RuntimeIdentity = "SP-B1-C-ALT-NODE-D" + telemetry.Summary.AppliedIdentity = "SP-B1-C-ALT-NODE-D" + telemetry.Summary.RuntimeOutcomeAvailable = telemetryBool(true) + telemetry.Summary.Provenance["runtime_outcome_available"] = "executor.receipt" + telemetry.Summary.Caps = map[string]int64{"state_rows": 32} + telemetry.Summary.Provenance["caps.state_rows"] = "policy.state_cap" + delete(telemetry.Summary.Provenance, "caps.state") + telemetry.Diagnostic = ordinaryDiagnostic() + telemetry.Diagnostic.Counters.Ordinary.PeakState = telemetryInt64(33) + record := CaseResult{ + Dataset: "fixture", + Name: "candidate", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{FixtureTier: "envelope", FallbackExpectation: "forbidden"}, + Environment: &RunEnvironment{PoolSize: 1, SessionMemoryCeilingBytes: 1 << 20, PoolMemoryCeilingBytes: 1 << 20}, + TraversalTelemetry: &telemetry, + PostgresMetrics: &PostgresPlanMetrics{}, + Optimization: &translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: "SP", Applied: "SP-B1-C-ALT-NODE-D", + }}, + }, + } + record.TraversalTelemetry.Diagnostic.Counters.Workspace = &TraversalWorkspaceCounters{ + SessionPeakBytes: telemetryInt64(4096), PoolPeakBytes: telemetryInt64(4096), + } + record.TraversalTelemetry.Diagnostic.RequiredFamilies = append( + record.TraversalTelemetry.Diagnostic.RequiredFamilies, + TraversalTelemetryFamilyWorkspace, + ) + record.TraversalTelemetry.Diagnostic.Provenance["workspace.session_peak_bytes"] = "test.session" + record.TraversalTelemetry.Diagnostic.Provenance["workspace.pool_peak_bytes"] = "test.pool" + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + passingReportPath := filepath.Join(t.TempDir(), "cap-plus-one.json") + passed, err := createResourceGateReport(artifact, passingReportPath) + require.NoError(t, err) + passingReportRaw, err := os.ReadFile(passingReportPath) + require.NoError(t, err) + var passingReport ResourceGateReport + require.NoError(t, json.Unmarshal(passingReportRaw, &passingReport)) + require.True(t, passed, passingReport.Cases) + + telemetry.Diagnostic.Counters.Ordinary.PeakState = telemetryInt64(34) + record.TraversalTelemetry = &telemetry + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + passed, err = createResourceGateReport(artifact, filepath.Join(t.TempDir(), "overflow.json")) + require.NoError(t, err) + require.False(t, passed) + + telemetry.Diagnostic.Counters.Ordinary.PeakState = telemetryInt64(32) + telemetry.Summary.AppliedIdentity = "SP-S4-C-D" + record.TraversalTelemetry = &telemetry + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + passed, err = createResourceGateReport(artifact, filepath.Join(t.TempDir(), "identity.json")) + require.NoError(t, err) + require.False(t, passed) +} + +// TestResourceGateRequiresDiagnosticTelemetryForBidirectionalCandidates verifies +// opaque function work cannot qualify from outer EXPLAIN evidence alone. +func TestResourceGateRequiresDiagnosticTelemetryForBidirectionalCandidates(t *testing.T) { + artifact := filepath.Join(t.TempDir(), "records.jsonl") + record := CaseResult{ + Dataset: "fixture", + Name: "missing-telemetry", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{FixtureTier: "normal"}, + PostgresMetrics: &PostgresPlanMetrics{}, + Optimization: &translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: "SP", Applied: "SP-B2-C-MIN-LEVEL-D", + }}, + }, + } + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + passed, err := createResourceGateReport(artifact, filepath.Join(t.TempDir(), "missing.json")) + require.NoError(t, err) + require.False(t, passed) + + telemetry := validTraversalTelemetry() + telemetry.Level = TraversalTelemetryLevelDiagnostic + telemetry.Summary.RequestedIdentity = "SP-B2-C-MIN-LEVEL-D" + telemetry.Summary.PlannedIdentities = []string{"SP-B2-C-MIN-LEVEL-D", "SP-S4-C-D"} + telemetry.Summary.EmittedIdentity = "sp-bidirectional-tournament-v1" + telemetry.Summary.RuntimeIdentity = "SP-B2-C-MIN-LEVEL-D" + telemetry.Summary.AppliedIdentity = "SP-B2-C-MIN-LEVEL-D" + telemetry.Diagnostic = ordinaryDiagnostic() + telemetry.Diagnostic.CounterStatus = TraversalTelemetryCounterStatusHiddenUnavailable + telemetry.Diagnostic.IncompleteReasons = []string{"function scan hides invocation counters"} + record.TraversalTelemetry = &telemetry + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + output := filepath.Join(t.TempDir(), "incomplete.json") + passed, err = createResourceGateReport(artifact, output) + require.NoError(t, err) + require.False(t, passed) + + var report ResourceGateReport + raw, err := os.ReadFile(output) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(raw, &report)) + require.Contains(t, report.Cases[0].Reasons, "candidate qualification requires complete executor counters; diagnostic status is hidden_counters_unavailable") +} + +func TestResourceGateRejectsDeclaredMemoryCeilingsWithoutMeasuredWorkspace(t *testing.T) { + artifact := filepath.Join(t.TempDir(), "records.jsonl") + record := CaseResult{ + Dataset: "fixture", Name: "declared-only", ExecutionMode: ModePostgresSQL, Status: StatusOK, + Shape: WorkloadShape{FixtureTier: "normal"}, PostgresMetrics: &PostgresPlanMetrics{}, + Environment: &RunEnvironment{PoolSize: 1, SessionMemoryCeilingBytes: 1024, PoolMemoryCeilingBytes: 4096}, + Optimization: &translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{{Family: "SP", Applied: "SP-S4-C-D"}}}, + } + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + output := filepath.Join(t.TempDir(), "report.json") + passed, err := createResourceGateReport(artifact, output) + require.NoError(t, err) + require.False(t, passed) + + var report ResourceGateReport + raw, err := os.ReadFile(output) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(raw, &report)) + require.Contains(t, report.Cases[0].Reasons, "declared workspace memory ceilings lack measured session and pool high-water evidence") +} + +func TestResourceGateRequiresCompleteOrientationPolicyAndExactBranchAttribution(t *testing.T) { + artifact := filepath.Join(t.TempDir(), "records.jsonl") + telemetry := validTraversalTelemetry() + telemetry.Level = TraversalTelemetryLevelDiagnostic + telemetry.Summary.RequestedIdentity = "EXPANSION-SUFFIX-SEEDED-REVERSE" + telemetry.Summary.PlannedIdentities = []string{"EXPANSION-SUFFIX-SEEDED-REVERSE", "EXPANSION-STEPWISE-FORWARD"} + telemetry.Summary.EmittedIdentity = "orientation-probe-v1" + telemetry.Summary.RuntimeIdentity = "EXPANSION-SUFFIX-SEEDED-REVERSE" + telemetry.Summary.AppliedIdentity = "EXPANSION-SUFFIX-SEEDED-REVERSE" + telemetry.Summary.SelectorVersion = "orientation-probe-v1" + telemetry.Diagnostic = ordinaryDiagnostic() + telemetry.Diagnostic.CounterStatus = TraversalTelemetryCounterStatusPlanPartial + telemetry.Diagnostic.IncompleteReasons = []string{"plan evidence only"} + telemetry.Diagnostic.PlanReplay = &TraversalPlanReplayEvidence{ + Source: "test", Counters: map[string]int64{"orientation_executed_candidate_rows": 1}, Flags: map[string]bool{}, + Provenance: map[string]string{"counters.orientation_executed_candidate_rows": "test.marker"}, + } + record := CaseResult{ + Dataset: "fixture", Name: "orientation", ExecutionMode: ModePostgresSQL, Status: StatusOK, + Shape: WorkloadShape{FixtureTier: "normal"}, PostgresMetrics: &PostgresPlanMetrics{}, TraversalTelemetry: &telemetry, + Optimization: &translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: "fixed_suffix_expansion", Applied: "EXPANSION-SUFFIX-SEEDED-REVERSE", EmittedPolicy: "orientation-probe-v1", + }}}, + } + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + passed, err := createResourceGateReport(artifact, filepath.Join(t.TempDir(), "report.json")) + require.NoError(t, err) + require.False(t, passed) +} + +func TestResourceGateScopesStressFallbackToDeclaredExpectation(t *testing.T) { + artifact := filepath.Join(t.TempDir(), "records.jsonl") + withoutExpectation := CaseResult{ + Dataset: "fixture", Name: "stress-no-overflow", ExecutionMode: ModePostgresSQL, Status: StatusOK, + Shape: WorkloadShape{FixtureTier: "stress"}, PostgresMetrics: &PostgresPlanMetrics{}, + Optimization: &translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{{Family: "SP", Applied: "SP-S0"}}}, + } + require.NoError(t, writeJSONLFile(artifact, []CaseResult{withoutExpectation})) + passed, err := createResourceGateReport(artifact, filepath.Join(t.TempDir(), "no-expectation.json")) + require.NoError(t, err) + require.True(t, passed) + + withExpectation := withoutExpectation + withExpectation.Name = "stress-overflow" + withExpectation.Shape.FallbackExpectation = "required" + telemetry := validTraversalTelemetry() + telemetry.Summary.FallbackExecuted = telemetryBool(false) + withExpectation.TraversalTelemetry = &telemetry + require.NoError(t, writeJSONLFile(artifact, []CaseResult{withExpectation})) + passed, err = createResourceGateReport(artifact, filepath.Join(t.TempDir(), "expected.json")) + require.NoError(t, err) + require.False(t, passed) +} + +func TestResourceGateValidatesExactOrientationMarkersAndProbeCounts(t *testing.T) { + probeCounters := map[string]int64{ + "orientation_executed_candidate_rows": 1, + "orientation_executed_incumbent_rows": 0, + "orientation_root_probe_loops": 1, + "orientation_suffix_probe_loops": 1, + "orientation_boundary_probe_loops": 1, + "orientation_forward_degree_probe_loops": 1, + "orientation_reverse_degree_probe_loops": 1, + "orientation_decision_loops": 1, + "orientation_candidate_branch_loops": 1, + "orientation_incumbent_branch_loops": 0, + } + diagnostic := &TraversalExecutionDiagnostic{PlanReplay: &TraversalPlanReplayEvidence{Counters: probeCounters}} + gateCase := &ResourceGateCase{} + appendOrientationAttributionReasons(gateCase, diagnostic) + require.Empty(t, gateCase.Reasons) + + probeCounters["orientation_executed_incumbent_rows"] = 1 + probeCounters["orientation_root_probe_loops"] = 2 + delete(probeCounters, "orientation_suffix_probe_loops") + appendOrientationAttributionReasons(gateCase, diagnostic) + require.Contains(t, strings.Join(gateCase.Reasons, "\n"), "exactly one selected arm") + require.Contains(t, strings.Join(gateCase.Reasons, "\n"), "executed more than once") + require.Contains(t, strings.Join(gateCase.Reasons, "\n"), "no execution-count evidence") + + probeCounters["orientation_executed_incumbent_rows"] = 0 + probeCounters["orientation_root_probe_loops"] = 1 + probeCounters["orientation_suffix_probe_loops"] = 1 + probeCounters["orientation_incumbent_branch_loops"] = 1 + gateCase.Reasons = nil + appendOrientationAttributionReasons(gateCase, diagnostic) + require.Contains(t, gateCase.Reasons, "orientation incumbent arm performed work while the candidate was selected") +} + +func TestResourceGateRequiresSingularInlineASPBranchAndInactiveArm(t *testing.T) { + gateCase := &ResourceGateCase{} + diagnostic := &TraversalExecutionDiagnostic{PlanReplay: &TraversalPlanReplayEvidence{Counters: map[string]int64{ + "asp_i1_candidate_marker_rows": 1, + "asp_i1_fallback_marker_rows": 0, + "asp_i1_candidate_branch_rows": 1, + "asp_i1_fallback_branch_rows": 0, + "asp_i1_candidate_executor_loops": 1, + "asp_i1_fallback_executor_loops": 0, + }}} + appendInlineASPAttributionReasons(gateCase, diagnostic) + require.Empty(t, gateCase.Reasons) + + for _, missing := range []string{"asp_i1_candidate_branch_rows", "asp_i1_fallback_branch_rows"} { + value := diagnostic.PlanReplay.Counters[missing] + delete(diagnostic.PlanReplay.Counters, missing) + missingCase := &ResourceGateCase{} + appendInlineASPAttributionReasons(missingCase, diagnostic) + require.Contains(t, missingCase.Reasons, "inline ASP execution is missing exact candidate or fallback output-branch row evidence") + diagnostic.PlanReplay.Counters[missing] = value + } + for _, missing := range []string{"asp_i1_candidate_executor_loops", "asp_i1_fallback_executor_loops"} { + value := diagnostic.PlanReplay.Counters[missing] + delete(diagnostic.PlanReplay.Counters, missing) + missingCase := &ResourceGateCase{} + appendInlineASPAttributionReasons(missingCase, diagnostic) + require.Contains(t, missingCase.Reasons, "inline ASP execution is missing exact candidate or fallback executor-loop evidence") + diagnostic.PlanReplay.Counters[missing] = value + } + + diagnostic.PlanReplay.Counters["asp_i1_fallback_executor_loops"] = 1 + executedInactiveCase := &ResourceGateCase{} + appendInlineASPAttributionReasons(executedInactiveCase, diagnostic) + require.Contains(t, executedInactiveCase.Reasons, "inline ASP fallback executor ran while the candidate was selected") + diagnostic.PlanReplay.Counters["asp_i1_fallback_executor_loops"] = 0 + + diagnostic.PlanReplay.Counters["asp_i1_fallback_marker_rows"] = 1 + diagnostic.PlanReplay.Counters["asp_i1_fallback_branch_rows"] = 1 + appendInlineASPAttributionReasons(gateCase, diagnostic) + require.Contains(t, gateCase.Reasons, "inline ASP execution must attribute exactly one candidate or fallback marker") + require.Contains(t, gateCase.Reasons, "inline ASP fallback output arm emitted rows while the candidate was selected") +} + +func TestResourceGateScopesGuardedI1TelemetryAndInactiveArm(t *testing.T) { + require.False(t, telemetryRequiredForArchitecture(string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness))) + require.False(t, telemetryRequiredForArchitecture(string(optimize.ShortestPathExecutorASPI1DAG))) + require.True(t, telemetryRequiredForRecord( + guardedI1ResourceRecord(string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness)), + string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + )) + + gateCase := &ResourceGateCase{} + diagnostic := &TraversalExecutionDiagnostic{PlanReplay: &TraversalPlanReplayEvidence{Counters: map[string]int64{ + "asp_i1_candidate_marker_rows": 1, + "asp_i1_fallback_marker_rows": 0, + "asp_i1_candidate_branch_rows": 1, + "asp_i1_fallback_branch_rows": 0, + "asp_i1_candidate_executor_loops": 1, + "asp_i1_fallback_executor_loops": 0, + }}} + appendInlinePredecessorAttributionReasons(gateCase, diagnostic, "inline canonical SP") + require.Empty(t, gateCase.Reasons) + + diagnostic.PlanReplay.Counters["asp_i1_fallback_marker_rows"] = 1 + diagnostic.PlanReplay.Counters["asp_i1_fallback_branch_rows"] = 1 + appendInlinePredecessorAttributionReasons(gateCase, diagnostic, "inline canonical SP") + require.Contains(t, gateCase.Reasons, "inline canonical SP execution must attribute exactly one candidate or fallback marker") + require.Contains(t, gateCase.Reasons, "inline canonical SP fallback output arm emitted rows while the candidate was selected") +} + +func TestResourceGateDoesNotRequireGuardedTelemetryForExplicitI1References(t *testing.T) { + artifact := filepath.Join(t.TempDir(), "records.jsonl") + record := CaseResult{ + Dataset: "fixture", + Name: "explicit-references", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{FixtureTier: "normal"}, + PostgresMetrics: &PostgresPlanMetrics{}, + PostgresReferences: []PostgresReferenceResult{ + { + Name: "sp-i1-reference", Architecture: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + FullComparator: true, PostgresMetrics: &PostgresPlanMetrics{}, + }, + { + Name: "asp-i1-reference", Architecture: string(optimize.ShortestPathExecutorASPI1DAG), + FullComparator: true, PostgresMetrics: &PostgresPlanMetrics{}, + }, + }, + } + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + output := filepath.Join(t.TempDir(), "report.json") + passed, err := createResourceGateReport(artifact, output) + require.NoError(t, err) + require.True(t, passed) + + var report ResourceGateReport + raw, err := os.ReadFile(output) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(raw, &report)) + require.Len(t, report.Cases, 3) + for _, gateCase := range report.Cases { + require.True(t, gateCase.Passed, "%+v", gateCase) + require.NotContains(t, gateCase.Reasons, "required traversal execution telemetry is missing") + } +} + +func TestResourceGateBindsGuardedI1PolicyAndCounterNamespace(t *testing.T) { + tests := []struct { + name string + architecture string + mutate func(*CaseResult) + passed bool + reason string + }{ + { + name: "canonical SP valid", + architecture: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + passed: true, + }, + { + name: "ASP valid", + architecture: string(optimize.ShortestPathExecutorASPI1DAG), + passed: true, + }, + { + name: "canonical SP missing outcome policy", + architecture: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + mutate: func(record *CaseResult) { + record.Optimization.TargetOutcomes[0].EmittedPolicy = "" + }, + reason: "inline canonical SP production architecture requires emitted policy", + }, + { + name: "canonical SP wrong telemetry policy", + architecture: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + mutate: func(record *CaseResult) { + record.TraversalTelemetry.Summary.EmittedIdentity = optimize.ShortestPathPolicyASPI1GuardedV1 + }, + reason: "inline canonical SP production telemetry requires emitted identity", + }, + { + name: "canonical SP wrong counter namespace", + architecture: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + mutate: func(record *CaseResult) { + diagnostic := record.TraversalTelemetry.Diagnostic + diagnostic.Counters.InlineASP = diagnostic.Counters.InlineShortestPath + diagnostic.Counters.InlineShortestPath = nil + diagnostic.RequiredFamilies = []TraversalTelemetryFamily{TraversalTelemetryFamilyASP, TraversalTelemetryFamilyHydration} + diagnostic.Provenance = guardedI1CounterProvenance("inline_asp") + }, + reason: "inline canonical SP production telemetry requires inline_shortest_path counters", + }, + { + name: "canonical SP missing contract counter family", + architecture: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + mutate: func(record *CaseResult) { + record.TraversalTelemetry.Diagnostic.RequiredFamilies = []TraversalTelemetryFamily{TraversalTelemetryFamilyHydration} + }, + reason: `inline canonical SP production telemetry requires declared counter family "shortest_path"`, + }, + { + name: "ASP missing hydration family", + architecture: string(optimize.ShortestPathExecutorASPI1DAG), + mutate: func(record *CaseResult) { + diagnostic := record.TraversalTelemetry.Diagnostic + diagnostic.RequiredFamilies = []TraversalTelemetryFamily{TraversalTelemetryFamilyASP} + diagnostic.Counters.Hydration = nil + }, + reason: "inline ASP production telemetry requires declared hydration counters for its observation mode", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + record := guardedI1ResourceRecord(test.architecture) + if test.mutate != nil { + test.mutate(&record) + } + artifact := filepath.Join(t.TempDir(), "records.jsonl") + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + output := filepath.Join(t.TempDir(), "report.json") + passed, err := createResourceGateReport(artifact, output) + require.NoError(t, err) + require.Equal(t, test.passed, passed) + + var report ResourceGateReport + raw, err := os.ReadFile(output) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(raw, &report)) + require.Len(t, report.Cases, 1) + if test.reason != "" { + require.Contains(t, strings.Join(report.Cases[0].Reasons, "\n"), test.reason) + } + }) + } +} + +func guardedI1ResourceRecord(architecture string) CaseResult { + contract, _ := guardedInlineResourceContractForArchitecture(architecture) + fallback := string(optimize.ShortestPathExecutorS4CanonicalWitness) + requiredFamily := TraversalTelemetryFamilySP + observationMode := "one_path" + if architecture == string(optimize.ShortestPathExecutorASPI1DAG) { + fallback = string(optimize.ShortestPathExecutorASPA1DAG) + requiredFamily = TraversalTelemetryFamilyASP + observationMode = "all_paths" + } + + inlineCounters := &InlinePredecessorTraversalCounters{ + DistanceRows: telemetryInt64(3), + PredecessorRows: telemetryInt64(2), + EnumerationRows: telemetryInt64(1), + OutputPaths: telemetryInt64(1), + OutputBytes: telemetryInt64(64), + CandidateMarkerRows: telemetryInt64(1), + FallbackMarkerRows: telemetryInt64(0), + CandidateBranchRows: telemetryInt64(1), + FallbackBranchRows: telemetryInt64(0), + CandidateExecutorLoops: telemetryInt64(1), + FallbackExecutorLoops: telemetryInt64(0), + } + diagnosticCounters := TraversalDiagnosticCounters{} + if requiredFamily == TraversalTelemetryFamilySP { + diagnosticCounters.InlineShortestPath = inlineCounters + } else { + diagnosticCounters.InlineASP = inlineCounters + } + diagnosticCounters.Hydration = &TraversalHydrationCounters{ + PathCount: telemetryInt64(1), NodeLookups: telemetryInt64(2), EdgeLookups: telemetryInt64(1), + Loops: telemetryInt64(1), Rows: telemetryInt64(1), TimeNS: telemetryInt64(100), Bytes: telemetryInt64(64), + } + planCounters := map[string]int64{ + "asp_i1_distance_rows": 3, + "asp_i1_predecessor_rows": 2, + "asp_i1_enumeration_rows": 1, + "asp_i1_output_rows": 1, + "asp_i1_candidate_marker_rows": 1, + "asp_i1_fallback_marker_rows": 0, + "asp_i1_candidate_branch_rows": 1, + "asp_i1_fallback_branch_rows": 0, + "asp_i1_candidate_executor_loops": 1, + "asp_i1_fallback_executor_loops": 0, + } + planProvenance := map[string]string{} + for name := range planCounters { + planProvenance["counters."+name] = "test.plan." + name + } + + telemetry := validTraversalTelemetry() + telemetry.Level = TraversalTelemetryLevelDiagnostic + telemetry.Summary.RequestedIdentity = architecture + telemetry.Summary.PlannedIdentities = []string{architecture, fallback} + telemetry.Summary.EmittedIdentity = contract.policy + telemetry.Summary.RuntimeIdentity = architecture + telemetry.Summary.AppliedIdentity = architecture + telemetry.Summary.ObservationMode = observationMode + telemetry.Summary.RuntimeOutcomeAvailable = telemetryBool(true) + telemetry.Summary.Caps = map[string]int64{ + "state_rows": 100, "predecessor_rows": 100, "output_rows": 100, "output_bytes": 1024, + } + telemetry.Summary.Provenance["observation_mode"] = "test.observation" + telemetry.Summary.Provenance["runtime_outcome_available"] = "test.receipt" + for capName := range telemetry.Summary.Caps { + telemetry.Summary.Provenance["caps."+capName] = "test.cap." + capName + } + telemetry.Diagnostic = &TraversalExecutionDiagnostic{ + InvocationID: "guarded-i1-resource", + ConnectionID: "backend-1", + TimedSample: telemetryBool(false), + RequiredFamilies: []TraversalTelemetryFamily{requiredFamily, TraversalTelemetryFamilyHydration}, + Counters: diagnosticCounters, + CounterStatus: TraversalTelemetryCounterStatusComplete, + PlanReplay: &TraversalPlanReplayEvidence{ + Source: "test-plan", Counters: planCounters, Provenance: planProvenance, + }, + Provenance: guardedI1CounterProvenance(contract.namespace), + } + + return CaseResult{ + Dataset: "fixture", + Name: "guarded-i1", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{FixtureTier: "normal", FallbackExpectation: "forbidden"}, + PostgresMetrics: &PostgresPlanMetrics{}, + TraversalTelemetry: &telemetry, + Optimization: &translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: contract.family, Candidate: architecture, Selected: architecture, Applied: architecture, + EmittedPolicy: contract.policy, + }}}, + } +} + +func inlineI1CounterProvenance(namespace string) map[string]string { + provenance := map[string]string{} + for _, name := range []string{ + "distance_rows", "predecessor_rows", "enumeration_rows", "output_paths", "output_bytes", + "candidate_marker_rows", "fallback_marker_rows", "candidate_branch_rows", "fallback_branch_rows", + "candidate_executor_loops", "fallback_executor_loops", + } { + provenance[namespace+"."+name] = "test." + namespace + "." + name + } + return provenance +} + +func guardedI1CounterProvenance(namespace string) map[string]string { + provenance := inlineI1CounterProvenance(namespace) + for _, name := range []string{"path_count", "node_lookups", "edge_lookups", "loops", "rows", "time_ns", "bytes"} { + provenance["hydration."+name] = "test.hydration." + name + } + return provenance +} diff --git a/cmd/graphbench/results.go b/cmd/graphbench/results.go index f333b327..44d7720e 100644 --- a/cmd/graphbench/results.go +++ b/cmd/graphbench/results.go @@ -17,92 +17,640 @@ package main import ( + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "fmt" "io" "os" "path/filepath" + "slices" "sort" "time" "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/specterops/dawgs/drivers/pg" + "github.com/specterops/dawgs/testutil" ) const ( - StatusOK = "ok" - StatusRowMismatch = "row_mismatch" - StatusError = "error" + // StatusOK marks a benchmark case whose execution and expectations succeeded. + StatusOK = "ok" + + // StatusRowMismatch marks a benchmark case whose observed row count differed from its expectation. + StatusRowMismatch = "row_mismatch" + + // StatusError marks a benchmark case that failed during execution. + StatusError = "error" + + // StatusNotImplemented marks a benchmark case unsupported by the selected backend. StatusNotImplemented = "not_implemented" ) +// DurationStats summarizes warmup policy, measured latency samples, quantiles, and sample sufficiency. type DurationStats struct { - Iterations int `json:"iterations"` - Median time.Duration `json:"median"` - P95 time.Duration `json:"p95"` - Max time.Duration `json:"max"` + // Iterations records the number of measured iterations. + Iterations int `json:"iterations"` + // WarmupIterations records the untimed iterations run before measurement. + WarmupIterations int `json:"warmup_iterations"` + // Median records the median observed duration. + Median time.Duration `json:"median"` + // P95 records the 95th-percentile observed duration. + P95 time.Duration `json:"p95"` + // P99 records the 99th-percentile duration. + P99 time.Duration `json:"p99"` + // P99Gated reports whether the sample count is sufficient to enforce the P99 noise threshold. + P99Gated bool `json:"p99_gated"` + // Max records the longest observed duration. + Max time.Duration `json:"max"` + // Samples contains the individual measurements. + Samples []LatencySample `json:"samples,omitempty"` +} + +// RuntimeReceiptEvent records one ordered executor transition observed during +// a measured traversal invocation. Multiple events preserve nested fallback +// chains such as I1 -> S4 -> S3 without reducing them to the terminal arm. +type RuntimeReceiptEvent struct { + // InvocationID binds this event to the session-local timed invocation that emitted it. + InvocationID string `json:"invocation_id,omitempty"` + Ordinal int `json:"ordinal"` + RuntimeIdentity string `json:"runtime_identity"` + RuntimeBranch string `json:"runtime_branch"` + FallbackExecuted bool `json:"fallback_executed"` +} + +// LatencySample records one labeled duration and its measurement order. +type LatencySample struct { + // Round identifies the measurement round. + Round int `json:"round"` + // Block identifies the measurement block used to control carryover effects. + Block int `json:"block,omitempty"` + // Arm identifies the measurement arm that produced the sample. + Arm string `json:"arm,omitempty"` + // ArmOrder records the arm's position within its balanced measurement block. + ArmOrder int `json:"arm_order,omitempty"` + // RunUUID links the sample to its resumable benchmark run series. + RunUUID string `json:"run_uuid,omitempty"` + // Iteration identifies the measured iteration within its worker or round. + Iteration int `json:"iteration"` + // Case identifies the workload whose iteration produced the sample. + Case string `json:"case"` + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Backend identifies the execution backend. + Backend ExecutionMode `json:"backend"` + // ConnectionID records the backend session that executed the measured iteration. + ConnectionID string `json:"connection_id,omitempty"` + // Classification records the assigned measurement or result class. + Classification string `json:"classification"` + // Duration records elapsed time for this observation. + Duration time.Duration `json:"duration"` + // RequestedIdentity records the candidate arm selected for this case. + RequestedIdentity string `json:"requested_identity,omitempty"` + // RuntimeIdentity records the executor observed by the case's isolated runtime attestation. + RuntimeIdentity string `json:"runtime_identity,omitempty"` + // RuntimeBranch records the singular preflight, recursive, incumbent, or fallback branch observed for the case. + RuntimeBranch string `json:"runtime_branch,omitempty"` + // FallbackExecuted records whether the candidate delegated to its exact incumbent. + FallbackExecuted *bool `json:"fallback_executed,omitempty"` + // RuntimeAttestation identifies the boundary that supplied runtime identity. + RuntimeAttestation string `json:"runtime_attestation,omitempty"` + // RuntimeInvocationID uniquely identifies the session-local timed invocation. + RuntimeInvocationID string `json:"runtime_invocation_id,omitempty"` + // RuntimeReceiptEvents preserves the complete ordered runtime branch chain + // for this exact measured invocation. + RuntimeReceiptEvents []RuntimeReceiptEvent `json:"runtime_receipt_events,omitempty"` +} + +// ConcurrencySample records one concurrent worker iteration and its connection and latency stages. +type ConcurrencySample struct { + // Worker identifies the concurrent worker that produced the sample. + Worker int `json:"worker"` + // Iteration identifies the measured iteration within its worker or round. + Iteration int `json:"iteration"` + // ConnectionID records the PostgreSQL backend session assigned to the worker iteration. + ConnectionID string `json:"connection_id"` + // Classification records the assigned measurement or result class. + Classification string `json:"classification"` + // PoolWait records latency spent acquiring a database connection from the pool. + PoolWait time.Duration `json:"pool_wait"` + // Transaction records latency spent beginning and configuring the transaction. + Transaction time.Duration `json:"transaction_setup"` + // ExecuteDrain records latency spent executing and draining all rows. + ExecuteDrain time.Duration `json:"execute_decode_drain"` + // Total records the total elapsed duration. + Total time.Duration `json:"total"` +} + +// ConcurrencyBlock summarizes all samples and connection usage for one concurrency level. +type ConcurrencyBlock struct { + // Concurrency records the worker count exercised by the block. + Concurrency int `json:"concurrency"` + // PoolSize sets the database connection-pool size. + PoolSize int `json:"pool_size"` + // Operations records successful query operations completed by a concurrency block. + Operations int `json:"operations"` + // Wall records end-to-end wall time for a concurrency block. + Wall time.Duration `json:"wall"` + // QPS reports completed query iterations per second. + QPS float64 `json:"qps"` + // Samples contains the individual measurements. + Samples []ConcurrencySample `json:"samples"` +} + +// PostgresReferenceResult records one independent PostgreSQL reference arm's identity, plan, observations, and timings. +type PostgresReferenceResult struct { + // SchemaVersion identifies the PostgreSQL reference-result schema revision. + SchemaVersion int `json:"schema_version"` + // Name identifies the independently measured reference arm. + Name string `json:"name"` + // LegacyName retains a compatibility alias for the reference arm. + LegacyName string `json:"legacy_name,omitempty"` + // Architecture identifies the executor architecture. + Architecture string `json:"architecture"` + // ImplementationID provides a versioned identity for the measured reference algorithm and materializer. + ImplementationID string `json:"implementation_id"` + // StateShape describes recursive state retained by the reference implementation. + StateShape string `json:"state_shape"` + // ObservationShape describes normalized values returned by the reference boundary. + ObservationShape string `json:"observation_shape"` + // SemanticValidation identifies the exact observation contract enforced for the reference. + SemanticValidation string `json:"semantic_validation"` + // Boundary identifies the measured execution boundary. + Boundary string `json:"boundary"` + // TimingBoundary describes which reference stages contribute to latency samples. + TimingBoundary string `json:"timing_boundary"` + // FullComparator indicates that the reference returns the complete public observation. + FullComparator bool `json:"full_comparator"` + // MeasurementOrder records the operation's position within its measurement round. + MeasurementOrder int `json:"measurement_order,omitempty"` + // AAAliasOf identifies the reference arm reused for an explicit A/A comparison. + AAAliasOf string `json:"aa_alias_of,omitempty"` + // SQL contains the rendered SQL statement. + SQL string `json:"sql"` + // SQLFingerprint identifies normalized SQL without retaining the statement text. + SQLFingerprint string `json:"sql_fingerprint"` + // RowCount records the number of rows produced. + RowCount int64 `json:"row_count"` + // ObservedRows contains stable serialized observations used for correctness comparison. + ObservedRows []string `json:"observed_rows,omitempty"` + // Stats contains latency statistics for the enclosing result or reference. + Stats DurationStats `json:"stats"` + // PostgresPlan contains normalized PostgreSQL text-plan lines. + PostgresPlan []string `json:"postgres_plan,omitempty"` + // PostgresPlanJSON contains structured PostgreSQL EXPLAIN evidence. + PostgresPlanJSON json.RawMessage `json:"postgres_plan_json,omitempty"` + // PostgresMetrics contains normalized PostgreSQL plan resource metrics. + PostgresMetrics *PostgresPlanMetrics `json:"postgres_metrics,omitempty"` + // TraversalTelemetry contains lightweight execution identity and optional untimed diagnostic counters. + TraversalTelemetry *TraversalExecutionTelemetry `json:"traversal_execution_telemetry,omitempty"` + // traversalTelemetryParameters retains invocation parameters only until all + // timed samples finish and the optional replay is attached. + traversalTelemetryParameters map[string]any +} + +// CompileSample breaks one Cypher compilation into parse, translate, and render stages. +type CompileSample struct { + // Iteration identifies the measured iteration within its worker or round. + Iteration int `json:"iteration"` + // Parse records Cypher parse latency. + Parse time.Duration `json:"parse"` + // Optimize records query optimization latency. + Optimize time.Duration `json:"optimize"` + // TranslateIncludingOptimize records combined translation and optimization latency. + TranslateIncludingOptimize time.Duration `json:"translate_including_optimize"` + // Render records SQL rendering latency after translation. + Render time.Duration `json:"render"` + // Total records the total elapsed duration. + Total time.Duration `json:"total"` + // Allocations records allocation count while measuring the client-side stage. + Allocations uint64 `json:"allocations"` + // AllocatedBytes records bytes allocated while measuring the client-side stage. + AllocatedBytes uint64 `json:"allocated_bytes"` } +// ClientWaterfall summarizes compile and raw-request samples at the client boundary. +type ClientWaterfall struct { + // IntervalsOverlap warns that nested compilation stages cannot be summed as exclusive costs. + IntervalsOverlap bool `json:"intervals_overlap"` + // Notes contains human-readable caveats attached to the artifact or case. + Notes string `json:"notes"` + // Samples contains the individual measurements. + Samples []CompileSample `json:"samples"` +} + +// BoundarySample breaks one raw PostgreSQL request into client-side latency stages. +type BoundarySample struct { + // Iteration identifies the measured iteration within its worker or round. + Iteration int `json:"iteration"` + // PoolWait records latency spent acquiring a PostgreSQL connection from the pool. + PoolWait time.Duration `json:"pool_wait"` + // Transaction records latency spent beginning and configuring the transaction. + Transaction time.Duration `json:"transaction_setup"` + // BindPrepare records PostgreSQL bind and statement-prepare latency. + BindPrepare time.Duration `json:"bind_prepare"` + // FirstRow records latency until the first result row becomes available. + FirstRow time.Duration `json:"first_row"` + // AllRowsDecode records client time to decode the complete result set. + AllRowsDecode time.Duration `json:"all_rows_decode"` + // DrainClose records latency spent draining remaining rows and closing the iterator. + DrainClose time.Duration `json:"drain_close"` + // Total records the total elapsed duration. + Total time.Duration `json:"total"` + // Rows records the number of rows decoded during this boundary measurement. + Rows int64 `json:"rows"` + // Allocations records allocation count while measuring the client-side stage. + Allocations uint64 `json:"allocations"` + // AllocatedBytes records bytes allocated while measuring the client-side stage. + AllocatedBytes uint64 `json:"allocated_bytes"` +} + +// PostgresBoundaryWaterfall summarizes PostgreSQL planning, execution, and client overhead samples. +type PostgresBoundaryWaterfall struct { + // Boundary identifies the measured execution boundary. + Boundary string `json:"boundary"` + // SQLFingerprint identifies normalized SQL without retaining the statement text. + SQLFingerprint string `json:"sql_fingerprint"` + // WarmupIterations records the untimed iterations run before measurement. + WarmupIterations int `json:"warmup_iterations"` + // MeasurementOrder records the operation's position within its measurement round. + MeasurementOrder int `json:"measurement_order,omitempty"` + // Samples contains the individual measurements. + Samples []BoundarySample `json:"samples"` +} + +// PostgresPlanMetrics aggregates structural, cardinality, timing, and buffer evidence from a PostgreSQL plan. type PostgresPlanMetrics struct { - PlanningMS *float64 `json:"planning_ms,omitempty"` + // PlanningMS records PostgreSQL planning time in milliseconds. + PlanningMS *float64 `json:"planning_ms,omitempty"` + // ExecutionMS records PostgreSQL execution time in milliseconds. ExecutionMS *float64 `json:"execution_ms,omitempty"` - Buffers Buffers `json:"buffers,omitempty"` + // Buffers contains shared, local, and temporary buffer activity attributed to the plan. + Buffers Buffers `json:"buffers,omitempty"` + // TempFiles records temporary files created by the backend session. + TempFiles int64 `json:"temp_files,omitempty"` + // TempBytes records temporary bytes written by the backend session. + TempBytes int64 `json:"temp_bytes,omitempty"` + // WALRecords records write-ahead-log records attributed to the plan node. + WALRecords int64 `json:"wal_records,omitempty"` + // WALBytes records write-ahead-log bytes attributed to the plan node. + WALBytes int64 `json:"wal_bytes,omitempty"` + // RootRows records rows emitted by root selection in the PostgreSQL plan. + RootRows int64 `json:"root_rows,omitempty"` + // RecursiveRows records rows emitted by recursive traversal state. + RecursiveRows int64 `json:"recursive_rows,omitempty"` + // RecursiveLoops records loops performed by recursive plan nodes. + RecursiveLoops int64 `json:"recursive_loops,omitempty"` + // FrontierRows records rows retained in the active traversal frontier. + FrontierRows int64 `json:"frontier_rows,omitempty"` + // WitnessRows records rows retained for shortest-path witness reconstruction. + WitnessRows int64 `json:"witness_rows,omitempty"` + // MeetingRows records bidirectional search rows where frontiers meet. + MeetingRows int64 `json:"meeting_rows,omitempty"` + // HydrationRows records rows processed while hydrating paths from ID trails. + HydrationRows int64 `json:"hydration_rows,omitempty"` + // ForwardEdgeProbes records relationship probes performed by forward search. + ForwardEdgeProbes int64 `json:"forward_edge_probes,omitempty"` + // ReverseEdgeProbes records relationship probes performed by reverse search. + ReverseEdgeProbes int64 `json:"reverse_edge_probes,omitempty"` + // RootLookupLoops records repeated plan loops used to locate traversal roots. + RootLookupLoops int64 `json:"root_lookup_loops,omitempty"` + // BoundaryLookupLoops records loops used to resolve traversal boundaries. + BoundaryLookupLoops int64 `json:"boundary_lookup_loops,omitempty"` + // HydrationLoops records loops performed while hydrating search results. + HydrationLoops int64 `json:"hydration_loops,omitempty"` + // EndpointProbeRows records rows examined by endpoint preflight probing. + EndpointProbeRows int64 `json:"endpoint_probe_rows,omitempty"` + // ReverseStateProbeRows records reverse-search state rows examined by probing. + ReverseStateProbeRows int64 `json:"reverse_state_probe_rows,omitempty"` + // EndpointGuardOverflow reports whether endpoint-seeded search exceeded its configured guard. + EndpointGuardOverflow bool `json:"endpoint_guard_overflow,omitempty"` + // StateGuardOverflow reports whether recursive state exceeded its configured guard. + StateGuardOverflow bool `json:"state_guard_overflow,omitempty"` + // ExpansionFallbackExecuted reports whether guarded expansion switched to its exact fallback executor. + ExpansionFallbackExecuted bool `json:"expansion_fallback_executed,omitempty"` + // PlanNodes lists normalized PostgreSQL plan-node metrics in traversal order. + PlanNodes []PostgresPlanNodeMetric `json:"plan_nodes,omitempty"` + // Provenance maps derived metric names to the plan evidence used to compute them. + Provenance map[string]string `json:"provenance,omitempty"` +} + +// PostgresPlanNodeMetric captures one PostgreSQL plan node's identity, counters, and buffers. +type PostgresPlanNodeMetric struct { + // PlanNodeID identifies this node within the normalized pre-order plan tree. + PlanNodeID int64 `json:"plan_node_id,omitempty"` + // ParentPlanNodeID identifies the direct parent node; the root has no parent. + ParentPlanNodeID int64 `json:"parent_plan_node_id,omitempty"` + // NodeType identifies the PostgreSQL plan node type. + NodeType string `json:"node_type"` + // ParentRelationship identifies the relationship by which this plan node is attached to its parent. + ParentRelationship string `json:"parent_relationship,omitempty"` + // CTEName names the recursive common-table expression referenced by the plan node. + CTEName string `json:"cte_name,omitempty"` + // RelationName identifies the PostgreSQL relation scanned by the plan node. + RelationName string `json:"relation_name,omitempty"` + // Alias contains the display alias assigned to the plan node. + Alias string `json:"alias,omitempty"` + // IndexName names the PostgreSQL index scanned by the plan node. + IndexName string `json:"index_name,omitempty"` + // FunctionName identifies a SQL function invoked by a Function Scan without exposing its internal work. + FunctionName string `json:"function_name,omitempty"` + // SubplanName names an initplan, subplan, or CTE body used for stable branch attribution. + SubplanName string `json:"subplan_name,omitempty"` + // PlanRows records the planner's estimated rows for the plan node. + PlanRows int64 `json:"plan_rows,omitempty"` + // PlanWidth records the planner's estimated row width in bytes. + PlanWidth int64 `json:"plan_width,omitempty"` + // ActualRows records rows actually emitted by the plan node. + ActualRows int64 `json:"actual_rows,omitempty"` + // ActualLoops records how many times the PostgreSQL plan node executed. + ActualLoops int64 `json:"actual_loops,omitempty"` + // RowsRemovedByFilter records rows PostgreSQL reports as rejected by this node's filter. + RowsRemovedByFilter int64 `json:"rows_removed_by_filter,omitempty"` + // ActualTotalMS records total observed time for the PostgreSQL plan node. + ActualTotalMS float64 `json:"actual_total_ms,omitempty"` + // Buffers contains shared, local, and temporary buffer activity attributed to the plan. + Buffers Buffers `json:"buffers,omitempty"` + // Provenance identifies the plan evidence from which this node metric was measured. + Provenance string `json:"provenance"` } +// Buffers contains PostgreSQL buffer activity split by storage class and operation. type Buffers struct { - SharedHit int64 `json:"shared_hit,omitempty"` - SharedRead int64 `json:"shared_read,omitempty"` + // SharedHit records shared PostgreSQL buffer cache hits. + SharedHit int64 `json:"shared_hit,omitempty"` + // SharedRead records shared PostgreSQL buffers read by the plan. + SharedRead int64 `json:"shared_read,omitempty"` + // SharedDirtied records shared PostgreSQL buffers dirtied by the plan. SharedDirtied int64 `json:"shared_dirtied,omitempty"` - TempRead int64 `json:"temp_read,omitempty"` - TempWritten int64 `json:"temp_written,omitempty"` + // SharedWritten records shared PostgreSQL buffers written by the plan. + SharedWritten int64 `json:"shared_written,omitempty"` + // LocalHit records local PostgreSQL buffer cache hits. + LocalHit int64 `json:"local_hit,omitempty"` + // LocalRead records local PostgreSQL buffers read by the plan. + LocalRead int64 `json:"local_read,omitempty"` + // LocalDirtied records local PostgreSQL buffers dirtied by the plan. + LocalDirtied int64 `json:"local_dirtied,omitempty"` + // LocalWritten records local PostgreSQL buffers written by the plan. + LocalWritten int64 `json:"local_written,omitempty"` + // TempRead records temporary PostgreSQL buffers read by the plan. + TempRead int64 `json:"temp_read,omitempty"` + // TempWritten records temporary PostgreSQL buffers written by the plan. + TempWritten int64 `json:"temp_written,omitempty"` } +// CaseResult records one workload execution with provenance, observations, plan evidence, and latency samples. type CaseResult struct { - Source string `json:"source"` - Dataset string `json:"dataset"` - Name string `json:"name"` - Category string `json:"category"` - ExecutionMode ExecutionMode `json:"execution_mode"` - Status string `json:"status"` - Cypher string `json:"cypher"` - Params map[string]any `json:"params,omitempty"` - NodeParams map[string]string `json:"node_params,omitempty"` - ExpectedRowCount *int64 `json:"expected_row_count,omitempty"` - RowCount int64 `json:"row_count,omitempty"` - Stats DurationStats `json:"stats,omitempty"` - SQL string `json:"sql,omitempty"` - PostgresPlan []string `json:"postgres_plan,omitempty"` - PostgresMetrics *PostgresPlanMetrics `json:"postgres_metrics,omitempty"` - Neo4jPlan *Neo4jPlanNode `json:"neo4j_plan,omitempty"` - Neo4jOperators []string `json:"neo4j_operators,omitempty"` - Optimization *translate.OptimizationSummary `json:"optimization,omitempty"` - Baseline *BaselineComparison `json:"baseline,omitempty"` - FallbackReason string `json:"fallback_reason,omitempty"` - Error string `json:"error,omitempty"` + // Metadata captures build and baseline metadata. + Metadata testutil.BaselineMetadata `json:"metadata"` + // Environment captures the environment in which the measurement ran. + Environment *RunEnvironment `json:"environment,omitempty"` + // PostgresEnvironment captures PostgreSQL settings required for comparability. + PostgresEnvironment *PostgresEnvironment `json:"postgres_environment,omitempty"` + // Fixture captures the fixture identity and cardinality contract. + Fixture *FixtureMetadata `json:"fixture,omitempty"` + // Source identifies the source corpus file. + Source string `json:"source"` + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // WorkloadSHA256 binds the result to the case declaration and execution mode. + WorkloadSHA256 string `json:"workload_sha256"` + // Category groups cases by workload category. + Category string `json:"category"` + // Shape describes the workload shape used for selection and comparison. + Shape WorkloadShape `json:"shape"` + // ExecutionMode identifies the backend execution mode that produced the case result. + ExecutionMode ExecutionMode `json:"execution_mode"` + // Status records the execution outcome. + Status string `json:"status"` + // Cypher contains the Cypher statement under test. + Cypher string `json:"cypher"` + // Params supplies literal query parameters. + Params map[string]any `json:"params,omitempty"` + // NodeParams maps query parameters to fixture node keys. + NodeParams map[string]string `json:"node_params,omitempty"` + // NodeListParams maps query parameters to ordered fixture node-key lists. + NodeListParams map[string][]string `json:"node_list_params,omitempty"` + // ExpectedRowCount sets the required result-row count when known. + ExpectedRowCount *int64 `json:"expected_row_count,omitempty"` + // ObservedRows contains stable serialized observations used for correctness comparison. + ObservedRows []string `json:"observed_rows,omitempty"` + // RowCount records the number of rows produced. + RowCount int64 `json:"row_count,omitempty"` + // MatchedCount records entities selected by the measured mutation. + MatchedCount *int64 `json:"matched_count,omitempty"` + // AffectedCount records entities actually changed by the measured mutation. + AffectedCount *int64 `json:"affected_count,omitempty"` + // PostState contains the observed results of post-write validation queries. + PostState []StateQueryResult `json:"post_state,omitempty"` + // Stats contains latency statistics for the enclosing result or reference. + Stats DurationStats `json:"stats,omitempty"` + // Concurrency contains opt-in worker-count measurements for this case. + Concurrency []ConcurrencyBlock `json:"concurrency,omitempty"` + // PostgresReferences contains independent PostgreSQL reference results for the case. + PostgresReferences []PostgresReferenceResult `json:"postgres_references,omitempty"` + // ClientWaterfall contains Cypher compilation and client-boundary timing samples. + ClientWaterfall *ClientWaterfall `json:"client_waterfall,omitempty"` + // RawPGXWaterfall contains raw PGX boundary timings used for PostgreSQL cost attribution. + RawPGXWaterfall *PostgresBoundaryWaterfall `json:"raw_pgx_waterfall,omitempty"` + // RawPGXRoundTrip records legacy aggregate raw-PGX round-trip latency. + RawPGXRoundTrip *PostgresBoundaryWaterfall `json:"raw_pgx_round_trip,omitempty"` + // SQL contains the rendered SQL statement. + SQL string `json:"sql,omitempty"` + // SQLFingerprint identifies normalized SQL without retaining the statement text. + SQLFingerprint string `json:"sql_fingerprint,omitempty"` + // PostgresPlan contains normalized PostgreSQL text-plan lines. + PostgresPlan []string `json:"postgres_plan,omitempty"` + // PostgresPlanJSON contains structured PostgreSQL EXPLAIN evidence. + PostgresPlanJSON json.RawMessage `json:"postgres_plan_json,omitempty"` + // PostgresMetrics contains normalized PostgreSQL plan resource metrics. + PostgresMetrics *PostgresPlanMetrics `json:"postgres_metrics,omitempty"` + // TraversalTelemetry contains lightweight execution identity and optional untimed diagnostic counters. + TraversalTelemetry *TraversalExecutionTelemetry `json:"traversal_execution_telemetry,omitempty"` + // Neo4jPlan contains the normalized Neo4j operator tree. + Neo4jPlan *Neo4jPlanNode `json:"neo4j_plan,omitempty"` + // Neo4jOperators lists normalized Neo4j operators found in the captured plan. + Neo4jOperators []string `json:"neo4j_operators,omitempty"` + // Optimization captures translation optimization and lowering decisions. + Optimization *translate.OptimizationSummary `json:"optimization,omitempty"` + // ParseCache reports parse-cache hit and miss statistics for the case. + ParseCache *pg.ParseCacheStats `json:"parse_cache,omitempty"` + // Baseline contains the latency comparison with a matching baseline record. + Baseline *BaselineComparison `json:"baseline,omitempty"` + // FallbackReason explains why execution used a fallback architecture. + FallbackReason string `json:"fallback_reason,omitempty"` + // ExistingGraph selects read-only execution against a pre-existing graph. + ExistingGraph *ExistingGraphRun `json:"existing_graph,omitempty"` + // Error records the failure message when the operation did not succeed. + Error string `json:"error,omitempty"` + // StableObservation reports whether ObservedRows contains a backend-independent normalized result. + StableObservation bool `json:"observation_captured,omitempty"` } +// StateQueryResult records a post-write validation query's row count and optional scalar value. +type StateQueryResult struct { + // Name labels the post-write state assertion that produced this result. + Name string `json:"name"` + // RowCount records the number of rows produced. + RowCount int64 `json:"row_count"` + // ScalarInt contains the observed scalar value when the state query expects one. + ScalarInt *int64 `json:"scalar_int,omitempty"` +} + +// BaselineComparison compares current median latency with a previously recorded baseline. type BaselineComparison struct { + // BaselineMedian records the median latency loaded from the comparison baseline. BaselineMedian time.Duration `json:"baseline_median"` - CurrentMedian time.Duration `json:"current_median"` - Change time.Duration `json:"change"` - Ratio float64 `json:"ratio"` + // CurrentMedian records the median latency measured by the current run. + CurrentMedian time.Duration `json:"current_median"` + // Change records current latency relative to the selected baseline. + Change time.Duration `json:"change"` + // Ratio reports the candidate-to-baseline latency ratio. + Ratio float64 `json:"ratio"` +} + +// validateBackendObservations checks row counts and stable observations across successful backend results. +func validateBackendObservations(records []CaseResult) error { + // observationKey identifies one dataset, case, backend, and round during observation validation. + type observationKey struct { + // dataset names the fixture shared by observations compared across backends. + dataset string + // name identifies the workload case compared across backends. + name string + } + + postgres := map[observationKey][]string{} + for _, record := range records { + if record.ExecutionMode == ModePostgresSQL && record.Status == StatusOK && record.StableObservation && record.ObservedRows != nil { + postgres[observationKey{ + dataset: record.Dataset, + name: record.Name, + }] = record.ObservedRows + } + } + + for _, record := range records { + if record.ExecutionMode != ModeNeo4j || record.Status != StatusOK || !record.StableObservation || record.ObservedRows == nil { + continue + } + key := observationKey{ + dataset: record.Dataset, + name: record.Name, + } + if expected, found := postgres[key]; found && !slices.Equal(expected, record.ObservedRows) { + return fmt.Errorf("backend observations differ for %s/%s: postgres=%v neo4j=%v", record.Dataset, record.Name, expected, record.ObservedRows) + } + } + + return nil } +// newCaseResult initializes workload identity, expectations, observation policy, and successful status for one case. func newCaseResult(testCase ScaleCase, mode ExecutionMode, params map[string]any) CaseResult { return CaseResult{ Source: testCase.Source, Dataset: testCase.Dataset, Name: testCase.Name, + WorkloadSHA256: scaleCaseWorkloadIdentity(testCase, mode), Category: testCase.Category, + Shape: testCase.Shape, ExecutionMode: mode, Status: StatusOK, Cypher: testCase.Cypher, Params: params, NodeParams: testCase.NodeParams, + NodeListParams: testCase.NodeListParams, ExpectedRowCount: testCase.Expected.RowCount, + StableObservation: testCase.Expected.ResultKind == "id_rows" || + testCase.Expected.ResultKind == "scalar" || + (testCase.Expected.ResultKind == "path_set" && (len(testCase.Expected.PathRows) > 0 || + testCase.Expected.RowCount != nil && *testCase.Expected.RowCount == 0)), + } +} + +// scaleCaseWorkloadIdentity hashes the logical workload fields that must match across artifacts. +func scaleCaseWorkloadIdentity(testCase ScaleCase, mode ExecutionMode) string { + payload := struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // Source identifies the source corpus file. + Source string `json:"source"` + // Backend identifies the execution backend. + Backend ExecutionMode `json:"backend"` + // Case contains the complete workload declaration included in the identity digest. + Case ScaleCase `json:"case"` + }{ + Version: 1, + Source: testCase.Source, + Backend: mode, + Case: testCase, + } + raw, err := json.Marshal(payload) + if err != nil { + return "" + } + digest := sha256.Sum256(raw) + return hex.EncodeToString(digest[:]) +} + +// attachFixtureMetadata adds fixture metadata to the owning artifact. +func attachFixtureMetadata(record *CaseResult, fixture FixtureMetadata) { + if record == nil { + return } + record.Fixture = &fixture + payload := struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // LogicalWorkloadSHA256 identifies query semantics independently of runtime measurements. + LogicalWorkloadSHA256 string `json:"logical_workload_sha256"` + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Checksum binds fixture identity to its canonical logical contents. + Checksum string `json:"checksum"` + // NodeCount records logical fixture nodes declared or loaded. + NodeCount int `json:"node_count"` + // EdgeCount records logical fixture relationships declared or loaded. + EdgeCount int `json:"edge_count"` + // PhysicalNodeCount records physical node rows present in the backend fixture. + PhysicalNodeCount int64 `json:"physical_node_count,omitempty"` + // PhysicalEdgeCount records physical relationship rows present in the backend fixture. + PhysicalEdgeCount int64 `json:"physical_edge_count,omitempty"` + // Configuration captures the generator parameters that define the fixture shape. + Configuration string `json:"configuration,omitempty"` + // Shortest contains expectations derived from a generated shortest-path fixture. + Shortest *ShortestFixtureExpectations `json:"shortest,omitempty"` + // FixedSuffixExpansion contains expectations derived from a fixed-suffix expansion fixture. + FixedSuffixExpansion *FixedSuffixExpansionFixtureExpectations `json:"fixed_suffix_expansion,omitempty"` + // EndpointSeededExpansion contains expectations derived from an endpoint-seeded expansion fixture. + EndpointSeededExpansion *EndpointSeededExpansionFixtureExpectations `json:"endpoint_seeded_expansion,omitempty"` + }{ + Version: 1, + LogicalWorkloadSHA256: record.WorkloadSHA256, + Dataset: fixture.Dataset, + Checksum: fixture.Checksum, + NodeCount: fixture.NodeCount, + EdgeCount: fixture.EdgeCount, + PhysicalNodeCount: fixture.PhysicalNodeCount, + PhysicalEdgeCount: fixture.PhysicalEdgeCount, + Configuration: fixture.Configuration, + Shortest: fixture.Shortest, + FixedSuffixExpansion: fixture.FixedSuffixExpansion, + EndpointSeededExpansion: fixture.EndpointSeededExpansion, + } + raw, err := json.Marshal(payload) + if err != nil { + record.WorkloadSHA256 = "" + return + } + digest := sha256.Sum256(raw) + record.WorkloadSHA256 = hex.EncodeToString(digest[:]) } +// computeDurationStats validates measured durations and derives median, tail, maximum, and labeled sample data. func computeDurationStats(durations []time.Duration) (DurationStats, error) { if len(durations) == 0 { return DurationStats{}, fmt.Errorf("duration stats require at least one duration") @@ -115,14 +663,91 @@ func computeDurationStats(durations []time.Duration) (DurationStats, error) { n := len(sortedDurations) p95Index := (95*n+99)/100 - 1 + p99Index := (99*n+99)/100 - 1 return DurationStats{ Iterations: n, Median: sortedDurations[n/2], P95: sortedDurations[p95Index], + P99: sortedDurations[p99Index], + P99Gated: n >= 10_000, Max: sortedDurations[n-1], + Samples: func() []LatencySample { + samples := make([]LatencySample, len(durations)) + for idx, duration := range durations { + samples[idx] = LatencySample{ + Round: 1, + Iteration: idx + 1, + Classification: "warm", + Duration: duration, + } + } + return samples + }(), }, nil } +// labelLatencySamples attaches backend, dataset, and case identity to every latency sample in stats. +func labelLatencySamples(stats *DurationStats, mode ExecutionMode, testCase ScaleCase) { + for idx := range stats.Samples { + stats.Samples[idx].Backend = mode + stats.Samples[idx].Case = testCase.Name + stats.Samples[idx].Dataset = testCase.Dataset + } +} + +// setSampleRound assigns a measurement round to every latency sample in stats. +func setSampleRound(stats *DurationStats, round int) { + for idx := range stats.Samples { + stats.Samples[idx].Round = round + } +} + +// setSampleRunMetadata copies run, arm, block, and round identity onto every latency sample in stats. +func setSampleRunMetadata(stats *DurationStats, environment RunEnvironment) { + for idx := range stats.Samples { + stats.Samples[idx].Round = environment.Round + stats.Samples[idx].Block = environment.Block + stats.Samples[idx].Arm = environment.Arm + stats.Samples[idx].ArmOrder = environment.ArmOrder + stats.Samples[idx].RunUUID = environment.RunUUID + } +} + +// setSampleTraversalRuntimeMetadata binds every timed sample to the singular +// invocation-local replay outcome obtained for the same case, parameters, SQL, +// and physical session. This supports diagnostics but deliberately does not +// claim per-timed-invocation attribution; promotion gates require the stronger +// "timed_invocation" attestation. +func setSampleTraversalRuntimeMetadata(stats *DurationStats, telemetry *TraversalExecutionTelemetry) { + if stats == nil || telemetry == nil { + return + } + for idx := range stats.Samples { + if stats.Samples[idx].RuntimeAttestation == "timed_invocation" { + continue + } + stats.Samples[idx].RequestedIdentity = telemetry.Summary.RequestedIdentity + stats.Samples[idx].RuntimeIdentity = telemetry.Summary.RuntimeIdentity + stats.Samples[idx].RuntimeBranch = telemetry.Summary.RuntimeBranch + stats.Samples[idx].FallbackExecuted = telemetry.Summary.FallbackExecuted + stats.Samples[idx].RuntimeAttestation = "same_case_invocation_local_replay" + } +} + +// setCaseRunMetadata assigns case run metadata across the supplied records. +func setCaseRunMetadata(record *CaseResult, metadata testutil.BaselineMetadata, environment RunEnvironment) { + if record == nil { + return + } + record.Metadata = metadata + record.Environment = &environment + setSampleRunMetadata(&record.Stats, environment) + for idx := range record.PostgresReferences { + setSampleRunMetadata(&record.PostgresReferences[idx].Stats, environment) + } +} + +// applyRowExpectation marks a successful result as mismatched when its row count violates the declared expectation. func applyRowExpectation(result *CaseResult) { if result.ExpectedRowCount != nil && result.RowCount != *result.ExpectedRowCount { result.Status = StatusRowMismatch @@ -130,6 +755,7 @@ func applyRowExpectation(result *CaseResult) { } } +// writeJSONLFile writes records to standard output or replaces the requested JSON Lines artifact. func writeJSONLFile(path string, records []CaseResult) (err error) { if path == "" { return writeJSONL(os.Stdout, records) @@ -152,6 +778,95 @@ func writeJSONLFile(path string, records []CaseResult) (err error) { return writeJSONL(output, records) } +// appendJSONLFile validates compatibility with existing records before appending new JSON Lines entries. +func appendJSONLFile(path string, records []CaseResult) (err error) { + if path == "" { + return errors.New("append JSONL path must not be empty") + } + if err := ensureOutputDir(path); err != nil { + return err + } + + if existing, readErr := readJSONLFile(path); readErr == nil { + if err := validateJSONLAppend(existing, records); err != nil { + return err + } + } else if !errors.Is(readErr, os.ErrNotExist) { + return readErr + } + + output, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o600) + if err != nil { + return err + } + defer func() { + if closeErr := output.Close(); err == nil && closeErr != nil { + err = closeErr + } + }() + return writeJSONL(output, records) +} + +// validateJSONLAppend ensures appended records share run identity and do not duplicate case rounds. +func validateJSONLAppend(existing, appended []CaseResult) error { + if len(existing) == 0 || len(appended) == 0 { + return nil + } + + left, right := existing[0].Environment, appended[0].Environment + if left == nil || right == nil { + return errors.New("append JSONL requires run environment metadata") + } + if left.RunUUID != right.RunUUID || left.Arm != right.Arm || left.BinarySHA256 != right.BinarySHA256 || left.DirtyDiffSHA256 != right.DirtyDiffSHA256 { + return fmt.Errorf("append JSONL run identity mismatch: existing run=%q arm=%q binary=%q diff=%q, appended run=%q arm=%q binary=%q diff=%q", + left.RunUUID, left.Arm, left.BinarySHA256, left.DirtyDiffSHA256, + right.RunUUID, right.Arm, right.BinarySHA256, right.DirtyDiffSHA256) + } + + // recordKey identifies one run, dataset, case, mode, and round during append validation. + type recordKey struct { + // dataset names the fixture component of the append-deduplication key. + dataset string + // name identifies the workload case within its dataset. + name string + // mode separates records for different execution backends within the same round. + mode ExecutionMode + // round identifies the measurement round used to balance execution order. + round int + } + seen := make(map[recordKey]struct{}, len(existing)) + for _, record := range existing { + round := 0 + if record.Environment != nil { + round = record.Environment.Round + } + seen[recordKey{ + dataset: record.Dataset, + name: record.Name, + mode: record.ExecutionMode, + round: round, + }] = struct{}{} + } + for _, record := range appended { + round := 0 + if record.Environment != nil { + round = record.Environment.Round + } + key := recordKey{ + dataset: record.Dataset, + name: record.Name, + mode: record.ExecutionMode, + round: round, + } + if _, duplicate := seen[key]; duplicate { + return fmt.Errorf("append JSONL duplicate record for %s/%s/%s round %d", key.dataset, key.name, key.mode, key.round) + } + seen[key] = struct{}{} + } + return nil +} + +// writeJSONL encodes each case result as one JSON Lines record in input order. func writeJSONL(w io.Writer, records []CaseResult) error { encoder := json.NewEncoder(w) for _, record := range records { @@ -163,6 +878,7 @@ func writeJSONL(w io.Writer, records []CaseResult) error { return nil } +// readJSONLFile reads JSON Lines file and propagates I/O or decoding failures. func readJSONLFile(path string) ([]CaseResult, error) { input, err := os.Open(path) if err != nil { @@ -184,6 +900,7 @@ func readJSONLFile(path string) ([]CaseResult, error) { return nil, err } + normalizeHistoricalReferences(&record) records = append(records, record) } @@ -191,6 +908,42 @@ func readJSONLFile(path string) ([]CaseResult, error) { return records, nil } +// normalizeHistoricalReferences canonicalizes historical references for stable comparison. +func normalizeHistoricalReferences(record *CaseResult) { + for idx := range record.PostgresReferences { + reference := &record.PostgresReferences[idx] + if reference.SchemaVersion != 0 { + continue + } + reference.SchemaVersion = 1 + switch reference.Name { + case "complete_reference_s1_array_cte": + reference.LegacyName = reference.Name + reference.Name = "s3_unidirectional_trail_cte" + reference.Architecture = "SP-S3-U-NE" + reference.ImplementationID = "inline_recursive_cte_unidirectional_v1" + case "candidate_s2_bidirectional_cte": + reference.LegacyName = reference.Name + reference.Name = "s3_bidirectional_trail_cte" + reference.Architecture = "SP-S3-B" + reference.ImplementationID = "inline_recursive_cte_bidirectional_trails_v1" + } + if reference.StateShape == "" { + reference.StateShape = "legacy_unspecified" + } + if reference.ObservationShape == "" { + reference.ObservationShape = reference.Boundary + } + if reference.SemanticValidation == "" { + reference.SemanticValidation = "legacy_row_count_only" + if !reference.FullComparator { + reference.SemanticValidation = "row_count_stability" + } + } + } +} + +// ensureOutputDir creates the parent directory needed for an output file. func ensureOutputDir(path string) error { dir := filepath.Dir(path) if dir == "." || dir == "" { @@ -200,6 +953,7 @@ func ensureOutputDir(path string) error { return os.MkdirAll(dir, 0o755) } +// applyBaseline attaches median latency deltas and ratios from matching baseline records. func applyBaseline(path string, records []CaseResult) error { baseline, err := readJSONLFile(path) if err != nil { @@ -229,6 +983,7 @@ func applyBaseline(path string, records []CaseResult) error { return nil } +// resultKey joins result identity fields into the append-validation key. func resultKey(dataset, name string, mode ExecutionMode) string { return dataset + "\x00" + name + "\x00" + string(mode) } diff --git a/cmd/graphbench/results_test.go b/cmd/graphbench/results_test.go index 0ee87344..60134938 100644 --- a/cmd/graphbench/results_test.go +++ b/cmd/graphbench/results_test.go @@ -17,18 +17,51 @@ package main import ( + "path/filepath" "testing" "time" "github.com/stretchr/testify/require" ) +// TestAppendJSONLFileValidatesRunIdentityAndDuplicateRounds verifies append-only accumulation across rounds while rejecting duplicate keys and changes to arm or run UUID. +func TestAppendJSONLFileValidatesRunIdentityAndDuplicateRounds(t *testing.T) { + path := filepath.Join(t.TempDir(), "rounds.jsonl") + record := func(round int, arm, runUUID, binary string) CaseResult { + return CaseResult{ + Dataset: "fixture", + Name: "case", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Environment: &RunEnvironment{ + Round: round, + Arm: arm, + RunUUID: runUUID, + BinarySHA256: binary, + DirtyDiffSHA256: "diff", + }, + } + } + + require.NoError(t, appendJSONLFile(path, []CaseResult{record(1, "candidate", "run-1", "binary")})) + require.NoError(t, appendJSONLFile(path, []CaseResult{record(2, "candidate", "run-1", "binary")})) + records, err := readJSONLFile(path) + require.NoError(t, err) + require.Len(t, records, 2) + + require.ErrorContains(t, appendJSONLFile(path, []CaseResult{record(2, "candidate", "run-1", "binary")}), "duplicate record") + require.ErrorContains(t, appendJSONLFile(path, []CaseResult{record(3, "incumbent", "run-1", "binary")}), "run identity mismatch") + require.ErrorContains(t, appendJSONLFile(path, []CaseResult{record(3, "candidate", "run-2", "binary")}), "run identity mismatch") +} + +// TestComputeDurationStatsRejectsEmptyDurations verifies that aggregate statistics cannot be fabricated without at least one timing observation. func TestComputeDurationStatsRejectsEmptyDurations(t *testing.T) { _, err := computeDurationStats(nil) require.ErrorContains(t, err, "at least one duration") } +// TestComputeDurationStatsCopiesAndSortsDurations verifies aggregate values, preservation of input/sample order, default warm labels, backend metadata, and round relabeling. func TestComputeDurationStatsCopiesAndSortsDurations(t *testing.T) { durations := []time.Duration{ 30 * time.Millisecond, @@ -42,12 +75,47 @@ func TestComputeDurationStatsCopiesAndSortsDurations(t *testing.T) { require.Equal(t, 3, stats.Iterations) require.Equal(t, 20*time.Millisecond, stats.Median) require.Equal(t, 30*time.Millisecond, stats.P95) + require.Equal(t, 30*time.Millisecond, stats.P99) + require.False(t, stats.P99Gated) require.Equal(t, 30*time.Millisecond, stats.Max) require.Equal(t, 30*time.Millisecond, durations[0]) require.Equal(t, 10*time.Millisecond, durations[1]) require.Equal(t, 20*time.Millisecond, durations[2]) + require.Equal(t, []LatencySample{ + { + Round: 1, + Iteration: 1, + Classification: "warm", + Duration: 30 * time.Millisecond, + }, + { + Round: 1, + Iteration: 2, + Classification: "warm", + Duration: 10 * time.Millisecond, + }, + { + Round: 1, + Iteration: 3, + Classification: "warm", + Duration: 20 * time.Millisecond, + }, + }, stats.Samples) + + labelLatencySamples(&stats, ModePostgresSQL, ScaleCase{ + Name: "case", + Dataset: "fixture", + }) + require.Equal(t, ModePostgresSQL, stats.Samples[0].Backend) + require.Equal(t, "case", stats.Samples[0].Case) + require.Equal(t, "fixture", stats.Samples[0].Dataset) + + setSampleRound(&stats, 7) + require.Equal(t, 7, stats.Samples[0].Round) + require.Equal(t, 7, stats.Samples[2].Round) } +// TestComputeDurationStatsUsesNearestRankP95 verifies that twenty ordered samples select the nineteenth value for P95 while retaining the twentieth as maximum. func TestComputeDurationStatsUsesNearestRankP95(t *testing.T) { durations := make([]time.Duration, 20) for idx := range durations { @@ -60,3 +128,86 @@ func TestComputeDurationStatsUsesNearestRankP95(t *testing.T) { require.Equal(t, 19*time.Millisecond, stats.P95) require.Equal(t, 20*time.Millisecond, stats.Max) } + +// TestCheckStateExpectationChecksRowsAndScalar verifies simultaneous row/scalar acceptance and a scalar-specific diagnostic on mismatch. +func TestCheckStateExpectationChecksRowsAndScalar(t *testing.T) { + rowCount := int64(1) + scalar := int64(3) + + require.NoError(t, checkStateExpectation( + StateQueryResult{ + RowCount: 1, + ScalarInt: &scalar, + }, + ExpectedResult{ + RowCount: &rowCount, + ScalarInt: &scalar, + }, + )) + + wrong := int64(4) + require.ErrorContains(t, checkStateExpectation( + StateQueryResult{ + RowCount: 1, + ScalarInt: &scalar, + }, + ExpectedResult{ScalarInt: &wrong}, + ), "expected scalar integer 4") +} + +// TestValidateBackendObservationsPreservesDuplicateStableRows verifies multiset semantics: equal duplicate rows match across backends, but dropping one duplicate does not. +func TestValidateBackendObservationsPreservesDuplicateStableRows(t *testing.T) { + records := []CaseResult{ + { + Dataset: "fixture", + Name: "case", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + StableObservation: true, + ObservedRows: []string{`["a"]`, `["a"]`}, + }, + { + Dataset: "fixture", + Name: "case", + ExecutionMode: ModeNeo4j, + Status: StatusOK, + StableObservation: true, + ObservedRows: []string{`["a"]`, `["a"]`}, + }, + } + require.NoError(t, validateBackendObservations(records)) + + records[1].ObservedRows = []string{`["a"]`} + require.ErrorContains(t, validateBackendObservations(records), "backend observations differ") +} + +// TestNewCaseResultCrossChecksExactPathSets verifies that path observations +// become stable cross-backend evidence only when an exact nonempty path set or +// an exact empty result is declared. +func TestNewCaseResultCrossChecksExactPathSets(t *testing.T) { + record := newCaseResult(ScaleCase{ + Expected: ExpectedResult{ + ResultKind: "path_set", + }, + }, ModePostgresSQL, nil) + require.False(t, record.StableObservation) + + record = newCaseResult(ScaleCase{ + Expected: ExpectedResult{ + ResultKind: "path_set", + PathRows: []ExpectedPath{{ + Nodes: []string{"start"}, + }}, + }, + }, ModePostgresSQL, nil) + require.True(t, record.StableObservation) + + zero := int64(0) + record = newCaseResult(ScaleCase{ + Expected: ExpectedResult{ + ResultKind: "path_set", + RowCount: &zero, + }, + }, ModePostgresSQL, nil) + require.True(t, record.StableObservation) +} diff --git a/cmd/graphbench/run_lock.go b/cmd/graphbench/run_lock.go new file mode 100644 index 00000000..d267a48d --- /dev/null +++ b/cmd/graphbench/run_lock.go @@ -0,0 +1,54 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "os" + "path/filepath" + "syscall" +) + +// destructiveRunLock holds the filesystem lock that serializes destructive benchmark runs. +type destructiveRunLock struct { + // file owns the lock file descriptor until the destructive run completes. + file *os.File +} + +// acquireDestructiveRunLock acquires a nonblocking filesystem lock that serializes destructive runs. +func acquireDestructiveRunLock(path string) (*destructiveRunLock, error) { + if path == "" { + return nil, fmt.Errorf("destructive lock path must not be empty") + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return nil, fmt.Errorf("create destructive lock directory: %w", err) + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, fmt.Errorf("open destructive lock: %w", err) + } + if err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + _ = file.Close() + return nil, fmt.Errorf("another GraphBench process holds destructive lock %s: %w", path, err) + } + if err := file.Truncate(0); err == nil { + _, _ = fmt.Fprintf(file, "pid=%d\n", os.Getpid()) + } + return &destructiveRunLock{file: file}, nil +} + +// Close releases the advisory process lock and closes its file descriptor. +func (s *destructiveRunLock) Close() error { + if s == nil || s.file == nil { + return nil + } + unlockErr := syscall.Flock(int(s.file.Fd()), syscall.LOCK_UN) + closeErr := s.file.Close() + if unlockErr != nil { + return unlockErr + } + return closeErr +} diff --git a/cmd/graphbench/run_lock_test.go b/cmd/graphbench/run_lock_test.go new file mode 100644 index 00000000..d3dc322c --- /dev/null +++ b/cmd/graphbench/run_lock_test.go @@ -0,0 +1,24 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestDestructiveRunLockRejectsOverlap verifies that a held lock prevents a second destructive GraphBench process from using the same lock path. +func TestDestructiveRunLockRejectsOverlap(t *testing.T) { + path := filepath.Join(t.TempDir(), "graphbench.lock") + first, err := acquireDestructiveRunLock(path) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, first.Close()) }) + + _, err = acquireDestructiveRunLock(path) + require.ErrorContains(t, err, "another GraphBench process") +} diff --git a/cmd/graphbench/scale_corpus_contract_test.go b/cmd/graphbench/scale_corpus_contract_test.go new file mode 100644 index 00000000..40645008 --- /dev/null +++ b/cmd/graphbench/scale_corpus_contract_test.go @@ -0,0 +1,641 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "slices" + "strings" + "testing" + + "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/drivers/pg" + "github.com/specterops/dawgs/testutil" + "github.com/stretchr/testify/require" +) + +// scaleCorpusRequiredIDs lists representative corpus cases required by the regression contract. +var scaleCorpusRequiredIDs = []string{ + "REC-01", "REC-02", "REC-04", "REC-06", "REC-08", + "TRUST-01", "TRUST-02", + "PRUNE-01", "PRUNE-02", "PRUNE-03", "PRUNE-04", + "HOP-01", "HOP-02", "HOP-03", "HOP-04", "HOP-05", "HOP-07", "HOP-09", + "SCAN-01", "SCAN-02", "SCAN-03", "SCAN-04", "SCAN-05", "SCAN-07", "SCAN-08", + "LOOKUP-02", "LOOKUP-04", "LOOKUP-05", "LOOKUP-09", "LOOKUP-11", "LOOKUP-13", "LOOKUP-15", "LOOKUP-16", +} + +// TestGeneratedScaleCasesParseAndExecuteRealBackends verifies that each generated family has parseable Cypher and an explicit support decision for PostgreSQL and Neo4j. +func TestGeneratedScaleCasesParseAndExecuteRealBackends(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + + covered := map[string]int{} + for _, testCase := range corpus.Cases { + if !strings.HasPrefix(testCase.Dataset, "generated_shortest_paths_") && !strings.HasPrefix(testCase.Dataset, "generated_fixed_suffix_expansion_") && !strings.HasPrefix(testCase.Dataset, "generated_endpoint_seeded_expansion_") { + continue + } + _, err := frontend.ParseCypher(frontend.NewContext(), testCase.Cypher) + require.NoError(t, err, testCase.Name) + _, postgresUnsupported := testCase.UnsupportedReason(ModePostgresSQL) + _, neo4jUnsupported := testCase.UnsupportedReason(ModeNeo4j) + require.True(t, testCase.Supports(ModePostgresSQL) || postgresUnsupported, testCase.Name) + require.True(t, testCase.Supports(ModeNeo4j) || neo4jUnsupported, testCase.Name) + if strings.HasPrefix(testCase.Dataset, "generated_shortest_paths_") { + covered["shortest"]++ + } else if strings.HasPrefix(testCase.Dataset, "generated_fixed_suffix_expansion_") { + covered["fixed_suffix_expansion"]++ + } else { + covered["endpoint_seeded_expansion"]++ + } + } + require.Positive(t, covered["shortest"]) + require.Positive(t, covered["fixed_suffix_expansion"]) + require.Positive(t, covered["endpoint_seeded_expansion"]) +} + +// TestGeneratedFixedSuffixV3OrientationCorpusFreezesTrainingAndHoldoutMatrices +// verifies exact cohort sizes, independent training dimensions, fresh holdout +// depths, canonical cohort tags, and graph-derived result cardinalities. +func TestGeneratedFixedSuffixV3OrientationCorpusFreezesTrainingAndHoldoutMatrices(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + + type fraction struct{ reachable, fanout int } + trainingDepths := map[int]bool{} + trainingFanouts := map[int]bool{} + trainingFractions := map[fraction]bool{} + trainingDisconnected := map[int]bool{} + trainingFanIn := map[int]bool{} + trainingMultiplicity := map[int]bool{} + trainingRoots := map[int]bool{} + trainingObservations := map[string]bool{} + trainingBoundaryControls := map[[2]bool]bool{} + trainingZeroDepth := map[bool]bool{} + trainingPayloads := map[int]bool{} + holdoutDepths := map[int]bool{} + declaredCohort := map[performanceKey]string{} + trainingCount, holdoutCount := 0, 0 + + for _, testCase := range corpus.Cases { + if !strings.HasPrefix(testCase.Dataset, "generated_fixed_suffix_expansion_v3_") { + continue + } + + config, ok := parseFixedSuffixExpansionV3DatasetName(testCase.Dataset) + require.True(t, ok, testCase.Name) + require.NotNil(t, testCase.Expected.RowCount, testCase.Name) + require.NotNil(t, testCase.Shape.MaxDepth, testCase.Name) + require.Equal(t, config.ExpansionDepth, *testCase.Shape.MaxDepth, testCase.Name) + if testCase.Expected.ResultKind == "path_set" { + require.Len(t, testCase.Expected.PathRows, int(*testCase.Expected.RowCount), + testCase.Name+" must predeclare every stable path observation") + require.True(t, newCaseResult(testCase, ModePostgresSQL, testCase.Params).StableObservation, testCase.Name) + } + declaredCohort[performanceKey{dataset: testCase.Dataset, name: testCase.Name, backend: ModePostgresSQL}] = testCase.Shape.QualificationSplit + + metadata, err := fixtureMetadata("unused", testCase.Dataset) + require.NoError(t, err, testCase.Name) + require.NotNil(t, metadata.FixedSuffixExpansion, testCase.Name) + require.Equal(t, metadata.FixedSuffixExpansion.CompleteOutputTrails, *testCase.Expected.RowCount, testCase.Name) + + trainingTag := slices.Contains(testCase.Tags, "orientation-v2-training") + holdoutTag := slices.Contains(testCase.Tags, "orientation-v2-holdout") + require.NotEqual(t, trainingTag, holdoutTag, testCase.Name) + switch testCase.Shape.QualificationSplit { + case "training": + require.True(t, trainingTag, testCase.Name) + require.False(t, holdoutTag, testCase.Name) + trainingCount++ + trainingDepths[config.ExpansionDepth] = true + trainingFanouts[config.Fanout] = true + trainingFractions[fraction{reachable: *config.ExactReachableSuffixSources, fanout: config.Fanout}] = true + trainingDisconnected[config.DisconnectedSuffixSources] = true + trainingFanIn[config.ReverseFanIn] = true + trainingMultiplicity[config.SuffixPathsPerBoundary] = true + trainingRoots[config.RootMatchCount] = true + observation := "endpoint" + if testCase.Observes.Paths { + observation = "path" + } + trainingObservations[observation] = true + trainingBoundaryControls[[2]bool{config.AddProductiveBoundaryCycle, config.AddProductiveBoundarySelfLoop}] = true + trainingZeroDepth[*config.RootHasZeroDepthSuffix] = true + trainingPayloads[config.PropertyPayloadSize] = true + case "holdout": + require.False(t, trainingTag, testCase.Name) + require.True(t, holdoutTag, testCase.Name) + holdoutCount++ + holdoutDepths[config.ExpansionDepth] = true + default: + t.Fatalf("%s has invalid v3 orientation split %q", testCase.Name, testCase.Shape.QualificationSplit) + } + } + + require.Equal(t, 8, trainingCount) + require.Equal(t, 4, holdoutCount) + require.GreaterOrEqual(t, len(trainingDepths), 4) + require.GreaterOrEqual(t, len(trainingFanouts), 4) + require.GreaterOrEqual(t, len(trainingFractions), 4) + require.GreaterOrEqual(t, len(trainingDisconnected), 4) + require.GreaterOrEqual(t, len(trainingFanIn), 3) + require.GreaterOrEqual(t, len(trainingMultiplicity), 3) + require.GreaterOrEqual(t, len(trainingRoots), 4) + require.Equal(t, map[string]bool{"endpoint": true, "path": true}, trainingObservations) + require.Equal(t, map[bool]bool{false: true, true: true}, trainingZeroDepth) + require.GreaterOrEqual(t, len(trainingPayloads), 3) + for _, combination := range [][2]bool{{false, false}, {true, false}, {false, true}, {true, true}} { + require.True(t, trainingBoundaryControls[combination], "missing cycle/self-loop combination %v", combination) + } + require.Equal(t, map[int]bool{7: true, 11: true, 13: true, 15: true}, holdoutDepths) + for depth := range holdoutDepths { + require.False(t, trainingDepths[depth], "holdout depth %d is already present in training", depth) + } + require.Len(t, orientationV2CanonicalCases, len(declaredCohort)) + for _, frozen := range orientationV2CanonicalCases { + require.Equal(t, frozen.split, declaredCohort[performanceKey{dataset: frozen.dataset, name: frozen.name, backend: ModePostgresSQL}], frozen.name) + } + canonical, err := canonicalOrientationV2Cohort() + require.NoError(t, err) + _, trainingSelection, err := selectScaleCorpus(corpus, CorpusSelectors{Tags: []string{"orientation-v2-training"}}) + require.NoError(t, err) + require.Equal(t, canonical.trainingDeclarationSHA256, trainingSelection.DeclarationSHA256) + _, confirmationSelection, err := selectScaleCorpus(corpus, CorpusSelectors{Tags: []string{"orientation-v2-training", "orientation-v2-holdout"}}) + require.NoError(t, err) + require.Equal(t, canonical.declarationSHA256, confirmationSelection.DeclarationSHA256) +} + +// TestEndpointSeededExpansionCorpusCoversGuardOutcomes verifies corpus representatives for admitted execution plus endpoint-guard and state-guard overflow fallbacks. +func TestEndpointSeededExpansionCorpusCoversGuardOutcomes(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + required := map[string]bool{"guard-admitted": false, "endpoint-guard-overflow": false, "state-guard-overflow": false} + for _, testCase := range corpus.Cases { + if testCase.Category != "generated_endpoint_seeded_expansion" { + continue + } + for tag := range required { + if slices.Contains(testCase.Tags, tag) { + required[tag] = true + } + } + } + for tag, found := range required { + require.True(t, found, "endpoint-seeded corpus is missing %s", tag) + } +} + +// TestGeneratedShortestDistanceCorpusCoversQualificationEnvelope verifies distance cases spanning deep, wide, inbound, disconnected, cyclic, parallel-edge, and self-loop shapes. +func TestGeneratedShortestDistanceCorpusCoversQualificationEnvelope(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + + requiredTags := map[string]bool{ + "depth-32": false, "depth-64": false, "fanout-512": false, "fanout-1000": false, + "inbound": false, "disconnected": false, "cycle": false, "parallel-edges": false, "self-loop": false, + } + for _, testCase := range corpus.Cases { + if testCase.Category != "generated_shortest_path" || !slices.Contains(testCase.Tags, "distance") { + continue + } + for tag := range requiredTags { + if slices.Contains(testCase.Tags, tag) { + requiredTags[tag] = true + } + } + } + for tag, covered := range requiredTags { + require.True(t, covered, "shortest distance corpus is missing %s", tag) + } +} + +// TestGeneratedShortestPathCorpusCoversMaterializerEnvelope verifies hydrated-path cases spanning deep, wide, inbound, zero-depth, disconnected, cyclic, parallel-edge, and self-loop shapes. +func TestGeneratedShortestPathCorpusCoversMaterializerEnvelope(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + + requiredTags := map[string]bool{ + "depth-32": false, "depth-64": false, "fanout-512": false, "fanout-1000": false, + "inbound": false, "zero-depth": false, "disconnected": false, "cycle": false, "parallel-edges": false, "self-loop": false, + } + for _, testCase := range corpus.Cases { + if testCase.Category != "generated_shortest_path" || !slices.Contains(testCase.Tags, "path") { + continue + } + for tag := range requiredTags { + if slices.Contains(testCase.Tags, tag) { + requiredTags[tag] = true + } + } + } + for tag, covered := range requiredTags { + require.True(t, covered, "shortest path corpus is missing %s", tag) + } +} + +// TestGeneratedAllShortestCorpusCoversInlineQualificationEnvelope keeps the +// training corpus broad enough to qualify early-stop behavior independently +// from the frozen depth-8 holdouts. Cap-threshold branch execution is covered +// by the live guarded-statement integration tests because corpus cases do not +// override immutable production caps. +func TestGeneratedAllShortestCorpusCoversInlineQualificationEnvelope(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + + requiredTraining := map[string]bool{ + "early-depth-1": false, "early-depth-2": false, "early-depth-3": false, + "max-16": false, "max-64": false, "inbound": false, + "cycle-dead-tail": false, "reconvergence": false, "disconnected": false, + } + hasQualifiedHoldout := false + for _, testCase := range corpus.Cases { + if testCase.Category != "generated_shortest_path_v2" || !slices.Contains(testCase.Tags, "all-shortest") { + continue + } + if testCase.Shape.QualificationSplit == "holdout" && testCase.Shape.RelationshipKindCount == 1 && testCase.Shape.MaxDepth != nil && *testCase.Shape.MaxDepth >= 3 { + hasQualifiedHoldout = true + } + if testCase.Shape.QualificationSplit != "training" { + continue + } + for tag := range requiredTraining { + if slices.Contains(testCase.Tags, tag) { + requiredTraining[tag] = true + } + } + } + for tag, covered := range requiredTraining { + require.True(t, covered, "all-shortest training corpus is missing %s", tag) + } + require.True(t, hasQualifiedHoldout, "all-shortest corpus lacks a typed single-kind holdout at maximum depth 3 or greater") +} + +// scaleCorpusCaseID joins a scale case's dataset and name into its contract identifier. +func scaleCorpusCaseID(name string) string { + if separator := strings.IndexByte(name, '_'); separator >= 0 { + return name[:separator] + } + return name +} + +// scaleCorpusRequiredIDSet returns the required representative scale-case identifiers as a set. +func scaleCorpusRequiredIDSet() map[string]struct{} { + required := make(map[string]struct{}, len(scaleCorpusRequiredIDs)) + for _, id := range scaleCorpusRequiredIDs { + required[id] = struct{}{} + } + return required +} + +// TestScaleCorpusRequiredRepresentativesDeclareCardinality verifies every required query-form tag is present and declares row counts or complete mutation cardinalities. +func TestScaleCorpusRequiredRepresentativesDeclareCardinality(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + + required := scaleCorpusRequiredIDSet() + covered := map[string]int{} + for _, testCase := range corpus.Cases { + id := scaleCorpusCaseID(testCase.Name) + if _, isRequired := required[id]; !isRequired { + continue + } + + covered[id]++ + require.Contains(t, testCase.Tags, id, "%s must retain its stable query-form tag", testCase.Name) + if testCase.WriteScenario == nil { + require.NotNil(t, testCase.Expected.RowCount, "%s must declare expected row cardinality", testCase.Name) + } else { + require.NotNil(t, testCase.WriteScenario.ExpectedMatched, "%s must declare expected matched cardinality", testCase.Name) + require.NotNil(t, testCase.WriteScenario.ExpectedAffected, "%s must declare expected affected cardinality", testCase.Name) + } + } + + for _, id := range scaleCorpusRequiredIDs { + require.Positive(t, covered[id], "required scale corpus is missing %s", id) + } +} + +// TestScaleCorpusDistinguishesProjectionClasses verifies that ID-only, shallow, and fully hydrated tags agree with result kind and observation requirements. +func TestScaleCorpusDistinguishesProjectionClasses(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + + requiredClasses := map[string]bool{ + "projection-id-only": false, + "projection-shallow-ids-kind": false, + "projection-full-hydration": false, + } + for _, testCase := range corpus.Cases { + for _, tag := range testCase.Tags { + if _, required := requiredClasses[tag]; !required { + continue + } + + requiredClasses[tag] = true + switch tag { + case "projection-id-only": + require.Equal(t, "id_set", testCase.Expected.ResultKind) + require.False(t, testCase.Observes.Nodes) + require.False(t, testCase.Observes.Relationships) + require.False(t, testCase.Observes.Properties) + case "projection-shallow-ids-kind": + require.Equal(t, "shallow_ids_kind", testCase.Expected.ResultKind) + require.False(t, testCase.Observes.Nodes) + require.False(t, testCase.Observes.Relationships) + require.False(t, testCase.Observes.Properties) + case "projection-full-hydration": + require.True(t, testCase.Observes.Nodes || testCase.Observes.Relationships) + require.True(t, testCase.Observes.Properties) + } + } + } + + for projectionClass, found := range requiredClasses { + require.True(t, found, "scale corpus is missing %s", projectionClass) + } +} + +// TestFixedSuffixExpansionIDRowsUseStableFixtureIdentitiesAndPreserveDuplicates verifies four identical logical endpoint pairs remain explicit expected rows rather than being deduplicated or backend-ID based. +func TestFixedSuffixExpansionIDRowsUseStableFixtureIdentitiesAndPreserveDuplicates(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + + for _, testCase := range corpus.Cases { + if testCase.Name != "fixed_suffix_expansion_endpoint_ids" { + continue + } + + require.Equal(t, [][]string{ + {"fse-head", "fse-terminal"}, + {"fse-head", "fse-terminal"}, + {"fse-head", "fse-terminal"}, + {"fse-head", "fse-terminal"}, + }, testCase.Expected.IDRows) + return + } + + t.Fatal("fixed_suffix_expansion_endpoint_ids case not found") +} + +// TestGeneratedSPI1InboundV1CorpusFreezesTrainingAndUnopenedHoldoutMatrices +// verifies the preregistered canonical-witness cohort without executing or +// inspecting any holdout timing. The contract binds exact generated topology, +// stable path observations, split tags, query identity, and selection digests. +func TestGeneratedSPI1InboundV1CorpusFreezesTrainingAndUnopenedHoldoutMatrices(t *testing.T) { + const ( + query = "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN p" + querySHA256 = "1024577967901503995d4ec0c76540e96b65f4d25e015ccb6eeffb500a5596f9" + ) + + type expectedCase struct { + dataset string + config testutil.ShortestPathScaleV2Config + fixtureSHA256 string + split string + target string + resultDepth int + stateClass string + extraTags []string + } + expected := map[string]expectedCase{ + "GSP-I1-V1-TRAIN-D04-FI016-full": { + dataset: "generated_shortest_paths_v2_d4_o0_r4_fo0_fi16_l2_k0_t0_w0_x4_p0_c0_s0", + config: spI1InboundFixtureConfig(4, 4, 16, 2, 4), + fixtureSHA256: "29b0c923d7e3312ba1f19d09076006692dfc66379a524d160da8d74d9c7c3889", + split: "training", + target: "sp-v2-inbound-end", + resultDepth: 4, + stateClass: "inbound_predecessor_full_depth_fanin_16", + }, + "GSP-I1-V1-TRAIN-D16-FI256-early-d04": { + dataset: "generated_shortest_paths_v2_d16_o0_r8_fo0_fi256_l8_k0_t0_w0_x16_p0_c0_s0", + config: spI1InboundFixtureConfig(16, 8, 256, 8, 16), + fixtureSHA256: "a297da4f7be1cb8621d173cd763e1fcc902b560e23d8fdfbbc9565d10c308bce", + split: "training", + target: "sp-v2-inbound-linear-04", + resultDepth: 4, + stateClass: "inbound_predecessor_early_target_fanin_256", + extraTags: []string{"early-target", "early-depth-4"}, + }, + "GSP-I1-V1-TRAIN-D16-FI256-full": { + dataset: "generated_shortest_paths_v2_d16_o0_r8_fo0_fi256_l8_k0_t0_w0_x16_p0_c0_s0", + config: spI1InboundFixtureConfig(16, 8, 256, 8, 16), + fixtureSHA256: "a297da4f7be1cb8621d173cd763e1fcc902b560e23d8fdfbbc9565d10c308bce", + split: "training", + target: "sp-v2-inbound-end", + resultDepth: 16, + stateClass: "inbound_predecessor_full_depth_fanin_256", + }, + "GSP-I1-V1-TRAIN-D16-FI256-disconnected": { + dataset: "generated_shortest_paths_v2_d16_o0_r8_fo0_fi256_l8_k0_t0_w0_x16_p0_c0_s0", + config: spI1InboundFixtureConfig(16, 8, 256, 8, 16), + fixtureSHA256: "a297da4f7be1cb8621d173cd763e1fcc902b560e23d8fdfbbc9565d10c308bce", + split: "training", + target: "sp-v2-disconnected-end", + resultDepth: -1, + stateClass: "inbound_predecessor_disconnected_fanin_256", + extraTags: []string{"disconnected", "max-miss"}, + }, + "GSP-I1-V1-HOLDOUT-D08-FI031-full": { + dataset: "generated_shortest_paths_v2_d8_o0_r3_fo0_fi31_l3_k0_t0_w0_x7_p0_c0_s0", + config: spI1InboundFixtureConfig(8, 3, 31, 3, 7), + fixtureSHA256: "47acf96f7862e639a8a33bc28f2c9b9e4457320e44c8e88b0b06ab2f25691e63", + split: "holdout", + target: "sp-v2-inbound-end", + resultDepth: 8, + stateClass: "inbound_predecessor_full_depth_fanin_31", + }, + "GSP-I1-V1-HOLDOUT-D32-FI191-full": { + dataset: "generated_shortest_paths_v2_d32_o0_r11_fo0_fi191_l21_k0_t0_w0_x13_p0_c0_s0", + config: spI1InboundFixtureConfig(32, 11, 191, 21, 13), + fixtureSHA256: "da33b5d223d8513ff4af240613a8f976e12be6b398536bb9b4f8d5a184d9443b", + split: "holdout", + target: "sp-v2-inbound-end", + resultDepth: 32, + stateClass: "inbound_predecessor_full_depth_fanin_191", + }, + "GSP-I1-V1-HOLDOUT-D32-FI191-disconnected": { + dataset: "generated_shortest_paths_v2_d32_o0_r11_fo0_fi191_l21_k0_t0_w0_x13_p0_c0_s0", + config: spI1InboundFixtureConfig(32, 11, 191, 21, 13), + fixtureSHA256: "da33b5d223d8513ff4af240613a8f976e12be6b398536bb9b4f8d5a184d9443b", + split: "holdout", + target: "sp-v2-disconnected-end", + resultDepth: -1, + stateClass: "inbound_predecessor_disconnected_fanin_191", + extraTags: []string{"disconnected", "max-miss"}, + }, + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + seen := map[string]bool{} + trainingDepths, holdoutDepths := map[int]bool{}, map[int]bool{} + trainingCount, holdoutCount := 0, 0 + for _, testCase := range corpus.Cases { + trainingTag := slices.Contains(testCase.Tags, "sp-i1-inbound-v1-training") + holdoutTag := slices.Contains(testCase.Tags, "sp-i1-inbound-v1-holdout") + if !trainingTag && !holdoutTag { + continue + } + require.NotEqual(t, trainingTag, holdoutTag, testCase.Name) + contract, found := expected[testCase.Name] + require.True(t, found, "unexpected SP-I1 inbound-v1 declaration %s", testCase.Name) + require.False(t, seen[testCase.Name], testCase.Name) + seen[testCase.Name] = true + + require.True(t, strings.HasSuffix(testCase.Source, "/cases/generated_sp_i1_inbound_v1.json"), testCase.Name) + require.Equal(t, contract.dataset, testCase.Dataset, testCase.Name) + require.Equal(t, "generated_shortest_path_v2", testCase.Category, testCase.Name) + require.Equal(t, query, testCase.Cypher, testCase.Name) + require.Equal(t, querySHA256, pg.TraversalPolicyQuerySHA256(testCase.Cypher), testCase.Name) + require.Equal(t, map[string]string{"root_id": "sp-v2-inbound-root", "end_id": contract.target}, testCase.NodeParams, testCase.Name) + require.Equal(t, []ExecutionMode{ModePostgresSQL, ModeNeo4j}, testCase.CandidateModes, testCase.Name) + require.Empty(t, testCase.UnsupportedModes, testCase.Name) + require.Equal(t, ObservedValues{Paths: true, Nodes: true, Relationships: true, Properties: true}, testCase.Observes, testCase.Name) + + shape := testCase.Shape + require.Equal(t, contract.split, shape.QualificationSplit, testCase.Name) + require.Equal(t, "forbidden", shape.FallbackExpectation, testCase.Name) + require.Equal(t, "bound_id", shape.RootPredicate, testCase.Name) + require.Equal(t, "bound_id", shape.TerminalPredicate, testCase.Name) + require.Equal(t, []string{"Traverse"}, shape.EdgeKinds, testCase.Name) + require.Equal(t, "inbound", shape.Direction, testCase.Name) + require.Equal(t, 1, shape.RelationshipKindCount, testCase.Name) + require.Equal(t, "normal", shape.FixtureTier, testCase.Name) + require.Equal(t, contract.stateClass, shape.ExpectedStateClass, testCase.Name) + require.NotNil(t, shape.MinDepth, testCase.Name) + require.NotNil(t, shape.MaxDepth, testCase.Name) + require.Equal(t, 1, *shape.MinDepth, testCase.Name) + require.Equal(t, 64, *shape.MaxDepth, testCase.Name) + require.True(t, shape.PathMaterializationRequired, testCase.Name) + + config, ok := parseShortestPathV2DatasetName(testCase.Dataset) + require.True(t, ok, testCase.Name) + require.Equal(t, contract.config, config, testCase.Name) + metadata, err := fixtureMetadata("unused", testCase.Dataset) + require.NoError(t, err, testCase.Name) + require.Equal(t, contract.fixtureSHA256, metadata.Checksum, testCase.Name) + require.NotNil(t, metadata.Shortest, testCase.Name) + require.Equal(t, int64(config.Depth), metadata.Shortest.ExpectedMinimumDistance, testCase.Name) + + expectedTags := []string{"generated", "v2", "normal-tier", "path", "inbound", "hidden-fan-in"} + expectedTags = append(expectedTags, contract.extraTags...) + if contract.split == "training" { + trainingCount++ + trainingDepths[config.Depth] = true + expectedTags = append(expectedTags, "sp-i1-inbound-v1-training") + } else { + holdoutCount++ + holdoutDepths[config.Depth] = true + expectedTags = append(expectedTags, "holdout", "sp-i1-inbound-v1-holdout") + } + require.Equal(t, expectedTags, testCase.Tags, testCase.Name) + + require.NotNil(t, testCase.Expected.RowCount, testCase.Name) + require.Equal(t, "path_set", testCase.Expected.ResultKind, testCase.Name) + if contract.resultDepth < 0 { + require.Zero(t, *testCase.Expected.RowCount, testCase.Name) + require.Empty(t, testCase.Expected.PathRows, testCase.Name) + require.Equal(t, "empty", shape.ResultCardinalityClass, testCase.Name) + } else { + require.Equal(t, int64(1), *testCase.Expected.RowCount, testCase.Name) + require.Equal(t, []ExpectedPath{spI1InboundExpectedPath(config.Depth, contract.resultDepth)}, testCase.Expected.PathRows, testCase.Name) + require.Equal(t, "singleton", shape.ResultCardinalityClass, testCase.Name) + } + } + + require.Len(t, seen, 7) + for name := range expected { + require.True(t, seen[name], "missing SP-I1 inbound-v1 declaration %s", name) + } + require.Len(t, spI1CanonicalCases, len(expected)) + canonicalSeen := map[string]bool{} + for _, canonical := range spI1CanonicalCases { + contract, found := expected[canonical.name] + require.True(t, found, "unexpected frozen SP-I1 case %s", canonical.name) + require.False(t, canonicalSeen[canonical.name], canonical.name) + canonicalSeen[canonical.name] = true + require.Equal(t, contract.dataset, canonical.dataset, canonical.name) + require.Equal(t, contract.split, canonical.split, canonical.name) + } + require.Equal(t, seen, canonicalSeen, "corpus and qualification reporter must freeze the same SP-I1 cases") + require.Equal(t, 4, trainingCount) + require.Equal(t, 3, holdoutCount) + require.Equal(t, map[int]bool{4: true, 16: true}, trainingDepths) + require.Equal(t, map[int]bool{8: true, 32: true}, holdoutDepths) + for depth := range holdoutDepths { + require.False(t, trainingDepths[depth], "holdout depth %d is present in training", depth) + } + + training, trainingSelection, err := selectScaleCorpus(corpus, CorpusSelectors{Tags: []string{"sp-i1-inbound-v1-training"}}) + require.NoError(t, err) + require.Len(t, training.Cases, 4) + require.True(t, trainingSelection.DiagnosticOnly) + require.Equal(t, 8, trainingSelection.SelectedDeclarationCount) + require.Equal(t, "1162e6563678dad742d8fe89d250936862b4a73deab247cde4b5ddebdfdd93ce", trainingSelection.DeclarationSHA256) + require.Equal(t, "cc07b55331e15f4e268043d1ed36abf7deec7217771a1b30913db6e738d27f7a", resolvedSelectionSHA256(trainingSelection.Resolved)) + require.Equal(t, "3da3c4b1cea3fa64fbaa1958f7bf8048639241522ccf6e46defd10d2d8c9ccd6", spI1InboundRuntimeCorpusIdentity(training)) + + confirmation, confirmationSelection, err := selectScaleCorpus(corpus, CorpusSelectors{Tags: []string{"sp-i1-inbound-v1-training", "sp-i1-inbound-v1-holdout"}}) + require.NoError(t, err) + require.Len(t, confirmation.Cases, 7) + require.True(t, confirmationSelection.DiagnosticOnly) + require.Equal(t, 14, confirmationSelection.SelectedDeclarationCount) + require.Equal(t, "31f6041f342b3ed8059d4d1396a76f073c3fc877472d06632a8bad16b5a4cbfd", confirmationSelection.DeclarationSHA256) + require.Equal(t, "16a8756a7c32695f0314b3552c80d2a500226c7a44c57847c916a96e775aa0c5", resolvedSelectionSHA256(confirmationSelection.Resolved)) + require.Equal(t, "219ee26cae52d8b81c6c91f9c517692c544ef4cec1aa9b9314fbc4e8f5ad3c5c", spI1InboundRuntimeCorpusIdentity(confirmation)) +} + +func spI1InboundFixtureConfig(depth, rootFanIn, intermediateFanIn, fanInLevel, disconnectedWidth int) testutil.ShortestPathScaleV2Config { + return testutil.ShortestPathScaleV2Config{ + Depth: depth, + ReverseRootFanIn: rootFanIn, + IntermediateReverseFanIn: intermediateFanIn, + FanInLevel: fanInLevel, + DisconnectedWidth: disconnectedWidth, + } +} + +func spI1InboundExpectedPath(fixtureDepth, resultDepth int) ExpectedPath { + nodes := []string{"sp-v2-inbound-root"} + for level := 1; level < resultDepth; level++ { + nodes = append(nodes, fmt.Sprintf("sp-v2-inbound-linear-%02d", level)) + } + if resultDepth == fixtureDepth { + nodes = append(nodes, "sp-v2-inbound-end") + } else { + nodes = append(nodes, fmt.Sprintf("sp-v2-inbound-linear-%02d", resultDepth)) + } + kinds := make([]string, resultDepth) + keys := make([]string, resultDepth) + for idx := range resultDepth { + kinds[idx] = "Traverse" + keys[idx] = fmt.Sprintf("inbound-primary-%02d", fixtureDepth-idx) + } + return ExpectedPath{Nodes: nodes, RelationshipKinds: kinds, RelationshipKeys: keys} +} + +// spI1InboundRuntimeCorpusIdentity normalizes the package-test corpus root to +// the repository-root spelling used by GraphBench capture commands. +func spI1InboundRuntimeCorpusIdentity(corpus ScaleCorpus) string { + canonical := ScaleCorpus{Cases: append([]ScaleCase(nil), corpus.Cases...)} + for idx := range canonical.Cases { + if offset := strings.Index(canonical.Cases[idx].Source, "benchmark/testdata/scale/"); offset >= 0 { + canonical.Cases[idx].Source = canonical.Cases[idx].Source[offset:] + } + } + return corpusIdentity(canonical) +} diff --git a/cmd/graphbench/selection.go b/cmd/graphbench/selection.go new file mode 100644 index 00000000..7c843dda --- /dev/null +++ b/cmd/graphbench/selection.go @@ -0,0 +1,252 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "slices" + "sort" +) + +// selectionManifestVersion identifies the serialized schema revision for selection manifest. +const selectionManifestVersion = 2 + +// CorpusSelectors contains exact dataset, category, case, and tag filters supplied by the user. +type CorpusSelectors struct { + // Cases lists exact case names requested by the user. + Cases []string `json:"cases,omitempty"` + // Datasets lists exact dataset selectors supplied by the user. + Datasets []string `json:"datasets,omitempty"` + // Categories lists workload categories used to filter the corpus. + Categories []string `json:"categories,omitempty"` + // Tags lists exact tag selectors supplied by the user. + Tags []string `json:"tags,omitempty"` +} + +// ResolvedCaseSelector identifies a selected case together with its declared category. +type ResolvedCaseSelector struct { + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // Category groups cases by workload category. + Category string `json:"category"` +} + +// SelectionManifest records requested filters, resolved workloads, and completeness evidence for one run. +type SelectionManifest struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // Requested preserves the exact corpus filters supplied by the user. + Requested CorpusSelectors `json:"requested"` + // Resolved lists exact case selectors retained after corpus filtering. + Resolved []ResolvedCaseSelector `json:"resolved"` + // DiagnosticOnly marks a selection that is informative but ineligible for complete gating. + DiagnosticOnly bool `json:"diagnostic_only"` + // FullDeclarationCount records all case/backend declarations before selection. + FullDeclarationCount int `json:"full_declaration_count"` + // SelectedDeclarationCount records declarations retained by the resolved selection. + SelectedDeclarationCount int `json:"selected_declaration_count"` + // OmittedDeclarationCount records declarations omitted by the resolved selection. + OmittedDeclarationCount int `json:"omitted_declaration_count"` + // ProtectedDeclarationCount records protocol-only declarations omitted before ordinary selector resolution. + ProtectedDeclarationCount int `json:"protected_declaration_count,omitempty"` + // ProtectedDeclarationSHA256 identifies the exact protocol-only declarations omitted from the runnable universe. + ProtectedDeclarationSHA256 string `json:"protected_declaration_sha256,omitempty"` + // DeclarationSHA256 identifies the canonical set of declared workloads. + DeclarationSHA256 string `json:"declaration_sha256"` +} + +// validateSelectionManifestAccounting distinguishes protocol-protected +// omissions from ordinary filtered omissions. An unfiltered artifact remains +// complete only when every omitted declaration is explicitly protected and +// bound by one digest. +func validateSelectionManifestAccounting(manifest SelectionManifest) error { + if manifest.Version != selectionManifestVersion || manifest.FullDeclarationCount < 1 || + manifest.SelectedDeclarationCount < 1 || manifest.OmittedDeclarationCount < 0 || + manifest.FullDeclarationCount != manifest.SelectedDeclarationCount+manifest.OmittedDeclarationCount || + manifest.ProtectedDeclarationCount < 0 || manifest.ProtectedDeclarationCount > manifest.OmittedDeclarationCount { + return fmt.Errorf("selection manifest has inconsistent declaration accounting") + } + if manifest.ProtectedDeclarationCount == 0 { + if manifest.ProtectedDeclarationSHA256 != "" { + return fmt.Errorf("selection manifest has a protected digest without protected declarations") + } + } else if !lowercaseSHA256(manifest.ProtectedDeclarationSHA256) { + return fmt.Errorf("selection manifest lacks a valid protected declaration digest") + } + if !manifest.DiagnosticOnly && manifest.OmittedDeclarationCount != manifest.ProtectedDeclarationCount { + return fmt.Errorf("complete selection manifest contains non-protected omissions") + } + return nil +} + +// selectScaleCorpus filters corpus cases and returns both selected cases and a hashed selection manifest. +func selectScaleCorpus(corpus ScaleCorpus, selectors CorpusSelectors) (ScaleCorpus, SelectionManifest, error) { + if err := validateCorpusSelectors(corpus, selectors); err != nil { + return ScaleCorpus{}, SelectionManifest{}, err + } + return selectScaleCorpusValidated(corpus, selectors) +} + +// selectScaleCorpusValidated resolves selectors against a universe whose +// selector names have already been validated. Keeping validation separate lets +// protocol-only workloads remain known selectors while ordinary execution +// deliberately omits them from its runnable universe. +func selectScaleCorpusValidated(corpus ScaleCorpus, selectors CorpusSelectors) (ScaleCorpus, SelectionManifest, error) { + filtered := len(selectors.Cases)+len(selectors.Datasets)+len(selectors.Categories)+len(selectors.Tags) > 0 + manifest := SelectionManifest{ + Version: selectionManifestVersion, + Requested: selectors, + DiagnosticOnly: filtered, + FullDeclarationCount: len(corpus.DeclaredBackends()), + } + selected := ScaleCorpus{} + for _, testCase := range corpus.Cases { + if matchesSelectors(testCase, selectors) { + selected.Cases = append(selected.Cases, testCase) + manifest.Resolved = append(manifest.Resolved, ResolvedCaseSelector{ + Dataset: testCase.Dataset, + Name: testCase.Name, + Category: testCase.Category, + }) + } + } + if len(selected.Cases) == 0 { + return ScaleCorpus{}, SelectionManifest{}, fmt.Errorf("selectors resolved to an empty corpus") + } + manifest.SelectedDeclarationCount = len(selected.DeclaredBackends()) + manifest.OmittedDeclarationCount = manifest.FullDeclarationCount - manifest.SelectedDeclarationCount + manifest.DeclarationSHA256 = declarationSHA256(selected.DeclaredBackends()) + return selected, manifest, nil +} + +// validateCorpusSelectors rejects duplicate, ambiguous, or unknown exact selectors. +func validateCorpusSelectors(corpus ScaleCorpus, selectors CorpusSelectors) error { + caseMatches := map[string][]ScaleCase{} + datasets := map[string]struct{}{} + categories := map[string]struct{}{} + tags := map[string]struct{}{} + for _, testCase := range corpus.Cases { + caseMatches[testCase.Name] = append(caseMatches[testCase.Name], testCase) + datasets[testCase.Dataset] = struct{}{} + categories[testCase.Category] = struct{}{} + for _, tag := range testCase.Tags { + tags[tag] = struct{}{} + } + } + for _, name := range selectors.Cases { + matches := caseMatches[name] + if len(matches) == 0 { + return fmt.Errorf("unknown case selector %q", name) + } + if len(matches) != 1 { + return fmt.Errorf("ambiguous case selector %q resolves to %d cases", name, len(matches)) + } + } + for _, selector := range []struct { + // kind names the selector dimension for validation errors. + kind string + // values contains the requested selectors to validate in this dimension. + values []string + // known indexes accepted selector values for exact validation. + known map[string]struct{} + }{ + { + kind: "dataset", + values: selectors.Datasets, + known: datasets, + }, + { + kind: "category", + values: selectors.Categories, + known: categories, + }, + { + kind: "tag", + values: selectors.Tags, + known: tags, + }, + } { + for _, value := range selector.values { + if _, found := selector.known[value]; !found { + return fmt.Errorf("unknown %s selector %q", selector.kind, value) + } + } + } + return nil +} + +// matchesSelectors reports whether a scale case matches every nonempty selector dimension. +func matchesSelectors(testCase ScaleCase, selectors CorpusSelectors) bool { + if len(selectors.Cases) > 0 && !slices.Contains(selectors.Cases, testCase.Name) { + return false + } + if len(selectors.Datasets) > 0 && !slices.Contains(selectors.Datasets, testCase.Dataset) { + return false + } + if len(selectors.Categories) > 0 && !slices.Contains(selectors.Categories, testCase.Category) { + return false + } + if len(selectors.Tags) > 0 { + matched := false + for _, tag := range selectors.Tags { + matched = matched || slices.Contains(testCase.Tags, tag) + } + if !matched { + return false + } + } + return true +} + +// selectionIdentity returns the common selection manifest shared by every artifact record. +func selectionIdentity(records []CaseResult) (SelectionManifest, error) { + var selected *SelectionManifest + for _, record := range records { + if record.Environment == nil || record.Environment.Selection == nil { + return SelectionManifest{}, fmt.Errorf("%s/%s has no selection manifest", record.Dataset, record.Name) + } + if selected == nil { + copy := *record.Environment.Selection + selected = © + continue + } + current := record.Environment.Selection + if selected.Version != current.Version || selected.DeclarationSHA256 != current.DeclarationSHA256 || + selected.DiagnosticOnly != current.DiagnosticOnly || selected.FullDeclarationCount != current.FullDeclarationCount || + selected.SelectedDeclarationCount != current.SelectedDeclarationCount || selected.OmittedDeclarationCount != current.OmittedDeclarationCount || + selected.ProtectedDeclarationCount != current.ProtectedDeclarationCount || + selected.ProtectedDeclarationSHA256 != current.ProtectedDeclarationSHA256 || + resolvedSelectionSHA256(selected.Resolved) != resolvedSelectionSHA256(current.Resolved) { + return SelectionManifest{}, fmt.Errorf("artifact contains inconsistent selection manifests") + } + } + + if selected == nil { + return SelectionManifest{}, fmt.Errorf("artifact contains no records") + } + return *selected, nil +} + +// resolvedSelectionSHA256 hashes selected dataset, case, and category tuples in deterministic order. +func resolvedSelectionSHA256(resolved []ResolvedCaseSelector) string { + items := append([]ResolvedCaseSelector(nil), resolved...) + sort.Slice(items, func(i, j int) bool { + if items[i].Dataset != items[j].Dataset { + return items[i].Dataset < items[j].Dataset + } + return items[i].Name < items[j].Name + }) + + digest := sha256.New() + for _, item := range items { + fmt.Fprintf(digest, "%s\x00%s\x00%s\n", item.Dataset, item.Name, item.Category) + } + return hex.EncodeToString(digest.Sum(nil)) +} diff --git a/cmd/graphbench/sp_i1_qualification.go b/cmd/graphbench/sp_i1_qualification.go new file mode 100644 index 00000000..16e0c433 --- /dev/null +++ b/cmd/graphbench/sp_i1_qualification.go @@ -0,0 +1,1910 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "os" + "os/exec" + "path/filepath" + "reflect" + "slices" + "sort" + "strings" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" +) + +const ( + spI1QualificationVersion = 1 + spI1FreezeVersion = 1 + spI1TrainingTag = "sp-i1-inbound-v1-training" + spI1HoldoutTag = "sp-i1-inbound-v1-holdout" + spI1QuerySHA256 = "1024577967901503995d4ec0c76540e96b65f4d25e015ccb6eeffb500a5596f9" + spI1TrainingCorpusSHA256 = "3da3c4b1cea3fa64fbaa1958f7bf8048639241522ccf6e46defd10d2d8c9ccd6" + spI1FullCorpusSHA256 = "219ee26cae52d8b81c6c91f9c517692c544ef4cec1aa9b9314fbc4e8f5ad3c5c" + spI1TrainingResolvedSHA = "cc07b55331e15f4e268043d1ed36abf7deec7217771a1b30913db6e738d27f7a" + spI1FullResolvedSHA = "16a8756a7c32695f0314b3552c80d2a500226c7a44c57847c916a96e775aa0c5" +) + +var spI1CanonicalCases = []struct { + dataset string + name string + split string +}{ + {"generated_shortest_paths_v2_d4_o0_r4_fo0_fi16_l2_k0_t0_w0_x4_p0_c0_s0", "GSP-I1-V1-TRAIN-D04-FI016-full", "training"}, + {"generated_shortest_paths_v2_d16_o0_r8_fo0_fi256_l8_k0_t0_w0_x16_p0_c0_s0", "GSP-I1-V1-TRAIN-D16-FI256-early-d04", "training"}, + {"generated_shortest_paths_v2_d16_o0_r8_fo0_fi256_l8_k0_t0_w0_x16_p0_c0_s0", "GSP-I1-V1-TRAIN-D16-FI256-full", "training"}, + {"generated_shortest_paths_v2_d16_o0_r8_fo0_fi256_l8_k0_t0_w0_x16_p0_c0_s0", "GSP-I1-V1-TRAIN-D16-FI256-disconnected", "training"}, + {"generated_shortest_paths_v2_d8_o0_r3_fo0_fi31_l3_k0_t0_w0_x7_p0_c0_s0", "GSP-I1-V1-HOLDOUT-D08-FI031-full", "holdout"}, + {"generated_shortest_paths_v2_d32_o0_r11_fo0_fi191_l21_k0_t0_w0_x13_p0_c0_s0", "GSP-I1-V1-HOLDOUT-D32-FI191-full", "holdout"}, + {"generated_shortest_paths_v2_d32_o0_r11_fo0_fi191_l21_k0_t0_w0_x13_p0_c0_s0", "GSP-I1-V1-HOLDOUT-D32-FI191-disconnected", "holdout"}, +} + +type spI1CanonicalCohort struct { + keys map[performanceKey]struct{} + trainingKeys map[performanceKey]struct{} + holdoutKeys map[performanceKey]struct{} + declarationSHA256 string + trainingDeclarationSHA256 string + holdoutDeclarationSHA256 string + trainingCorpusSHA256 string + fullCorpusSHA256 string + trainingResolvedSHA256 string + fullResolvedSHA256 string +} + +type spI1CanonicalDeclaration struct { + testCase ScaleCase + fixture FixtureMetadata +} + +func canonicalSPI1Declarations() (map[performanceKey]spI1CanonicalDeclaration, error) { + repositoryRoot := strings.TrimSpace(commandOutput("git", "rev-parse", "--show-toplevel")) + if repositoryRoot == "" || repositoryRoot == "unknown" { + return nil, fmt.Errorf("locate repository root for frozen SP-I1 declarations") + } + corpus, err := loadScaleCorpus(filepath.Join(repositoryRoot, "benchmark", "testdata", "scale")) + if err != nil { + return nil, fmt.Errorf("load frozen SP-I1 declarations: %w", err) + } + cohort, err := canonicalSPI1Cohort() + if err != nil { + return nil, err + } + declarations := make(map[performanceKey]spI1CanonicalDeclaration, len(cohort.keys)) + for _, testCase := range corpus.Cases { + key := performanceKey{dataset: testCase.Dataset, name: testCase.Name, backend: ModePostgresSQL} + if _, expected := cohort.keys[key]; !expected { + continue + } + if _, duplicate := declarations[key]; duplicate { + return nil, fmt.Errorf("frozen SP-I1 corpus duplicates %s/%s", key.dataset, key.name) + } + fixture, err := fixtureMetadata("unused", testCase.Dataset) + if err != nil { + return nil, fmt.Errorf("derive frozen SP-I1 fixture %s: %w", testCase.Dataset, err) + } + declarations[key] = spI1CanonicalDeclaration{testCase: testCase, fixture: fixture} + } + if len(declarations) != len(cohort.keys) { + return nil, fmt.Errorf("frozen SP-I1 corpus omits canonical declarations") + } + return declarations, nil +} + +func canonicalSPI1Cohort() (spI1CanonicalCohort, error) { + cohort := spI1CanonicalCohort{ + keys: map[performanceKey]struct{}{}, trainingKeys: map[performanceKey]struct{}{}, holdoutKeys: map[performanceKey]struct{}{}, + trainingCorpusSHA256: spI1TrainingCorpusSHA256, fullCorpusSHA256: spI1FullCorpusSHA256, + trainingResolvedSHA256: spI1TrainingResolvedSHA, fullResolvedSHA256: spI1FullResolvedSHA, + } + var full, training, holdout []DeclaredCaseBackend + for _, testCase := range spI1CanonicalCases { + key := performanceKey{dataset: testCase.dataset, name: testCase.name, backend: ModePostgresSQL} + if _, duplicate := cohort.keys[key]; duplicate || !strings.HasPrefix(testCase.dataset, "generated_shortest_paths_v2_") { + return spI1CanonicalCohort{}, fmt.Errorf("frozen SP-I1 cohort contains an invalid declaration") + } + cohort.keys[key] = struct{}{} + for _, backend := range []ExecutionMode{ModePostgresSQL, ModeNeo4j} { + item := DeclaredCaseBackend{Dataset: key.dataset, Name: key.name, Backend: backend} + full = append(full, item) + if testCase.split == "training" { + training = append(training, item) + } else if testCase.split == "holdout" { + holdout = append(holdout, item) + } else { + return spI1CanonicalCohort{}, fmt.Errorf("frozen SP-I1 cohort contains an invalid split") + } + } + if testCase.split == "training" { + cohort.trainingKeys[key] = struct{}{} + } else { + cohort.holdoutKeys[key] = struct{}{} + } + } + if len(cohort.trainingKeys) != 4 || len(cohort.holdoutKeys) != 3 || len(cohort.keys) != 7 { + return spI1CanonicalCohort{}, fmt.Errorf("frozen SP-I1 cohort must contain exactly 4 training and 3 holdout cases") + } + cohort.declarationSHA256 = declarationSHA256(full) + cohort.trainingDeclarationSHA256 = declarationSHA256(training) + cohort.holdoutDeclarationSHA256 = declarationSHA256(holdout) + return cohort, nil +} + +func spI1QualificationCaps() map[string]int64 { + return map[string]int64{ + "state_limit": 100_000, + "predecessor_limit": 100_000, + "enumeration_limit": 100_000, + "output_bytes_limit": 64 * 1024 * 1024, + } +} + +func spI1TelemetryCaps() map[string]int64 { + return map[string]int64{ + "state_rows": 100_000, + "predecessor_rows": 100_000, + "output_rows": 100_000, + "output_bytes": 64 * 1024 * 1024, + } +} + +type SPI1QualificationOptions struct { + Seed int64 + Confidence float64 + BootstrapCount int + Protocol string + // Training evidence paths make confirmation independently recompute the + // discovery decision instead of trusting only a mutable report and freeze. + TrainingBaselinePath string + TrainingCandidatePath string + TrainingResourcePath string + // SourceArchiveSHA256 binds the report to git archive HEAD. Report-mode + // callers populate it from the current committed tree; tests may supply a + // synthetic digest without invoking Git. + SourceArchiveSHA256 string + Freeze *SPI1QualificationFreezeManifest + Discovery *SPI1QualificationReport +} + +type SPI1QualificationCase struct { + Dataset string `json:"dataset"` + Name string `json:"name"` + QualificationSplit string `json:"qualification_split"` + Rounds int `json:"matched_rounds"` + BaselineSamples int `json:"baseline_samples"` + CandidateSamples int `json:"candidate_samples"` + MedianRatio RatioInterval `json:"median_ratio_to_s4"` + MedianSaving DurationInterval `json:"median_saving_vs_s4"` + P95Ratio RatioInterval `json:"p95_ratio_to_s4"` + Material bool `json:"material"` + P95Contained bool `json:"p95_contained"` + ResourcePassed bool `json:"resource_passed"` + RuntimeBranch string `json:"runtime_branch"` + Passed bool `json:"passed"` + Reasons []string `json:"reasons,omitempty"` +} + +type SPI1QualificationReport struct { + Version int `json:"version"` + Protocol string `json:"protocol"` + Baseline string `json:"baseline"` + Candidate string `json:"candidate"` + Policy string `json:"policy"` + QuerySHA256 string `json:"query_sha256"` + Seed int64 `json:"seed"` + Confidence float64 `json:"confidence_level"` + BootstrapCount int `json:"bootstrap_count"` + MaterialityRatio float64 `json:"materiality_ratio_upper_limit"` + MaterialityAbsolute time.Duration `json:"materiality_absolute_lower_limit"` + P95RatioLimit float64 `json:"p95_ratio_upper_limit"` + Caps map[string]int64 `json:"caps"` + SourceCommit string `json:"source_commit"` + SourceArchiveSHA256 string `json:"source_archive_sha256"` + DirtyDiffSHA256 string `json:"dirty_diff_sha256"` + BinarySHA256 string `json:"binary_sha256"` + CorpusSHA256 string `json:"corpus_sha256"` + CohortDeclarationSHA256 string `json:"cohort_declaration_sha256"` + ResolvedSelectionSHA256 string `json:"resolved_selection_sha256"` + TrainingDeclarationSHA256 string `json:"training_declaration_sha256"` + HoldoutDeclarationSHA256 string `json:"holdout_declaration_sha256"` + FullDeclarationSHA256 string `json:"full_declaration_sha256"` + TrainingCorpusSHA256 string `json:"training_corpus_sha256"` + FullCorpusSHA256 string `json:"full_corpus_sha256"` + BaselineArtifactSHA256 string `json:"baseline_artifact_sha256,omitempty"` + CandidateArtifactSHA256 string `json:"candidate_artifact_sha256,omitempty"` + ResourceReportSHA256 string `json:"resource_report_sha256,omitempty"` + FreezeManifestSHA256 string `json:"freeze_manifest_sha256,omitempty"` + EvidencePassed bool `json:"evidence_passed"` + TrainingCases int `json:"training_cases"` + HoldoutCases int `json:"holdout_cases"` + TrainingPassed bool `json:"training_passed"` + HoldoutPassed bool `json:"holdout_passed"` + QualificationPassed bool `json:"qualification_passed"` + Cases []SPI1QualificationCase `json:"cases"` +} + +type SPI1QualificationFreezeManifest struct { + Version int `json:"version"` + Baseline string `json:"baseline"` + Candidate string `json:"candidate"` + Policy string `json:"policy"` + QuerySHA256 string `json:"query_sha256"` + Caps map[string]int64 `json:"caps"` + Seed int64 `json:"seed"` + Confidence float64 `json:"confidence_level"` + BootstrapCount int `json:"bootstrap_count"` + SourceCommit string `json:"source_commit"` + SourceArchiveSHA256 string `json:"source_archive_sha256"` + DirtyDiffSHA256 string `json:"dirty_diff_sha256"` + BinarySHA256 string `json:"binary_sha256"` + TrainingDeclarationSHA256 string `json:"training_declaration_sha256"` + HoldoutDeclarationSHA256 string `json:"holdout_declaration_sha256"` + FullDeclarationSHA256 string `json:"full_declaration_sha256"` + TrainingCorpusSHA256 string `json:"training_corpus_sha256"` + FullCorpusSHA256 string `json:"full_corpus_sha256"` + TrainingResolvedSHA256 string `json:"training_resolved_selection_sha256"` + FullResolvedSHA256 string `json:"full_resolved_selection_sha256"` + BaselineArtifactSHA256 string `json:"baseline_artifact_sha256"` + CandidateArtifactSHA256 string `json:"candidate_artifact_sha256"` + ResourceReportSHA256 string `json:"resource_report_sha256"` + DiscoveryReportSHA256 string `json:"discovery_report_sha256"` + TrainingPassed bool `json:"training_passed"` +} + +type spI1EvidenceIdentity struct { + sourceCommit string + dirtyDiffSHA256 string + binarySHA256 string + corpusSHA256 string + declarationSHA256 string + resolvedSHA256 string +} + +func sourceArchiveSHA256() (string, error) { + archive, err := exec.Command("git", "archive", "--format=tar", "HEAD").Output() + if err != nil { + return "", fmt.Errorf("archive source commit: %w", err) + } + digest := sha256.Sum256(archive) + return hex.EncodeToString(digest[:]), nil +} + +func equalSPI1Caps(left, right map[string]int64) bool { + if len(left) != len(right) { + return false + } + for name, value := range left { + if right[name] != value { + return false + } + } + return true +} + +type spI1ProtocolRequirements struct { + minimumWarmups int + minimumRounds int + maximumRounds int + minimumSamples int + protectedCount int + protectedSHA string + expectedKeys map[performanceKey]struct{} + declarationSHA string + corpusSHA string + resolvedSHA string +} + +type spI1QualificationSeries struct { + baseline roundSamples + candidate roundSamples + runtimeBranch string + resourcePassed bool +} + +func spI1Requirements(protocol string, cohort spI1CanonicalCohort) (spI1ProtocolRequirements, error) { + switch protocol { + case referencePairProtocolDiscovery: + return spI1ProtocolRequirements{ + minimumWarmups: 5, + minimumRounds: 5, + maximumRounds: 20, + minimumSamples: 10, + protectedCount: 2 * len(cohort.holdoutKeys), + protectedSHA: cohort.holdoutDeclarationSHA256, + expectedKeys: cohort.trainingKeys, + declarationSHA: cohort.trainingDeclarationSHA256, + corpusSHA: cohort.trainingCorpusSHA256, + resolvedSHA: cohort.trainingResolvedSHA256, + }, nil + case referencePairProtocolConfirmation: + return spI1ProtocolRequirements{ + minimumWarmups: 20, + minimumRounds: 10, + maximumRounds: 20, + minimumSamples: 50, + expectedKeys: cohort.keys, + declarationSHA: cohort.declarationSHA256, + corpusSHA: cohort.fullCorpusSHA256, + resolvedSHA: cohort.fullResolvedSHA256, + }, nil + default: + return spI1ProtocolRequirements{}, fmt.Errorf("unsupported SP-I1 qualification protocol %q", protocol) + } +} + +func buildSPI1QualificationReport( + baseline, candidate []CaseResult, + resource ResourceGateReport, + options SPI1QualificationOptions, +) (SPI1QualificationReport, error) { + if options.Confidence != defaultConfidenceLevel || math.IsNaN(options.Confidence) || math.IsInf(options.Confidence, 0) { + return SPI1QualificationReport{}, fmt.Errorf("SP-I1 qualification confidence must be the frozen %.4f", defaultConfidenceLevel) + } + if options.Seed != 1 { + return SPI1QualificationReport{}, fmt.Errorf("SP-I1 qualification bootstrap seed must be the frozen value 1") + } + if options.BootstrapCount == 0 { + options.BootstrapCount = defaultBootstrapCount + } + if options.BootstrapCount != defaultBootstrapCount { + return SPI1QualificationReport{}, fmt.Errorf("SP-I1 qualification bootstrap count must be the frozen value %d", defaultBootstrapCount) + } + if options.Protocol == "" { + options.Protocol = referencePairProtocolConfirmation + } + if !lowercaseSHA256(options.SourceArchiveSHA256) { + return SPI1QualificationReport{}, fmt.Errorf("SP-I1 source archive digest is missing or malformed") + } + + cohort, err := canonicalSPI1Cohort() + if err != nil { + return SPI1QualificationReport{}, err + } + requirements, err := spI1Requirements(options.Protocol, cohort) + if err != nil { + return SPI1QualificationReport{}, err + } + identity, err := validateSPI1EvidenceIdentity(baseline, candidate, requirements) + if err != nil { + return SPI1QualificationReport{}, err + } + series, keys, err := collectSPI1QualificationSeries(baseline, candidate, resource, requirements) + if err != nil { + return SPI1QualificationReport{}, err + } + + report := SPI1QualificationReport{ + Version: spI1QualificationVersion, + Protocol: options.Protocol, + Baseline: string(optimize.ShortestPathExecutorS4CanonicalWitness), + Candidate: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + Policy: optimize.ShortestPathPolicyI1CanonicalGuardedV1, + QuerySHA256: spI1QuerySHA256, + Seed: options.Seed, + Confidence: options.Confidence, + BootstrapCount: options.BootstrapCount, + MaterialityRatio: 0.95, + MaterialityAbsolute: 100 * time.Microsecond, + P95RatioLimit: 1.05, + Caps: spI1QualificationCaps(), + SourceCommit: identity.sourceCommit, + SourceArchiveSHA256: options.SourceArchiveSHA256, + DirtyDiffSHA256: identity.dirtyDiffSHA256, + BinarySHA256: identity.binarySHA256, + CorpusSHA256: identity.corpusSHA256, + CohortDeclarationSHA256: identity.declarationSHA256, + ResolvedSelectionSHA256: identity.resolvedSHA256, + TrainingDeclarationSHA256: cohort.trainingDeclarationSHA256, + HoldoutDeclarationSHA256: cohort.holdoutDeclarationSHA256, + FullDeclarationSHA256: cohort.declarationSHA256, + TrainingCorpusSHA256: cohort.trainingCorpusSHA256, + FullCorpusSHA256: cohort.fullCorpusSHA256, + EvidencePassed: true, + TrainingPassed: true, + HoldoutPassed: true, + } + if options.Protocol == referencePairProtocolConfirmation { + if err := validateSPI1Freeze(options.Freeze, options.Discovery, report, cohort); err != nil { + return SPI1QualificationReport{}, err + } + } + + gateOptions := PerfGateOptions{ + Seed: options.Seed, + Confidence: options.Confidence, + BootstrapCount: options.BootstrapCount, + } + for index, key := range keys { + current := series[key] + baselineRounds, candidateRounds := matchedRounds(current.baseline, current.candidate) + if !slices.Equal(sortedRounds(current.baseline), sortedRounds(current.candidate)) || + len(baselineRounds) != len(current.baseline) || len(candidateRounds) != len(current.candidate) { + return SPI1QualificationReport{}, fmt.Errorf("%s/%s SP-I1 arms do not contain identical nonempty round sets", key.dataset, key.name) + } + rounds := sortedRounds(baselineRounds) + if len(rounds) < requirements.minimumRounds || len(rounds) > requirements.maximumRounds { + return SPI1QualificationReport{}, fmt.Errorf( + "%s/%s requires %d-%d matched SP-I1 rounds, got %d", + key.dataset, key.name, requirements.minimumRounds, requirements.maximumRounds, len(rounds), + ) + } + for _, round := range rounds { + if len(baselineRounds[round]) < requirements.minimumSamples || len(candidateRounds[round]) < requirements.minimumSamples { + return SPI1QualificationReport{}, fmt.Errorf( + "%s/%s round %d requires at least %d warm samples per SP-I1 arm, got %d/%d", + key.dataset, key.name, round, requirements.minimumSamples, + len(baselineRounds[round]), len(candidateRounds[round]), + ) + } + } + if err := validatePairedOrderEvidence(baseline, candidate, key, rounds, requirements.minimumWarmups); err != nil { + return SPI1QualificationReport{}, fmt.Errorf("invalid SP-I1 paired evidence: %w", err) + } + + split := "training" + if _, holdout := cohort.holdoutKeys[key]; holdout { + split = "holdout" + } + seed := options.Seed + int64(index)*7919 + gateCase := SPI1QualificationCase{ + Dataset: key.dataset, + Name: key.name, + QualificationSplit: split, + Rounds: len(rounds), + BaselineSamples: sampleCount(baselineRounds), + CandidateSamples: sampleCount(candidateRounds), + MedianRatio: bootstrapRoundMedianRatio(baselineRounds, candidateRounds, seed, gateOptions), + MedianSaving: bootstrapRoundMedianSaving(baselineRounds, candidateRounds, seed+1, gateOptions), + P95Ratio: bootstrapStratifiedP95Ratio(baselineRounds, candidateRounds, seed+2, gateOptions), + ResourcePassed: current.resourcePassed, + RuntimeBranch: current.runtimeBranch, + Passed: true, + } + gateCase.Material = gateCase.MedianRatio.Upper <= report.MaterialityRatio || + gateCase.MedianSaving.Lower >= report.MaterialityAbsolute + gateCase.P95Contained = gateCase.P95Ratio.Upper <= report.P95RatioLimit + if !gateCase.Material { + gateCase.Passed = false + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf( + "median improvement is not material: ratio upper %.4f > %.4f and saving lower %s < %s", + gateCase.MedianRatio.Upper, report.MaterialityRatio, + gateCase.MedianSaving.Lower, report.MaterialityAbsolute, + )) + } + if !gateCase.P95Contained { + gateCase.Passed = false + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf( + "p95 ratio upper %.4f exceeds %.4f", gateCase.P95Ratio.Upper, report.P95RatioLimit, + )) + } + if !gateCase.ResourcePassed { + gateCase.Passed = false + gateCase.Reasons = append(gateCase.Reasons, "candidate resource evidence did not pass") + } + + switch split { + case "training": + report.TrainingCases++ + report.TrainingPassed = report.TrainingPassed && gateCase.Passed + case "holdout": + report.HoldoutCases++ + report.HoldoutPassed = report.HoldoutPassed && gateCase.Passed + } + report.Cases = append(report.Cases, gateCase) + } + report.TrainingPassed = report.TrainingPassed && report.TrainingCases == len(cohort.trainingKeys) + report.HoldoutPassed = report.HoldoutPassed && report.HoldoutCases == len(cohort.holdoutKeys) + if options.Protocol == referencePairProtocolDiscovery { + report.HoldoutPassed = false + } + report.QualificationPassed = report.EvidencePassed && report.TrainingPassed && report.HoldoutPassed + return report, nil +} + +func validateSPI1EvidenceIdentity( + baseline, candidate []CaseResult, + requirements spI1ProtocolRequirements, +) (spI1EvidenceIdentity, error) { + if err := validatePerformanceWorkloadIdentity(baseline, candidate); err != nil { + return spI1EvidenceIdentity{}, err + } + baselineHost, err := artifactHostFingerprint(baseline) + if err != nil { + return spI1EvidenceIdentity{}, fmt.Errorf("SP-I1 baseline host: %w", err) + } + candidateHost, err := artifactHostFingerprint(candidate) + if err != nil { + return spI1EvidenceIdentity{}, fmt.Errorf("SP-I1 candidate host: %w", err) + } + if baselineHost != candidateHost { + return spI1EvidenceIdentity{}, fmt.Errorf("SP-I1 baseline and candidate host identities differ") + } + + identity := spI1EvidenceIdentity{} + for _, artifact := range []struct { + name string + records []CaseResult + }{ + {name: "baseline", records: baseline}, + {name: "candidate", records: candidate}, + } { + selection, err := selectionIdentity(artifact.records) + if err != nil { + return spI1EvidenceIdentity{}, fmt.Errorf("SP-I1 %s selection: %w", artifact.name, err) + } + if err := validateSPI1Selection(selection, requirements); err != nil { + return spI1EvidenceIdentity{}, fmt.Errorf("SP-I1 %s selection: %w", artifact.name, err) + } + currentIdentity := spI1EvidenceIdentity{ + declarationSHA256: selection.DeclarationSHA256, + resolvedSHA256: resolvedSelectionSHA256(selection.Resolved), + } + for _, record := range artifact.records { + if record.Environment == nil || record.PostgresEnvironment == nil { + return spI1EvidenceIdentity{}, fmt.Errorf("%s/%s %s arm lacks source or PostgreSQL environment identity", record.Dataset, record.Name, artifact.name) + } + current := spI1EvidenceIdentity{ + sourceCommit: strings.TrimSpace(record.Environment.SourceCommit), + dirtyDiffSHA256: record.Environment.DirtyDiffSHA256, + binarySHA256: record.Environment.BinarySHA256, + corpusSHA256: record.Environment.CorpusSHA256, + declarationSHA256: selection.DeclarationSHA256, + resolvedSHA256: currentIdentity.resolvedSHA256, + } + if current.sourceCommit == "" || current.sourceCommit == "unknown" || + !lowercaseSHA256(current.dirtyDiffSHA256) || !lowercaseSHA256(current.binarySHA256) || + !lowercaseSHA256(current.corpusSHA256) { + return spI1EvidenceIdentity{}, fmt.Errorf("%s/%s %s arm lacks frozen source, diff, binary, or corpus identity", record.Dataset, record.Name, artifact.name) + } + if current.corpusSHA256 != requirements.corpusSHA { + return spI1EvidenceIdentity{}, fmt.Errorf("%s/%s %s arm corpus digest is not the exact frozen SP-I1 cohort", record.Dataset, record.Name, artifact.name) + } + if identity.sourceCommit == "" { + identity = current + } else if identity != current { + return spI1EvidenceIdentity{}, fmt.Errorf("SP-I1 artifacts mix source, diff, binary, corpus, declaration, or selection identities") + } + } + } + if identity.declarationSHA256 != requirements.declarationSHA || identity.resolvedSHA256 != requirements.resolvedSHA { + return spI1EvidenceIdentity{}, fmt.Errorf("SP-I1 artifacts do not bind the exact frozen declaration and resolved selection") + } + for key := range requirements.expectedKeys { + baselinePostgres, err := postgresTimingEnvironmentSHA256ForKey(baseline, key) + if err != nil { + return spI1EvidenceIdentity{}, err + } + candidatePostgres, err := postgresTimingEnvironmentSHA256ForKey(candidate, key) + if err != nil { + return spI1EvidenceIdentity{}, err + } + baselineFixture, err := fixtureSHA256ForKey(baseline, key) + if err != nil { + return spI1EvidenceIdentity{}, err + } + candidateFixture, err := fixtureSHA256ForKey(candidate, key) + if err != nil { + return spI1EvidenceIdentity{}, err + } + if !lowercaseSHA256(baselinePostgres) || baselinePostgres != candidatePostgres { + return spI1EvidenceIdentity{}, fmt.Errorf("%s/%s SP-I1 PostgreSQL timing environments differ between arms", key.dataset, key.name) + } + if !lowercaseSHA256(baselineFixture) || baselineFixture != candidateFixture { + return spI1EvidenceIdentity{}, fmt.Errorf("%s/%s SP-I1 fixture identities differ between arms", key.dataset, key.name) + } + baselineSQL, err := spI1SQLFingerprintForKey(baseline, key) + if err != nil { + return spI1EvidenceIdentity{}, err + } + candidateSQL, err := spI1SQLFingerprintForKey(candidate, key) + if err != nil { + return spI1EvidenceIdentity{}, err + } + if baselineSQL == candidateSQL { + return spI1EvidenceIdentity{}, fmt.Errorf("%s/%s SP-I1 arms use the same SQL fingerprint", key.dataset, key.name) + } + if err := validateOrientationExactObservations(key, baseline, candidate); err != nil { + return spI1EvidenceIdentity{}, fmt.Errorf("SP-I1 exact observations: %w", err) + } + } + return identity, nil +} + +func spI1SQLFingerprintForKey(records []CaseResult, key performanceKey) (string, error) { + fingerprint := "" + for _, record := range records { + if record.Dataset != key.dataset || record.Name != key.name || record.ExecutionMode != key.backend { + continue + } + if fingerprint != "" && fingerprint != record.SQLFingerprint { + return "", fmt.Errorf("%s/%s changes SQL fingerprint within one SP-I1 arm", key.dataset, key.name) + } + fingerprint = record.SQLFingerprint + } + if !lowercaseSHA256(fingerprint) { + return "", fmt.Errorf("%s/%s lacks one stable SP-I1 SQL fingerprint", key.dataset, key.name) + } + return fingerprint, nil +} + +func validateSPI1Selection(selection SelectionManifest, requirements spI1ProtocolRequirements) error { + if selection.Version != selectionManifestVersion || !selection.DiagnosticOnly || + selection.SelectedDeclarationCount != 2*len(requirements.expectedKeys) || + selection.FullDeclarationCount != selection.SelectedDeclarationCount+selection.OmittedDeclarationCount || + selection.ProtectedDeclarationCount != requirements.protectedCount || + selection.ProtectedDeclarationSHA256 != requirements.protectedSHA || + len(selection.Resolved) != len(requirements.expectedKeys) || + selection.DeclarationSHA256 != requirements.declarationSHA || + resolvedSelectionSHA256(selection.Resolved) != requirements.resolvedSHA { + return fmt.Errorf("selection manifest does not bind the exact frozen cohort") + } + resolved := make(map[performanceKey]struct{}, len(selection.Resolved)) + for _, item := range selection.Resolved { + if item.Category != "generated_shortest_path_v2" { + return fmt.Errorf("selection contains non-SP-I1 category %q", item.Category) + } + key := performanceKey{dataset: item.Dataset, name: item.Name, backend: ModePostgresSQL} + if _, duplicate := resolved[key]; duplicate { + return fmt.Errorf("selection contains duplicate %s/%s", item.Dataset, item.Name) + } + resolved[key] = struct{}{} + } + if !orientationV2KeySetsEqual(resolved, requirements.expectedKeys) { + return fmt.Errorf("selection does not contain the exact frozen SP-I1 cases") + } + return nil +} + +func collectSPI1QualificationSeries( + baseline, candidate []CaseResult, + resource ResourceGateReport, + requirements spI1ProtocolRequirements, +) (map[performanceKey]*spI1QualificationSeries, []performanceKey, error) { + if err := validateSPI1GlobalInvocationIDs(baseline, candidate); err != nil { + return nil, nil, err + } + declarations, err := canonicalSPI1Declarations() + if err != nil { + return nil, nil, err + } + baselineKeys, baselineRounds, err := collectSPI1Artifact("baseline", baseline, requirements, declarations) + if err != nil { + return nil, nil, err + } + candidateKeys, candidateRounds, err := collectSPI1Artifact("candidate", candidate, requirements, declarations) + if err != nil { + return nil, nil, err + } + if !orientationV2KeySetsEqual(baselineKeys, requirements.expectedKeys) || + !orientationV2KeySetsEqual(candidateKeys, requirements.expectedKeys) { + return nil, nil, fmt.Errorf("SP-I1 artifacts do not contain the exact protocol cohort") + } + if err := validateSPI1RunSchedule(baseline, candidate, requirements); err != nil { + return nil, nil, err + } + resourcePassed, err := validateSPI1ResourceCases(resource, candidate, requirements) + if err != nil { + return nil, nil, err + } + + series := make(map[performanceKey]*spI1QualificationSeries, len(requirements.expectedKeys)) + for key := range requirements.expectedKeys { + current := &spI1QualificationSeries{ + baseline: roundSamples{}, + candidate: roundSamples{}, + resourcePassed: resourcePassed[key], + } + series[key] = current + for round, record := range baselineRounds[key] { + appendSPI1WarmSamples(current.baseline, round, record) + } + for round, record := range candidateRounds[key] { + appendSPI1WarmSamples(current.candidate, round, record) + branch := record.TraversalTelemetry.Summary.RuntimeBranch + if current.runtimeBranch != "" && current.runtimeBranch != branch { + return nil, nil, fmt.Errorf("%s/%s changes SP-I1 runtime branch across rounds", key.dataset, key.name) + } + current.runtimeBranch = branch + } + if current.runtimeBranch == "" { + return nil, nil, fmt.Errorf("%s/%s has no attributable SP-I1 candidate runtime", key.dataset, key.name) + } + } + return series, sortedPerformanceKeys(requirements.expectedKeys), nil +} + +// validateSPI1GlobalInvocationIDs prevents one genuine timed receipt from +// being copied into another case, round, or arm. The attestor emits globally +// unique invocation IDs, so the complete paired study must not reuse one. +func validateSPI1GlobalInvocationIDs(artifacts ...[]CaseResult) error { + seen := map[string]struct{}{} + for _, records := range artifacts { + for _, record := range records { + for _, sample := range record.Stats.Samples { + if sample.Classification != "warm" { + continue + } + invocationID := strings.TrimSpace(sample.RuntimeInvocationID) + if invocationID == "" { + return fmt.Errorf("%s/%s warm sample lacks a global timed invocation identity", record.Dataset, record.Name) + } + if _, duplicate := seen[invocationID]; duplicate { + return fmt.Errorf("SP-I1 evidence reuses timed invocation identity %q across the paired study", invocationID) + } + seen[invocationID] = struct{}{} + } + } + } + return nil +} + +type spI1InvocationIdentity struct { + round, block, order int + arm, runUUID string + startedAt, endedAt time.Time +} + +func validateSPI1RunSchedule(baseline, candidate []CaseResult, requirements spI1ProtocolRequirements) error { + collect := func(arm string, records []CaseResult) (map[int]spI1InvocationIdentity, error) { + invocations := map[int]spI1InvocationIdentity{} + caseCounts := map[int]int{} + for _, record := range records { + if record.Environment == nil { + return nil, fmt.Errorf("%s/%s %s arm lacks invocation chronology", record.Dataset, record.Name, arm) + } + environment := record.Environment + identity := spI1InvocationIdentity{ + round: environment.Round, block: environment.Block, order: environment.ArmOrder, + arm: environment.Arm, runUUID: environment.RunUUID, + startedAt: environment.StartedAt, endedAt: environment.EndedAt, + } + if identity.startedAt.IsZero() || identity.endedAt.IsZero() || identity.endedAt.Before(identity.startedAt) { + return nil, fmt.Errorf("SP-I1 %s round %d has malformed invocation timestamps", arm, identity.round) + } + if prior, found := invocations[identity.round]; found && prior != identity { + return nil, fmt.Errorf("SP-I1 %s round %d mixes invocation identities", arm, identity.round) + } + invocations[identity.round] = identity + caseCounts[identity.round]++ + } + for round, count := range caseCounts { + if count != len(requirements.expectedKeys) { + return nil, fmt.Errorf("SP-I1 %s round %d contains %d cases, expected %d", arm, round, count, len(requirements.expectedKeys)) + } + } + return invocations, nil + } + left, err := collect("baseline", baseline) + if err != nil { + return err + } + right, err := collect("candidate", candidate) + if err != nil { + return err + } + if len(left) != len(right) || len(left) < requirements.minimumRounds || len(left) > requirements.maximumRounds { + return fmt.Errorf("SP-I1 artifacts do not contain one complete paired invocation schedule") + } + runUUID := "" + var priorEnded time.Time + for round := 1; round <= len(left); round++ { + baselineInvocation, baselineFound := left[round] + candidateInvocation, candidateFound := right[round] + if !baselineFound || !candidateFound { + return fmt.Errorf("SP-I1 invocation schedule must use contiguous rounds starting at 1") + } + expectedBaselineOrder, expectedCandidateOrder := 1, 2 + if round%2 == 0 { + expectedBaselineOrder, expectedCandidateOrder = 2, 1 + } + if baselineInvocation.block != round || candidateInvocation.block != round || + baselineInvocation.arm != "sp-i1-s4" || candidateInvocation.arm != "sp-i1-candidate" || + baselineInvocation.order != expectedBaselineOrder || candidateInvocation.order != expectedCandidateOrder || + baselineInvocation.runUUID == "" || baselineInvocation.runUUID != candidateInvocation.runUUID { + return fmt.Errorf("SP-I1 round %d does not match the frozen alternating two-arm schedule", round) + } + if runUUID == "" { + runUUID = baselineInvocation.runUUID + } else if runUUID != baselineInvocation.runUUID { + return fmt.Errorf("SP-I1 artifacts mix run UUIDs across rounds") + } + first, second := baselineInvocation, candidateInvocation + if candidateInvocation.order == 1 { + first, second = candidateInvocation, baselineInvocation + } + if first.endedAt.After(second.startedAt) { + return fmt.Errorf("SP-I1 round %d arm timestamps contradict the declared execution order", round) + } + if !priorEnded.IsZero() && priorEnded.After(first.startedAt) { + return fmt.Errorf("SP-I1 round %d overlaps or predates the prior round", round) + } + priorEnded = second.endedAt + } + return nil +} + +func collectSPI1Artifact( + arm string, + records []CaseResult, + requirements spI1ProtocolRequirements, + declarations map[performanceKey]spI1CanonicalDeclaration, +) (map[performanceKey]struct{}, map[performanceKey]map[int]CaseResult, error) { + if len(records) == 0 { + return nil, nil, fmt.Errorf("SP-I1 %s artifact is empty", arm) + } + keys := map[performanceKey]struct{}{} + rounds := map[performanceKey]map[int]CaseResult{} + for _, record := range records { + key := performanceKey{dataset: record.Dataset, name: record.Name, backend: record.ExecutionMode} + if _, expected := requirements.expectedKeys[key]; !expected { + return nil, nil, fmt.Errorf("SP-I1 %s artifact contains unexpected case %s/%s", arm, key.dataset, key.name) + } + declaration, found := declarations[key] + if !found { + return nil, nil, fmt.Errorf("SP-I1 %s artifact has no frozen declaration for %s/%s", arm, key.dataset, key.name) + } + if err := validateSPI1Record(record, arm, declaration); err != nil { + return nil, nil, err + } + round, err := orientationV2RecordRound(record) + if err != nil { + return nil, nil, err + } + if rounds[key] == nil { + rounds[key] = map[int]CaseResult{} + } + if _, duplicate := rounds[key][round]; duplicate { + return nil, nil, fmt.Errorf("%s/%s %s artifact duplicates round %d", key.dataset, key.name, arm, round) + } + rounds[key][round] = record + keys[key] = struct{}{} + } + return keys, rounds, nil +} + +func appendSPI1WarmSamples(series roundSamples, round int, record CaseResult) { + for _, sample := range record.Stats.Samples { + if sample.Classification == "warm" && sample.Duration > 0 { + series[round] = append(series[round], sample.Duration) + } + } +} + +func validateSPI1Record(record CaseResult, arm string, declaration spI1CanonicalDeclaration) error { + if record.ExecutionMode != ModePostgresSQL || record.Status != StatusOK || + record.Environment == nil || record.PostgresEnvironment == nil || record.Fixture == nil || + record.TraversalTelemetry == nil || record.Optimization == nil || record.PostgresMetrics == nil { + return fmt.Errorf("%s/%s %s arm lacks a successful telemetry-bearing PostgreSQL record", record.Dataset, record.Name, arm) + } + if record.Environment.ArtifactSchemaVersion != 2 || record.Environment.PoolSize != 1 || + len(record.Environment.Concurrency) != 0 || record.Environment.ExistingGraph || + record.Environment.Protocol != "fixed_confirmation" { + return fmt.Errorf("%s/%s %s arm lacks the schema-v2 single-session fixed-confirmation contract", record.Dataset, record.Name, arm) + } + if record.Fixture.Dataset != record.Dataset || !lowercaseSHA256(record.Fixture.Checksum) || + !record.Fixture.PhysicalValidated || record.Fixture.PhysicalNodeCount != int64(record.Fixture.NodeCount) || + record.Fixture.PhysicalEdgeCount != int64(record.Fixture.EdgeCount) || + record.Fixture.Checksum != declaration.fixture.Checksum || + record.Fixture.NodeCount != declaration.fixture.NodeCount || record.Fixture.EdgeCount != declaration.fixture.EdgeCount || + record.Fixture.Configuration != declaration.fixture.Configuration || + !reflect.DeepEqual(record.Fixture.Shortest, declaration.fixture.Shortest) || + record.Fixture.NodeRelationBytes <= 0 || record.Fixture.EdgeRelationBytes <= 0 { + return fmt.Errorf("%s/%s %s arm lacks one exact physically validated fixture", record.Dataset, record.Name, arm) + } + if !strings.EqualFold(strings.TrimSpace(record.PostgresEnvironment.TransactionIsolation), "repeatable read") { + return fmt.Errorf("%s/%s %s arm was not measured under Repeatable Read", record.Dataset, record.Name, arm) + } + testCase := declaration.testCase + testCase.Source = record.Source + expectedRecord := newCaseResult(testCase, ModePostgresSQL, nil) + attachFixtureMetadata(&expectedRecord, *record.Fixture) + if filepath.Base(record.Source) != "generated_sp_i1_inbound_v1.json" || + record.Category != testCase.Category || record.Cypher != testCase.Cypher || sqlFingerprint(record.Cypher) != spI1QuerySHA256 || + !lowercaseSHA256(record.WorkloadSHA256) || !lowercaseSHA256(record.SQLFingerprint) || + record.WorkloadSHA256 != expectedRecord.WorkloadSHA256 || + record.SQL == "" || sqlFingerprint(record.SQL) != record.SQLFingerprint || + !reflect.DeepEqual(record.NodeParams, testCase.NodeParams) || + !reflect.DeepEqual(record.NodeListParams, testCase.NodeListParams) || + !reflect.DeepEqual(record.Shape, testCase.Shape) { + return fmt.Errorf("%s/%s %s arm lacks the frozen inbound SP-I1 workload identity", record.Dataset, record.Name, arm) + } + minimumDepth, maximumDepth := 0, 0 + if record.Shape.MinDepth != nil { + minimumDepth = *record.Shape.MinDepth + } + if record.Shape.MaxDepth != nil { + maximumDepth = *record.Shape.MaxDepth + } + if record.Shape.QualificationSplit != "training" && record.Shape.QualificationSplit != "holdout" || + record.Shape.FallbackExpectation != "forbidden" || record.Shape.Direction != "inbound" || + record.Shape.RelationshipKindCount != 1 || !slices.Equal(record.Shape.EdgeKinds, []string{"Traverse"}) || + minimumDepth != 1 || maximumDepth != 64 || !record.Shape.PathMaterializationRequired { + return fmt.Errorf("%s/%s %s arm changes the frozen inbound one-path shape", record.Dataset, record.Name, arm) + } + expectedSplit := testCase.Shape.QualificationSplit + if record.Shape.QualificationSplit != expectedSplit { + return fmt.Errorf("%s/%s %s arm changes the frozen qualification split", record.Dataset, record.Name, arm) + } + expectedRows := *testCase.Expected.RowCount + if !record.StableObservation || record.RowCount != expectedRows || record.ExpectedRowCount == nil || + *record.ExpectedRowCount != expectedRows { + return fmt.Errorf("%s/%s %s arm lacks the exact stable path observation contract", record.Dataset, record.Name, arm) + } + if err := validateExpectedObservations(testCase.Expected, record.ObservedRows); err != nil { + return fmt.Errorf("%s/%s %s arm changes the frozen path observation: %w", record.Dataset, record.Name, arm, err) + } + if len(record.Concurrency) != 0 || len(record.PostgresReferences) != 0 || record.ClientWaterfall != nil || + record.RawPGXWaterfall != nil || record.RawPGXRoundTrip != nil || record.Baseline != nil { + return fmt.Errorf("%s/%s %s arm mixes SP-I1 timing with supplemental measurements", record.Dataset, record.Name, arm) + } + if err := ValidateTraversalExecutionTelemetry(record.TraversalTelemetry); err != nil { + return fmt.Errorf("%s/%s %s arm telemetry: %w", record.Dataset, record.Name, arm, err) + } + if err := validateSPI1Runtime(record, arm); err != nil { + return err + } + return nil +} + +func validateSPI1Runtime(record CaseResult, arm string) error { + summary := record.TraversalTelemetry.Summary + if summary.RuntimeOutcomeAvailable == nil || !*summary.RuntimeOutcomeAvailable || + summary.Overflow == nil || summary.FallbackExecuted == nil || *summary.Overflow || *summary.FallbackExecuted || + summary.WouldSelectIdentity != "" || summary.ObservationMode != string(optimize.ShortestPathObservationOnePath) || + summary.SchedulerVersion != string(optimize.ShortestPathSchedulerSingleEndedLevel) { + return fmt.Errorf("%s/%s %s arm lacks one non-fallback one-path runtime outcome", record.Dataset, record.Name, arm) + } + outcome, ok := singleTraversalOutcome(record.Optimization.TargetOutcomes) + if !ok || outcome.Family != "SP" { + return fmt.Errorf("%s/%s %s arm lacks one exact SP lowering outcome", record.Dataset, record.Name, arm) + } + baseline := string(optimize.ShortestPathExecutorS4CanonicalWitness) + candidate := string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness) + plannedIdentities := spI1ShortestPathPlannedIdentities() + outcomeDepthsExact := outcome.MinimumDepth != nil && *outcome.MinimumDepth == 1 && + outcome.MaximumDepth != nil && *outcome.MaximumDepth == 64 + outcomeShapeExact := outcome.Lowering == optimize.LoweringShortestPathExecutor && outcome.TargetKind == "traversal" && + outcome.ObservationMode == string(optimize.ShortestPathObservationOnePath) && outcome.Direction == "inbound" && + outcome.PhysicalExpansion == "end_id" && outcome.RelationshipKindCount == 1 && !outcome.UntypedRelationship && + outcome.TopologyClassification == "physical_inbound_deep" && outcome.SelectionMode == "forced_tool" && + outcome.Scheduler == string(optimize.ShortestPathSchedulerSingleEndedLevel) && outcomeDepthsExact && + outcome.Eligible != nil && *outcome.Eligible && outcome.StaticallyEligible != nil && *outcome.StaticallyEligible + if !outcomeShapeExact { + return fmt.Errorf("%s/%s %s arm changes the frozen SP-I1 lowering shape", record.Dataset, record.Name, arm) + } + switch arm { + case "baseline": + if summary.RequestedIdentity != baseline || summary.EmittedIdentity != baseline || + summary.RuntimeIdentity != baseline || summary.AppliedIdentity != baseline || + !slices.Equal(summary.PlannedIdentities, plannedIdentities) || + summary.SelectorVersion != "sp-tool-v1" || + summary.ExecutionBoundary != optimize.ShortestPathExecutorS4CanonicalWitness.ExecutionBoundary() || + summary.RuntimeBranch != "selected" || + outcome.Candidate != "" || outcome.Selected != baseline || outcome.Applied != baseline || outcome.Fallback != "SP-S0" || + !slices.Equal(outcome.PlannedCandidates, plannedIdentities) || + outcome.ExecutionBoundary != "stored_helper" || outcome.SelectorVersion != "sp-tool-v1" || + outcome.EmittedPolicy != "" || len(outcome.EmittedCandidates) != 0 || + outcome.StateLimit != 100_000 || outcome.FrontierLimit != 100_000 || outcome.PredecessorLimit != 100_000 || + outcome.EnumerationLimit != 100_000 || outcome.OutputBytesLimit != 64*1024*1024 { + return fmt.Errorf("%s/%s baseline arm did not execute exact forced S4", record.Dataset, record.Name) + } + case "candidate": + expectedBranch := "inline_canonical_witness" + if record.RowCount == 0 { + expectedBranch = "inline_canonical_no_path" + } + if summary.RequestedIdentity != candidate || summary.EmittedIdentity != optimize.ShortestPathPolicyI1CanonicalGuardedV1 || + summary.RuntimeIdentity != candidate || summary.AppliedIdentity != candidate || + !slices.Equal(summary.PlannedIdentities, plannedIdentities) || + summary.SelectorVersion != "sp-i1-canonical-tool-v1" || + summary.ExecutionBoundary != optimize.ExpansionSearchExecutionBoundaryGuardedDualArm || + summary.RuntimeBranch != expectedBranch || + !equalSPI1Caps(summary.Caps, spI1TelemetryCaps()) || + !slices.Contains(summary.PlannedIdentities, baseline) || !slices.Contains(summary.PlannedIdentities, candidate) || + outcome.Candidate != candidate || outcome.Selected != candidate || outcome.Applied != candidate || + outcome.Fallback != baseline || outcome.EmittedPolicy != optimize.ShortestPathPolicyI1CanonicalGuardedV1 || + !slices.Equal(outcome.PlannedCandidates, plannedIdentities) || + !slices.Equal(outcome.EmittedCandidates, []string{candidate, baseline}) || + outcome.ExecutionBoundary != optimize.ExpansionSearchExecutionBoundaryGuardedDualArm || + outcome.SelectorVersion != "sp-i1-canonical-tool-v1" || + outcome.StateLimit != spI1QualificationCaps()["state_limit"] || + outcome.PredecessorLimit != spI1QualificationCaps()["predecessor_limit"] || + outcome.EnumerationLimit != spI1QualificationCaps()["enumeration_limit"] || + outcome.OutputBytesLimit != spI1QualificationCaps()["output_bytes_limit"] || outcome.FrontierLimit != 0 { + return fmt.Errorf("%s/%s candidate arm did not execute exact guarded canonical I1", record.Dataset, record.Name) + } + diagnostic := record.TraversalTelemetry.Diagnostic + if record.TraversalTelemetry.Level != TraversalTelemetryLevelDiagnostic || diagnostic == nil || + diagnostic.CounterStatus != TraversalTelemetryCounterStatusComplete || diagnostic.Counters.InlineShortestPath == nil || + !slices.Contains(diagnostic.RequiredFamilies, TraversalTelemetryFamilySP) || + !slices.Contains(diagnostic.RequiredFamilies, TraversalTelemetryFamilyHydration) { + return fmt.Errorf("%s/%s candidate arm lacks complete typed canonical-I1 resource telemetry", record.Dataset, record.Name) + } + inline := diagnostic.Counters.InlineShortestPath + outputRows, outputPresent := int64(0), false + if diagnostic.PlanReplay != nil { + outputRows, outputPresent = diagnostic.PlanReplay.Counters["asp_i1_output_rows"] + } + if inline.OutputPaths == nil || *inline.OutputPaths != record.RowCount || !outputPresent || outputRows != record.RowCount { + return fmt.Errorf("%s/%s candidate arm runtime branch does not bind the exact output observation", record.Dataset, record.Name) + } + default: + return fmt.Errorf("unknown SP-I1 arm %q", arm) + } + if err := validateSPI1SampleRuntime(record, arm); err != nil { + return err + } + return nil +} + +// spI1ShortestPathPlannedIdentities mirrors the optimizer's complete SP search +// space. Planned candidates describe every executor considered by lowering; +// emitted candidates and the runtime receipt separately attest the exact +// guarded two-arm statement that executed. +func spI1ShortestPathPlannedIdentities() []string { + return []string{ + string(optimize.ShortestPathExecutorIncumbentWorkspace), + string(optimize.ShortestPathExecutorS0Direct), + string(optimize.ShortestPathExecutorS1ArrayBFS), + string(optimize.ShortestPathExecutorS2TraceRelation), + string(optimize.ShortestPathExecutorS3Unidirectional), + string(optimize.ShortestPathExecutorS3EdgeM0), + string(optimize.ShortestPathExecutorS4CanonicalDistance), + string(optimize.ShortestPathExecutorS4CanonicalWitness), + string(optimize.ShortestPathExecutorI1CanonicalDistance), + string(optimize.ShortestPathExecutorI1CanonicalWitness), + string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + string(optimize.ShortestPathExecutorB1AlternatingNodeDistance), + string(optimize.ShortestPathExecutorB1AlternatingNodeWitness), + string(optimize.ShortestPathExecutorB2SmallerCurrentLevelDistance), + string(optimize.ShortestPathExecutorB2SmallerCurrentLevelWitness), + } +} + +func validateSPI1SampleRuntime(record CaseResult, arm string) error { + summary := record.TraversalTelemetry.Summary + if record.Environment == nil || record.Stats.Iterations < 1 || record.Stats.WarmupIterations != record.Environment.WarmupIterations || + record.Stats.Median <= 0 || record.Stats.P95 <= 0 { + return fmt.Errorf("%s/%s %s arm has malformed iteration or warmup evidence", record.Dataset, record.Name, arm) + } + expectedArm := "sp-i1-s4" + if arm == "candidate" { + expectedArm = "sp-i1-candidate" + } + if record.Environment.Arm != expectedArm || record.Environment.Round < 1 || record.Environment.Block != record.Environment.Round || + record.Environment.ArmOrder < 1 || record.Environment.ArmOrder > 2 || strings.TrimSpace(record.Environment.RunUUID) == "" { + return fmt.Errorf("%s/%s %s arm has malformed frozen run metadata", record.Dataset, record.Name, arm) + } + warmSamples, coldSamples := 0, 0 + iterations := map[int]struct{}{} + invocations := map[string]struct{}{} + for _, sample := range record.Stats.Samples { + if sample.Duration <= 0 || sample.Dataset != record.Dataset || sample.Case != record.Name || sample.Backend != ModePostgresSQL || + sample.Round != record.Environment.Round || sample.Block != record.Environment.Block || sample.Arm != record.Environment.Arm || + sample.ArmOrder != record.Environment.ArmOrder || sample.RunUUID != record.Environment.RunUUID || strings.TrimSpace(sample.ConnectionID) == "" { + return fmt.Errorf("%s/%s %s arm has a sample outside its frozen invocation identity", record.Dataset, record.Name, arm) + } + switch sample.Classification { + case "cold": + if sample.Iteration != 0 { + return fmt.Errorf("%s/%s %s arm cold sample has a nonzero iteration", record.Dataset, record.Name, arm) + } + coldSamples++ + continue + case "warm": + default: + return fmt.Errorf("%s/%s %s arm contains an unexpected sample classification", record.Dataset, record.Name, arm) + } + warmSamples++ + if sample.Iteration < 1 || sample.Iteration > record.Stats.Iterations { + return fmt.Errorf("%s/%s %s arm has an out-of-range warm iteration", record.Dataset, record.Name, arm) + } + if _, duplicate := iterations[sample.Iteration]; duplicate { + return fmt.Errorf("%s/%s %s arm duplicates warm iteration %d", record.Dataset, record.Name, arm, sample.Iteration) + } + iterations[sample.Iteration] = struct{}{} + if sample.RequestedIdentity != summary.RequestedIdentity || sample.RuntimeIdentity != summary.RuntimeIdentity || + sample.FallbackExecuted == nil || *sample.FallbackExecuted != *summary.FallbackExecuted { + return fmt.Errorf("%s/%s %s arm warm sample contradicts its runtime summary", record.Dataset, record.Name, arm) + } + if sample.RuntimeAttestation != "timed_invocation" { + return fmt.Errorf("%s/%s %s arm warm sample lacks timed-invocation attribution", record.Dataset, record.Name, arm) + } + if strings.TrimSpace(sample.RuntimeInvocationID) == "" { + return fmt.Errorf("%s/%s %s arm warm sample lacks a timed invocation identity", record.Dataset, record.Name, arm) + } + if _, duplicate := invocations[sample.RuntimeInvocationID]; duplicate { + return fmt.Errorf("%s/%s %s arm reuses timed invocation identity %q", record.Dataset, record.Name, arm, sample.RuntimeInvocationID) + } + invocations[sample.RuntimeInvocationID] = struct{}{} + expectedBranch := summary.RuntimeBranch + if arm == "baseline" { + expectedBranch = "compact_workspace_witness" + if record.RowCount == 0 { + expectedBranch = "compact_no_path" + } + } + if sample.RuntimeBranch != expectedBranch || len(sample.RuntimeReceiptEvents) != 1 || + sample.RuntimeReceiptEvents[0].InvocationID != sample.RuntimeInvocationID || sample.RuntimeReceiptEvents[0].FallbackExecuted { + return fmt.Errorf("%s/%s %s arm warm sample has a non-canonical runtime receipt", record.Dataset, record.Name, arm) + } + if err := validateRuntimeReceiptEvents(sample.RuntimeReceiptEvents, sample.RuntimeIdentity, sample.RuntimeBranch, sample.FallbackExecuted); err != nil { + return fmt.Errorf("%s/%s %s arm warm sample receipt: %w", record.Dataset, record.Name, arm, err) + } + } + if coldSamples != 1 || warmSamples != record.Stats.Iterations || len(record.Stats.Samples) != record.Stats.Iterations+1 { + return fmt.Errorf("%s/%s %s arm must contain one cold and exactly %d unique warm samples", record.Dataset, record.Name, arm, record.Stats.Iterations) + } + return nil +} + +func validateSPI1ResourceCases( + report ResourceGateReport, + candidate []CaseResult, + requirements spI1ProtocolRequirements, +) (map[performanceKey]bool, error) { + if report.Version != resourceGateVersion { + return nil, fmt.Errorf("SP-I1 resource report version must be %d", resourceGateVersion) + } + type recordKey struct { + performanceKey + round, block, order int + runUUID, arm string + } + expected := map[recordKey]CaseResult{} + for _, record := range candidate { + if record.Environment == nil { + return nil, fmt.Errorf("%s/%s candidate resource record lacks run identity", record.Dataset, record.Name) + } + key := recordKey{ + performanceKey: performanceKey{dataset: record.Dataset, name: record.Name, backend: ModePostgresSQL}, + round: record.Environment.Round, block: record.Environment.Block, order: record.Environment.ArmOrder, + runUUID: record.Environment.RunUUID, arm: record.Environment.Arm, + } + if _, duplicate := expected[key]; duplicate { + return nil, fmt.Errorf("SP-I1 candidate artifact duplicates a resource record identity") + } + expected[key] = record + } + actual := map[recordKey]struct{}{} + passed := map[performanceKey]bool{} + for key := range requirements.expectedKeys { + passed[key] = true + } + cohort, err := canonicalSPI1Cohort() + if err != nil { + return nil, err + } + allPassed := true + for _, gateCase := range report.Cases { + key := performanceKey{dataset: gateCase.Dataset, name: gateCase.Name, backend: ModePostgresSQL} + if _, expected := requirements.expectedKeys[key]; !expected || gateCase.Reference != "" { + return nil, fmt.Errorf("SP-I1 resource report contains an unexpected production or reference case %s/%s", gateCase.Dataset, gateCase.Name) + } + identity := recordKey{ + performanceKey: key, round: gateCase.Round, block: gateCase.Block, order: gateCase.ArmOrder, + runUUID: gateCase.RunUUID, arm: gateCase.Arm, + } + record, found := expected[identity] + if !found { + return nil, fmt.Errorf("SP-I1 resource case %s/%s round %d does not bind an exact candidate record", gateCase.Dataset, gateCase.Name, gateCase.Round) + } + if _, duplicate := actual[identity]; duplicate { + return nil, fmt.Errorf("SP-I1 resource report duplicates %s/%s round %d", gateCase.Dataset, gateCase.Name, gateCase.Round) + } + actual[identity] = struct{}{} + recomputed := evaluateProductionResourceGateCase(record) + if !reflect.DeepEqual(gateCase, recomputed) { + return nil, fmt.Errorf("SP-I1 resource case %s/%s round %d differs from the decision recomputed from its candidate record", gateCase.Dataset, gateCase.Name, gateCase.Round) + } + expectedSplit := "training" + if _, holdout := cohort.holdoutKeys[key]; holdout { + expectedSplit = "holdout" + } + if gateCase.Architecture != string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness) || + gateCase.FallbackArchitecture != "" || gateCase.QualificationSplit != expectedSplit || + gateCase.Tier != "normal" || !equalSPI1Caps(gateCase.NumericLimits, spI1TelemetryCaps()) || + gateCase.Passed != (len(gateCase.Reasons) == 0) || + !reflect.DeepEqual(gateCase.RuntimeReceiptChains, runtimeReceiptChains(record.Stats.Samples)) { + return nil, fmt.Errorf("SP-I1 resource case %s/%s does not bind exact guarded-I1 limits and split", gateCase.Dataset, gateCase.Name) + } + observations := traversalNumericObservations(record.TraversalTelemetry.Diagnostic.Counters) + if len(gateCase.NumericObserved) != len(spI1TelemetryCaps()) { + return nil, fmt.Errorf("SP-I1 resource case %s/%s has unexpected numeric observations", gateCase.Dataset, gateCase.Name) + } + for name := range spI1TelemetryCaps() { + observed, found := gateCase.NumericObserved[name] + expectedObserved, expectedFound := observations[name] + if !found || !expectedFound || observed != expectedObserved || observed < 0 { + return nil, fmt.Errorf("SP-I1 resource case %s/%s has invalid %s observation", gateCase.Dataset, gateCase.Name, name) + } + } + passed[key] = passed[key] && gateCase.Passed + allPassed = allPassed && gateCase.Passed + } + if len(actual) != len(expected) { + return nil, fmt.Errorf("SP-I1 resource report has %d exact record decisions, expected %d", len(actual), len(expected)) + } + for key := range requirements.expectedKeys { + if _, found := passed[key]; !found { + return nil, fmt.Errorf("SP-I1 resource report omits %s/%s", key.dataset, key.name) + } + } + if report.Passed != allPassed { + return nil, fmt.Errorf("SP-I1 resource report aggregate disposition contradicts its cases") + } + return passed, nil +} + +func validateSPI1Freeze( + freeze *SPI1QualificationFreezeManifest, + discovery *SPI1QualificationReport, + report SPI1QualificationReport, + cohort spI1CanonicalCohort, +) error { + if err := validateSPI1FrozenDiscovery(freeze, discovery, cohort); err != nil { + return err + } + if report.Protocol != referencePairProtocolConfirmation || + report.SourceCommit != freeze.SourceCommit || report.SourceArchiveSHA256 != freeze.SourceArchiveSHA256 || + report.DirtyDiffSHA256 != freeze.DirtyDiffSHA256 || report.BinarySHA256 != freeze.BinarySHA256 || + report.QuerySHA256 != freeze.QuerySHA256 || report.Policy != freeze.Policy || + report.Baseline != freeze.Baseline || report.Candidate != freeze.Candidate || + report.CohortDeclarationSHA256 != freeze.FullDeclarationSHA256 || + report.CorpusSHA256 != freeze.FullCorpusSHA256 || report.ResolvedSelectionSHA256 != freeze.FullResolvedSHA256 || + report.Seed != freeze.Seed || report.Confidence != freeze.Confidence || report.BootstrapCount != freeze.BootstrapCount || + !equalSPI1Caps(report.Caps, freeze.Caps) { + return fmt.Errorf("SP-I1 confirmation identity differs from the frozen discovery") + } + return nil +} + +func validateSPI1FrozenDiscovery( + freeze *SPI1QualificationFreezeManifest, + discovery *SPI1QualificationReport, + cohort spI1CanonicalCohort, +) error { + if freeze == nil || discovery == nil { + return fmt.Errorf("SP-I1 confirmation requires a discovery report and freeze manifest") + } + baseline := string(optimize.ShortestPathExecutorS4CanonicalWitness) + candidate := string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness) + if freeze.Version != spI1FreezeVersion || freeze.Baseline != baseline || freeze.Candidate != candidate || + freeze.Policy != optimize.ShortestPathPolicyI1CanonicalGuardedV1 || freeze.QuerySHA256 != spI1QuerySHA256 || + freeze.Seed != 1 || freeze.Confidence != defaultConfidenceLevel || freeze.BootstrapCount != defaultBootstrapCount || + !equalSPI1Caps(freeze.Caps, spI1QualificationCaps()) || + freeze.TrainingDeclarationSHA256 != cohort.trainingDeclarationSHA256 || + freeze.HoldoutDeclarationSHA256 != cohort.holdoutDeclarationSHA256 || + freeze.FullDeclarationSHA256 != cohort.declarationSHA256 || + freeze.TrainingCorpusSHA256 != cohort.trainingCorpusSHA256 || freeze.FullCorpusSHA256 != cohort.fullCorpusSHA256 || + freeze.TrainingResolvedSHA256 != cohort.trainingResolvedSHA256 || freeze.FullResolvedSHA256 != cohort.fullResolvedSHA256 || + !lowercaseSHA256(freeze.SourceArchiveSHA256) || !lowercaseSHA256(freeze.DirtyDiffSHA256) || + !lowercaseSHA256(freeze.BinarySHA256) || !lowercaseSHA256(freeze.BaselineArtifactSHA256) || + !lowercaseSHA256(freeze.CandidateArtifactSHA256) || !lowercaseSHA256(freeze.ResourceReportSHA256) || + !lowercaseSHA256(freeze.DiscoveryReportSHA256) || strings.TrimSpace(freeze.SourceCommit) == "" { + return fmt.Errorf("SP-I1 freeze manifest does not bind the exact immutable study identity") + } + if freeze.DirtyDiffSHA256 != cleanWorkingTreeSHA256() { + return fmt.Errorf("SP-I1 freeze manifest was not created from a clean source tree") + } + if discovery.Version != spI1QualificationVersion || discovery.Protocol != referencePairProtocolDiscovery || + discovery.Baseline != freeze.Baseline || discovery.Candidate != freeze.Candidate || + discovery.Policy != freeze.Policy || discovery.QuerySHA256 != freeze.QuerySHA256 || + discovery.SourceCommit != freeze.SourceCommit || discovery.SourceArchiveSHA256 != freeze.SourceArchiveSHA256 || + discovery.DirtyDiffSHA256 != freeze.DirtyDiffSHA256 || discovery.BinarySHA256 != freeze.BinarySHA256 || + discovery.CohortDeclarationSHA256 != cohort.trainingDeclarationSHA256 || + discovery.ResolvedSelectionSHA256 != cohort.trainingResolvedSHA256 || + discovery.CorpusSHA256 != cohort.trainingCorpusSHA256 || + discovery.TrainingDeclarationSHA256 != cohort.trainingDeclarationSHA256 || + discovery.HoldoutDeclarationSHA256 != cohort.holdoutDeclarationSHA256 || + discovery.FullDeclarationSHA256 != cohort.declarationSHA256 || + discovery.TrainingCorpusSHA256 != cohort.trainingCorpusSHA256 || discovery.FullCorpusSHA256 != cohort.fullCorpusSHA256 || + discovery.BaselineArtifactSHA256 != freeze.BaselineArtifactSHA256 || + discovery.CandidateArtifactSHA256 != freeze.CandidateArtifactSHA256 || + discovery.ResourceReportSHA256 != freeze.ResourceReportSHA256 || + !equalSPI1Caps(discovery.Caps, freeze.Caps) || discovery.Seed != freeze.Seed || + discovery.Confidence != freeze.Confidence || discovery.BootstrapCount != freeze.BootstrapCount || + discovery.MaterialityRatio != 0.95 || discovery.MaterialityAbsolute != 100*time.Microsecond || + discovery.P95RatioLimit != 1.05 || !discovery.EvidencePassed || + discovery.TrainingCases != len(cohort.trainingKeys) || discovery.HoldoutCases != 0 || + discovery.HoldoutPassed || discovery.QualificationPassed || discovery.TrainingPassed != freeze.TrainingPassed { + return fmt.Errorf("SP-I1 discovery report does not prove the exact frozen training identity") + } + seen := map[performanceKey]struct{}{} + for _, entry := range discovery.Cases { + key := performanceKey{dataset: entry.Dataset, name: entry.Name, backend: ModePostgresSQL} + if entry.QualificationSplit != "training" { + return fmt.Errorf("SP-I1 discovery report contains non-training timing") + } + if _, expected := cohort.trainingKeys[key]; !expected { + return fmt.Errorf("SP-I1 discovery report contains unexpected case %s/%s", entry.Dataset, entry.Name) + } + if _, duplicate := seen[key]; duplicate { + return fmt.Errorf("SP-I1 discovery report duplicates case %s/%s", entry.Dataset, entry.Name) + } + expectedBranch := "inline_canonical_witness" + if strings.HasSuffix(entry.Name, "-disconnected") { + expectedBranch = "inline_canonical_no_path" + } + if !validSPI1RatioInterval(entry.MedianRatio) || !validSPI1RatioInterval(entry.P95Ratio) || + entry.MedianSaving.Lower > entry.MedianSaving.Estimate || entry.MedianSaving.Estimate > entry.MedianSaving.Upper || + entry.Material != (entry.MedianRatio.Upper <= discovery.MaterialityRatio || entry.MedianSaving.Lower >= discovery.MaterialityAbsolute) || + entry.P95Contained != (entry.P95Ratio.Upper <= discovery.P95RatioLimit) || + !entry.Passed || len(entry.Reasons) != 0 || !entry.Material || !entry.P95Contained || !entry.ResourcePassed || + entry.RuntimeBranch != expectedBranch || + entry.Rounds < 5 || entry.Rounds > 20 || entry.BaselineSamples < 50 || entry.CandidateSamples < 50 { + return fmt.Errorf("SP-I1 discovery report case %s/%s did not pass the frozen training gates", entry.Dataset, entry.Name) + } + seen[key] = struct{}{} + } + if !orientationV2KeySetsEqual(seen, cohort.trainingKeys) { + return fmt.Errorf("SP-I1 discovery report omits part of the exact training cohort") + } + if !freeze.TrainingPassed || !discovery.TrainingPassed { + return fmt.Errorf("SP-I1 training discovery did not pass") + } + return nil +} + +func validSPI1RatioInterval(interval RatioInterval) bool { + return interval.Lower > 0 && interval.Lower <= interval.Estimate && interval.Estimate <= interval.Upper && + !math.IsNaN(interval.Lower) && !math.IsNaN(interval.Estimate) && !math.IsNaN(interval.Upper) && + !math.IsInf(interval.Lower, 0) && !math.IsInf(interval.Estimate, 0) && !math.IsInf(interval.Upper, 0) +} + +// createSPI1QualificationReport loads and evaluates the staged two-arm +// qualification evidence, writes the report even for statistical failures, +// and freezes discovery before any holdout capture is authorized. +func createSPI1QualificationReport( + baselinePath, candidatePath, resourcePath, freezePath, discoveryPath, freezeOutputPath, outputPath string, + options SPI1QualificationOptions, +) (bool, error) { + if err := validateDistinctSPI1Paths(map[string]string{ + "baseline artifact": baselinePath, "candidate artifact": candidatePath, "resource report": resourcePath, + "freeze manifest": freezePath, "discovery report": discoveryPath, "freeze output": freezeOutputPath, "report output": outputPath, + }); err != nil { + return false, err + } + baseline, err := readJSONLFile(baselinePath) + if err != nil { + return false, fmt.Errorf("read SP-I1 baseline artifact: %w", err) + } + candidate, err := readJSONLFile(candidatePath) + if err != nil { + return false, fmt.Errorf("read SP-I1 candidate artifact: %w", err) + } + resource, err := loadSPI1ResourceReport(resourcePath) + if err != nil { + return false, err + } + baselineSHA256, err := fileSHA256(baselinePath) + if err != nil { + return false, err + } + candidateSHA256, err := fileSHA256(candidatePath) + if err != nil { + return false, err + } + resourceSHA256, err := fileSHA256(resourcePath) + if err != nil { + return false, err + } + if resource.ArtifactSHA256 != candidateSHA256 { + return false, fmt.Errorf("SP-I1 resource report is not bound to the exact candidate artifact") + } + + freezeSHA256 := "" + if freezePath != "" || discoveryPath != "" { + if freezePath == "" || discoveryPath == "" { + return false, fmt.Errorf("SP-I1 confirmation requires both freeze and discovery report paths") + } + freeze, digest, err := loadSPI1FreezeManifest(freezePath) + if err != nil { + return false, fmt.Errorf("read SP-I1 freeze manifest: %w", err) + } + discovery, err := loadSPI1QualificationReport(discoveryPath) + if err != nil { + return false, fmt.Errorf("read SP-I1 discovery report: %w", err) + } + discoverySHA256, err := fileSHA256(discoveryPath) + if err != nil { + return false, err + } + if discoverySHA256 != freeze.DiscoveryReportSHA256 { + return false, fmt.Errorf("SP-I1 discovery report digest does not match freeze manifest") + } + options.Freeze, options.Discovery = freeze, discovery + freezeSHA256 = digest + if err := validateSPI1FrozenTrainingEvidence( + freeze, discovery, + options.TrainingBaselinePath, options.TrainingCandidatePath, options.TrainingResourcePath, + ); err != nil { + return false, err + } + } + options.SourceArchiveSHA256, err = sourceArchiveSHA256() + if err != nil { + return false, err + } + report, err := buildSPI1QualificationReport(baseline, candidate, resource, options) + if err != nil { + return false, err + } + report.BaselineArtifactSHA256 = baselineSHA256 + report.CandidateArtifactSHA256 = candidateSHA256 + report.ResourceReportSHA256 = resourceSHA256 + report.FreezeManifestSHA256 = freezeSHA256 + if err := validateCurrentSPI1Source(report.SourceCommit, report.SourceArchiveSHA256, report.DirtyDiffSHA256, report.BinarySHA256); err != nil { + return false, err + } + if err := writeSPI1QualificationReport(outputPath, report); err != nil { + return false, err + } + if options.Protocol == referencePairProtocolDiscovery { + if err := writeSPI1FreezeManifest(freezeOutputPath, outputPath, report); err != nil { + return false, err + } + return report.TrainingPassed, nil + } + return report.QualificationPassed, nil +} + +// validateSPI1HoldoutCapture authorizes the exact frozen cohort before any +// database setup is allowed to begin. +func validateSPI1HoldoutCapture( + corpus ScaleCorpus, + freezePath, discoveryPath, trainingBaselinePath, trainingCandidatePath, trainingResourcePath string, +) error { + cohort, err := canonicalSPI1Cohort() + if err != nil { + return err + } + if err := validateSPI1Corpus(corpus, cohort); err != nil { + return err + } + freeze, _, err := loadSPI1FreezeManifest(freezePath) + if err != nil { + return fmt.Errorf("read SP-I1 freeze manifest: %w", err) + } + discovery, err := loadSPI1QualificationReport(discoveryPath) + if err != nil { + return fmt.Errorf("read SP-I1 discovery report: %w", err) + } + discoverySHA256, err := fileSHA256(discoveryPath) + if err != nil { + return err + } + if discoverySHA256 != freeze.DiscoveryReportSHA256 { + return fmt.Errorf("SP-I1 discovery report digest does not match freeze manifest") + } + if err := validateSPI1FrozenTrainingEvidence( + freeze, discovery, trainingBaselinePath, trainingCandidatePath, trainingResourcePath, + ); err != nil { + return err + } + if err := validateCurrentSPI1Source(freeze.SourceCommit, freeze.SourceArchiveSHA256, freeze.DirtyDiffSHA256, freeze.BinarySHA256); err != nil { + return err + } + return nil +} + +// validateSPI1FrozenTrainingEvidence reloads and recomputes the exact training +// closure named by the freeze. This prevents an internally consistent but +// hand-edited report/freeze pair from authorizing protected holdout timing. +func validateSPI1FrozenTrainingEvidence( + freeze *SPI1QualificationFreezeManifest, + discovery *SPI1QualificationReport, + baselinePath, candidatePath, resourcePath string, +) error { + cohort, err := canonicalSPI1Cohort() + if err != nil { + return err + } + if err := validateSPI1FrozenDiscovery(freeze, discovery, cohort); err != nil { + return err + } + if baselinePath == "" || candidatePath == "" || resourcePath == "" { + return fmt.Errorf("SP-I1 frozen discovery verification requires the three exact training evidence artifacts") + } + baselineSHA256, err := fileSHA256(baselinePath) + if err != nil { + return fmt.Errorf("hash frozen SP-I1 training baseline: %w", err) + } + candidateSHA256, err := fileSHA256(candidatePath) + if err != nil { + return fmt.Errorf("hash frozen SP-I1 training candidate: %w", err) + } + resourceSHA256, err := fileSHA256(resourcePath) + if err != nil { + return fmt.Errorf("hash frozen SP-I1 training resource report: %w", err) + } + if baselineSHA256 != freeze.BaselineArtifactSHA256 || candidateSHA256 != freeze.CandidateArtifactSHA256 || + resourceSHA256 != freeze.ResourceReportSHA256 { + return fmt.Errorf("SP-I1 frozen training evidence digests differ from the discovery freeze") + } + baseline, err := readJSONLFile(baselinePath) + if err != nil { + return fmt.Errorf("read frozen SP-I1 training baseline: %w", err) + } + candidate, err := readJSONLFile(candidatePath) + if err != nil { + return fmt.Errorf("read frozen SP-I1 training candidate: %w", err) + } + resource, err := loadSPI1ResourceReport(resourcePath) + if err != nil { + return err + } + if resource.ArtifactSHA256 != candidateSHA256 { + return fmt.Errorf("SP-I1 frozen training resource report is not bound to the candidate artifact") + } + recomputed, err := buildSPI1QualificationReport(baseline, candidate, resource, SPI1QualificationOptions{ + Seed: freeze.Seed, Confidence: freeze.Confidence, BootstrapCount: freeze.BootstrapCount, + Protocol: referencePairProtocolDiscovery, SourceArchiveSHA256: freeze.SourceArchiveSHA256, + }) + if err != nil { + return fmt.Errorf("recompute frozen SP-I1 training discovery: %w", err) + } + recomputed.BaselineArtifactSHA256 = baselineSHA256 + recomputed.CandidateArtifactSHA256 = candidateSHA256 + recomputed.ResourceReportSHA256 = resourceSHA256 + if !reflect.DeepEqual(recomputed, *discovery) { + return fmt.Errorf("SP-I1 discovery report differs from its recomputed frozen training evidence") + } + return nil +} + +func validateSPI1Corpus(corpus ScaleCorpus, cohort spI1CanonicalCohort) error { + if len(corpus.Cases) != len(cohort.keys) { + return fmt.Errorf("SP-I1 holdout capture requires exactly the frozen four-training/three-holdout cohort") + } + seen := map[performanceKey]struct{}{} + resolved := make([]ResolvedCaseSelector, 0, len(corpus.Cases)) + for _, testCase := range corpus.Cases { + key := performanceKey{dataset: testCase.Dataset, name: testCase.Name, backend: ModePostgresSQL} + if _, expected := cohort.keys[key]; !expected { + return fmt.Errorf("SP-I1 holdout capture contains unexpected case %s/%s", testCase.Dataset, testCase.Name) + } + if _, duplicate := seen[key]; duplicate { + return fmt.Errorf("SP-I1 holdout capture duplicates case %s/%s", testCase.Dataset, testCase.Name) + } + seen[key] = struct{}{} + if filepath.Base(testCase.Source) != "generated_sp_i1_inbound_v1.json" || + testCase.Category != "generated_shortest_path_v2" || sqlFingerprint(testCase.Cypher) != spI1QuerySHA256 || + testCase.Shape.FallbackExpectation != "forbidden" || testCase.Shape.Direction != "inbound" || + testCase.Shape.RelationshipKindCount != 1 || !slices.Equal(testCase.Shape.EdgeKinds, []string{"Traverse"}) || + testCase.Shape.MinDepth == nil || *testCase.Shape.MinDepth != 1 || + testCase.Shape.MaxDepth == nil || *testCase.Shape.MaxDepth != 64 || + !testCase.Shape.PathMaterializationRequired || + !slices.Equal(testCase.CandidateModes, []ExecutionMode{ModePostgresSQL, ModeNeo4j}) { + return fmt.Errorf("SP-I1 holdout capture changes frozen declaration %s/%s", testCase.Dataset, testCase.Name) + } + expectedSplit := "training" + if _, holdout := cohort.holdoutKeys[key]; holdout { + expectedSplit = "holdout" + } + if testCase.Shape.QualificationSplit != expectedSplit { + return fmt.Errorf("SP-I1 holdout capture changes frozen split for %s/%s", testCase.Dataset, testCase.Name) + } + resolved = append(resolved, ResolvedCaseSelector{Dataset: testCase.Dataset, Name: testCase.Name, Category: testCase.Category}) + } + if !orientationV2KeySetsEqual(seen, cohort.keys) || + declarationSHA256(corpus.DeclaredBackends()) != cohort.declarationSHA256 || + resolvedSelectionSHA256(resolved) != cohort.fullResolvedSHA256 || + corpusIdentity(corpus) != cohort.fullCorpusSHA256 { + return fmt.Errorf("SP-I1 holdout capture does not match the exact frozen declaration, selection, and corpus digests") + } + return nil +} + +func validateCurrentSPI1Source(sourceCommit, sourceArchive, dirtyDiff, binary string) error { + currentCommit := strings.TrimSpace(commandOutput("git", "rev-parse", "HEAD")) + currentArchive, err := sourceArchiveSHA256() + if err != nil { + return err + } + currentDiff := workingTreeSHA256() + currentBinary := executableSHA256() + if currentCommit == "" || currentCommit == "unknown" || sourceCommit != currentCommit || + !lowercaseSHA256(sourceArchive) || sourceArchive != currentArchive || + dirtyDiff != cleanWorkingTreeSHA256() || currentDiff != cleanWorkingTreeSHA256() || + !lowercaseSHA256(binary) || binary != currentBinary { + return fmt.Errorf("SP-I1 evidence requires the current clean committed source archive and exact running binary") + } + return nil +} + +func loadSPI1ResourceReport(path string) (ResourceGateReport, error) { + raw, err := os.ReadFile(path) + if err != nil { + return ResourceGateReport{}, fmt.Errorf("read SP-I1 resource report: %w", err) + } + report := ResourceGateReport{} + if err := json.Unmarshal(raw, &report); err != nil { + return ResourceGateReport{}, fmt.Errorf("decode SP-I1 resource report: %w", err) + } + if report.Version != resourceGateVersion || !lowercaseSHA256(report.ArtifactSHA256) { + return ResourceGateReport{}, fmt.Errorf("SP-I1 resource report must be checksummed schema v%d", resourceGateVersion) + } + return report, nil +} + +func loadSPI1QualificationReport(path string) (*SPI1QualificationReport, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + report := &SPI1QualificationReport{} + if err := json.Unmarshal(raw, report); err != nil { + return nil, fmt.Errorf("decode SP-I1 qualification report: %w", err) + } + return report, nil +} + +func loadSPI1FreezeManifest(path string) (*SPI1QualificationFreezeManifest, string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, "", err + } + manifest := &SPI1QualificationFreezeManifest{} + if err := json.Unmarshal(raw, manifest); err != nil { + return nil, "", fmt.Errorf("decode SP-I1 freeze manifest: %w", err) + } + digest := sha256.Sum256(raw) + return manifest, hex.EncodeToString(digest[:]), nil +} + +func writeSPI1QualificationReport(path string, report SPI1QualificationReport) (err error) { + if path == "" { + return fmt.Errorf("SP-I1 qualification requires an explicit report output path") + } + if err := ensureOutputDir(path); err != nil { + return err + } + output, err := os.Create(path) + if err != nil { + return err + } + defer func() { + if closeErr := output.Close(); err == nil && closeErr != nil { + err = closeErr + } + }() + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(report) +} + +func writeSPI1FreezeManifest(path, discoveryReportPath string, report SPI1QualificationReport) (err error) { + if path == "" || discoveryReportPath == "" { + return fmt.Errorf("SP-I1 discovery freeze requires report and manifest output paths") + } + cohort, err := canonicalSPI1Cohort() + if err != nil { + return err + } + if report.Protocol != referencePairProtocolDiscovery || report.CohortDeclarationSHA256 != cohort.trainingDeclarationSHA256 || + report.ResolvedSelectionSHA256 != cohort.trainingResolvedSHA256 || report.CorpusSHA256 != cohort.trainingCorpusSHA256 || + report.TrainingCases != len(cohort.trainingKeys) || report.HoldoutCases != 0 || + report.Seed != 1 || report.Confidence != defaultConfidenceLevel || report.BootstrapCount != defaultBootstrapCount || + report.DirtyDiffSHA256 != cleanWorkingTreeSHA256() || !equalSPI1Caps(report.Caps, spI1QualificationCaps()) || + !lowercaseSHA256(report.BaselineArtifactSHA256) || !lowercaseSHA256(report.CandidateArtifactSHA256) || + !lowercaseSHA256(report.ResourceReportSHA256) { + return fmt.Errorf("SP-I1 discovery freeze requires the exact clean training-only report") + } + discoveryReportSHA256, err := fileSHA256(discoveryReportPath) + if err != nil { + return err + } + manifest := SPI1QualificationFreezeManifest{ + Version: spI1FreezeVersion, + Baseline: report.Baseline, + Candidate: report.Candidate, + Policy: report.Policy, + QuerySHA256: report.QuerySHA256, + Caps: report.Caps, + Seed: report.Seed, + Confidence: report.Confidence, + BootstrapCount: report.BootstrapCount, + SourceCommit: report.SourceCommit, + SourceArchiveSHA256: report.SourceArchiveSHA256, + DirtyDiffSHA256: report.DirtyDiffSHA256, + BinarySHA256: report.BinarySHA256, + TrainingDeclarationSHA256: cohort.trainingDeclarationSHA256, + HoldoutDeclarationSHA256: cohort.holdoutDeclarationSHA256, + FullDeclarationSHA256: cohort.declarationSHA256, + TrainingCorpusSHA256: cohort.trainingCorpusSHA256, + FullCorpusSHA256: cohort.fullCorpusSHA256, + TrainingResolvedSHA256: cohort.trainingResolvedSHA256, + FullResolvedSHA256: cohort.fullResolvedSHA256, + BaselineArtifactSHA256: report.BaselineArtifactSHA256, + CandidateArtifactSHA256: report.CandidateArtifactSHA256, + ResourceReportSHA256: report.ResourceReportSHA256, + DiscoveryReportSHA256: discoveryReportSHA256, + TrainingPassed: report.TrainingPassed, + } + if err := ensureOutputDir(path); err != nil { + return err + } + output, err := os.Create(path) + if err != nil { + return err + } + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + encodeErr := encoder.Encode(manifest) + closeErr := output.Close() + if encodeErr != nil { + return encodeErr + } + return closeErr +} + +func validateDistinctSPI1Paths(paths map[string]string) error { + names := make([]string, 0, len(paths)) + for name, path := range paths { + if path != "" { + names = append(names, name) + } + } + sort.Strings(names) + type resolvedPath struct { + name string + info os.FileInfo + } + resolved := map[string]resolvedPath{} + var existing []resolvedPath + for _, name := range names { + absolute, err := filepath.Abs(filepath.Clean(paths[name])) + if err != nil { + return fmt.Errorf("resolve SP-I1 %s: %w", name, err) + } + if evaluated, err := filepath.EvalSymlinks(absolute); err == nil { + absolute = evaluated + } else if evaluatedParent, parentErr := filepath.EvalSymlinks(filepath.Dir(absolute)); parentErr == nil { + absolute = filepath.Join(evaluatedParent, filepath.Base(absolute)) + } + if prior, duplicate := resolved[absolute]; duplicate { + return fmt.Errorf("SP-I1 %s and %s must use distinct paths", prior.name, name) + } + current := resolvedPath{name: name} + if info, err := os.Stat(paths[name]); err == nil { + current.info = info + for _, prior := range existing { + if prior.info != nil && os.SameFile(prior.info, info) { + return fmt.Errorf("SP-I1 %s and %s must not alias the same file", prior.name, name) + } + } + existing = append(existing, current) + } else if !os.IsNotExist(err) { + return fmt.Errorf("inspect SP-I1 %s path: %w", name, err) + } + resolved[absolute] = current + } + return nil +} + +func selectedCorpusContainsSPI1Holdout(corpus ScaleCorpus) bool { + cohort, err := canonicalSPI1Cohort() + if err != nil { + return true + } + for _, testCase := range corpus.Cases { + key := performanceKey{dataset: testCase.Dataset, name: testCase.Name, backend: ModePostgresSQL} + if _, holdout := cohort.holdoutKeys[key]; holdout { + return true + } + } + return false +} + +// selectRunnableScaleCorpus keeps the protected SP-I1 holdout out of ordinary +// GraphBench selection. The holdout becomes selectable only through its exact +// protocol tag or an exact case name; database capture then passes through the +// freeze checks in main before any target is opened. +func selectRunnableScaleCorpus(corpus ScaleCorpus, selectors CorpusSelectors) (ScaleCorpus, SelectionManifest, error) { + if err := validateCorpusSelectors(corpus, selectors); err != nil { + return ScaleCorpus{}, SelectionManifest{}, err + } + includeProtected := slices.Contains(selectors.Tags, spI1HoldoutTag) + if !includeProtected && len(selectors.Cases) > 0 { + protectedNames := make(map[string]struct{}, len(spI1CanonicalCases)) + for _, testCase := range spI1CanonicalCases { + if testCase.split == "holdout" { + protectedNames[testCase.name] = struct{}{} + } + } + for _, name := range selectors.Cases { + if _, protected := protectedNames[name]; protected { + includeProtected = true + break + } + } + } + if includeProtected { + return selectScaleCorpusValidated(corpus, selectors) + } + + cohort, err := canonicalSPI1Cohort() + if err != nil { + return ScaleCorpus{}, SelectionManifest{}, err + } + filtered := ScaleCorpus{Cases: make([]ScaleCase, 0, len(corpus.Cases))} + protected := ScaleCorpus{Cases: make([]ScaleCase, 0, len(cohort.holdoutKeys))} + for _, testCase := range corpus.Cases { + key := performanceKey{dataset: testCase.Dataset, name: testCase.Name, backend: ModePostgresSQL} + if _, isProtected := cohort.holdoutKeys[key]; isProtected { + protected.Cases = append(protected.Cases, testCase) + continue + } + filtered.Cases = append(filtered.Cases, testCase) + } + selected, manifest, err := selectScaleCorpusValidated(filtered, selectors) + if err != nil { + return ScaleCorpus{}, SelectionManifest{}, err + } + manifest.FullDeclarationCount = len(corpus.DeclaredBackends()) + manifest.OmittedDeclarationCount = manifest.FullDeclarationCount - manifest.SelectedDeclarationCount + manifest.ProtectedDeclarationCount = len(protected.DeclaredBackends()) + manifest.ProtectedDeclarationSHA256 = declarationSHA256(protected.DeclaredBackends()) + return selected, manifest, nil +} + +func validateSPI1HoldoutCaptureConfig(cfg config) error { + if len(cfg.Modes) != 1 || cfg.Modes[0] != ModePostgresSQL || cfg.ExistingGraph || cfg.Discovery { + return fmt.Errorf("SP-I1 holdout capture requires one managed PostgreSQL fixed-confirmation mode") + } + if cfg.Iterations < 50 || cfg.WarmupIterations < 20 || cfg.PoolSize != 1 || len(cfg.Concurrency) != 0 { + return fmt.Errorf("SP-I1 holdout capture requires at least 50 samples, 20 warmups, pool size 1, and no concurrency block") + } + if cfg.Round < 1 || cfg.Round > 20 || cfg.Block != cfg.Round || cfg.ArmOrder < 1 || cfg.ArmOrder > 2 || + strings.TrimSpace(cfg.RunUUID) == "" { + return fmt.Errorf("SP-I1 holdout capture requires rounds 1-20, block equal to round, a two-arm order, and an explicit shared run UUID") + } + baseline := string(optimize.ShortestPathExecutorS4CanonicalWitness) + candidate := string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness) + expectedArm, expectedOrder := "", 0 + switch cfg.PostgresForceShortest { + case baseline: + expectedArm = "sp-i1-s4" + expectedOrder = 1 + if cfg.Round%2 == 0 { + expectedOrder = 2 + } + case candidate: + expectedArm = "sp-i1-candidate" + expectedOrder = 2 + if cfg.Round%2 == 0 { + expectedOrder = 1 + } + default: + return fmt.Errorf("SP-I1 holdout capture must force exact S4 or guarded canonical I1") + } + if cfg.Arm != expectedArm || cfg.ArmOrder != expectedOrder { + return fmt.Errorf("SP-I1 holdout capture round %d requires arm %q at order %d", cfg.Round, expectedArm, expectedOrder) + } + if !cfg.PostgresRepeatableRead || cfg.PostgresTraversalTelemetry != postgresTraversalTelemetryDiagnostic || + cfg.PostgresProductionManifest != "" || cfg.PostgresForceExpansion != "" || + cfg.PostgresExpansionOrientationShadow || cfg.PostgresExpansionOrientationTournament || + cfg.PostgresReferences || len(cfg.PostgresReferenceArms) != 0 || cfg.Baseline != "" || + cfg.BundleDir != "" || len(cfg.BundleEvidence) != 0 { + return fmt.Errorf("SP-I1 holdout capture requires forced Repeatable Read with diagnostic telemetry and no supplemental PostgreSQL arms") + } + if cfg.OutputJSONL == "" || cfg.Round > 1 && !cfg.AppendJSONL { + return fmt.Errorf("SP-I1 holdout capture requires a JSONL output and append mode after round 1") + } + return validateDistinctSPI1Paths(map[string]string{ + "freeze manifest": cfg.SPI1Freeze, "discovery report": cfg.SPI1DiscoveryReport, + "training baseline artifact": cfg.SPI1TrainingBaseline, + "training candidate artifact": cfg.SPI1TrainingCandidate, + "training resource report": cfg.SPI1TrainingResource, + "capture JSONL": cfg.OutputJSONL, "capture summary": cfg.Summary, "capture JSON summary": cfg.SummaryJSON, + }) +} diff --git a/cmd/graphbench/sp_i1_qualification_test.go b/cmd/graphbench/sp_i1_qualification_test.go new file mode 100644 index 00000000..4ddeb00c --- /dev/null +++ b/cmd/graphbench/sp_i1_qualification_test.go @@ -0,0 +1,661 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/stretchr/testify/require" +) + +func TestSPI1QualificationDiscoveryPassesTrainingWithoutOpeningHoldout(t *testing.T) { + baseline, candidate, resource := spI1QualificationTestArtifacts(t, referencePairProtocolDiscovery) + report, err := buildSPI1QualificationReport(baseline, candidate, resource, SPI1QualificationOptions{ + Seed: 1, Confidence: defaultConfidenceLevel, BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolDiscovery, SourceArchiveSHA256: strings.Repeat("a", 64), + }) + require.NoError(t, err) + require.True(t, report.EvidencePassed) + require.True(t, report.TrainingPassed) + require.False(t, report.HoldoutPassed) + require.False(t, report.QualificationPassed) + require.Equal(t, 4, report.TrainingCases) + require.Zero(t, report.HoldoutCases) + require.Len(t, report.Cases, 4) + require.Equal(t, spI1QualificationCaps(), report.Caps) + for _, gateCase := range report.Cases { + require.True(t, gateCase.Passed, gateCase.Reasons) + require.Equal(t, "training", gateCase.QualificationSplit) + require.LessOrEqual(t, gateCase.MedianRatio.Upper, 0.95) + require.LessOrEqual(t, gateCase.P95Ratio.Upper, 1.05) + } +} + +func TestTimedRuntimeAttestationIdentityIncludesExactS4Baseline(t *testing.T) { + baseline := string(optimize.ShortestPathExecutorS4CanonicalWitness) + translation := translate.Result{Optimization: translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: "SP", Selected: baseline, + }}}} + require.Equal(t, baseline, timedRuntimeAttestationIdentity(translation)) +} + +func TestSPI1QualificationConfirmationRequiresAndPassesFrozenDiscovery(t *testing.T) { + trainingBaseline, trainingCandidate, trainingResource := spI1QualificationTestArtifacts(t, referencePairProtocolDiscovery) + discovery, err := buildSPI1QualificationReport(trainingBaseline, trainingCandidate, trainingResource, SPI1QualificationOptions{ + Seed: 1, Confidence: defaultConfidenceLevel, BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolDiscovery, SourceArchiveSHA256: strings.Repeat("a", 64), + }) + require.NoError(t, err) + discovery.BaselineArtifactSHA256 = strings.Repeat("1", 64) + discovery.CandidateArtifactSHA256 = strings.Repeat("2", 64) + discovery.ResourceReportSHA256 = strings.Repeat("3", 64) + freeze := spI1QualificationTestFreeze(t, discovery) + + baseline, candidate, resource := spI1QualificationTestArtifacts(t, referencePairProtocolConfirmation) + report, err := buildSPI1QualificationReport(baseline, candidate, resource, SPI1QualificationOptions{ + Seed: 1, Confidence: defaultConfidenceLevel, BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolConfirmation, SourceArchiveSHA256: strings.Repeat("a", 64), + Freeze: &freeze, Discovery: &discovery, + }) + require.NoError(t, err) + require.True(t, report.TrainingPassed) + require.True(t, report.HoldoutPassed) + require.True(t, report.QualificationPassed) + require.Equal(t, 4, report.TrainingCases) + require.Equal(t, 3, report.HoldoutCases) + require.Len(t, report.Cases, 7) +} + +func TestSPI1QualificationRejectsUnattestedCandidateAndFreezeMutation(t *testing.T) { + baseline, candidate, resource := spI1QualificationTestArtifacts(t, referencePairProtocolDiscovery) + candidate[0].Stats.Samples[1].RuntimeAttestation = "same_case_invocation_local_replay" + candidate[0].Stats.Samples[1].RuntimeReceiptEvents = nil + _, err := buildSPI1QualificationReport(baseline, candidate, resource, SPI1QualificationOptions{ + Seed: 1, Confidence: defaultConfidenceLevel, BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolDiscovery, SourceArchiveSHA256: strings.Repeat("a", 64), + }) + require.ErrorContains(t, err, "timed-invocation attribution") + + trainingBaseline, trainingCandidate, trainingResource := spI1QualificationTestArtifacts(t, referencePairProtocolDiscovery) + discovery, err := buildSPI1QualificationReport(trainingBaseline, trainingCandidate, trainingResource, SPI1QualificationOptions{ + Seed: 1, Confidence: defaultConfidenceLevel, BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolDiscovery, SourceArchiveSHA256: strings.Repeat("a", 64), + }) + require.NoError(t, err) + discovery.BaselineArtifactSHA256 = strings.Repeat("1", 64) + discovery.CandidateArtifactSHA256 = strings.Repeat("2", 64) + discovery.ResourceReportSHA256 = strings.Repeat("3", 64) + freeze := spI1QualificationTestFreeze(t, discovery) + freeze.QuerySHA256 = strings.Repeat("f", 64) + cohort, err := canonicalSPI1Cohort() + require.NoError(t, err) + require.Error(t, validateSPI1FrozenDiscovery(&freeze, &discovery, cohort)) +} + +func TestSPI1QualificationClassifiesBoundResourceFailure(t *testing.T) { + baseline, candidate, resource := spI1QualificationTestArtifacts(t, referencePairProtocolDiscovery) + candidate[0].PostgresMetrics.Buffers.TempWritten = 1 + resource.Cases[0] = evaluateProductionResourceGateCase(candidate[0]) + resource.Passed = false + report, err := buildSPI1QualificationReport(baseline, candidate, resource, SPI1QualificationOptions{ + Seed: 1, Confidence: defaultConfidenceLevel, BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolDiscovery, SourceArchiveSHA256: strings.Repeat("a", 64), + }) + require.NoError(t, err) + require.False(t, report.TrainingPassed) + require.False(t, report.QualificationPassed) + found := false + for _, gateCase := range report.Cases { + found = found || strings.Contains(strings.Join(gateCase.Reasons, "\n"), "candidate resource evidence did not pass") + } + require.True(t, found) +} + +func TestSPI1QualificationRejectsCanonicalEvidenceAndScheduleTampering(t *testing.T) { + tests := map[string]func([]CaseResult, []CaseResult, *ResourceGateReport){ + "canonical observation": func(baseline, _ []CaseResult, _ *ResourceGateReport) { + baseline[0].ObservedRows = []string{`[{"nodes":[],"relationships":[]}]`} + }, + "duplicate warm iteration": func(_ []CaseResult, candidate []CaseResult, _ *ResourceGateReport) { + candidate[0].Stats.Samples[2].Iteration = 1 + }, + "duplicate timed invocation": func(_ []CaseResult, candidate []CaseResult, _ *ResourceGateReport) { + candidate[0].Stats.Samples[2].RuntimeInvocationID = candidate[0].Stats.Samples[1].RuntimeInvocationID + candidate[0].Stats.Samples[2].RuntimeReceiptEvents[0].InvocationID = candidate[0].Stats.Samples[1].RuntimeInvocationID + }, + "cross-record timed invocation replay": func(baseline, candidate []CaseResult, _ *ResourceGateReport) { + candidate[1].Stats.Samples[1].RuntimeInvocationID = baseline[0].Stats.Samples[1].RuntimeInvocationID + candidate[1].Stats.Samples[1].RuntimeReceiptEvents[0].InvocationID = baseline[0].Stats.Samples[1].RuntimeInvocationID + }, + "contradictory arm chronology": func(baseline, candidate []CaseResult, _ *ResourceGateReport) { + started := baseline[0].Environment.StartedAt.Add(-2 * time.Second) + for index := range candidate { + if candidate[index].Environment.Round == 1 { + candidate[index].Environment.StartedAt = started + candidate[index].Environment.EndedAt = started.Add(time.Second) + } + } + }, + "unbound resource round": func(_, _ []CaseResult, resource *ResourceGateReport) { + resource.Cases[0].Round = 99 + }, + "substituted resource receipt": func(_, _ []CaseResult, resource *ResourceGateReport) { + resource.Cases[0].RuntimeReceiptChains[0][0].RuntimeBranch = "substituted" + }, + "cleared resource spill": func(_, candidate []CaseResult, _ *ResourceGateReport) { + candidate[0].PostgresMetrics.Buffers.TempWritten = 1 + }, + "reachable relabeled no path": func(_, candidate []CaseResult, _ *ResourceGateReport) { + candidate[0].TraversalTelemetry.Summary.RuntimeBranch = "inline_canonical_no_path" + for index := range candidate[0].Stats.Samples { + if candidate[0].Stats.Samples[index].Classification == "warm" { + candidate[0].Stats.Samples[index].RuntimeBranch = "inline_canonical_no_path" + candidate[0].Stats.Samples[index].RuntimeReceiptEvents[0].RuntimeBranch = "inline_canonical_no_path" + } + } + }, + "no path relabeled witness": func(_, candidate []CaseResult, _ *ResourceGateReport) { + for recordIndex := range candidate { + if !strings.HasSuffix(candidate[recordIndex].Name, "-disconnected") { + continue + } + candidate[recordIndex].TraversalTelemetry.Summary.RuntimeBranch = "inline_canonical_witness" + for sampleIndex := range candidate[recordIndex].Stats.Samples { + if candidate[recordIndex].Stats.Samples[sampleIndex].Classification == "warm" { + candidate[recordIndex].Stats.Samples[sampleIndex].RuntimeBranch = "inline_canonical_witness" + candidate[recordIndex].Stats.Samples[sampleIndex].RuntimeReceiptEvents[0].RuntimeBranch = "inline_canonical_witness" + } + } + return + } + }, + "output counter differs from observation": func(_, candidate []CaseResult, _ *ResourceGateReport) { + *candidate[0].TraversalTelemetry.Diagnostic.Counters.InlineShortestPath.OutputPaths = 0 + }, + "supplemental planned arm": func(_, candidate []CaseResult, _ *ResourceGateReport) { + candidate[0].TraversalTelemetry.Summary.PlannedIdentities = append(candidate[0].TraversalTelemetry.Summary.PlannedIdentities, "SP-B1-extra") + }, + "reduced planned search space": func(baseline, _ []CaseResult, _ *ResourceGateReport) { + baseline[0].TraversalTelemetry.Summary.PlannedIdentities = []string{ + string(optimize.ShortestPathExecutorS4CanonicalWitness), + string(optimize.ShortestPathExecutorIncumbentWorkspace), + } + }, + } + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + baseline, candidate, resource := spI1QualificationTestArtifacts(t, referencePairProtocolDiscovery) + mutate(baseline, candidate, &resource) + _, err := buildSPI1QualificationReport(baseline, candidate, resource, SPI1QualificationOptions{ + Seed: 1, Confidence: defaultConfidenceLevel, BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolDiscovery, SourceArchiveSHA256: strings.Repeat("a", 64), + }) + require.Error(t, err) + }) + } +} + +func TestSPI1QualificationFreezesStatisticalPolicyAndDiscoverySemantics(t *testing.T) { + baseline, candidate, resource := spI1QualificationTestArtifacts(t, referencePairProtocolDiscovery) + for _, options := range []SPI1QualificationOptions{ + {Seed: 2, Confidence: defaultConfidenceLevel, BootstrapCount: defaultBootstrapCount}, + {Seed: 1, Confidence: 0.95, BootstrapCount: defaultBootstrapCount}, + {Seed: 1, Confidence: defaultConfidenceLevel, BootstrapCount: 1}, + } { + options.Protocol = referencePairProtocolDiscovery + options.SourceArchiveSHA256 = strings.Repeat("a", 64) + _, err := buildSPI1QualificationReport(baseline, candidate, resource, options) + require.Error(t, err) + } + + discovery, err := buildSPI1QualificationReport(baseline, candidate, resource, SPI1QualificationOptions{ + Seed: 1, Confidence: defaultConfidenceLevel, BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolDiscovery, SourceArchiveSHA256: strings.Repeat("a", 64), + }) + require.NoError(t, err) + discovery.BaselineArtifactSHA256 = strings.Repeat("1", 64) + discovery.CandidateArtifactSHA256 = strings.Repeat("2", 64) + discovery.ResourceReportSHA256 = strings.Repeat("3", 64) + freeze := spI1QualificationTestFreeze(t, discovery) + discovery.Cases[0].P95Ratio.Upper = 2 + cohort, err := canonicalSPI1Cohort() + require.NoError(t, err) + require.Error(t, validateSPI1FrozenDiscovery(&freeze, &discovery, cohort)) +} + +func TestSPI1FrozenTrainingEvidenceIsRecomputedFromNamedArtifacts(t *testing.T) { + baseline, candidate, resource := spI1QualificationTestArtifacts(t, referencePairProtocolDiscovery) + directory := t.TempDir() + baselinePath := filepath.Join(directory, "s4.jsonl") + candidatePath := filepath.Join(directory, "i1.jsonl") + resourcePath := filepath.Join(directory, "resource.json") + require.NoError(t, writeJSONLFile(baselinePath, baseline)) + require.NoError(t, writeJSONLFile(candidatePath, candidate)) + candidateSHA256, err := fileSHA256(candidatePath) + require.NoError(t, err) + resource.ArtifactSHA256 = candidateSHA256 + resourceRaw, err := json.MarshalIndent(resource, "", " ") + require.NoError(t, err) + require.NoError(t, os.WriteFile(resourcePath, append(resourceRaw, '\n'), 0o600)) + + discovery, err := buildSPI1QualificationReport(baseline, candidate, resource, SPI1QualificationOptions{ + Seed: 1, Confidence: defaultConfidenceLevel, BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolDiscovery, SourceArchiveSHA256: strings.Repeat("a", 64), + }) + require.NoError(t, err) + discovery.BaselineArtifactSHA256, err = fileSHA256(baselinePath) + require.NoError(t, err) + discovery.CandidateArtifactSHA256 = candidateSHA256 + discovery.ResourceReportSHA256, err = fileSHA256(resourcePath) + require.NoError(t, err) + freeze := spI1QualificationTestFreeze(t, discovery) + require.NoError(t, validateSPI1FrozenTrainingEvidence(&freeze, &discovery, baselinePath, candidatePath, resourcePath)) + + forged := discovery + forged.Cases = append([]SPI1QualificationCase(nil), discovery.Cases...) + forged.Cases[0].MedianRatio = RatioInterval{Lower: 0.801, Estimate: 0.801, Upper: 0.801} + require.ErrorContains(t, + validateSPI1FrozenTrainingEvidence(&freeze, &forged, baselinePath, candidatePath, resourcePath), + "differs from its recomputed", + ) +} + +func TestSPI1PathsRejectHardlinkAliases(t *testing.T) { + directory := t.TempDir() + input := filepath.Join(directory, "input.json") + alias := filepath.Join(directory, "alias.json") + require.NoError(t, os.WriteFile(input, []byte("{}"), 0o600)) + require.NoError(t, os.Link(input, alias)) + require.Error(t, validateDistinctSPI1Paths(map[string]string{"input": input, "output": alias})) +} + +func TestSPI1HoldoutCaptureProfileAcceptsBothBalancedArmsAndRejectsDrift(t *testing.T) { + baseline := string(optimize.ShortestPathExecutorS4CanonicalWitness) + candidate := string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness) + valid := func(executor, arm string, round, order int) config { + return config{ + Modes: []ExecutionMode{ModePostgresSQL}, Iterations: 50, WarmupIterations: 20, + Round: round, Block: round, Arm: arm, ArmOrder: order, RunUUID: "sp-i1-confirmation", + PoolSize: 1, PostgresForceShortest: executor, PostgresRepeatableRead: true, + PostgresTraversalTelemetry: postgresTraversalTelemetryDiagnostic, + OutputJSONL: fmt.Sprintf(".coverage/sp-i1-%s-%d.jsonl", arm, round), AppendJSONL: round > 1, + SPI1Freeze: ".coverage/sp-i1-freeze.json", SPI1DiscoveryReport: ".coverage/sp-i1-discovery.json", + SPI1TrainingBaseline: ".coverage/sp-i1-training-s4.jsonl", + SPI1TrainingCandidate: ".coverage/sp-i1-training-i1.jsonl", + SPI1TrainingResource: ".coverage/sp-i1-training-resource.json", + } + } + for _, cfg := range []config{ + valid(baseline, "sp-i1-s4", 1, 1), + valid(candidate, "sp-i1-candidate", 1, 2), + valid(baseline, "sp-i1-s4", 2, 2), + valid(candidate, "sp-i1-candidate", 2, 1), + } { + require.NoError(t, validateSPI1HoldoutCaptureConfig(cfg)) + } + + tests := map[string]func(*config){ + "wrong backend": func(cfg *config) { cfg.Modes = []ExecutionMode{ModeNeo4j} }, + "existing graph": func(cfg *config) { cfg.ExistingGraph = true }, + "too few samples": func(cfg *config) { cfg.Iterations = 49 }, + "too few warmups": func(cfg *config) { cfg.WarmupIterations = 19 }, + "pool larger than one": func(cfg *config) { cfg.PoolSize = 2 }, + "concurrency": func(cfg *config) { cfg.Concurrency = []int{2} }, + "round above maximum": func(cfg *config) { cfg.Round, cfg.Block = 21, 21 }, + "mismatched block": func(cfg *config) { cfg.Block = 2 }, + "missing run UUID": func(cfg *config) { cfg.RunUUID = "" }, + "wrong arm label": func(cfg *config) { cfg.Arm = "baseline" }, + "wrong arm order": func(cfg *config) { cfg.ArmOrder = 2 }, + "wrong executor": func(cfg *config) { cfg.PostgresForceShortest = "SP-S3-U-E+MAT-M0" }, + "read committed": func(cfg *config) { cfg.PostgresRepeatableRead = false }, + "summary telemetry": func(cfg *config) { cfg.PostgresTraversalTelemetry = postgresTraversalTelemetrySummary }, + "supplemental references": func(cfg *config) { cfg.PostgresReferences = true }, + "missing output": func(cfg *config) { cfg.OutputJSONL = "" }, + "path alias": func(cfg *config) { cfg.OutputJSONL = cfg.SPI1Freeze }, + } + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + cfg := valid(baseline, "sp-i1-s4", 1, 1) + mutate(&cfg) + require.Error(t, validateSPI1HoldoutCaptureConfig(cfg)) + }) + } + t.Run("round after one requires append", func(t *testing.T) { + cfg := valid(baseline, "sp-i1-s4", 2, 2) + cfg.AppendJSONL = false + require.Error(t, validateSPI1HoldoutCaptureConfig(cfg)) + }) +} + +func TestSPI1HoldoutDetectionAndCorpusBindingIgnoreMutableTagAlone(t *testing.T) { + full, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + training, _, err := selectScaleCorpus(full, CorpusSelectors{Tags: []string{"sp-i1-inbound-v1-training"}}) + require.NoError(t, err) + require.False(t, selectedCorpusContainsSPI1Holdout(training)) + + confirmation, _, err := selectScaleCorpus(full, CorpusSelectors{Tags: []string{"sp-i1-inbound-v1-training", "sp-i1-inbound-v1-holdout"}}) + require.NoError(t, err) + require.True(t, selectedCorpusContainsSPI1Holdout(confirmation)) + for index := range confirmation.Cases { + confirmation.Cases[index].Source = strings.TrimPrefix(confirmation.Cases[index].Source, "../../") + confirmation.Cases[index].Tags = nil + } + require.True(t, selectedCorpusContainsSPI1Holdout(confirmation), "canonical key detection must not depend on tags") + + // Restore exact declarations before checking the complete frozen corpus. + exact, _, err := selectScaleCorpus(full, CorpusSelectors{Tags: []string{"sp-i1-inbound-v1-training", "sp-i1-inbound-v1-holdout"}}) + require.NoError(t, err) + for index := range exact.Cases { + exact.Cases[index].Source = strings.TrimPrefix(exact.Cases[index].Source, "../../") + } + cohort, err := canonicalSPI1Cohort() + require.NoError(t, err) + require.NoError(t, validateSPI1Corpus(exact, cohort)) + + omitted := ScaleCorpus{Cases: append([]ScaleCase(nil), exact.Cases[:len(exact.Cases)-1]...)} + require.Error(t, validateSPI1Corpus(omitted, cohort)) + mutated := ScaleCorpus{Cases: append([]ScaleCase(nil), exact.Cases...)} + mutated.Cases[0].Cypher += " " + require.Error(t, validateSPI1Corpus(mutated, cohort)) +} + +func TestRunnableCorpusExcludesSPI1HoldoutUntilExactOptIn(t *testing.T) { + full, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + + ordinary, manifest, err := selectRunnableScaleCorpus(full, CorpusSelectors{}) + require.NoError(t, err) + require.False(t, selectedCorpusContainsSPI1Holdout(ordinary)) + require.False(t, manifest.DiagnosticOnly) + require.Equal(t, manifest.FullDeclarationCount, manifest.SelectedDeclarationCount+manifest.OmittedDeclarationCount) + require.Equal(t, 6, manifest.OmittedDeclarationCount) + require.Equal(t, 6, manifest.ProtectedDeclarationCount) + require.True(t, lowercaseSHA256(manifest.ProtectedDeclarationSHA256)) + require.True(t, selectedCorpusContainsTag(ordinary, spI1TrainingTag)) + + for name, selectors := range map[string]CorpusSelectors{ + "generic holdout tag": {Tags: []string{"holdout"}}, + "broad category": {Categories: []string{"generated_shortest_path_v2"}}, + } { + t.Run(name, func(t *testing.T) { + selected, _, err := selectRunnableScaleCorpus(full, selectors) + require.NoError(t, err) + require.False(t, selectedCorpusContainsSPI1Holdout(selected)) + }) + } + + exactTag, _, err := selectRunnableScaleCorpus(full, CorpusSelectors{Tags: []string{spI1HoldoutTag}}) + require.NoError(t, err) + require.Len(t, exactTag.Cases, 3) + require.True(t, selectedCorpusContainsSPI1Holdout(exactTag)) + + exactCase, _, err := selectRunnableScaleCorpus(full, CorpusSelectors{Cases: []string{spI1CanonicalCases[4].name}}) + require.NoError(t, err) + require.Len(t, exactCase.Cases, 1) + require.True(t, selectedCorpusContainsSPI1Holdout(exactCase)) +} + +func spI1QualificationTestArtifacts(t *testing.T, protocol string) ([]CaseResult, []CaseResult, ResourceGateReport) { + t.Helper() + full, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + tags := []string{"sp-i1-inbound-v1-training"} + rounds, samples, warmups := 5, 10, 5 + corpusSHA256 := spI1TrainingCorpusSHA256 + if protocol == referencePairProtocolConfirmation { + tags = append(tags, "sp-i1-inbound-v1-holdout") + rounds, samples, warmups = 10, 50, 20 + corpusSHA256 = spI1FullCorpusSHA256 + } + selected, selection, err := selectRunnableScaleCorpus(full, CorpusSelectors{Tags: tags}) + require.NoError(t, err) + + var baseline, candidate []CaseResult + resource := ResourceGateReport{Version: resourceGateVersion, ArtifactSHA256: strings.Repeat("9", 64), Passed: true} + for _, testCase := range selected.Cases { + fixture, err := fixtureMetadata("unused", testCase.Dataset) + require.NoError(t, err) + fixture.PhysicalValidated = true + fixture.PhysicalNodeCount = int64(fixture.NodeCount) + fixture.PhysicalEdgeCount = int64(fixture.EdgeCount) + fixture.NodeRelationBytes = int64(fixture.NodeCount) * 1024 + fixture.EdgeRelationBytes = int64(fixture.EdgeCount) * 1024 + for round := 1; round <= rounds; round++ { + left, right := spI1QualificationTestRecords(t, testCase, fixture, selection, corpusSHA256, round, samples, warmups) + baseline = append(baseline, left) + candidate = append(candidate, right) + allObserved := traversalNumericObservations(right.TraversalTelemetry.Diagnostic.Counters) + observed := make(map[string]int64, len(spI1TelemetryCaps())) + for name := range spI1TelemetryCaps() { + observed[name] = allObserved[name] + } + resource.Cases = append(resource.Cases, ResourceGateCase{ + Dataset: right.Dataset, Name: right.Name, Tier: right.Shape.FixtureTier, + Round: right.Environment.Round, Block: right.Environment.Block, RunUUID: right.Environment.RunUUID, + Arm: right.Environment.Arm, ArmOrder: right.Environment.ArmOrder, + QualificationSplit: right.Shape.QualificationSplit, + Architecture: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + Passed: true, + NumericLimits: spI1TelemetryCaps(), + NumericObserved: observed, + RuntimeReceiptChains: runtimeReceiptChains(right.Stats.Samples), + }) + } + } + return baseline, candidate, resource +} + +func spI1QualificationTestRecords( + t *testing.T, + testCase ScaleCase, + fixture FixtureMetadata, + selection SelectionManifest, + corpusSHA256 string, + round, samples, warmups int, +) (CaseResult, CaseResult) { + t.Helper() + baselineIdentity := string(optimize.ShortestPathExecutorS4CanonicalWitness) + candidateIdentity := string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness) + baselineOrder, candidateOrder := 1, 2 + if round%2 == 0 { + baselineOrder, candidateOrder = 2, 1 + } + rowCount := int64(1) + var observed []string + if len(testCase.Expected.PathRows) == 1 { + expectedPath := testCase.Expected.PathRows[0] + path := stablePathObservation{ + Nodes: make([]stableNodeObservation, len(expectedPath.Nodes)), + Relationships: make([]stableRelationshipObservation, len(expectedPath.RelationshipKinds)), + } + for index, identity := range expectedPath.Nodes { + path.Nodes[index].Identity = identity + } + for index, kind := range expectedPath.RelationshipKinds { + path.Relationships[index] = stableRelationshipObservation{ + Identity: expectedPath.RelationshipKeys[index], Start: expectedPath.Nodes[index], + End: expectedPath.Nodes[index+1], Kind: kind, + } + } + raw, err := json.Marshal([]any{path}) + require.NoError(t, err) + observed = []string{string(raw)} + } + if strings.HasSuffix(testCase.Name, "-disconnected") { + rowCount, observed = 0, nil + } + falseValue, trueValue := false, true + makeSamples := func(arm string, order int, duration time.Duration, requested, branch, attestation string) []LatencySample { + result := make([]LatencySample, samples+1) + result[0] = LatencySample{ + Round: round, Block: round, Arm: arm, ArmOrder: order, RunUUID: "sp-i1-test-run", + Iteration: 0, Case: testCase.Name, Dataset: testCase.Dataset, Backend: ModePostgresSQL, + ConnectionID: "101", Classification: "cold", Duration: 2 * duration, + } + for index := range samples { + invocationID := fmt.Sprintf("sp-i1-test-%s-%s-%d-%d", arm, testCase.Name, round, index+1) + result[index+1] = LatencySample{ + Round: round, Block: round, Arm: arm, ArmOrder: order, RunUUID: "sp-i1-test-run", + Iteration: index + 1, Case: testCase.Name, Dataset: testCase.Dataset, Backend: ModePostgresSQL, + ConnectionID: "101", Classification: "warm", Duration: duration, + RequestedIdentity: requested, RuntimeIdentity: requested, RuntimeBranch: branch, + FallbackExecuted: &falseValue, RuntimeAttestation: attestation, RuntimeInvocationID: invocationID, + } + if attestation == "timed_invocation" { + result[index+1].RuntimeReceiptEvents = []RuntimeReceiptEvent{{ + InvocationID: invocationID, Ordinal: 1, RuntimeIdentity: requested, RuntimeBranch: branch, FallbackExecuted: false, + }} + } + } + return result + } + baseEnvironment := RunEnvironment{ + ArtifactSchemaVersion: 2, CorpusSHA256: corpusSHA256, + SourceCommit: "deadbeef", DirtyDiffSHA256: cleanWorkingTreeSHA256(), BinarySHA256: strings.Repeat("b", 64), + GOOS: "linux", GOARCH: "amd64", CPUCount: 8, CPUModel: "test-cpu", Kernel: "test-kernel", + CgroupCPU: "max 100000", CgroupMemory: "max", CPUGovernor: "performance", + RunUUID: "sp-i1-test-run", Block: round, Round: round, WarmupIterations: warmups, + Selection: &selection, PoolSize: 1, Protocol: "fixed_confirmation", + } + postgresEnvironment := &PostgresEnvironment{ + Version: "PostgreSQL test", Database: "dawgs", PlanCacheMode: "auto", TransactionIsolation: "repeatable read", + WorkMem: "64MB", TempFileLimit: "1GB", GraphPartitionCount: 1, DatabaseOID: 42, + Autovacuum: "on", NodeRelationBytes: fixture.NodeRelationBytes, EdgeRelationBytes: fixture.EdgeRelationBytes, + AnalyzeState: "edge:analyzed,node:analyzed", SchemaFingerprint: strings.Repeat("c", 64), IndexFingerprint: strings.Repeat("d", 64), + } + base := newCaseResult(testCase, ModePostgresSQL, nil) + base.RowCount = rowCount + base.ObservedRows = append([]string(nil), observed...) + base.Status = StatusOK + base.WorkloadSHA256 = scaleCaseWorkloadIdentity(testCase, ModePostgresSQL) + attachFixtureMetadata(&base, fixture) + base.PostgresEnvironment = postgresEnvironment + + baseline := base + baseline.Environment = cloneSPI1TestEnvironment(baseEnvironment, "sp-i1-s4", baselineOrder) + firstStarted := time.Unix(1_700_000_000+int64(round)*10, 0) + baselineStarted, candidateStarted := firstStarted, firstStarted.Add(2*time.Second) + if candidateOrder == 1 { + candidateStarted, baselineStarted = firstStarted, firstStarted.Add(2*time.Second) + } + baseline.Environment.StartedAt, baseline.Environment.EndedAt = baselineStarted, baselineStarted.Add(time.Second) + baseline.SQL = "select 's4:' || " + fmt.Sprintf("%q", testCase.Name) + baseline.SQLFingerprint = sqlFingerprint(baseline.SQL) + baselineBranch := "compact_workspace_witness" + if rowCount == 0 { + baselineBranch = "compact_no_path" + } + baseline.Stats = DurationStats{ + Iterations: samples, WarmupIterations: warmups, Median: 10 * time.Millisecond, P95: 10 * time.Millisecond, + Samples: makeSamples("sp-i1-s4", baselineOrder, 10*time.Millisecond, baselineIdentity, baselineBranch, "timed_invocation"), + } + baselineOutcome := translate.TargetLoweringOutcome{ + Lowering: optimize.LoweringShortestPathExecutor, TargetKind: "traversal", Family: "SP", + Selected: baselineIdentity, Applied: baselineIdentity, Fallback: "SP-S0", + PlannedCandidates: spI1ShortestPathPlannedIdentities(), SelectorVersion: "sp-tool-v1", + ExecutionBoundary: "stored_helper", ObservationMode: "one_path", Scheduler: "single_ended_level", + Direction: "inbound", PhysicalExpansion: "end_id", RelationshipKindCount: 1, + TopologyClassification: "physical_inbound_deep", SelectionMode: "forced_tool", + Eligible: &trueValue, StaticallyEligible: &trueValue, + MinimumDepth: traversalTelemetryPointer(int64(1)), MaximumDepth: traversalTelemetryPointer(int64(64)), + StateLimit: 100_000, FrontierLimit: 100_000, PredecessorLimit: 100_000, + EnumerationLimit: 100_000, OutputBytesLimit: 64 * 1024 * 1024, + } + baseline.Optimization = &translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{baselineOutcome}} + baselineMetrics := PostgresPlanMetrics{Provenance: map[string]string{}} + baseline.PostgresMetrics = &baselineMetrics + baselineTelemetry, err := buildPostgresCaseTraversalTelemetry(*baseline.Optimization, baselineMetrics, "101", TraversalTelemetryLevelDiagnostic) + require.NoError(t, err) + baseline.TraversalTelemetry = baselineTelemetry + + candidate := base + candidate.Environment = cloneSPI1TestEnvironment(baseEnvironment, "sp-i1-candidate", candidateOrder) + candidate.Environment.StartedAt, candidate.Environment.EndedAt = candidateStarted, candidateStarted.Add(time.Second) + candidate.SQL = "select 'i1:' || " + fmt.Sprintf("%q", testCase.Name) + candidate.SQLFingerprint = sqlFingerprint(candidate.SQL) + candidateBranch := "inline_canonical_witness" + if rowCount == 0 { + candidateBranch = "inline_canonical_no_path" + } + candidate.Stats = DurationStats{ + Iterations: samples, WarmupIterations: warmups, Median: 8 * time.Millisecond, P95: 8 * time.Millisecond, + Samples: makeSamples("sp-i1-candidate", candidateOrder, 8*time.Millisecond, candidateIdentity, candidateBranch, "timed_invocation"), + } + candidateOutcome := translate.TargetLoweringOutcome{ + Lowering: optimize.LoweringShortestPathExecutor, TargetKind: "traversal", Family: "SP", + Candidate: candidateIdentity, Selected: candidateIdentity, Applied: candidateIdentity, + Fallback: baselineIdentity, PlannedCandidates: spI1ShortestPathPlannedIdentities(), + EmittedCandidates: []string{candidateIdentity, baselineIdentity}, EmittedPolicy: optimize.ShortestPathPolicyI1CanonicalGuardedV1, + SelectorVersion: "sp-i1-canonical-tool-v1", ExecutionBoundary: optimize.ExpansionSearchExecutionBoundaryGuardedDualArm, + ObservationMode: "one_path", Scheduler: "single_ended_level", Direction: "inbound", PhysicalExpansion: "end_id", + RelationshipKindCount: 1, TopologyClassification: "physical_inbound_deep", SelectionMode: "forced_tool", + Eligible: &trueValue, StaticallyEligible: &trueValue, + MinimumDepth: traversalTelemetryPointer(int64(1)), MaximumDepth: traversalTelemetryPointer(int64(64)), + StateLimit: 100_000, PredecessorLimit: 100_000, + EnumerationLimit: 100_000, OutputBytesLimit: 64 * 1024 * 1024, + } + candidate.Optimization = &translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{candidateOutcome}} + candidateMetrics := PostgresPlanMetrics{Provenance: map[string]string{}, HydrationRows: rowCount, HydrationLoops: rowCount, PlanNodes: []PostgresPlanNodeMetric{ + inlinePredecessorPlanNode("asp_i1_distance_bounded", 32, 1), + inlinePredecessorPlanNode("asp_i1_predecessor_bounded", 16, 1), + inlinePredecessorPlanNode("asp_i1_paths_bounded", rowCount, 1), + inlinePredecessorPlanNode("asp_i1_shortest", rowCount, 1), + inlinePredecessorPlanNode("asp_i1_candidate_marker", 1, 1), + inlinePredecessorPlanNode("asp_i1_fallback_marker", 0, 1), + inlinePredecessorPlanNode("asp_i1_candidate_rows", rowCount, 1), + inlinePredecessorPlanNode("asp_i1_fallback_rows", 0, 1), + inlinePredecessorMarkerGateNode("candidate", 1, 1), + inlinePredecessorMarkerGateNode("fallback", 0, 1), + inlinePredecessorExecutorNode("candidate", 1), + inlinePredecessorExecutorNode("fallback", 0), + }} + candidate.PostgresMetrics = &candidateMetrics + candidateTelemetry, err := buildPostgresCaseTraversalTelemetry(*candidate.Optimization, candidateMetrics, "101", TraversalTelemetryLevelDiagnostic) + require.NoError(t, err) + enrichInlinePredecessorTraversalTelemetry(candidateTelemetry, candidateMetrics, rowCount, observed) + require.NoError(t, candidateTelemetry.Validate()) + candidate.TraversalTelemetry = candidateTelemetry + return baseline, candidate +} + +func cloneSPI1TestEnvironment(environment RunEnvironment, arm string, order int) *RunEnvironment { + copy := environment + copy.Arm = arm + copy.ArmOrder = order + return © +} + +func spI1QualificationTestFreeze(t *testing.T, discovery SPI1QualificationReport) SPI1QualificationFreezeManifest { + t.Helper() + cohort, err := canonicalSPI1Cohort() + require.NoError(t, err) + return SPI1QualificationFreezeManifest{ + Version: spI1FreezeVersion, Baseline: discovery.Baseline, Candidate: discovery.Candidate, + Policy: discovery.Policy, QuerySHA256: discovery.QuerySHA256, Caps: discovery.Caps, + Seed: discovery.Seed, Confidence: discovery.Confidence, BootstrapCount: discovery.BootstrapCount, + SourceCommit: discovery.SourceCommit, SourceArchiveSHA256: discovery.SourceArchiveSHA256, + DirtyDiffSHA256: discovery.DirtyDiffSHA256, BinarySHA256: discovery.BinarySHA256, + TrainingDeclarationSHA256: cohort.trainingDeclarationSHA256, + HoldoutDeclarationSHA256: cohort.holdoutDeclarationSHA256, + FullDeclarationSHA256: cohort.declarationSHA256, + TrainingCorpusSHA256: cohort.trainingCorpusSHA256, + FullCorpusSHA256: cohort.fullCorpusSHA256, + TrainingResolvedSHA256: cohort.trainingResolvedSHA256, + FullResolvedSHA256: cohort.fullResolvedSHA256, + BaselineArtifactSHA256: discovery.BaselineArtifactSHA256, + CandidateArtifactSHA256: discovery.CandidateArtifactSHA256, + ResourceReportSHA256: discovery.ResourceReportSHA256, + DiscoveryReportSHA256: strings.Repeat("4", 64), + TrainingPassed: discovery.TrainingPassed, + } +} diff --git a/cmd/graphbench/statistical_evidence.go b/cmd/graphbench/statistical_evidence.go new file mode 100644 index 00000000..9c7c1e37 --- /dev/null +++ b/cmd/graphbench/statistical_evidence.go @@ -0,0 +1,750 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "sort" + "strings" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" +) + +const ( + // defaultConfidenceLevel is the default confidence used by qualification reports. + defaultConfidenceLevel = 0.975 + // minimumTimingNoiseRatio is the smallest relative timing floor accepted for promotion decisions. + minimumTimingNoiseRatio = 0.05 + // minimumTimingNoiseAbsolute is the smallest absolute timing floor accepted for promotion decisions. + minimumTimingNoiseAbsolute = 100 * time.Microsecond +) + +// benchmarkHostIdentity contains stable host properties that must match an A/A calibration. +type benchmarkHostIdentity struct { + GOOS string `json:"goos"` + GOARCH string `json:"goarch"` + CPUCount int `json:"cpu_count"` + CPUModel string `json:"cpu_model"` + Kernel string `json:"kernel"` + CgroupCPU string `json:"cgroup_cpu,omitempty"` + CgroupMemory string `json:"cgroup_memory,omitempty"` + CPUGovernor string `json:"cpu_governor,omitempty"` +} + +// artifactHostFingerprint returns one stable host fingerprint for all PostgreSQL timing records. +func artifactHostFingerprint(records []CaseResult) (string, error) { + fingerprint := "" + found := false + for _, record := range records { + if record.ExecutionMode != ModePostgresSQL || !hasWarmLatencySample(record) { + continue + } + if record.Environment == nil { + return "", fmt.Errorf("%s/%s has no run environment for host calibration", record.Dataset, record.Name) + } + identity := benchmarkHostIdentity{ + GOOS: strings.TrimSpace(record.Environment.GOOS), + GOARCH: strings.TrimSpace(record.Environment.GOARCH), + CPUCount: record.Environment.CPUCount, + CPUModel: strings.TrimSpace(record.Environment.CPUModel), + Kernel: strings.TrimSpace(record.Environment.Kernel), + CgroupCPU: strings.TrimSpace(record.Environment.CgroupCPU), + CgroupMemory: strings.TrimSpace(record.Environment.CgroupMemory), + CPUGovernor: strings.TrimSpace(record.Environment.CPUGovernor), + } + if identity.GOOS == "" || identity.GOARCH == "" || identity.CPUCount < 1 || identity.CPUModel == "" || identity.Kernel == "" { + return "", fmt.Errorf("%s/%s has incomplete host identity", record.Dataset, record.Name) + } + raw, err := json.Marshal(identity) + if err != nil { + return "", err + } + digest := sha256.Sum256(raw) + current := hex.EncodeToString(digest[:]) + if fingerprint != "" && current != fingerprint { + return "", fmt.Errorf("PostgreSQL timing artifact mixes host identities") + } + fingerprint = current + found = true + } + if !found { + return "", fmt.Errorf("artifact has no PostgreSQL warm timing records for host calibration") + } + + return fingerprint, nil +} + +func hasWarmLatencySample(record CaseResult) bool { + for _, sample := range record.Stats.Samples { + if sample.Classification == "warm" && sample.Duration > 0 { + return true + } + } + return false +} + +// validateAAResolutionEvidence verifies schema, checksum, confidence, host, and per-case metric integrity. +func validateAAResolutionEvidence(report *AAResolutionReport, records []CaseResult, confidence float64) error { + if report == nil { + return fmt.Errorf("host A/A resolution report is required") + } + if report.Version != aaReportVersion { + return fmt.Errorf("A/A report version must be %d", aaReportVersion) + } + if report.Confidence <= 0 || report.Confidence >= 1 || math.IsNaN(report.Confidence) || report.Confidence < confidence { + return fmt.Errorf("A/A confidence %.4f is below requested confidence %.4f", report.Confidence, confidence) + } + if !validSHA256(report.ArtifactSHA256) { + return fmt.Errorf("A/A artifact SHA-256 is missing or malformed") + } + hostFingerprint, err := artifactHostFingerprint(records) + if err != nil { + return err + } + if !validSHA256(report.HostFingerprint) || report.HostFingerprint != hostFingerprint { + return fmt.Errorf("A/A host fingerprint does not match timing artifact host") + } + if report.MinimumRounds < minimumGateRounds || report.MinimumSamplesPerArmPerRound < 10 || !report.OrderBalanced { + return fmt.Errorf("A/A report lacks the balanced discovery evidence protocol") + } + if len(report.Cases) == 0 { + return fmt.Errorf("A/A report contains no case resolution evidence") + } + + seen := map[performanceKey]struct{}{} + for _, entry := range report.Cases { + key := performanceKey{dataset: entry.Dataset, name: entry.Name, backend: entry.Backend} + if entry.Dataset == "" || entry.Name == "" || entry.Backend != ModePostgresSQL { + return fmt.Errorf("A/A report contains malformed case identity") + } + if strings.TrimSpace(entry.WorkloadSHA256) == "" { + return fmt.Errorf("A/A case %s/%s has no workload identity", key.dataset, key.name) + } + if _, duplicate := seen[key]; duplicate { + return fmt.Errorf("A/A report contains duplicate case %s/%s/%s", key.dataset, key.name, key.backend) + } + seen[key] = struct{}{} + if entry.Rounds < minimumGateRounds || entry.SamplesPerArm < entry.Rounds*report.MinimumSamplesPerArmPerRound { + return fmt.Errorf("A/A case %s/%s lacks discovery-grade rounds or samples", key.dataset, key.name) + } + if err := validateAAMetric(entry.P50); err != nil { + return fmt.Errorf("A/A case %s/%s p50: %w", key.dataset, key.name, err) + } + if err := validateAAMetric(entry.P95); err != nil { + return fmt.Errorf("A/A case %s/%s p95: %w", key.dataset, key.name, err) + } + for _, record := range records { + if record.Dataset == key.dataset && record.Name == key.name && record.ExecutionMode == key.backend && record.WorkloadSHA256 != entry.WorkloadSHA256 { + return fmt.Errorf("A/A workload identity does not match %s/%s/%s", key.dataset, key.name, key.backend) + } + } + } + + return nil +} + +func workloadSHA256ForKey(records []CaseResult, key performanceKey) (string, error) { + identity := "" + for _, record := range records { + if record.Dataset != key.dataset || record.Name != key.name || record.ExecutionMode != key.backend { + continue + } + if record.WorkloadSHA256 == "" { + return "", fmt.Errorf("%s/%s/%s has no workload identity", key.dataset, key.name, key.backend) + } + if identity != "" && identity != record.WorkloadSHA256 { + return "", fmt.Errorf("%s/%s/%s mixes workload identities", key.dataset, key.name, key.backend) + } + identity = record.WorkloadSHA256 + } + if identity == "" { + return "", fmt.Errorf("%s/%s/%s has no workload record", key.dataset, key.name, key.backend) + } + return identity, nil +} + +func postgresTimingEnvironmentSHA256ForKey(records []CaseResult, key performanceKey) (string, error) { + identity := "" + found, missing := false, false + for _, record := range records { + if record.Dataset != key.dataset || record.Name != key.name || record.ExecutionMode != key.backend { + continue + } + found = true + if record.PostgresEnvironment == nil { + missing = true + continue + } + value := *record.PostgresEnvironment + value.AnalyzeState = normalizedAnalyzeState(value.AnalyzeState) + raw, err := json.Marshal(value) + if err != nil { + return "", fmt.Errorf("encode %s/%s/%s PostgreSQL timing environment: %w", key.dataset, key.name, key.backend, err) + } + digest := sha256.Sum256(raw) + current := hex.EncodeToString(digest[:]) + if identity != "" && identity != current { + return "", fmt.Errorf("%s/%s/%s mixes PostgreSQL timing environments", key.dataset, key.name, key.backend) + } + identity = current + } + if !found { + return "", fmt.Errorf("%s/%s/%s has no workload record", key.dataset, key.name, key.backend) + } + if missing && identity != "" { + return "", fmt.Errorf("%s/%s/%s has partially missing PostgreSQL timing environment", key.dataset, key.name, key.backend) + } + return identity, nil +} + +func fixtureSHA256ForKey(records []CaseResult, key performanceKey) (string, error) { + identity := "" + found, missing := false, false + for _, record := range records { + if record.Dataset != key.dataset || record.Name != key.name || record.ExecutionMode != key.backend { + continue + } + found = true + if record.Fixture == nil { + missing = true + continue + } + raw, err := json.Marshal(record.Fixture) + if err != nil { + return "", fmt.Errorf("encode %s/%s/%s fixture: %w", key.dataset, key.name, key.backend, err) + } + digest := sha256.Sum256(raw) + current := hex.EncodeToString(digest[:]) + if identity != "" && identity != current { + return "", fmt.Errorf("%s/%s/%s mixes fixture identities", key.dataset, key.name, key.backend) + } + identity = current + } + if !found { + return "", fmt.Errorf("%s/%s/%s has no workload record", key.dataset, key.name, key.backend) + } + if missing && identity != "" { + return "", fmt.Errorf("%s/%s/%s has partially missing fixture identity", key.dataset, key.name, key.backend) + } + return identity, nil +} + +func normalizedAnalyzeState(value string) string { + if strings.TrimSpace(value) == "" { + return "" + } + entries := strings.Split(value, ",") + for index, entry := range entries { + relation, state, found := strings.Cut(strings.TrimSpace(entry), ":") + if !found { + entries[index] = relation + continue + } + state = strings.TrimSpace(state) + if state != "" && state != "never" { + state = "analyzed" + } + entries[index] = relation + ":" + state + } + sort.Strings(entries) + return strings.Join(entries, ",") +} + +func validateAAMetric(metric AAMetricResolution) error { + if metric.Ratio.Estimate <= 0 || metric.Ratio.Lower <= 0 || metric.Ratio.Upper <= 0 || + metric.Ratio.Lower > metric.Ratio.Estimate || metric.Ratio.Estimate > metric.Ratio.Upper || + math.IsNaN(metric.Ratio.Estimate) || math.IsNaN(metric.Ratio.Lower) || math.IsNaN(metric.Ratio.Upper) || + math.IsInf(metric.Ratio.Estimate, 0) || math.IsInf(metric.Ratio.Lower, 0) || math.IsInf(metric.Ratio.Upper, 0) { + return fmt.Errorf("ratio interval is malformed") + } + if metric.RatioResolution < 0 || math.IsNaN(metric.RatioResolution) || math.IsInf(metric.RatioResolution, 0) || metric.AbsoluteResolution < 0 { + return fmt.Errorf("resolution is malformed") + } + if metric.AbsoluteChange.Lower > metric.AbsoluteChange.Estimate || metric.AbsoluteChange.Estimate > metric.AbsoluteChange.Upper || + metric.AbsoluteResolution < max(absDuration(metric.AbsoluteChange.Lower), absDuration(metric.AbsoluteChange.Upper)) { + return fmt.Errorf("absolute-change interval is malformed") + } + return nil +} + +// aaTimingFloor returns host-derived per-case noise with the mandatory relative and absolute minimums. +func aaTimingFloor(report *AAResolutionReport, key performanceKey, p95 bool, configuredRatio float64) (float64, time.Duration, error) { + for _, entry := range report.Cases { + if entry.Dataset != key.dataset || entry.Name != key.name || entry.Backend != key.backend { + continue + } + metric := entry.P50 + if p95 { + metric = entry.P95 + } + return max(minimumTimingNoiseRatio, configuredRatio, metric.RatioResolution), + max(minimumTimingNoiseAbsolute, metric.AbsoluteResolution), nil + } + + return 0, 0, fmt.Errorf("A/A report has no resolution evidence for %s/%s/%s", key.dataset, key.name, key.backend) +} + +func validSHA256(value string) bool { + if len(value) != sha256.Size*2 { + return false + } + _, err := hex.DecodeString(value) + return err == nil +} + +// timingTier requires a stable, explicit normal, envelope, or stress classification across artifacts. +func timingTier(key performanceKey, artifacts ...[]CaseResult) (string, error) { + tier := "" + found := false + for _, records := range artifacts { + for _, record := range records { + if record.Dataset != key.dataset || record.Name != key.name || record.ExecutionMode != key.backend { + continue + } + current := record.Shape.FixtureTier + if current != "normal" && current != "envelope" && current != "stress" { + return "", fmt.Errorf("%s/%s/%s has missing or unsupported fixture tier %q", key.dataset, key.name, key.backend, current) + } + if tier != "" && tier != current { + return "", fmt.Errorf("%s/%s/%s changes fixture tier across artifacts", key.dataset, key.name, key.backend) + } + tier = current + found = true + } + } + if !found { + return "unknown", nil + } + return tier, nil +} + +// qualificationSplit requires one stable training, holdout, or diagnostic +// partition for prioritized traversal records. The split is part of the +// workload declaration and may not drift between benchmark arms or rounds. +// Legacy non-traversal records may omit it. +func qualificationSplit(key performanceKey, artifacts ...[]CaseResult) (string, error) { + split := "" + found := false + for _, records := range artifacts { + for _, record := range records { + if record.Dataset != key.dataset || record.Name != key.name || record.ExecutionMode != key.backend { + continue + } + current := record.Shape.QualificationSplit + if current == "" { + if prioritizedTraversalRecord(record) { + return "", fmt.Errorf("%s/%s/%s has no frozen qualification split", key.dataset, key.name, key.backend) + } + continue + } + if current != "training" && current != "holdout" && current != "diagnostic" { + return "", fmt.Errorf("%s/%s/%s has unsupported qualification split %q", key.dataset, key.name, key.backend, current) + } + if split != "" && split != current { + return "", fmt.Errorf("%s/%s/%s changes qualification split across artifacts", key.dataset, key.name, key.backend) + } + split = current + found = true + } + } + if !found { + return "legacy", nil + } + return split, nil +} + +// prioritizedTraversalCategory identifies result families introduced by the +// traversal-priority qualification program. Their split remains mandatory +// even when an artifact was assembled outside the scale-corpus loader. +func prioritizedTraversalCategory(category string) bool { + switch category { + case "generated_shortest_path_v2", "generated_all_shortest_path_v2", "expand_into_one_hop", "generated_endpoint_seeded_expansion", "generated_fixed_suffix_expansion_v2", "orientation_shadow": + return true + default: + return false + } +} + +// prioritizedTraversalRecord also recognizes the fixed-suffix v2 and +// boundary datasets whose category intentionally remains compatible with the +// original corpus. Artifact consumers must not mistake that shared category +// for permission to omit the frozen qualification split. +func prioritizedTraversalRecord(record CaseResult) bool { + if prioritizedTraversalCategory(record.Category) { + return true + } + + return record.Category == "generated_fixed_suffix_expansion" && + (strings.HasPrefix(record.Dataset, "generated_fixed_suffix_expansion_v2_") || + strings.HasPrefix(record.Dataset, "generated_fixed_suffix_expansion_v3_") || + strings.HasPrefix(record.Name, "GFSE-V2-") || + strings.HasPrefix(record.Name, "GFSE-V3-") || + strings.HasPrefix(record.Name, "GFSE-BOUNDARY-")) +} + +// prioritizedTraversalKey reports whether either artifact identifies a +// matched performance key as part of the traversal qualification program. +// Looking at both artifacts makes the gate fail closed if one side drops or +// changes the category while preserving the logical case identity. +func prioritizedTraversalKey(key performanceKey, artifacts ...[]CaseResult) bool { + for _, records := range artifacts { + for _, record := range records { + if record.Dataset == key.dataset && record.Name == key.name && record.ExecutionMode == key.backend && prioritizedTraversalRecord(record) { + return true + } + } + } + + return false +} + +// TraversalQualificationStatus reports independent selector-training and +// frozen-holdout coverage for one concrete traversal candidate family. +type TraversalQualificationStatus struct { + Family string `json:"family"` + TrainingCases int `json:"training_cases"` + HoldoutCases int `json:"holdout_cases"` + TrainingPassed bool `json:"training_passed"` + HoldoutPassed bool `json:"holdout_passed"` + Passed bool `json:"passed"` +} + +// traversalQualificationFamily returns the most specific stable candidate +// identity available for a matched key. Candidate/right artifacts take +// precedence over incumbent/left artifacts. A conservative semantic family +// remains available for externally assembled artifacts without optimizer or +// runtime telemetry. +func traversalQualificationFamily(key performanceKey, artifacts ...[]CaseResult) string { + for artifactIdx := len(artifacts) - 1; artifactIdx >= 0; artifactIdx-- { + for _, record := range artifacts[artifactIdx] { + if record.Dataset != key.dataset || record.Name != key.name || record.ExecutionMode != key.backend { + continue + } + if record.TraversalTelemetry != nil { + summary := record.TraversalTelemetry.Summary + for _, identity := range []string{summary.EmittedIdentity, summary.SelectorVersion} { + if isOrientationProbePolicy(identity) { + return identity + } + } + if identity := summary.RequestedIdentity; prioritizedTraversalIdentity(identity) { + branch := summary.RuntimeBranch + if branch != "" && branch != "runtime_outcome_unavailable" && branch != "mixed" { + return identity + "@" + branch + } + return identity + } + } + if record.Optimization != nil { + for outcomeIdx := len(record.Optimization.TargetOutcomes) - 1; outcomeIdx >= 0; outcomeIdx-- { + outcome := record.Optimization.TargetOutcomes[outcomeIdx] + for _, identity := range []string{outcome.Candidate, outcome.EmittedPolicy, outcome.PlannedPolicy, outcome.Applied, outcome.Selected} { + if prioritizedTraversalIdentity(identity) { + return identity + } + } + } + } + } + } + for artifactIdx := len(artifacts) - 1; artifactIdx >= 0; artifactIdx-- { + for _, record := range artifacts[artifactIdx] { + if record.Dataset != key.dataset || record.Name != key.name || record.ExecutionMode != key.backend || record.Optimization == nil { + continue + } + for _, outcome := range record.Optimization.TargetOutcomes { + if outcome.TargetKind != "" && outcome.TargetKind != "traversal" { + continue + } + if outcome.Family == "SP" || outcome.Family == "ASP" || strings.Contains(outcome.Family, "expansion") { + return outcome.Family + } + } + } + } + + for _, records := range artifacts { + for _, record := range records { + if record.Dataset != key.dataset || record.Name != key.name || record.ExecutionMode != key.backend { + continue + } + if strings.HasPrefix(record.Dataset, "generated_fixed_suffix_expansion_v3_") || strings.HasPrefix(record.Name, "GFSE-V3-") { + return string(optimize.ExpansionSearchPolicyOrientationProbeV2) + } + switch record.Category { + case "generated_shortest_path_v2", "generated_all_shortest_path_v2": + if strings.Contains(strings.ToLower(record.Cypher), "allshortestpaths") || strings.Contains(strings.ToLower(record.Name), "all-shortest") { + return "ASP" + } + return "SP" + case "generated_endpoint_seeded_expansion": + return "fixed_prefix_terminal_expansion" + case "generated_fixed_suffix_expansion", "generated_fixed_suffix_expansion_v2", "orientation_shadow": + return "orientation-probe-v1" + case "expand_into_one_hop": + return "expand-into-study-v1" + } + } + } + + return "prioritized_traversal" +} + +// validateCandidateRuntimeEvidence rejects performance attribution to an +// experimental traversal arm unless every warm sample is bound to one +// singular, non-fallback runtime outcome for that measured invocation. A +// same-case diagnostic replay is useful resource evidence but is not allowed +// to attest latency samples because concurrent graph changes or cap outcomes +// could select a different branch. +func validateCandidateRuntimeEvidence(records []CaseResult, key performanceKey) error { + for _, record := range records { + if record.Dataset != key.dataset || record.Name != key.name || record.ExecutionMode != key.backend || !requiresCandidateRuntimeEvidence(record) { + continue + } + if record.TraversalTelemetry == nil { + return fmt.Errorf("candidate traversal has no runtime telemetry") + } + summary := record.TraversalTelemetry.Summary + if summary.RuntimeOutcomeAvailable == nil || !*summary.RuntimeOutcomeAvailable || summary.RuntimeIdentity == "" || summary.RuntimeBranch == "" || summary.RuntimeBranch == "mixed" || summary.RuntimeBranch == "runtime_outcome_unavailable" { + return fmt.Errorf("candidate traversal runtime outcome is unavailable or mixed") + } + if summary.FallbackExecuted == nil { + return fmt.Errorf("candidate traversal fallback outcome is unavailable") + } + if *summary.FallbackExecuted { + return fmt.Errorf("candidate traversal executed exact fallback %q", summary.FallbackIdentity) + } + for _, sample := range record.Stats.Samples { + if sample.Classification != "warm" || sample.Duration <= 0 { + continue + } + if sample.RequestedIdentity != summary.RequestedIdentity || sample.RuntimeIdentity != summary.RuntimeIdentity || sample.RuntimeBranch != summary.RuntimeBranch || sample.FallbackExecuted == nil || *sample.FallbackExecuted || sample.RuntimeAttestation != "timed_invocation" { + return fmt.Errorf("warm sample lacks matching singular runtime attribution") + } + if err := validateRuntimeReceiptEvents(sample.RuntimeReceiptEvents, sample.RuntimeIdentity, sample.RuntimeBranch, sample.FallbackExecuted); err != nil { + return fmt.Errorf("warm sample runtime receipt chain: %w", err) + } + } + } + return nil +} + +func validateRuntimeReceiptEvents(events []RuntimeReceiptEvent, runtimeIdentity, runtimeBranch string, fallbackExecuted *bool) error { + if len(events) == 0 { + return fmt.Errorf("event chain is missing") + } + for idx, event := range events { + if event.Ordinal != idx+1 || event.RuntimeIdentity == "" || event.RuntimeBranch == "" { + return fmt.Errorf("event chain is not contiguous") + } + } + terminal := events[len(events)-1] + if terminal.RuntimeIdentity != runtimeIdentity || terminal.RuntimeBranch != runtimeBranch { + return fmt.Errorf("terminal event does not match runtime outcome") + } + if fallbackExecuted == nil || terminal.FallbackExecuted != *fallbackExecuted { + return fmt.Errorf("terminal event does not match fallback outcome") + } + return nil +} + +func runtimeReceiptChains(samples []LatencySample) [][]RuntimeReceiptEvent { + chains := make([][]RuntimeReceiptEvent, 0) + for _, sample := range samples { + if len(sample.RuntimeReceiptEvents) == 0 { + continue + } + chains = append(chains, append([]RuntimeReceiptEvent(nil), sample.RuntimeReceiptEvents...)) + } + return chains +} + +func caseRuntimeReceiptChains(records []CaseResult, key performanceKey) [][]RuntimeReceiptEvent { + chains := make([][]RuntimeReceiptEvent, 0) + for _, record := range records { + if record.Dataset == key.dataset && record.Name == key.name && record.ExecutionMode == key.backend { + chains = append(chains, runtimeReceiptChains(record.Stats.Samples)...) + } + } + return chains +} + +func requiresCandidateRuntimeEvidence(record CaseResult) bool { + if record.TraversalTelemetry != nil { + summary := record.TraversalTelemetry.Summary + if isOrientationProbePolicy(summary.EmittedIdentity) || isOrientationProbePolicy(summary.SelectorVersion) { + return true + } + requested := summary.RequestedIdentity + if strings.HasPrefix(requested, "SP-B") || strings.HasPrefix(requested, "ASP-B") || isOrientationProbePolicy(requested) { + return true + } + } + if record.Optimization == nil { + return false + } + for _, outcome := range record.Optimization.TargetOutcomes { + for _, identity := range []string{outcome.Candidate, outcome.EmittedPolicy, outcome.Selected} { + if strings.HasPrefix(identity, "SP-B") || strings.HasPrefix(identity, "ASP-B") || isOrientationProbePolicy(identity) { + return true + } + } + } + return false +} + +func prioritizedTraversalIdentity(identity string) bool { + return strings.HasPrefix(identity, "SP-") || + strings.HasPrefix(identity, "ASP-") || + strings.HasPrefix(identity, "EXPANSION-") || + isOrientationProbePolicy(identity) +} + +// promotionTimingSplit reports whether a frozen qualification partition may +// contribute timing evidence to a promotion decision. Diagnostic records are +// still checked for correctness and resource behavior, but never tune or +// qualify a production selector. +func promotionTimingSplit(split string) bool { + return split != "diagnostic" +} + +type pairedRoundEvidence struct { + Block int + ArmOrder int + RunUUID string + Arm string + Warmups int +} + +// validatePairedOrderEvidence verifies matched block identity and balanced two-arm ordering for the requested rounds. +func validatePairedOrderEvidence(left, right []CaseResult, key performanceKey, rounds []int, minimumWarmups int) error { + leftEvidence, err := collectPairedRoundEvidence(left, key) + if err != nil { + return err + } + rightEvidence, err := collectPairedRoundEvidence(right, key) + if err != nil { + return err + } + leftFirst := 0 + for _, round := range rounds { + leftRound, leftOK := leftEvidence[round] + rightRound, rightOK := rightEvidence[round] + if !leftOK || !rightOK { + return fmt.Errorf("%s/%s round %d lacks paired order evidence", key.dataset, key.name, round) + } + if leftRound.Warmups < minimumWarmups || rightRound.Warmups < minimumWarmups { + return fmt.Errorf("%s/%s round %d requires at least %d warmups per arm, got %d/%d", key.dataset, key.name, round, minimumWarmups, leftRound.Warmups, rightRound.Warmups) + } + if leftRound.Block < 1 || leftRound.Block != rightRound.Block { + return fmt.Errorf("%s/%s round %d has missing or mismatched paired block", key.dataset, key.name, round) + } + if leftRound.RunUUID == "" || leftRound.RunUUID != rightRound.RunUUID { + return fmt.Errorf("%s/%s round %d has missing or mismatched paired run UUID", key.dataset, key.name, round) + } + if leftRound.Arm == "" || rightRound.Arm == "" || leftRound.Arm == "unlabeled" || rightRound.Arm == "unlabeled" || leftRound.Arm == rightRound.Arm { + return fmt.Errorf("%s/%s round %d has missing or indistinct arm identity", key.dataset, key.name, round) + } + if !((leftRound.ArmOrder == 1 && rightRound.ArmOrder == 2) || (leftRound.ArmOrder == 2 && rightRound.ArmOrder == 1)) { + return fmt.Errorf("%s/%s round %d lacks a complete two-arm order", key.dataset, key.name, round) + } + if leftRound.ArmOrder == 1 { + leftFirst++ + } + } + rightFirst := len(rounds) - leftFirst + if leftFirst-rightFirst > 1 || rightFirst-leftFirst > 1 { + return fmt.Errorf("%s/%s paired arm order is not balanced: %d/%d", key.dataset, key.name, leftFirst, rightFirst) + } + + return nil +} + +func collectPairedRoundEvidence(records []CaseResult, key performanceKey) (map[int]pairedRoundEvidence, error) { + evidence := map[int]pairedRoundEvidence{} + for _, record := range records { + if record.Dataset != key.dataset || record.Name != key.name || record.ExecutionMode != key.backend { + continue + } + warmups := record.Stats.WarmupIterations + if record.Environment != nil { + if warmups != 0 && record.Environment.WarmupIterations != 0 && warmups != record.Environment.WarmupIterations { + return nil, fmt.Errorf("%s/%s has inconsistent warmup evidence", key.dataset, key.name) + } + if warmups == 0 { + warmups = record.Environment.WarmupIterations + } + } + for _, sample := range record.Stats.Samples { + if sample.Classification != "warm" || sample.Duration <= 0 { + continue + } + round := sample.Round + current := pairedRoundEvidence{ + Block: sample.Block, ArmOrder: sample.ArmOrder, RunUUID: sample.RunUUID, Arm: sample.Arm, Warmups: warmups, + } + if record.Environment != nil { + if round == 0 { + round = record.Environment.Round + } + if current.Block == 0 { + current.Block = record.Environment.Block + } + if current.ArmOrder == 0 { + current.ArmOrder = record.Environment.ArmOrder + } + if current.RunUUID == "" { + current.RunUUID = record.Environment.RunUUID + } + if current.Arm == "" { + current.Arm = record.Environment.Arm + } + } + if round < 1 { + return nil, fmt.Errorf("%s/%s has warm sample without a round", key.dataset, key.name) + } + if prior, found := evidence[round]; found && prior != current { + return nil, fmt.Errorf("%s/%s round %d has inconsistent paired order metadata", key.dataset, key.name, round) + } + evidence[round] = current + } + } + return evidence, nil +} + +// sortedPerformanceKeys returns stable keys from a set. +func sortedPerformanceKeys(values map[performanceKey]struct{}) []performanceKey { + keys := make([]performanceKey, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { + if keys[i].dataset != keys[j].dataset { + return keys[i].dataset < keys[j].dataset + } + if keys[i].name != keys[j].name { + return keys[i].name < keys[j].name + } + return keys[i].backend < keys[j].backend + }) + return keys +} diff --git a/cmd/graphbench/summary.go b/cmd/graphbench/summary.go index ba21fd9a..53928495 100644 --- a/cmd/graphbench/summary.go +++ b/cmd/graphbench/summary.go @@ -24,51 +24,128 @@ import ( "sort" "strings" "time" + + "github.com/specterops/dawgs/testutil" ) +// Summary aggregates benchmark records into cases, modes, improvements, and cost models. type Summary struct { - GeneratedAt time.Time `json:"generated_at"` - Modes []ModeSummary `json:"modes"` - Cases []CaseSummary `json:"cases"` - Regressions []BaselineEntry `json:"regressions,omitempty"` + // GeneratedAt records when the summary was assembled. + GeneratedAt time.Time `json:"generated_at"` + // Metadata captures build and baseline metadata. + Metadata testutil.BaselineMetadata `json:"metadata"` + // Modes lists aggregate mode summaries in deterministic report order. + Modes []ModeSummary `json:"modes"` + // Cases contains per-workload aggregates in deterministic report order. + Cases []CaseSummary `json:"cases"` + // Regressions lists baseline comparisons classified as regressions. + Regressions []BaselineEntry `json:"regressions,omitempty"` + // Improvements lists baseline comparisons classified as improvements. Improvements []BaselineEntry `json:"improvements,omitempty"` + // CostModels lists per-case client/backend latency attribution models. + CostModels []CostModelCase `json:"cost_models,omitempty"` +} + +// CostModelCase attributes one case's end-to-end latency across compile and backend boundary components. +type CostModelCase struct { + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // Boundary identifies the measured execution boundary. + Boundary string `json:"boundary"` + // E2EMedian records median end-to-end latency attributed by the cost model. + E2EMedian time.Duration `json:"e2e_median"` + // Attribution reports the fraction of median end-to-end latency explained by measured components. + Attribution float64 `json:"attribution"` + // Components lists cost-model components in display order. + Components []CostModelComponent `json:"components"` } +// CostModelComponent attributes a duration and share to one benchmark boundary component. +type CostModelComponent struct { + // Name labels the measured latency component shown in the cost model. + Name string `json:"name"` + // Interval states whether the component is exclusive, derived, or inclusive and overlapping. + Interval string `json:"interval"` + // Median records the median observed duration. + Median time.Duration `json:"median"` + // P95 records the component's 95th-percentile observed duration. + P95 time.Duration `json:"p95"` + // Rows records the result cardinality observed alongside the component measurement. + Rows int64 `json:"rows,omitempty"` + // ShareOfE2E reports this component's fraction of end-to-end latency. + ShareOfE2E float64 `json:"share_of_e2e,omitempty"` + // Confidence describes whether the component is directly observed, derived, or diagnostic. + Confidence string `json:"confidence"` +} + +// ModeSummary aggregates sample and latency statistics for one execution mode. type ModeSummary struct { - Mode ExecutionMode `json:"mode"` - Total int `json:"total"` - OK int `json:"ok"` - RowMismatch int `json:"row_mismatch"` - Error int `json:"error"` - NotImplemented int `json:"not_implemented"` + // Mode identifies the backend whose result statuses are aggregated. + Mode ExecutionMode `json:"mode"` + // Total counts all results emitted for the execution mode. + Total int `json:"total"` + // OK counts successful results for an execution mode. + OK int `json:"ok"` + // RowMismatch counts results whose row cardinality differed from expectation. + RowMismatch int `json:"row_mismatch"` + // Error counts results that failed during backend execution. + Error int `json:"error"` + // NotImplemented counts cases unsupported by the execution mode. + NotImplemented int `json:"not_implemented"` } +// CaseSummary aggregates all backend results for one dataset case. type CaseSummary struct { - Source string `json:"source"` - Dataset string `json:"dataset"` - Name string `json:"name"` - Category string `json:"category"` - Modes map[ExecutionMode]ModeCaseCell `json:"modes"` + // Source identifies the source corpus file. + Source string `json:"source"` + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // Category groups cases by workload category. + Category string `json:"category"` + // Modes maps execution mode to its status, statistics, and baseline comparison. + Modes map[ExecutionMode]ModeCaseCell `json:"modes"` } +// ModeCaseCell contains the status, statistics, and baseline comparison rendered in one summary cell. type ModeCaseCell struct { - Status string `json:"status"` - Rows int64 `json:"rows,omitempty"` - Median time.Duration `json:"median,omitempty"` - Baseline *BaselineComparison `json:"baseline,omitempty"` - FallbackReason string `json:"fallback_reason,omitempty"` - Error string `json:"error,omitempty"` + // Status records the execution outcome. + Status string `json:"status"` + // Rows records the row count returned for this case and execution mode. + Rows int64 `json:"rows,omitempty"` + // Median records the median observed duration. + Median time.Duration `json:"median,omitempty"` + // Baseline contains the latency comparison with a matching baseline record. + Baseline *BaselineComparison `json:"baseline,omitempty"` + // FallbackReason explains why execution used a fallback architecture. + FallbackReason string `json:"fallback_reason,omitempty"` + // Error records the failure message when the operation did not succeed. + Error string `json:"error,omitempty"` + // RuntimeReceiptChains preserves every measured invocation's complete + // ordered traversal branch chain. + RuntimeReceiptChains [][]RuntimeReceiptEvent `json:"runtime_receipt_chains,omitempty"` } +// BaselineEntry stores one case/backend baseline median used for future comparison. type BaselineEntry struct { - Dataset string `json:"dataset"` - Name string `json:"name"` - Mode ExecutionMode `json:"mode"` + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // Mode identifies the backend to which the baseline comparison applies. + Mode ExecutionMode `json:"mode"` + // BaselineMedian records the median latency loaded from the comparison baseline. BaselineMedian time.Duration `json:"baseline_median"` - CurrentMedian time.Duration `json:"current_median"` - Ratio float64 `json:"ratio"` + // CurrentMedian records the median latency measured by the current run. + CurrentMedian time.Duration `json:"current_median"` + // Ratio reports the candidate-to-baseline latency ratio. + Ratio float64 `json:"ratio"` } +// buildSummary aggregates benchmark records by case and mode and derives boundary cost models. func buildSummary(records []CaseResult) Summary { var ( summary = Summary{ @@ -79,6 +156,9 @@ func buildSummary(records []CaseResult) Summary { ) for _, record := range records { + if summary.Metadata == (testutil.BaselineMetadata{}) { + summary.Metadata = record.Metadata + } modeSummary := modeSummaries[record.ExecutionMode] if modeSummary == nil { modeSummary = &ModeSummary{Mode: record.ExecutionMode} @@ -114,12 +194,13 @@ func buildSummary(records []CaseResult) Summary { } caseSummary.Modes[record.ExecutionMode] = ModeCaseCell{ - Status: record.Status, - Rows: record.RowCount, - Median: record.Stats.Median, - Baseline: record.Baseline, - FallbackReason: record.FallbackReason, - Error: record.Error, + Status: record.Status, + Rows: record.RowCount, + Median: record.Stats.Median, + Baseline: record.Baseline, + FallbackReason: record.FallbackReason, + Error: record.Error, + RuntimeReceiptChains: runtimeReceiptChains(record.Stats.Samples), } if record.Baseline != nil { @@ -137,6 +218,9 @@ func buildSummary(records []CaseResult) Summary { summary.Improvements = append(summary.Improvements, entry) } } + if record.RawPGXWaterfall != nil && len(record.RawPGXWaterfall.Samples) > 0 { + summary.CostModels = append(summary.CostModels, buildBoundaryCostModel(record)) + } } for _, modeSummary := range modeSummaries { @@ -163,9 +247,119 @@ func buildSummary(records []CaseResult) Summary { sortBaselineEntries(summary.Regressions, true) sortBaselineEntries(summary.Improvements, false) + sort.Slice(summary.CostModels, func(i, j int) bool { + if summary.CostModels[i].Dataset != summary.CostModels[j].Dataset { + return summary.CostModels[i].Dataset < summary.CostModels[j].Dataset + } + return summary.CostModels[i].Name < summary.CostModels[j].Name + }) return summary } +// buildBoundaryCostModel attributes end-to-end latency among compile, driver, planning, execution, and decode stages. +func buildBoundaryCostModel(record CaseResult) CostModelCase { + samples := record.RawPGXWaterfall.Samples + total := boundaryDurations(samples, func(sample BoundarySample) time.Duration { return sample.Total }) + e2e := durationFromQuantile(total, 0.50) + components := []struct { + // name labels the latency component in the rendered cost model. + name string + // values contains the observed durations attributed to the component. + values []time.Duration + }{ + { + name: "Pool acquisition", + values: boundaryDurations(samples, func(sample BoundarySample) time.Duration { return sample.PoolWait }), + }, + { + name: "Transaction setup", + values: boundaryDurations(samples, func(sample BoundarySample) time.Duration { return sample.Transaction }), + }, + { + name: "Bind/prepare", + values: boundaryDurations(samples, func(sample BoundarySample) time.Duration { return sample.BindPrepare }), + }, + { + name: "First-row transfer/decode", + values: boundaryDurations(samples, func(sample BoundarySample) time.Duration { return sample.FirstRow }), + }, + { + name: "Remaining transfer/decode", + values: boundaryDurations(samples, func(sample BoundarySample) time.Duration { return sample.AllRowsDecode }), + }, + { + name: "Drain/close", + values: boundaryDurations(samples, func(sample BoundarySample) time.Duration { return sample.DrainClose }), + }, + } + model := CostModelCase{ + Dataset: record.Dataset, + Name: record.Name, + Boundary: record.RawPGXWaterfall.Boundary, + E2EMedian: e2e, + } + var attributed time.Duration + for _, component := range components { + median := durationFromQuantile(component.values, 0.50) + attributed += median + model.Components = append(model.Components, CostModelComponent{ + Name: component.name, + Interval: "exclusive", + Median: median, + P95: durationFromQuantile(component.values, 0.95), + Rows: samples[0].Rows, + ShareOfE2E: durationShare(median, e2e), + Confidence: "raw-pgx observed boundary", + }) + } + residual := e2e - attributed + if residual < 0 { + residual = 0 + } + model.Components = append(model.Components, CostModelComponent{ + Name: "Unexplained residual", + Interval: "derived", + Median: residual, + ShareOfE2E: durationShare(residual, e2e), + Confidence: "derived", + }) + model.Attribution = durationShare(e2e-residual, e2e) + if record.PostgresMetrics != nil && record.PostgresMetrics.ExecutionMS != nil { + server := time.Duration(*record.PostgresMetrics.ExecutionMS * float64(time.Millisecond)) + model.Components = append(model.Components, CostModelComponent{ + Name: "Server execution", + Interval: "inclusive/overlapping", + Median: server, + ShareOfE2E: durationShare(server, e2e), + Confidence: "single EXPLAIN diagnostic", + }) + } + return model +} + +// boundaryDurations extracts positive boundary-stage durations from benchmark samples. +func boundaryDurations(samples []BoundarySample, selectDuration func(BoundarySample) time.Duration) []time.Duration { + values := make([]time.Duration, len(samples)) + for idx, sample := range samples { + values[idx] = selectDuration(sample) + } + return values +} + +// durationFromQuantile converts a floating-point duration quantile to time.Duration. +func durationFromQuantile(values []time.Duration, probability float64) time.Duration { + return time.Duration(durationQuantile(values, probability)) +} + +// durationShare returns a component's fraction of total latency. +func durationShare(component, total time.Duration) float64 { + if total <= 0 { + return 0 + } + return float64(component) / float64(total) +} + +// sortBaselineEntries orders baseline entries by dataset, case, and execution mode. func sortBaselineEntries(entries []BaselineEntry, descending bool) { sort.Slice(entries, func(i, j int) bool { if descending { @@ -176,6 +370,7 @@ func sortBaselineEntries(entries []BaselineEntry, descending bool) { }) } +// writeMarkdownSummaryFile creates a Markdown summary file and propagates write or close failures. func writeMarkdownSummaryFile(path string, summary Summary) error { if err := ensureOutputDir(path); err != nil { return err @@ -190,6 +385,7 @@ func writeMarkdownSummaryFile(path string, summary Summary) error { return writeMarkdownSummary(output, summary) } +// writeJSONSummaryFile creates a JSON summary file and propagates encode or close failures. func writeJSONSummaryFile(path string, summary Summary) error { if err := ensureOutputDir(path); err != nil { return err @@ -206,9 +402,11 @@ func writeJSONSummaryFile(path string, summary Summary) error { return encoder.Encode(summary) } +// writeMarkdownSummary renders benchmark overview, case matrix, improvements, and cost models as Markdown. func writeMarkdownSummary(w io.Writer, summary Summary) error { fmt.Fprintf(w, "# GraphBench Summary\n\n") fmt.Fprintf(w, "Generated: %s\n\n", summary.GeneratedAt.Format(time.RFC3339)) + fmt.Fprintf(w, "DAWGS version: `%s`\n\n", summary.Metadata.DAWGSVersion) fmt.Fprintf(w, "## Modes\n\n") fmt.Fprintf(w, "| Mode | Total | OK | Row Mismatch | Error | Not Implemented |\n") @@ -246,10 +444,23 @@ func writeMarkdownSummary(w io.Writer, summary Summary) error { fmt.Fprintf(w, "\n## Baseline Improvements\n\n") writeBaselineTable(w, summary.Improvements) } + if len(summary.CostModels) > 0 { + fmt.Fprintf(w, "\n## Raw PostgreSQL Cost Models\n\n") + for _, model := range summary.CostModels { + fmt.Fprintf(w, "### %s / %s\n\n", escapeMarkdown(model.Dataset), escapeMarkdown(model.Name)) + fmt.Fprintf(w, "Boundary attribution: %.1f%% of %s.\n\n", model.Attribution*100, formatDuration(model.E2EMedian)) + fmt.Fprintf(w, "| Component | Interval | Median | p95 | Share of E2E | Confidence |\n") + fmt.Fprintf(w, "| --- | --- | ---: | ---: | ---: | --- |\n") + for _, component := range model.Components { + fmt.Fprintf(w, "| %s | %s | %s | %s | %.1f%% | %s |\n", escapeMarkdown(component.Name), component.Interval, formatDuration(component.Median), formatDuration(component.P95), component.ShareOfE2E*100, escapeMarkdown(component.Confidence)) + } + } + } return nil } +// writeBaselineTable renders baseline comparisons for one summary section. func writeBaselineTable(w io.Writer, entries []BaselineEntry) { fmt.Fprintf(w, "| Case | Dataset | Mode | Baseline | Current | Ratio |\n") fmt.Fprintf(w, "| --- | --- | --- | ---: | ---: | ---: |\n") @@ -265,6 +476,7 @@ func writeBaselineTable(w io.Writer, entries []BaselineEntry) { } } +// formatModeCell formats one backend result and its baseline comparison for Markdown. func formatModeCell(cell ModeCaseCell) string { if cell.Status == "" { return "-" @@ -296,6 +508,7 @@ func formatModeCell(cell ModeCaseCell) string { return escapeMarkdown(strings.Join(parts, "; ")) } +// formatDuration formats a duration for compact benchmark tables. func formatDuration(duration time.Duration) string { ms := float64(duration.Microseconds()) / 1000.0 if ms < 1 { @@ -308,6 +521,7 @@ func formatDuration(duration time.Duration) string { return fmt.Sprintf("%.0fms", ms) } +// escapeMarkdown escapes table delimiters and normalizes line breaks for Markdown cells. func escapeMarkdown(value string) string { return strings.ReplaceAll(value, "|", "\\|") } diff --git a/cmd/graphbench/summary_test.go b/cmd/graphbench/summary_test.go index e7a080bd..acb5b7a4 100644 --- a/cmd/graphbench/summary_test.go +++ b/cmd/graphbench/summary_test.go @@ -25,6 +25,7 @@ import ( "github.com/stretchr/testify/require" ) +// TestApplyBaseline verifies that matching dataset/name/backend records receive the expected 1.5 ratio and five-millisecond absolute change. func TestApplyBaseline(t *testing.T) { var ( dir = t.TempDir() @@ -57,6 +58,7 @@ func TestApplyBaseline(t *testing.T) { require.Equal(t, 5*time.Millisecond, records[0].Baseline.Change) } +// TestBuildSummarySortsCaseSourceTieBreaker verifies deterministic source-path ordering when dataset, case name, and backend keys are otherwise identical. func TestBuildSummarySortsCaseSourceTieBreaker(t *testing.T) { summary := buildSummary([]CaseResult{ { @@ -80,6 +82,7 @@ func TestBuildSummarySortsCaseSourceTieBreaker(t *testing.T) { require.Equal(t, "cases/b.json", summary.Cases[1].Source) } +// TestWriteMarkdownSummary verifies that one row combines PostgreSQL timing/cardinality with an unavailable local-executor status and leaves absent backends blank. func TestWriteMarkdownSummary(t *testing.T) { var ( summary = buildSummary([]CaseResult{ @@ -110,3 +113,32 @@ func TestWriteMarkdownSummary(t *testing.T) { require.NoError(t, writeMarkdownSummary(&output, summary)) require.Contains(t, output.String(), "| case | base | counts | 2.0ms; rows=1 | not_implemented; local traversal executor unavailable | - |") } + +// TestBuildSummaryIncludesExclusiveRawPGXCostModel verifies that mutually exclusive boundary components reconcile to total latency and retain an explicit residual component. +func TestBuildSummaryIncludesExclusiveRawPGXCostModel(t *testing.T) { + record := CaseResult{ + Dataset: "base", + Name: "large", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + RawPGXWaterfall: &PostgresBoundaryWaterfall{ + Boundary: "raw", + Samples: []BoundarySample{{ + PoolWait: time.Millisecond, + Transaction: time.Millisecond, + BindPrepare: 2 * time.Millisecond, + FirstRow: 2 * time.Millisecond, + AllRowsDecode: 3 * time.Millisecond, + DrainClose: time.Millisecond, + Total: 10 * time.Millisecond, + Rows: 1000, + }}, + }, + } + + summary := buildSummary([]CaseResult{record}) + require.Len(t, summary.CostModels, 1) + require.Equal(t, 10*time.Millisecond, summary.CostModels[0].E2EMedian) + require.InDelta(t, 1.0, summary.CostModels[0].Attribution, 0.0001) + require.Equal(t, "Unexplained residual", summary.CostModels[0].Components[6].Name) +} diff --git a/cmd/graphbench/traversal_telemetry.go b/cmd/graphbench/traversal_telemetry.go new file mode 100644 index 00000000..bcc679da --- /dev/null +++ b/cmd/graphbench/traversal_telemetry.go @@ -0,0 +1,656 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "slices" + "strings" +) + +const ( + // TraversalExecutionTelemetrySchemaVersion is the current serialized telemetry schema revision. + TraversalExecutionTelemetrySchemaVersion = 2 + + // TraversalTelemetryLevelSummary records only the production execution identity and outcome. + TraversalTelemetryLevelSummary TraversalTelemetryLevel = "summary" + // TraversalTelemetryLevelDiagnostic adds counters from a separate untimed replay. + TraversalTelemetryLevelDiagnostic TraversalTelemetryLevel = "diagnostic" + + // TraversalTelemetryCounterStatusComplete records a replay with every declared family populated by invocation-local counters. + TraversalTelemetryCounterStatusComplete TraversalTelemetryCounterStatus = "complete" + // TraversalTelemetryCounterStatusPlanPartial records honest SQL-visible EXPLAIN evidence that is insufficient for qualification. + TraversalTelemetryCounterStatusPlanPartial TraversalTelemetryCounterStatus = "plan_derived_partial" + // TraversalTelemetryCounterStatusHiddenUnavailable records a function-backed executor whose internal work counters were unavailable. + TraversalTelemetryCounterStatusHiddenUnavailable TraversalTelemetryCounterStatus = "hidden_counters_unavailable" + + // TraversalTelemetryFamilyOrdinary identifies ordinary DFS or recursive-CTE traversal work. + TraversalTelemetryFamilyOrdinary TraversalTelemetryFamily = "ordinary" + // TraversalTelemetryFamilyOrientation identifies runtime orientation-policy work. + TraversalTelemetryFamilyOrientation TraversalTelemetryFamily = "orientation" + // TraversalTelemetryFamilySP identifies singleton shortest-path work. + TraversalTelemetryFamilySP TraversalTelemetryFamily = "shortest_path" + // TraversalTelemetryFamilyASP identifies all-shortest-path work. + TraversalTelemetryFamilyASP TraversalTelemetryFamily = "all_shortest_paths" + // TraversalTelemetryFamilyHydration identifies post-discovery path hydration work. + TraversalTelemetryFamilyHydration TraversalTelemetryFamily = "hydration" + // TraversalTelemetryFamilyWorkspace identifies measured session and pool workspace high-water marks. + TraversalTelemetryFamilyWorkspace TraversalTelemetryFamily = "workspace" +) + +// TraversalTelemetryLevel identifies whether a record contains only lightweight summary data or an untimed diagnostic replay. +type TraversalTelemetryLevel string + +// TraversalTelemetryFamily identifies a counter family required for an invocation. +type TraversalTelemetryFamily string + +// TraversalTelemetryCounterStatus identifies whether an untimed replay exposes every required invocation-local counter. +type TraversalTelemetryCounterStatus string + +// TraversalExecutionTelemetry records versioned execution identity and optional diagnostic replay counters. +type TraversalExecutionTelemetry struct { + // SchemaVersion identifies the serialized telemetry schema revision. + SchemaVersion int `json:"schema_version"` + // Level identifies the instrumentation boundary represented by this record. + Level TraversalTelemetryLevel `json:"level"` + // Summary contains lightweight data captured for the production invocation. + Summary TraversalExecutionSummary `json:"summary"` + // Diagnostic contains counters from a separate untimed replay when Level is diagnostic. + Diagnostic *TraversalExecutionDiagnostic `json:"diagnostic,omitempty"` +} + +// TraversalExecutionSummary identifies the planned and executed traversal policy without detailed work counters. +type TraversalExecutionSummary struct { + RequestedIdentity string `json:"requested_identity"` + PlannedIdentities []string `json:"planned_identities"` + EmittedIdentity string `json:"emitted_identity"` + RuntimeIdentity string `json:"runtime_identity"` + AppliedIdentity string `json:"applied_identity"` + SelectorVersion string `json:"selector_version"` + SchedulerVersion string `json:"scheduler_version"` + ExecutionBoundary string `json:"execution_boundary,omitempty"` + // ObservationMode identifies whether the public boundary consumes scalar, + // ordered-ID, or hydrated path values. + ObservationMode string `json:"observation_mode,omitempty"` + Caps map[string]int64 `json:"caps"` + // RuntimeOutcomeAvailable distinguishes executor evidence from a + // translator prediction. When false, runtime-dependent facts stay unset. + RuntimeOutcomeAvailable *bool `json:"runtime_outcome_available,omitempty"` + RuntimeBranch string `json:"runtime_branch"` + Overflow *bool `json:"overflow"` + FallbackExecuted *bool `json:"fallback_executed"` + FallbackIdentity string `json:"fallback_identity,omitempty"` + // WouldSelectIdentity records a shadow policy choice while RuntimeIdentity + // and AppliedIdentity remain bound to the only executed incumbent arm. + WouldSelectIdentity string `json:"would_select_identity,omitempty"` + // Provenance maps summary field paths to the optimizer, SQL branch, function, or executor fact that produced them. + Provenance map[string]string `json:"provenance"` +} + +// TraversalExecutionDiagnostic contains counters from one tool-only replay, separate from all timed samples. +type TraversalExecutionDiagnostic struct { + // InvocationID uniquely identifies the diagnostic invocation and its session-local workspace. + InvocationID string `json:"invocation_id"` + // ConnectionID identifies the same backend connection used by the production invocation. + ConnectionID string `json:"connection_id"` + // TimedSample is required and must be false so replay resources cannot be attributed to latency samples. + TimedSample *bool `json:"timed_sample"` + // RequiredFamilies declares exactly which counter groups must be complete for this invocation. + RequiredFamilies []TraversalTelemetryFamily `json:"required_families"` + Counters TraversalDiagnosticCounters `json:"counters"` + // CounterStatus distinguishes qualification-complete invocation metrics from partial plan evidence or opaque function work. + CounterStatus TraversalTelemetryCounterStatus `json:"counter_status"` + // IncompleteReasons explains why a diagnostic replay cannot qualify when CounterStatus is not complete. + IncompleteReasons []string `json:"incomplete_reasons,omitempty"` + // PlanReplay records only counters PostgreSQL exposes through the separate TIMING OFF JSON EXPLAIN replay. + PlanReplay *TraversalPlanReplayEvidence `json:"plan_replay,omitempty"` + // Provenance maps diagnostic counter paths to the function, CTE, or executor metric that produced them. + Provenance map[string]string `json:"provenance"` +} + +// TraversalPlanReplayEvidence contains honest SQL-visible counters without pretending an outer Function Scan exposes hidden executor work. +type TraversalPlanReplayEvidence struct { + // Source identifies the exact untimed diagnostic boundary. + Source string `json:"source"` + // Counters contains only values with explicit PostgreSQL plan provenance. + Counters map[string]int64 `json:"counters,omitempty"` + // Flags contains only boolean outcomes observable from named plan branches or guards. + Flags map[string]bool `json:"flags,omitempty"` + // Provenance maps every counter and flag to its JSON EXPLAIN derivation. + Provenance map[string]string `json:"provenance"` +} + +// TraversalDiagnosticCounters groups independent runtime counter families. +type TraversalDiagnosticCounters struct { + Ordinary *OrdinaryTraversalCounters `json:"ordinary,omitempty"` + Orientation *OrientationTraversalCounters `json:"orientation,omitempty"` + ShortestPath *ShortestPathTraversalCounters `json:"shortest_path,omitempty"` + AllShortestPaths *AllShortestPathsTraversalCounters `json:"all_shortest_paths,omitempty"` + InlineASP *InlinePredecessorTraversalCounters `json:"inline_asp,omitempty"` + InlineShortestPath *InlinePredecessorTraversalCounters `json:"inline_shortest_path,omitempty"` + Hydration *TraversalHydrationCounters `json:"hydration,omitempty"` + Workspace *TraversalWorkspaceCounters `json:"workspace,omitempty"` +} + +// InlinePredecessorTraversalCounters records the complete set of bounded +// relations and complementary branch markers exposed by an inline I1 +// predecessor statement. ASP and canonical one-witness policies serialize +// into separate fields so their resource evidence cannot be interchanged. +type InlinePredecessorTraversalCounters struct { + DistanceRows *int64 `json:"distance_rows"` + PredecessorRows *int64 `json:"predecessor_rows"` + EnumerationRows *int64 `json:"enumeration_rows"` + OutputPaths *int64 `json:"output_paths"` + OutputBytes *int64 `json:"output_bytes"` + CandidateMarkerRows *int64 `json:"candidate_marker_rows"` + FallbackMarkerRows *int64 `json:"fallback_marker_rows"` + CandidateBranchRows *int64 `json:"candidate_branch_rows"` + FallbackBranchRows *int64 `json:"fallback_branch_rows"` + CandidateExecutorLoops *int64 `json:"candidate_executor_loops"` + FallbackExecutorLoops *int64 `json:"fallback_executor_loops"` +} + +// InlineASPTraversalCounters preserves the source-level name used by existing +// ASP telemetry producers while sharing the exact bounded-relation schema. +type InlineASPTraversalCounters = InlinePredecessorTraversalCounters + +// OrdinaryTraversalCounters records DFS or recursive-CTE discovery work. +type OrdinaryTraversalCounters struct { + Roots *int64 `json:"roots"` + EdgeCandidates *int64 `json:"edge_candidates"` + AdmittedStates *int64 `json:"admitted_states"` + RelationshipRepeatRejects *int64 `json:"relationship_repeat_rejects"` + RecursiveRows *int64 `json:"recursive_rows"` + PeakState *int64 `json:"peak_state"` + EmittedTrails *int64 `json:"emitted_trails"` + HydrationRows *int64 `json:"hydration_rows"` +} + +// OrientationTraversalCounters records bounded policy probes and selected-branch work. +type OrientationTraversalCounters struct { + ForwardSeeds *int64 `json:"forward_seeds"` + ReverseSeeds *int64 `json:"reverse_seeds"` + DuplicateSeeds *int64 `json:"duplicate_seeds"` + SuffixRows *int64 `json:"suffix_rows"` + DistinctBoundaries *int64 `json:"distinct_boundaries"` + TypedDirectionalDegreeSamples *int64 `json:"typed_directional_degree_samples"` + ForwardDegreeSamples *int64 `json:"forward_degree_samples"` + ReverseDegreeSamples *int64 `json:"reverse_degree_samples"` + ShallowSurvivalRows *int64 `json:"shallow_survival_rows"` + ShallowSurvival *float64 `json:"shallow_survival"` + ProbeRows *int64 `json:"probe_rows"` + ProbeTimeNS *int64 `json:"probe_time_ns"` + ProbeBufferHits *int64 `json:"probe_buffer_hits"` + ProbeBufferReads *int64 `json:"probe_buffer_reads"` + ForwardScore *float64 `json:"forward_score"` + ReverseScore *float64 `json:"reverse_score"` + SelectedSide string `json:"selected_side"` + SentinelOverflow *bool `json:"sentinel_overflow"` + BranchLoops *int64 `json:"branch_loops"` +} + +// ShortestPathLevelCounters records one scheduler action and the two-sided frontier state it observed. +type ShortestPathLevelCounters struct { + SearchID int64 `json:"search_id"` + ActionIndex int64 `json:"action_index"` + Side string `json:"side"` + Action string `json:"action"` + Depth *int64 `json:"depth"` + FrontierRows *int64 `json:"frontier_rows"` + CandidateEdges *int64 `json:"candidate_edges"` + DistinctNewNodes *int64 `json:"distinct_new_nodes"` + SeenRows *int64 `json:"seen_rows"` + QueueRows *int64 `json:"queue_rows"` + PredecessorRows *int64 `json:"predecessor_rows"` + MeetingCandidates *int64 `json:"meeting_candidates"` + // Provenance names the invocation-local stage or executor metric that produced this level row. + Provenance string `json:"provenance"` +} + +// ShortestPathTraversalCounters records bidirectional scheduler, frontier, and witness work. +type ShortestPathTraversalCounters struct { + SchedulerActions *int64 `json:"scheduler_actions"` + Levels []ShortestPathLevelCounters `json:"levels"` + CandidateEdges *int64 `json:"candidate_edges"` + DistinctNewNodes *int64 `json:"distinct_new_nodes"` + SeenPeak *int64 `json:"seen_peak"` + FrontierPeak *int64 `json:"frontier_peak"` + QueuePeak *int64 `json:"queue_peak"` + PredecessorPeak *int64 `json:"predecessor_peak"` + MeetingCandidates *int64 `json:"meeting_candidates"` + FrozenDistance *int64 `json:"frozen_distance"` + WitnessRows *int64 `json:"witness_rows"` + FallbackExecuted *bool `json:"fallback_executed"` +} + +// AllShortestPathsTraversalCounters records SP search work plus predecessor and output enumeration work. +type AllShortestPathsTraversalCounters struct { + Search ShortestPathTraversalCounters `json:"search"` + SameDepthPredecessorAdditions *int64 `json:"same_depth_predecessor_additions"` + PredecessorPeak *int64 `json:"predecessor_peak"` + MeetingNodes *int64 `json:"meeting_nodes"` + CutDepth *int64 `json:"cut_depth"` + PathCountEstimate *int64 `json:"path_count_estimate"` + PathCountSaturated *bool `json:"path_count_saturated"` + EnumeratedCandidates *int64 `json:"enumerated_candidates"` + DuplicateRejects *int64 `json:"duplicate_rejects"` + OutputPaths *int64 `json:"output_paths"` + OutputEdgeCells *int64 `json:"output_edge_cells"` + OutputBytes *int64 `json:"output_bytes"` +} + +// TraversalHydrationCounters records post-discovery materialization separately from traversal work. +type TraversalHydrationCounters struct { + PathCount *int64 `json:"path_count"` + NodeLookups *int64 `json:"node_lookups"` + EdgeLookups *int64 `json:"edge_lookups"` + Loops *int64 `json:"loops"` + Rows *int64 `json:"rows"` + TimeNS *int64 `json:"time_ns"` + Bytes *int64 `json:"bytes"` +} + +// TraversalWorkspaceCounters records measured high-water memory attributed to +// one diagnostic invocation and to all simultaneously active pool sessions. +type TraversalWorkspaceCounters struct { + SessionPeakBytes *int64 `json:"session_peak_bytes"` + PoolPeakBytes *int64 `json:"pool_peak_bytes"` +} + +// ValidateTraversalExecutionTelemetry rejects incomplete or contradictory telemetry. +func ValidateTraversalExecutionTelemetry(telemetry *TraversalExecutionTelemetry) error { + if telemetry == nil { + return fmt.Errorf("traversal execution telemetry is missing") + } + + return telemetry.Validate() +} + +// Validate rejects unsupported schema versions, incomplete summaries, timed diagnostic replays, and missing counters or provenance. +func (s TraversalExecutionTelemetry) Validate() error { + var problems []string + + if s.SchemaVersion != TraversalExecutionTelemetrySchemaVersion { + problems = append(problems, fmt.Sprintf("schema_version must be %d", TraversalExecutionTelemetrySchemaVersion)) + } + if s.Level != TraversalTelemetryLevelSummary && s.Level != TraversalTelemetryLevelDiagnostic { + problems = append(problems, "level must be summary or diagnostic") + } + + validateTraversalSummary(s.Summary, &problems) + + switch s.Level { + case TraversalTelemetryLevelSummary: + if s.Diagnostic != nil { + problems = append(problems, "summary telemetry must not contain a diagnostic replay") + } + case TraversalTelemetryLevelDiagnostic: + validateTraversalDiagnostic(s.Diagnostic, &problems) + } + + if len(problems) > 0 { + return fmt.Errorf("invalid traversal execution telemetry: %s", strings.Join(problems, "; ")) + } + + return nil +} + +func validateTraversalSummary(summary TraversalExecutionSummary, problems *[]string) { + requireText("summary.requested_identity", summary.RequestedIdentity, problems) + if len(summary.PlannedIdentities) == 0 { + *problems = append(*problems, "summary.planned_identities is missing") + } + planned := map[string]struct{}{} + for idx, identity := range summary.PlannedIdentities { + requireText(fmt.Sprintf("summary.planned_identities[%d]", idx), identity, problems) + if _, duplicate := planned[identity]; duplicate { + *problems = append(*problems, fmt.Sprintf("summary.planned_identities contains duplicate %q", identity)) + } + planned[identity] = struct{}{} + } + requireText("summary.emitted_identity", summary.EmittedIdentity, problems) + runtimeOutcomeAvailable := summary.RuntimeOutcomeAvailable == nil || *summary.RuntimeOutcomeAvailable + if runtimeOutcomeAvailable { + requireText("summary.runtime_identity", summary.RuntimeIdentity, problems) + requireText("summary.applied_identity", summary.AppliedIdentity, problems) + } else { + if summary.RuntimeIdentity != "" || summary.AppliedIdentity != "" { + *problems = append(*problems, "summary unavailable runtime outcome must not assert runtime or applied identity") + } + if summary.RuntimeBranch != "runtime_outcome_unavailable" { + *problems = append(*problems, "summary unavailable runtime outcome must use runtime_outcome_unavailable branch") + } + if summary.Overflow != nil || summary.FallbackExecuted != nil || summary.FallbackIdentity != "" { + *problems = append(*problems, "summary unavailable runtime outcome must not assert overflow or fallback facts") + } + } + requireText("summary.selector_version", summary.SelectorVersion, problems) + requireText("summary.scheduler_version", summary.SchedulerVersion, problems) + requireText("summary.runtime_branch", summary.RuntimeBranch, problems) + if runtimeOutcomeAvailable { + requirePointer("summary.overflow", summary.Overflow, problems) + requirePointer("summary.fallback_executed", summary.FallbackExecuted, problems) + } + if runtimeOutcomeAvailable && summary.RuntimeIdentity != "" { + if _, ok := planned[summary.RuntimeIdentity]; !ok { + *problems = append(*problems, "summary.runtime_identity is not a planned identity") + } + } + if runtimeOutcomeAvailable && summary.FallbackExecuted != nil && *summary.FallbackExecuted { + requireText("summary.fallback_identity", summary.FallbackIdentity, problems) + if summary.FallbackIdentity != "" { + if _, ok := planned[summary.FallbackIdentity]; !ok { + *problems = append(*problems, "summary.fallback_identity is not a planned identity") + } + if summary.AppliedIdentity != summary.FallbackIdentity { + *problems = append(*problems, "summary.applied_identity must equal fallback_identity when fallback executes") + } + } + } else if runtimeOutcomeAvailable && summary.FallbackExecuted != nil && summary.AppliedIdentity != "" && summary.RuntimeIdentity != "" && summary.AppliedIdentity != summary.RuntimeIdentity { + *problems = append(*problems, "summary.applied_identity must equal runtime_identity when fallback does not execute") + } + if summary.WouldSelectIdentity != "" { + if _, ok := planned[summary.WouldSelectIdentity]; !ok { + *problems = append(*problems, "summary.would_select_identity is not a planned identity") + } + requireProvenance("summary.would_select_identity", summary.Provenance["would_select_identity"], problems) + } + + for _, path := range []string{ + "requested_identity", "planned_identities", "emitted_identity", "runtime_identity", "applied_identity", + "selector_version", "scheduler_version", "runtime_branch", + } { + requireProvenance("summary."+path, summary.Provenance[path], problems) + } + if summary.RuntimeOutcomeAvailable != nil { + requireProvenance("summary.runtime_outcome_available", summary.Provenance["runtime_outcome_available"], problems) + } + if summary.ObservationMode != "" { + requireProvenance("summary.observation_mode", summary.Provenance["observation_mode"], problems) + } + if runtimeOutcomeAvailable { + for _, path := range []string{"overflow", "fallback_executed"} { + requireProvenance("summary."+path, summary.Provenance[path], problems) + } + } + for capName := range summary.Caps { + requireProvenance("summary.caps."+capName, summary.Provenance["caps."+capName], problems) + } + if runtimeOutcomeAvailable && summary.FallbackExecuted != nil && *summary.FallbackExecuted { + requireProvenance("summary.fallback_identity", summary.Provenance["fallback_identity"], problems) + } +} + +func validateTraversalDiagnostic(diagnostic *TraversalExecutionDiagnostic, problems *[]string) { + if diagnostic == nil { + *problems = append(*problems, "diagnostic replay is missing") + return + } + + requireText("diagnostic.invocation_id", diagnostic.InvocationID, problems) + requireText("diagnostic.connection_id", diagnostic.ConnectionID, problems) + requirePointer("diagnostic.timed_sample", diagnostic.TimedSample, problems) + if diagnostic.TimedSample != nil && *diagnostic.TimedSample { + *problems = append(*problems, "diagnostic.timed_sample must be false") + } + if len(diagnostic.RequiredFamilies) == 0 { + *problems = append(*problems, "diagnostic.required_families is missing") + } + counterStatus := diagnostic.CounterStatus + if counterStatus == "" { + // Version-one in-memory callers predate the explicit completeness field; + // their fully populated typed counters retain complete semantics. + counterStatus = TraversalTelemetryCounterStatusComplete + } + if counterStatus != TraversalTelemetryCounterStatusComplete && + counterStatus != TraversalTelemetryCounterStatusPlanPartial && + counterStatus != TraversalTelemetryCounterStatusHiddenUnavailable { + *problems = append(*problems, "diagnostic.counter_status is unsupported") + } + if counterStatus != TraversalTelemetryCounterStatusComplete && len(diagnostic.IncompleteReasons) == 0 { + *problems = append(*problems, "diagnostic.incomplete_reasons is missing for incomplete counters") + } + if counterStatus == TraversalTelemetryCounterStatusPlanPartial && diagnostic.PlanReplay == nil { + *problems = append(*problems, "diagnostic.plan_replay is missing for plan-derived counters") + } + if diagnostic.PlanReplay != nil { + validateTraversalPlanReplay(diagnostic.PlanReplay, problems) + } + + seen := map[TraversalTelemetryFamily]struct{}{} + for _, family := range diagnostic.RequiredFamilies { + if _, duplicate := seen[family]; duplicate { + *problems = append(*problems, fmt.Sprintf("diagnostic.required_families contains duplicate %q", family)) + continue + } + seen[family] = struct{}{} + + if counterStatus != TraversalTelemetryCounterStatusComplete { + continue + } + + switch family { + case TraversalTelemetryFamilyOrdinary: + validateOrdinaryCounters(diagnostic.Counters.Ordinary, diagnostic.Provenance, problems) + case TraversalTelemetryFamilyOrientation: + validateOrientationCounters(diagnostic.Counters.Orientation, diagnostic.Provenance, problems) + case TraversalTelemetryFamilySP: + if diagnostic.Counters.InlineShortestPath != nil { + validateInlinePredecessorCounters("inline_shortest_path", diagnostic.Counters.InlineShortestPath, diagnostic.Provenance, problems) + } else { + validateShortestPathCounters("shortest_path", diagnostic.Counters.ShortestPath, diagnostic.Provenance, problems) + } + case TraversalTelemetryFamilyASP: + if diagnostic.Counters.InlineASP != nil { + validateInlinePredecessorCounters("inline_asp", diagnostic.Counters.InlineASP, diagnostic.Provenance, problems) + } else { + validateAllShortestPathsCounters(diagnostic.Counters.AllShortestPaths, diagnostic.Provenance, problems) + } + case TraversalTelemetryFamilyHydration: + validateHydrationCounters(diagnostic.Counters.Hydration, diagnostic.Provenance, problems) + case TraversalTelemetryFamilyWorkspace: + validateWorkspaceCounters(diagnostic.Counters.Workspace, diagnostic.Provenance, problems) + default: + *problems = append(*problems, fmt.Sprintf("diagnostic.required_families contains unsupported family %q", family)) + } + } + + if counterStatus != TraversalTelemetryCounterStatusComplete { + return + } + + for family, present := range map[TraversalTelemetryFamily]bool{ + TraversalTelemetryFamilyOrdinary: diagnostic.Counters.Ordinary != nil, + TraversalTelemetryFamilyOrientation: diagnostic.Counters.Orientation != nil, + TraversalTelemetryFamilySP: diagnostic.Counters.ShortestPath != nil || diagnostic.Counters.InlineShortestPath != nil, + TraversalTelemetryFamilyASP: diagnostic.Counters.AllShortestPaths != nil || diagnostic.Counters.InlineASP != nil, + TraversalTelemetryFamilyHydration: diagnostic.Counters.Hydration != nil, + TraversalTelemetryFamilyWorkspace: diagnostic.Counters.Workspace != nil, + } { + if present && !slices.Contains(diagnostic.RequiredFamilies, family) { + *problems = append(*problems, fmt.Sprintf("diagnostic counter family %q is present but not declared", family)) + } + } +} + +func validateInlinePredecessorCounters(prefix string, counters *InlinePredecessorTraversalCounters, provenance map[string]string, problems *[]string) { + if counters == nil { + *problems = append(*problems, "diagnostic.counters."+prefix+" is missing") + return + } + requireCounters(prefix, provenance, problems, map[string]*int64{ + "distance_rows": counters.DistanceRows, "predecessor_rows": counters.PredecessorRows, + "enumeration_rows": counters.EnumerationRows, "output_paths": counters.OutputPaths, + "output_bytes": counters.OutputBytes, "candidate_marker_rows": counters.CandidateMarkerRows, + "fallback_marker_rows": counters.FallbackMarkerRows, "candidate_branch_rows": counters.CandidateBranchRows, + "fallback_branch_rows": counters.FallbackBranchRows, "candidate_executor_loops": counters.CandidateExecutorLoops, + "fallback_executor_loops": counters.FallbackExecutorLoops, + }) +} + +func validateTraversalPlanReplay(replay *TraversalPlanReplayEvidence, problems *[]string) { + if replay == nil { + return + } + requireText("diagnostic.plan_replay.source", replay.Source, problems) + if len(replay.Counters) == 0 && len(replay.Flags) == 0 { + *problems = append(*problems, "diagnostic.plan_replay contains no observable counters or flags") + } + for name := range replay.Counters { + requireProvenance("diagnostic.plan_replay.counters."+name, replay.Provenance["counters."+name], problems) + } + for name := range replay.Flags { + requireProvenance("diagnostic.plan_replay.flags."+name, replay.Provenance["flags."+name], problems) + } +} + +func validateOrdinaryCounters(counters *OrdinaryTraversalCounters, provenance map[string]string, problems *[]string) { + if counters == nil { + *problems = append(*problems, "diagnostic.counters.ordinary is missing") + return + } + + requireCounters("ordinary", provenance, problems, map[string]*int64{ + "roots": counters.Roots, "edge_candidates": counters.EdgeCandidates, "admitted_states": counters.AdmittedStates, + "relationship_repeat_rejects": counters.RelationshipRepeatRejects, "recursive_rows": counters.RecursiveRows, + "peak_state": counters.PeakState, "emitted_trails": counters.EmittedTrails, "hydration_rows": counters.HydrationRows, + }) +} + +func validateOrientationCounters(counters *OrientationTraversalCounters, provenance map[string]string, problems *[]string) { + if counters == nil { + *problems = append(*problems, "diagnostic.counters.orientation is missing") + return + } + + requireCounters("orientation", provenance, problems, map[string]*int64{ + "forward_seeds": counters.ForwardSeeds, "reverse_seeds": counters.ReverseSeeds, "duplicate_seeds": counters.DuplicateSeeds, + "suffix_rows": counters.SuffixRows, "distinct_boundaries": counters.DistinctBoundaries, + "typed_directional_degree_samples": counters.TypedDirectionalDegreeSamples, "probe_rows": counters.ProbeRows, + "forward_degree_samples": counters.ForwardDegreeSamples, "reverse_degree_samples": counters.ReverseDegreeSamples, + "shallow_survival_rows": counters.ShallowSurvivalRows, + "probe_time_ns": counters.ProbeTimeNS, "probe_buffer_hits": counters.ProbeBufferHits, + "probe_buffer_reads": counters.ProbeBufferReads, "branch_loops": counters.BranchLoops, + }) + requirePointerAndProvenance("orientation.shallow_survival", counters.ShallowSurvival, provenance, problems) + requirePointerAndProvenance("orientation.forward_score", counters.ForwardScore, provenance, problems) + requirePointerAndProvenance("orientation.reverse_score", counters.ReverseScore, provenance, problems) + requireText("diagnostic.counters.orientation.selected_side", counters.SelectedSide, problems) + requireProvenance("diagnostic.counters.orientation.selected_side", provenance["orientation.selected_side"], problems) + requirePointerAndProvenance("orientation.sentinel_overflow", counters.SentinelOverflow, provenance, problems) +} + +func validateShortestPathCounters(prefix string, counters *ShortestPathTraversalCounters, provenance map[string]string, problems *[]string) { + if counters == nil { + *problems = append(*problems, "diagnostic.counters."+prefix+" is missing") + return + } + + requireCounters(prefix, provenance, problems, map[string]*int64{ + "scheduler_actions": counters.SchedulerActions, "candidate_edges": counters.CandidateEdges, + "distinct_new_nodes": counters.DistinctNewNodes, "seen_peak": counters.SeenPeak, "frontier_peak": counters.FrontierPeak, + "queue_peak": counters.QueuePeak, "predecessor_peak": counters.PredecessorPeak, "meeting_candidates": counters.MeetingCandidates, + "frozen_distance": counters.FrozenDistance, "witness_rows": counters.WitnessRows, + }) + requirePointerAndProvenance(prefix+".fallback_executed", counters.FallbackExecuted, provenance, problems) + if len(counters.Levels) == 0 { + *problems = append(*problems, "diagnostic.counters."+prefix+".levels is missing") + } + for idx, level := range counters.Levels { + levelPath := fmt.Sprintf("diagnostic.counters.%s.levels[%d]", prefix, idx) + requireText(levelPath+".side", level.Side, problems) + requireText(levelPath+".action", level.Action, problems) + requirePointer(levelPath+".depth", level.Depth, problems) + requirePointer(levelPath+".frontier_rows", level.FrontierRows, problems) + requirePointer(levelPath+".candidate_edges", level.CandidateEdges, problems) + requirePointer(levelPath+".distinct_new_nodes", level.DistinctNewNodes, problems) + requirePointer(levelPath+".seen_rows", level.SeenRows, problems) + requirePointer(levelPath+".queue_rows", level.QueueRows, problems) + requirePointer(levelPath+".predecessor_rows", level.PredecessorRows, problems) + requirePointer(levelPath+".meeting_candidates", level.MeetingCandidates, problems) + requireProvenance(levelPath, level.Provenance, problems) + } +} + +func validateAllShortestPathsCounters(counters *AllShortestPathsTraversalCounters, provenance map[string]string, problems *[]string) { + if counters == nil { + *problems = append(*problems, "diagnostic.counters.all_shortest_paths is missing") + return + } + + validateShortestPathCounters("all_shortest_paths.search", &counters.Search, provenance, problems) + requireCounters("all_shortest_paths", provenance, problems, map[string]*int64{ + "same_depth_predecessor_additions": counters.SameDepthPredecessorAdditions, "predecessor_peak": counters.PredecessorPeak, + "meeting_nodes": counters.MeetingNodes, "cut_depth": counters.CutDepth, "path_count_estimate": counters.PathCountEstimate, + "enumerated_candidates": counters.EnumeratedCandidates, "duplicate_rejects": counters.DuplicateRejects, + "output_paths": counters.OutputPaths, "output_edge_cells": counters.OutputEdgeCells, "output_bytes": counters.OutputBytes, + }) + requirePointerAndProvenance("all_shortest_paths.path_count_saturated", counters.PathCountSaturated, provenance, problems) +} + +func validateHydrationCounters(counters *TraversalHydrationCounters, provenance map[string]string, problems *[]string) { + if counters == nil { + *problems = append(*problems, "diagnostic.counters.hydration is missing") + return + } + + requireCounters("hydration", provenance, problems, map[string]*int64{ + "path_count": counters.PathCount, "node_lookups": counters.NodeLookups, "edge_lookups": counters.EdgeLookups, + "loops": counters.Loops, "rows": counters.Rows, "time_ns": counters.TimeNS, "bytes": counters.Bytes, + }) +} + +func validateWorkspaceCounters(counters *TraversalWorkspaceCounters, provenance map[string]string, problems *[]string) { + if counters == nil { + *problems = append(*problems, "diagnostic.counters.workspace is missing") + return + } + + requireCounters("workspace", provenance, problems, map[string]*int64{ + "session_peak_bytes": counters.SessionPeakBytes, + "pool_peak_bytes": counters.PoolPeakBytes, + }) +} + +func requireCounters(prefix string, provenance map[string]string, problems *[]string, counters map[string]*int64) { + for name, value := range counters { + requirePointerAndProvenance(prefix+"."+name, value, provenance, problems) + } +} + +func requirePointerAndProvenance[T any](path string, value *T, provenance map[string]string, problems *[]string) { + requirePointer("diagnostic.counters."+path, value, problems) + requireProvenance("diagnostic.counters."+path, provenance[path], problems) +} + +func requirePointer[T any](path string, value *T, problems *[]string) { + if value == nil { + *problems = append(*problems, path+" is missing") + } +} + +func requireText(path, value string, problems *[]string) { + if strings.TrimSpace(value) == "" { + *problems = append(*problems, path+" is missing") + } +} + +func requireProvenance(path, value string, problems *[]string) { + if strings.TrimSpace(value) == "" { + *problems = append(*problems, path+" provenance is missing") + } +} diff --git a/cmd/graphbench/traversal_telemetry_test.go b/cmd/graphbench/traversal_telemetry_test.go new file mode 100644 index 00000000..b890bf1b --- /dev/null +++ b/cmd/graphbench/traversal_telemetry_test.go @@ -0,0 +1,165 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestTraversalExecutionTelemetrySummaryValidation(t *testing.T) { + telemetry := validTraversalTelemetry() + + require.NoError(t, telemetry.Validate()) + + telemetry.Summary.Overflow = nil + err := telemetry.Validate() + require.ErrorContains(t, err, "summary.overflow is missing") + + telemetry = validTraversalTelemetry() + delete(telemetry.Summary.Provenance, "runtime_identity") + err = telemetry.Validate() + require.ErrorContains(t, err, "summary.runtime_identity provenance is missing") +} + +func TestTraversalExecutionTelemetrySummaryRejectsContradictoryIdentityChain(t *testing.T) { + telemetry := validTraversalTelemetry() + telemetry.Summary.RuntimeIdentity = "unplanned-v1" + + require.ErrorContains(t, telemetry.Validate(), "summary.runtime_identity is not a planned identity") + + telemetry = validTraversalTelemetry() + telemetry.Summary.FallbackExecuted = telemetryBool(true) + telemetry.Summary.FallbackIdentity = "incumbent-v1" + telemetry.Summary.Provenance["fallback_identity"] = "executor.fallback_identity" + + require.ErrorContains(t, telemetry.Validate(), "summary.applied_identity must equal fallback_identity") +} + +func TestTraversalExecutionTelemetryDiagnosticRequiresPointerCountersAndProvenance(t *testing.T) { + telemetry := validTraversalTelemetry() + telemetry.Level = TraversalTelemetryLevelDiagnostic + telemetry.Diagnostic = ordinaryDiagnostic() + + require.NoError(t, telemetry.Validate()) + + telemetry.Diagnostic.Counters.Ordinary.RecursiveRows = nil + err := telemetry.Validate() + require.ErrorContains(t, err, "diagnostic.counters.ordinary.recursive_rows is missing") + + telemetry.Diagnostic.Counters.Ordinary.RecursiveRows = telemetryInt64(0) + delete(telemetry.Diagnostic.Provenance, "ordinary.recursive_rows") + err = telemetry.Validate() + require.ErrorContains(t, err, "diagnostic.counters.ordinary.recursive_rows provenance is missing") +} + +func TestTraversalExecutionTelemetryDiagnosticCannotBeTimed(t *testing.T) { + telemetry := validTraversalTelemetry() + telemetry.Level = TraversalTelemetryLevelDiagnostic + telemetry.Diagnostic = ordinaryDiagnostic() + telemetry.Diagnostic.TimedSample = telemetryBool(true) + + require.ErrorContains(t, telemetry.Validate(), "diagnostic.timed_sample must be false") +} + +func TestTraversalExecutionTelemetryAttachmentsSerializeVersionedSchema(t *testing.T) { + telemetry := validTraversalTelemetry() + encoded, err := json.Marshal(struct { + Case CaseResult `json:"case"` + Reference PostgresReferenceResult `json:"reference"` + }{ + Case: CaseResult{TraversalTelemetry: &telemetry}, + Reference: PostgresReferenceResult{TraversalTelemetry: &telemetry}, + }) + + require.NoError(t, err) + require.Contains(t, string(encoded), `"traversal_execution_telemetry":{"schema_version":2`) +} + +func validTraversalTelemetry() TraversalExecutionTelemetry { + return TraversalExecutionTelemetry{ + SchemaVersion: TraversalExecutionTelemetrySchemaVersion, + Level: TraversalTelemetryLevelSummary, + Summary: TraversalExecutionSummary{ + RequestedIdentity: "requested-v1", + PlannedIdentities: []string{"candidate-v1", "incumbent-v1"}, + EmittedIdentity: "policy-v1", + RuntimeIdentity: "candidate-v1", + AppliedIdentity: "candidate-v1", + SelectorVersion: "selector-v1", + SchedulerVersion: "scheduler-v1", + Caps: map[string]int64{"state": 32}, + RuntimeBranch: "candidate", + Overflow: telemetryBool(false), + FallbackExecuted: telemetryBool(false), + Provenance: map[string]string{ + "requested_identity": "optimizer.request", + "planned_identities": "optimizer.candidates", + "emitted_identity": "translator.policy", + "runtime_identity": "executor.branch", + "applied_identity": "executor.applied", + "selector_version": "optimizer.selector", + "scheduler_version": "executor.scheduler", + "caps.state": "policy.state_cap", + "runtime_branch": "executor.branch", + "overflow": "executor.guard", + "fallback_executed": "executor.fallback", + }, + }, + } +} + +func ordinaryDiagnostic() *TraversalExecutionDiagnostic { + provenance := map[string]string{} + for _, name := range []string{ + "roots", "edge_candidates", "admitted_states", "relationship_repeat_rejects", "recursive_rows", + "peak_state", "emitted_trails", "hydration_rows", + } { + provenance["ordinary."+name] = "traversal_recursive_cte." + name + } + + return &TraversalExecutionDiagnostic{ + InvocationID: "invocation-1", + ConnectionID: "backend-123", + TimedSample: telemetryBool(false), + RequiredFamilies: []TraversalTelemetryFamily{TraversalTelemetryFamilyOrdinary}, + CounterStatus: TraversalTelemetryCounterStatusComplete, + Counters: TraversalDiagnosticCounters{ + Ordinary: &OrdinaryTraversalCounters{ + Roots: telemetryInt64(0), + EdgeCandidates: telemetryInt64(0), + AdmittedStates: telemetryInt64(0), + RelationshipRepeatRejects: telemetryInt64(0), + RecursiveRows: telemetryInt64(0), + PeakState: telemetryInt64(0), + EmittedTrails: telemetryInt64(0), + HydrationRows: telemetryInt64(0), + }, + }, + Provenance: provenance, + } +} + +func telemetryInt64(value int64) *int64 { + return &value +} + +func telemetryBool(value bool) *bool { + return &value +} diff --git a/cmd/graphbench/types.go b/cmd/graphbench/types.go index c941a01a..0b275157 100644 --- a/cmd/graphbench/types.go +++ b/cmd/graphbench/types.go @@ -20,14 +20,22 @@ import ( "fmt" "slices" "strings" + + "github.com/specterops/dawgs/testutil" ) const ( - ModePostgresSQL ExecutionMode = "postgres_sql" + // ModePostgresSQL selects translated PostgreSQL execution. + ModePostgresSQL ExecutionMode = "postgres_sql" + + // ModeLocalTraversal selects in-process traversal execution. ModeLocalTraversal ExecutionMode = "local_traversal" - ModeNeo4j ExecutionMode = "neo4j" + + // ModeNeo4j selects Neo4j execution. + ModeNeo4j ExecutionMode = "neo4j" ) +// validExecutionModes lists every execution mode accepted by graphbench. var validExecutionModes = []ExecutionMode{ ModePostgresSQL, ModeLocalTraversal, @@ -36,10 +44,12 @@ var validExecutionModes = []ExecutionMode{ type ExecutionMode string +// Valid reports whether the execution mode is one of the supported backend modes. func (s ExecutionMode) Valid() bool { return slices.Contains(validExecutionModes, s) } +// parseExecutionMode returns the execution mode named by text or an error for unsupported values. func parseExecutionMode(raw string) (ExecutionMode, error) { mode := ExecutionMode(strings.TrimSpace(raw)) if mode.Valid() { @@ -49,56 +59,216 @@ func parseExecutionMode(raw string) (ExecutionMode, error) { return "", fmt.Errorf("unsupported execution mode %q", raw) } +// ScaleCorpus contains the ordered benchmark cases loaded from the scale corpus. type ScaleCorpus struct { + // Cases contains loaded workloads in deterministic corpus order. Cases []ScaleCase } +// DeclaredCaseBackend identifies one case/backend combination and any declared unsupported reason. +type DeclaredCaseBackend struct { + // Dataset identifies the fixture dataset. + Dataset string + // Name identifies the case or record within its dataset. + Name string + // Backend identifies the execution backend. + Backend ExecutionMode + // UnsupportedReason explains why a declared case cannot run on the selected backend. + UnsupportedReason string +} + +// DeclaredBackends expands a scale case into the backend declarations consumed during gate validation. +func (s ScaleCorpus) DeclaredBackends() []DeclaredCaseBackend { + declared := make([]DeclaredCaseBackend, 0, len(s.Cases)*2) + for _, testCase := range s.Cases { + for _, backend := range testCase.CandidateModes { + declared = append(declared, DeclaredCaseBackend{ + Dataset: testCase.Dataset, + Name: testCase.Name, + Backend: backend, + }) + } + for backend, reason := range testCase.UnsupportedModes { + declared = append(declared, DeclaredCaseBackend{ + Dataset: testCase.Dataset, + Name: testCase.Name, + Backend: backend, + UnsupportedReason: reason, + }) + } + } + return declared +} + +// ScaleCaseFile models the JSON envelope containing a group of scale cases. type ScaleCaseFile struct { + // Cases contains the workload declarations decoded from one corpus file. Cases []ScaleCase `json:"cases"` } +// ScaleCase declares one executable workload, its parameters, backend support, and exact expectations. type ScaleCase struct { - Source string `json:"-"` - Name string `json:"name"` - Dataset string `json:"dataset"` - Category string `json:"category"` - Cypher string `json:"cypher"` - Params map[string]any `json:"params,omitempty"` - NodeParams map[string]string `json:"node_params,omitempty"` - Expected ExpectedResult `json:"expected"` - Observes ObservedValues `json:"observes"` - Shape WorkloadShape `json:"shape"` - CandidateModes []ExecutionMode `json:"candidate_modes"` - Tags []string `json:"tags,omitempty"` - ReferenceDesign *ReferenceDesign `json:"reference_design,omitempty"` + // Source identifies the source corpus file. + Source string `json:"-"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Category groups cases by workload category. + Category string `json:"category"` + // Cypher contains the Cypher statement under test. + Cypher string `json:"cypher"` + // Params supplies literal query parameters. + Params testutil.Params `json:"params,omitempty"` + // NodeParams maps query parameters to fixture node keys. + NodeParams map[string]string `json:"node_params,omitempty"` + // NodeListParams maps query parameters to ordered fixture node-key lists. + NodeListParams map[string][]string `json:"node_list_params,omitempty"` + // GeneratedNodeListParams maps query parameters to generated fixture node sets. + GeneratedNodeListParams map[string]testutil.GeneratedNodeListParam `json:"generated_node_list_params,omitempty"` + // Expected defines the required observable result. + Expected ExpectedResult `json:"expected"` + // Observes identifies the normalized observation contract declared by the scale case. + Observes ObservedValues `json:"observes"` + // Shape describes the workload shape used for selection and comparison. + Shape WorkloadShape `json:"shape"` + // CandidateModes lists backends expected to participate in cross-backend comparison. + CandidateModes []ExecutionMode `json:"candidate_modes"` + // UnsupportedModes maps unsupported execution modes to their declared reasons. + UnsupportedModes map[ExecutionMode]string `json:"unsupported_modes,omitempty"` + // Tags lists selectors attached to the case. + Tags []string `json:"tags,omitempty"` + // ReferenceDesign documents reference arms and validation boundaries applicable to the scale case. + ReferenceDesign *ReferenceDesign `json:"reference_design,omitempty"` + // WriteScenario defines the mutation and post-state checks measured for the scale case. + WriteScenario *WriteScenario `json:"write_scenario,omitempty"` } +// ExpectedResult defines the row cardinality and normalized scalar, ID-row, or path observations a case must return. type ExpectedResult struct { - RowCount *int64 `json:"row_count,omitempty"` + // RowCount records the number of rows produced. + RowCount *int64 `json:"row_count,omitempty"` + // ScalarInt sets the required scalar result when ResultKind is scalar_int. + ScalarInt *int64 `json:"scalar_int,omitempty"` + // ResultKind identifies how returned values must be normalized. ResultKind string `json:"result_kind,omitempty"` + // IDRows contains the expected ordered identifier rows. + IDRows [][]string `json:"id_rows,omitempty"` + // PathRows contains the expected stable paths. + PathRows []ExpectedPath `json:"path_rows,omitempty"` } +// ExpectedPath defines one expected stable node and relationship sequence. +type ExpectedPath struct { + // Nodes contains the stable node sequence. + Nodes []string `json:"nodes"` + // RelationshipKinds contains the expected relationship-kind sequence. + RelationshipKinds []string `json:"relationship_kinds"` + // RelationshipKeys contains the expected fixture relationship-key sequence. + RelationshipKeys []string `json:"relationship_keys,omitempty"` +} + +// WriteScenario defines a measured mutation and the state checks that validate it. +type WriteScenario struct { + // SelectionCypher contains the write-selection Cypher statement. + SelectionCypher string `json:"selection_cypher"` + // Params supplies literal query parameters. + Params testutil.Params `json:"params,omitempty"` + // NodeParams maps query parameters to fixture node keys. + NodeParams map[string]string `json:"node_params,omitempty"` + // NodeListParams maps query parameters to ordered fixture node-key lists. + NodeListParams map[string][]string `json:"node_list_params,omitempty"` + // GeneratedNodeListParams maps query parameters to generated fixture node sets. + GeneratedNodeListParams map[string]testutil.GeneratedNodeListParam `json:"generated_node_list_params,omitempty"` + // AffectedEntity identifies the entity class counted after a write. + AffectedEntity string `json:"affected_entity"` + // ExpectedMatched sets the required number of matched entities. + ExpectedMatched *int64 `json:"expected_matched"` + // ExpectedAffected sets the required number of affected entities. + ExpectedAffected *int64 `json:"expected_affected"` + // PostState defines the state query evaluated after a write. + PostState []ScaleStateQuery `json:"post_state"` +} + +// ScaleStateQuery defines a post-mutation query and its scalar or row-count expectation. +type ScaleStateQuery struct { + // Name labels the post-write state assertion in diagnostics and results. + Name string `json:"name"` + // Cypher contains the Cypher statement under test. + Cypher string `json:"cypher"` + // Params supplies literal query parameters. + Params testutil.Params `json:"params,omitempty"` + // NodeParams maps query parameters to fixture node keys. + NodeParams map[string]string `json:"node_params,omitempty"` + // NodeListParams maps query parameters to ordered fixture node-key lists. + NodeListParams map[string][]string `json:"node_list_params,omitempty"` + // GeneratedNodeListParams maps query parameters to generated fixture node sets. + GeneratedNodeListParams map[string]testutil.GeneratedNodeListParam `json:"generated_node_list_params,omitempty"` + // Expected defines the required observable result. + Expected ExpectedResult `json:"expected"` +} + +// ObservedValues declares which entity and path features a case exposes for normalized comparison. type ObservedValues struct { - Paths bool `json:"paths"` - Nodes bool `json:"nodes"` + // Paths reports whether the normalized result includes materialized paths. + Paths bool `json:"paths"` + // Nodes reports whether the normalized result includes node values. + Nodes bool `json:"nodes"` + // Relationships reports whether the normalized result includes relationship values. Relationships bool `json:"relationships"` - Properties bool `json:"properties"` + // Properties reports whether normalized entity observations include properties. + Properties bool `json:"properties"` } +// WorkloadShape describes traversal depth, direction, projection, and expected complexity. type WorkloadShape struct { - RootPredicate string `json:"root_predicate,omitempty"` - TerminalPredicate string `json:"terminal_predicate,omitempty"` - EdgeKinds []string `json:"edge_kinds,omitempty"` - MinDepth *int `json:"min_depth,omitempty"` - MaxDepth *int `json:"max_depth,omitempty"` - PathMaterializationRequired bool `json:"path_materialization_required"` + // QualificationSplit identifies whether a topology bucket is training, + // holdout, or a diagnostic boundary. Selector tuning must not consume + // holdout records. + QualificationSplit string `json:"qualification_split,omitempty"` + // FallbackExpectation is the typed runtime contract for candidate execution: + // forbidden, required, or allowed. Prioritized corpus declarations receive a + // deterministic value during loading when older files omit it. + FallbackExpectation string `json:"fallback_expectation,omitempty"` + // RootPredicate describes how the traversal root is constrained. + RootPredicate string `json:"root_predicate,omitempty"` + // TerminalPredicate describes how the traversal terminal is constrained. + TerminalPredicate string `json:"terminal_predicate,omitempty"` + // EdgeKinds lists the relationship kinds traversed by the workload. + EdgeKinds []string `json:"edge_kinds,omitempty"` + // Direction sets the traversal direction. + Direction string `json:"direction,omitempty"` + // RelationshipKindCount records the number of relationship kinds in the workload. + RelationshipKindCount int `json:"relationship_kind_count,omitempty"` + // FixtureTier identifies the fixture scale tier. + FixtureTier string `json:"fixture_tier,omitempty"` + // ExpectedStateClass identifies the expected recursive-state complexity class. + ExpectedStateClass string `json:"expected_state_class,omitempty"` + // ResultCardinalityClass identifies the expected result-cardinality class. + ResultCardinalityClass string `json:"result_cardinality_class,omitempty"` + // MinDepth is the shallowest traversal depth permitted by the workload. + MinDepth *int `json:"min_depth,omitempty"` + // MaxDepth sets the maximum traversal depth. + MaxDepth *int `json:"max_depth,omitempty"` + // PathMaterializationRequired reports whether the workload must materialize complete paths. + PathMaterializationRequired bool `json:"path_materialization_required"` } +// ReferenceDesign documents the independent reference implementations applicable to a case. type ReferenceDesign struct { + // AGERelevance documents how the reference design relates to Apache AGE execution. AGERelevance []string `json:"age_relevance,omitempty"` - Notes string `json:"notes,omitempty"` + // Notes contains human-readable caveats attached to the artifact or case. + Notes string `json:"notes,omitempty"` } +// Supports reports whether the case declares the requested execution mode as a candidate backend. func (s ScaleCase) Supports(mode ExecutionMode) bool { return slices.Contains(s.CandidateModes, mode) } + +// UnsupportedReason returns the declared reason that a scale case cannot run in the requested mode. +func (s ScaleCase) UnsupportedReason(mode ExecutionMode) (string, bool) { + reason, unsupported := s.UnsupportedModes[mode] + return reason, unsupported +} diff --git a/cmd/graphbench/waterfall.go b/cmd/graphbench/waterfall.go new file mode 100644 index 00000000..0d3cc561 --- /dev/null +++ b/cmd/graphbench/waterfall.go @@ -0,0 +1,204 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "fmt" + "runtime" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" +) + +// measureCompileWaterfall times Cypher parse, translate, and SQL rendering separately. +func measureCompileWaterfall( + ctx context.Context, + cypherQuery string, + params map[string]any, + kindMapper pgsql.KindMapper, + graphID int32, + iterations int, + toolOptions translate.ToolOptions, +) (ClientWaterfall, error) { + waterfall := ClientWaterfall{ + IntervalsOverlap: true, + Notes: "translate_including_optimize repeats optimization internally; parse, optimize, translate, and render must not be summed as an additive client attribution", + Samples: make([]CompileSample, 0, iterations), + } + for iteration := 1; iteration <= iterations; iteration++ { + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + totalStart := time.Now() + + parseStart := time.Now() + query, err := frontend.ParseCypher(frontend.NewContext(), cypherQuery) + if err != nil { + return ClientWaterfall{}, fmt.Errorf("parse: %w", err) + } + parseDuration := time.Since(parseStart) + + optimizeStart := time.Now() + if _, err := optimize.Optimize(query); err != nil { + return ClientWaterfall{}, fmt.Errorf("optimize: %w", err) + } + optimizeDuration := time.Since(optimizeStart) + + translateStart := time.Now() + var translation translate.Result + if !hasForcedToolOptions(toolOptions) { + translation, err = translate.Translate(ctx, query, kindMapper, params, graphID) + } else { + translation, err = translate.TranslateForTool(ctx, query, kindMapper, params, graphID, toolOptions) + } + if err != nil { + return ClientWaterfall{}, fmt.Errorf("translate: %w", err) + } + translateDuration := time.Since(translateStart) + + renderStart := time.Now() + if _, err := translate.Translated(translation); err != nil { + return ClientWaterfall{}, fmt.Errorf("render: %w", err) + } + renderDuration := time.Since(renderStart) + totalDuration := time.Since(totalStart) + runtime.ReadMemStats(&after) + + waterfall.Samples = append(waterfall.Samples, CompileSample{ + Iteration: iteration, + Parse: parseDuration, + Optimize: optimizeDuration, + TranslateIncludingOptimize: translateDuration, + Render: renderDuration, + Total: totalDuration, + Allocations: after.Mallocs - before.Mallocs, + AllocatedBytes: after.TotalAlloc - before.TotalAlloc, + }) + } + return waterfall, nil +} + +// measureRawPGXWaterfall times PostgreSQL bind, first row, drain, and close stages separately. +func measureRawPGXWaterfall(ctx context.Context, pool *pgxpool.Pool, sqlQuery string, params map[string]any, warmupIterations, iterations int, isolation ...pgx.TxIsoLevel) (PostgresBoundaryWaterfall, error) { + if warmupIterations < 0 || iterations < 1 { + return PostgresBoundaryWaterfall{}, fmt.Errorf("invalid raw pgx warmup/iteration counts") + } + run := func(iteration int, retain bool) (BoundarySample, error) { + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + totalStart := time.Now() + acquireStart := time.Now() + connection, err := pool.Acquire(ctx) + if err != nil { + return BoundarySample{}, err + } + defer connection.Release() + + poolWait := time.Since(acquireStart) + transactionStart := time.Now() + // DAWGS read queries may invoke the incumbent shortest-path workspace, + // whose SQL performs session-local DDL/DML. Use a rollback-only + // read-write transaction so the raw boundary can execute the identical + // translated SQL without committing state. + txOptions := pgx.TxOptions{AccessMode: pgx.ReadWrite} + if len(isolation) > 0 { + txOptions.IsoLevel = isolation[0] + } + tx, err := connection.BeginTx(ctx, txOptions) + if err != nil { + return BoundarySample{}, err + } + defer func() { _ = tx.Rollback(ctx) }() + + transactionDuration := time.Since(transactionStart) + bindStart := time.Now() + queryArgs := []any{pgx.QueryExecModeCacheStatement, pgx.QueryResultFormats{pgx.BinaryFormatCode}} + if len(params) > 0 { + queryArgs = append(queryArgs, pgx.NamedArgs(params)) + } + rows, err := tx.Query(ctx, sqlQuery, queryArgs...) + if err != nil { + return BoundarySample{}, err + } + bindDuration := time.Since(bindStart) + firstRowStart := time.Now() + var rowCount int64 + if rows.Next() { + rowCount++ + if _, err := rows.Values(); err != nil { + rows.Close() + return BoundarySample{}, err + } + } + firstRowDuration := time.Since(firstRowStart) + allRowsStart := time.Now() + for rows.Next() { + rowCount++ + if _, err := rows.Values(); err != nil { + rows.Close() + return BoundarySample{}, err + } + } + allRowsDuration := time.Since(allRowsStart) + drainStart := time.Now() + rows.Close() + if err := rows.Err(); err != nil { + return BoundarySample{}, err + } + if err := tx.Rollback(ctx); err != nil && err != pgx.ErrTxClosed { + return BoundarySample{}, err + } + drainDuration := time.Since(drainStart) + runtime.ReadMemStats(&after) + sample := BoundarySample{ + Iteration: iteration, + PoolWait: poolWait, + Transaction: transactionDuration, + BindPrepare: bindDuration, + FirstRow: firstRowDuration, + AllRowsDecode: allRowsDuration, + DrainClose: drainDuration, + Total: time.Since(totalStart), + Rows: rowCount, + } + if retain { + sample.Allocations = after.Mallocs - before.Mallocs + sample.AllocatedBytes = after.TotalAlloc - before.TotalAlloc + } + return sample, nil + } + for idx := 0; idx < warmupIterations; idx++ { + if _, err := run(-(idx + 1), false); err != nil { + return PostgresBoundaryWaterfall{}, err + } + } + result := PostgresBoundaryWaterfall{ + Boundary: "identical translated SQL through raw pgx pool/transaction/decode/drain", + SQLFingerprint: sqlFingerprint(sqlQuery), + WarmupIterations: warmupIterations, + Samples: make([]BoundarySample, 0, iterations), + } + var expectedRows int64 = -1 + for iteration := 1; iteration <= iterations; iteration++ { + sample, err := run(iteration, true) + if err != nil { + return PostgresBoundaryWaterfall{}, err + } + if expectedRows < 0 { + expectedRows = sample.Rows + } + if sample.Rows != expectedRows { + return PostgresBoundaryWaterfall{}, fmt.Errorf("raw pgx row count changed from %d to %d", expectedRows, sample.Rows) + } + result.Samples = append(result.Samples, sample) + } + return result, nil +} diff --git a/cmd/graphbench/waterfall_test.go b/cmd/graphbench/waterfall_test.go new file mode 100644 index 00000000..85c511f4 --- /dev/null +++ b/cmd/graphbench/waterfall_test.go @@ -0,0 +1,29 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "testing" + + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/specterops/dawgs/drivers/pg/pgutil" + "github.com/stretchr/testify/require" +) + +// TestMeasureCompileWaterfallMarksOverlappingIntervals verifies that compile phase timings are labeled non-additive and each requested sample records elapsed time and allocations. +func TestMeasureCompileWaterfallMarksOverlappingIntervals(t *testing.T) { + waterfall, err := measureCompileWaterfall(context.Background(), "MATCH (n) RETURN id(n)", nil, pgutil.NewInMemoryKindMapper(), 1, 2, translate.ToolOptions{}) + + require.NoError(t, err) + require.True(t, waterfall.IntervalsOverlap) + require.Contains(t, waterfall.Notes, "must not be summed") + require.Len(t, waterfall.Samples, 2) + for _, sample := range waterfall.Samples { + require.Positive(t, sample.Total) + require.Positive(t, sample.Allocations) + } +} diff --git a/cmd/integrationguard/main.go b/cmd/integrationguard/main.go new file mode 100644 index 00000000..c3f1f802 --- /dev/null +++ b/cmd/integrationguard/main.go @@ -0,0 +1,25 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "os" + + "github.com/specterops/dawgs/databaseguard" +) + +// main runs the integrationguard command. +func main() { + if err := databaseguard.Validate( + os.Getenv("CONNECTION_STRING"), + os.Getenv(databaseguard.AllowDestructiveEnv), + os.Getenv(databaseguard.DisposableTargetsEnv), + ); err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/cmd/plancorpus/README.md b/cmd/plancorpus/README.md index 75d8ee64..a4c8a477 100644 --- a/cmd/plancorpus/README.md +++ b/cmd/plancorpus/README.md @@ -3,20 +3,35 @@ `plancorpus` captures query-plan diagnostics for the shared integration corpus. It reads `integration/testdata/cases` and `integration/testdata/templates`, loads the same datasets and inline fixtures used by the integration tests, and writes backend-specific JSONL plan records plus markdown and JSON summaries. +Fixture-backed `node_params` and `node_list_params` are resolved after each +fixture load, preserving ID-anchored production query shapes in captured plans. Use this command to baseline PostgreSQL translator and optimizer changes. PostgreSQL captures include translated SQL, `EXPLAIN` output, plan operator counts, estimated plan cost, recursive CTE indicators, path materialization indicators, -planned lowerings, applied lowerings, skipped lowerings, and skipped-lowering reasons. Neo4j captures include logical -plan operator trees for cross-backend plan-shape comparison. +planned lowerings, applied lowerings, skipped lowerings, and skipped-lowering reasons. Neo4j read captures use `PROFILE` +after execution and retain ordered operators, estimated and actual rows, DB and page-cache hits, loops, and operator +time when the server exposes them. Writes remain `EXPLAIN`-only. + +Every run also writes a semantic PostgreSQL/Neo4j delta over the union of captured workloads. The delta is keyed by +workload hash and source revision, fingerprints each backend plan, compares access side, physical direction, predicate +placement, endpoint binding, traversal family, estimates, and PostgreSQL planned/emitted/fallback identities, and ranks +the largest disagreements. A missing or failed backend remains an explicit incomplete pair; it is never discarded by an +intersection-only comparison. Runtime-arm attribution remains GraphBench's responsibility. ## Usage ```bash -PG_CONNECTION_STRING="postgres://postgres:password@localhost/db" \ -NEO4J_CONNECTION_STRING="neo4j://neo4j:password@localhost:7687" \ -go run ./cmd/plancorpus +DAWGS_INTEGRATION_ALLOW_DESTRUCTIVE=1 \ +DAWGS_INTEGRATION_DISPOSABLE_TARGETS="postgresql://localhost:5432/db,neo4j://localhost:7687/" \ + PG_CONNECTION_STRING="postgres://postgres:password@localhost/db" \ + NEO4J_CONNECTION_STRING="neo4j://neo4j:password@localhost:7687" \ + go run ./cmd/plancorpus ``` +Plan capture reloads fixtures and refuses to open a selected backend unless the destructive acknowledgement is set and +its exact credential-free target is allowlisted. PostgreSQL aliases and omitted default ports are canonicalized; +multi-host PostgreSQL URLs are accepted only when every fallback resolves to the same target. + Useful flags: | Flag | Default | Description | @@ -28,16 +43,20 @@ Useful flags: | `-neo4j-connection` | `NEO4J_CONNECTION_STRING` | Neo4j backend | | `-summary` | `.coverage/plan-corpus-summary.md` | Markdown summary | | `-summary-json` | `.coverage/plan-corpus-summary.json` | JSON summary | +| `-plan-delta-json` | `.coverage/plan-corpus-delta.json` | Versioned paired semantic delta, including incomplete backend pairs | | `-top` | `25` | Number of expensive PostgreSQL plans to include in summaries | +| `-dawgs-version` | auto-detected | DAWGS source version recorded in output | ## Reviewing Captures The markdown summary is intended for human review. It ranks the highest-cost PostgreSQL plans, reports feature counts such as `Recursive Union`, `SubPlan`, and `Function Scan on unnest`, and summarizes planned/applied/skipped lowerings. -The JSON summary is intended for automation and baseline comparison. For optimizer work, check that intentional SQL +The JSON summary and paired delta are intended for automation and baseline comparison. For optimizer work, check that intentional SQL shape changes are explained and that skipped-lowering accounting remains actionable. A planned lowering without a matching applied lowering should either have a specific skipped reason or indicate a translator consumption bug. +Both per-query JSONL records and summaries include the DAWGS source version +needed to compare captures made from different worktrees. Expected capture errors should be limited to invalid-query cases surfaced by the integration corpus or backend-specific syntax differences. Unexpected capture errors should be treated as validation failures for planner or translator work. diff --git a/cmd/plancorpus/capture.go b/cmd/plancorpus/capture.go index d05a7046..f1214cb5 100644 --- a/cmd/plancorpus/capture.go +++ b/cmd/plancorpus/capture.go @@ -7,14 +7,17 @@ import ( "os" "path/filepath" "sort" + "strconv" "strings" "github.com/jackc/pgx/v5/pgxpool" neo4jcore "github.com/neo4j/neo4j-go-driver/v5/neo4j" "github.com/specterops/dawgs" "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/cypher/models/cypher" "github.com/specterops/dawgs/cypher/models/pgsql/optimize" "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/specterops/dawgs/databaseguard" "github.com/specterops/dawgs/drivers/neo4j" "github.com/specterops/dawgs/drivers/pg" "github.com/specterops/dawgs/graph" @@ -22,22 +25,34 @@ import ( "github.com/specterops/dawgs/util/size" ) +// defaultGraphName names the isolated graph populated while capturing corpus plans. const defaultGraphName = "integration_test" +// captureSpec binds a requested driver name to the connection string used for capture. type captureSpec struct { + // DriverName identifies the database driver selected for this capture. DriverName string + // Connection contains the backend connection string. Connection string } +// backendCapture owns one plan-capture backend and its graph database handle. type backendCapture struct { - spec captureSpec - db graph.Database - pgDriver *pg.Driver - pgGraphID int32 + // spec identifies the backend connection and driver being captured. + spec captureSpec + // db provides graph transactions for fixture preparation and query execution. + db graph.Database + // pgDriver provides PostgreSQL graph access and kind mapping. + pgDriver *pg.Driver + // pgGraphID selects the PostgreSQL graph partition cleared, populated, and queried during capture. + pgGraphID int32 + // neo4jDriver owns the Neo4j connection used for plan capture. neo4jDriver neo4jcore.Driver + // neo4jDBName selects the Neo4j database used for plan capture. neo4jDBName string } +// driverFromConnectionString selects a graph driver from the connection URI scheme. func driverFromConnectionString(connStr string) (string, error) { u, err := url.Parse(connStr) if err != nil { @@ -54,7 +69,12 @@ func driverFromConnectionString(connStr string) (string, error) { } } +// captureCorpus loads each required fixture and captures every corpus query for one backend. func captureCorpus(ctx context.Context, datasetDir string, suite corpus, spec captureSpec) ([]PlanRecord, error) { + if err := databaseguard.ValidateEnvironment(spec.Connection); err != nil { + return nil, fmt.Errorf("refuse destructive plan-corpus target: %w", err) + } + backend, err := openBackend(ctx, suite, spec) if err != nil { return nil, err @@ -87,23 +107,28 @@ func captureCorpus(ctx context.Context, datasetDir string, suite corpus, spec ca for _, file := range group.files { for _, testCase := range file.Cases { + var idMap opengraph.IDMap if testCase.Fixture == nil { if err := ensureDatasetLoaded(); err != nil { return nil, err } } else { - if err := loadCommittedFixture(ctx, backend.db, testCase.Fixture); err != nil { + if idMap, err = loadCommittedFixture(ctx, backend.db, testCase.Fixture); err != nil { return nil, err } datasetLoaded = false } + params, err := resolveFixtureParams(testCase.Params, testCase.NodeParams, testCase.NodeListParams, idMap) + if err != nil { + return nil, fmt.Errorf("%s/%s: %w", file.path, testCase.Name, err) + } record := backend.capture(ctx, CorpusQuery{ Source: file.path, Dataset: datasetName, Name: testCase.Name, Cypher: testCase.Cypher, - Params: testCase.Params, + Params: params, }) records = append(records, record) } @@ -123,15 +148,25 @@ func captureCorpus(ctx context.Context, datasetDir string, suite corpus, spec ca if err != nil { return nil, fmt.Errorf("%s/%s/%s: %w", file.path, family.Name, variant.Name, err) } - if err := loadCommittedFixture(ctx, backend.db, family.Fixture); err != nil { + idMap, err := loadCommittedFixture(ctx, backend.db, family.Fixture) + if err != nil { return nil, err } + params, err := resolveFixtureParams( + mergeParams(family.Params, variant.Params), + mergeStringMap(family.NodeParams, variant.NodeParams), + mergeStringListMap(family.NodeListParams, variant.NodeListParams), + idMap, + ) + if err != nil { + return nil, fmt.Errorf("%s/%s/%s: %w", file.path, family.Name, variant.Name, err) + } record := backend.capture(ctx, CorpusQuery{ Source: file.path, Name: fileName + "/" + family.Name + "/" + variant.Name, Cypher: rendered, - Params: mergeParams(family.Params, variant.Params), + Params: params, }) records = append(records, record) } @@ -141,7 +176,7 @@ func captureCorpus(ctx context.Context, datasetDir string, suite corpus, spec ca if family.Fixture == nil { return nil, fmt.Errorf("%s/%s has no fixture", file.path, family.Name) } - if err := loadCommittedFixture(ctx, backend.db, family.Fixture); err != nil { + if _, err := loadCommittedFixture(ctx, backend.db, family.Fixture); err != nil { return nil, err } @@ -160,6 +195,7 @@ func captureCorpus(ctx context.Context, datasetDir string, suite corpus, spec ca return records, nil } +// openBackend opens the requested graph backend, asserts the capture schema, and retains driver-specific plan handles. func openBackend(ctx context.Context, suite corpus, spec captureSpec) (*backendCapture, error) { cfg := dawgs.Config{ GraphQueryMemoryLimit: size.Gibibyte, @@ -232,6 +268,7 @@ func openBackend(ctx context.Context, suite corpus, spec captureSpec) (*backendC return backend, nil } +// close closes the backend driver resources owned by a capture. func (s *backendCapture) close(ctx context.Context) { if s.neo4jDriver != nil { _ = s.neo4jDriver.Close() @@ -241,14 +278,17 @@ func (s *backendCapture) close(ctx context.Context) { } } +// capture captures one query plan with driver, workload, and fixture metadata. func (s *backendCapture) capture(ctx context.Context, query CorpusQuery) PlanRecord { record := PlanRecord{ - Driver: s.spec.DriverName, - Source: query.Source, - Dataset: query.Dataset, - Name: query.Name, - Cypher: query.Cypher, - Params: query.Params, + SchemaVersion: planRecordSchemaVersion, + Driver: s.spec.DriverName, + Source: query.Source, + Dataset: query.Dataset, + Name: query.Name, + WorkloadSHA256: workloadFingerprint(query), + Cypher: query.Cypher, + Params: query.Params, } switch s.spec.DriverName { @@ -257,10 +297,13 @@ func (s *backendCapture) capture(ctx context.Context, query CorpusQuery) PlanRec case neo4j.DriverName: s.captureNeo4j(query.Cypher, query.Params, &record) } + record.PGPlanFingerprint = postgresPlanFingerprint(record.PGPlan) + record.Neo4jPlanFingerprint = neo4jPlanFingerprint(record.Neo4jPlan) return record } +// capturePostgres translates a Cypher query and attaches PostgreSQL EXPLAIN evidence to its record. func (s *backendCapture) capturePostgres(ctx context.Context, cypherQuery string, params map[string]any, record *PlanRecord) { regularQuery, err := frontend.ParseCypher(frontend.NewContext(), cypherQuery) if err != nil { @@ -307,14 +350,27 @@ func (s *backendCapture) capturePostgres(ctx context.Context, cypherQuery string record.Optimization = &translation.Optimization } +// captureNeo4j runs PROFILE for reads and EXPLAIN for writes, then attaches its normalized operator tree. func (s *backendCapture) captureNeo4j(cypherQuery string, params map[string]any, record *PlanRecord) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), cypherQuery) + if err != nil { + record.Error = err.Error() + return + } + write := regularQueryHasUpdates(regularQuery) + accessMode := neo4jcore.AccessModeRead + command := "PROFILE " + if write { + accessMode = neo4jcore.AccessModeWrite + command = "EXPLAIN " + } session := s.neo4jDriver.NewSession(neo4jcore.SessionConfig{ - AccessMode: neo4jcore.AccessModeWrite, + AccessMode: accessMode, DatabaseName: s.neo4jDBName, }) defer session.Close() - result, err := session.Run("EXPLAIN "+cypherWithoutTerminator(cypherQuery), params) + result, err := session.Run(command+cypherWithoutTerminator(cypherQuery), params) if err != nil { record.Error = err.Error() return @@ -326,20 +382,50 @@ func (s *backendCapture) captureNeo4j(cypherQuery string, params map[string]any, return } - if plan := summary.Plan(); plan != nil { + if profile := summary.Profile(); profile != nil { + planNode := convertNeo4jProfile(profile) + record.Neo4jPlan = &planNode + record.Neo4jOperators = neo4jOperators(planNode) + } else if plan := summary.Plan(); plan != nil { planNode := convertNeo4jPlan(plan) record.Neo4jPlan = &planNode record.Neo4jOperators = neo4jOperators(planNode) } } +// regularQueryHasUpdates reports whether any query part contains a mutation. +func regularQueryHasUpdates(query *cypher.RegularQuery) bool { + if query == nil || query.SingleQuery == nil { + return false + } + if single := query.SingleQuery.SinglePartQuery; single != nil { + return len(single.UpdatingClauses) > 0 + } + multi := query.SingleQuery.MultiPartQuery + if multi == nil { + return false + } + for _, part := range multi.Parts { + if part != nil && len(part.UpdatingClauses) > 0 { + return true + } + } + return multi.SinglePartQuery != nil && len(multi.SinglePartQuery.UpdatingClauses) > 0 +} + +// neo4jPlanDriverConfig contains a Neo4j server URI and optional target database parsed from a connection string. type neo4jPlanDriverConfig struct { - Target string - Username string - Password string + // Target contains the Neo4j server URI without a database path. + Target string + // Username contains the Neo4j username decoded from the connection URI. + Username string + // Password contains the Neo4j password decoded from the connection URI. + Password string + // DatabaseName selects the Neo4j database targeted by the session. DatabaseName string } +// parseNeo4jPlanDriverConfig parses a Neo4j connection string while preserving its server URI and database path. func parseNeo4jPlanDriverConfig(connStr string) (neo4jPlanDriverConfig, error) { connectionURL, err := url.Parse(connStr) if err != nil { @@ -376,6 +462,7 @@ func parseNeo4jPlanDriverConfig(connStr string) (neo4jPlanDriverConfig, error) { }, nil } +// neo4jDatabaseName returns the optional single-segment database name encoded in a Neo4j URI path. func neo4jDatabaseName(connectionURL *url.URL) (string, error) { databasePath := strings.Trim(connectionURL.EscapedPath(), "/") if databasePath == "" { @@ -397,6 +484,7 @@ func neo4jDatabaseName(connectionURL *url.URL) (string, error) { return databaseName, nil } +// openNeo4jPlanDriver parses the capture connection settings and returns a driver together with the selected Neo4j database name. func openNeo4jPlanDriver(connStr string) (neo4jcore.Driver, string, error) { cfg, err := parseNeo4jPlanDriverConfig(connStr) if err != nil { @@ -414,12 +502,45 @@ func openNeo4jPlanDriver(connStr string) (neo4jcore.Driver, string, error) { return driver, cfg.DatabaseName, nil } +// clearGraph removes relationships before nodes, using PostgreSQL partition truncation when available. func clearGraph(ctx context.Context, db graph.Database) error { + if pgDriver, isPostgres := db.(*pg.Driver); isPostgres { + graphTarget, hasDefaultGraph := pgDriver.DefaultGraph() + if !hasDefaultGraph { + return fmt.Errorf("PostgreSQL default graph is not set") + } + + return clearPostgresGraph(ctx, db, graphTarget.ID) + } + + return db.WriteTransaction(ctx, func(tx graph.Transaction) error { + if err := tx.Relationships().Delete(); err != nil { + return fmt.Errorf("delete relationships: %w", err) + } + + if err := tx.Nodes().Delete(); err != nil { + return fmt.Errorf("delete nodes: %w", err) + } + + return nil + }) +} + +// clearPostgresGraph truncates one PostgreSQL graph's edge and node partitions in a transaction. +func clearPostgresGraph(ctx context.Context, db graph.Database, graphID int32) error { return db.WriteTransaction(ctx, func(tx graph.Transaction) error { - return tx.Nodes().Delete() + statement := fmt.Sprintf("truncate table edge_%d, node_%d", graphID, graphID) + result := tx.Raw(statement, nil) + result.Close() + if err := result.Error(); err != nil { + return fmt.Errorf("execute PostgreSQL graph reset: %w", err) + } + + return nil }) } +// loadDataset decodes and loads a named fixture dataset into an empty graph. func loadDataset(ctx context.Context, db graph.Database, datasetDir, name string) error { f, err := os.Open(filepath.Join(datasetDir, name+".json")) if err != nil { @@ -433,27 +554,36 @@ func loadDataset(ctx context.Context, db graph.Database, datasetDir, name string return nil } -func loadCommittedFixture(ctx context.Context, db graph.Database, fixture *opengraph.Graph) error { +// loadCommittedFixture loads an inline fixture graph and returns its stable key-to-ID mapping. +func loadCommittedFixture(ctx context.Context, db graph.Database, fixture *opengraph.Graph) (opengraph.IDMap, error) { if fixture == nil { - return fmt.Errorf("fixture is nil") + return nil, fmt.Errorf("fixture is nil") } if err := clearGraph(ctx, db); err != nil { - return err + return nil, err } - return db.WriteTransaction(ctx, func(tx graph.Transaction) error { - _, err := opengraph.WriteGraphTx(tx, fixture) + var idMap opengraph.IDMap + if err := db.WriteTransaction(ctx, func(tx graph.Transaction) error { + var err error + idMap, err = opengraph.WriteGraphTx(tx, fixture) return err - }) + }); err != nil { + return nil, err + } + + return idMap, nil } +// convertNeo4jPlan recursively converts a Neo4j plan into the stable serialized plan-node schema. func convertNeo4jPlan(plan neo4jcore.Plan) Neo4jPlanNode { node := Neo4jPlanNode{ - Operator: plan.Operator(), + Operator: normalizeNeo4jOperator(plan.Operator()), Arguments: stringifyArguments(plan.Arguments()), Identifiers: append([]string(nil), plan.Identifiers()...), } + node.EstimatedRows = neo4jArgumentFloat(node.Arguments, "EstimatedRows") for _, child := range plan.Children() { node.Children = append(node.Children, convertNeo4jPlan(child)) @@ -462,6 +592,60 @@ func convertNeo4jPlan(plan neo4jcore.Plan) Neo4jPlanNode { return node } +// convertNeo4jProfile recursively converts executed read-plan evidence. +func convertNeo4jProfile(plan neo4jcore.ProfiledPlan) Neo4jPlanNode { + rows, dbHits := plan.Records(), plan.DbHits() + node := Neo4jPlanNode{ + Operator: normalizeNeo4jOperator(plan.Operator()), + Arguments: stringifyArguments(plan.Arguments()), + Identifiers: append([]string(nil), plan.Identifiers()...), + ActualRows: &rows, + DBHits: optionalNonnegativeInt64(dbHits), + PageCacheHits: optionalNonnegativeInt64(plan.PageCacheHits()), + PageCacheMisses: optionalNonnegativeInt64(plan.PageCacheMisses()), + TimeNS: optionalNonnegativeInt64(plan.Time()), + } + node.EstimatedRows = neo4jArgumentFloat(node.Arguments, "EstimatedRows") + for _, child := range plan.Children() { + node.Children = append(node.Children, convertNeo4jProfile(child)) + } + return node +} + +// optionalNonnegativeInt64 distinguishes unavailable profiler values from zero. +func optionalNonnegativeInt64(value int64) *int64 { + if value < 0 { + return nil + } + return &value +} + +// neo4jArgumentFloat parses an optional numeric plan argument. +func neo4jArgumentFloat(arguments map[string]string, key string) *float64 { + value, found := arguments[key] + if !found { + return nil + } + parsed, err := strconv.ParseFloat(value, 64) + if err != nil { + return nil + } + return &parsed +} + +// normalizeNeo4jOperator removes repeated backend suffixes and applies exactly one. +func normalizeNeo4jOperator(operator string) string { + operator = strings.TrimSpace(operator) + for strings.HasSuffix(operator, "@neo4j") { + operator = strings.TrimSuffix(operator, "@neo4j") + } + if operator == "" { + return "" + } + return operator + "@neo4j" +} + +// stringifyArguments converts plan arguments to stable strings in a fresh map. func stringifyArguments(arguments map[string]any) map[string]string { if len(arguments) == 0 { return nil @@ -474,6 +658,7 @@ func stringifyArguments(arguments map[string]any) map[string]string { return values } +// postgresOperators extracts normalized operator names from PostgreSQL text plans. func postgresOperators(plan []string) []string { operators := make([]string, 0, len(plan)) for _, line := range plan { @@ -491,6 +676,7 @@ func postgresOperators(plan []string) []string { return operators } +// neo4jOperators flattens a Neo4j plan tree into sorted unique operator names. func neo4jOperators(root Neo4jPlanNode) []string { var ( operators []string @@ -498,7 +684,7 @@ func neo4jOperators(root Neo4jPlanNode) []string { ) walk = func(node Neo4jPlanNode) { - operators = append(operators, node.Operator) + operators = append(operators, normalizeNeo4jOperator(node.Operator)) for _, child := range node.Children { walk(child) } @@ -507,6 +693,7 @@ func neo4jOperators(root Neo4jPlanNode) []string { return operators } +// loweringNames returns sorted unique names of applied SQL lowering decisions. func loweringNames(decisions []optimize.LoweringDecision) []string { if len(decisions) == 0 { return nil @@ -529,6 +716,7 @@ func loweringNames(decisions []optimize.LoweringDecision) []string { return names } +// cypherWithoutTerminator trims surrounding whitespace and one trailing Cypher semicolon. func cypherWithoutTerminator(cypherQuery string) string { return strings.TrimSuffix(strings.TrimSpace(cypherQuery), ";") } diff --git a/cmd/plancorpus/corpus.go b/cmd/plancorpus/corpus.go index 46fdd4e0..b398f6a1 100644 --- a/cmd/plancorpus/corpus.go +++ b/cmd/plancorpus/corpus.go @@ -10,66 +10,120 @@ import ( "github.com/specterops/dawgs/graph" "github.com/specterops/dawgs/opengraph" + "github.com/specterops/dawgs/testutil" ) +// corpus contains loaded corpus queries and their dataset definitions. type corpus struct { - caseGroups map[string]*caseGroup - datasetNames []string + // caseGroups indexes loaded corpus cases by dataset name. + caseGroups map[string]*caseGroup + // datasetNames lists fixture datasets in deterministic plan-capture order. + datasetNames []string + // templateFiles retains decoded template files for corpus expansion. templateFiles []templateFile - nodeKinds graph.Kinds - edgeKinds graph.Kinds + // nodeKinds contains every node kind declared by loaded fixtures. + nodeKinds graph.Kinds + // edgeKinds contains every relationship kind declared by loaded fixtures. + edgeKinds graph.Kinds } +// caseGroup models a case-group entry in a scale-corpus JSON file. type caseGroup struct { + // dataset names the fixture shared by every case file in the group. dataset string - files []caseFile + // files retains source case files contributing to a dataset group. + files []caseFile } +// caseFile models the top-level groups in a scale-corpus case file. type caseFile struct { - path string - Dataset string `json:"dataset"` - Cases []caseEntry `json:"cases"` + // path retains the source path used in errors and provenance. + path string + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Cases contains query cases declared by this source file. + Cases []caseEntry `json:"cases"` } +// caseEntry models one named query case and its parameter declarations. type caseEntry struct { - Name string `json:"name"` - Cypher string `json:"cypher"` - Params map[string]any `json:"params,omitempty"` + // Name identifies the query case within its dataset. + Name string `json:"name"` + // Cypher contains the Cypher statement under test. + Cypher string `json:"cypher"` + // Params supplies literal query parameters. + Params testutil.Params `json:"params,omitempty"` + // NodeParams maps query parameters to fixture node keys. + NodeParams map[string]string `json:"node_params,omitempty"` + // NodeListParams maps query parameters to ordered fixture node-key lists. + NodeListParams map[string][]string `json:"node_list_params,omitempty"` + // Fixture captures the fixture identity and cardinality contract. Fixture *opengraph.Graph `json:"fixture,omitempty"` } +// templateFile models template and metamorphic query families from a corpus template file. type templateFile struct { - path string - Families []templateFamily `json:"families,omitempty"` + // path retains the source path used in errors and provenance. + path string + // Families lists query-template families decoded from the file. + Families []templateFamily `json:"families,omitempty"` + // Metamorphic lists metamorphic query families decoded from the file. Metamorphic []metamorphicFamily `json:"metamorphic,omitempty"` } +// templateFamily defines a base query and the variants rendered from it. type templateFamily struct { - Name string `json:"name"` - Template string `json:"template"` - Params map[string]any `json:"params,omitempty"` - Fixture *opengraph.Graph `json:"fixture,omitempty"` + // Name identifies the query-template family in expanded case names. + Name string `json:"name"` + // Template contains the Cypher template rendered for each variant. + Template string `json:"template"` + // Params supplies literal query parameters. + Params testutil.Params `json:"params,omitempty"` + // NodeParams maps query parameters to fixture node keys. + NodeParams map[string]string `json:"node_params,omitempty"` + // NodeListParams maps query parameters to ordered fixture node-key lists. + NodeListParams map[string][]string `json:"node_list_params,omitempty"` + // Fixture captures the fixture identity and cardinality contract. + Fixture *opengraph.Graph `json:"fixture,omitempty"` + // Variants lists substitutions rendered from the base query template. Variants []templateVariant `json:"variants"` } +// templateVariant defines one named substitution set for a query template. type templateVariant struct { - Name string `json:"name"` - Vars map[string]string `json:"vars"` - Params map[string]any `json:"params,omitempty"` + // Name identifies this substitution set in the rendered case name. + Name string `json:"name"` + // Vars maps template placeholders to replacement text. + Vars map[string]string `json:"vars"` + // Params supplies literal query parameters. + Params testutil.Params `json:"params,omitempty"` + // NodeParams maps query parameters to fixture node keys. + NodeParams map[string]string `json:"node_params,omitempty"` + // NodeListParams maps query parameters to ordered fixture node-key lists. + NodeListParams map[string][]string `json:"node_list_params,omitempty"` } +// metamorphicFamily groups semantically equivalent queries used for plan comparison. type metamorphicFamily struct { - Name string `json:"name"` - Fixture *opengraph.Graph `json:"fixture,omitempty"` + // Name identifies the family of queries expected to remain semantically equivalent. + Name string `json:"name"` + // Fixture captures the fixture identity and cardinality contract. + Fixture *opengraph.Graph `json:"fixture,omitempty"` + // Queries lists semantically equivalent queries in the metamorphic family. Queries []metamorphicQuery `json:"queries"` } +// metamorphicQuery defines one named query in a metamorphic family. type metamorphicQuery struct { - Name string `json:"name"` - Cypher string `json:"cypher"` - Params map[string]any `json:"params,omitempty"` + // Name identifies one query variant within its metamorphic family. + Name string `json:"name"` + // Cypher contains the Cypher statement under test. + Cypher string `json:"cypher"` + // Params supplies literal query parameters. + Params testutil.Params `json:"params,omitempty"` } +// loadCorpus loads case, template, and dataset-kind declarations from a corpus directory. func loadCorpus(datasetDir string) (corpus, error) { var loaded corpus loaded.caseGroups = map[string]*caseGroup{} @@ -88,6 +142,7 @@ func loadCorpus(datasetDir string) (corpus, error) { return loaded, nil } +// loadCaseFiles decodes case files and indexes them by dataset while retaining source paths. func (s *corpus) loadCaseFiles(datasetDir string) error { paths, err := filepath.Glob(filepath.Join(datasetDir, "cases", "*.json")) if err != nil { @@ -123,6 +178,7 @@ func (s *corpus) loadCaseFiles(datasetDir string) error { return nil } +// loadTemplateFiles renders template variants and metamorphic families into executable corpus cases. func (s *corpus) loadTemplateFiles(datasetDir string) error { paths, err := filepath.Glob(filepath.Join(datasetDir, "templates", "*.json")) if err != nil { @@ -149,6 +205,7 @@ func (s *corpus) loadTemplateFiles(datasetDir string) error { return nil } +// loadDatasetKinds loads fixture graphs and accumulates the node and relationship kinds they declare. func (s *corpus) loadDatasetKinds(datasetDir string) error { for _, datasetName := range s.datasetNames { path := filepath.Join(datasetDir, datasetName+".json") @@ -174,6 +231,7 @@ func (s *corpus) loadDatasetKinds(datasetDir string) error { return nil } +// addFixtureKinds unions a fixture's node and relationship kinds into the corpus kind sets. func (s *corpus) addFixtureKinds(fixture *opengraph.Graph) { if fixture == nil { return @@ -184,6 +242,7 @@ func (s *corpus) addFixtureKinds(fixture *opengraph.Graph) { s.edgeKinds = s.edgeKinds.Add(edgeKinds...) } +// decodeJSONFile reads a JSON file and decodes it into the supplied destination. func decodeJSONFile(path string, target any) error { raw, err := os.ReadFile(path) if err != nil { @@ -195,6 +254,7 @@ func decodeJSONFile(path string, target any) error { return nil } +// renderTemplate substitutes every named placeholder and rejects any unresolved template markers. func renderTemplate(template string, vars map[string]string) (string, error) { rendered := template for name, value := range vars { @@ -206,6 +266,7 @@ func renderTemplate(template string, vars map[string]string) (string, error) { return rendered, nil } +// mergeParams returns a copied parameter map in which override values take precedence. func mergeParams(base, overrides map[string]any) map[string]any { if len(base) == 0 && len(overrides) == 0 { return nil @@ -220,3 +281,73 @@ func mergeParams(base, overrides map[string]any) map[string]any { } return merged } + +// mergeStringMap returns a copied string map in which override values take precedence. +func mergeStringMap(base, overrides map[string]string) map[string]string { + if len(base) == 0 && len(overrides) == 0 { + return nil + } + + merged := make(map[string]string, len(base)+len(overrides)) + for key, value := range base { + merged[key] = value + } + for key, value := range overrides { + merged[key] = value + } + return merged +} + +// mergeStringListMap returns a deep-enough copy of string-list parameters with overrides applied. +func mergeStringListMap(base, overrides map[string][]string) map[string][]string { + if len(base) == 0 && len(overrides) == 0 { + return nil + } + + merged := make(map[string][]string, len(base)+len(overrides)) + for key, value := range base { + merged[key] = append([]string(nil), value...) + } + for key, value := range overrides { + merged[key] = append([]string(nil), value...) + } + return merged +} + +// resolveFixtureParams replaces symbolic node keys and key lists with fixture database identifiers. +func resolveFixtureParams( + params map[string]any, + nodeParams map[string]string, + nodeListParams map[string][]string, + idMap opengraph.IDMap, +) (map[string]any, error) { + resolved := make(map[string]any, len(params)+len(nodeParams)+len(nodeListParams)) + for name, value := range params { + resolved[name] = value + } + + for paramName, fixtureID := range nodeParams { + id, found := idMap[fixtureID] + if !found { + return nil, fmt.Errorf("node parameter %q references unknown fixture ID %q", paramName, fixtureID) + } + resolved[paramName] = id.Int64() + } + + for paramName, fixtureIDs := range nodeListParams { + ids := make([]int64, len(fixtureIDs)) + for idx, fixtureID := range fixtureIDs { + id, found := idMap[fixtureID] + if !found { + return nil, fmt.Errorf("node list parameter %q references unknown fixture ID %q", paramName, fixtureID) + } + ids[idx] = id.Int64() + } + resolved[paramName] = ids + } + + if len(resolved) == 0 { + return nil, nil + } + return resolved, nil +} diff --git a/cmd/plancorpus/corpus_test.go b/cmd/plancorpus/corpus_test.go index 141fa515..5b8996d3 100644 --- a/cmd/plancorpus/corpus_test.go +++ b/cmd/plancorpus/corpus_test.go @@ -4,9 +4,13 @@ import ( "path/filepath" "testing" + "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/opengraph" "github.com/stretchr/testify/require" ) +// TestLoadCorpus verifies that integration fixtures populate case groups, datasets, templates, and both node and edge kind catalogs. func TestLoadCorpus(t *testing.T) { suite, err := loadCorpus(filepath.Join("..", "..", "integration", "testdata")) require.NoError(t, err) @@ -18,6 +22,26 @@ func TestLoadCorpus(t *testing.T) { require.NotEmpty(t, suite.edgeKinds) } +// TestCorpusTemplatesParse verifies that every declared template variant renders without placeholders and parses as Cypher. +func TestCorpusTemplatesParse(t *testing.T) { + suite, err := loadCorpus(filepath.Join("..", "..", "integration", "testdata")) + require.NoError(t, err) + + for _, file := range suite.templateFiles { + for _, family := range file.Families { + for _, variant := range family.Variants { + t.Run(family.Name+"/"+variant.Name, func(t *testing.T) { + rendered, err := renderTemplate(family.Template, variant.Vars) + require.NoError(t, err) + _, err = frontend.ParseCypher(frontend.NewContext(), rendered) + require.NoError(t, err) + }) + } + } + } +} + +// TestRenderTemplateRequiresAllPlaceholders verifies successful substitution and rejection when any template marker remains unresolved. func TestRenderTemplateRequiresAllPlaceholders(t *testing.T) { rendered, err := renderTemplate("match ({{name}}) return {{name}}", map[string]string{"name": "n"}) require.NoError(t, err) @@ -27,8 +51,28 @@ func TestRenderTemplateRequiresAllPlaceholders(t *testing.T) { require.ErrorContains(t, err, "unresolved placeholders") } +// TestMergeParams verifies right-hand override precedence, retention of unrelated values, and a nil result for two absent maps. func TestMergeParams(t *testing.T) { merged := mergeParams(map[string]any{"a": 1, "b": 2}, map[string]any{"b": 3}) require.Equal(t, map[string]any{"a": 1, "b": 3}, merged) require.Nil(t, mergeParams(nil, nil)) } + +// TestResolveFixtureParams verifies scalar/list key resolution to ordered int64 IDs and reports an unknown fixture key. +func TestResolveFixtureParams(t *testing.T) { + params, err := resolveFixtureParams( + map[string]any{"literal": "value"}, + map[string]string{"start_id": "start"}, + map[string][]string{"end_ids": {"end", "start"}}, + opengraph.IDMap{"start": graph.ID(11), "end": graph.ID(22)}, + ) + require.NoError(t, err) + require.Equal(t, map[string]any{ + "literal": "value", + "start_id": int64(11), + "end_ids": []int64{22, 11}, + }, params) + + _, err = resolveFixtureParams(nil, map[string]string{"missing": "unknown"}, nil, opengraph.IDMap{}) + require.ErrorContains(t, err, "unknown fixture ID") +} diff --git a/cmd/plancorpus/destructive_guard_test.go b/cmd/plancorpus/destructive_guard_test.go new file mode 100644 index 00000000..a027522d --- /dev/null +++ b/cmd/plancorpus/destructive_guard_test.go @@ -0,0 +1,26 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "testing" + + "github.com/specterops/dawgs/databaseguard" + "github.com/stretchr/testify/require" +) + +// TestCaptureCorpusRequiresTargetAuthorization verifies that plan capture refuses an unallowlisted PostgreSQL target before loading destructive fixture data. +func TestCaptureCorpusRequiresTargetAuthorization(t *testing.T) { + t.Setenv(databaseguard.AllowDestructiveEnv, "") + t.Setenv(databaseguard.DisposableTargetsEnv, "") + + _, err := captureCorpus(context.Background(), "", corpus{}, captureSpec{ + DriverName: pgDriverName(), + Connection: "postgresql://user:secret@localhost/dawgs", + }) + require.ErrorContains(t, err, "refuse destructive plan-corpus") +} diff --git a/cmd/plancorpus/dormant_forms_guard_test.go b/cmd/plancorpus/dormant_forms_guard_test.go new file mode 100644 index 00000000..45bdb5c1 --- /dev/null +++ b/cmd/plancorpus/dormant_forms_guard_test.go @@ -0,0 +1,60 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestDormantFormsStayOutOfPlanCorpus verifies that active cases, variants, and metamorphic query names never expose FUTURE-prefixed query forms. +func TestDormantFormsStayOutOfPlanCorpus(t *testing.T) { + suite, err := loadCorpus("../../integration/testdata") + require.NoError(t, err) + + for _, group := range suite.caseGroups { + for _, file := range group.files { + for _, testCase := range file.Cases { + requireNoDormantPlanQueryFormID(t, file.path+" case", testCase.Name) + } + } + } + + for _, file := range suite.templateFiles { + for _, family := range file.Families { + requireNoDormantPlanQueryFormID(t, file.path+" family", family.Name) + for _, variant := range family.Variants { + requireNoDormantPlanQueryFormID(t, file.path+" variant", variant.Name) + } + } + for _, family := range file.Metamorphic { + requireNoDormantPlanQueryFormID(t, file.path+" metamorphic family", family.Name) + for _, query := range family.Queries { + requireNoDormantPlanQueryFormID(t, file.path+" metamorphic query", query.Name) + } + } + } +} + +// requireNoDormantPlanQueryFormID rejects a corpus field containing the reserved FUTURE marker, independent of letter case. +func requireNoDormantPlanQueryFormID(t *testing.T, field, value string) { + t.Helper() + require.False(t, strings.Contains(strings.ToUpper(value), "FUTURE-"), + "%s %q places a dormant query form in the active plan corpus", field, value) +} diff --git a/cmd/plancorpus/main.go b/cmd/plancorpus/main.go index 152a0beb..32d73e11 100644 --- a/cmd/plancorpus/main.go +++ b/cmd/plancorpus/main.go @@ -8,29 +8,47 @@ import ( "io" "os" "path/filepath" + + "github.com/specterops/dawgs/testutil" ) +// commandConfig contains plancorpus command-line inputs and output selections. type commandConfig struct { - DatasetDir string - OutputDir string + // DatasetDir locates fixture datasets loaded before plan capture. + DatasetDir string + // OutputDir selects the directory that receives captured plans and summaries. + OutputDir string + // SummaryMarkdown selects the Markdown plan-summary destination. SummaryMarkdown string - SummaryJSON string - Connection string - PGConnection string + // SummaryJSON selects the JSON summary destination. + SummaryJSON string + // PlanDeltaJSON selects the versioned paired plan-delta destination. + PlanDeltaJSON string + // Connection contains the backend connection string. + Connection string + // PGConnection contains the PostgreSQL connection string. + PGConnection string + // Neo4jConnection contains the Neo4j connection string. Neo4jConnection string - TopPlans int + // TopPlans limits expensive PostgreSQL plans included in the summary. + TopPlans int + // DAWGSVersion records the DAWGS source version attached to artifact provenance. + DAWGSVersion string } +// main runs the plancorpus command. func main() { cfg := commandConfig{} flag.StringVar(&cfg.DatasetDir, "dataset-dir", "integration/testdata", "integration testdata directory") flag.StringVar(&cfg.OutputDir, "output-dir", ".coverage", "directory for JSONL plan captures") flag.StringVar(&cfg.SummaryMarkdown, "summary", "", "markdown summary path (default: output-dir/plan-corpus-summary.md)") flag.StringVar(&cfg.SummaryJSON, "summary-json", "", "JSON summary path (default: output-dir/plan-corpus-summary.json)") + flag.StringVar(&cfg.PlanDeltaJSON, "plan-delta-json", "", "paired semantic plan-delta path (default: output-dir/plan-corpus-delta.json)") flag.StringVar(&cfg.Connection, "connection", os.Getenv("CONNECTION_STRING"), "single backend connection string") flag.StringVar(&cfg.PGConnection, "pg-connection", os.Getenv("PG_CONNECTION_STRING"), "PostgreSQL connection string") flag.StringVar(&cfg.Neo4jConnection, "neo4j-connection", os.Getenv("NEO4J_CONNECTION_STRING"), "Neo4j connection string") flag.IntVar(&cfg.TopPlans, "top", defaultTopPlans, "number of expensive PostgreSQL plans to include in summaries") + flag.StringVar(&cfg.DAWGSVersion, "dawgs-version", "", "DAWGS source version (auto-detected when empty)") flag.Parse() if err := run(context.Background(), cfg); err != nil { @@ -39,6 +57,7 @@ func main() { } } +// run captures plans for each configured backend and writes aggregate summaries. func run(ctx context.Context, cfg commandConfig) error { specs, err := captureSpecs(cfg) if err != nil { @@ -55,12 +74,17 @@ func run(ctx context.Context, cfg commandConfig) error { } var allRecords []PlanRecord + metadata := testutil.ResolveBaselineMetadata(cfg.DAWGSVersion) for _, spec := range specs { records, err := captureCorpus(ctx, cfg.DatasetDir, suite, spec) if err != nil { return err } + for idx := range records { + records[idx].Metadata = metadata + } + outputPath := filepath.Join(cfg.OutputDir, "plan-corpus-"+spec.DriverName+".jsonl") if err := writePlanRecords(outputPath, records); err != nil { return err @@ -81,10 +105,22 @@ func run(ctx context.Context, cfg commandConfig) error { if err := writeSummaryFiles(cfg.SummaryMarkdown, cfg.SummaryJSON, summary); err != nil { return err } + planDelta, err := buildPlanDeltaReport(allRecords) + if err != nil { + return err + } + if cfg.PlanDeltaJSON == "" { + cfg.PlanDeltaJSON = filepath.Join(cfg.OutputDir, "plan-corpus-delta.json") + } + if err := writePlanDeltaReport(cfg.PlanDeltaJSON, planDelta); err != nil { + return err + } fmt.Fprintf(os.Stderr, "wrote summaries to %s and %s\n", cfg.SummaryMarkdown, cfg.SummaryJSON) + fmt.Fprintf(os.Stderr, "wrote paired plan delta to %s\n", cfg.PlanDeltaJSON) return nil } +// captureSpecs validates connection inputs and returns one deterministic capture specification per driver. func captureSpecs(cfg commandConfig) ([]captureSpec, error) { specsByDriver := map[string]captureSpec{} @@ -129,14 +165,17 @@ func captureSpecs(cfg commandConfig) ([]captureSpec, error) { return specs, nil } +// pgDriverName returns the registered driver name for PostgreSQL connections. func pgDriverName() string { return "pg" } +// neo4jDriverName returns the registered driver name for Neo4j connections. func neo4jDriverName() string { return "neo4j" } +// writePlanRecords creates a JSON Lines artifact and writes every captured plan record to it. func writePlanRecords(path string, records []PlanRecord) error { out, err := os.Create(path) if err != nil { @@ -146,6 +185,7 @@ func writePlanRecords(path string, records []PlanRecord) error { return writePlanRecordsTo(out, path, records) } +// writePlanRecordsTo encodes plan records as JSON Lines and reports both encode and close failures. func writePlanRecordsTo(out io.WriteCloser, path string, records []PlanRecord) error { encoder := json.NewEncoder(out) for _, record := range records { @@ -162,6 +202,7 @@ func writePlanRecordsTo(out io.WriteCloser, path string, records []PlanRecord) e return nil } +// writeSummaryFiles writes the requested Markdown and JSON plan summaries and closes each output. func writeSummaryFiles(markdownPath, jsonPath string, summary PlanSummary) error { if markdownPath != "" { out, err := os.Create(markdownPath) diff --git a/cmd/plancorpus/main_test.go b/cmd/plancorpus/main_test.go index 17aca49a..18829fd0 100644 --- a/cmd/plancorpus/main_test.go +++ b/cmd/plancorpus/main_test.go @@ -7,18 +7,25 @@ import ( "path/filepath" "testing" + "github.com/specterops/dawgs/cypher/frontend" "github.com/stretchr/testify/require" ) +// closeErrorWriter wraps an in-memory buffer and injects a Close error for output tests. type closeErrorWriter struct { + // Buffer captures bytes written before the injected Close failure. bytes.Buffer + + // err is returned after serialization attempts to close the destination. err error } +// Close returns the injected failure used to verify output finalization errors. func (s *closeErrorWriter) Close() error { return s.err } +// TestCaptureSpecs verifies that backend-specific connection flags override the generic URI and produce PostgreSQL then Neo4j capture specs. func TestCaptureSpecs(t *testing.T) { specs, err := captureSpecs(commandConfig{ Connection: "neo4j://neo4j:password@localhost:7687", @@ -35,32 +42,42 @@ func TestCaptureSpecs(t *testing.T) { }}, specs) } +// TestCaptureSpecsRequiresConnection verifies that capture cannot proceed when no generic or backend-specific connection URI is supplied. func TestCaptureSpecsRequiresConnection(t *testing.T) { _, err := captureSpecs(commandConfig{}) require.ErrorContains(t, err, "no connection string supplied") } +// TestWritePlanRecordsWritesJSONLines verifies the stable JSON Lines schema, including source query identity and default metadata. func TestWritePlanRecordsWritesJSONLines(t *testing.T) { path := filepath.Join(t.TempDir(), "records.jsonl") err := writePlanRecords(path, []PlanRecord{{ - Driver: "pg", - Source: "cases/example.json", - Name: "example", - Cypher: "MATCH (n) RETURN n", + SchemaVersion: planRecordSchemaVersion, + Driver: "pg", + Source: "cases/example.json", + Name: "example", + WorkloadSHA256: "workload", + Cypher: "MATCH (n) RETURN n", }}) require.NoError(t, err) contents, err := os.ReadFile(path) require.NoError(t, err) require.JSONEq(t, `{ + "schema_version": 2, "driver": "pg", "source": "cases/example.json", "name": "example", - "cypher": "MATCH (n) RETURN n" + "workload_sha256": "workload", + "cypher": "MATCH (n) RETURN n", + "metadata": { + "dawgs_version": "" + } }`, string(bytes.TrimSpace(contents))) } +// TestWritePlanRecordsToReturnsCloseError verifies that destination close failures retain the output path in their diagnostic. func TestWritePlanRecordsToReturnsCloseError(t *testing.T) { writer := &closeErrorWriter{err: errors.New("close failed")} @@ -70,6 +87,7 @@ func TestWritePlanRecordsToReturnsCloseError(t *testing.T) { require.ErrorContains(t, err, "close failed") } +// TestWritePlanRecordsToClosesAfterEncodeError verifies that encoding and close failures are joined so cleanup is attempted without losing the primary serialization error. func TestWritePlanRecordsToClosesAfterEncodeError(t *testing.T) { writer := &closeErrorWriter{err: errors.New("close failed")} @@ -85,6 +103,7 @@ func TestWritePlanRecordsToClosesAfterEncodeError(t *testing.T) { require.ErrorContains(t, err, "close failed") } +// TestDriverFromConnectionString verifies PostgreSQL and all supported Neo4j routing schemes and rejects an unrelated database protocol. func TestDriverFromConnectionString(t *testing.T) { driverName, err := driverFromConnectionString("postgresql://postgres:password@localhost/db") require.NoError(t, err) @@ -104,11 +123,19 @@ func TestDriverFromConnectionString(t *testing.T) { require.ErrorContains(t, err, "unknown connection string scheme") } +// TestParseNeo4jPlanDriverConfigPreservesURI verifies credentials extraction while preserving routing security, host, query, and an optional single database name. func TestParseNeo4jPlanDriverConfigPreservesURI(t *testing.T) { testCases := []struct { - name string - connStr string - expectedTarget string + // name identifies the routing form in subtest diagnostics. + name string + + // connStr is the credential-bearing URI accepted by the parser. + connStr string + + // expectedTarget is the credential-free driver URI after database-path extraction. + expectedTarget string + + // expectedDatabase is the optional database parsed from the sole path segment. expectedDatabase string }{{ name: "plain routing", @@ -139,6 +166,7 @@ func TestParseNeo4jPlanDriverConfigPreservesURI(t *testing.T) { } } +// TestParseNeo4jPlanDriverConfigRejectsNestedDatabasePath verifies that literal and percent-encoded nested paths cannot masquerade as one Neo4j database name. func TestParseNeo4jPlanDriverConfigRejectsNestedDatabasePath(t *testing.T) { for _, connStr := range []string{ "neo4j://neo4j:password@localhost:7687/db/extra", @@ -148,3 +176,22 @@ func TestParseNeo4jPlanDriverConfigRejectsNestedDatabasePath(t *testing.T) { require.ErrorContains(t, err, "single database name") } } + +// TestRegularQueryHasUpdatesDistinguishesReadProfilesFromWriteExplains verifies +// the PlanCorpus Neo4j command boundary cannot execute mutations during capture. +func TestRegularQueryHasUpdatesDistinguishesReadProfilesFromWriteExplains(t *testing.T) { + for _, testCase := range []struct { + query string + write bool + }{{query: "MATCH (n) RETURN n", write: false}, {query: "CREATE (n) RETURN n", write: true}, {query: "MATCH (n) WITH n SET n.x = 1 RETURN n", write: true}} { + parsed, err := frontend.ParseCypher(frontend.NewContext(), testCase.query) + require.NoError(t, err) + require.Equal(t, testCase.write, regularQueryHasUpdates(parsed), testCase.query) + } +} + +// TestNormalizeNeo4jOperatorAppliesOneSuffix verifies historical doubled backend suffixes are canonicalized. +func TestNormalizeNeo4jOperatorAppliesOneSuffix(t *testing.T) { + require.Equal(t, "ShortestPath@neo4j", normalizeNeo4jOperator("ShortestPath@neo4j@neo4j")) + require.Equal(t, "ShortestPath@neo4j", normalizeNeo4jOperator("ShortestPath")) +} diff --git a/cmd/plancorpus/plan_delta.go b/cmd/plancorpus/plan_delta.go new file mode 100644 index 00000000..d8c6859c --- /dev/null +++ b/cmd/plancorpus/plan_delta.go @@ -0,0 +1,762 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "os" + "reflect" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/specterops/dawgs/cypher/models/pgsql/translate" +) + +const planDeltaSchemaVersion = 2 + +var planRowsPattern = regexp.MustCompile(`\brows=([0-9]+)\b`) + +// workloadFingerprint hashes backend-independent query identity and parameter +// type shape. Physical fixture IDs are deliberately excluded so captures from +// independently loaded backends still pair. +func workloadFingerprint(query CorpusQuery) string { + parameterTypes := make(map[string]string, len(query.Params)) + for name, value := range query.Params { + if value == nil { + parameterTypes[name] = "nil" + } else { + parameterTypes[name] = reflect.TypeOf(value).String() + } + } + + return jsonFingerprint(struct { + Source string `json:"source"` + Dataset string `json:"dataset,omitempty"` + Name string `json:"name"` + Cypher string `json:"cypher"` + ParameterTypes map[string]string `json:"parameter_types,omitempty"` + }{ + Source: query.Source, + Dataset: query.Dataset, + Name: query.Name, + Cypher: strings.TrimSpace(query.Cypher), + ParameterTypes: parameterTypes, + }) +} + +// postgresPlanFingerprint hashes one normalized PostgreSQL text plan. +func postgresPlanFingerprint(plan []string) string { + if len(plan) == 0 { + return "" + } + return jsonFingerprint(plan) +} + +// neo4jPlanFingerprint hashes one normalized Neo4j plan tree. +func neo4jPlanFingerprint(plan *Neo4jPlanNode) string { + if plan == nil { + return "" + } + type fingerprintNode struct { + Operator string `json:"operator"` + Arguments map[string]string `json:"arguments,omitempty"` + Identifiers []string `json:"identifiers,omitempty"` + Children []fingerprintNode `json:"children,omitempty"` + } + var project func(Neo4jPlanNode) fingerprintNode + project = func(node Neo4jPlanNode) fingerprintNode { + projected := fingerprintNode{ + Operator: normalizeNeo4jOperator(node.Operator), + Arguments: structuralNeo4jArguments(node.Arguments), + Identifiers: append([]string(nil), node.Identifiers...), + } + for _, child := range node.Children { + projected.Children = append(projected.Children, project(child)) + } + return projected + } + return jsonFingerprint(project(*plan)) +} + +// structuralNeo4jArguments removes execution-only counters from a PROFILE so +// the plan fingerprint remains stable when the same operator tree is replayed. +func structuralNeo4jArguments(arguments map[string]string) map[string]string { + if len(arguments) == 0 { + return nil + } + filtered := map[string]string{} + for name, value := range arguments { + canonical := strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(name, "_", ""), " ", "")) + switch canonical { + case "rows", "dbhits", "pagecachehits", "pagecachemisses", "time", "timens", "actualrows", "actualloops": + continue + default: + filtered[name] = value + } + } + if len(filtered) == 0 { + return nil + } + return filtered +} + +// jsonFingerprint returns a stable SHA-256 digest for a JSON-serializable value. +func jsonFingerprint(value any) string { + raw, err := json.Marshal(value) + if err != nil { + return "" + } + digest := sha256.Sum256(raw) + return hex.EncodeToString(digest[:]) +} + +// buildPlanDeltaReport pairs records by workload union so a missing backend is +// preserved as evidence instead of disappearing through intersection-only reporting. +func buildPlanDeltaReport(records []PlanRecord) (PlanDeltaReport, error) { + type pair struct { + postgres *PlanRecord + neo4j *PlanRecord + } + + type pairKey struct { + workload string + revision string + } + pairs := map[pairKey]pair{} + for idx := range records { + record := &records[idx] + if record.WorkloadSHA256 == "" { + record.WorkloadSHA256 = workloadFingerprint(CorpusQuery{ + Source: record.Source, + Dataset: record.Dataset, + Name: record.Name, + Cypher: record.Cypher, + Params: record.Params, + }) + } + key := pairKey{workload: record.WorkloadSHA256, revision: record.Metadata.DAWGSVersion} + next := pairs[key] + switch record.Driver { + case pgDriverName(): + if next.postgres != nil { + return PlanDeltaReport{}, fmt.Errorf("duplicate PostgreSQL plan for workload %s at source revision %q", record.WorkloadSHA256, key.revision) + } + next.postgres = record + case neo4jDriverName(): + if next.neo4j != nil { + return PlanDeltaReport{}, fmt.Errorf("duplicate Neo4j plan for workload %s at source revision %q", record.WorkloadSHA256, key.revision) + } + next.neo4j = record + default: + return PlanDeltaReport{}, fmt.Errorf("unsupported plan-delta driver %q", record.Driver) + } + pairs[key] = next + } + + report := PlanDeltaReport{Version: planDeltaSchemaVersion} + for key, next := range pairs { + identity := next.postgres + if identity == nil { + identity = next.neo4j + } + delta := PlanDeltaRecord{ + Dataset: identity.Dataset, + Source: identity.Source, + Name: identity.Name, + WorkloadSHA256: key.workload, + SourceRevision: key.revision, + } + if next.postgres != nil { + plan := semanticPostgresPlan(*next.postgres) + delta.Postgres = &plan + } + if next.neo4j != nil { + plan := semanticNeo4jPlan(*next.neo4j) + delta.Neo4j = &plan + } + delta.Complete, delta.IncompleteReason = planDeltaCompleteness(delta) + if delta.Postgres != nil && delta.Neo4j != nil { + delta.OppositeStartingSides = comparableDifferent(accessSide(delta.Postgres.StartingAccess), accessSide(delta.Neo4j.StartingAccess)) + delta.OppositePhysicalDirections = comparableDifferent(delta.Postgres.PhysicalDirection, delta.Neo4j.PhysicalDirection) + delta.Neo4jReorderedPattern = neo4jReorderedPattern(identity.Cypher, delta.Neo4j.StartingAccess) + delta.ChosenSideDidLessObservedWork = lessObservedSeedWork(delta.Neo4j) + delta.SeedEstimateQError = estimateQError(delta.Postgres.EstimatedSeeds, delta.Neo4j.EstimatedSeeds) + delta.TraversalEstimateQError = estimateQError(delta.Postgres.EstimatedTraversal, delta.Neo4j.EstimatedTraversal) + delta.OutputEstimateQError = estimateQError(delta.Postgres.EstimatedOutput, delta.Neo4j.EstimatedOutput) + delta.PredicatePlacementMoved = predicatePlacementMoved(delta.Postgres.PredicatePlacement, delta.Neo4j.PredicatePlacement) + delta.HydrationEstimateQError = estimateQError(delta.Postgres.EstimatedHydration, delta.Neo4j.EstimatedHydration) + } + delta.PairSHA256 = planDeltaPairFingerprint(delta) + report.Records = append(report.Records, delta) + } + + sort.Slice(report.Records, func(i, j int) bool { + left, right := report.Records[i], report.Records[j] + if left.Dataset != right.Dataset { + return left.Dataset < right.Dataset + } + if left.Source != right.Source { + return left.Source < right.Source + } + return left.Name < right.Name + }) + report.RankedFindings = rankPlanDeltaFindings(report.Records) + return report, nil +} + +// planDeltaPairFingerprint binds source and both backend plan identities without embedding raw plans. +func planDeltaPairFingerprint(delta PlanDeltaRecord) string { + postgresFingerprint, neo4jFingerprint := "", "" + if delta.Postgres != nil { + postgresFingerprint = delta.Postgres.PlanFingerprint + } + if delta.Neo4j != nil { + neo4jFingerprint = delta.Neo4j.PlanFingerprint + } + return jsonFingerprint(struct { + Dataset string `json:"dataset,omitempty"` + Source string `json:"source"` + Name string `json:"name"` + WorkloadSHA256 string `json:"workload_sha256"` + SourceRevision string `json:"source_revision,omitempty"` + PostgresFingerprint string `json:"postgres_plan_fingerprint,omitempty"` + Neo4jFingerprint string `json:"neo4j_plan_fingerprint,omitempty"` + }{ + Dataset: delta.Dataset, Source: delta.Source, Name: delta.Name, + WorkloadSHA256: delta.WorkloadSHA256, SourceRevision: delta.SourceRevision, + PostgresFingerprint: postgresFingerprint, Neo4jFingerprint: neo4jFingerprint, + }) +} + +// accessSide maps backend-specific access labels onto a root/terminal side when possible. +func accessSide(access string) string { + lower := strings.ToLower(access) + switch { + case strings.Contains(lower, "terminal"), strings.Contains(lower, "target"), strings.Contains(lower, " n1"), strings.Contains(lower, "(n1"): + return "terminal" + case strings.Contains(lower, "root"), strings.Contains(lower, "source"), strings.Contains(lower, " n0"), strings.Contains(lower, "(n0"): + return "root" + default: + return "" + } +} + +// planDeltaCompleteness reports whether both sides contain successful plan evidence. +func planDeltaCompleteness(delta PlanDeltaRecord) (bool, string) { + var reasons []string + if delta.Postgres == nil { + reasons = append(reasons, "missing_postgres") + } else if delta.Postgres.Error != "" || delta.Postgres.PlanFingerprint == "" { + reasons = append(reasons, "failed_postgres") + } + if delta.Neo4j == nil { + reasons = append(reasons, "missing_neo4j") + } else if delta.Neo4j.Error != "" || delta.Neo4j.PlanFingerprint == "" { + reasons = append(reasons, "failed_neo4j") + } + return len(reasons) == 0, strings.Join(reasons, ",") +} + +// comparableDifferent compares nonempty semantic labels. +func comparableDifferent(left, right string) bool { + return left != "" && right != "" && left != right +} + +// neo4jReorderedPattern reports a conservative endpoint reversal relative to the textual first relationship. +func neo4jReorderedPattern(cypherQuery, startingAccess string) bool { + if logicalDirection(cypherQuery) == "" { + return false + } + return accessSide(startingAccess) == "terminal" +} + +// lessObservedSeedWork compares profiled leaf work only when both endpoint leaves expose it. +func lessObservedSeedWork(plan *SemanticPlan) *bool { + if plan == nil || plan.ObservedSeedWork == nil || plan.ObservedAlternativeSeedWork == nil { + return nil + } + value := *plan.ObservedSeedWork <= *plan.ObservedAlternativeSeedWork + return &value +} + +// estimateQError reports symmetric disagreement between two positive backend estimates. +func estimateQError(left, right *float64) *float64 { + if left == nil || right == nil || *left <= 0 || *right <= 0 { + return nil + } + value := math.Max(*left / *right, *right / *left) + return &value +} + +// predicatePlacementMoved compares normalized predicate-bearing stage families rather than raw backend syntax. +func predicatePlacementMoved(postgres, neo4j []string) bool { + if len(postgres) == 0 && len(neo4j) == 0 { + return false + } + postgresStages := normalizedPredicateStages(postgres) + neo4jStages := normalizedPredicateStages(neo4j) + return !reflect.DeepEqual(postgresStages, neo4jStages) +} + +// normalizedPredicateStages reduces backend syntax to access/filter/join stage counts. +func normalizedPredicateStages(stages []string) map[string]int { + normalized := map[string]int{} + for _, stage := range stages { + lower := strings.ToLower(stage) + switch { + case strings.Contains(lower, "join filter"), strings.Contains(lower, "apply"): + normalized["join"]++ + case strings.Contains(lower, "index cond"), strings.Contains(lower, "seek"): + normalized["access"]++ + default: + normalized["filter"]++ + } + } + return normalized +} + +// neo4jNodeObservedWork prefers DB hits and otherwise uses profiled output rows. +func neo4jNodeObservedWork(node Neo4jPlanNode) *int64 { + if node.DBHits != nil { + return node.DBHits + } + return node.ActualRows +} + +// rankPlanDeltaFindings produces category-local scores and a stable global review order. +func rankPlanDeltaFindings(records []PlanDeltaRecord) []PlanDeltaFinding { + var findings []PlanDeltaFinding + appendFinding := func(record PlanDeltaRecord, category string, score float64, summary string) { + findings = append(findings, PlanDeltaFinding{ + Category: category, Dataset: record.Dataset, Source: record.Source, Name: record.Name, + PairSHA256: record.PairSHA256, Score: score, Summary: summary, + }) + } + for _, record := range records { + if !record.Complete { + appendFinding(record, "incomplete_pair", math.MaxFloat64, record.IncompleteReason) + continue + } + if record.OppositeStartingSides { + summary := "backends start from opposite endpoint sides" + if record.ChosenSideDidLessObservedWork != nil { + summary += fmt.Sprintf("; Neo4j lower-work choice=%t", *record.ChosenSideDidLessObservedWork) + } + appendFinding(record, "opposite_starting_side", 1, summary) + } + for category, value := range map[string]*float64{ + "seed_estimate_disagreement": record.SeedEstimateQError, + "traversal_estimate_disagreement": record.TraversalEstimateQError, + "output_estimate_disagreement": record.OutputEstimateQError, + "hydration_estimate_disagreement": record.HydrationEstimateQError, + } { + if value != nil && *value > 1 { + appendFinding(record, category, *value, fmt.Sprintf("backend estimate Q-error %.4g", *value)) + } + } + if record.PredicatePlacementMoved { + appendFinding(record, "predicate_placement_move", 1, "predicate-bearing stage families differ") + } + if record.Postgres != nil && (record.Postgres.FallbackReason != "" || len(record.Postgres.ProbeCaps) > 0) { + summary := "bounded candidate or fallback is present" + if record.Postgres.FallbackReason != "" { + summary = "fallback: " + record.Postgres.FallbackReason + } + appendFinding(record, "fallback_or_cap", float64(len(record.Postgres.ProbeCaps)+1), summary) + } + } + categoryPriority := map[string]int{ + "incomplete_pair": 0, "fallback_or_cap": 1, "opposite_starting_side": 2, + "traversal_estimate_disagreement": 3, "seed_estimate_disagreement": 4, + "output_estimate_disagreement": 5, "predicate_placement_move": 6, "hydration_estimate_disagreement": 7, + } + sort.Slice(findings, func(i, j int) bool { + leftPriority, rightPriority := categoryPriority[findings[i].Category], categoryPriority[findings[j].Category] + if leftPriority != rightPriority { + return leftPriority < rightPriority + } + if findings[i].Score != findings[j].Score { + return findings[i].Score > findings[j].Score + } + if findings[i].Dataset != findings[j].Dataset { + return findings[i].Dataset < findings[j].Dataset + } + if findings[i].Source != findings[j].Source { + return findings[i].Source < findings[j].Source + } + return findings[i].Name < findings[j].Name + }) + for idx := range findings { + findings[idx].Rank = idx + 1 + } + return findings +} + +// semanticPostgresPlan projects PostgreSQL operators and translator outcomes +// onto backend-neutral traversal stages. +func semanticPostgresPlan(record PlanRecord) SemanticPlan { + plan := SemanticPlan{ + Driver: record.Driver, + PlanFingerprint: record.PGPlanFingerprint, + LogicalDirection: logicalDirection(record.Cypher), + PhysicalDirection: postgresPhysicalDirection(record.PGPlan), + PredicatePlacement: postgresPredicatePlacement(record.PGPlan), + EndpointBinding: postgresEndpointBinding(record.PGPlan), + OperatorFamily: postgresOperatorFamily(record.PGPlan), + RuntimeIdentityKnown: false, + Error: record.Error, + RawOptimization: record.Optimization, + } + accesses := postgresAccesses(record.PGPlan) + if len(accesses) > 0 { + plan.StartingAccess = accesses[0] + plan.EstimatedSeeds = postgresRowsEstimate(accesses[0]) + } + if len(accesses) > 1 { + plan.TerminalAccess = accesses[1] + } + if len(record.PGPlan) > 0 { + plan.EstimatedOutput = postgresRowsEstimate(record.PGPlan[0]) + } + for _, line := range record.PGPlan { + lower := strings.ToLower(line) + if strings.Contains(line, "Recursive Union") || strings.Contains(lower, "shortest_path") { + plan.EstimatedTraversal = postgresRowsEstimate(line) + } + if plan.EstimatedHydration == nil && (strings.Contains(lower, "hydrat") || strings.Contains(lower, "materializ")) { + plan.EstimatedHydration = postgresRowsEstimate(line) + } + } + plan.PlannedIdentity, plan.EmittedIdentity, plan.PlannedCandidates, plan.EmittedCandidates, + plan.FallbackIdentity, plan.FallbackReason, plan.SelectorVersion, plan.ProbeCaps = postgresPlanIdentities(record.Optimization) + return plan +} + +// semanticNeo4jPlan projects the ordered Neo4j tree onto comparable stages. +func semanticNeo4jPlan(record PlanRecord) SemanticPlan { + plan := SemanticPlan{ + Driver: record.Driver, + PlanFingerprint: record.Neo4jPlanFingerprint, + LogicalDirection: logicalDirection(record.Cypher), + PhysicalDirection: neo4jPhysicalDirection(record.Neo4jPlan), + PredicatePlacement: neo4jPredicatePlacement(record.Neo4jPlan), + EndpointBinding: neo4jEndpointBinding(record.Neo4jPlan), + OperatorFamily: neo4jOperatorFamily(record.Neo4jPlan), + RuntimeIdentityKnown: false, + Error: record.Error, + } + if record.Neo4jPlan == nil { + return plan + } + leaves := neo4jLeaves(*record.Neo4jPlan) + if len(leaves) > 0 { + plan.StartingAccess = neo4jAccessLabel(leaves[0]) + plan.EstimatedSeeds = neo4jEstimatedRows(leaves[0]) + plan.ObservedSeedWork = neo4jNodeObservedWork(leaves[0]) + } + if len(leaves) > 1 { + plan.TerminalAccess = neo4jAccessLabel(leaves[1]) + plan.ObservedAlternativeSeedWork = neo4jNodeObservedWork(leaves[1]) + } + plan.EstimatedOutput = neo4jEstimatedRows(*record.Neo4jPlan) + plan.ActualOutput = record.Neo4jPlan.ActualRows + plan.OutputQError = qError(plan.EstimatedOutput, plan.ActualOutput) + var traversal *Neo4jPlanNode + walkNeo4jPlan(*record.Neo4jPlan, func(node Neo4jPlanNode) { + if traversal == nil && (strings.Contains(node.Operator, "Expand") || strings.Contains(node.Operator, "ShortestPath")) { + copyNode := node + traversal = ©Node + } + lower := strings.ToLower(node.Operator + " " + node.Arguments["Details"]) + if strings.Contains(lower, "project") || strings.Contains(lower, "materializ") || strings.Contains(lower, "path") && !strings.Contains(lower, "shortestpath") { + if plan.EstimatedHydration == nil { + plan.EstimatedHydration = neo4jEstimatedRows(node) + } + if plan.ObservedHydrationRows == nil { + plan.ObservedHydrationRows = node.ActualRows + } + } + }) + if traversal != nil { + plan.EstimatedTraversal = neo4jEstimatedRows(*traversal) + plan.ObservedTraversalWork = traversal.DBHits + if strings.Contains(traversal.Operator, "ShortestPath") { + plan.InternalTraversalWork = "opaque" + } + } + return plan +} + +// postgresPlanIdentities returns selected/emitted identities, complete candidate sets, fallback, selector, and bounded probe caps. +func postgresPlanIdentities(optimization *translate.OptimizationSummary) (string, string, []string, []string, string, string, string, map[string]int64) { + if optimization == nil { + return "", "", nil, nil, "", "", "", nil + } + for _, outcome := range optimization.TargetOutcomes { + if outcome.Family == "SP" || outcome.Family == "ASP" || strings.Contains(outcome.Family, "expansion") { + caps := map[string]int64{} + if outcome.ProbeCaps != nil { + caps["root_rows"] = outcome.ProbeCaps.RootRowLimit + caps["reverse_seed_rows"] = outcome.ProbeCaps.ReverseSeedRowLimit + caps["directional_degree_rows"] = outcome.ProbeCaps.DirectionalDegreeRowLimit + caps["survival_rows"] = outcome.ProbeCaps.SurvivalRowLimit + } + if outcome.StateLimit > 0 { + caps["state_rows"] = outcome.StateLimit + } + if outcome.EndpointLimit > 0 { + caps["endpoint_rows"] = outcome.EndpointLimit + } + for name, value := range caps { + if value <= 0 { + delete(caps, name) + } + } + if len(caps) == 0 { + caps = nil + } + return outcome.Selected, outcome.Applied, + append([]string(nil), outcome.PlannedCandidates...), append([]string(nil), outcome.EmittedCandidates...), + outcome.Fallback, outcome.SkipReason, outcome.SelectorVersion, caps + } + } + return "", "", nil, nil, "", "", "", nil +} + +// postgresAccesses returns leaf access lines in execution order. +func postgresAccesses(plan []string) []string { + var accesses []string + for idx := len(plan) - 1; idx >= 0; idx-- { + line := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(plan[idx]), "->")) + if strings.Contains(line, " Scan") && !strings.Contains(line, "CTE Scan") && !strings.Contains(line, "Subquery Scan") { + accesses = append(accesses, line) + } + } + return accesses +} + +// postgresRowsEstimate extracts a planner row estimate from a text-plan line. +func postgresRowsEstimate(line string) *float64 { + match := planRowsPattern.FindStringSubmatch(line) + if len(match) != 2 { + return nil + } + value, err := strconv.ParseFloat(match[1], 64) + if err != nil { + return nil + } + return &value +} + +// postgresPhysicalDirection identifies the adjacency endpoint used by a plan. +func postgresPhysicalDirection(plan []string) string { + joined := strings.ToLower(strings.Join(plan, "\n")) + start, end := strings.Contains(joined, "start_id"), strings.Contains(joined, "end_id") + switch { + case start && end: + return "mixed" + case start: + return "start_id" + case end: + return "end_id" + default: + return "" + } +} + +// postgresPredicatePlacement lists plan stages containing filters. +func postgresPredicatePlacement(plan []string) []string { + var stages []string + for _, line := range plan { + trimmed := strings.TrimSpace(line) + if strings.Contains(trimmed, "Filter:") || strings.Contains(trimmed, "Index Cond:") || strings.Contains(trimmed, "Join Filter:") { + stages = append(stages, trimmed) + } + } + return stages +} + +// postgresEndpointBinding classifies evidence that a bound endpoint pair is materialized. +func postgresEndpointBinding(plan []string) string { + joined := strings.ToLower(strings.Join(plan, "\n")) + if strings.Contains(joined, "pair_filter") || strings.Contains(joined, "cartesian") { + return "both_before_traversal" + } + if strings.Contains(joined, "terminal_filter") { + return "terminal_before_traversal" + } + return "" +} + +// postgresOperatorFamily classifies PostgreSQL traversal execution. +func postgresOperatorFamily(plan []string) string { + joined := strings.ToLower(strings.Join(plan, "\n")) + switch { + case strings.Contains(joined, "all_shortest_paths"): + return "all_shortest_paths" + case strings.Contains(joined, "shortest_path"): + return "shortest_path" + case strings.Contains(joined, "recursive union"): + return "ordinary_expand" + case strings.Contains(joined, "edge"): + return "fixed_hop" + default: + return "" + } +} + +// logicalDirection extracts the first directed relationship orientation. +func logicalDirection(cypherQuery string) string { + compact := strings.ReplaceAll(cypherQuery, " ", "") + switch { + case strings.Contains(compact, "]->"): + return "outbound" + case strings.Contains(compact, "<-["): + return "inbound" + case strings.Contains(compact, "]-[") || strings.Contains(compact, "]-"): + return "directionless" + default: + return "" + } +} + +// neo4jLeaves returns leaf operators in backend child order. +func neo4jLeaves(root Neo4jPlanNode) []Neo4jPlanNode { + var leaves []Neo4jPlanNode + walkNeo4jPlan(root, func(node Neo4jPlanNode) { + if len(node.Children) == 0 { + leaves = append(leaves, node) + } + }) + return leaves +} + +// walkNeo4jPlan visits a plan in parent-before-child order while retaining backend child order. +func walkNeo4jPlan(root Neo4jPlanNode, visit func(Neo4jPlanNode)) { + visit(root) + for _, child := range root.Children { + walkNeo4jPlan(child, visit) + } +} + +// neo4jAccessLabel renders an access operator with its stable details. +func neo4jAccessLabel(node Neo4jPlanNode) string { + details := node.Arguments["Details"] + if details == "" { + return node.Operator + } + return node.Operator + ": " + details +} + +// neo4jEstimatedRows returns an estimate from a typed field or stable argument. +func neo4jEstimatedRows(node Neo4jPlanNode) *float64 { + if node.EstimatedRows != nil { + return node.EstimatedRows + } + value, err := strconv.ParseFloat(node.Arguments["EstimatedRows"], 64) + if err != nil { + return nil + } + return &value +} + +// neo4jPhysicalDirection classifies expansion direction from operator details. +func neo4jPhysicalDirection(root *Neo4jPlanNode) string { + if root == nil { + return "" + } + var directions []string + walkNeo4jPlan(*root, func(node Neo4jPlanNode) { + if !strings.Contains(node.Operator, "Expand") && !strings.Contains(node.Operator, "ShortestPath") { + return + } + details := strings.ToLower(node.Arguments["Details"]) + switch { + case strings.Contains(details, "incoming") || strings.Contains(details, "<-"): + directions = append(directions, "incoming") + case strings.Contains(details, "outgoing") || strings.Contains(details, "->"): + directions = append(directions, "outgoing") + } + }) + if len(directions) == 0 { + return "" + } + for _, direction := range directions[1:] { + if direction != directions[0] { + return "mixed" + } + } + return directions[0] +} + +// neo4jPredicatePlacement lists operators whose details expose predicates. +func neo4jPredicatePlacement(root *Neo4jPlanNode) []string { + if root == nil { + return nil + } + var stages []string + walkNeo4jPlan(*root, func(node Neo4jPlanNode) { + if strings.Contains(node.Operator, "Filter") || strings.Contains(node.Operator, "Seek") { + stages = append(stages, neo4jAccessLabel(node)) + } + }) + return stages +} + +// neo4jEndpointBinding recognizes the pair-producing plan boundary. +func neo4jEndpointBinding(root *Neo4jPlanNode) string { + if root == nil { + return "" + } + bound := "" + walkNeo4jPlan(*root, func(node Neo4jPlanNode) { + if strings.Contains(node.Operator, "CartesianProduct") || strings.Contains(node.Operator, "Apply") { + bound = "both_before_traversal" + } + }) + return bound +} + +// neo4jOperatorFamily classifies Neo4j traversal operators. +func neo4jOperatorFamily(root *Neo4jPlanNode) string { + if root == nil { + return "" + } + family := "" + walkNeo4jPlan(*root, func(node Neo4jPlanNode) { + switch { + case strings.Contains(node.Operator, "ShortestPath"): + family = "shortest_path" + case family == "" && strings.Contains(node.Operator, "VarLengthExpand"): + family = "ordinary_expand" + case family == "" && strings.Contains(node.Operator, "Expand"): + family = "fixed_hop" + } + }) + return family +} + +// qError returns symmetric estimate error when both values are positive. +func qError(estimated *float64, actual *int64) *float64 { + if estimated == nil || actual == nil || *estimated <= 0 || *actual <= 0 { + return nil + } + value := math.Max(*estimated/float64(*actual), float64(*actual)/(*estimated)) + return &value +} + +// writePlanDeltaReport writes one indented, newline-terminated paired report. +func writePlanDeltaReport(path string, report PlanDeltaReport) error { + raw, err := json.MarshalIndent(report, "", " ") + if err != nil { + return err + } + if err := os.WriteFile(path, append(raw, '\n'), 0o644); err != nil { + return fmt.Errorf("write plan delta %s: %w", path, err) + } + return nil +} diff --git a/cmd/plancorpus/plan_delta_test.go b/cmd/plancorpus/plan_delta_test.go new file mode 100644 index 00000000..2f329e2a --- /dev/null +++ b/cmd/plancorpus/plan_delta_test.go @@ -0,0 +1,162 @@ +package main + +import ( + "path/filepath" + "testing" + + "github.com/specterops/dawgs/testutil" + "github.com/stretchr/testify/require" +) + +// TestBuildPlanDeltaReportPairsByWorkloadAndPreservesSemanticDifferences verifies +// stable pairing, plan fingerprints, direction classification, and opaque Neo4j +// shortest-path work. +func TestBuildPlanDeltaReportPairsByWorkloadAndPreservesSemanticDifferences(t *testing.T) { + query := CorpusQuery{ + Source: "cases/shortest.json", + Dataset: "shortest", + Name: "bound", + Cypher: "MATCH p = shortestPath((root)-[*1..4]->(terminal)) RETURN p", + Params: map[string]any{"root_id": int64(1), "terminal_id": int64(2)}, + } + workload := workloadFingerprint(query) + pgPlan := []string{ + "Function Scan on shortest_path_compact (cost=0.25..0.26 rows=1 width=8)", + "Index Scan using node_id_idx on node root (cost=0.10..1.00 rows=1 width=8)", + "Index Cond: (start_id = root.id)", + } + neoPlan := &Neo4jPlanNode{ + Operator: "ProduceResults", + Arguments: map[string]string{"EstimatedRows": "1"}, + Children: []Neo4jPlanNode{{ + Operator: "ShortestPath", + Arguments: map[string]string{"EstimatedRows": "1", "Details": "(terminal)<-[*]-(root)"}, + Children: []Neo4jPlanNode{{ + Operator: "NodeByIdSeek", + Arguments: map[string]string{"Details": "terminal"}, + }}, + }}, + } + records := []PlanRecord{{ + SchemaVersion: planRecordSchemaVersion, + Driver: pgDriverName(), + Source: query.Source, + Dataset: query.Dataset, + Name: query.Name, + WorkloadSHA256: workload, + Cypher: query.Cypher, + PGPlan: pgPlan, + PGPlanFingerprint: postgresPlanFingerprint(pgPlan), + }, { + SchemaVersion: planRecordSchemaVersion, + Driver: neo4jDriverName(), + Source: query.Source, + Dataset: query.Dataset, + Name: query.Name, + WorkloadSHA256: workload, + Cypher: query.Cypher, + Neo4jPlan: neoPlan, + Neo4jPlanFingerprint: neo4jPlanFingerprint(neoPlan), + }} + + report, err := buildPlanDeltaReport(records) + require.NoError(t, err) + require.Equal(t, planDeltaSchemaVersion, report.Version) + require.Len(t, report.Records, 1) + delta := report.Records[0] + require.True(t, delta.Complete) + require.Empty(t, delta.IncompleteReason) + require.Equal(t, "shortest_path", delta.Postgres.OperatorFamily) + require.Equal(t, "shortest_path", delta.Neo4j.OperatorFamily) + require.Equal(t, "opaque", delta.Neo4j.InternalTraversalWork) + require.True(t, delta.OppositeStartingSides) + require.NotEmpty(t, delta.Postgres.PlanFingerprint) + require.NotEmpty(t, delta.Neo4j.PlanFingerprint) + require.NotEmpty(t, delta.PairSHA256) + require.NotEmpty(t, report.RankedFindings) + require.Equal(t, "opposite_starting_side", report.RankedFindings[0].Category) +} + +// TestBuildPlanDeltaReportKeepsSourceRevisionsSeparate verifies captures from different source trees cannot silently pair. +func TestBuildPlanDeltaReportKeepsSourceRevisionsSeparate(t *testing.T) { + postgres := PlanRecord{ + Driver: pgDriverName(), Source: "cases/a.json", Name: "a", WorkloadSHA256: "workload", + PGPlanFingerprint: "pg-plan", Metadata: testutil.BaselineMetadata{DAWGSVersion: "revision-a"}, + } + neo4j := PlanRecord{ + Driver: neo4jDriverName(), Source: "cases/a.json", Name: "a", WorkloadSHA256: "workload", + Neo4jPlanFingerprint: "neo-plan", Metadata: testutil.BaselineMetadata{DAWGSVersion: "revision-b"}, + } + report, err := buildPlanDeltaReport([]PlanRecord{postgres, neo4j}) + require.NoError(t, err) + require.Len(t, report.Records, 2) + require.False(t, report.Records[0].Complete) + require.False(t, report.Records[1].Complete) +} + +// TestBuildPlanDeltaReportRetainsIncompletePairs verifies union-based pairing. +func TestBuildPlanDeltaReportRetainsIncompletePairs(t *testing.T) { + report, err := buildPlanDeltaReport([]PlanRecord{{ + Driver: pgDriverName(), + Source: "cases/a.json", + Name: "a", + WorkloadSHA256: "workload", + PGPlan: []string{"Result (cost=0.00..0.01 rows=1 width=4)"}, + PGPlanFingerprint: "pg-plan", + }}) + + require.NoError(t, err) + require.Len(t, report.Records, 1) + require.False(t, report.Records[0].Complete) + require.Equal(t, "missing_neo4j", report.Records[0].IncompleteReason) + require.NotNil(t, report.Records[0].Postgres) + require.Nil(t, report.Records[0].Neo4j) +} + +// TestBuildPlanDeltaReportRejectsDuplicateBackendSides verifies ambiguous pairing fails closed. +func TestBuildPlanDeltaReportRejectsDuplicateBackendSides(t *testing.T) { + _, err := buildPlanDeltaReport([]PlanRecord{{Driver: pgDriverName(), WorkloadSHA256: "same"}, {Driver: pgDriverName(), WorkloadSHA256: "same"}}) + require.ErrorContains(t, err, "duplicate PostgreSQL") +} + +// TestWritePlanDeltaReportWritesVersionedJSON verifies portable serialization. +func TestWritePlanDeltaReportWritesVersionedJSON(t *testing.T) { + path := filepath.Join(t.TempDir(), "delta.json") + require.NoError(t, writePlanDeltaReport(path, PlanDeltaReport{Version: planDeltaSchemaVersion})) + require.FileExists(t, path) +} + +// TestWorkloadFingerprintIgnoresPhysicalValuesButIncludesTypeShape verifies independently loaded backend IDs pair safely. +func TestWorkloadFingerprintIgnoresPhysicalValuesButIncludesTypeShape(t *testing.T) { + base := CorpusQuery{Source: "cases/a.json", Name: "a", Cypher: "RETURN $id", Params: map[string]any{"id": int64(1)}} + otherID := base + otherID.Params = map[string]any{"id": int64(999)} + otherType := base + otherType.Params = map[string]any{"id": "1"} + + require.Equal(t, workloadFingerprint(base), workloadFingerprint(otherID)) + require.NotEqual(t, workloadFingerprint(base), workloadFingerprint(otherType)) +} + +// TestNeo4jPlanFingerprintExcludesProfileMeasurements verifies replay counters do not make an identical plan shape look like a different plan. +func TestNeo4jPlanFingerprintExcludesProfileMeasurements(t *testing.T) { + firstRows, secondRows := int64(1), int64(99) + first := &Neo4jPlanNode{ + Operator: "ProduceResults@neo4j", + Arguments: map[string]string{"EstimatedRows": "1", "Rows": "1", "Details": "n"}, + ActualRows: &firstRows, + DBHits: &firstRows, + Children: []Neo4jPlanNode{{Operator: "NodeByLabelScan", Arguments: map[string]string{"Details": "n:Node"}}}, + } + second := &Neo4jPlanNode{ + Operator: "ProduceResults@neo4j@neo4j", + Arguments: map[string]string{"EstimatedRows": "1", "Rows": "99", "Details": "n"}, + ActualRows: &secondRows, + DBHits: &secondRows, + Children: []Neo4jPlanNode{{Operator: "NodeByLabelScan@neo4j", Arguments: map[string]string{"Details": "n:Node"}}}, + } + + require.Equal(t, neo4jPlanFingerprint(first), neo4jPlanFingerprint(second)) + second.Children[0].Operator = "NodeIndexSeek" + require.NotEqual(t, neo4jPlanFingerprint(first), neo4jPlanFingerprint(second)) +} diff --git a/cmd/plancorpus/report.go b/cmd/plancorpus/report.go index 654067cc..e2104d1f 100644 --- a/cmd/plancorpus/report.go +++ b/cmd/plancorpus/report.go @@ -10,56 +10,96 @@ import ( "strings" "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/specterops/dawgs/testutil" ) +// defaultTopPlans limits an unconfigured report to its 25 most expensive PostgreSQL plans. const defaultTopPlans = 25 +// postgresCostPattern extracts the total-cost upper bound from a PostgreSQL plan's cost range. var postgresCostPattern = regexp.MustCompile(`cost=[0-9.]+\.\.([0-9.]+)`) +// PlanSummary aggregates captured plans by driver, lowering, and cost. type PlanSummary struct { - Drivers []DriverSummary `json:"drivers"` - TopPostgresPlans []CostedPlan `json:"top_postgres_plans,omitempty"` - PostgresOperators []Count `json:"postgres_operators,omitempty"` - Neo4jOperators []Count `json:"neo4j_operators,omitempty"` - PlannedLowerings []Count `json:"planned_lowerings,omitempty"` - AppliedLowerings []Count `json:"applied_lowerings,omitempty"` - SkippedLowerings []Count `json:"skipped_lowerings,omitempty"` - SkippedReasons []Count `json:"skipped_reasons,omitempty"` - FeatureCounts []Count `json:"feature_counts,omitempty"` - Errors []PlanError `json:"errors,omitempty"` + // Metadata captures build and baseline metadata. + Metadata testutil.BaselineMetadata `json:"metadata"` + // Drivers lists driver summaries in deterministic display order. + Drivers []DriverSummary `json:"drivers"` + // TopPostgresPlans lists the highest-cost PostgreSQL plans selected for the summary. + TopPostgresPlans []CostedPlan `json:"top_postgres_plans,omitempty"` + // PostgresOperators counts normalized PostgreSQL plan operators. + PostgresOperators []Count `json:"postgres_operators,omitempty"` + // Neo4jOperators lists normalized Neo4j operators found in the captured plan. + Neo4jOperators []Count `json:"neo4j_operators,omitempty"` + // PlannedLowerings lists SQL lowering opportunities identified before optimization. + PlannedLowerings []Count `json:"planned_lowerings,omitempty"` + // AppliedLowerings lists SQL lowerings actually applied during translation. + AppliedLowerings []Count `json:"applied_lowerings,omitempty"` + // SkippedLowerings lists identified SQL lowerings not applied. + SkippedLowerings []Count `json:"skipped_lowerings,omitempty"` + // SkippedReasons counts reasons identified lowerings were not applied. + SkippedReasons []Count `json:"skipped_reasons,omitempty"` + // FeatureCounts counts captured plans containing each normalized plan feature. + FeatureCounts []Count `json:"feature_counts,omitempty"` + // Errors lists failures observed while processing the record. + Errors []PlanError `json:"errors,omitempty"` } +// DriverSummary aggregates plan counts and operators for one database driver. type DriverSummary struct { - Driver string `json:"driver"` - Records int `json:"records"` - Errors int `json:"errors"` + // Driver identifies the database driver that produced the plan or summary. + Driver string `json:"driver"` + // Records counts captured plan records produced by the driver. + Records int `json:"records"` + // Errors counts plan-capture failures reported by the driver. + Errors int `json:"errors"` } +// Count pairs a label with an aggregate count for serialized summaries. type Count struct { - Name string `json:"name"` - Count int `json:"count"` + // Name labels the operator, lowering, feature, or reason being counted. + Name string `json:"name"` + // Count records how many plan records contributed the named item. + Count int `json:"count"` } +// CostedPlan identifies a captured plan and its parsed PostgreSQL estimated cost. type CostedPlan struct { - Cost float64 `json:"cost"` - Driver string `json:"driver"` - Source string `json:"source"` - Dataset string `json:"dataset,omitempty"` - Name string `json:"name"` - Cypher string `json:"cypher"` - PlanRoot string `json:"plan_root"` + // Cost records the PostgreSQL planner's estimated total cost. + Cost float64 `json:"cost"` + // Driver identifies the database driver that produced the plan or summary. + Driver string `json:"driver"` + // Source identifies the source corpus file. + Source string `json:"source"` + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset,omitempty"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // Cypher contains the Cypher statement under test. + Cypher string `json:"cypher"` + // PlanRoot identifies the root operator of the captured plan. + PlanRoot string `json:"plan_root"` + // PlannedLowerings lists SQL lowering opportunities identified before optimization. PlannedLowerings []string `json:"planned_lowerings,omitempty"` + // AppliedLowerings lists SQL lowerings actually applied during translation. AppliedLowerings []string `json:"applied_lowerings,omitempty"` + // SkippedLowerings lists identified SQL lowerings not applied. SkippedLowerings []string `json:"skipped_lowerings,omitempty"` } +// PlanError records the driver, query, and failure for a plan that could not be summarized. type PlanError struct { + // Driver identifies the database driver that produced the plan or summary. Driver string `json:"driver"` + // Source identifies the source corpus file. Source string `json:"source"` - Name string `json:"name"` - Error string `json:"error"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // Error records the failure message when the operation did not succeed. + Error string `json:"error"` } +// buildSummary aggregates plan records by driver, operator, lowering, error, and estimated cost. func buildSummary(records []PlanRecord, topN int) PlanSummary { if topN <= 0 { topN = defaultTopPlans @@ -74,11 +114,15 @@ func buildSummary(records []PlanRecord, topN int) PlanSummary { skippedLoweringCounts = map[string]int{} skippedReasonCounts = map[string]int{} featureCounts = map[string]int{} + summaryMetadata testutil.BaselineMetadata errors []PlanError topPG []CostedPlan ) for _, record := range records { + if summaryMetadata == (testutil.BaselineMetadata{}) { + summaryMetadata = record.Metadata + } driver := driverCounts[record.Driver] if driver == nil { driver = &DriverSummary{Driver: record.Driver} @@ -150,6 +194,7 @@ func buildSummary(records []PlanRecord, topN int) PlanSummary { } return PlanSummary{ + Metadata: summaryMetadata, Drivers: sortedDriverSummaries(driverCounts), TopPostgresPlans: topPG, PostgresOperators: sortedCounts(postgresOperatorCounts), @@ -163,6 +208,7 @@ func buildSummary(records []PlanRecord, topN int) PlanSummary { } } +// skippedLoweringLabels renders skipped lowering names and reasons as stable report labels, preserving their plan order. func skippedLoweringLabels(lowerings []translate.SkippedLowering) []string { if len(lowerings) == 0 { return nil @@ -176,6 +222,7 @@ func skippedLoweringLabels(lowerings []translate.SkippedLowering) []string { return labels } +// postgresEstimatedCost extracts the PostgreSQL planner's estimated total cost from plan text. func postgresEstimatedCost(planRoot string) float64 { match := postgresCostPattern.FindStringSubmatch(planRoot) if len(match) != 2 { @@ -189,6 +236,7 @@ func postgresEstimatedCost(planRoot string) float64 { return cost } +// normalizePostgresOperator removes plan decoration so equivalent PostgreSQL operator lines share one name. func normalizePostgresOperator(operator string) string { operator = strings.TrimSpace(operator) if operator == "" { @@ -206,6 +254,7 @@ func normalizePostgresOperator(operator string) string { return operator } +// sortedDriverSummaries returns driver summaries ordered by driver name. func sortedDriverSummaries(drivers map[string]*DriverSummary) []DriverSummary { sorted := make([]DriverSummary, 0, len(drivers)) for _, summary := range drivers { @@ -217,6 +266,7 @@ func sortedDriverSummaries(drivers map[string]*DriverSummary) []DriverSummary { return sorted } +// sortedCounts converts a count map to descending-count, name-tiebroken entries. func sortedCounts(counts map[string]int) []Count { sorted := make([]Count, 0, len(counts)) for name, count := range counts { @@ -234,12 +284,14 @@ func sortedCounts(counts map[string]int) []Count { return sorted } +// writeJSONSummary encodes a plan summary as indented JSON. func writeJSONSummary(w io.Writer, summary PlanSummary) error { encoder := json.NewEncoder(w) encoder.SetIndent("", " ") return encoder.Encode(summary) } +// writeMarkdownSummary renders aggregate counts, expensive plans, and errors as Markdown. func writeMarkdownSummary(w io.Writer, summary PlanSummary) error { writef := func(format string, args ...any) error { _, err := fmt.Fprintf(w, format, args...) @@ -272,6 +324,9 @@ func writeMarkdownSummary(w io.Writer, summary PlanSummary) error { if err := writeln("# Cypher Plan Corpus Summary"); err != nil { return err } + if err := writef("\nDAWGS version: `%s`\n", summary.Metadata.DAWGSVersion); err != nil { + return err + } if err := writeln("\n## Drivers\n\n| Driver | Records | Errors |\n| --- | ---: | ---: |"); err != nil { return err } @@ -341,6 +396,7 @@ func writeMarkdownSummary(w io.Writer, summary PlanSummary) error { return nil } +// markdownCell escapes table delimiters and line breaks for a Markdown cell. func markdownCell(value string) string { value = strings.ReplaceAll(value, "\n", " ") value = strings.ReplaceAll(value, "|", "\\|") diff --git a/cmd/plancorpus/types.go b/cmd/plancorpus/types.go index 9c4fa662..254120b9 100644 --- a/cmd/plancorpus/types.go +++ b/cmd/plancorpus/types.go @@ -1,37 +1,235 @@ package main -import "github.com/specterops/dawgs/cypher/models/pgsql/translate" +import ( + "encoding/json" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/specterops/dawgs/testutil" +) + +const planRecordSchemaVersion = 2 + +// PlanRecord captures a query plan together with workload, fixture, and environment identity. type PlanRecord struct { - Driver string `json:"driver"` - Source string `json:"source"` - Dataset string `json:"dataset,omitempty"` - Name string `json:"name"` - Cypher string `json:"cypher"` - Params map[string]any `json:"params,omitempty"` - SQL string `json:"sql,omitempty"` - PGPlan []string `json:"pg_plan,omitempty"` - PGOperators []string `json:"pg_operators,omitempty"` - Neo4jPlan *Neo4jPlanNode `json:"neo4j_plan,omitempty"` - Neo4jOperators []string `json:"neo4j_operators,omitempty"` - PlannedLowerings []string `json:"planned_lowerings,omitempty"` - AppliedLowerings []string `json:"applied_lowerings,omitempty"` - SkippedLowerings []translate.SkippedLowering `json:"skipped_lowerings,omitempty"` - Optimization *translate.OptimizationSummary `json:"optimization,omitempty"` - Error string `json:"error,omitempty"` + // SchemaVersion identifies the serialized plan-record schema revision. + SchemaVersion int `json:"schema_version"` + // Metadata captures build and baseline metadata. + Metadata testutil.BaselineMetadata `json:"metadata"` + // Driver identifies the database driver that produced the plan or summary. + Driver string `json:"driver"` + // Source identifies the source corpus file. + Source string `json:"source"` + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset,omitempty"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // WorkloadSHA256 identifies the backend-independent source workload. + WorkloadSHA256 string `json:"workload_sha256"` + // Cypher contains the Cypher statement under test. + Cypher string `json:"cypher"` + // Params supplies literal query parameters. + Params map[string]any `json:"params,omitempty"` + // SQL contains the rendered SQL statement. + SQL string `json:"sql,omitempty"` + // PGPlan contains the normalized PostgreSQL text plan. + PGPlan []string `json:"pg_plan,omitempty"` + // PGPlanFingerprint identifies the normalized PostgreSQL plan without retaining another copy. + PGPlanFingerprint string `json:"pg_plan_fingerprint,omitempty"` + // PGOperators lists normalized PostgreSQL operators found in the captured plan. + PGOperators []string `json:"pg_operators,omitempty"` + // Neo4jPlan contains the normalized Neo4j operator tree. + Neo4jPlan *Neo4jPlanNode `json:"neo4j_plan,omitempty"` + // Neo4jPlanFingerprint identifies the normalized Neo4j plan tree. + Neo4jPlanFingerprint string `json:"neo4j_plan_fingerprint,omitempty"` + // Neo4jOperators lists normalized Neo4j operators found in the captured plan. + Neo4jOperators []string `json:"neo4j_operators,omitempty"` + // PlannedLowerings lists SQL lowering opportunities identified before optimization. + PlannedLowerings []string `json:"planned_lowerings,omitempty"` + // AppliedLowerings lists SQL lowerings actually applied during translation. + AppliedLowerings []string `json:"applied_lowerings,omitempty"` + // SkippedLowerings lists identified SQL lowerings not applied. + SkippedLowerings []translate.SkippedLowering `json:"skipped_lowerings,omitempty"` + // Optimization captures translation optimization and lowering decisions. + Optimization *translate.OptimizationSummary `json:"optimization,omitempty"` + // Error records the failure message when the operation did not succeed. + Error string `json:"error,omitempty"` } +// Neo4jPlanNode models the recursive operator tree returned by Neo4j EXPLAIN. type Neo4jPlanNode struct { - Operator string `json:"operator"` - Arguments map[string]string `json:"arguments,omitempty"` - Identifiers []string `json:"identifiers,omitempty"` - Children []Neo4jPlanNode `json:"children,omitempty"` + // Operator identifies the backend plan operator at this node. + Operator string `json:"operator"` + // Arguments maps backend plan argument names to stable string representations. + Arguments map[string]string `json:"arguments,omitempty"` + // Identifiers lists variables or identifiers referenced by the Neo4j plan node. + Identifiers []string `json:"identifiers,omitempty"` + // Children contains child Neo4j plan operators in backend order. + Children []Neo4jPlanNode `json:"children,omitempty"` + // EstimatedRows records planner cardinality when exposed by the server. + EstimatedRows *float64 `json:"estimated_rows,omitempty"` + // ActualRows records profiled output cardinality when this is an executed read plan. + ActualRows *int64 `json:"actual_rows,omitempty"` + // DBHits records profiled store accesses when exposed by the server. + DBHits *int64 `json:"db_hits,omitempty"` + // PageCacheHits records profiled page-cache hits when exposed by the server. + PageCacheHits *int64 `json:"page_cache_hits,omitempty"` + // PageCacheMisses records profiled page-cache misses when exposed by the server. + PageCacheMisses *int64 `json:"page_cache_misses,omitempty"` + // TimeNS records profiled operator time in nanoseconds when exposed by the server. + TimeNS *int64 `json:"time_ns,omitempty"` +} + +// PlanDeltaReport contains backend-paired semantic plan comparisons without +// treating backend-specific operator counters as interchangeable. +type PlanDeltaReport struct { + // Version identifies the serialized plan-delta schema revision. + Version int `json:"version"` + // Records contains complete and explicitly incomplete backend pairs. + Records []PlanDeltaRecord `json:"records"` + // RankedFindings prioritizes semantic disagreements and qualification cases. + RankedFindings []PlanDeltaFinding `json:"ranked_findings,omitempty"` +} + +// PlanDeltaFinding ranks one cross-backend semantic observation for review. +type PlanDeltaFinding struct { + // Rank is the one-based position after stable severity ordering. + Rank int `json:"rank"` + // Category identifies the semantic disagreement being ranked. + Category string `json:"category"` + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset,omitempty"` + // Source identifies the corpus declaration. + Source string `json:"source"` + // Name identifies the workload case. + Name string `json:"name"` + // PairSHA256 identifies the exact paired record. + PairSHA256 string `json:"pair_sha256"` + // Score is a category-local descending severity score. + Score float64 `json:"score"` + // Summary is a compact stable explanation of the finding. + Summary string `json:"summary"` +} + +// PlanDeltaRecord compares one source workload across PostgreSQL and Neo4j. +type PlanDeltaRecord struct { + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset,omitempty"` + // Source identifies the source corpus declaration. + Source string `json:"source"` + // Name identifies the case within its source. + Name string `json:"name"` + // WorkloadSHA256 identifies the backend-independent source workload. + WorkloadSHA256 string `json:"workload_sha256"` + // SourceRevision identifies the DAWGS source used for capture. + SourceRevision string `json:"source_revision,omitempty"` + // PairSHA256 binds workload, source revision, and both backend plan fingerprints. + PairSHA256 string `json:"pair_sha256"` + // Postgres records the PostgreSQL side when captured. + Postgres *SemanticPlan `json:"postgres,omitempty"` + // Neo4j records the Neo4j side when captured. + Neo4j *SemanticPlan `json:"neo4j,omitempty"` + // Complete reports whether both backend plans were captured successfully. + Complete bool `json:"complete"` + // IncompleteReason explains a missing or failed backend side. + IncompleteReason string `json:"incomplete_reason,omitempty"` + // OppositeStartingSides reports a material starting-side disagreement. + OppositeStartingSides bool `json:"opposite_starting_sides,omitempty"` + // OppositePhysicalDirections reports a physical adjacency disagreement. + OppositePhysicalDirections bool `json:"opposite_physical_directions,omitempty"` + // Neo4jReorderedPattern reports that Neo4j started from the opposite logical endpoint. + Neo4jReorderedPattern bool `json:"neo4j_reordered_pattern,omitempty"` + // ChosenSideDidLessObservedWork reports whether Neo4j's first leaf had no more profiled work than the alternative leaf. + ChosenSideDidLessObservedWork *bool `json:"chosen_side_did_less_observed_work,omitempty"` + // SeedEstimateQError reports symmetric disagreement between backend seed estimates. + SeedEstimateQError *float64 `json:"seed_estimate_q_error,omitempty"` + // TraversalEstimateQError reports symmetric disagreement between backend traversal estimates. + TraversalEstimateQError *float64 `json:"traversal_estimate_q_error,omitempty"` + // OutputEstimateQError reports symmetric disagreement between backend output estimates. + OutputEstimateQError *float64 `json:"output_estimate_q_error,omitempty"` + // PredicatePlacementMoved reports a backend disagreement in predicate-bearing stages. + PredicatePlacementMoved bool `json:"predicate_placement_moved,omitempty"` + // HydrationEstimateQError reports symmetric disagreement in identifiable hydration work. + HydrationEstimateQError *float64 `json:"hydration_estimate_q_error,omitempty"` +} + +// SemanticPlan normalizes one backend plan into comparable traversal stages. +type SemanticPlan struct { + // Driver identifies the backend that produced this plan. + Driver string `json:"driver"` + // PlanFingerprint identifies the complete normalized backend plan. + PlanFingerprint string `json:"plan_fingerprint"` + // StartingAccess describes the first observed leaf access. + StartingAccess string `json:"starting_access,omitempty"` + // TerminalAccess describes the opposite endpoint access when identifiable. + TerminalAccess string `json:"terminal_access,omitempty"` + // LogicalDirection describes the query's directed traversal orientation. + LogicalDirection string `json:"logical_direction,omitempty"` + // PhysicalDirection identifies start_id or end_id adjacency use. + PhysicalDirection string `json:"physical_direction,omitempty"` + // PredicatePlacement lists stages carrying predicates or filters. + PredicatePlacement []string `json:"predicate_placement,omitempty"` + // EndpointBinding reports whether both endpoints are available before traversal. + EndpointBinding string `json:"endpoint_binding,omitempty"` + // OperatorFamily classifies ordinary expansion, SP, ASP, or fixed-hop work. + OperatorFamily string `json:"operator_family,omitempty"` + // EstimatedSeeds records a comparable seed estimate when exposed. + EstimatedSeeds *float64 `json:"estimated_seeds,omitempty"` + // EstimatedTraversal records a comparable traversal estimate when exposed. + EstimatedTraversal *float64 `json:"estimated_traversal,omitempty"` + // EstimatedOutput records a comparable output estimate when exposed. + EstimatedOutput *float64 `json:"estimated_output,omitempty"` + // EstimatedHydration records rows at an identifiable hydration/materialization stage. + EstimatedHydration *float64 `json:"estimated_hydration,omitempty"` + // ActualOutput records profiled output rows when exposed. + ActualOutput *int64 `json:"actual_output,omitempty"` + // ObservedSeedWork records actual rows or store hits at the selected seed leaf when exposed. + ObservedSeedWork *int64 `json:"observed_seed_work,omitempty"` + // ObservedAlternativeSeedWork records the comparable opposite leaf's work when exposed. + ObservedAlternativeSeedWork *int64 `json:"observed_alternative_seed_work,omitempty"` + // ObservedTraversalWork records profiled traversal DB hits when exposed. + ObservedTraversalWork *int64 `json:"observed_traversal_work,omitempty"` + // ObservedHydrationRows records profiled hydration rows when exposed. + ObservedHydrationRows *int64 `json:"observed_hydration_rows,omitempty"` + // OutputQError records estimate error when both estimate and actual output exist. + OutputQError *float64 `json:"output_q_error,omitempty"` + // PlannedIdentity records the optimizer-selected CySQL candidate. + PlannedIdentity string `json:"planned_identity,omitempty"` + // EmittedIdentity records the candidate actually emitted by translation. + EmittedIdentity string `json:"emitted_identity,omitempty"` + // PlannedCandidates lists the complete typed candidate set. + PlannedCandidates []string `json:"planned_candidates,omitempty"` + // EmittedCandidates lists the arms present in translated SQL. + EmittedCandidates []string `json:"emitted_candidates,omitempty"` + // FallbackIdentity records the exact incumbent chain declared by translation. + FallbackIdentity string `json:"fallback_identity,omitempty"` + // FallbackReason records static qualification failure or guarded fallback intent. + FallbackReason string `json:"fallback_reason,omitempty"` + // SelectorVersion identifies the policy that produced the plan. + SelectorVersion string `json:"selector_version,omitempty"` + // ProbeCaps records bounded runtime evidence limits declared by the plan. + ProbeCaps map[string]int64 `json:"probe_caps,omitempty"` + // RuntimeIdentityKnown is false for PlanCorpus because execution telemetry is GraphBench authority. + RuntimeIdentityKnown bool `json:"runtime_identity_known"` + // InternalTraversalWork marks backend work that profiling cannot expose. + InternalTraversalWork string `json:"internal_traversal_work,omitempty"` + // Error retains a capture failure without dropping the pair. + Error string `json:"error,omitempty"` + // RawOptimization retains typed translation diagnostics for PostgreSQL. + RawOptimization *translate.OptimizationSummary `json:"raw_optimization,omitempty"` + // PlanJSON optionally retains a stable semantic projection for downstream tools. + PlanJSON json.RawMessage `json:"plan_json,omitempty"` } +// CorpusQuery defines one corpus query and the fixture parameters needed to execute it. type CorpusQuery struct { - Source string + // Source identifies the source corpus file. + Source string + // Dataset identifies the fixture dataset. Dataset string - Name string - Cypher string - Params map[string]any + // Name identifies the case or record within its dataset. + Name string + // Cypher contains the Cypher statement under test. + Cypher string + // Params supplies literal query parameters. + Params map[string]any } diff --git a/cypher/frontend/expression.go b/cypher/frontend/expression.go index 994105bc..bdb692b4 100644 --- a/cypher/frontend/expression.go +++ b/cypher/frontend/expression.go @@ -423,6 +423,7 @@ func (s *NonArithmeticOperatorExpressionVisitor) EnterOC_PropertyKeyName(ctx *pa s.ctx.Enter(&SymbolicNameOrReservedWordVisitor{}) } +// ExitOC_PropertyKeyName assigns the parsed key to the property lookup under construction. func (s *NonArithmeticOperatorExpressionVisitor) ExitOC_PropertyKeyName(ctx *parser.OC_PropertyKeyNameContext) { - s.PropertyKeyName = s.ctx.Exit().(*SymbolicNameOrReservedWordVisitor).Name + s.PropertyKeyName = extractPropertyKeyName(s.ctx, ctx) } diff --git a/cypher/frontend/literal.go b/cypher/frontend/literal.go index 735785a9..4797d304 100644 --- a/cypher/frontend/literal.go +++ b/cypher/frontend/literal.go @@ -44,8 +44,9 @@ func (s *MapLiteralVisitor) EnterOC_PropertyKeyName(ctx *parser.OC_PropertyKeyNa s.ctx.Enter(&SymbolicNameOrReservedWordVisitor{}) } +// ExitOC_PropertyKeyName decodes and retains the key for the next map-literal entry. func (s *MapLiteralVisitor) ExitOC_PropertyKeyName(ctx *parser.OC_PropertyKeyNameContext) { - s.nextPropertyKey = s.ctx.Exit().(*SymbolicNameOrReservedWordVisitor).Name + s.nextPropertyKey = cypher.UnescapePropertyKeyName(s.ctx.Exit().(*SymbolicNameOrReservedWordVisitor).Name) } func (s *MapLiteralVisitor) EnterOC_Expression(ctx *parser.OC_ExpressionContext) { diff --git a/cypher/frontend/property_key.go b/cypher/frontend/property_key.go new file mode 100644 index 00000000..c4fd6841 --- /dev/null +++ b/cypher/frontend/property_key.go @@ -0,0 +1,21 @@ +package frontend + +import ( + "github.com/specterops/dawgs/cypher/models/cypher" + "github.com/specterops/dawgs/cypher/parser" +) + +// extractPropertyKeyName decodes a parsed property-key token and records a syntax error when the decoded key is invalid. +func extractPropertyKeyName(ctx *Context, cypherCtx *parser.OC_PropertyKeyNameContext) string { + name := cypher.UnescapePropertyKeyName(ctx.Exit().(*SymbolicNameOrReservedWordVisitor).Name) + if err := cypher.ValidatePropertyKeyName(name); err != nil { + ctx.AddErrors(SyntaxError{ + Line: cypherCtx.GetStart().GetLine(), + Column: cypherCtx.GetStart().GetColumn(), + OffendingSymbol: cypherCtx.GetText(), + Message: err.Error(), + }) + } + + return name +} diff --git a/cypher/frontend/property_key_test.go b/cypher/frontend/property_key_test.go new file mode 100644 index 00000000..ea169b68 --- /dev/null +++ b/cypher/frontend/property_key_test.go @@ -0,0 +1,111 @@ +package frontend_test + +import ( + "testing" + + "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/cypher/models/cypher" + "github.com/specterops/dawgs/cypher/models/walk" + "github.com/stretchr/testify/require" +) + +// TestParsePropertyLookupStoresRawPropertyKeyNames verifies that lookup tokens are decoded before storage in the AST. +func TestParsePropertyLookupStoresRawPropertyKeyNames(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), "RETURN n.match, n.`a-aaa`, n.`has``tick`, n.` `") + require.NoError(t, err) + + var symbols []string + err = walk.CypherStructural(regularQuery, walk.NewSimpleVisitor[cypher.SyntaxNode](func(node cypher.SyntaxNode, _ walk.VisitorHandler) { + if propertyLookup, typeOK := node.(*cypher.PropertyLookup); typeOK { + symbols = append(symbols, propertyLookup.Symbol) + } + })) + require.NoError(t, err) + + require.Equal(t, []string{"match", "a-aaa", "has`tick", " "}, symbols) +} + +// TestParsePropertyLookupStoresQuotePropertyKeyNames verifies that quote characters survive property-key parsing unchanged. +func TestParsePropertyLookupStoresQuotePropertyKeyNames(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), "RETURN n.`'`, n.`\"`") + require.NoError(t, err) + + var symbols []string + err = walk.CypherStructural(regularQuery, walk.NewSimpleVisitor[cypher.SyntaxNode](func(node cypher.SyntaxNode, _ walk.VisitorHandler) { + if propertyLookup, typeOK := node.(*cypher.PropertyLookup); typeOK { + symbols = append(symbols, propertyLookup.Symbol) + } + })) + require.NoError(t, err) + + require.Equal(t, []string{"'", "\""}, symbols) +} + +// TestParsePropertyLookupStoresUnicodePropertyKeyNames verifies the Unicode classes accepted in raw property keys. +func TestParsePropertyLookupStoresUnicodePropertyKeyNames(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), "RETURN n.\u2118, n.a\u00b7, n.a\u0301, n.a\u093e, n.a$, n.`a\u20dd`") + require.NoError(t, err) + + var symbols []string + err = walk.CypherStructural(regularQuery, walk.NewSimpleVisitor[cypher.SyntaxNode](func(node cypher.SyntaxNode, _ walk.VisitorHandler) { + if propertyLookup, typeOK := node.(*cypher.PropertyLookup); typeOK { + symbols = append(symbols, propertyLookup.Symbol) + } + })) + require.NoError(t, err) + + require.Equal(t, []string{"\u2118", "a\u00b7", "a\u0301", "a\u093e", "a$", "a\u20dd"}, symbols) +} + +// TestParseMapLiteralStoresRawPropertyKeyNames verifies that map keys are decoded before storage in the AST. +func TestParseMapLiteralStoresRawPropertyKeyNames(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), "RETURN {match: 1, `a-aaa`: 2, `has``tick`: 3, ``: 4, ` `: 5}") + require.NoError(t, err) + + var keys []string + err = walk.CypherStructural(regularQuery, walk.NewSimpleVisitor[cypher.SyntaxNode](func(node cypher.SyntaxNode, _ walk.VisitorHandler) { + if mapItem, typeOK := node.(*cypher.MapItem); typeOK { + keys = append(keys, mapItem.Key) + } + })) + require.NoError(t, err) + + require.ElementsMatch(t, []string{"match", "a-aaa", "has`tick", "", " "}, keys) +} + +// TestParseMapLiteralStoresQuotePropertyKeyNames verifies that quote characters survive map-key parsing unchanged. +func TestParseMapLiteralStoresQuotePropertyKeyNames(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), "RETURN {`'`: 1, `\"`: 2}") + require.NoError(t, err) + + var keys []string + err = walk.CypherStructural(regularQuery, walk.NewSimpleVisitor[cypher.SyntaxNode](func(node cypher.SyntaxNode, _ walk.VisitorHandler) { + if mapItem, typeOK := node.(*cypher.MapItem); typeOK { + keys = append(keys, mapItem.Key) + } + })) + require.NoError(t, err) + + require.ElementsMatch(t, []string{"'", "\""}, keys) +} + +// TestParseRejectsEmptyPropertyKeyNames verifies that empty escaped keys are rejected in every property-key position. +func TestParseRejectsEmptyPropertyKeyNames(t *testing.T) { + testCases := []struct { + // name labels the property-key syntax under test. + name string + // query contains an empty escaped key in the named syntax position. + query string + }{ + {name: "property lookup", query: "RETURN n.``"}, + {name: "set property", query: "MATCH (n) SET n.`` = 'value'"}, + {name: "remove property", query: "MATCH (n) REMOVE n.``"}, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + _, err := frontend.ParseCypher(frontend.NewContext(), testCase.query) + require.ErrorContains(t, err, cypher.ErrEmptyPropertyKeyName.Error()) + }) + } +} diff --git a/cypher/frontend/query.go b/cypher/frontend/query.go index 27045fab..36a338ed 100644 --- a/cypher/frontend/query.go +++ b/cypher/frontend/query.go @@ -706,6 +706,7 @@ func (s *PropertyExpressionVisitor) EnterOC_PropertyKeyName(ctx *parser.OC_Prope s.ctx.Enter(&SymbolicNameOrReservedWordVisitor{}) } +// ExitOC_PropertyKeyName assigns the parsed key to the property expression under construction. func (s *PropertyExpressionVisitor) ExitOC_PropertyKeyName(ctx *parser.OC_PropertyKeyNameContext) { - s.PropertyLookup.SetSymbol(s.ctx.Exit().(*SymbolicNameOrReservedWordVisitor).Name) + s.PropertyLookup.SetSymbol(extractPropertyKeyName(s.ctx, ctx)) } diff --git a/cypher/models/cypher/format/format.go b/cypher/models/cypher/format/format.go index 0173915c..7e12a9cd 100644 --- a/cypher/models/cypher/format/format.go +++ b/cypher/models/cypher/format/format.go @@ -12,6 +12,7 @@ import ( "github.com/specterops/dawgs/graph" ) +// strippedLiteral replaces literal values when emitting a privacy-preserving Cypher query. const strippedLiteral = "$STRIPPED" func writeJoinedKinds(output io.Writer, delimiter string, kinds graph.Kinds) error { @@ -317,6 +318,7 @@ func (s Emitter) formatWhere(output io.Writer, whereClause *cypher.Where) error return nil } +// formatMapLiteral renders a Cypher map literal with each property key escaped as needed. func (s Emitter) formatMapLiteral(output io.Writer, mapLiteral cypher.MapLiteral) error { if _, err := io.WriteString(output, "{"); err != nil { return err @@ -332,7 +334,7 @@ func (s Emitter) formatMapLiteral(output io.Writer, mapLiteral cypher.MapLiteral first = false } - if _, err := io.WriteString(output, key); err != nil { + if _, err := io.WriteString(output, cypher.EscapePropertyKeyName(key)); err != nil { return err } @@ -446,6 +448,7 @@ func (s Emitter) formatLiteral(output io.Writer, literal *cypher.Literal) error return nil } +// WriteExpression renders an expression and its nested operands as Cypher syntax. func (s Emitter) WriteExpression(output io.Writer, expression cypher.Expression) error { switch typedExpression := expression.(type) { case *cypher.ProjectionItem: @@ -633,7 +636,11 @@ func (s Emitter) WriteExpression(output io.Writer, expression cypher.Expression) return err } - if _, err := io.WriteString(output, typedExpression.Symbol); err != nil { + if err := cypher.ValidatePropertyKeyName(typedExpression.Symbol); err != nil { + return err + } + + if _, err := io.WriteString(output, cypher.EscapePropertyKeyName(typedExpression.Symbol)); err != nil { return err } diff --git a/cypher/models/cypher/format/format_test.go b/cypher/models/cypher/format/format_test.go index b10e8001..44474401 100644 --- a/cypher/models/cypher/format/format_test.go +++ b/cypher/models/cypher/format/format_test.go @@ -44,6 +44,97 @@ func TestCypherEmitter_FormatsMapLiteralInKeyOrder(t *testing.T) { require.Equal(t, "{a: 1, b: 2}", buffer.String()) } +// TestCypherEmitter_FormatsMapLiteralPropertyKeys verifies that map keys are emitted bare or escaped according to property-key grammar. +func TestCypherEmitter_FormatsMapLiteralPropertyKeys(t *testing.T) { + var ( + buffer = &bytes.Buffer{} + emitter = format.NewCypherEmitter(false) + ) + + err := emitter.WriteExpression(buffer, cypher.MapLiteral{ + "match": cypher.NewLiteral(1, false), + "a-aaa": cypher.NewLiteral(2, false), + "has`tick": cypher.NewLiteral(3, false), + "": cypher.NewLiteral(4, false), + " ": cypher.NewLiteral(5, false), + "'": cypher.NewLiteral(6, false), + }) + + require.NoError(t, err) + require.Equal(t, "{``: 4, ` `: 5, `'`: 6, `a-aaa`: 2, `has``tick`: 3, match: 1}", buffer.String()) +} + +// TestCypherEmitter_FormatsPropertyLookupKeys verifies canonical rendering of bare and escaped lookup keys. +func TestCypherEmitter_FormatsPropertyLookupKeys(t *testing.T) { + testCases := []struct { + // name labels the property-key form under test. + name string + // symbol is the raw property key stored in the AST. + symbol string + // expected is the canonical rendered property lookup. + expected string + }{ + { + name: "simple key", + symbol: "name", + expected: "n.name", + }, + { + name: "reserved word key", + symbol: "match", + expected: "n.match", + }, + { + name: "key with hyphen", + symbol: "a-aaa", + expected: "n.`a-aaa`", + }, + { + name: "key with backtick", + symbol: "has`tick", + expected: "n.`has``tick`", + }, + { + name: "key with single quote", + symbol: "'", + expected: "n.`'`", + }, + { + name: "whitespace-only key", + symbol: " ", + expected: "n.` `", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + buffer := &bytes.Buffer{} + emitter := format.NewCypherEmitter(false) + + err := emitter.WriteExpression(buffer, &cypher.PropertyLookup{ + Atom: cypher.NewVariableWithSymbol("n"), + Symbol: testCase.symbol, + }) + + require.NoError(t, err) + require.Equal(t, testCase.expected, buffer.String()) + }) + } +} + +// TestCypherEmitter_RejectsEmptyPropertyLookupKey verifies that an empty raw lookup key cannot be rendered. +func TestCypherEmitter_RejectsEmptyPropertyLookupKey(t *testing.T) { + buffer := &bytes.Buffer{} + emitter := format.NewCypherEmitter(false) + + err := emitter.WriteExpression(buffer, &cypher.PropertyLookup{ + Atom: cypher.NewVariableWithSymbol("n"), + Symbol: "", + }) + + require.ErrorIs(t, err, cypher.ErrEmptyPropertyKeyName) +} + func TestCypherEmitter_MapLiteralPropagatesExpressionError(t *testing.T) { var ( buffer = &bytes.Buffer{} diff --git a/cypher/models/cypher/functions.go b/cypher/models/cypher/functions.go index 21dcac2b..96e42117 100644 --- a/cypher/models/cypher/functions.go +++ b/cypher/models/cypher/functions.go @@ -1,46 +1,126 @@ package cypher const ( - CountFunction = "count" - DateFunction = "date" - TimeFunction = "time" - LocalTimeFunction = "localtime" - DateTimeFunction = "datetime" - LocalDateTimeFunction = "localdatetime" - DurationFunction = "duration" - IdentityFunction = "id" - ToLowerFunction = "tolower" - ToUpperFunction = "toupper" - NodeLabelsFunction = "labels" - EdgeTypeFunction = "type" - StartNodeFunction = "startnode" - EndNodeFunction = "endnode" + // CountFunction identifies the Cypher aggregate that counts non-null values or rows. + CountFunction = "count" + + // DateFunction identifies the Cypher constructor for date values. + DateFunction = "date" + + // TimeFunction identifies the Cypher constructor for zoned time values. + TimeFunction = "time" + + // LocalTimeFunction identifies the Cypher constructor for local time values. + LocalTimeFunction = "localtime" + + // DateTimeFunction identifies the Cypher constructor for zoned date-time values. + DateTimeFunction = "datetime" + + // LocalDateTimeFunction identifies the Cypher constructor for local date-time values. + LocalDateTimeFunction = "localdatetime" + + // DurationFunction identifies the Cypher constructor for duration values. + DurationFunction = "duration" + + // IdentityFunction identifies the Cypher function that returns an entity ID. + IdentityFunction = "id" + + // ToLowerFunction identifies the Cypher function that lowercases text. + ToLowerFunction = "tolower" + + // ToUpperFunction identifies the Cypher function that uppercases text. + ToUpperFunction = "toupper" + + // NodeLabelsFunction identifies the Cypher function that returns a node's labels. + NodeLabelsFunction = "labels" + + // EdgeTypeFunction identifies the Cypher function that returns a relationship's type. + EdgeTypeFunction = "type" + + // StartNodeFunction identifies the Cypher function that returns a relationship's start node. + StartNodeFunction = "startnode" + + // EndNodeFunction identifies the Cypher function that returns a relationship's end node. + EndNodeFunction = "endnode" + + // StringSplitToArrayFunction identifies the Cypher function that splits text into a list. StringSplitToArrayFunction = "split" - ToStringFunction = "tostring" - ToIntegerFunction = "tointeger" - ListSizeFunction = "size" - HeadFunction = "head" - TailFunction = "tail" - NodesFunction = "nodes" - RelationshipsFunction = "relationships" - CoalesceFunction = "coalesce" - CollectFunction = "collect" - SumFunction = "sum" - AvgFunction = "avg" - MinFunction = "min" - MaxFunction = "max" - - // ITTC - Instant Type; Temporal Component (https://neo4j.com/docs/cypher-manual/current/functions/temporal/) - ITTCYear = "year" - ITTCMonth = "month" - ITTCDay = "day" - ITTCHour = "hour" - ITTCMinute = "minute" - ITTCSecond = "second" - ITTCMillisecond = "millisecond" - ITTCMicrosecond = "microsecond" - ITTCNanosecond = "nanosecond" - ITTCTimeZone = "timezone" - ITTCEpochSeconds = "epochseconds" + + // ToStringFunction identifies the Cypher function that converts a value to text. + ToStringFunction = "tostring" + + // ToIntegerFunction identifies the Cypher function that converts a value to an integer. + ToIntegerFunction = "tointeger" + + // ListSizeFunction identifies the Cypher function that returns the size of a list or string. + ListSizeFunction = "size" + + // HeadFunction identifies the Cypher function that returns the first list element. + HeadFunction = "head" + + // TailFunction identifies the Cypher function that returns all but the first list element. + TailFunction = "tail" + + // NodesFunction identifies the Cypher function that returns a path's nodes in order. + NodesFunction = "nodes" + + // RelationshipsFunction identifies the Cypher function that returns a path's relationships in order. + RelationshipsFunction = "relationships" + + // PathLengthFunction identifies the Cypher function that returns the number of relationships in a path. + PathLengthFunction = "length" + + // CoalesceFunction identifies the Cypher function that returns the first non-null argument. + CoalesceFunction = "coalesce" + + // CollectFunction identifies the Cypher aggregate that collects values into a list. + CollectFunction = "collect" + + // SumFunction identifies the Cypher aggregate that sums numeric values. + SumFunction = "sum" + + // AvgFunction identifies the Cypher aggregate that averages numeric values. + AvgFunction = "avg" + + // MinFunction identifies the Cypher aggregate that returns the minimum value. + MinFunction = "min" + + // MaxFunction identifies the Cypher aggregate that returns the maximum value. + MaxFunction = "max" + + // ITTCYear identifies the year component of a Cypher instant value. + ITTCYear = "year" + + // ITTCMonth identifies the month component of a Cypher instant value. + ITTCMonth = "month" + + // ITTCDay identifies the day component of a Cypher instant value. + ITTCDay = "day" + + // ITTCHour identifies the hour component of a Cypher instant value. + ITTCHour = "hour" + + // ITTCMinute identifies the minute component of a Cypher instant value. + ITTCMinute = "minute" + + // ITTCSecond identifies the second component of a Cypher instant value. + ITTCSecond = "second" + + // ITTCMillisecond identifies the millisecond component of a Cypher instant value. + ITTCMillisecond = "millisecond" + + // ITTCMicrosecond identifies the microsecond component of a Cypher instant value. + ITTCMicrosecond = "microsecond" + + // ITTCNanosecond identifies the nanosecond component of a Cypher instant value. + ITTCNanosecond = "nanosecond" + + // ITTCTimeZone identifies the time-zone component of a Cypher instant value. + ITTCTimeZone = "timezone" + + // ITTCEpochSeconds identifies the epoch-seconds component of a Cypher instant value. + ITTCEpochSeconds = "epochseconds" + + // ITTCEpochMilliseconds identifies the epoch-milliseconds component of a Cypher instant value. ITTCEpochMilliseconds = "epochmillis" ) diff --git a/cypher/models/cypher/model.go b/cypher/models/cypher/model.go index 173919b8..9154f465 100644 --- a/cypher/models/cypher/model.go +++ b/cypher/models/cypher/model.go @@ -1306,8 +1306,13 @@ func (s *ProjectionItem) copy() *ProjectionItem { } } +// PropertyLookup represents access to a named property on an expression. type PropertyLookup struct { - Atom Expression + // Atom is the expression whose property is accessed. + Atom Expression + + // Symbol is the raw property key, not an already-rendered Cypher token. + // Callers should not pre-wrap names in backticks; formatting handles that. Symbol string } diff --git a/cypher/models/cypher/property_key.go b/cypher/models/cypher/property_key.go new file mode 100644 index 00000000..3cb92f19 --- /dev/null +++ b/cypher/models/cypher/property_key.go @@ -0,0 +1,86 @@ +package cypher + +import ( + "errors" + "strings" + "unicode" +) + +// ErrEmptyPropertyKeyName reports that a property-key token decoded to an empty name. +var ErrEmptyPropertyKeyName = errors.New("property key name must not be empty") + +// isCypherIDStart reports whether char may begin an unescaped Cypher identifier. +func isCypherIDStart(char rune) bool { + return unicode.IsLetter(char) || unicode.In(char, unicode.Nl, unicode.Other_ID_Start) +} + +// isCypherIDContinue reports whether char may follow the first rune of an unescaped Cypher identifier. +func isCypherIDContinue(char rune) bool { + return isCypherIDStart(char) || unicode.In(char, unicode.Mn, unicode.Mc, unicode.Nd, unicode.Pc, unicode.Other_ID_Continue) +} + +// isCypherSymbolStart reports whether char may begin an unescaped symbolic name, including connector punctuation. +func isCypherSymbolStart(char rune) bool { + return isCypherIDStart(char) || unicode.In(char, unicode.Pc) +} + +// isCypherSymbolPart reports whether char may appear after the first rune of an unescaped symbolic name. +func isCypherSymbolPart(char rune) bool { + return isCypherIDContinue(char) || unicode.In(char, unicode.Sc) +} + +// CanEmitBarePropertyKeyName returns true when a raw property key can be emitted without backticks. +// +// This is specific to Cypher property-key position, such as n.name and {name: value}. Property keys use +// oC_PropertyKeyName -> oC_SchemaName, where reserved words are valid bare names, unlike variable or parameter +// symbols. Empty keys and keys containing characters outside the unescaped symbolic-name grammar return false; non-empty +// keys outside the bare grammar are still representable by EscapePropertyKeyName using backticks. +func CanEmitBarePropertyKeyName(name string) bool { + if name == "" { + return false + } + + for idx, char := range name { + if idx == 0 { + if !isCypherSymbolStart(char) { + return false + } + } else if !isCypherSymbolPart(char) { + return false + } + } + + return true +} + +// ValidatePropertyKeyName rejects empty decoded property-key names. +func ValidatePropertyKeyName(name string) error { + if name == "" { + return ErrEmptyPropertyKeyName + } + + return nil +} + +// EscapePropertyKeyName formats a raw property key as a Cypher property-key token. +func EscapePropertyKeyName(name string) string { + if CanEmitBarePropertyKeyName(name) { + return name + } + + return "`" + strings.ReplaceAll(name, "`", "``") + "`" +} + +// IsEscapedPropertyKeyName returns true when name is wrapped in Cypher backtick delimiters. +func IsEscapedPropertyKeyName(name string) bool { + return len(name) >= 2 && name[0] == '`' && name[len(name)-1] == '`' +} + +// UnescapePropertyKeyName decodes a Cypher property-key token into the raw property key it names. +func UnescapePropertyKeyName(name string) string { + if !IsEscapedPropertyKeyName(name) { + return name + } + + return strings.ReplaceAll(name[1:len(name)-1], "``", "`") +} diff --git a/cypher/models/cypher/property_key_test.go b/cypher/models/cypher/property_key_test.go new file mode 100644 index 00000000..7d7c2f25 --- /dev/null +++ b/cypher/models/cypher/property_key_test.go @@ -0,0 +1,110 @@ +package cypher_test + +import ( + "testing" + + "github.com/specterops/dawgs/cypher/models/cypher" + "github.com/stretchr/testify/require" +) + +// TestCanEmitBarePropertyKeyName verifies the Unicode and punctuation rules for unescaped property keys. +func TestCanEmitBarePropertyKeyName(t *testing.T) { + testCases := []struct { + // name labels the property-key form under test. + name string + // input is the decoded property-key name. + input string + // expected indicates whether input may be rendered without backticks. + expected bool + }{ + {name: "simple", input: "name", expected: true}, + {name: "underscore", input: "object_id", expected: true}, + {name: "reserved word allowed in property key position", input: "match", expected: true}, + {name: "other id start", input: "\u2118", expected: true}, + {name: "other id continue", input: "a\u00b7", expected: true}, + {name: "nonspacing mark part", input: "a\u0301", expected: true}, + {name: "spacing mark part", input: "a\u093e", expected: true}, + {name: "currency symbol part", input: "a$", expected: true}, + {name: "empty", input: "", expected: false}, + {name: "dash", input: "a-aaa", expected: false}, + {name: "starts digit", input: "1name", expected: false}, + {name: "starts currency symbol", input: "$a", expected: false}, + {name: "literal backtick", input: "has`tick", expected: false}, + {name: "enclosing mark part", input: "a\u20dd", expected: false}, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + require.Equal(t, testCase.expected, cypher.CanEmitBarePropertyKeyName(testCase.input)) + }) + } +} + +// TestEscapePropertyKeyName verifies canonical quoting and embedded-backtick escaping for property keys. +func TestEscapePropertyKeyName(t *testing.T) { + testCases := []struct { + // name labels the property-key form under test. + name string + // input is the decoded property-key name. + input string + // expected is the canonical property-key token. + expected string + }{ + {name: "simple", input: "name", expected: "name"}, + {name: "reserved word allowed in property key position", input: "match", expected: "match"}, + {name: "other id start", input: "\u2118", expected: "\u2118"}, + {name: "other id continue", input: "a\u00b7", expected: "a\u00b7"}, + {name: "nonspacing mark part", input: "a\u0301", expected: "a\u0301"}, + {name: "spacing mark part", input: "a\u093e", expected: "a\u093e"}, + {name: "currency symbol part", input: "a$", expected: "a$"}, + {name: "enclosing mark part", input: "a\u20dd", expected: "`a\u20dd`"}, + {name: "dash", input: "a-aaa", expected: "`a-aaa`"}, + {name: "embedded backtick", input: "has`tick", expected: "`has``tick`"}, + {name: "starts backtick", input: "`starts-tick", expected: "```starts-tick`"}, + {name: "wrapped backticks", input: "`super-wrapped`", expected: "```super-wrapped```"}, + {name: "single backtick", input: "`", expected: "````"}, + {name: "single quote", input: "'", expected: "`'`"}, + {name: "double quote", input: "\"", expected: "`\"`"}, + {name: "whitespace-only", input: " ", expected: "` `"}, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + require.Equal(t, testCase.expected, cypher.EscapePropertyKeyName(testCase.input)) + }) + } +} + +// TestValidatePropertyKeyName verifies that only empty decoded property-key names are invalid. +func TestValidatePropertyKeyName(t *testing.T) { + require.NoError(t, cypher.ValidatePropertyKeyName(" ")) + require.ErrorIs(t, cypher.ValidatePropertyKeyName(""), cypher.ErrEmptyPropertyKeyName) +} + +// TestUnescapePropertyKeyName verifies decoding of quoted keys and doubled backticks. +func TestUnescapePropertyKeyName(t *testing.T) { + testCases := []struct { + // name labels the property-key token under test. + name string + // input is the rendered property-key token. + input string + // expected is the decoded property-key name. + expected string + }{ + {name: "simple", input: "name", expected: "name"}, + {name: "dash", input: "`a-aaa`", expected: "a-aaa"}, + {name: "embedded backtick", input: "`has``tick`", expected: "has`tick"}, + {name: "starts backtick", input: "```starts-tick`", expected: "`starts-tick"}, + {name: "wrapped backticks", input: "```super-wrapped```", expected: "`super-wrapped`"}, + {name: "single backtick", input: "````", expected: "`"}, + {name: "single quote", input: "`'`", expected: "'"}, + {name: "double quote", input: "`\"`", expected: "\""}, + {name: "empty", input: "``", expected: ""}, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + require.Equal(t, testCase.expected, cypher.UnescapePropertyKeyName(testCase.input)) + }) + } +} diff --git a/cypher/models/pgsql/format/format.go b/cypher/models/pgsql/format/format.go index 9cbed49c..acd89395 100644 --- a/cypher/models/pgsql/format/format.go +++ b/cypher/models/pgsql/format/format.go @@ -8,11 +8,38 @@ import ( "github.com/specterops/dawgs/cypher/models/pgsql" ) +// OutputBuilder accumulates PostgreSQL text with graph targeting and optional parameter materialization. type OutputBuilder struct { + // MaterializeParameters substitutes configured values for parameter references during rendering. MaterializeParameters bool - StripLiterals bool - parameters map[string]any - builder *strings.Builder + // StripLiterals records the requested literal-redaction mode for formatter configuration. + StripLiterals bool + // TargetGraphID selects the concrete graph partitions used to render persistent node and edge references. + TargetGraphID int32 + // parameters contains values substituted when MaterializeParameters is enabled. + parameters map[string]any + // builder accumulates the rendered PostgreSQL text. + builder *strings.Builder +} + +// formatIdentifier preserves the wildcard and quotes names containing characters outside the formatter's unquoted ASCII subset. +func formatIdentifier(identifier pgsql.Identifier) string { + value := identifier.String() + if value == pgsql.WildcardIdentifier.String() { + return value + } + + for idx, character := range value { + valid := character == '_' || character >= 'a' && character <= 'z' || character >= 'A' && character <= 'Z' + if idx > 0 { + valid = valid || character >= '0' && character <= '9' || character == '$' + } + if !valid { + return `"` + strings.ReplaceAll(value, `"`, `""`) + `"` + } + } + + return value } func NewOutputBuilder() *OutputBuilder { @@ -28,6 +55,14 @@ func (s *OutputBuilder) WithMaterializedParameters(parameters map[string]any) *O return s } +// WithTargetGraph renders persistent node and edge references against the +// concrete target partitions. Graph-local IDs are not globally unique, and a +// concrete relation also lets PostgreSQL avoid planning unrelated partitions. +func (s *OutputBuilder) WithTargetGraph(graphID int32) *OutputBuilder { + s.TargetGraphID = graphID + return s +} + func (s *OutputBuilder) HasOutput() bool { return s.builder.Len() != 0 } @@ -51,6 +86,7 @@ func (s *OutputBuilder) Build() string { return s.builder.String() } +// formatSlice writes a typed PostgreSQL array literal from a Go slice. func formatSlice[T any, TS []T](builder *OutputBuilder, slice TS, dataType pgsql.DataType) error { builder.Write("array [") @@ -68,6 +104,7 @@ func formatSlice[T any, TS []T](builder *OutputBuilder, slice TS, dataType pgsql return nil } +// formatValue writes a supported scalar or slice value as a PostgreSQL literal. func formatValue(builder *OutputBuilder, value any) error { switch typedValue := value.(type) { case uint: @@ -134,6 +171,7 @@ func formatValue(builder *OutputBuilder, value any) error { return nil } +// formatLiteral writes a literal value and its explicit PostgreSQL cast when required. func formatLiteral(builder *OutputBuilder, literal pgsql.Literal) error { if literal.Null { builder.Write("null") @@ -148,6 +186,7 @@ func formatLiteral(builder *OutputBuilder, literal pgsql.Literal) error { return formatValue(builder, literal.Value) } +// formatCase validates paired conditions and results before writing a CASE expression in clause order. func formatCase(builder *OutputBuilder, caseExpr pgsql.Case) error { if len(caseExpr.Conditions) != len(caseExpr.Then) { return fmt.Errorf("case expression has %d conditions and %d then expressions", len(caseExpr.Conditions), len(caseExpr.Then)) @@ -190,6 +229,7 @@ func formatCase(builder *OutputBuilder, caseExpr pgsql.Case) error { return nil } +// formatNode dispatches a PostgreSQL syntax node to the formatter for its concrete AST type. func formatNode(builder *OutputBuilder, rootExpr pgsql.SyntaxNode) error { exprStack := []pgsql.SyntaxNode{ rootExpr, @@ -254,6 +294,15 @@ func formatNode(builder *OutputBuilder, rootExpr pgsql.SyntaxNode) error { if !typedNextExpr.Bare { exprStack = append(exprStack, pgsql.FormattingLiteral(")")) } + if len(typedNextExpr.OrderBy) > 0 { + for idx := len(typedNextExpr.OrderBy) - 1; idx >= 0; idx-- { + exprStack = append(exprStack, typedNextExpr.OrderBy[idx]) + if idx > 0 { + exprStack = append(exprStack, pgsql.FormattingLiteral(", ")) + } + } + exprStack = append(exprStack, pgsql.FormattingLiteral(" order by ")) + } for idx := len(typedNextExpr.Parameters) - 1; idx >= 0; idx-- { exprStack = append(exprStack, typedNextExpr.Parameters[idx]) @@ -277,7 +326,7 @@ func formatNode(builder *OutputBuilder, rootExpr pgsql.SyntaxNode) error { builder.Write(typedNextExpr.String()) case pgsql.Identifier: - builder.Write(typedNextExpr) + builder.Write(formatIdentifier(typedNextExpr)) case pgsql.CompoundIdentifier: for idx := len(typedNextExpr) - 1; idx >= 0; idx-- { @@ -329,7 +378,13 @@ func formatNode(builder *OutputBuilder, rootExpr pgsql.SyntaxNode) error { exprStack = append(exprStack, typedNextExpr.Binding.Value, pgsql.FormattingLiteral(" ")) } - exprStack = append(exprStack, typedNextExpr.Name) + tableName := typedNextExpr.Name + if builder.TargetGraphID != 0 && len(tableName) == 1 && + (tableName[0] == pgsql.TableNode || tableName[0] == pgsql.TableEdge) { + tableName = pgsql.CompoundIdentifier{pgsql.Identifier(fmt.Sprintf("%s_%d", tableName[0], builder.TargetGraphID))} + } + + exprStack = append(exprStack, tableName) case pgsql.LateralSubquery: if typedNextExpr.Binding.Set { @@ -438,7 +493,7 @@ func formatNode(builder *OutputBuilder, rootExpr pgsql.SyntaxNode) error { return fmt.Errorf("conflict target has both columns and an 'on constraint' expression set") } - exprStack = append(exprStack, typedNextExpr.Constraint, pgsql.FormattingLiteral("on constraint ")) + exprStack = append(exprStack, pgsql.FormattingLiteral(typedNextExpr.Constraint.String()), pgsql.FormattingLiteral("on constraint ")) } case *pgsql.AliasedExpression: @@ -538,12 +593,23 @@ func formatNode(builder *OutputBuilder, rootExpr pgsql.SyntaxNode) error { return fmt.Errorf("edge array from path IDs has no path expression") } - exprStack = append( - exprStack, - pgsql.FormattingLiteral(") with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id)"), - typedNextExpr.PathIDs, - pgsql.FormattingLiteral("(select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest("), - ) + if typedNextExpr.GraphID == nil { + exprStack = append( + exprStack, + pgsql.FormattingLiteral(") with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id)"), + typedNextExpr.PathIDs, + pgsql.FormattingLiteral("(select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest("), + ) + } else { + exprStack = append( + exprStack, + pgsql.FormattingLiteral(")"), + typedNextExpr.GraphID, + pgsql.FormattingLiteral(") with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id and _edge.graph_id = "), + typedNextExpr.PathIDs, + pgsql.FormattingLiteral("(select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest("), + ) + } case pgsql.Parameter: if builder.MaterializeParameters { @@ -619,6 +685,7 @@ func Expression(expression pgsql.SyntaxNode, builder *OutputBuilder) (string, er return builder.Build(), nil } +// formatSelect writes a SELECT expression with its projection, sources, predicates, grouping, and ordering. func formatSelect(builder *OutputBuilder, selectStmt pgsql.Select) error { builder.Write("select ") @@ -663,6 +730,7 @@ func formatSelect(builder *OutputBuilder, selectStmt pgsql.Select) error { return nil } +// formatGroupBy writes comma-separated GROUP BY expressions when grouping is present. func formatGroupBy(builder *OutputBuilder, groupByExpressions []pgsql.Expression) error { for idx, groupByExpression := range groupByExpressions { if idx > 0 { @@ -677,6 +745,7 @@ func formatGroupBy(builder *OutputBuilder, groupByExpressions []pgsql.Expression return nil } +// formatFromClauses writes comma-separated FROM sources and their joins. func formatFromClauses(builder *OutputBuilder, fromClauses []pgsql.FromClause) error { for idx, fromClause := range fromClauses { if idx > 0 { @@ -726,6 +795,7 @@ func formatFromClauses(builder *OutputBuilder, fromClauses []pgsql.FromClause) e return nil } +// formatTableAlias writes an alias and its optional record-shape column list. func formatTableAlias(builder *OutputBuilder, tableAlias pgsql.TableAlias) error { builder.Write(tableAlias.Name) @@ -748,6 +818,7 @@ func formatTableAlias(builder *OutputBuilder, tableAlias pgsql.TableAlias) error return nil } +// formatCommonTableExpressions writes a WITH clause and each materialization-qualified CTE. func formatCommonTableExpressions(builder *OutputBuilder, commonTableExpressions pgsql.With) error { // Only write "with" if there are actually expressions if len(commonTableExpressions.Expressions) == 0 { @@ -794,6 +865,7 @@ func formatCommonTableExpressions(builder *OutputBuilder, commonTableExpressions return nil } +// formatSetExpression dispatches rendering for SELECT, nested query, values, and set-operation operands. func formatSetExpression(builder *OutputBuilder, expression pgsql.SetExpression) error { switch typedSetExpression := expression.(type) { case pgsql.Query: @@ -850,7 +922,7 @@ func formatSetExpression(builder *OutputBuilder, expression pgsql.SetExpression) return fmt.Errorf("set operation for query may not be both ALL and DISTINCT") } - if err := formatSetExpression(builder, typedSetExpression.LOperand); err != nil { + if err := formatSetOperationOperand(builder, typedSetExpression.LOperand); err != nil { return err } @@ -870,7 +942,7 @@ func formatSetExpression(builder *OutputBuilder, expression pgsql.SetExpression) builder.Write("distinct ") } - if err := formatSetExpression(builder, typedSetExpression.ROperand); err != nil { + if err := formatSetOperationOperand(builder, typedSetExpression.ROperand); err != nil { return err } @@ -890,6 +962,20 @@ func formatSetExpression(builder *OutputBuilder, expression pgsql.SetExpression) return nil } +// formatSetOperationOperand parenthesizes query operands so their WITH, ORDER BY, and limits remain scoped to the operand. +func formatSetOperationOperand(builder *OutputBuilder, operand pgsql.SetExpression) error { + if _, isQuery := operand.(pgsql.Query); !isQuery { + return formatSetExpression(builder, operand) + } + builder.Write("(") + if err := formatSetExpression(builder, operand); err != nil { + return err + } + builder.Write(")") + return nil +} + +// formatMergeStatement writes a MERGE statement with matched and unmatched actions. func formatMergeStatement(builder *OutputBuilder, merge pgsql.Merge) error { builder.Write("merge ") @@ -999,6 +1085,7 @@ func formatMergeStatement(builder *OutputBuilder, merge pgsql.Merge) error { return nil } +// formatInsertStatement writes an INSERT source, conflict action, and optional RETURNING projection. func formatInsertStatement(builder *OutputBuilder, insert pgsql.Insert) error { builder.Write("insert into ") @@ -1061,6 +1148,7 @@ func formatInsertStatement(builder *OutputBuilder, insert pgsql.Insert) error { return nil } +// formatUpdateStatement writes an UPDATE target, assignments, sources, predicate, and optional RETURNING projection. func formatUpdateStatement(builder *OutputBuilder, update pgsql.Update) error { builder.Write("update ") @@ -1113,6 +1201,7 @@ func formatUpdateStatement(builder *OutputBuilder, update pgsql.Update) error { return nil } +// formatDeleteStatement writes a DELETE target, USING sources, predicate, and optional RETURNING projection. func formatDeleteStatement(builder *OutputBuilder, sqlDelete pgsql.Delete) error { builder.Write("delete from ") diff --git a/cypher/models/pgsql/format/format_test.go b/cypher/models/pgsql/format/format_test.go index 86b9629d..80478cfe 100644 --- a/cypher/models/pgsql/format/format_test.go +++ b/cypher/models/pgsql/format/format_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/require" ) +// mustAsLiteral converts value to a PostgreSQL literal and panics if the value type is unsupported. func mustAsLiteral(value any) pgsql.Literal { if literal, err := pgsql.AsLiteral(value); err != nil { panic(fmt.Sprintf("%v", err)) @@ -26,6 +27,17 @@ func TestFormat_TypeCastedParenthetical(t *testing.T) { require.Equal(t, "('str')::text", formattedQuery) } +// TestFormat_QuotesExpressionShapedIdentifiers verifies that identifier text resembling an expression remains an identifier. +func TestFormat_QuotesExpressionShapedIdentifiers(t *testing.T) { + formatted, err := format.Expression( + pgsql.CompoundIdentifier{"s0", "id(n)"}, + format.NewOutputBuilder(), + ) + + require.NoError(t, err) + require.Equal(t, `s0."id(n)"`, formatted) +} + func TestFormat_Case(t *testing.T) { formattedQuery, err := format.Expression(pgsql.Case{ Conditions: []pgsql.Expression{ @@ -118,6 +130,27 @@ func TestFormat_LateralSubqueryJoin(t *testing.T) { require.Equal(t, "select n.id, e.id from node n join lateral (select e.id from edge e where e.start_id = n.id offset 0) e on true;", formattedQuery) } +// TestFormat_FunctionAggregateOrderBy verifies that aggregate input ordering renders inside the function call. +func TestFormat_FunctionAggregateOrderBy(t *testing.T) { + formattedQuery, err := format.Statement(pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{ + pgsql.FunctionCall{ + Function: pgsql.FunctionArrayAggregate, + Parameters: []pgsql.Expression{pgsql.CompoundIdentifier{"edge", "id"}}, + OrderBy: []*pgsql.OrderBy{{ + Expression: pgsql.Identifier("ordinality"), + Ascending: true, + }}, + }, + }, + }, + }, format.NewOutputBuilder()) + + require.NoError(t, err) + require.Equal(t, "select array_agg(edge.id order by ordinality);", formattedQuery) +} + func TestFormat_Delete(t *testing.T) { formattedQuery, err := format.Statement(pgsql.Delete{ From: []pgsql.TableReference{{ @@ -664,6 +697,44 @@ func TestFormat_CTEs(t *testing.T) { require.Equal(t, "with recursive expansion_1(root_id, next_id, depth, stop, is_cycle, path) as materialized (select r.start_id, r.end_id, 1, false, r.start_id = r.end_id, array [r.id] from edge r join node a on a.id = r.start_id where a.kind_ids operator (pg_catalog.&&) array [23]::int2[] union all select expansion_1.root_id, r.end_id, expansion_1.depth + 1, b.kind_ids operator (pg_catalog.&&) array [24]::int2[], r.id = any(expansion_1.path), expansion_1.path || r.id from expansion_1 join edge r on r.start_id = expansion_1.next_id join node b on b.id = r.end_id where not expansion_1.is_cycle and not expansion_1.stop) select a.properties, b.properties from expansion_1 join node a on a.id = expansion_1.root_id join node b on b.id = expansion_1.next_id where not expansion_1.is_cycle and expansion_1.stop;", formattedQuery) } +// TestFormat_SetOperationParenthesizesQueryOperand verifies that a query operand retains its WITH clause under a set operation. +func TestFormat_SetOperationParenthesizesQueryOperand(t *testing.T) { + formattedQuery, err := format.Statement(pgsql.Query{ + Body: pgsql.SetOperation{ + Operator: pgsql.OperatorUnion, + All: true, + LOperand: pgsql.Select{ + Projection: pgsql.Projection{mustAsLiteral(1)}, + }, + ROperand: pgsql.Query{ + CommonTableExpressions: &pgsql.With{ + Expressions: []pgsql.CommonTableExpression{{ + Alias: pgsql.TableAlias{ + Name: "value", + }, + Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{mustAsLiteral(2)}, + }, + }, + }}, + }, + Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.Wildcard{}}, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{ + Name: pgsql.CompoundIdentifier{"value"}, + }, + }}, + }, + }, + }, + }, format.NewOutputBuilder()) + + require.NoError(t, err) + require.Equal(t, "select 1 union all (with value as (select 2) select * from value);", formattedQuery) +} + func TestFormat_QueryInjection(t *testing.T) { query := pgsql.Query{ Body: pgsql.Select{ diff --git a/cypher/models/pgsql/functions.go b/cypher/models/pgsql/functions.go index 3d0b5f4e..75333023 100644 --- a/cypher/models/pgsql/functions.go +++ b/cypher/models/pgsql/functions.go @@ -1,55 +1,176 @@ package pgsql const ( - FunctionUnidirectionalASPHarness Identifier = "unidirectional_asp_harness" - FunctionUnidirectionalSPHarness Identifier = "unidirectional_sp_harness" - FunctionBidirectionalASPHarness Identifier = "bidirectional_asp_harness" - FunctionBidirectionalSPHarness Identifier = "bidirectional_sp_harness" + // FunctionUnidirectionalASPHarness identifies the SQL harness for unidirectional all-shortest-path search. + FunctionUnidirectionalASPHarness Identifier = "unidirectional_asp_harness" + + // FunctionUnidirectionalSPHarness identifies the SQL harness for unidirectional single-shortest-path search. + FunctionUnidirectionalSPHarness Identifier = "unidirectional_sp_harness" + + // FunctionBidirectionalASPHarness identifies the SQL harness for bidirectional all-shortest-path search. + FunctionBidirectionalASPHarness Identifier = "bidirectional_asp_harness" + + // FunctionBidirectionalSPHarness identifies the SQL harness for bidirectional single-shortest-path search. + FunctionBidirectionalSPHarness Identifier = "bidirectional_sp_harness" + + // FunctionAllShortestPathsDAG identifies the SQL helper that materializes every shortest path from a predecessor DAG. + FunctionAllShortestPathsDAG Identifier = "all_shortest_paths_dag" + + // FunctionShortestPathCompact identifies the SQL helper that materializes one compact shortest-path witness. + FunctionShortestPathCompact Identifier = "shortest_path_compact" + + // FunctionShortestPathB1StrictAlternating identifies compact bidirectional search with strict node alternation. + FunctionShortestPathB1StrictAlternating Identifier = "shortest_path_b1_strict_alternating" + + // FunctionShortestPathB2SmallerCurrentLevel identifies compact bidirectional search that expands the smaller current level. + FunctionShortestPathB2SmallerCurrentLevel Identifier = "shortest_path_b2_smaller_current_level" + + // FunctionAllShortestPathsB1StrictAlternating identifies two-sided predecessor-DAG enumeration with strict node alternation. + FunctionAllShortestPathsB1StrictAlternating Identifier = "all_shortest_paths_b1_strict_alternating" + + // FunctionAllShortestPathsB2SmallerCurrentLevel identifies two-sided predecessor-DAG enumeration that expands the smaller current level. + FunctionAllShortestPathsB2SmallerCurrentLevel Identifier = "all_shortest_paths_b2_smaller_current_level" + + // FunctionShortestPathSelfEndpointError identifies the SQL helper that raises an invalid self-endpoint error. FunctionShortestPathSelfEndpointError Identifier = "shortest_path_self_endpoint_error" - FunctionIntArrayUnique Identifier = "uniq" - FunctionIntArraySort Identifier = "sort" - FunctionJSONBToTextArray Identifier = "jsonb_to_text_array" - FunctionJSONBArrayElementsText Identifier = "jsonb_array_elements_text" - FunctionJSONBBuildObject Identifier = "jsonb_build_object" - FunctionJSONBArrayLength Identifier = "jsonb_array_length" - FunctionJSONBTypeof Identifier = "jsonb_typeof" - FunctionToJSONB Identifier = "to_jsonb" - FunctionCypherContains Identifier = "cypher_contains" - FunctionCypherStartsWith Identifier = "cypher_starts_with" - FunctionCypherEndsWith Identifier = "cypher_ends_with" - FunctionCypherMin Identifier = "cypher_min" - FunctionCypherMax Identifier = "cypher_max" - FunctionArrayLength Identifier = "array_length" - FunctionCardinality Identifier = "cardinality" - FunctionArrayAggregate Identifier = "array_agg" - FunctionArrayRemove Identifier = "array_remove" - FunctionMin Identifier = "min" - FunctionMax Identifier = "max" - FunctionSum Identifier = "sum" - FunctionAvg Identifier = "avg" - FunctionLocalTimestamp Identifier = "localtimestamp" - FunctionLocalTime Identifier = "localtime" - FunctionCurrentTime Identifier = "current_time" - FunctionCurrentDate Identifier = "current_date" - FunctionNow Identifier = "now" - FunctionToLower Identifier = "lower" - FunctionToUpper Identifier = "upper" - FunctionCoalesce Identifier = "coalesce" - FunctionReplace Identifier = "replace" - FunctionUnnest Identifier = "unnest" - FunctionNextValue Identifier = "nextval" - FunctionPGGetSerialSequence Identifier = "pg_get_serial_sequence" - FunctionJSONBSet Identifier = "jsonb_set" - FunctionCount Identifier = "count" - FunctionStringToArray Identifier = "string_to_array" - FunctionEdgesToPath Identifier = "edges_to_path" - FunctionOrderedEdgesToPath Identifier = "ordered_edges_to_path" - FunctionNodesToPath Identifier = "nodes_to_path" - FunctionKindName Identifier = "kind_name" - FunctionStartNode Identifier = "start_node" - FunctionEndNode Identifier = "end_node" - FunctionExtract Identifier = "extract" - FunctionGenerateSubscripts Identifier = "generate_subscripts" + + // FunctionIntArrayUnique identifies the SQL helper that removes duplicate integer-array values. + FunctionIntArrayUnique Identifier = "uniq" + + // FunctionIntArraySort identifies the SQL helper that orders integer-array values. + FunctionIntArraySort Identifier = "sort" + + // FunctionJSONBToTextArray identifies the SQL helper that converts a JSONB array to text[]. + FunctionJSONBToTextArray Identifier = "jsonb_to_text_array" + + // FunctionJSONBArrayElementsText identifies PostgreSQL's JSONB array-element text expansion function. + FunctionJSONBArrayElementsText Identifier = "jsonb_array_elements_text" + + // FunctionJSONBBuildObject identifies PostgreSQL's JSONB object constructor. + FunctionJSONBBuildObject Identifier = "jsonb_build_object" + + // FunctionJSONBArrayLength identifies PostgreSQL's JSONB array-length function. + FunctionJSONBArrayLength Identifier = "jsonb_array_length" + + // FunctionJSONBTypeof identifies PostgreSQL's JSONB type-inspection function. + FunctionJSONBTypeof Identifier = "jsonb_typeof" + + // FunctionToJSONB identifies PostgreSQL's conversion to JSONB. + FunctionToJSONB Identifier = "to_jsonb" + + // FunctionCypherContains identifies the SQL helper implementing Cypher CONTAINS semantics. + FunctionCypherContains Identifier = "cypher_contains" + + // FunctionCypherStartsWith identifies the SQL helper implementing Cypher STARTS WITH semantics. + FunctionCypherStartsWith Identifier = "cypher_starts_with" + + // FunctionCypherEndsWith identifies the SQL helper implementing Cypher ENDS WITH semantics. + FunctionCypherEndsWith Identifier = "cypher_ends_with" + + // FunctionCypherMin identifies the SQL aggregate implementing Cypher minimum semantics. + FunctionCypherMin Identifier = "cypher_min" + + // FunctionCypherMax identifies the SQL aggregate implementing Cypher maximum semantics. + FunctionCypherMax Identifier = "cypher_max" + + // FunctionArrayLength identifies PostgreSQL's dimension-aware array-length function. + FunctionArrayLength Identifier = "array_length" + + // FunctionCardinality identifies PostgreSQL's total array-element count function. + FunctionCardinality Identifier = "cardinality" + + // FunctionArrayAggregate identifies PostgreSQL's array aggregation function. + FunctionArrayAggregate Identifier = "array_agg" + + // FunctionArrayRemove identifies PostgreSQL's array element-removal function. + FunctionArrayRemove Identifier = "array_remove" + + // FunctionMin identifies PostgreSQL's minimum aggregate. + FunctionMin Identifier = "min" + + // FunctionMax identifies PostgreSQL's maximum aggregate. + FunctionMax Identifier = "max" + + // FunctionSum identifies PostgreSQL's sum aggregate. + FunctionSum Identifier = "sum" + + // FunctionAvg identifies PostgreSQL's average aggregate. + FunctionAvg Identifier = "avg" + + // FunctionLocalTimestamp identifies PostgreSQL's local timestamp constructor. + FunctionLocalTimestamp Identifier = "localtimestamp" + + // FunctionLocalTime identifies PostgreSQL's local time constructor. + FunctionLocalTime Identifier = "localtime" + + // FunctionCurrentTime identifies PostgreSQL's current zoned time value. + FunctionCurrentTime Identifier = "current_time" + + // FunctionCurrentDate identifies PostgreSQL's current date value. + FunctionCurrentDate Identifier = "current_date" + + // FunctionNow identifies PostgreSQL's current transaction timestamp function. + FunctionNow Identifier = "now" + + // FunctionToLower identifies PostgreSQL's lowercase text function. + FunctionToLower Identifier = "lower" + + // FunctionToUpper identifies PostgreSQL's uppercase text function. + FunctionToUpper Identifier = "upper" + + // FunctionCoalesce identifies PostgreSQL's first-non-null expression. + FunctionCoalesce Identifier = "coalesce" + + // FunctionNullIf identifies PostgreSQL's NULLIF function for nulling matching scalar values. + FunctionNullIf Identifier = "nullif" + + // FunctionReplace identifies PostgreSQL's substring-replacement function. + FunctionReplace Identifier = "replace" + + // FunctionUnnest identifies PostgreSQL's array-to-row expansion function. + FunctionUnnest Identifier = "unnest" + + // FunctionNextValue identifies PostgreSQL's sequence increment function. + FunctionNextValue Identifier = "nextval" + + // FunctionPGGetSerialSequence identifies PostgreSQL's serial-sequence lookup function. + FunctionPGGetSerialSequence Identifier = "pg_get_serial_sequence" + + // FunctionJSONBSet identifies PostgreSQL's JSONB path-update function. + FunctionJSONBSet Identifier = "jsonb_set" + + // FunctionCount identifies PostgreSQL's count aggregate. + FunctionCount Identifier = "count" + + // FunctionStringToArray identifies PostgreSQL's delimiter-based text-to-array function. + FunctionStringToArray Identifier = "string_to_array" + + // FunctionEdgesToPath identifies the SQL helper that builds a path from unordered edge composites. + FunctionEdgesToPath Identifier = "edges_to_path" + + // FunctionOrderedEdgesToPath identifies the SQL helper that builds a path from ordered edge composites. + FunctionOrderedEdgesToPath Identifier = "ordered_edges_to_path" + + // FunctionOrderedEdgeIDsToPath identifies the SQL helper that hydrates an ordered edge-ID array into a path. + FunctionOrderedEdgeIDsToPath Identifier = "ordered_edge_ids_to_path" + + // FunctionNodesToPath identifies the SQL helper that builds a path from ordered node composites. + FunctionNodesToPath Identifier = "nodes_to_path" + + // FunctionKindName identifies the SQL helper that resolves a kind ID to its name. + FunctionKindName Identifier = "kind_name" + + // FunctionStartNode identifies the SQL helper that hydrates a relationship's start node. + FunctionStartNode Identifier = "start_node" + + // FunctionEndNode identifies the SQL helper that hydrates a relationship's end node. + FunctionEndNode Identifier = "end_node" + + // FunctionExtract identifies PostgreSQL's temporal component-extraction function. + FunctionExtract Identifier = "extract" + + // FunctionGenerateSubscripts identifies PostgreSQL's array-index generation function. + FunctionGenerateSubscripts Identifier = "generate_subscripts" ) func IsAggregateFunction(function Identifier) bool { diff --git a/cypher/models/pgsql/model.go b/cypher/models/pgsql/model.go index 5c7096f4..cddcefd8 100644 --- a/cypher/models/pgsql/model.go +++ b/cypher/models/pgsql/model.go @@ -404,8 +404,12 @@ func (s *Parenthetical) AsExpression() Expression { return s } +// EdgeArrayFromPathIDs hydrates edge composites from a path's ordered edge identifiers. type EdgeArrayFromPathIDs struct { + // PathIDs is the ordered edge-ID array to hydrate. PathIDs Expression + // GraphID identifies the graph whose edge IDs are hydrated into edge composites. + GraphID Expression } func (s *EdgeArrayFromPathIDs) NodeType() string { @@ -419,9 +423,16 @@ func (s *EdgeArrayFromPathIDs) AsExpression() Expression { type JoinType int const ( + // JoinTypeInner retains rows that satisfy the join constraint on both sides. JoinTypeInner JoinType = iota + + // JoinTypeLeftOuter retains every left row even when no right row matches. JoinTypeLeftOuter + + // JoinTypeRightOuter retains every right row even when no left row matches. JoinTypeRightOuter + + // JoinTypeFullOuter retains unmatched rows from both sides. JoinTypeFullOuter ) @@ -456,16 +467,26 @@ func (s OrderBy) NodeType() string { type WindowFrameUnit int const ( + // WindowFrameUnitRows measures frame boundaries in physical rows. WindowFrameUnitRows WindowFrameUnit = iota + + // WindowFrameUnitRange measures frame boundaries by ordering-key value ranges. WindowFrameUnitRange + + // WindowFrameUnitGroups measures frame boundaries in peer groups. WindowFrameUnitGroups ) type WindowFrameBoundaryType int const ( + // WindowFrameBoundaryTypeCurrentRow anchors a window boundary at the current row or peer group. WindowFrameBoundaryTypeCurrentRow WindowFrameBoundaryType = iota + + // WindowFrameBoundaryTypePreceding places a window boundary before the current row. WindowFrameBoundaryTypePreceding + + // WindowFrameBoundaryTypeFollowing places a window boundary after the current row. WindowFrameBoundaryTypeFollowing ) @@ -568,13 +589,22 @@ func AsParameter(identifier Identifier, value any) (*Parameter, error) { return parameter, nil } +// FunctionCall represents a PostgreSQL function invocation and its aggregate or window options. type FunctionCall struct { - Bare bool - Distinct bool - Function Identifier + // Bare omits the usual argument parentheses for SQL keyword-like functions. + Bare bool + // Distinct deduplicates argument rows before aggregate evaluation. + Distinct bool + // Function identifies the PostgreSQL function to invoke. + Function Identifier + // Parameters contains the function arguments in call order. Parameters []Expression - Over *Window - CastType DataType + // OrderBy orders aggregate inputs before the function is evaluated. + OrderBy []*OrderBy + // Over supplies the window specification for a window-function call. + Over *Window + // CastType records the function result type known to the translator. + CastType DataType } func (s FunctionCall) AsAssignment() Assignment { diff --git a/cypher/models/pgsql/optimize/analysis_test.go b/cypher/models/pgsql/optimize/analysis_test.go index e55dfab7..61fa0474 100644 --- a/cypher/models/pgsql/optimize/analysis_test.go +++ b/cypher/models/pgsql/optimize/analysis_test.go @@ -9,18 +9,20 @@ import ( "github.com/stretchr/testify/require" ) -const adcsQuery = ` -MATCH (n:Group) -WHERE n.objectid = 'S-1-5-21-2643190041-1319121918-239771340-513' -MATCH p1 = (n)-[:MemberOf*0..]->()-[:Enroll]->(ca:EnterpriseCA)-[:TrustedForNTAuth]->(:NTAuthStore)-[:NTAuthStoreFor]->(d:Domain) -MATCH p2 = (n)-[:MemberOf*0..]->()-[:GenericAll|Enroll|AllExtendedRights]->(ct:CertTemplate)-[:PublishedTo]->(ca)-[:IssuedSignedBy|EnterpriseCAFor*1..]->(:RootCA)-[:RootCAFor]->(d) -WHERE ct.authenticationenabled = true -AND ct.requiresmanagerapproval = false -AND ct.enrolleesuppliessubject = true -AND (ct.schemaversion = 1 OR ct.authorizedsignatures = 0) +// fixedSuffixExpansionQuery exercises one variable expansion followed by a three-edge typed suffix. +const fixedSuffixExpansionQuery = ` +MATCH (root:ExpansionRoot) +WHERE root.root_key = 'root' +MATCH p1 = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) +MATCH p2 = (root)-[:Expand*0..16]->()-[:OptionA|OptionB|OptionC]->(predicate:PredicateNode)-[:JoinSuffix]->(head)-[:HeadToBridge|HeadToAlternateBridge*1..16]->(:BridgeNode)-[:ReachTerminal]->(terminal) +WHERE predicate.eligible = true +AND predicate.requires_review = false +AND predicate.allows_direct = true +AND (predicate.version = 1 OR predicate.required_approvals = 0) RETURN p1, p2 ` +// analyzeCypher parses query, runs optimizer analysis, and requires both stages to succeed. func analyzeCypher(t *testing.T, query string) Analysis { t.Helper() @@ -30,6 +32,7 @@ func analyzeCypher(t *testing.T, query string) Analysis { return Analyze(regularQuery) } +// requireBinding requires an analyzed binding with the expected symbol and kind. func requireBinding(t *testing.T, bindings []Binding, symbol string, kind BindingKind) { t.Helper() @@ -42,6 +45,7 @@ func requireBinding(t *testing.T, bindings []Binding, symbol string, kind Bindin t.Fatalf("expected binding %s:%s in %#v", symbol, kind, bindings) } +// requirePathVariable requires a path variable with the expected relationship count and range shape. func requirePathVariable(t *testing.T, pathVariables []PathVariable, symbol string, relationshipCount int, expectedVariableLength bool) { t.Helper() @@ -56,10 +60,11 @@ func requirePathVariable(t *testing.T, pathVariables []PathVariable, symbol stri t.Fatalf("expected path variable %s in %#v", symbol, pathVariables) } -func TestAnalyzeIdentifiesEligibleADCSRegion(t *testing.T) { +// TestAnalyzeIdentifiesEligibleFixedSuffixExpansionRegion verifies that analysis isolates the variable expansion and its fixed suffix. +func TestAnalyzeIdentifiesEligibleFixedSuffixExpansionRegion(t *testing.T) { t.Parallel() - analysis := analyzeCypher(t, adcsQuery) + analysis := analyzeCypher(t, fixedSuffixExpansionQuery) require.Len(t, analysis.QueryParts, 1) @@ -77,13 +82,13 @@ func TestAnalyzeIdentifiesEligibleADCSRegion(t *testing.T) { require.Len(t, region.Clauses, 3) require.Len(t, region.BindingOccurrences, 10) require.Len(t, region.Predicates, 2) - require.Equal(t, []string{"n"}, region.Predicates[0].Dependencies) - require.Equal(t, []string{"ct"}, region.Predicates[1].Dependencies) + require.Equal(t, []string{"root"}, region.Predicates[0].Dependencies) + require.Equal(t, []string{"predicate"}, region.Predicates[1].Dependencies) - requireBinding(t, region.Bindings, "n", BindingKindNode) - requireBinding(t, region.Bindings, "ca", BindingKindNode) - requireBinding(t, region.Bindings, "ct", BindingKindNode) - requireBinding(t, region.Bindings, "d", BindingKindNode) + requireBinding(t, region.Bindings, "root", BindingKindNode) + requireBinding(t, region.Bindings, "head", BindingKindNode) + requireBinding(t, region.Bindings, "predicate", BindingKindNode) + requireBinding(t, region.Bindings, "terminal", BindingKindNode) requireBinding(t, region.Bindings, "p1", BindingKindPath) requireBinding(t, region.Bindings, "p2", BindingKindPath) @@ -132,18 +137,19 @@ func TestAnalyzeSegmentsRegionsAtSemanticBarriers(t *testing.T) { require.Equal(t, []string{"m"}, secondPart.ProjectionDependencies) } +// TestAnalysisDiagnosticsAreStable verifies that diagnostic ordering and query coordinates remain deterministic. func TestAnalysisDiagnosticsAreStable(t *testing.T) { t.Parallel() var ( - analysis = analyzeCypher(t, adcsQuery) + analysis = analyzeCypher(t, fixedSuffixExpansionQuery) diagnostics = strings.Join(analysis.Diagnostics(), "\n") ) require.Contains(t, diagnostics, "query_part[0] kind=single projection_deps=p1,p2") require.Contains(t, diagnostics, "region[0] part=0 clauses=0..2 matches=3") - require.Contains(t, diagnostics, "bindings=n:node,p1:path,ca:node,d:node,p2:path,ct:node") + require.Contains(t, diagnostics, "bindings=root:node,p1:path,head:node,terminal:node,p2:path,predicate:node") require.Contains(t, diagnostics, "paths=p1,p2") - require.Contains(t, diagnostics, "predicates=n,ct") + require.Contains(t, diagnostics, "predicates=root,predicate") require.Contains(t, diagnostics, "barrier[0] part=0 clause=3 kind=return deps=p1,p2") } diff --git a/cypher/models/pgsql/optimize/expansion_orientation.go b/cypher/models/pgsql/optimize/expansion_orientation.go new file mode 100644 index 00000000..bef7a4e1 --- /dev/null +++ b/cypher/models/pgsql/optimize/expansion_orientation.go @@ -0,0 +1,98 @@ +package optimize + +// contiguousExpansionOrientationCandidate contains the common typed metadata +// for one variable expansion and an adjacent fixed seed region. Prefix- and +// suffix-specific analyzers retain their own correctness facts and fallback +// reasons, then use this type to produce a consistent public decision. +type contiguousExpansionOrientationCandidate struct { + Target TraversalStepTarget + Family string + PlannedPolicy ExpansionSearchPolicy + EmittedPolicy ExpansionSearchPolicy + PlannedCandidates []ExpansionSearchStrategy + EmittedCandidates []ExpansionSearchStrategy + CandidateStrategy ExpansionSearchStrategy + ProbeCaps ExpansionSearchProbeCaps + Admission ExpansionSearchAdmission + PrefixStartStep int + PrefixEndStep int + PrefixLength int + SuffixStartStep int + SuffixEndStep int + SuffixLength int + SeedPredicateClass string + EndpointLimit int64 +} + +// contiguousExpansionOrientationQualification contains analysis results that +// remain specific to the fixed-prefix or fixed-suffix correctness envelope. +type contiguousExpansionOrientationQualification struct { + SelectedStrategy ExpansionSearchStrategy + StructurallyEligible bool + StaticallyEligible bool + EligibilityFacts []ExpansionSearchEligibilityFact + HasFinalLimit bool + ObservationMode ExpansionSearchObservationMode + LogicalDirection string + MinimumDepth int64 + MaximumDepth int64 + SelectionMode string + SelectorVersion string + FallbackReason string +} + +// decision combines common orientation metadata with family-specific +// qualification without conflating a planned policy with emitted SQL. +func (s contiguousExpansionOrientationCandidate) decision(qualification contiguousExpansionOrientationQualification) ExpansionSearchStrategyDecision { + return ExpansionSearchStrategyDecision{ + Target: s.Target, + Family: s.Family, + PlannedPolicy: s.PlannedPolicy, + EmittedPolicy: s.EmittedPolicy, + PlannedCandidates: s.PlannedCandidates, + EmittedCandidates: s.EmittedCandidates, + CandidateStrategy: s.CandidateStrategy, + SelectedStrategy: qualification.SelectedStrategy, + StructurallyEligible: qualification.StructurallyEligible, + StaticallyEligible: qualification.StaticallyEligible, + EligibilityFacts: qualification.EligibilityFacts, + ProbeCaps: s.ProbeCaps, + Admission: s.Admission, + SuffixStartStep: s.SuffixStartStep, + SuffixEndStep: s.SuffixEndStep, + SuffixLength: s.SuffixLength, + PrefixStartStep: s.PrefixStartStep, + PrefixEndStep: s.PrefixEndStep, + PrefixLength: s.PrefixLength, + SeedPredicateClass: s.SeedPredicateClass, + EndpointLimit: s.EndpointLimit, + StateLimit: s.Admission.StateLimit, + HasFinalLimit: qualification.HasFinalLimit, + ObservationMode: qualification.ObservationMode, + LogicalDirection: qualification.LogicalDirection, + MinimumDepth: qualification.MinimumDepth, + MaximumDepth: qualification.MaximumDepth, + SelectionMode: qualification.SelectionMode, + SelectorVersion: qualification.SelectorVersion, + FallbackStrategy: s.Admission.FallbackStrategy, + FallbackReason: qualification.FallbackReason, + } +} + +// setExpansionSearchExpectedEmission keeps compile-time emission metadata in +// sync after statement-wide safety and observation checks change selection. +// It describes statement shape only; execution telemetry records the arm that +// actually ran. +func setExpansionSearchExpectedEmission(decision *ExpansionSearchStrategyDecision) { + decision.EmittedPolicy = "" + decision.EmittedCandidates = []ExpansionSearchStrategy{decision.SelectedStrategy} + decision.ExecutionBoundary = ExpansionSearchExecutionBoundaryInlineStatement + if decision.SelectedStrategy == ExpansionSearchEndpointSeededReverse && decision.StructurallyEligible { + decision.EmittedPolicy = ExpansionSearchPolicyEndpointGuardV1 + decision.EmittedCandidates = []ExpansionSearchStrategy{ + ExpansionSearchStepwiseForward, + ExpansionSearchEndpointSeededReverse, + } + decision.ExecutionBoundary = ExpansionSearchExecutionBoundaryGuardedDualArm + } +} diff --git a/cypher/models/pgsql/optimize/lowering.go b/cypher/models/pgsql/optimize/lowering.go index 20b3b2dc..6b7bbb6c 100644 --- a/cypher/models/pgsql/optimize/lowering.go +++ b/cypher/models/pgsql/optimize/lowering.go @@ -6,20 +6,62 @@ import ( ) const ( - LoweringProjectionPruning = "ProjectionPruning" - LoweringLatePathMaterialization = "LatePathMaterialization" - LoweringExpandIntoDetection = "ExpandIntoDetection" - LoweringTraversalDirection = "TraversalDirectionSelection" - LoweringShortestPathStrategy = "ShortestPathStrategySelection" - LoweringShortestPathFilter = "ShortestPathFilterMaterialization" - LoweringLimitPushdown = "LimitPushdown" - LoweringExpansionSuffixPushdown = "ExpansionSuffixPushdown" - LoweringPredicatePlacement = "PredicatePlacement" - LoweringCountStoreFastPath = "CountStoreFastPath" - LoweringCollectIDMembership = "CollectIDMembership" - LoweringAggregateTraversalCount = "AggregateTraversalCount" - LoweringExactRangeExpansion = "ExactRangeExpansion" + // LoweringProjectionPruning identifies removal of traversal fields that downstream clauses do not consume. + LoweringProjectionPruning = "ProjectionPruning" + + // LoweringLatePathMaterialization identifies deferral of path hydration until a consumer requires it. + LoweringLatePathMaterialization = "LatePathMaterialization" + + // LoweringExpandIntoDetection identifies traversal steps whose two endpoints are already bound. + LoweringExpandIntoDetection = "ExpandIntoDetection" + + // LoweringTraversalDirection identifies selection of the lower-cost logical traversal direction. + LoweringTraversalDirection = "TraversalDirectionSelection" + + // LoweringShortestPathStrategy identifies unidirectional or bidirectional shortest-path selection. + LoweringShortestPathStrategy = "ShortestPathStrategySelection" + + // LoweringShortestPathFilter identifies materialization of reusable shortest-path endpoint filters. + LoweringShortestPathFilter = "ShortestPathFilterMaterialization" + + // LoweringLimitPushdown identifies limits moved into a traversal or shortest-path harness. + LoweringLimitPushdown = "LimitPushdown" + + // LoweringExpansionSuffixPushdown identifies fixed-suffix predicates moved closer to variable expansion. + LoweringExpansionSuffixPushdown = "ExpansionSuffixPushdown" + + // LoweringPredicatePlacement identifies attachment of predicates to the earliest safe traversal step. + LoweringPredicatePlacement = "PredicatePlacement" + + // LoweringCountStoreFastPath identifies count queries satisfied directly from graph statistics. + LoweringCountStoreFastPath = "CountStoreFastPath" + + // LoweringCollectIDMembership identifies membership checks rewritten over collected scalar IDs. + LoweringCollectIDMembership = "CollectIDMembership" + + // LoweringAggregateTraversalCount identifies traversal counts lowered without materializing result rows. + LoweringAggregateTraversalCount = "AggregateTraversalCount" + + // LoweringExactRangeExpansion identifies short exact ranges expanded into fixed traversal steps. + LoweringExactRangeExpansion = "ExactRangeExpansion" + + // LoweringPathRelationshipPredicate identifies relationship quantifiers attached to path state. LoweringPathRelationshipPredicate = "PathRelationshipPredicate" + + // LoweringFieldRequirements identifies analysis that records which representation each binding consumer needs. + LoweringFieldRequirements = "FieldRequirements" + + // LoweringShortestPathExecutor identifies selection of a physical shortest-path executor. + LoweringShortestPathExecutor = "ShortestPathExecutorDecision" + + // LoweringExpansionSearchStrategy identifies selection of a physical variable-expansion search strategy. + LoweringExpansionSearchStrategy = "ExpansionSearchStrategyDecision" + + // LoweringEndpointResolution identifies planned bounded endpoint-resolution analysis. + LoweringEndpointResolution = "EndpointResolutionDecision" + + // LoweringTraversalPredicateClassification identifies planned traversal-predicate locality analysis. + LoweringTraversalPredicateClassification = "TraversalPredicateClassificationDecision" ) type LoweringDecision struct { @@ -72,8 +114,13 @@ type ProjectionPruningDecision struct { type LatePathMaterializationMode string const ( - LatePathMaterializationPathEdgeID LatePathMaterializationMode = "path_edge_id" + // LatePathMaterializationPathEdgeID carries a path as ordered edge IDs until hydration. + LatePathMaterializationPathEdgeID LatePathMaterializationMode = "path_edge_id" + + // LatePathMaterializationExpansionPath carries recursive expansion path state until hydration. LatePathMaterializationExpansionPath LatePathMaterializationMode = "expansion_path" + + // LatePathMaterializationEdgeComposite defers hydration of an edge composite. LatePathMaterializationEdgeComposite LatePathMaterializationMode = "edge_composite" ) @@ -95,7 +142,10 @@ type TraversalDirectionDecision struct { type ShortestPathStrategy string const ( - ShortestPathStrategyBidirectional ShortestPathStrategy = "bidirectional" + // ShortestPathStrategyBidirectional searches simultaneously from both endpoints. + ShortestPathStrategyBidirectional ShortestPathStrategy = "bidirectional" + + // ShortestPathStrategyUnidirectional searches from one endpoint toward the other. ShortestPathStrategyUnidirectional ShortestPathStrategy = "unidirectional" ) @@ -105,10 +155,318 @@ type ShortestPathStrategyDecision struct { Reason string `json:"reason,omitempty"` } +type ShortestPathExecutor string + +const ( + // ShortestPathPolicyASPI1GuardedV1 identifies the bounded inline + // predecessor-DAG candidate with an exact A1 fallback. + ShortestPathPolicyASPI1GuardedV1 = "asp-i1-guarded-v1" + + // ShortestPathPolicyI1CanonicalGuardedV1 identifies the bounded inline + // canonical-witness candidate with an exact compact S4 fallback. + ShortestPathPolicyI1CanonicalGuardedV1 = "sp-i1-canonical-guarded-v1" + + // ShortestPathSelectorStaticV6 identifies the evidence-gated production + // selector for the qualified inbound, typed, single-kind canonical witness + // envelope. The automatic selector remains sp-static-v5-contained until a + // complete production evidence manifest activates this version. + ShortestPathSelectorStaticV6 = "sp-static-v6" + + // ShortestPathExecutorIncumbentWorkspace selects the existing workspace-table executor. + ShortestPathExecutorIncumbentWorkspace ShortestPathExecutor = "SP-S0" + + // ShortestPathExecutorS1ArrayBFS selects breadth-first search with path state held in arrays. + ShortestPathExecutorS1ArrayBFS ShortestPathExecutor = "SP-S1" + + // ShortestPathExecutorS2TraceRelation selects breadth-first search backed by a trace relation. + ShortestPathExecutorS2TraceRelation ShortestPathExecutor = "SP-S2" + + // ShortestPathExecutorS3Unidirectional selects the unidirectional scalar-distance executor. + ShortestPathExecutorS3Unidirectional ShortestPathExecutor = "SP-S3-U-D" + + // ShortestPathExecutorS3EdgeM0 selects unidirectional edge-trail search with deferred path materialization. + ShortestPathExecutorS3EdgeM0 ShortestPathExecutor = "SP-S3-U-E+MAT-M0" + + // ShortestPathExecutorS0Direct selects the direct preflight executor with workspace fallback. + ShortestPathExecutorS0Direct ShortestPathExecutor = "SP-S0-DIRECT" + + // ShortestPathExecutorS4CanonicalDistance selects canonical compact search for distance-only observations. + ShortestPathExecutorS4CanonicalDistance ShortestPathExecutor = "SP-S4-C-D" + + // ShortestPathExecutorS4CanonicalWitness selects canonical compact search with witness materialization. + ShortestPathExecutorS4CanonicalWitness ShortestPathExecutor = "SP-S4-C-WE+MAT-M0" + + // ShortestPathExecutorASPA1DAG selects all-shortest-path enumeration from a predecessor DAG. + ShortestPathExecutorASPA1DAG ShortestPathExecutor = "ASP-A1-DAG" + + // ShortestPathExecutorI1CanonicalDistance selects an inline recursive SQL + // distance search. The distinct identity prevents evidence collected at an + // inline statement boundary from being attributed to a helper function. + ShortestPathExecutorI1CanonicalDistance ShortestPathExecutor = "SP-I1-C-D" + + // ShortestPathExecutorI1CanonicalWitness selects inline recursive SQL with + // ordered edge-ID witness state and late M0 path materialization. + ShortestPathExecutorI1CanonicalWitness ShortestPathExecutor = "SP-I1-U-E+MAT-M0" + + // ShortestPathExecutorI1CanonicalPredecessorWitness selects guarded inline + // minimum-distance/predecessor discovery, one deterministic witness, and an + // exact compact S4 fallback. It is intentionally distinct from the legacy + // unguarded relationship-trail I1 identity above. + ShortestPathExecutorI1CanonicalPredecessorWitness ShortestPathExecutor = "SP-I1-C-WE+MAT-M0" + + // ShortestPathExecutorASPI1DAG selects inline predecessor-DAG discovery and + // late M0 materialization for all shortest paths. + ShortestPathExecutorASPI1DAG ShortestPathExecutor = "ASP-I1-U-DAG+MAT-M0" + + // ShortestPathExecutorB1AlternatingNodeDistance reserves compact bidirectional + // distance search with strict node-at-a-time alternation. + ShortestPathExecutorB1AlternatingNodeDistance ShortestPathExecutor = "SP-B1-C-ALT-NODE-D" + + // ShortestPathExecutorB1AlternatingNodeWitness reserves compact bidirectional + // witness search with strict node-at-a-time alternation and deferred materialization. + ShortestPathExecutorB1AlternatingNodeWitness ShortestPathExecutor = "SP-B1-C-ALT-NODE-WE+MAT-M0" + + // ShortestPathExecutorB2SmallerCurrentLevelDistance reserves compact bidirectional + // distance search that expands the smaller current level. + ShortestPathExecutorB2SmallerCurrentLevelDistance ShortestPathExecutor = "SP-B2-C-MIN-LEVEL-D" + + // ShortestPathExecutorB2SmallerCurrentLevelWitness reserves compact bidirectional + // witness search that expands the smaller current level and defers materialization. + ShortestPathExecutorB2SmallerCurrentLevelWitness ShortestPathExecutor = "SP-B2-C-MIN-LEVEL-WE+MAT-M0" + + // ShortestPathExecutorASPB1AlternatingNodeDAG reserves all-shortest-path DAG + // enumeration with strict node-at-a-time alternation. + ShortestPathExecutorASPB1AlternatingNodeDAG ShortestPathExecutor = "ASP-B1-DAG-ALT-NODE" + + // ShortestPathExecutorASPB2SmallerCurrentLevelDAG reserves all-shortest-path DAG + // enumeration that expands the smaller current level. + ShortestPathExecutorASPB2SmallerCurrentLevelDAG ShortestPathExecutor = "ASP-B2-DAG-MIN-LEVEL" +) + +// ShortestPathScheduler identifies the frontier scheduling policy used by a +// shortest-path executor independently of its result-observation contract. +type ShortestPathScheduler string + +const ( + // ShortestPathSchedulerSingleEndedLevel expands one complete level from a single frontier. + ShortestPathSchedulerSingleEndedLevel ShortestPathScheduler = "single_ended_level" + + // ShortestPathSchedulerStrictAlternatingNode alternates one node expansion from each frontier. + ShortestPathSchedulerStrictAlternatingNode ShortestPathScheduler = "strict_alternating_node" + + // ShortestPathSchedulerSmallerCurrentLevel expands the smaller of the two current frontier levels. + ShortestPathSchedulerSmallerCurrentLevel ShortestPathScheduler = "smaller_current_level" +) + +// Scheduler reports the stable frontier scheduler associated with this executor. +func (s ShortestPathExecutor) Scheduler() ShortestPathScheduler { + switch s { + case ShortestPathExecutorS3Unidirectional, + ShortestPathExecutorS3EdgeM0, + ShortestPathExecutorS4CanonicalDistance, + ShortestPathExecutorS4CanonicalWitness, + ShortestPathExecutorASPA1DAG, + ShortestPathExecutorI1CanonicalDistance, + ShortestPathExecutorI1CanonicalWitness, + ShortestPathExecutorI1CanonicalPredecessorWitness, + ShortestPathExecutorASPI1DAG: + return ShortestPathSchedulerSingleEndedLevel + case ShortestPathExecutorB1AlternatingNodeDistance, + ShortestPathExecutorB1AlternatingNodeWitness, + ShortestPathExecutorASPB1AlternatingNodeDAG: + return ShortestPathSchedulerStrictAlternatingNode + case ShortestPathExecutorB2SmallerCurrentLevelDistance, + ShortestPathExecutorB2SmallerCurrentLevelWitness, + ShortestPathExecutorASPB2SmallerCurrentLevelDAG: + return ShortestPathSchedulerSmallerCurrentLevel + default: + return "" + } +} + +// ExecutionBoundary reports the SQL boundary represented by the executor +// identity. Benchmark and promotion artifacts must match this value. +func (s ShortestPathExecutor) ExecutionBoundary() string { + switch s { + case ShortestPathExecutorS3Unidirectional, + ShortestPathExecutorS3EdgeM0, + ShortestPathExecutorI1CanonicalDistance, + ShortestPathExecutorI1CanonicalWitness, + ShortestPathExecutorI1CanonicalPredecessorWitness, + ShortestPathExecutorASPI1DAG: + return "inline_statement" + default: + return "stored_helper" + } +} + +type ShortestPathObservationMode string + +const ( + // ShortestPathObservationDistance indicates that only shortest-path length is consumed. + ShortestPathObservationDistance ShortestPathObservationMode = "distance" + + // ShortestPathObservationOnePath indicates that one shortest-path witness is consumed. + ShortestPathObservationOnePath ShortestPathObservationMode = "one_path" + + // ShortestPathObservationAllPaths indicates that every shortest-path witness is consumed. + ShortestPathObservationAllPaths ShortestPathObservationMode = "all_paths" + + // ShortestPathObservationUnknown indicates that analysis could not classify downstream path use. + ShortestPathObservationUnknown ShortestPathObservationMode = "unknown" +) + +const ( + // ShortestPathFallbackAllShortestPaths records an all-shortest-path query lacking singleton endpoints required by specialized execution. + ShortestPathFallbackAllShortestPaths = "all_shortest_paths" + + // ShortestPathFallbackCorrelatedEndpoints rejects endpoint sources not proven uncorrelated, such as UNWIND or later query parts. + ShortestPathFallbackCorrelatedEndpoints = "correlated_endpoints" + + // ShortestPathFallbackMultipleEndpointPairs rejects specialized execution when additional row sources prevent proving one endpoint pair. + ShortestPathFallbackMultipleEndpointPairs = "multiple_endpoint_pairs" + + // ShortestPathFallbackNonSingletonID rejects an endpoint whose ID is not statically singleton. + ShortestPathFallbackNonSingletonID = "non_singleton_id" + + // ShortestPathFallbackMultipleIDEqualities rejects an endpoint constrained by competing ID equalities. + ShortestPathFallbackMultipleIDEqualities = "multiple_id_equalities" + + // ShortestPathFallbackPathPredicate rejects a predicate that observes the materialized path. + ShortestPathFallbackPathPredicate = "path_predicate" + + // ShortestPathFallbackRelationshipPredicate rejects a predicate on the traversed relationship. + ShortestPathFallbackRelationshipPredicate = "relationship_predicate" + + // ShortestPathFallbackRelationshipVariable rejects an observed relationship binding. + ShortestPathFallbackRelationshipVariable = "relationship_variable" + + // ShortestPathFallbackDirectionless rejects a directionless shortest-path expansion. + ShortestPathFallbackDirectionless = "directionless" + + // ShortestPathFallbackOptionalMatch rejects shortest-path work under OPTIONAL MATCH semantics. + ShortestPathFallbackOptionalMatch = "optional_match" + + // ShortestPathFallbackUnsupportedDepth rejects a depth range unsupported by the candidate executor. + ShortestPathFallbackUnsupportedDepth = "unsupported_depth" + + // ShortestPathFallbackMutation rejects specialized execution for a statement containing updates. + ShortestPathFallbackMutation = "mutation" + + // ShortestPathFallbackMultiplePathCalls rejects statements containing more than one shortest-path pattern. + ShortestPathFallbackMultiplePathCalls = "multiple_path_calls" + + // ShortestPathFallbackDeepInboundUnqualified rejects an unqualified deep inbound traversal. + ShortestPathFallbackDeepInboundUnqualified = "deep_inbound_unqualified" + + // ShortestPathFallbackNonSingleKindPathState rejects compact path state without one relationship kind. + ShortestPathFallbackNonSingleKindPathState = "non_single_kind_path_state_unqualified" + + // ShortestPathFallbackTournamentUnqualified records that no experimental candidate won qualification. + ShortestPathFallbackTournamentUnqualified = "tournament_unqualified" +) + +type ShortestPathPhysicalExpansion string + +const ( + // ShortestPathPhysicalExpansionStartID joins recursive expansion through each edge's start ID. + ShortestPathPhysicalExpansionStartID ShortestPathPhysicalExpansion = "start_id" + + // ShortestPathPhysicalExpansionEndID joins recursive expansion through each edge's end ID. + ShortestPathPhysicalExpansionEndID ShortestPathPhysicalExpansion = "end_id" +) + +type ShortestPathTopologyClassification string + +const ( + // ShortestPathTopologyPhysicalOutbound classifies traversal aligned with stored edge direction. + ShortestPathTopologyPhysicalOutbound ShortestPathTopologyClassification = "physical_outbound" + + // ShortestPathTopologyPhysicalInboundShallow classifies a shallow traversal against stored edge direction. + ShortestPathTopologyPhysicalInboundShallow ShortestPathTopologyClassification = "physical_inbound_shallow" + + // ShortestPathTopologyPhysicalInboundDeep classifies a deep traversal against stored edge direction. + ShortestPathTopologyPhysicalInboundDeep ShortestPathTopologyClassification = "physical_inbound_deep" + + // ShortestPathTopologyDirectionless classifies traversal that may follow either stored direction. + ShortestPathTopologyDirectionless ShortestPathTopologyClassification = "directionless" +) + +// ShortestPathEligibilityFact records one named qualification check for an executor candidate. +type ShortestPathEligibilityFact struct { + // Name identifies the qualification check. + Name string `json:"name"` + // Eligible reports whether the candidate passed the named check. + Eligible bool `json:"eligible"` +} + +// ShortestPathExecutorDecision records either a qualified static executor or +// the incumbent fallback, keeping every eligibility and fallback fact visible. +type ShortestPathExecutorDecision struct { + // Target locates the traversal step governed by this decision. + Target TraversalStepTarget `json:"target"` + // Family names the executor-selection family that produced the decision. + Family string `json:"family"` + // PlannedCandidates lists the executors considered in preference order. + PlannedCandidates []ShortestPathExecutor `json:"planned_candidates"` + // SelectedExecutor is the executor chosen after qualification. + SelectedExecutor ShortestPathExecutor `json:"selected_executor"` + // ExecutionBoundary distinguishes inline statement SQL from stored helper + // execution. Promotion evidence must match this boundary exactly. + ExecutionBoundary string `json:"execution_boundary"` + // Scheduler identifies the selected executor's frontier scheduling policy. + Scheduler ShortestPathScheduler `json:"scheduler,omitempty"` + // ObservationMode describes how downstream clauses consume the shortest path. + ObservationMode ShortestPathObservationMode `json:"observation_mode"` + // Direction is the logical direction of the traversal. + Direction graph.Direction `json:"direction"` + // PhysicalExpansion identifies which stored edge endpoint advances the search. + PhysicalExpansion ShortestPathPhysicalExpansion `json:"physical_expansion"` + // RelationshipKindCount is the number of statically resolved relationship kinds. + RelationshipKindCount int `json:"relationship_kind_count"` + // UntypedRelationship reports whether the pattern omitted relationship kinds. + UntypedRelationship bool `json:"untyped_relationship"` + // TopologyClassification summarizes logical direction, physical direction, and depth. + TopologyClassification ShortestPathTopologyClassification `json:"topology_classification"` + // Eligibility records each qualification check and its result. + Eligibility []ShortestPathEligibilityFact `json:"eligibility"` + // StructurallyEligible reports whether the query shape can use the candidate executor. + StructurallyEligible bool `json:"structurally_eligible"` + // StaticallyEligible reports whether known literals and kinds satisfy executor constraints. + StaticallyEligible bool `json:"statically_eligible"` + // MinimumDepth is the inclusive lower traversal-depth bound. + MinimumDepth int64 `json:"minimum_depth"` + // MaximumDepth is the inclusive upper traversal-depth bound. + MaximumDepth int64 `json:"maximum_depth"` + // StateLimit caps state admitted by bounded experimental executors. + StateLimit int64 `json:"state_limit,omitempty"` + // FrontierLimit caps current and queued frontier rows independently of seen state. + FrontierLimit int64 `json:"frontier_limit,omitempty"` + // PredecessorLimit caps retained witness predecessor rows independently of discovery state. + PredecessorLimit int64 `json:"predecessor_limit,omitempty"` + // EnumerationLimit caps distinct ordered all-shortest-path arrays before exact fallback. + EnumerationLimit int64 `json:"enumeration_limit,omitempty"` + // OutputBytesLimit caps staged all-shortest-path array bytes before exact fallback. + OutputBytesLimit int64 `json:"output_bytes_limit,omitempty"` + // SelectorVersion identifies the policy version that ranked the candidates. + SelectorVersion string `json:"selector_version"` + // SelectionMode records whether selection was automatic or forced by tooling. + SelectionMode string `json:"selection_mode"` + // FallbackExecutor is used when the preferred candidate cannot be applied. + FallbackExecutor ShortestPathExecutor `json:"fallback_executor"` + // FallbackReason explains why the preferred candidate was not selected. + FallbackReason string `json:"fallback_reason"` + // ExperimentalWinner reports whether an experimental candidate beat the incumbent. + ExperimentalWinner bool `json:"experimental_winner,omitempty"` +} + type ShortestPathFilterMode string const ( - ShortestPathFilterTerminal ShortestPathFilterMode = "terminal" + // ShortestPathFilterTerminal materializes candidate terminal IDs independently of roots. + ShortestPathFilterTerminal ShortestPathFilterMode = "terminal" + + // ShortestPathFilterEndpointPair materializes admissible root-terminal ID pairs. ShortestPathFilterEndpointPair ShortestPathFilterMode = "endpoint_pair" ) @@ -121,7 +479,10 @@ type ShortestPathFilterDecision struct { type LimitPushdownMode string const ( - LimitPushdownTraversalCTE LimitPushdownMode = "traversal_cte" + // LimitPushdownTraversalCTE applies a row limit inside an ordinary traversal CTE. + LimitPushdownTraversalCTE LimitPushdownMode = "traversal_cte" + + // LimitPushdownShortestPathHarness applies a row limit inside a shortest-path harness. LimitPushdownShortestPathHarness LimitPushdownMode = "shortest_path_harness" ) @@ -130,14 +491,312 @@ type LimitPushdownDecision struct { Mode LimitPushdownMode `json:"mode"` } +// ExpansionSuffixPushdownDecision describes a fixed traversal suffix evaluated for supplemental search. type ExpansionSuffixPushdownDecision struct { - Target TraversalStepTarget `json:"target"` - SuffixLength int `json:"suffix_length"` - SuffixStartStep int `json:"suffix_start_step"` - SuffixEndStep int `json:"suffix_end_step"` + // Target locates the variable expansion followed by the fixed suffix. + Target TraversalStepTarget `json:"target"` + // SuffixLength is the number of fixed traversal steps eligible for pushdown. + SuffixLength int `json:"suffix_length"` + // SuffixStartStep identifies the first fixed traversal step in the suffix. + SuffixStartStep int `json:"suffix_start_step"` + // SuffixEndStep identifies the final fixed traversal step in the suffix. + SuffixEndStep int `json:"suffix_end_step"` + // ApplySupplemental reports whether translation should emit the supplemental suffix-search branch. + ApplySupplemental bool `json:"apply_supplemental"` + // Reason explains why supplemental suffix search was enabled or withheld. + Reason string `json:"reason,omitempty"` + // PredicateAttachments lists predicates assigned to scopes within the fixed suffix. PredicateAttachments []PredicateAttachment `json:"predicate_attachments,omitempty"` } +type ExpansionSearchStrategy string + +const ( + // ExpansionSearchStepwiseForward selects the incumbent left-to-right expansion plan. + ExpansionSearchStepwiseForward ExpansionSearchStrategy = "EXPANSION-STEPWISE-FORWARD" + + // ExpansionSearchLateHydratedForward selects forward search with deferred entity hydration. + ExpansionSearchLateHydratedForward ExpansionSearchStrategy = "EXPANSION-LATE-HYDRATED-FORWARD" + + // ExpansionSearchFactoredSuffixForward selects forward search with a factored fixed suffix. + ExpansionSearchFactoredSuffixForward ExpansionSearchStrategy = "EXPANSION-FACTORED-SUFFIX-FORWARD" + + // ExpansionSearchSuffixSeededReverse selects reverse probing seeded from a selective fixed suffix. + ExpansionSearchSuffixSeededReverse ExpansionSearchStrategy = "EXPANSION-SUFFIX-SEEDED-REVERSE" + + // ExpansionSearchEndpointSeededReverse selects reverse probing seeded from selective terminal endpoints. + ExpansionSearchEndpointSeededReverse ExpansionSearchStrategy = "EXPANSION-ENDPOINT-SEEDED-REVERSE" + + // ExpansionSearchBackwardViabilityForward selects forward expansion gated by backward reachability. + ExpansionSearchBackwardViabilityForward ExpansionSearchStrategy = "EXPANSION-BACKWARD-VIABILITY-FORWARD" +) + +// ExpansionSearchPolicy identifies a runtime policy independently of the +// expansion arm that the policy may execute. +type ExpansionSearchPolicy string + +const ( + // ExpansionSearchPolicyEndpointGuardV1 identifies the shipped endpoint and + // reverse-state sentinel policy. It is distinct from topology orientation, + // which requires root, suffix, and directional-degree probes. + ExpansionSearchPolicyEndpointGuardV1 ExpansionSearchPolicy = "endpoint-state-guard-v1" + + // ExpansionSearchPolicyOrientationProbeV1 selects an ordinary-expansion + // orientation from bounded, same-statement topology probes. + ExpansionSearchPolicyOrientationProbeV1 ExpansionSearchPolicy = "orientation-probe-v1" + + // ExpansionSearchPolicyOrientationProbeV2 selects an ordinary-expansion + // orientation using depth-weighted forward work and the same bounded, + // same-statement topology probes as v1. + ExpansionSearchPolicyOrientationProbeV2 ExpansionSearchPolicy = "orientation-probe-v2" + + // ExpansionSearchOrientationRootRowLimit caps complete forward-root evidence + // for the initial fixed-suffix orientation tournament. + ExpansionSearchOrientationRootRowLimit int64 = 512 + + // ExpansionSearchOrientationReverseSeedRowLimit caps complete fixed-suffix + // row evidence while preserving duplicate suffix paths. + ExpansionSearchOrientationReverseSeedRowLimit int64 = 512 + + // ExpansionSearchOrientationDirectionalDegreeRowLimit caps each typed + // directional adjacency probe independently. + ExpansionSearchOrientationDirectionalDegreeRowLimit int64 = 16_384 + + // ExpansionSearchOrientationStateLimit caps admitted reverse recursive state. + ExpansionSearchOrientationStateLimit int64 = 4_096 + + // ExpansionSearchOrientationReverseScoreMultiplier is the reverse side of + // orientation-probe-v1's strict 3/4 hysteresis comparison. + ExpansionSearchOrientationReverseScoreMultiplier int64 = 4 + + // ExpansionSearchOrientationForwardScoreMultiplier is the incumbent side + // of orientation-probe-v1's strict 3/4 hysteresis comparison. + ExpansionSearchOrientationForwardScoreMultiplier int64 = 3 + + // ExpansionSearchOrientationV2ReverseScoreMultiplier is the reverse side + // of orientation-probe-v2's strict 3/4 hysteresis comparison. + ExpansionSearchOrientationV2ReverseScoreMultiplier int64 = 4 + + // ExpansionSearchOrientationV2ForwardScoreMultiplier is the incumbent side + // of orientation-probe-v2's strict 3/4 hysteresis comparison. + ExpansionSearchOrientationV2ForwardScoreMultiplier int64 = 3 + + // ExpansionSearchExecutionBoundaryInlineStatement identifies one emitted + // expansion traversal arm in the translated statement. + ExpansionSearchExecutionBoundaryInlineStatement = "inline_statement" + + // ExpansionSearchExecutionBoundaryGuardedDualArm identifies a + // same-statement expansion policy with exact candidate and fallback arms. + ExpansionSearchExecutionBoundaryGuardedDualArm = "guarded_dual_arm" +) + +// ExpansionSearchProbeCaps records the maximum complete evidence admitted by +// an orientation policy. SQL probes use cap+1 sentinels to detect overflow. +type ExpansionSearchProbeCaps struct { + // RootRowLimit caps forward-root evidence. + RootRowLimit int64 `json:"root_row_limit,omitempty"` + // ReverseSeedRowLimit caps terminal or fixed-suffix seed evidence. + ReverseSeedRowLimit int64 `json:"reverse_seed_row_limit,omitempty"` + // DirectionalDegreeRowLimit caps typed first-hop adjacency evidence. + DirectionalDegreeRowLimit int64 `json:"directional_degree_row_limit,omitempty"` + // SurvivalRowLimit caps optional one-level survival evidence. + SurvivalRowLimit int64 `json:"survival_row_limit,omitempty"` +} + +// ExpansionSearchAdmission records the exact gate and fallback for a +// specialized orientation arm. +type ExpansionSearchAdmission struct { + // StateLimit caps specialized search state before incumbent fallback. + StateLimit int64 `json:"state_limit,omitempty"` + // RequiresCompleteProbes requires every candidate input probe to remain at + // or below its declared cap before specialized rows may be exposed. + RequiresCompleteProbes bool `json:"requires_complete_probes,omitempty"` + // FallbackStrategy names the exact incumbent used when admission fails. + FallbackStrategy ExpansionSearchStrategy `json:"fallback_strategy,omitempty"` +} + +type ExpansionSearchObservationMode string + +const ( + // ExpansionSearchObservationEndpointIDs indicates that downstream clauses consume only endpoint IDs. + ExpansionSearchObservationEndpointIDs ExpansionSearchObservationMode = "endpoint_ids" + + // ExpansionSearchObservationOrderedPathIDs indicates that downstream clauses consume ordered path IDs. + ExpansionSearchObservationOrderedPathIDs ExpansionSearchObservationMode = "ordered_path_ids" + + // ExpansionSearchObservationFullPath indicates that downstream clauses consume hydrated path values. + ExpansionSearchObservationFullPath ExpansionSearchObservationMode = "full_path" + + // ExpansionSearchObservationUnsupported indicates an observation pattern unsupported by specialized search. + ExpansionSearchObservationUnsupported ExpansionSearchObservationMode = "unsupported" +) + +// ExpansionSearchEligibilityFact records one named qualification check for a search strategy. +type ExpansionSearchEligibilityFact struct { + // Name identifies the qualification check. + Name string `json:"name"` + // Eligible reports whether the strategy passed the named check. + Eligible bool `json:"eligible"` +} + +const ( + // ExpansionSearchFallbackNoFixedSuffix rejects a strategy that requires a fixed suffix when none exists. + ExpansionSearchFallbackNoFixedSuffix = "no_fixed_suffix" + + // ExpansionSearchFallbackSuffixTooShort rejects a fixed suffix below the strategy's minimum length. + ExpansionSearchFallbackSuffixTooShort = "suffix_too_short" + + // ExpansionSearchFallbackOptionalMatch rejects a rewrite that would alter OPTIONAL MATCH behavior. + ExpansionSearchFallbackOptionalMatch = "optional_match" + + // ExpansionSearchFallbackShortestPath rejects ordinary-expansion strategies for shortestPath patterns. + ExpansionSearchFallbackShortestPath = "shortest_path" + + // ExpansionSearchFallbackAllShortestPaths rejects ordinary-expansion strategies for allShortestPaths patterns. + ExpansionSearchFallbackAllShortestPaths = "all_shortest_paths" + + // ExpansionSearchFallbackDirectionlessExpansion rejects a directionless variable expansion. + ExpansionSearchFallbackDirectionlessExpansion = "directionless_expansion" + + // ExpansionSearchFallbackDirectionlessSuffix rejects a directionless edge in the fixed suffix. + ExpansionSearchFallbackDirectionlessSuffix = "directionless_suffix" + + // ExpansionSearchFallbackUnboundedDepth rejects an expansion without a finite maximum depth. + ExpansionSearchFallbackUnboundedDepth = "unbounded_depth" + + // ExpansionSearchFallbackUnsupportedDepth rejects a depth range the candidate cannot preserve. + ExpansionSearchFallbackUnsupportedDepth = "unsupported_depth" + + // ExpansionSearchFallbackMultipleVariableExpansions rejects regions containing more than one variable expansion. + ExpansionSearchFallbackMultipleVariableExpansions = "multiple_variable_expansions" + + // ExpansionSearchFallbackCorrelatedSuffix rejects a fixed suffix that reuses an outer binding. + ExpansionSearchFallbackCorrelatedSuffix = "correlated_suffix" + + // ExpansionSearchFallbackCrossRegionPredicate rejects predicates spanning the variable and fixed regions. + ExpansionSearchFallbackCrossRegionPredicate = "cross_region_predicate" + + // ExpansionSearchFallbackPathDependentPredicate rejects predicates that depend on accumulated path state. + ExpansionSearchFallbackPathDependentPredicate = "path_dependent_predicate" + + // ExpansionSearchFallbackRelationshipVariable rejects an observed relationship binding in the variable expansion or fixed suffix. + ExpansionSearchFallbackRelationshipVariable = "relationship_variable" + + // ExpansionSearchFallbackRelationshipPredicate rejects relationship predicates in the variable expansion or fixed suffix. + ExpansionSearchFallbackRelationshipPredicate = "relationship_predicate" + + // ExpansionSearchFallbackLimitPushdownConflict rejects a rewrite that conflicts with an existing limit pushdown. + ExpansionSearchFallbackLimitPushdownConflict = "limit_pushdown_conflict" + + // ExpansionSearchFallbackUnsupportedObservation rejects downstream uses the candidate cannot reconstruct. + ExpansionSearchFallbackUnsupportedObservation = "unsupported_observation" + + // ExpansionSearchFallbackMutation rejects specialized search for a statement containing updates. + ExpansionSearchFallbackMutation = "mutation" + + // ExpansionSearchFallbackNonDeterministicPredicate rejects a seed predicate that cannot be safely reordered. + ExpansionSearchFallbackNonDeterministicPredicate = "non_deterministic_predicate" + + // ExpansionSearchFallbackUnboundRoot rejects a strategy that requires a previously bound expansion root. + ExpansionSearchFallbackUnboundRoot = "unbound_root" + + // ExpansionSearchFallbackTournamentUnqualified records that no specialized strategy passed qualification. + ExpansionSearchFallbackTournamentUnqualified = "tournament_unqualified" + + // ExpansionSearchFallbackNoFixedPrefix rejects a strategy that requires a fixed prefix when none exists. + ExpansionSearchFallbackNoFixedPrefix = "no_fixed_prefix" + + // ExpansionSearchFallbackExpansionNotTerminal rejects endpoint seeding when the expansion is not terminal. + ExpansionSearchFallbackExpansionNotTerminal = "expansion_not_terminal" + + // ExpansionSearchFallbackPrefixTooLong rejects a prefix that is not exactly one fixed hop. + ExpansionSearchFallbackPrefixTooLong = "prefix_too_long" + + // ExpansionSearchFallbackDirectionlessPrefix rejects a directionless edge in the fixed prefix. + ExpansionSearchFallbackDirectionlessPrefix = "directionless_prefix" + + // ExpansionSearchFallbackTerminalNotSelective rejects endpoint seeding without a selective terminal predicate. + ExpansionSearchFallbackTerminalNotSelective = "terminal_not_selective" + + // ExpansionSearchFallbackCorrelatedTerminal rejects a pre-bound terminal or a terminal predicate that depends on another binding. + ExpansionSearchFallbackCorrelatedTerminal = "correlated_terminal" + + // ExpansionSearchFallbackZeroDepth rejects a rewrite that cannot preserve zero-length paths. + ExpansionSearchFallbackZeroDepth = "zero_depth" +) + +// ExpansionSearchStrategyDecision records qualification and selection details for one variable expansion. +type ExpansionSearchStrategyDecision struct { + // Target locates the variable-expansion step governed by this decision. + Target TraversalStepTarget `json:"target"` + // Family names the search-strategy family that produced the decision. + Family string `json:"family"` + // PlannedPolicy identifies the runtime policy intended for this candidate + // family, whether or not translation currently emits it. + PlannedPolicy ExpansionSearchPolicy `json:"planned_policy,omitempty"` + // EmittedPolicy identifies the runtime policy actually present in emitted + // SQL. It remains empty for a single forced arm or incumbent-only SQL. + EmittedPolicy ExpansionSearchPolicy `json:"emitted_policy,omitempty"` + // PlannedCandidates lists the strategies considered in preference order. + PlannedCandidates []ExpansionSearchStrategy `json:"planned_candidates"` + // EmittedCandidates lists the arms present in the translated statement. + // Runtime telemetry, not this field, records which arm executed. + EmittedCandidates []ExpansionSearchStrategy `json:"emitted_candidates,omitempty"` + // ExecutionBoundary describes the SQL boundary that contains the emitted + // expansion arm or guarded policy. + ExecutionBoundary string `json:"execution_boundary,omitempty"` + // ProbeCaps records bounded evidence inputs for the planned policy. + ProbeCaps ExpansionSearchProbeCaps `json:"probe_caps"` + // Admission records the exact specialized-state gate and fallback chain. + Admission ExpansionSearchAdmission `json:"admission"` + // CandidateStrategy is the specialized strategy proposed by structural analysis. + CandidateStrategy ExpansionSearchStrategy `json:"candidate_strategy,omitempty"` + // SelectedStrategy is the strategy chosen after all qualification checks. + SelectedStrategy ExpansionSearchStrategy `json:"selected_strategy"` + // StructurallyEligible reports whether the traversal shape supports the candidate. + StructurallyEligible bool `json:"structurally_eligible"` + // StaticallyEligible reports whether known bounds and predicates support the candidate. + StaticallyEligible bool `json:"statically_eligible"` + // EligibilityFacts records each qualification check and its result. + EligibilityFacts []ExpansionSearchEligibilityFact `json:"eligibility_facts"` + // SuffixStartStep is the first traversal step in the fixed suffix. + SuffixStartStep int `json:"suffix_start_step,omitempty"` + // SuffixEndStep is the last traversal step in the fixed suffix. + SuffixEndStep int `json:"suffix_end_step,omitempty"` + // SuffixLength is the number of traversal steps in the fixed suffix. + SuffixLength int `json:"suffix_length,omitempty"` + // PrefixStartStep is the first traversal step in the fixed prefix. + PrefixStartStep int `json:"prefix_start_step,omitempty"` + // PrefixEndStep is the last traversal step in the fixed prefix. + PrefixEndStep int `json:"prefix_end_step,omitempty"` + // PrefixLength is the number of traversal steps in the fixed prefix. + PrefixLength int `json:"prefix_length,omitempty"` + // SeedPredicateClass describes the predicate used to bound reverse search seeds. + SeedPredicateClass string `json:"seed_predicate_class,omitempty"` + // EndpointLimit caps terminal endpoints admitted into endpoint-seeded search. + EndpointLimit int64 `json:"endpoint_limit,omitempty"` + // StateLimit caps reverse-search states admitted before falling back. + StateLimit int64 `json:"state_limit,omitempty"` + // HasFinalLimit reports whether the terminal projection has a row limit. + HasFinalLimit bool `json:"has_final_limit,omitempty"` + // ObservationMode describes the representation required by downstream consumers. + ObservationMode ExpansionSearchObservationMode `json:"observation_mode"` + // LogicalDirection records the variable expansion's Cypher direction. + LogicalDirection string `json:"logical_direction"` + // MinimumDepth is the inclusive lower expansion-depth bound. + MinimumDepth int64 `json:"minimum_depth"` + // MaximumDepth is the inclusive upper expansion-depth bound, or zero when unbounded. + MaximumDepth int64 `json:"maximum_depth,omitempty"` + // SelectionMode records whether selection was automatic or forced by tooling. + SelectionMode string `json:"selection_mode"` + // SelectorVersion identifies the policy version that ranked the candidates. + SelectorVersion string `json:"selector_version"` + // FallbackStrategy is used when the specialized candidate cannot be applied. + FallbackStrategy ExpansionSearchStrategy `json:"fallback_strategy"` + // FallbackReason explains why the specialized candidate was not selected. + FallbackReason string `json:"fallback_reason"` +} + type PredicatePlacementDecision struct { Target TraversalStepTarget `json:"target"` Attachment PredicateAttachment `json:"attachment"` @@ -147,6 +806,7 @@ type PredicatePlacementDecision struct { type PatternPredicatePlacementMode string const ( + // PatternPredicatePlacementExistence lowers a pattern predicate as an existence test. PatternPredicatePlacementExistence PatternPredicatePlacementMode = "existence" ) @@ -158,7 +818,10 @@ type PatternPredicatePlacementDecision struct { type CountStoreFastPathTarget string const ( + // CountStoreFastPathNode reads a node count directly from graph statistics. CountStoreFastPathNode CountStoreFastPathTarget = "node" + + // CountStoreFastPathEdge reads a relationship count directly from graph statistics. CountStoreFastPathEdge CountStoreFastPathTarget = "edge" ) @@ -191,6 +854,57 @@ type AggregateTraversalCountDecision struct { Target TraversalStepTarget `json:"target"` } +type FieldRequirement string + +const ( + // FieldRequirementEntityID requires only the scalar entity identifier. + FieldRequirementEntityID FieldRequirement = "entity_id" + + // FieldRequirementKinds requires the entity kind array in addition to identity. + FieldRequirementKinds FieldRequirement = "kinds" + + // FieldRequirementProperties requires the entity property document in addition to identity. + FieldRequirementProperties FieldRequirement = "properties" + + // FieldRequirementFullEntity requires the complete node or relationship composite. + FieldRequirementFullEntity FieldRequirement = "full_entity" + + // FieldRequirementRelationshipIDs requires relationship IDs without hydrated relationship composites. + FieldRequirementRelationshipIDs FieldRequirement = "relationship_ids" + + // FieldRequirementOrderedPathEdgeIDs requires edge IDs in path traversal order. + FieldRequirementOrderedPathEdgeIDs FieldRequirement = "ordered_path_edge_ids" + + // FieldRequirementFullPath requires the complete hydrated path composite. + FieldRequirementFullPath FieldRequirement = "full_path" +) + +// FieldRequirementUse records the representation required at one ordered use of a binding. +type FieldRequirementUse struct { + // Ordinal orders this use relative to the other uses in its query part. + Ordinal int `json:"ordinal"` + // Fields lists the binding components consumed at this use. + Fields []FieldRequirement `json:"fields"` + // Internal reports whether the requirement is internal to translation rather than an external consumer. + Internal bool `json:"internal,omitempty"` +} + +// FieldRequirementDecision is analysis metadata only. Phase 6B consumes this +// staged information when it is safe to lower a composite binding to scalar +// state; recording it here intentionally does not change SQL semantics. +type FieldRequirementDecision struct { + // QueryPartIndex identifies the query part containing the analyzed binding. + QueryPartIndex int `json:"query_part_index"` + // Symbol is the Cypher binding whose representation requirements were analyzed. + Symbol string `json:"symbol"` + // Fields is the union of binding components required by all uses. + Fields []FieldRequirement `json:"fields"` + // Uses preserves the ordered evidence contributing to Fields. + Uses []FieldRequirementUse `json:"uses"` + // LastUse is the greatest use ordinal observed for the binding. + LastUse int `json:"last_use"` +} + type AggregateTraversalCountShape struct { QueryPartIndex int SourceSymbol string @@ -211,23 +925,49 @@ type AggregateTraversalCountShape struct { Target TraversalStepTarget } +// LoweringPlan records lowering analyses and semantic or physical decisions for a query. type LoweringPlan struct { - ProjectionPruning []ProjectionPruningDecision `json:"projection_pruning,omitempty"` - LatePathMaterialization []LatePathMaterializationDecision `json:"late_path_materialization,omitempty"` - ExpandInto []ExpandIntoDecision `json:"expand_into,omitempty"` - TraversalDirection []TraversalDirectionDecision `json:"traversal_direction,omitempty"` - ShortestPathStrategy []ShortestPathStrategyDecision `json:"shortest_path_strategy,omitempty"` - ShortestPathFilter []ShortestPathFilterDecision `json:"shortest_path_filter,omitempty"` - LimitPushdown []LimitPushdownDecision `json:"limit_pushdown,omitempty"` - ExpansionSuffixPushdown []ExpansionSuffixPushdownDecision `json:"expansion_suffix_pushdown,omitempty"` - PredicatePlacement []PredicatePlacementDecision `json:"predicate_placement,omitempty"` - PatternPredicate []PatternPredicatePlacementDecision `json:"pattern_predicate_placement,omitempty"` - CountStoreFastPath []CountStoreFastPathDecision `json:"count_store_fast_path,omitempty"` - ExactRangeExpansion []ExactRangeExpansionDecision `json:"exact_range_expansion,omitempty"` + // ProjectionPruning records traversal fields that downstream clauses do not require. + ProjectionPruning []ProjectionPruningDecision `json:"projection_pruning,omitempty"` + // LatePathMaterialization records path values whose hydration can be deferred. + LatePathMaterialization []LatePathMaterializationDecision `json:"late_path_materialization,omitempty"` + // ExpandInto records traversal steps whose endpoints are both already bound. + ExpandInto []ExpandIntoDecision `json:"expand_into,omitempty"` + // TraversalDirection records planned logical direction changes. + TraversalDirection []TraversalDirectionDecision `json:"traversal_direction,omitempty"` + // ShortestPathStrategy records directional search choices for shortest-path steps. + ShortestPathStrategy []ShortestPathStrategyDecision `json:"shortest_path_strategy,omitempty"` + // ShortestPathFilter records endpoint filters selected for materialization. + ShortestPathFilter []ShortestPathFilterDecision `json:"shortest_path_filter,omitempty"` + // LimitPushdown records row limits that may safely constrain traversal work. + LimitPushdown []LimitPushdownDecision `json:"limit_pushdown,omitempty"` + // ExpansionSuffixPushdown records fixed suffixes considered for supplemental filtering, including withheld candidates. + ExpansionSuffixPushdown []ExpansionSuffixPushdownDecision `json:"expansion_suffix_pushdown,omitempty"` + // PredicatePlacement records the earliest safe traversal scope for attached predicates. + PredicatePlacement []PredicatePlacementDecision `json:"predicate_placement,omitempty"` + // PatternPredicate records existence lowering selected for pattern predicates. + PatternPredicate []PatternPredicatePlacementDecision `json:"pattern_predicate_placement,omitempty"` + // CountStoreFastPath records counts answerable directly from graph statistics. + CountStoreFastPath []CountStoreFastPathDecision `json:"count_store_fast_path,omitempty"` + // ExactRangeExpansion records short fixed-depth ranges selected for unrolling. + ExactRangeExpansion []ExactRangeExpansionDecision `json:"exact_range_expansion,omitempty"` + // PathRelationshipPredicate records relationship quantifiers attached to carried path state. PathRelationshipPredicate []PathRelationshipPredicateDecision `json:"path_relationship_predicate,omitempty"` - AggregateTraversalCount []AggregateTraversalCountDecision `json:"aggregate_traversal_count,omitempty"` + // AggregateTraversalCount records traversals lowered directly to aggregate counts. + AggregateTraversalCount []AggregateTraversalCountDecision `json:"aggregate_traversal_count,omitempty"` + // FieldRequirements records downstream representation needs for each analyzed binding. + FieldRequirements []FieldRequirementDecision `json:"field_requirements,omitempty"` + // ShortestPathExecutor records physical executor choices for shortest-path steps. + ShortestPathExecutor []ShortestPathExecutorDecision `json:"shortest_path_executor,omitempty"` + // ExpansionSearchStrategy records physical search choices for variable expansions. + ExpansionSearchStrategy []ExpansionSearchStrategyDecision `json:"expansion_search_strategy,omitempty"` + // EndpointResolution records planned-only bounded endpoint materialization for SP/ASP traversals. + EndpointResolution []EndpointResolutionDecision `json:"endpoint_resolution,omitempty"` + // TraversalPredicate records conservative locality and universality classifications. + TraversalPredicate []TraversalPredicateDecision `json:"traversal_predicate,omitempty"` } +// Empty reports whether the plan contains no lowering-analysis or decision entries. func (s LoweringPlan) Empty() bool { return len(s.ProjectionPruning) == 0 && len(s.LatePathMaterialization) == 0 && @@ -242,9 +982,15 @@ func (s LoweringPlan) Empty() bool { len(s.CountStoreFastPath) == 0 && len(s.ExactRangeExpansion) == 0 && len(s.PathRelationshipPredicate) == 0 && - len(s.AggregateTraversalCount) == 0 + len(s.AggregateTraversalCount) == 0 && + len(s.FieldRequirements) == 0 && + len(s.ShortestPathExecutor) == 0 && + len(s.ExpansionSearchStrategy) == 0 && + len(s.EndpointResolution) == 0 && + len(s.TraversalPredicate) == 0 } +// Decisions returns one summary entry for each lowering category present in the plan. func (s LoweringPlan) Decisions() []LoweringDecision { var decisions []LoweringDecision add := func(name string, applied bool) { @@ -266,6 +1012,11 @@ func (s LoweringPlan) Decisions() []LoweringDecision { add(LoweringExactRangeExpansion, len(s.ExactRangeExpansion) > 0) add(LoweringPathRelationshipPredicate, len(s.PathRelationshipPredicate) > 0) add(LoweringAggregateTraversalCount, len(s.AggregateTraversalCount) > 0) + add(LoweringFieldRequirements, len(s.FieldRequirements) > 0) + add(LoweringShortestPathExecutor, len(s.ShortestPathExecutor) > 0) + add(LoweringExpansionSearchStrategy, len(s.ExpansionSearchStrategy) > 0) + add(LoweringEndpointResolution, len(s.EndpointResolution) > 0) + add(LoweringTraversalPredicateClassification, len(s.TraversalPredicate) > 0) return decisions } @@ -322,6 +1073,7 @@ func IndexPatternPredicateTargets(query *cypher.RegularQuery) map[*cypher.Patter return targets } +// indexReadingClauseTargets maps each pattern in readingClauses to stable source coordinates. func indexReadingClauseTargets(targets map[*cypher.PatternPart]PatternTarget, queryPartIndex int, readingClauses []*cypher.ReadingClause) { for clauseIndex, readingClause := range readingClauses { if readingClause == nil || readingClause.Match == nil { @@ -338,6 +1090,7 @@ func indexReadingClauseTargets(targets map[*cypher.PatternPart]PatternTarget, qu } } +// indexQueryPartPatternPredicateTargets assigns stable target coordinates to pattern predicates in one query part. func indexQueryPartPatternPredicateTargets(targets map[*cypher.PatternPredicate]PatternTarget, queryPartIndex int, queryPart cypher.SyntaxNode) { for _, indexedPredicate := range indexedPatternPredicatesInQueryPart(queryPart) { targets[indexedPredicate.Predicate] = PatternTarget{ diff --git a/cypher/models/pgsql/optimize/lowering_plan.go b/cypher/models/pgsql/optimize/lowering_plan.go index 61df8831..4d433406 100644 --- a/cypher/models/pgsql/optimize/lowering_plan.go +++ b/cypher/models/pgsql/optimize/lowering_plan.go @@ -1,6 +1,7 @@ package optimize import ( + "slices" "strings" "github.com/specterops/dawgs/cypher/models/cypher" @@ -8,39 +9,92 @@ import ( "github.com/specterops/dawgs/graph" ) +// sourceTraversalStep groups the node and relationship patterns that make up one analyzed traversal step. type sourceTraversalStep struct { - LeftNode *cypher.NodePattern + // LeftNode is the node pattern immediately preceding Relationship in source syntax. + LeftNode *cypher.NodePattern + // Relationship is the edge pattern connecting the two endpoints. Relationship *cypher.RelationshipPattern - RightNode *cypher.NodePattern + // RightNode is the node pattern immediately following Relationship in source syntax. + RightNode *cypher.NodePattern } +// boundSourceSelectivity ranks how strongly known constraints bound a traversal source. type boundSourceSelectivity int const ( - traversalDirectionReasonRightBound = "right_bound" - traversalDirectionReasonRightConstrained = "right_constrained" - traversalDirectionReasonRightPredicate = "right_predicate" + // traversalDirectionReasonRightBound explains a direction flip toward an already bound right endpoint. + traversalDirectionReasonRightBound = "right_bound" + + // traversalDirectionReasonRightConstrained explains a direction flip toward a constrained right endpoint. + traversalDirectionReasonRightConstrained = "right_constrained" + + // traversalDirectionReasonRightPredicate explains a direction flip toward a right endpoint with a selective predicate. + traversalDirectionReasonRightPredicate = "right_predicate" + + // traversalDirectionReasonTerminalKindOnlyEstimateWide explains rejection of a terminal kind whose estimate is too broad. traversalDirectionReasonTerminalKindOnlyEstimateWide = "terminal kind-only estimate too broad" - traversalDirectionReasonBoundSourceSelective = "bound source estimate selective" + // traversalDirectionReasonBoundSourceSelective explains retention of a sufficiently selective bound source. + traversalDirectionReasonBoundSourceSelective = "bound source estimate selective" + + // shortestPathStrategyReasonBoundEndpointPairs selects bidirectional search for materialized endpoint pairs. shortestPathStrategyReasonBoundEndpointPairs = "bound_endpoint_pairs" + + // shortestPathStrategyReasonEndpointPredicates selects bidirectional search for predicates on both endpoints. shortestPathStrategyReasonEndpointPredicates = "endpoint_predicates" - shortestPathFilterReasonTerminalPredicate = "terminal_predicate" + // shortestPathFilterReasonTerminalPredicate materializes a filter for a selective terminal predicate. + shortestPathFilterReasonTerminalPredicate = "terminal_predicate" + + // shortestPathFilterReasonEndpointPairPredicates materializes a filter for correlated endpoint-pair predicates. shortestPathFilterReasonEndpointPairPredicates = "endpoint_pair_predicates" ) const ( + // boundSourceSelectivityNone indicates that no useful source constraint was found. boundSourceSelectivityNone boundSourceSelectivity = iota + + // boundSourceSelectivityKindOnly indicates that only a node-kind predicate constrains the source. boundSourceSelectivityKindOnly + + // boundSourceSelectivityPredicate indicates that a non-unique predicate constrains the source. boundSourceSelectivityPredicate + + // boundSourceSelectivityUnique indicates that a unique lookup constrains the source. boundSourceSelectivityUnique + + // boundSourceSelectivityLimited indicates that a row limit bounds the source. boundSourceSelectivityLimited + + // boundSourceSelectivityTopN indicates that an ordered or aggregate projection with a limit bounds the source. boundSourceSelectivityTopN ) -const maxExactRangeExpansionDepth int64 = 2 +const ( + // maxExactRangeExpansionDepth is the largest exact range expanded into fixed traversal steps. + maxExactRangeExpansionDepth int64 = 2 + + // defaultShortestPathExpansionDepth supplies the maximum depth for an otherwise open shortest-path range. + defaultShortestPathExpansionDepth int64 = 15 + // defaultShortestPathStateLimit caps intermediate states admitted by guarded experimental executors. + defaultShortestPathStateLimit int64 = 100_000 + + // defaultShortestPathFrontierLimit independently caps queued/current frontier state. + defaultShortestPathFrontierLimit int64 = 100_000 + + // defaultShortestPathPredecessorLimit independently caps retained witness predecessors. + defaultShortestPathPredecessorLimit int64 = 100_000 + + // defaultAllShortestPathsEnumerationLimit independently caps staged distinct path arrays. + defaultAllShortestPathsEnumerationLimit int64 = 100_000 + + // defaultAllShortestPathsOutputBytesLimit independently caps staged ordered edge-array bytes. + defaultAllShortestPathsOutputBytesLimit int64 = 64 * 1024 * 1024 +) + +// BuildLoweringPlan analyzes a query and selects safe semantic and physical lowering decisions. func BuildLoweringPlan(query *cypher.RegularQuery, predicateAttachments []PredicateAttachment) (LoweringPlan, error) { if query == nil || query.SingleQuery == nil { return LoweringPlan{}, nil @@ -93,9 +147,13 @@ func BuildLoweringPlan(query *cypher.RegularQuery, predicateAttachments []Predic attachPredicatePlacementsToSuffixPushdowns(&plan) appendCountStoreFastPathDecisions(&plan, query) appendAggregateTraversalCountDecisions(&plan, query) + finalizeShortestPathExecutorDecisions(&plan, query) + finalizeExpansionSearchStrategyDecisions(&plan, query) + finalizeTraversalEnvelopeDecisions(&plan, query) return plan, nil } +// appendQueryPartLowerings runs every lowering analysis for one query part and appends its decisions to plan. func appendQueryPartLowerings( plan *LoweringPlan, queryPartIndex int, @@ -117,16 +175,1222 @@ func appendQueryPartLowerings( appendLatePathMaterializationDecisions(plan, queryPartIndex, readingClauses, sourceReferences) appendPatternPredicateProjectionLowerings(plan, queryPartIndex, queryPart, sourceReferences) appendPatternPredicatePlacementDecisions(plan, queryPartIndex, queryPart) - appendExpandIntoDecisions(plan, queryPartIndex, readingClauses) + appendExpandIntoDecisions(plan, queryPartIndex, readingClauses, initialDeclaredSymbols) appendTraversalDirectionDecisions(plan, queryPartIndex, readingClauses, bindingPredicateSymbols(predicateAttachments, queryPartIndex), initialDeclaredSymbols, initialSelectivity) shortestPathSearchSymbols := shortestPathSearchPredicateSymbols(readingClauses) appendShortestPathStrategyDecisions(plan, queryPartIndex, readingClauses, shortestPathSearchSymbols) appendShortestPathFilterDecisions(plan, queryPartIndex, readingClauses, shortestPathSearchSymbols) + appendShortestPathExecutorDecisions(plan, queryPartIndex, queryPart, readingClauses, sourceReferences) + appendEndpointResolutionDecisions(plan, queryPartIndex, queryPart, readingClauses, initialDeclaredSymbols) + appendTraversalPredicateDecisions(plan, queryPartIndex, queryPart, readingClauses) appendLimitPushdownDecisions(plan, queryPartIndex, queryPart, readingClauses) - appendExpansionSuffixPushdownDecisions(plan, queryPartIndex, readingClauses) + appendExpansionSuffixPushdownDecisions(plan, queryPartIndex, readingClauses, sourceReferences) + appendEndpointSeededExpansionDecisions(plan, queryPartIndex, queryPart, readingClauses, sourceReferences, initialDeclaredSymbols) + appendExpansionSearchStrategyDecisions(plan, queryPartIndex, queryPart, readingClauses, sourceReferences, initialDeclaredSymbols) + fieldRequirements, err := collectFieldRequirements(queryPartIndex, queryPart) + if err != nil { + return err + } + plan.FieldRequirements = append(plan.FieldRequirements, fieldRequirements...) + applyShortestPathObservationModes(plan, queryPartIndex, readingClauses, fieldRequirements) + applyExpansionSearchObservationModes(plan, queryPartIndex, readingClauses, fieldRequirements) return nil } +// appendEndpointSeededExpansionDecisions qualifies terminal expansions with a fixed prefix for guarded reverse search. +func appendEndpointSeededExpansionDecisions(plan *LoweringPlan, queryPartIndex int, queryPart cypher.SyntaxNode, readingClauses []*cypher.ReadingClause, sourceReferences map[string]struct{}, initialDeclaredSymbols map[string]struct{}) { + _, updatingClauses := queryPartProjection(queryPart) + declaredSymbols := copyStringSet(initialDeclaredSymbols) + for clauseIndex, readingClause := range readingClauses { + if readingClause == nil || readingClause.Match == nil { + continue + } + searchSymbols := shortestPathSearchPredicateSymbols([]*cypher.ReadingClause{readingClause}) + idEqualities := singletonIDEqualityCounts(readingClause.Match.Where) + for patternIndex, patternPart := range readingClause.Match.Pattern { + steps := traversalStepsForPattern(patternPart) + variableExpansions := 0 + for _, step := range steps { + if step.Relationship != nil && step.Relationship.Range != nil { + variableExpansions++ + } + } + for stepIndex, step := range steps { + if step.Relationship == nil || step.Relationship.Range == nil || stepIndex == 0 { + continue + } + target := PatternTarget{QueryPartIndex: queryPartIndex, ClauseIndex: clauseIndex, PatternIndex: patternIndex}.TraversalStep(stepIndex) + prefixLength := stepIndex + terminal := stepIndex == len(steps)-1 + directedPrefix := true + prefixFixed := true + for _, prefixStep := range steps[:stepIndex] { + directedPrefix = directedPrefix && prefixStep.Relationship != nil && prefixStep.Relationship.Direction != graph.DirectionBoth + prefixFixed = prefixFixed && prefixStep.Relationship != nil && prefixStep.Relationship.Range == nil + } + minDepth := int64(1) + if step.Relationship.Range.StartIndex != nil { + minDepth = *step.Relationship.Range.StartIndex + } + maxDepth := int64(15) + if step.Relationship.Range.EndIndex != nil { + maxDepth = *step.Relationship.Range.EndIndex + } + terminalSymbol := variableSymbol(step.RightNode.Variable) + _, propertySearch := searchSymbols[terminalSymbol] + idSearch := idEqualities[terminalSymbol] == 1 + seedClass := "" + if idSearch { + seedClass = "id_equality" + } else if propertySearch { + seedClass = endpointSeedPredicateClass(readingClause.Match.Where, terminalSymbol) + } + terminalSelective := idSearch || propertySearch + terminalCorrelated := symbolDeclared(declaredSymbols, terminalSymbol) + terminalPredicateLocal := predicateTermsForSymbolAreLocal(readingClause.Match.Where, terminalSymbol) + relationshipPredicate := step.Relationship.Properties != nil || syntaxDependsOn(readingClause.Match.Where, variableSymbol(step.Relationship.Variable)) + pathDependentPredicate := patternPart != nil && patternPart.Variable != nil && syntaxDependsOn(readingClause.Match.Where, patternPart.Variable.Symbol) + deterministicPredicates := !syntaxContainsNonIdentityFunctionInvocation(patternPart) && !syntaxContainsNonIdentityFunctionInvocation(readingClause.Match.Where) + observation := ExpansionSearchObservationEndpointIDs + if patternPart != nil && patternPart.Variable != nil && referencesSourceIdentifier(sourceReferences, patternPart.Variable.Symbol) { + observation = ExpansionSearchObservationFullPath + } + facts := []ExpansionSearchEligibilityFact{ + {Name: "read_only", Eligible: updatingClauses == 0}, + {Name: "non_optional", Eligible: !readingClause.Match.Optional}, + {Name: "ordinary_path", Eligible: patternPart != nil && !patternPart.ShortestPathPattern && !patternPart.AllShortestPathsPattern}, + {Name: "single_variable_expansion_in_region", Eligible: variableExpansions == 1}, + {Name: "terminal_expansion", Eligible: terminal}, + {Name: "exact_one_hop_prefix", Eligible: prefixLength == 1 && prefixFixed}, + {Name: "directed_prefix", Eligible: directedPrefix}, + {Name: "directed_expansion", Eligible: step.Relationship.Direction != graph.DirectionBoth}, + {Name: "supported_effective_depth", Eligible: maxDepth >= minDepth && maxDepth <= 64}, + {Name: "minimum_depth_one", Eligible: minDepth >= 1}, + {Name: "terminal_unbound", Eligible: !terminalCorrelated}, + {Name: "selective_terminal_predicate", Eligible: terminalSelective}, + {Name: "terminal_predicate_local", Eligible: terminalPredicateLocal}, + {Name: "single_relationship_kind", Eligible: len(step.Relationship.Kinds) == 1}, + {Name: "no_relationship_variable", Eligible: step.Relationship.Variable == nil}, + {Name: "no_relationship_predicate", Eligible: !relationshipPredicate}, + {Name: "no_path_dependent_predicate", Eligible: !pathDependentPredicate}, + {Name: "deterministic_predicates", Eligible: deterministicPredicates}, + {Name: "supported_observation", Eligible: observation != ExpansionSearchObservationUnsupported}, + } + eligible := expansionSearchFactsEligible(facts) + fallbackReason := ExpansionSearchFallbackTournamentUnqualified + switch { + case updatingClauses > 0: + fallbackReason = ExpansionSearchFallbackMutation + case readingClause.Match.Optional: + fallbackReason = ExpansionSearchFallbackOptionalMatch + case !terminal: + fallbackReason = ExpansionSearchFallbackExpansionNotTerminal + case prefixLength == 0: + fallbackReason = ExpansionSearchFallbackNoFixedPrefix + case prefixLength != 1 || !prefixFixed: + fallbackReason = ExpansionSearchFallbackPrefixTooLong + case !directedPrefix: + fallbackReason = ExpansionSearchFallbackDirectionlessPrefix + case step.Relationship.Direction == graph.DirectionBoth: + fallbackReason = ExpansionSearchFallbackDirectionlessExpansion + case minDepth < 1: + fallbackReason = ExpansionSearchFallbackZeroDepth + case maxDepth < minDepth || maxDepth > 64: + fallbackReason = ExpansionSearchFallbackUnsupportedDepth + case variableExpansions != 1: + fallbackReason = ExpansionSearchFallbackMultipleVariableExpansions + case terminalCorrelated || !terminalPredicateLocal: + fallbackReason = ExpansionSearchFallbackCorrelatedTerminal + case !terminalSelective: + fallbackReason = ExpansionSearchFallbackTerminalNotSelective + case len(step.Relationship.Kinds) != 1: + fallbackReason = ExpansionSearchFallbackTournamentUnqualified + case step.Relationship.Variable != nil: + fallbackReason = ExpansionSearchFallbackRelationshipVariable + case relationshipPredicate: + fallbackReason = ExpansionSearchFallbackRelationshipPredicate + case pathDependentPredicate: + fallbackReason = ExpansionSearchFallbackPathDependentPredicate + case !deterministicPredicates: + fallbackReason = ExpansionSearchFallbackNonDeterministicPredicate + } + selected := ExpansionSearchStepwiseForward + selectionMode := "incumbent_default" + if eligible { + selected = ExpansionSearchEndpointSeededReverse + selectionMode = "static_guarded" + fallbackReason = "" + } + projection, _ := queryPartProjection(queryPart) + candidate := contiguousExpansionOrientationCandidate{ + Target: target, + Family: "fixed_prefix_terminal_expansion", + PlannedPolicy: ExpansionSearchPolicyEndpointGuardV1, + PlannedCandidates: []ExpansionSearchStrategy{ExpansionSearchStepwiseForward, ExpansionSearchEndpointSeededReverse}, + EmittedCandidates: []ExpansionSearchStrategy{ExpansionSearchStepwiseForward}, + CandidateStrategy: ExpansionSearchEndpointSeededReverse, + ProbeCaps: ExpansionSearchProbeCaps{ + ReverseSeedRowLimit: 32, + }, + Admission: ExpansionSearchAdmission{ + StateLimit: 4096, + RequiresCompleteProbes: true, + FallbackStrategy: ExpansionSearchStepwiseForward, + }, + PrefixStartStep: 0, + PrefixEndStep: stepIndex - 1, + PrefixLength: prefixLength, + SeedPredicateClass: seedClass, + EndpointLimit: 32, + } + if eligible { + candidate.EmittedPolicy = ExpansionSearchPolicyEndpointGuardV1 + candidate.EmittedCandidates = []ExpansionSearchStrategy{ExpansionSearchStepwiseForward, ExpansionSearchEndpointSeededReverse} + } + plan.ExpansionSearchStrategy = append(plan.ExpansionSearchStrategy, candidate.decision(contiguousExpansionOrientationQualification{ + SelectedStrategy: selected, + StructurallyEligible: eligible, + StaticallyEligible: eligible, + EligibilityFacts: facts, + HasFinalLimit: projection != nil && projection.Limit != nil, + ObservationMode: observation, + LogicalDirection: step.Relationship.Direction.String(), + MinimumDepth: minDepth, + MaximumDepth: maxDepth, + SelectionMode: selectionMode, + SelectorVersion: "endpoint-seeded-guarded-v1", + FallbackReason: fallbackReason, + })) + } + declarePatternSymbols(declaredSymbols, patternPart) + } + declareWhereSymbols(declaredSymbols, readingClause.Match) + } +} + +// predicateTermsForSymbolAreLocal reports whether every predicate mentioning symbol depends on no other binding. +func predicateTermsForSymbolAreLocal(where *cypher.Where, symbol string) bool { + if where == nil || symbol == "" { + return true + } + for _, expression := range where.Expressions { + for _, term := range cypherConjunctionTerms(expression) { + dependencies := sortedDependencies(term) + if !slices.Contains(dependencies, symbol) { + continue + } + for _, dependency := range dependencies { + if dependency != symbol { + return false + } + } + } + } + return true +} + +// endpointSeedPredicateClass classifies a terminal property comparison as equality, suffix matching, or generic search. +func endpointSeedPredicateClass(where *cypher.Where, symbol string) string { + if where == nil { + return "" + } + for _, expression := range where.Expressions { + for _, term := range cypherConjunctionTerms(expression) { + comparison, ok := term.(*cypher.Comparison) + if !ok || comparison == nil || len(comparison.Partials) != 1 { + continue + } + partial := comparison.Partials[0] + leftSymbol, leftOK := propertyLookupVariableSymbol(comparison.Left) + rightSymbol, rightOK := propertyLookupVariableSymbol(partial.Right) + if (leftOK && leftSymbol == symbol && !expressionReferencesAnySource(partial.Right)) || (rightOK && rightSymbol == symbol && !expressionReferencesAnySource(comparison.Left)) { + switch partial.Operator { + case cypher.OperatorEquals: + return "property_equality" + case cypher.OperatorEndsWith: + return "property_ends_with" + default: + return "property_search" + } + } + } + } + return "" +} + +// hasExpansionSearchDecision reports whether plan already contains a search decision for target. +func hasExpansionSearchDecision(plan *LoweringPlan, target TraversalStepTarget) bool { + for _, decision := range plan.ExpansionSearchStrategy { + if decision.Target == target { + return true + } + } + return false +} + +// appendExpansionSearchStrategyDecisions qualifies variable expansions for fixed-suffix search strategies. +func appendExpansionSearchStrategyDecisions(plan *LoweringPlan, queryPartIndex int, queryPart cypher.SyntaxNode, readingClauses []*cypher.ReadingClause, sourceReferences map[string]struct{}, initialDeclaredSymbols map[string]struct{}) { + _, updatingClauses := queryPartProjection(queryPart) + declaredSymbols := copyStringSet(initialDeclaredSymbols) + queryPartVariableExpansions := 0 + for _, readingClause := range readingClauses { + if readingClause == nil || readingClause.Match == nil { + continue + } + for _, patternPart := range readingClause.Match.Pattern { + for _, step := range traversalStepsForPattern(patternPart) { + if step.Relationship != nil && step.Relationship.Range != nil { + queryPartVariableExpansions++ + } + } + } + } + for clauseIndex, readingClause := range readingClauses { + if readingClause == nil || readingClause.Match == nil { + continue + } + for patternIndex, patternPart := range readingClause.Match.Pattern { + steps := traversalStepsForPattern(patternPart) + deterministicPredicates := !syntaxContainsFunctionInvocation(patternPart) && !syntaxContainsFunctionInvocation(readingClause.Match.Where) + pathDependentPredicate := patternPart != nil && patternPart.Variable != nil && syntaxDependsOn(readingClause.Match.Where, patternPart.Variable.Symbol) + for stepIndex, step := range steps { + if step.Relationship == nil || step.Relationship.Range == nil { + continue + } + target := PatternTarget{ + QueryPartIndex: queryPartIndex, + ClauseIndex: clauseIndex, + PatternIndex: patternIndex, + }.TraversalStep(stepIndex) + if hasExpansionSearchDecision(plan, target) { + continue + } + limitConflict := hasLimitPushdownForTarget(plan, target) + suffixLength := fixedSuffixLength(steps[stepIndex+1:]) + suffixEnd := stepIndex + suffixLength + minDepth := int64(1) + if step.Relationship.Range.StartIndex != nil { + minDepth = *step.Relationship.Range.StartIndex + } + maxDepth := int64(0) + boundedDepth := step.Relationship.Range.EndIndex != nil + if boundedDepth { + maxDepth = *step.Relationship.Range.EndIndex + } + directedExpansion := step.Relationship.Direction != graph.DirectionBoth + directedSuffix := suffixLength > 0 + noSuffixRelationshipVariables := true + noRelationshipPredicates := step.Relationship.Properties == nil && !syntaxDependsOn(readingClause.Match.Where, variableSymbol(step.Relationship.Variable)) + suffixSteps := steps[stepIndex+1 : stepIndex+1+suffixLength] + uncorrelatedSuffix := true + for _, suffixStep := range suffixSteps { + directedSuffix = directedSuffix && suffixStep.Relationship.Direction != graph.DirectionBoth + noSuffixRelationshipVariables = noSuffixRelationshipVariables && suffixStep.Relationship.Variable == nil + noRelationshipPredicates = noRelationshipPredicates && suffixStep.Relationship.Properties == nil && !syntaxDependsOn(readingClause.Match.Where, variableSymbol(suffixStep.Relationship.Variable)) + uncorrelatedSuffix = uncorrelatedSuffix && !symbolDeclared(declaredSymbols, variableSymbol(suffixStep.Relationship.Variable)) && !symbolDeclared(declaredSymbols, variableSymbol(suffixStep.RightNode.Variable)) + } + noCrossRegionPredicate := !hasCrossRegionPredicate(readingClause.Match.Where, step, suffixSteps) + boundRoot := symbolDeclared(declaredSymbols, variableSymbol(step.LeftNode.Variable)) + observation := ExpansionSearchObservationEndpointIDs + if patternPart != nil && patternPart.Variable != nil && referencesSourceIdentifier(sourceReferences, patternPart.Variable.Symbol) { + observation = ExpansionSearchObservationFullPath + } + facts := []ExpansionSearchEligibilityFact{ + { + Name: "read_only", + Eligible: updatingClauses == 0, + }, + { + Name: "non_optional", + Eligible: !readingClause.Match.Optional, + }, + { + Name: "ordinary_path", + Eligible: patternPart != nil && !patternPart.ShortestPathPattern && !patternPart.AllShortestPathsPattern, + }, + { + Name: "single_variable_expansion", + Eligible: queryPartVariableExpansions == 1, + }, + { + Name: "bound_root", + Eligible: boundRoot, + }, + { + Name: "initial_variable_expansion", + Eligible: stepIndex == 0, + }, + { + Name: "directed_expansion", + Eligible: directedExpansion, + }, + { + Name: "bounded_supported_depth", + Eligible: boundedDepth && maxDepth >= minDepth && maxDepth <= 64, + }, + { + Name: "exact_three_hop_suffix", + Eligible: suffixLength == 3, + }, + { + Name: "qualified_fixed_suffix_topology", + Eligible: qualifiedFixedSuffixTopology(step, suffixSteps), + }, + { + Name: "directed_suffix", + Eligible: directedSuffix, + }, + { + Name: "no_relationship_variable", + Eligible: step.Relationship.Variable == nil && noSuffixRelationshipVariables, + }, + { + Name: "no_relationship_predicate", + Eligible: noRelationshipPredicates, + }, + { + Name: "uncorrelated_suffix", + Eligible: uncorrelatedSuffix, + }, + { + Name: "no_cross_region_predicate", + Eligible: noCrossRegionPredicate, + }, + { + Name: "no_path_dependent_predicate", + Eligible: !pathDependentPredicate, + }, + { + Name: "deterministic_predicates", + Eligible: deterministicPredicates, + }, + { + Name: "no_limit_pushdown_conflict", + Eligible: !limitConflict, + }, + { + Name: "supported_observation", + Eligible: observation != ExpansionSearchObservationUnsupported, + }, + } + eligible := true + for _, fact := range facts { + eligible = eligible && fact.Eligible + } + fallbackReason := ExpansionSearchFallbackTournamentUnqualified + switch { + case updatingClauses > 0: + fallbackReason = ExpansionSearchFallbackMutation + case readingClause.Match.Optional: + fallbackReason = ExpansionSearchFallbackOptionalMatch + case patternPart != nil && patternPart.AllShortestPathsPattern: + fallbackReason = ExpansionSearchFallbackAllShortestPaths + case patternPart != nil && patternPart.ShortestPathPattern: + fallbackReason = ExpansionSearchFallbackShortestPath + case queryPartVariableExpansions > 1: + fallbackReason = ExpansionSearchFallbackMultipleVariableExpansions + case stepIndex != 0: + fallbackReason = ExpansionSearchFallbackTournamentUnqualified + case !directedExpansion: + fallbackReason = ExpansionSearchFallbackDirectionlessExpansion + case !boundedDepth: + fallbackReason = ExpansionSearchFallbackUnboundedDepth + case maxDepth < minDepth || maxDepth > 64: + fallbackReason = ExpansionSearchFallbackUnsupportedDepth + case suffixLength == 0: + fallbackReason = ExpansionSearchFallbackNoFixedSuffix + case suffixLength < 3: + fallbackReason = ExpansionSearchFallbackSuffixTooShort + case suffixLength != 3: + fallbackReason = ExpansionSearchFallbackTournamentUnqualified + case !directedSuffix: + fallbackReason = ExpansionSearchFallbackDirectionlessSuffix + case !noRelationshipPredicates: + fallbackReason = ExpansionSearchFallbackRelationshipPredicate + case !uncorrelatedSuffix: + fallbackReason = ExpansionSearchFallbackCorrelatedSuffix + case !noCrossRegionPredicate: + fallbackReason = ExpansionSearchFallbackCrossRegionPredicate + case step.Relationship.Variable != nil || !noSuffixRelationshipVariables: + fallbackReason = ExpansionSearchFallbackRelationshipVariable + case pathDependentPredicate: + fallbackReason = ExpansionSearchFallbackPathDependentPredicate + case !deterministicPredicates: + fallbackReason = ExpansionSearchFallbackNonDeterministicPredicate + case limitConflict: + fallbackReason = ExpansionSearchFallbackLimitPushdownConflict + case !boundRoot && qualifiedFixedSuffixTopology(step, suffixSteps): + fallbackReason = ExpansionSearchFallbackUnboundRoot + } + candidate := contiguousExpansionOrientationCandidate{ + Target: target, + Family: "fixed_suffix_expansion", + PlannedPolicy: ExpansionSearchPolicyOrientationProbeV1, + PlannedCandidates: []ExpansionSearchStrategy{ + ExpansionSearchStepwiseForward, + ExpansionSearchLateHydratedForward, + ExpansionSearchFactoredSuffixForward, + ExpansionSearchSuffixSeededReverse, + ExpansionSearchBackwardViabilityForward, + }, + EmittedCandidates: []ExpansionSearchStrategy{ExpansionSearchStepwiseForward}, + CandidateStrategy: ExpansionSearchSuffixSeededReverse, + ProbeCaps: ExpansionSearchProbeCaps{ + RootRowLimit: ExpansionSearchOrientationRootRowLimit, + ReverseSeedRowLimit: ExpansionSearchOrientationReverseSeedRowLimit, + DirectionalDegreeRowLimit: ExpansionSearchOrientationDirectionalDegreeRowLimit, + }, + Admission: ExpansionSearchAdmission{ + StateLimit: ExpansionSearchOrientationStateLimit, + RequiresCompleteProbes: true, + FallbackStrategy: ExpansionSearchStepwiseForward, + }, + SuffixStartStep: stepIndex + 1, + SuffixEndStep: suffixEnd, + SuffixLength: suffixLength, + } + plan.ExpansionSearchStrategy = append(plan.ExpansionSearchStrategy, candidate.decision(contiguousExpansionOrientationQualification{ + SelectedStrategy: ExpansionSearchStepwiseForward, + StructurallyEligible: eligible, + StaticallyEligible: eligible, + EligibilityFacts: facts, + ObservationMode: observation, + LogicalDirection: step.Relationship.Direction.String(), + MinimumDepth: minDepth, + MaximumDepth: maxDepth, + SelectionMode: "incumbent_default", + SelectorVersion: "fixed-suffix-static-v1", + FallbackReason: fallbackReason, + })) + } + declarePatternSymbols(declaredSymbols, patternPart) + } + declareWhereSymbols(declaredSymbols, readingClause.Match) + } +} + +// syntaxContainsFunctionInvocation reports whether node contains any function invocation. +func syntaxContainsFunctionInvocation(node cypher.SyntaxNode) bool { + if node == nil { + return false + } + found := false + _ = walk.Cypher(node, walk.NewSimpleVisitor[cypher.SyntaxNode](func(node cypher.SyntaxNode, _ walk.VisitorHandler) { + if _, isFunction := node.(*cypher.FunctionInvocation); isFunction { + found = true + } + })) + return found +} + +// syntaxContainsNonIdentityFunctionInvocation reports whether node invokes a function other than id. +func syntaxContainsNonIdentityFunctionInvocation(node cypher.SyntaxNode) bool { + if node == nil { + return false + } + found := false + _ = walk.Cypher(node, walk.NewSimpleVisitor[cypher.SyntaxNode](func(node cypher.SyntaxNode, _ walk.VisitorHandler) { + if function, isFunction := node.(*cypher.FunctionInvocation); isFunction && function != nil && !strings.EqualFold(function.Name, cypher.IdentityFunction) { + found = true + } + })) + return found +} + +// symbolDeclared reports whether a non-empty symbol is present in the declaration set. +func symbolDeclared(declared map[string]struct{}, symbol string) bool { + if symbol == "" { + return false + } + _, found := declared[symbol] + return found +} + +// hasCrossRegionPredicate reports whether one predicate depends on both expansion and suffix bindings. +func hasCrossRegionPredicate(where *cypher.Where, expansion sourceTraversalStep, suffix []sourceTraversalStep) bool { + if where == nil { + return false + } + prefixSymbols := map[string]struct{}{} + suffixSymbols := map[string]struct{}{} + addSymbol(prefixSymbols, variableSymbol(expansion.LeftNode.Variable)) + addSymbol(prefixSymbols, variableSymbol(expansion.Relationship.Variable)) + addSymbol(prefixSymbols, variableSymbol(expansion.RightNode.Variable)) + for _, step := range suffix { + addSymbol(suffixSymbols, variableSymbol(step.Relationship.Variable)) + addSymbol(suffixSymbols, variableSymbol(step.RightNode.Variable)) + } + for _, expression := range where.Expressions { + var hasPrefix, hasSuffix bool + for _, dependency := range sortedDependencies(expression) { + if _, found := prefixSymbols[dependency]; found { + hasPrefix = true + } + if _, found := suffixSymbols[dependency]; found { + hasSuffix = true + } + } + if hasPrefix && hasSuffix { + return true + } + } + return false +} + +// fixedSuffixLength counts consecutive fixed relationship steps before the next range expansion. +func fixedSuffixLength(steps []sourceTraversalStep) int { + length := 0 + for _, step := range steps { + if step.Relationship == nil || step.Relationship.Range != nil { + break + } + length++ + } + return length +} + +// hasLimitPushdownForTarget reports whether target already has a planned limit pushdown. +func hasLimitPushdownForTarget(plan *LoweringPlan, target TraversalStepTarget) bool { + for _, decision := range plan.LimitPushdown { + if decision.Target == target { + return true + } + } + return false +} + +// qualifiedFixedSuffixTopology reports whether an outbound single-kind expansion has the required three-step typed suffix. +func qualifiedFixedSuffixTopology(expansion sourceTraversalStep, suffix []sourceTraversalStep) bool { + if len(suffix) != 3 || expansion.Relationship == nil || len(expansion.Relationship.Kinds) != 1 || expansion.Relationship.Direction != graph.DirectionOutbound { + return false + } + for _, step := range suffix { + if step.Relationship == nil || step.RightNode == nil || step.Relationship.Direction != graph.DirectionOutbound || len(step.Relationship.Kinds) != 1 || len(step.RightNode.Kinds) != 1 { + return false + } + } + return true +} + +// applyExpansionSearchObservationModes classifies each expansion by the fields its external consumers require. +func applyExpansionSearchObservationModes(plan *LoweringPlan, queryPartIndex int, readingClauses []*cypher.ReadingClause, requirements []FieldRequirementDecision) { + externalFieldsBySymbol := map[string]map[FieldRequirement]struct{}{} + for _, requirement := range requirements { + fields := map[FieldRequirement]struct{}{} + for _, use := range requirement.Uses { + if use.Internal { + continue + } + for _, field := range use.Fields { + fields[field] = struct{}{} + } + } + externalFieldsBySymbol[requirement.Symbol] = fields + } + for idx := range plan.ExpansionSearchStrategy { + decision := &plan.ExpansionSearchStrategy[idx] + if decision.Target.QueryPartIndex != queryPartIndex || decision.Target.Predicate || decision.Target.ClauseIndex >= len(readingClauses) { + continue + } + clause := readingClauses[decision.Target.ClauseIndex] + if clause == nil || clause.Match == nil || decision.Target.PatternIndex >= len(clause.Match.Pattern) { + continue + } + pattern := clause.Match.Pattern[decision.Target.PatternIndex] + if pattern == nil || pattern.Variable == nil { + decision.ObservationMode = ExpansionSearchObservationEndpointIDs + setExpansionSearchEligibilityFact(decision, "supported_observation", true) + continue + } + fields := externalFieldsBySymbol[pattern.Variable.Symbol] + switch { + case hasFieldRequirement(fields, FieldRequirementFullPath): + decision.ObservationMode = ExpansionSearchObservationFullPath + case hasFieldRequirement(fields, FieldRequirementOrderedPathEdgeIDs), hasFieldRequirement(fields, FieldRequirementRelationshipIDs): + decision.ObservationMode = ExpansionSearchObservationOrderedPathIDs + case hasFieldRequirement(fields, FieldRequirementFullEntity): + decision.ObservationMode = ExpansionSearchObservationFullPath + case len(fields) == 0: + decision.ObservationMode = ExpansionSearchObservationEndpointIDs + default: + decision.ObservationMode = ExpansionSearchObservationUnsupported + } + supported := decision.ObservationMode != ExpansionSearchObservationUnsupported + setExpansionSearchEligibilityFact(decision, "supported_observation", supported) + if !supported { + decision.StructurallyEligible = false + decision.StaticallyEligible = false + decision.SelectedStrategy = decision.FallbackStrategy + decision.FallbackReason = ExpansionSearchFallbackUnsupportedObservation + } + } +} + +// hasFieldRequirement reports whether fields contains the requested binding representation. +func hasFieldRequirement(fields map[FieldRequirement]struct{}, field FieldRequirement) bool { + _, found := fields[field] + return found +} + +// setExpansionSearchEligibilityFact updates a named qualification result +// already present on decision and reports whether that fact belongs to this +// candidate family. +func setExpansionSearchEligibilityFact(decision *ExpansionSearchStrategyDecision, name string, eligible bool) bool { + for idx := range decision.EligibilityFacts { + if decision.EligibilityFacts[idx].Name == name { + decision.EligibilityFacts[idx].Eligible = eligible + return true + } + } + return false +} + +// expansionSearchFactsEligible reports whether every recorded expansion-search qualification passed. +func expansionSearchFactsEligible(facts []ExpansionSearchEligibilityFact) bool { + for _, fact := range facts { + if !fact.Eligible { + return false + } + } + return true +} + +// applyShortestPathObservationModes classifies shortest-path consumers and updates their known-observation qualification. +func applyShortestPathObservationModes(plan *LoweringPlan, queryPartIndex int, readingClauses []*cypher.ReadingClause, requirements []FieldRequirementDecision) { + fieldsBySymbol := map[string]map[FieldRequirement]struct{}{} + for _, requirement := range requirements { + fields := map[FieldRequirement]struct{}{} + for _, field := range requirement.Fields { + fields[field] = struct{}{} + } + fieldsBySymbol[requirement.Symbol] = fields + } + for idx := range plan.ShortestPathExecutor { + decision := &plan.ShortestPathExecutor[idx] + if decision.Target.QueryPartIndex != queryPartIndex || decision.Target.Predicate { + continue + } + if decision.Target.ClauseIndex >= len(readingClauses) { + continue + } + clause := readingClauses[decision.Target.ClauseIndex] + if clause == nil || clause.Match == nil || decision.Target.PatternIndex >= len(clause.Match.Pattern) { + continue + } + pattern := clause.Match.Pattern[decision.Target.PatternIndex] + if pattern == nil || pattern.Variable == nil { + continue + } + fields := fieldsBySymbol[pattern.Variable.Symbol] + if pattern.AllShortestPathsPattern { + if _, fullPath := fields[FieldRequirementFullPath]; fullPath { + decision.ObservationMode = ShortestPathObservationAllPaths + } else if _, orderedIDs := fields[FieldRequirementOrderedPathEdgeIDs]; orderedIDs { + decision.ObservationMode = ShortestPathObservationAllPaths + } + } else if _, fullPath := fields[FieldRequirementFullPath]; fullPath { + decision.ObservationMode = ShortestPathObservationOnePath + } else if _, orderedIDs := fields[FieldRequirementOrderedPathEdgeIDs]; orderedIDs { + decision.ObservationMode = ShortestPathObservationDistance + } + setShortestPathEligibilityFact(decision, "known_observation_mode", decision.ObservationMode != ShortestPathObservationUnknown) + } +} + +// appendShortestPathExecutorDecisions records eligibility facts and incumbent executor decisions for shortest-path expansions. +func appendShortestPathExecutorDecisions(plan *LoweringPlan, queryPartIndex int, queryPart cypher.SyntaxNode, readingClauses []*cypher.ReadingClause, sourceReferences map[string]struct{}) { + var ( + shortestCalls int + patternSources int + hasUnwind bool + ) + for _, readingClause := range readingClauses { + if readingClause == nil { + continue + } + if readingClause.Unwind != nil { + hasUnwind = true + } + if readingClause.Match == nil { + continue + } + patternSources += len(readingClause.Match.Pattern) + for _, patternPart := range readingClause.Match.Pattern { + if patternPart != nil && (patternPart.ShortestPathPattern || patternPart.AllShortestPathsPattern) { + shortestCalls++ + } + } + } + _, updatingClauses := queryPartProjection(queryPart) + for clauseIndex, readingClause := range readingClauses { + if readingClause == nil || readingClause.Match == nil { + continue + } + for patternIndex, patternPart := range readingClause.Match.Pattern { + if patternPart == nil || (!patternPart.ShortestPathPattern && !patternPart.AllShortestPathsPattern) { + continue + } + steps := traversalStepsForPattern(patternPart) + idEqualities := singletonIDEqualityCounts(readingClause.Match.Where) + pathPredicate := syntaxDependsOn(readingClause.Match.Where, variableSymbol(patternPart.Variable)) + for stepIndex, step := range steps { + if step.Relationship == nil || step.Relationship.Range == nil { + continue + } + minDepth := int64(1) + if step.Relationship.Range.StartIndex != nil { + minDepth = *step.Relationship.Range.StartIndex + } + maxDepth := defaultShortestPathExpansionDepth + boundedDepth := step.Relationship.Range.EndIndex != nil + if boundedDepth { + maxDepth = *step.Relationship.Range.EndIndex + } + supportedDepth := (boundedDepth || patternPart.AllShortestPathsPattern) && (minDepth == 0 || minDepth == 1) && maxDepth >= minDepth && maxDepth <= 64 + directionSupported := step.Relationship.Direction != graph.DirectionBoth + relationshipVariableObserved := step.Relationship.Variable != nil && referencesSourceIdentifier(sourceReferences, step.Relationship.Variable.Symbol) + noRelationshipVariable := step.Relationship.Variable == nil || (patternPart.AllShortestPathsPattern && !relationshipVariableObserved) + leftIDCount := idEqualities[variableSymbol(step.LeftNode.Variable)] + rightIDCount := idEqualities[variableSymbol(step.RightNode.Variable)] + singletonIDs := leftIDCount == 1 && rightIDCount == 1 + uncorrelatedSource := queryPartIndex == 0 && !hasUnwind + singleEndpointPair := patternSources == 1 + physicalExpansion := ShortestPathPhysicalExpansionStartID + topologyClassification := ShortestPathTopologyPhysicalOutbound + if step.Relationship.Direction == graph.DirectionInbound { + physicalExpansion = ShortestPathPhysicalExpansionEndID + if maxDepth <= 1 { + topologyClassification = ShortestPathTopologyPhysicalInboundShallow + } else { + topologyClassification = ShortestPathTopologyPhysicalInboundDeep + } + } else if step.Relationship.Direction == graph.DirectionBoth { + topologyClassification = ShortestPathTopologyDirectionless + } + facts := []ShortestPathEligibilityFact{ + { + Name: "supported_shortest_path_mode", + Eligible: patternPart.ShortestPathPattern || patternPart.AllShortestPathsPattern, + }, + { + Name: "single_three_element_traversal", + Eligible: len(patternPart.PatternElements) == 3 && len(steps) == 1, + }, + { + Name: "non_optional", + Eligible: !readingClause.Match.Optional, + }, + { + Name: "directed", + Eligible: directionSupported, + }, + { + Name: "bounded_supported_depth", + Eligible: supportedDepth, + }, + { + Name: "no_relationship_variable", + Eligible: noRelationshipVariable, + }, + { + Name: "no_relationship_predicate", + Eligible: step.Relationship.Properties == nil, + }, + { + Name: "single_path_call", + Eligible: shortestCalls == 1, + }, + { + Name: "read_only", + Eligible: updatingClauses == 0, + }, + { + Name: "one_static_id_equality_per_endpoint", + Eligible: singletonIDs, + }, + { + Name: "no_path_predicate", + Eligible: !pathPredicate, + }, + { + Name: "uncorrelated_endpoint_source", + Eligible: uncorrelatedSource, + }, + { + Name: "single_endpoint_pair", + Eligible: singleEndpointPair, + }, + { + Name: "known_observation_mode", + Eligible: false, + }, + } + reason := ShortestPathFallbackTournamentUnqualified + switch { + case patternPart.AllShortestPathsPattern && !singletonIDs: + reason = ShortestPathFallbackAllShortestPaths + case readingClause.Match.Optional: + reason = ShortestPathFallbackOptionalMatch + case !directionSupported: + reason = ShortestPathFallbackDirectionless + case pathPredicate: + reason = ShortestPathFallbackPathPredicate + case !noRelationshipVariable: + reason = ShortestPathFallbackRelationshipVariable + case step.Relationship.Properties != nil: + reason = ShortestPathFallbackRelationshipPredicate + case !supportedDepth: + reason = ShortestPathFallbackUnsupportedDepth + case shortestCalls != 1: + reason = ShortestPathFallbackMultiplePathCalls + case updatingClauses != 0: + reason = ShortestPathFallbackMutation + case !uncorrelatedSource: + reason = ShortestPathFallbackCorrelatedEndpoints + case !singleEndpointPair: + reason = ShortestPathFallbackMultipleEndpointPairs + case leftIDCount > 1 || rightIDCount > 1: + reason = ShortestPathFallbackMultipleIDEqualities + case !singletonIDs: + reason = ShortestPathFallbackNonSingletonID + } + family := "SP" + plannedCandidates := []ShortestPathExecutor{ + ShortestPathExecutorIncumbentWorkspace, + ShortestPathExecutorS0Direct, + ShortestPathExecutorS1ArrayBFS, + ShortestPathExecutorS2TraceRelation, + ShortestPathExecutorS3Unidirectional, + ShortestPathExecutorS3EdgeM0, + ShortestPathExecutorS4CanonicalDistance, + ShortestPathExecutorS4CanonicalWitness, + ShortestPathExecutorI1CanonicalDistance, + ShortestPathExecutorI1CanonicalWitness, + ShortestPathExecutorI1CanonicalPredecessorWitness, + ShortestPathExecutorB1AlternatingNodeDistance, + ShortestPathExecutorB1AlternatingNodeWitness, + ShortestPathExecutorB2SmallerCurrentLevelDistance, + ShortestPathExecutorB2SmallerCurrentLevelWitness, + } + if patternPart.AllShortestPathsPattern { + family = "ASP" + plannedCandidates = []ShortestPathExecutor{ + ShortestPathExecutorIncumbentWorkspace, + ShortestPathExecutorASPA1DAG, + ShortestPathExecutorASPI1DAG, + ShortestPathExecutorASPB1AlternatingNodeDAG, + ShortestPathExecutorASPB2SmallerCurrentLevelDAG, + } + } + plan.ShortestPathExecutor = append(plan.ShortestPathExecutor, ShortestPathExecutorDecision{ + Target: PatternTarget{ + QueryPartIndex: queryPartIndex, + ClauseIndex: clauseIndex, + PatternIndex: patternIndex, + }.TraversalStep(stepIndex), + Family: family, + PlannedCandidates: plannedCandidates, + SelectedExecutor: ShortestPathExecutorIncumbentWorkspace, + ExecutionBoundary: "stored_helper", + ObservationMode: ShortestPathObservationUnknown, + Direction: step.Relationship.Direction, + PhysicalExpansion: physicalExpansion, + RelationshipKindCount: len(step.Relationship.Kinds), + UntypedRelationship: len(step.Relationship.Kinds) == 0, + TopologyClassification: topologyClassification, + Eligibility: facts, + StructurallyEligible: shortestPathFactsEligible(facts), + StaticallyEligible: false, + MinimumDepth: minDepth, + MaximumDepth: maxDepth, + StateLimit: defaultShortestPathStateLimit, + FrontierLimit: defaultShortestPathFrontierLimit, + PredecessorLimit: defaultShortestPathPredecessorLimit, + EnumerationLimit: defaultAllShortestPathsEnumerationLimit, + OutputBytesLimit: defaultAllShortestPathsOutputBytesLimit, + SelectorVersion: "sp-static-v3", + SelectionMode: "incumbent_default", + FallbackExecutor: ShortestPathExecutorIncumbentWorkspace, + FallbackReason: reason, + }) + } + } + } +} + +// shortestPathFactsEligible reports whether every recorded shortest-path qualification passed. +func shortestPathFactsEligible(facts []ShortestPathEligibilityFact) bool { + for _, fact := range facts { + if !fact.Eligible { + return false + } + } + return true +} + +// setShortestPathEligibilityFact replaces or appends one named executor qualification result. +func setShortestPathEligibilityFact(decision *ShortestPathExecutorDecision, name string, eligible bool) { + for idx := range decision.Eligibility { + if decision.Eligibility[idx].Name == name { + decision.Eligibility[idx].Eligible = eligible + return + } + } + decision.Eligibility = append(decision.Eligibility, ShortestPathEligibilityFact{ + Name: name, + Eligible: eligible, + }) +} + +// finalizeShortestPathExecutorDecisions applies statement-wide safety facts +// after every query part has been analyzed. Per-part counting can otherwise +// misclassify two shortest calls separated by WITH, or a shortest read followed +// by a mutation, as eligible singleton read-only execution. +func finalizeShortestPathExecutorDecisions(plan *LoweringPlan, query *cypher.RegularQuery) { + if plan == nil || query == nil || query.SingleQuery == nil { + return + } + defer func() { + for idx := range plan.ShortestPathExecutor { + decision := &plan.ShortestPathExecutor[idx] + decision.Scheduler = decision.SelectedExecutor.Scheduler() + decision.ExecutionBoundary = decision.SelectedExecutor.ExecutionBoundary() + } + }() + + var ( + shortestCalls int + updatingClauses int + ) + visitPart := func(part cypher.SyntaxNode, readingClauses []*cypher.ReadingClause) { + for _, readingClause := range readingClauses { + if readingClause == nil || readingClause.Match == nil { + continue + } + for _, patternPart := range readingClause.Match.Pattern { + if patternPart != nil && (patternPart.ShortestPathPattern || patternPart.AllShortestPathsPattern) { + shortestCalls++ + } + } + } + _, partUpdatingClauses := queryPartProjection(part) + updatingClauses += partUpdatingClauses + } + + if multiPart := query.SingleQuery.MultiPartQuery; multiPart != nil { + for _, part := range multiPart.Parts { + if part != nil { + visitPart(part, part.ReadingClauses) + } + } + if finalPart := multiPart.SinglePartQuery; finalPart != nil { + visitPart(finalPart, finalPart.ReadingClauses) + } + } else if singlePart := query.SingleQuery.SinglePartQuery; singlePart != nil { + visitPart(singlePart, singlePart.ReadingClauses) + } + + for idx := range plan.ShortestPathExecutor { + decision := &plan.ShortestPathExecutor[idx] + singlePathCall := shortestCalls == 1 + readOnly := updatingClauses == 0 + setShortestPathEligibilityFact(decision, "single_path_call", singlePathCall) + setShortestPathEligibilityFact(decision, "read_only", readOnly) + structurallyEligible := shortestPathFactsEligible(decision.Eligibility) + qualifiedPhysicalDepth := decision.Direction != graph.DirectionInbound || decision.MaximumDepth <= 1 + qualifiedPathKinds := decision.ObservationMode != ShortestPathObservationOnePath || (!decision.UntypedRelationship && decision.RelationshipKindCount == 1) + setShortestPathEligibilityFact(decision, "qualified_physical_expansion_depth", qualifiedPhysicalDepth) + setShortestPathEligibilityFact(decision, "qualified_one_path_kind_state", qualifiedPathKinds) + decision.StructurallyEligible = structurallyEligible + decision.StaticallyEligible = structurallyEligible && qualifiedPhysicalDepth && qualifiedPathKinds + + if !singlePathCall && (decision.FallbackReason == ShortestPathFallbackTournamentUnqualified || decision.FallbackReason == ShortestPathFallbackCorrelatedEndpoints) { + decision.FallbackReason = ShortestPathFallbackMultiplePathCalls + } else if !readOnly && decision.FallbackReason == ShortestPathFallbackTournamentUnqualified { + decision.FallbackReason = ShortestPathFallbackMutation + } + + if structurallyEligible && decision.ObservationMode == ShortestPathObservationAllPaths { + // The compact all-shortest search is deliberately narrower than the + // singleton witness executors. Minimum-depth zero and self-endpoint + // searches can require cyclic relationship-simple paths, which cannot + // use a minimum-node-depth predecessor DAG without changing semantics. + if decision.MinimumDepth != 1 { + decision.FallbackReason = ShortestPathFallbackUnsupportedDepth + continue + } + decision.SelectedExecutor = ShortestPathExecutorASPA1DAG + decision.StaticallyEligible = true + decision.SelectionMode = "static" + decision.SelectorVersion = "asp-static-v1" + decision.FallbackReason = "" + decision.ExperimentalWinner = true + continue + } + + if structurallyEligible { + if !qualifiedPhysicalDepth { + switch decision.ObservationMode { + case ShortestPathObservationDistance: + decision.SelectedExecutor = ShortestPathExecutorS4CanonicalDistance + case ShortestPathObservationOnePath: + decision.SelectedExecutor = ShortestPathExecutorS4CanonicalWitness + default: + decision.FallbackReason = ShortestPathFallbackDeepInboundUnqualified + continue + } + decision.SelectionMode = "static" + decision.SelectorVersion = "sp-static-v5-contained" + decision.StaticallyEligible = true + decision.FallbackReason = "" + decision.ExperimentalWinner = true + continue + } + if !qualifiedPathKinds { + if decision.ObservationMode == ShortestPathObservationOnePath { + decision.SelectedExecutor = ShortestPathExecutorS4CanonicalWitness + decision.SelectionMode = "static" + decision.SelectorVersion = "sp-static-v5-contained" + decision.StaticallyEligible = true + decision.FallbackReason = "" + decision.ExperimentalWinner = true + continue + } + decision.FallbackReason = ShortestPathFallbackNonSingleKindPathState + continue + } + switch decision.ObservationMode { + case ShortestPathObservationDistance: + decision.SelectedExecutor = ShortestPathExecutorS3Unidirectional + decision.SelectorVersion = "sp-static-v3" + case ShortestPathObservationOnePath: + // Restore the former, already-qualified S3 production envelope. + // Deep physical-inbound and non-single-kind witnesses remain on + // S4 above; expanding S3 into either shape would expose its + // unbounded relationship-trail state to a new workload class. + decision.SelectedExecutor = ShortestPathExecutorS3EdgeM0 + decision.SelectorVersion = "sp-static-v5-contained" + default: + continue + } + decision.SelectionMode = "static" + decision.FallbackReason = "" + decision.ExperimentalWinner = true + } + } +} + +// finalizeExpansionSearchStrategyDecisions applies statement-wide safety +// facts after all query parts and field requirements are known. The generic +// orientation tournament has a statement-wide single-expansion envelope; +// endpoint-seeded reverse retains its established per-region fact and guarded +// fallback across independent WITH-separated traversals. +func finalizeExpansionSearchStrategyDecisions(plan *LoweringPlan, query *cypher.RegularQuery) { + if plan == nil || query == nil || query.SingleQuery == nil { + return + } + var variableExpansions, updatingClauses int + visitPart := func(part cypher.SyntaxNode, readingClauses []*cypher.ReadingClause) { + for _, readingClause := range readingClauses { + if readingClause == nil || readingClause.Match == nil { + continue + } + for _, patternPart := range readingClause.Match.Pattern { + for _, step := range traversalStepsForPattern(patternPart) { + if step.Relationship != nil && step.Relationship.Range != nil { + variableExpansions++ + } + } + } + } + _, partUpdatingClauses := queryPartProjection(part) + updatingClauses += partUpdatingClauses + } + if multiPart := query.SingleQuery.MultiPartQuery; multiPart != nil { + for _, part := range multiPart.Parts { + if part != nil { + visitPart(part, part.ReadingClauses) + } + } + if finalPart := multiPart.SinglePartQuery; finalPart != nil { + visitPart(finalPart, finalPart.ReadingClauses) + } + } else if singlePart := query.SingleQuery.SinglePartQuery; singlePart != nil { + visitPart(singlePart, singlePart.ReadingClauses) + } + + for idx := range plan.ExpansionSearchStrategy { + decision := &plan.ExpansionSearchStrategy[idx] + singleExpansion := variableExpansions == 1 + readOnly := updatingClauses == 0 + hasStatementWideExpansionFact := setExpansionSearchEligibilityFact(decision, "single_variable_expansion", singleExpansion) + setExpansionSearchEligibilityFact(decision, "read_only", readOnly) + decision.StructurallyEligible = expansionSearchFactsEligible(decision.EligibilityFacts) + decision.StaticallyEligible = decision.StructurallyEligible + if !decision.StructurallyEligible && decision.SelectedStrategy == ExpansionSearchEndpointSeededReverse { + decision.SelectedStrategy = decision.FallbackStrategy + decision.SelectionMode = "incumbent_default" + } + if hasStatementWideExpansionFact && !singleExpansion && (decision.FallbackReason == "" || decision.FallbackReason == ExpansionSearchFallbackTournamentUnqualified || decision.FallbackReason == ExpansionSearchFallbackMultipleVariableExpansions || decision.FallbackReason == ExpansionSearchFallbackUnboundRoot) { + decision.FallbackReason = ExpansionSearchFallbackMultipleVariableExpansions + } else if !readOnly && (decision.FallbackReason == "" || decision.FallbackReason == ExpansionSearchFallbackTournamentUnqualified) { + decision.FallbackReason = ExpansionSearchFallbackMutation + } + setExpansionSearchExpectedEmission(decision) + } +} + +// syntaxDependsOn reports whether node references symbol as an external dependency. +func syntaxDependsOn(node cypher.SyntaxNode, symbol string) bool { + if symbol == "" { + return false + } + for _, dependency := range sortedDependencies(node) { + if dependency == symbol { + return true + } + } + return false +} + +// singletonIDEqualityCounts counts constant id(symbol) equalities for each symbol in where. +func singletonIDEqualityCounts(where *cypher.Where) map[string]int { + counts := map[string]int{} + if where == nil { + return counts + } + for _, expression := range where.Expressions { + for _, term := range cypherConjunctionTerms(expression) { + comparison, ok := term.(*cypher.Comparison) + if !ok || comparison == nil || len(comparison.Partials) != 1 || comparison.Partials[0].Operator != cypher.OperatorEquals { + continue + } + partial := comparison.Partials[0] + if symbol, ok := identityFunctionSymbol(comparison.Left); ok && expressionIsConstant(partial.Right) { + counts[symbol]++ + } + if symbol, ok := identityFunctionSymbol(partial.Right); ok && expressionIsConstant(comparison.Left) { + counts[symbol]++ + } + } + } + return counts +} + +// identityFunctionSymbol returns the variable named by a single-argument id invocation. +func identityFunctionSymbol(expression cypher.Expression) (string, bool) { + function, ok := expression.(*cypher.FunctionInvocation) + if !ok || function == nil || !strings.EqualFold(function.Name, cypher.IdentityFunction) || len(function.Arguments) != 1 { + return "", false + } + variable, ok := function.Arguments[0].(*cypher.Variable) + if !ok || variable == nil || variable.Symbol == "" { + return "", false + } + return variable.Symbol, true +} + +// appendExactRangeExpansionDecisions records safe short fixed-depth ranges throughout the reading clauses. func appendExactRangeExpansionDecisions(plan *LoweringPlan, queryPartIndex int, readingClauses []*cypher.ReadingClause) { for clauseIndex, readingClause := range readingClauses { if readingClause == nil || readingClause.Match == nil || readingClause.Match.Optional { @@ -143,6 +1407,7 @@ func appendExactRangeExpansionDecisions(plan *LoweringPlan, queryPartIndex int, } } +// appendPatternPredicateExactRangeExpansionDecisions records exact-range steps nested inside pattern predicates. func appendPatternPredicateExactRangeExpansionDecisions(plan *LoweringPlan, queryPartIndex int, queryPart cypher.SyntaxNode) { for _, indexedPredicate := range indexedPatternPredicatesInQueryPart(queryPart) { patternPart := patternPartForPredicate(indexedPredicate.Predicate) @@ -156,6 +1421,7 @@ func appendPatternPredicateExactRangeExpansionDecisions(plan *LoweringPlan, quer } } +// appendPatternExactRangeExpansionDecisions records exact-range steps in one pattern part. func appendPatternExactRangeExpansionDecisions(plan *LoweringPlan, target PatternTarget, patternPart *cypher.PatternPart) { for stepIndex, step := range traversalStepsForPattern(patternPart) { if exactRangeExpansionCandidate(patternPart, step) { @@ -167,6 +1433,7 @@ func appendPatternExactRangeExpansionDecisions(plan *LoweringPlan, target Patter } } +// exactRangeExpansionCandidate reports whether a non-shortest directed step has a small fixed depth safe to unroll. func exactRangeExpansionCandidate(patternPart *cypher.PatternPart, step sourceTraversalStep) bool { if patternPart == nil { return false @@ -184,6 +1451,7 @@ func exactRangeExpansionCandidate(patternPart *cypher.PatternPart, step sourceTr return depth >= 1 && depth <= maxExactRangeExpansionDepth } +// hasExactRangeExpansionDecision reports whether plan already unrolls target's exact range. func hasExactRangeExpansionDecision(plan *LoweringPlan, target TraversalStepTarget) bool { if plan == nil { return false @@ -210,13 +1478,19 @@ func ExactPatternRangeDepth(patternRange *cypher.PatternRange) int64 { return *patternRange.StartIndex } +// indexedQuantifier pairs a quantifier with its stable traversal-order index. type indexedQuantifier struct { - Index int + // Index is the quantifier's zero-based position in structural traversal order. + Index int + // Quantifier is the indexed Cypher predicate node. Quantifier *cypher.Quantifier } +// quantifierCollector records quantifiers in syntax traversal order. type quantifierCollector struct { + // VisitorHandler supplies cancellation and error propagation for the syntax walk. walk.VisitorHandler + // quantifiers accumulates visited quantifiers with their stable indexes. quantifiers []indexedQuantifier } @@ -232,6 +1506,7 @@ func (s *quantifierCollector) Enter(node cypher.SyntaxNode) { func (s *quantifierCollector) Visit(cypher.SyntaxNode) {} func (s *quantifierCollector) Exit(cypher.SyntaxNode) {} +// indexedQuantifiersInQueryPart returns all quantifiers in stable syntax traversal order. func indexedQuantifiersInQueryPart(queryPart cypher.SyntaxNode) []indexedQuantifier { if queryPart == nil { return nil @@ -248,6 +1523,7 @@ func indexedQuantifiersInQueryPart(queryPart cypher.SyntaxNode) []indexedQuantif return collector.quantifiers } +// quantifiersInSyntax returns the quantifier nodes contained in node in traversal order. func quantifiersInSyntax(node cypher.SyntaxNode) []*cypher.Quantifier { if node == nil { return nil @@ -271,6 +1547,7 @@ func quantifiersInSyntax(node cypher.SyntaxNode) []*cypher.Quantifier { return quantifiers } +// pathRelationshipQuantifierCandidate extracts the path and relationship symbols from a supported relationships(path) quantifier. func pathRelationshipQuantifierCandidate(quantifier *cypher.Quantifier) (string, string, bool) { if quantifier == nil || (quantifier.Type != cypher.QuantifierTypeAny && quantifier.Type != cypher.QuantifierTypeNone) || @@ -298,6 +1575,7 @@ func pathRelationshipQuantifierCandidate(quantifier *cypher.Quantifier) (string, return pathVariable.Symbol, bindingSymbol, true } +// appendPathRelationshipPredicateDecisions recognizes supported relationships(path) quantifiers and records their bindings. func appendPathRelationshipPredicateDecisions(plan *LoweringPlan, queryPartIndex int, queryPart cypher.SyntaxNode) { quantifierIndexes := map[*cypher.Quantifier]int{} for _, indexed := range indexedQuantifiersInQueryPart(queryPart) { @@ -340,6 +1618,7 @@ func appendPathRelationshipPredicateDecisions(plan *LoweringPlan, queryPartIndex } } +// appendProjectionPruningDecisions computes unused traversal bindings for each non-optional reading-clause pattern. func appendProjectionPruningDecisions(plan *LoweringPlan, queryPartIndex int, readingClauses []*cypher.ReadingClause, sourceReferences map[string]struct{}) { for clauseIndex, readingClause := range readingClauses { if readingClause == nil || readingClause.Match == nil || readingClause.Match.Optional { @@ -361,6 +1640,7 @@ func appendProjectionPruningDecisions(plan *LoweringPlan, queryPartIndex int, re } } +// appendPatternProjectionPruningDecisions records node, relationship, and path fields unused after each step in a pattern. func appendPatternProjectionPruningDecisions(plan *LoweringPlan, target PatternTarget, patternPart *cypher.PatternPart, steps []sourceTraversalStep, sourceReferences map[string]struct{}) { pathReferenced := referencesSourceIdentifier(sourceReferences, variableSymbol(patternPart.Variable)) @@ -397,6 +1677,7 @@ func appendPatternProjectionPruningDecisions(plan *LoweringPlan, target PatternT } } +// appendPatternPredicateProjectionLowerings applies projection analysis to traversal patterns nested in predicates. func appendPatternPredicateProjectionLowerings(plan *LoweringPlan, queryPartIndex int, queryPart cypher.SyntaxNode, sourceReferences map[string]struct{}) { for _, indexedPredicate := range indexedPatternPredicatesInQueryPart(queryPart) { var ( @@ -422,6 +1703,7 @@ func appendPatternPredicateProjectionLowerings(plan *LoweringPlan, queryPartInde } } +// appendPatternPredicatePlacementDecisions records existence lowering for pattern predicates in one query part. func appendPatternPredicatePlacementDecisions(plan *LoweringPlan, queryPartIndex int, queryPart cypher.SyntaxNode) { for _, indexedPredicate := range indexedPatternPredicatesInQueryPart(queryPart) { var ( @@ -462,6 +1744,7 @@ func appendPatternPredicatePlacementDecisions(plan *LoweringPlan, queryPartIndex } } +// appendLatePathMaterializationDecisions identifies path and edge values whose hydration can be deferred. func appendLatePathMaterializationDecisions(plan *LoweringPlan, queryPartIndex int, readingClauses []*cypher.ReadingClause, sourceReferences map[string]struct{}) { for clauseIndex, readingClause := range readingClauses { if readingClause == nil || readingClause.Match == nil || readingClause.Match.Optional { @@ -479,6 +1762,7 @@ func appendLatePathMaterializationDecisions(plan *LoweringPlan, queryPartIndex i } } +// appendPatternLatePathMaterializationDecisions records deferred materialization modes for one pattern's bindings. func appendPatternLatePathMaterializationDecisions(plan *LoweringPlan, target PatternTarget, patternPart *cypher.PatternPart, steps []sourceTraversalStep, sourceReferences map[string]struct{}) { pathReferenced := referencesSourceIdentifier(sourceReferences, variableSymbol(patternPart.Variable)) @@ -520,11 +1804,19 @@ func appendPatternLatePathMaterializationDecisions(plan *LoweringPlan, target Pa } } -func appendExpandIntoDecisions(plan *LoweringPlan, queryPartIndex int, readingClauses []*cypher.ReadingClause) { - declaredSymbols := map[string]struct{}{} +// appendExpandIntoDecisions records traversal steps whose left and right endpoints were already declared. +func appendExpandIntoDecisions(plan *LoweringPlan, queryPartIndex int, readingClauses []*cypher.ReadingClause, initialDeclaredSymbols map[string]struct{}) { + declaredSymbols := copyStringSet(initialDeclaredSymbols) for clauseIndex, readingClause := range readingClauses { - if readingClause == nil || readingClause.Match == nil { + if readingClause == nil { + continue + } + if readingClause.Unwind != nil { + addSymbol(declaredSymbols, variableSymbol(readingClause.Unwind.Variable)) + continue + } + if readingClause.Match == nil { continue } @@ -577,11 +1869,15 @@ func appendExpandIntoDecisions(plan *LoweringPlan, queryPartIndex int, readingCl } } +// declaredStepEndpoints snapshots visible symbols before each endpoint of a traversal step is declared. type declaredStepEndpoints struct { - BeforeLeftNode map[string]struct{} + // BeforeLeftNode contains symbols visible before the step's left endpoint declaration. + BeforeLeftNode map[string]struct{} + // BeforeRightNode contains symbols visible after the edge but before the right endpoint declaration. BeforeRightNode map[string]struct{} } +// declaredSymbolsBeforeStepEndpoints computes declaration snapshots for every traversal-step endpoint. func declaredSymbolsBeforeStepEndpoints(initial map[string]struct{}, steps []sourceTraversalStep) []declaredStepEndpoints { var ( declared = copyStringSet(initial) @@ -599,6 +1895,7 @@ func declaredSymbolsBeforeStepEndpoints(initial map[string]struct{}, steps []sou return endpoints } +// appendTraversalDirectionDecisions evaluates each step's bound endpoints and selectivity to choose its direction. func appendTraversalDirectionDecisions( plan *LoweringPlan, queryPartIndex int, @@ -668,6 +1965,7 @@ func appendTraversalDirectionDecisions( } } +// bindingPredicateSymbols returns predicate dependencies that reference declared bindings. func bindingPredicateSymbols(predicateAttachments []PredicateAttachment, queryPartIndex int) map[string]struct{} { symbols := map[string]struct{}{} @@ -684,6 +1982,7 @@ func bindingPredicateSymbols(predicateAttachments []PredicateAttachment, queryPa return symbols } +// copyBoundSourceSelectivity returns an independent copy of symbol selectivity rankings. func copyBoundSourceSelectivity(values map[string]boundSourceSelectivity) map[string]boundSourceSelectivity { copied := make(map[string]boundSourceSelectivity, len(values)) for key, value := range values { @@ -693,6 +1992,7 @@ func copyBoundSourceSelectivity(values map[string]boundSourceSelectivity) map[st return copied } +// carryProjectionSelectivity propagates source selectivity through a WITH projection and its aliases. func carryProjectionSelectivity( projection *cypher.Projection, incomingSymbols map[string]struct{}, @@ -733,6 +2033,7 @@ func carryProjectionSelectivity( return carriedSymbols, carriedSelectivity } +// projectionCarriesAllSymbols reports whether a projection uses the greedy asterisk form. func projectionCarriesAllSymbols(projection *cypher.Projection) bool { if projection == nil { return false @@ -753,6 +2054,7 @@ func projectionCarriesAllSymbols(projection *cypher.Projection) bool { return false } +// projectionCardinalitySelectivity classifies limited projections, ranking ordered or aggregate limits as top-N. func projectionCardinalitySelectivity(projection *cypher.Projection) boundSourceSelectivity { if projection == nil || projection.Limit == nil { return boundSourceSelectivityNone @@ -765,6 +2067,7 @@ func projectionCardinalitySelectivity(projection *cypher.Projection) boundSource return boundSourceSelectivityLimited } +// projectionHasAggregate reports whether any projection item contains an aggregate function. func projectionHasAggregate(projection *cypher.Projection) bool { if projection == nil { return false @@ -784,6 +2087,7 @@ func projectionHasAggregate(projection *cypher.Projection) bool { return false } +// expressionHasAggregate reports whether expression invokes a recognized aggregate function. func expressionHasAggregate(expression cypher.Expression) bool { switch typedExpression := expression.(type) { case *cypher.FunctionInvocation: @@ -793,6 +2097,7 @@ func expressionHasAggregate(expression cypher.Expression) bool { } } +// declareSelectiveMatchSymbols merges inferred node-property selectivity for a match into the symbol table. func declareSelectiveMatchSymbols(symbols map[string]boundSourceSelectivity, match *cypher.Match) { if match == nil { return @@ -826,6 +2131,7 @@ func declareSelectiveMatchSymbols(symbols map[string]boundSourceSelectivity, mat } } +// declareReadingClauseSymbols adds pattern bindings and WHERE dependencies from reading clauses. func declareReadingClauseSymbols(symbols map[string]struct{}, readingClauses []*cypher.ReadingClause) { for _, readingClause := range readingClauses { if readingClause != nil { @@ -834,6 +2140,7 @@ func declareReadingClauseSymbols(symbols map[string]struct{}, readingClauses []* } } +// declareReadingClauseSelectivity merges inferred selectivity from non-optional reading clauses. func declareReadingClauseSelectivity(symbols map[string]boundSourceSelectivity, readingClauses []*cypher.ReadingClause) { for _, readingClause := range readingClauses { if readingClause == nil || readingClause.Match == nil || readingClause.Match.Optional { @@ -844,6 +2151,7 @@ func declareReadingClauseSelectivity(symbols map[string]boundSourceSelectivity, } } +// nodePatternsForPattern returns every node pattern in chain order. func nodePatternsForPattern(patternPart *cypher.PatternPart) []*cypher.NodePattern { if patternPart == nil { return nil @@ -859,12 +2167,14 @@ func nodePatternsForPattern(patternPart *cypher.PatternPart) []*cypher.NodePatte return nodePatterns } +// mergeBoundSourceSelectivity retains the stronger selectivity rank for symbol. func mergeBoundSourceSelectivity(symbols map[string]boundSourceSelectivity, symbol string, selectivity boundSourceSelectivity) { if selectivity > symbols[symbol] { symbols[symbol] = selectivity } } +// propertyPredicateSelectivity returns the strongest property constraint on symbol in where. func propertyPredicateSelectivity(expression cypher.Expression) (string, boundSourceSelectivity, bool) { comparison, isComparison := expression.(*cypher.Comparison) if !isComparison || len(comparison.Partials) != 1 { @@ -887,6 +2197,7 @@ func propertyPredicateSelectivity(expression cypher.Expression) (string, boundSo return "", boundSourceSelectivityNone, false } +// propertyConstraintSelectivity returns the strongest selectivity inferred from constant-valued inline properties. func propertyConstraintSelectivity(expression cypher.Expression) boundSourceSelectivity { properties, ok := expression.(*cypher.Properties) if !ok || properties == nil || properties.Parameter != nil { @@ -903,6 +2214,7 @@ func propertyConstraintSelectivity(expression cypher.Expression) boundSourceSele return highest } +// propertySelectivity treats a constant objectid as unique and other constant property values as selective predicates. func propertySelectivity(property string, value cypher.Expression) boundSourceSelectivity { if strings.EqualFold(property, "objectid") && expressionIsConstant(value) { return boundSourceSelectivityUnique @@ -915,6 +2227,7 @@ func propertySelectivity(property string, value cypher.Expression) boundSourceSe return boundSourceSelectivityNone } +// expressionIsConstant reports whether expression is a non-null literal or parameter independent of row bindings. func expressionIsConstant(expression cypher.Expression) bool { switch typedExpression := expression.(type) { case *cypher.Literal: @@ -926,6 +2239,7 @@ func expressionIsConstant(expression cypher.Expression) bool { } } +// propertyLookupSymbol returns the variable whose property expression reads, when direct. func propertyLookupSymbol(expression cypher.Expression) (string, string, bool) { propertyLookup, isPropertyLookup := expression.(*cypher.PropertyLookup) if !isPropertyLookup || propertyLookup == nil { @@ -940,10 +2254,12 @@ func propertyLookupSymbol(expression cypher.Expression) (string, string, bool) { return variable.Symbol, propertyLookup.Symbol, true } +// nodePatternHasUniquePropertyConstraint reports whether node contains an inline property treated as unique. func nodePatternHasUniquePropertyConstraint(nodePattern *cypher.NodePattern) bool { return nodePattern != nil && propertyConstraintSelectivity(nodePattern.Properties) == boundSourceSelectivityUnique } +// nodePatternSelectivity ranks a node pattern from kind, inline-property, and attached-predicate constraints. func nodePatternSelectivity(nodePattern *cypher.NodePattern, hasAttachedPredicate bool) boundSourceSelectivity { if nodePattern == nil { return boundSourceSelectivityNone @@ -962,12 +2278,14 @@ func nodePatternSelectivity(nodePattern *cypher.NodePattern, hasAttachedPredicat return selectivity } +// mergeSelectivityValue raises current when next is the stronger source-selectivity rank. func mergeSelectivityValue(current *boundSourceSelectivity, next boundSourceSelectivity) { if next > *current { *current = next } } +// shortestPathSearchPredicateSymbols returns bindings constrained by search-compatible predicates in where. func shortestPathSearchPredicateSymbols(readingClauses []*cypher.ReadingClause) map[string]struct{} { symbols := map[string]struct{}{} @@ -984,6 +2302,7 @@ func shortestPathSearchPredicateSymbols(readingClauses []*cypher.ReadingClause) return symbols } +// addShortestPathSearchPredicateSymbols adds search-constrained symbols from one expression to output. func addShortestPathSearchPredicateSymbols(symbols map[string]struct{}, expression cypher.Expression) { for _, term := range cypherConjunctionTerms(expression) { if symbol, ok := shortestPathSearchPredicateSymbol(term); ok { @@ -992,6 +2311,7 @@ func addShortestPathSearchPredicateSymbols(symbols map[string]struct{}, expressi } } +// cypherConjunctionTerms flattens nested Cypher AND expressions into independent terms. func cypherConjunctionTerms(expression cypher.Expression) []cypher.Expression { if conjunction, isConjunction := expression.(*cypher.Conjunction); isConjunction { var terms []cypher.Expression @@ -1005,6 +2325,7 @@ func cypherConjunctionTerms(expression cypher.Expression) []cypher.Expression { return []cypher.Expression{expression} } +// shortestPathSearchPredicateSymbol extracts the endpoint symbol constrained by a supported search comparison. func shortestPathSearchPredicateSymbol(expression cypher.Expression) (string, bool) { comparison, isComparison := expression.(*cypher.Comparison) if !isComparison || len(comparison.Partials) != 1 { @@ -1027,6 +2348,7 @@ func shortestPathSearchPredicateSymbol(expression cypher.Expression) (string, bo return "", false } +// isEndpointSearchOperator reports whether an operator can constrain endpoint seed values. func isEndpointSearchOperator(operator cypher.Operator) bool { switch operator { case cypher.OperatorEquals, @@ -1045,6 +2367,7 @@ func isEndpointSearchOperator(operator cypher.Operator) bool { } } +// propertyLookupVariableSymbol returns the direct variable at the base of a property lookup. func propertyLookupVariableSymbol(expression cypher.Expression) (string, bool) { propertyLookup, isPropertyLookup := expression.(*cypher.PropertyLookup) if !isPropertyLookup || propertyLookup == nil { @@ -1059,11 +2382,13 @@ func propertyLookupVariableSymbol(expression cypher.Expression) (string, bool) { return variable.Symbol, true } +// expressionReferencesAnySource reports whether expression depends on a variable or property binding. func expressionReferencesAnySource(expression cypher.Expression) bool { references, err := collectReferencedSourceIdentifiers(expression) return err != nil || len(references) > 0 } +// traversalDirectionDecisionForStep chooses whether to reverse a step based on bound endpoints and estimated selectivity. func traversalDirectionDecisionForStep( target TraversalStepTarget, stepIndex int, @@ -1116,6 +2441,7 @@ func traversalDirectionDecisionForStep( return TraversalDirectionDecision{}, false } +// boundLeftExpansionDirectionDecisionForStep preserves a bound-left expansion unless terminal evidence justifies reversal. func boundLeftExpansionDirectionDecisionForStep( target TraversalStepTarget, patternPart *cypher.PatternPart, @@ -1182,6 +2508,7 @@ func boundLeftExpansionDirectionDecisionForStep( }, true } +// appendShortestPathStrategyDecisions records bidirectional search when endpoint evidence supports it. func appendShortestPathStrategyDecisions(plan *LoweringPlan, queryPartIndex int, readingClauses []*cypher.ReadingClause, predicateConstrainedSymbols map[string]struct{}) { declaredSymbols := map[string]struct{}{} @@ -1234,6 +2561,7 @@ func appendShortestPathStrategyDecisions(plan *LoweringPlan, queryPartIndex int, } } +// shortestPathStrategyDecisionForStep chooses bidirectional search when both endpoints provide usable evidence. func shortestPathStrategyDecisionForStep( target TraversalStepTarget, step sourceTraversalStep, @@ -1266,6 +2594,7 @@ func shortestPathStrategyDecisionForStep( return ShortestPathStrategyDecision{}, false } +// endpointHasSearchConstraint reports whether endpoint has an inline property or attached predicate constraint. func endpointHasSearchConstraint(nodePattern *cypher.NodePattern, symbol string, predicateConstrainedSymbols map[string]struct{}) bool { if nodePattern == nil { return false @@ -1274,6 +2603,7 @@ func endpointHasSearchConstraint(nodePattern *cypher.NodePattern, symbol string, return nodePattern.Properties != nil || referencesSourceIdentifier(predicateConstrainedSymbols, symbol) } +// endpointHasTerminalFilterConstraint reports whether endpoint has a kind, property, or attached predicate constraint useful as a terminal filter. func endpointHasTerminalFilterConstraint(nodePattern *cypher.NodePattern, symbol string, predicateConstrainedSymbols map[string]struct{}) bool { if nodePattern == nil { return false @@ -1282,6 +2612,7 @@ func endpointHasTerminalFilterConstraint(nodePattern *cypher.NodePattern, symbol return nodePatternHasConstraints(nodePattern) || referencesSourceIdentifier(predicateConstrainedSymbols, symbol) } +// appendShortestPathFilterDecisions records terminal and endpoint-pair filters worth materializing for shortest paths. func appendShortestPathFilterDecisions(plan *LoweringPlan, queryPartIndex int, readingClauses []*cypher.ReadingClause, predicateConstrainedSymbols map[string]struct{}) { declaredSymbols := map[string]struct{}{} @@ -1335,6 +2666,7 @@ func appendShortestPathFilterDecisions(plan *LoweringPlan, queryPartIndex int, r } } +// shortestPathFilterDecisionForStep chooses an endpoint-pair, terminal, or no filter for one shortest-path step. func shortestPathFilterDecisionForStep( plan *LoweringPlan, target TraversalStepTarget, @@ -1377,6 +2709,7 @@ func shortestPathFilterDecisionForStep( }, true } +// hasShortestPathBidirectionalStrategy reports whether target is planned for bidirectional shortest-path search. func hasShortestPathBidirectionalStrategy(plan *LoweringPlan, target TraversalStepTarget) bool { if plan == nil { return false @@ -1391,6 +2724,7 @@ func hasShortestPathBidirectionalStrategy(plan *LoweringPlan, target TraversalSt return false } +// appendLimitPushdownDecisions records a final literal limit that can safely bound traversal work. func appendLimitPushdownDecisions(plan *LoweringPlan, queryPartIndex int, queryPart cypher.SyntaxNode, readingClauses []*cypher.ReadingClause) { if !queryPartAllowsLimitPushdown(queryPart, readingClauses) { return @@ -1430,6 +2764,7 @@ func appendLimitPushdownDecisions(plan *LoweringPlan, queryPartIndex int, queryP } } +// queryPartAllowsLimitPushdown reports whether one reading clause with an unordered, non-distinct LIMIT and no SKIP or updates permits early limiting. func queryPartAllowsLimitPushdown(queryPart cypher.SyntaxNode, readingClauses []*cypher.ReadingClause) bool { projection, updatingClauseCount := queryPartProjection(queryPart) if projection == nil || @@ -1445,6 +2780,7 @@ func queryPartAllowsLimitPushdown(queryPart cypher.SyntaxNode, readingClauses [] return true } +// queryPartProjection returns a query part's terminal projection and number of updating clauses. func queryPartProjection(queryPart cypher.SyntaxNode) (*cypher.Projection, int) { switch typedQueryPart := queryPart.(type) { case *cypher.SinglePartQuery: @@ -1466,7 +2802,22 @@ func queryPartProjection(queryPart cypher.SyntaxNode) (*cypher.Projection, int) } } -func appendExpansionSuffixPushdownDecisions(plan *LoweringPlan, queryPartIndex int, readingClauses []*cypher.ReadingClause) { +// suffixBindingsObserved reports whether downstream syntax consumes a binding introduced in the fixed suffix. +func suffixBindingsObserved(patternPart *cypher.PatternPart, steps []sourceTraversalStep, references map[string]struct{}) bool { + if patternPart != nil && patternPart.Variable != nil && referencesSourceIdentifier(references, patternPart.Variable.Symbol) { + return true + } + for _, step := range steps { + if (step.Relationship != nil && step.Relationship.Variable != nil && referencesSourceIdentifier(references, step.Relationship.Variable.Symbol)) || + (step.RightNode != nil && step.RightNode.Variable != nil && referencesSourceIdentifier(references, step.RightNode.Variable.Symbol)) { + return true + } + } + return false +} + +// appendExpansionSuffixPushdownDecisions records fixed-suffix candidates evaluated for supplemental filtering. +func appendExpansionSuffixPushdownDecisions(plan *LoweringPlan, queryPartIndex int, readingClauses []*cypher.ReadingClause, sourceReferences map[string]struct{}) { declaredSymbols := map[string]struct{}{} for clauseIndex, readingClause := range readingClauses { @@ -1505,11 +2856,22 @@ func appendExpansionSuffixPushdownDecisions(plan *LoweringPlan, queryPartIndex i } if suffixLength := expansionSuffixPushdownLength(steps[stepIndex+1:]); suffixLength > 0 { + suffixSteps := steps[stepIndex+1 : stepIndex+1+suffixLength] + // Start with the measured fixed-suffix shape: an observed immediate + // continuation of three or more fixed hops. Shorter suffixes retain + // the established prefilter until their own decoy-density A/B exists. + observed := suffixLength >= 3 && suffixBindingsObserved(patternPart, suffixSteps, sourceReferences) + reason := "supplemental suffix prefilter retained for unobserved continuation" + if observed { + reason = "immediate observed continuation produces suffix rows" + } plan.ExpansionSuffixPushdown = append(plan.ExpansionSuffixPushdown, ExpansionSuffixPushdownDecision{ - Target: target, - SuffixLength: suffixLength, - SuffixStartStep: stepIndex + 1, - SuffixEndStep: stepIndex + suffixLength, + Target: target, + SuffixLength: suffixLength, + SuffixStartStep: stepIndex + 1, + SuffixEndStep: stepIndex + suffixLength, + ApplySupplemental: !observed, + Reason: reason, }) } } @@ -1521,11 +2883,13 @@ func appendExpansionSuffixPushdownDecisions(plan *LoweringPlan, queryPartIndex i } } +// expansionStepMayFlipForConstraintBalance reports whether reversal can move stronger constraints to the expansion root. func expansionStepMayFlipForConstraintBalance(stepIndex int, step sourceTraversalStep, declaredEndpoints declaredStepEndpoints) bool { _, mayFlip := traversalDirectionDecisionForStep(TraversalStepTarget{}, stepIndex, step, declaredEndpoints, false, false) return mayFlip } +// leftEndpointBoundForStep reports whether the left endpoint is available from prior scope or a preceding step. func leftEndpointBoundForStep(stepIndex int, step sourceTraversalStep, declaredEndpoints declaredStepEndpoints) bool { leftSymbol := variableSymbol(step.LeftNode.Variable) if leftSymbol == "" { @@ -1536,6 +2900,7 @@ func leftEndpointBoundForStep(stepIndex int, step sourceTraversalStep, declaredE return leftBound } +// hasTraversalDirectionFlip reports whether target has a planned logical direction reversal. func hasTraversalDirectionFlip(plan *LoweringPlan, target TraversalStepTarget) bool { if plan == nil { return false @@ -1550,11 +2915,15 @@ func hasTraversalDirectionFlip(plan *LoweringPlan, target TraversalStepTarget) b return false } +// bindingTargetKey uniquely identifies a binding within one query part. type bindingTargetKey struct { + // QueryPartIndex identifies the query part that owns the binding. QueryPartIndex int - Symbol string + // Symbol is the binding's Cypher variable name. + Symbol string } +// appendPredicatePlacementDecisions records the earliest traversal scope where each attached predicate is evaluable. func appendPredicatePlacementDecisions(plan *LoweringPlan, query *cypher.RegularQuery, predicateAttachments []PredicateAttachment) { if len(predicateAttachments) == 0 { return @@ -1585,6 +2954,7 @@ func appendPredicatePlacementDecisions(plan *LoweringPlan, query *cypher.Regular } } +// attachPredicatePlacementsToSuffixPushdowns copies relevant predicate attachments into each suffix-pushdown decision. func attachPredicatePlacementsToSuffixPushdowns(plan *LoweringPlan) { for suffixIdx := range plan.ExpansionSuffixPushdown { suffix := &plan.ExpansionSuffixPushdown[suffixIdx] @@ -1603,12 +2973,14 @@ func attachPredicatePlacementsToSuffixPushdowns(plan *LoweringPlan) { } } +// appendCountStoreFastPathDecisions records a single-part query answerable directly from node or relationship counts. func appendCountStoreFastPathDecisions(plan *LoweringPlan, query *cypher.RegularQuery) { if decision, ok := countStoreFastPathDecision(query); ok { plan.CountStoreFastPath = append(plan.CountStoreFastPath, decision) } } +// appendAggregateTraversalCountDecisions records variable traversals lowered to grouped aggregate counts. func appendAggregateTraversalCountDecisions(plan *LoweringPlan, query *cypher.RegularQuery) { if shape, ok := AggregateTraversalCountShapeForQuery(query); ok { plan.AggregateTraversalCount = append(plan.AggregateTraversalCount, AggregateTraversalCountDecision{ @@ -1688,6 +3060,7 @@ func AggregateTraversalCountShapeForQuery(query *cypher.RegularQuery) (Aggregate }, true } +// aggregateTraversalSourceMatch returns the match that establishes a traversal count's source binding. func aggregateTraversalSourceMatch(readingClause *cypher.ReadingClause) (*cypher.Match, *cypher.NodePattern, string, bool) { if readingClause == nil || readingClause.Match == nil { return nil, nil, "", false @@ -1713,6 +3086,7 @@ func aggregateTraversalSourceMatch(readingClause *cypher.ReadingClause) (*cypher return match, nodePattern, nodePattern.Variable.Symbol, true } +// aggregateTraversalMatch returns the single variable-length match eligible for aggregate counting. func aggregateTraversalMatch(readingClause *cypher.ReadingClause, sourceSymbol string) (*cypher.Match, *cypher.RelationshipPattern, *cypher.NodePattern, string, bool) { if readingClause == nil || readingClause.Match == nil { return nil, nil, nil, "", false @@ -1756,6 +3130,7 @@ func aggregateTraversalMatch(readingClause *cypher.ReadingClause, sourceSymbol s return match, relationship, rightNode, rightNode.Variable.Symbol, true } +// aggregateTraversalWithProjection validates the WITH projection and returns its count alias. func aggregateTraversalWithProjection(projection *cypher.Projection, sourceSymbol, terminalSymbol string) (string, bool) { if projection == nil || projection.All || projection.Order != nil || projection.Skip != nil || projection.Limit != nil || len(projection.Items) != 2 { return "", false @@ -1773,13 +3148,19 @@ func aggregateTraversalWithProjection(projection *cypher.Projection, sourceSymbo return countAlias, true } +// aggregateTraversalFinalProjectionShape describes the source and count columns required from the final projection. type aggregateTraversalFinalProjectionShape struct { + // SourceAlias is the output name of the traversal's source binding. SourceAlias string - CountAlias string + // CountAlias is the output name of the aggregate count binding. + CountAlias string + // ReturnCount reports whether the final projection includes the count binding. ReturnCount bool - Limit int64 + // Limit is the descending top-count bound applied by the final projection. + Limit int64 } +// aggregateTraversalFinalProjection validates the terminal projection and returns its aggregate-count output shape. func aggregateTraversalFinalProjection(queryPart *cypher.SinglePartQuery, sourceSymbol, countAlias string) (aggregateTraversalFinalProjectionShape, bool) { if queryPart == nil || len(queryPart.ReadingClauses) > 0 || len(queryPart.UpdatingClauses) > 0 || queryPart.Return == nil || queryPart.Return.Projection == nil { return aggregateTraversalFinalProjectionShape{}, false @@ -1843,6 +3224,7 @@ func aggregateTraversalFinalProjection(queryPart *cypher.SinglePartQuery, source return finalProjection, true } +// aggregateTraversalDepthBounds returns finite minimum and maximum depths for a countable relationship range. func aggregateTraversalDepthBounds(patternRange *cypher.PatternRange) (int64, int64, bool) { if patternRange == nil { return 0, 0, false @@ -1867,6 +3249,7 @@ func aggregateTraversalDepthBounds(patternRange *cypher.PatternRange) (int64, in return minDepth, maxDepth, true } +// projectionItemVariableSymbol returns the direct variable projected by item. func projectionItemVariableSymbol(expression cypher.Expression) (string, bool) { projectionItem, ok := expression.(*cypher.ProjectionItem) if !ok || projectionItem == nil || projectionItem.Alias != nil { @@ -1876,6 +3259,7 @@ func projectionItemVariableSymbol(expression cypher.Expression) (string, bool) { return expressionVariableSymbol(projectionItem.Expression) } +// projectionItemVariableSymbolAndAlias returns a projected variable and its effective output name. func projectionItemVariableSymbolAndAlias(expression cypher.Expression) (string, string, bool) { projectionItem, ok := expression.(*cypher.ProjectionItem) if !ok || projectionItem == nil { @@ -1899,6 +3283,7 @@ func projectionItemVariableSymbolAndAlias(expression cypher.Expression) (string, return symbol, alias, true } +// expressionVariableSymbol returns expression's direct variable symbol without following compound syntax. func expressionVariableSymbol(expression cypher.Expression) (string, bool) { variable, ok := expression.(*cypher.Variable) if !ok || variable == nil || variable.Symbol == "" { @@ -1908,6 +3293,7 @@ func expressionVariableSymbol(expression cypher.Expression) (string, bool) { return variable.Symbol, true } +// projectionItemCountAlias returns the alias of a supported count expression. func projectionItemCountAlias(expression cypher.Expression, terminalSymbol string) (string, bool) { projectionItem, ok := expression.(*cypher.ProjectionItem) if !ok || projectionItem == nil || projectionItem.Alias == nil || projectionItem.Alias.Symbol == "" { @@ -1927,6 +3313,7 @@ func projectionItemCountAlias(expression cypher.Expression, terminalSymbol strin return projectionItem.Alias.Symbol, true } +// aggregateTraversalCountArgumentMatches reports whether count observes the expected terminal binding or all rows. func aggregateTraversalCountArgumentMatches(expression cypher.Expression, terminalSymbol string) bool { if symbol, ok := expressionVariableSymbol(expression); ok { return symbol == terminalSymbol @@ -1936,6 +3323,7 @@ func aggregateTraversalCountArgumentMatches(expression cypher.Expression, termin return ok && rangeQuantifier != nil && rangeQuantifier.Value == cypher.TokenLiteralAsterisk } +// literalInt64 converts a non-negative integer literal to int64 when its value is representable. func literalInt64(expression cypher.Expression) (int64, bool) { literal, ok := expression.(*cypher.Literal) if !ok || literal == nil || literal.Null { @@ -1958,6 +3346,7 @@ func literalInt64(expression cypher.Expression) (int64, bool) { } } +// countStoreFastPathDecision recognizes a count query answerable from node or edge statistics. func countStoreFastPathDecision(query *cypher.RegularQuery) (CountStoreFastPathDecision, bool) { if query == nil || query.SingleQuery == nil || query.SingleQuery.SinglePartQuery == nil { return CountStoreFastPathDecision{}, false @@ -2041,6 +3430,7 @@ func countStoreFastPathDecision(query *cypher.RegularQuery) (CountStoreFastPathD }, true } +// simpleCountProjectionArgument extracts the direct variable or wildcard consumed by a lone count projection. func simpleCountProjectionArgument(returnClause *cypher.Return) (string, bool) { if returnClause == nil || returnClause.Projection == nil { return "", false @@ -2078,10 +3468,12 @@ func simpleCountProjectionArgument(returnClause *cypher.Return) (string, bool) { return "", false } +// constrainedCountFastPathEndpoint reports whether a node endpoint has constraints incompatible with count-store lookup. func constrainedCountFastPathEndpoint(nodePattern *cypher.NodePattern) bool { return nodePattern == nil || nodePattern.Variable != nil || len(nodePattern.Kinds) > 0 || nodePattern.Properties != nil } +// kindSymbols returns the string names of all non-nil kinds in declaration order. func kindSymbols(kinds graph.Kinds) []string { if len(kinds) == 0 { return nil @@ -2095,6 +3487,7 @@ func kindSymbols(kinds graph.Kinds) []string { return symbols } +// indexBindingTargets maps traversal-step node and relationship bindings to their first query-part target coordinates. func indexBindingTargets(query *cypher.RegularQuery) map[bindingTargetKey]TraversalStepTarget { targets := map[bindingTargetKey]TraversalStepTarget{} @@ -2121,6 +3514,7 @@ func indexBindingTargets(query *cypher.RegularQuery) map[bindingTargetKey]Traver return targets } +// indexReadingClauseBindingTargets adds first targets for traversal-step node and relationship bindings in readingClauses. func indexReadingClauseBindingTargets(targets map[bindingTargetKey]TraversalStepTarget, queryPartIndex int, readingClauses []*cypher.ReadingClause) { for clauseIndex, readingClause := range readingClauses { if readingClause == nil || readingClause.Match == nil { @@ -2144,6 +3538,7 @@ func indexReadingClauseBindingTargets(targets map[bindingTargetKey]TraversalStep } } +// setBindingTarget records target for a non-empty binding symbol without overwriting its first declaration. func setBindingTarget(targets map[bindingTargetKey]TraversalStepTarget, queryPartIndex int, symbol string, target TraversalStepTarget) { if symbol == "" { return @@ -2158,6 +3553,7 @@ func setBindingTarget(targets map[bindingTargetKey]TraversalStepTarget, queryPar } } +// expansionSuffixPushdownLength counts fixed directed steps following a variable expansion. func expansionSuffixPushdownLength(suffixSteps []sourceTraversalStep) int { var suffixLength int @@ -2172,6 +3568,7 @@ func expansionSuffixPushdownLength(suffixSteps []sourceTraversalStep) int { return suffixLength } +// declareMatchSymbols adds pattern bindings and WHERE dependencies from match to declared. func declareMatchSymbols(declared map[string]struct{}, match *cypher.Match) { if match == nil { return @@ -2184,6 +3581,7 @@ func declareMatchSymbols(declared map[string]struct{}, match *cypher.Match) { declareWhereSymbols(declared, match) } +// declarePatternSymbols adds path, node, and relationship bindings introduced by a pattern part. func declarePatternSymbols(declared map[string]struct{}, patternPart *cypher.PatternPart) { if patternPart == nil { return @@ -2203,26 +3601,31 @@ func declarePatternSymbols(declared map[string]struct{}, patternPart *cypher.Pat } } +// declareWhereSymbols adds variable dependencies referenced by a match predicate. func declareWhereSymbols(declared map[string]struct{}, match *cypher.Match) { for _, dependency := range dependenciesForMatch(match) { addSymbol(declared, dependency) } } +// nodePatternHasConstraints reports whether a node pattern declares kinds or inline properties. func nodePatternHasConstraints(nodePattern *cypher.NodePattern) bool { return nodePattern != nil && (len(nodePattern.Kinds) > 0 || nodePattern.Properties != nil) } +// relationshipPatternHasProperties reports whether a relationship pattern declares inline properties. func relationshipPatternHasProperties(relationshipPattern *cypher.RelationshipPattern) bool { return relationshipPattern != nil && relationshipPattern.Properties != nil } +// addSymbol inserts a non-empty symbol into a declaration set. func addSymbol(symbols map[string]struct{}, symbol string) { if symbol != "" { symbols[symbol] = struct{}{} } } +// copyStringSet returns an independent copy of a string membership set. func copyStringSet(values map[string]struct{}) map[string]struct{} { copied := make(map[string]struct{}, len(values)) for value := range values { @@ -2232,6 +3635,7 @@ func copyStringSet(values map[string]struct{}) map[string]struct{} { return copied } +// traversalStepsForPattern converts a pattern chain into ordered left-edge-right traversal steps. func traversalStepsForPattern(patternPart *cypher.PatternPart) []sourceTraversalStep { if patternPart == nil { return nil @@ -2272,6 +3676,7 @@ func traversalStepsForPattern(patternPart *cypher.PatternPart) []sourceTraversal return steps } +// variableSymbol returns variable's symbol or an empty string for a missing variable. func variableSymbol(variable *cypher.Variable) string { if variable == nil { return "" diff --git a/cypher/models/pgsql/optimize/optimizer_test.go b/cypher/models/pgsql/optimize/optimizer_test.go index 33848399..93883380 100644 --- a/cypher/models/pgsql/optimize/optimizer_test.go +++ b/cypher/models/pgsql/optimize/optimizer_test.go @@ -1,16 +1,21 @@ package optimize import ( + "encoding/json" + "fmt" "testing" "github.com/specterops/dawgs/cypher/frontend" "github.com/specterops/dawgs/cypher/models" "github.com/specterops/dawgs/cypher/models/cypher" "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/specterops/dawgs/graph" "github.com/stretchr/testify/require" ) +// testRule is a configurable optimizer rule used to assert rewrite ordering and error propagation. type testRule struct { + // name is the stable rule name returned to the optimizer. name string } @@ -22,6 +27,7 @@ func (s testRule) Apply(plan *Plan) (bool, error) { return false, nil } +// testBindingLookup supplies deterministic binding resolution to optimizer tests. type testBindingLookup map[pgsql.Identifier]pgsql.DataType func (s testBindingLookup) LookupDataType(identifier pgsql.Identifier) (pgsql.DataType, bool) { @@ -29,10 +35,11 @@ func (s testBindingLookup) LookupDataType(identifier pgsql.Identifier) (pgsql.Da return dataType, found } +// TestOptimizeCopiesAndAnalyzesQuery verifies that optimization preserves the input AST and records query-part metadata. func TestOptimizeCopiesAndAnalyzesQuery(t *testing.T) { t.Parallel() - regularQuery, err := frontend.ParseCypher(frontend.NewContext(), adcsQuery) + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), fixedSuffixExpansionQuery) require.NoError(t, err) plan, err := Optimize(regularQuery) @@ -48,23 +55,105 @@ func TestOptimizeCopiesAndAnalyzesQuery(t *testing.T) { require.Len(t, plan.PredicateAttachments, 2) } -func TestOptimizePlansADCSFanoutRewrite(t *testing.T) { +// TestFieldRequirementAnalysisDistinguishesObservationBoundaries verifies that each consumer requests only the binding fields it observes. +func TestFieldRequirementAnalysisDistinguishesObservationBoundaries(t *testing.T) { t.Parallel() - regularQuery, err := frontend.ParseCypher(frontend.NewContext(), adcsQuery) + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = (n:Group)-[r:MemberOf*1..]->(ca:EnterpriseCA) + WHERE n.objectid = 'source' + RETURN id(ca), labels(n), length(p) + `) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Contains(t, plan.LoweringPlan.Decisions(), LoweringDecision{Name: LoweringFieldRequirements}) + + bySymbol := map[string]FieldRequirementDecision{} + for _, decision := range plan.LoweringPlan.FieldRequirements { + bySymbol[decision.Symbol] = decision + } + + require.Contains(t, bySymbol["ca"].Fields, FieldRequirementEntityID) + require.NotContains(t, bySymbol["ca"].Fields, FieldRequirementFullEntity) + require.Contains(t, bySymbol["n"].Fields, FieldRequirementKinds) + require.Contains(t, bySymbol["n"].Fields, FieldRequirementProperties) + require.Contains(t, bySymbol["p"].Fields, FieldRequirementOrderedPathEdgeIDs) + require.NotContains(t, bySymbol["p"].Fields, FieldRequirementFullPath) +} + +// TestFieldRequirementAnalysisExpandsGreedyProjection verifies that RETURN * requires complete representations of visible bindings. +func TestFieldRequirementAnalysisExpandsGreedyProjection(t *testing.T) { + t.Parallel() + + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[r:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN * + `) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + + bySymbol := map[string]FieldRequirementDecision{} + for _, decision := range plan.LoweringPlan.FieldRequirements { + bySymbol[decision.Symbol] = decision + } + + require.NotContains(t, bySymbol, cypher.TokenLiteralAsterisk) + require.Contains(t, bySymbol["p"].Fields, FieldRequirementFullPath) + require.Contains(t, bySymbol["s"].Fields, FieldRequirementFullEntity) + require.Contains(t, bySymbol["e"].Fields, FieldRequirementFullEntity) + require.Contains(t, bySymbol["r"].Fields, FieldRequirementFullEntity) + require.Contains(t, bySymbol["r"].Fields, FieldRequirementRelationshipIDs) + require.Len(t, plan.LoweringPlan.ShortestPathExecutor, 1) + require.Equal(t, ShortestPathObservationOnePath, plan.LoweringPlan.ShortestPathExecutor[0].ObservationMode) +} + +// TestFieldRequirementAnalysisTreatsWithGreedyProjectionAsFullObservation verifies that WITH * prevents scalar-only path state. +func TestFieldRequirementAnalysisTreatsWithGreedyProjectionAsFullObservation(t *testing.T) { + t.Parallel() + + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + WITH * + RETURN length(p) + `) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + + for _, decision := range plan.LoweringPlan.FieldRequirements { + if decision.QueryPartIndex == 0 && decision.Symbol == "p" { + require.Contains(t, decision.Fields, FieldRequirementFullPath) + return + } + } + require.Fail(t, "missing path field-requirement decision") +} + +// TestOptimizePlansFixedSuffixFanoutRewrite verifies that an eligible terminal suffix receives supplemental pushdown metadata. +func TestOptimizePlansFixedSuffixFanoutRewrite(t *testing.T) { + t.Parallel() + + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), fixedSuffixExpansionQuery) require.NoError(t, err) plan, err := Optimize(regularQuery) require.NoError(t, err) - ctPredicate := PredicateAttachment{ + predicateAttachment := PredicateAttachment{ QueryPartIndex: 0, RegionIndex: 0, ClauseIndex: 2, ExpressionIndex: 0, Scope: PredicateAttachmentScopeBinding, - BindingSymbols: []string{"ct"}, - Dependencies: []string{"ct"}, + BindingSymbols: []string{"predicate"}, + Dependencies: []string{"predicate"}, } require.Contains(t, plan.LoweringPlan.Decisions(), LoweringDecision{Name: LoweringExpansionSuffixPushdown}) @@ -82,6 +171,7 @@ func TestOptimizePlansADCSFanoutRewrite(t *testing.T) { SuffixLength: 3, SuffixStartStep: 1, SuffixEndStep: 3, + Reason: "immediate observed continuation produces suffix rows", }) require.Contains(t, plan.LoweringPlan.ExpansionSuffixPushdown, ExpansionSuffixPushdownDecision{ Target: TraversalStepTarget{ @@ -93,7 +183,9 @@ func TestOptimizePlansADCSFanoutRewrite(t *testing.T) { SuffixLength: 2, SuffixStartStep: 1, SuffixEndStep: 2, - PredicateAttachments: []PredicateAttachment{ctPredicate}, + ApplySupplemental: true, + Reason: "supplemental suffix prefilter retained for unobserved continuation", + PredicateAttachments: []PredicateAttachment{predicateAttachment}, }) require.Contains(t, plan.LoweringPlan.ExpansionSuffixPushdown, ExpansionSuffixPushdownDecision{ Target: TraversalStepTarget{ @@ -102,9 +194,11 @@ func TestOptimizePlansADCSFanoutRewrite(t *testing.T) { PatternIndex: 0, StepIndex: 3, }, - SuffixLength: 1, - SuffixStartStep: 4, - SuffixEndStep: 4, + SuffixLength: 1, + SuffixStartStep: 4, + SuffixEndStep: 4, + ApplySupplemental: true, + Reason: "supplemental suffix prefilter retained for unobserved continuation", }) require.Contains(t, plan.LoweringPlan.ExpandInto, ExpandIntoDecision{ @@ -130,7 +224,7 @@ func TestOptimizePlansADCSFanoutRewrite(t *testing.T) { PatternIndex: 0, StepIndex: 1, }, - Attachment: ctPredicate, + Attachment: predicateAttachment, Placement: PredicateAttachmentScopeBinding, }) } @@ -163,6 +257,7 @@ func TestDefaultPredicateAttachmentRuleReportsSkippedWhenNoPredicatesExist(t *te require.Empty(t, plan.PredicateAttachments) } +// TestLoweringPlanReportsProjectionPruning verifies that unused traversal bindings produce explicit pruning decisions. func TestLoweringPlanReportsProjectionPruning(t *testing.T) { t.Parallel() @@ -174,7 +269,10 @@ func TestLoweringPlanReportsProjectionPruning(t *testing.T) { plan, err := Optimize(regularQuery) require.NoError(t, err) - require.Equal(t, []LoweringDecision{{Name: LoweringProjectionPruning}}, plan.LoweringPlan.Decisions()) + require.Equal(t, []LoweringDecision{ + {Name: LoweringProjectionPruning}, + {Name: LoweringFieldRequirements}, + }, plan.LoweringPlan.Decisions()) require.Equal(t, []ProjectionPruningDecision{{ Target: TraversalStepTarget{ QueryPartIndex: 0, @@ -484,6 +582,7 @@ func TestLoweringPlanReportsExactTwoHopRangeExpansion(t *testing.T) { }}, plan.LoweringPlan.ExactRangeExpansion) } +// TestExactRangeDependentPlanningRequiresDecision verifies that downstream planning changes only after exact-range expansion is selected. func TestExactRangeDependentPlanningRequiresDecision(t *testing.T) { t.Parallel() @@ -522,12 +621,14 @@ func TestExactRangeDependentPlanningRequiresDecision(t *testing.T) { Mode: LatePathMaterializationExpansionPath, }) - appendExpansionSuffixPushdownDecisions(&plan, 0, readingClauses) + appendExpansionSuffixPushdownDecisions(&plan, 0, readingClauses, nil) require.Contains(t, plan.ExpansionSuffixPushdown, ExpansionSuffixPushdownDecision{ - Target: target.TraversalStep(0), - SuffixLength: 1, - SuffixStartStep: 1, - SuffixEndStep: 1, + Target: target.TraversalStep(0), + SuffixLength: 1, + SuffixStartStep: 1, + SuffixEndStep: 1, + ApplySupplemental: true, + Reason: "supplemental suffix prefilter retained for unobserved continuation", }) }) @@ -550,7 +651,7 @@ func TestExactRangeDependentPlanningRequiresDecision(t *testing.T) { Mode: LatePathMaterializationPathEdgeID, }) - appendExpansionSuffixPushdownDecisions(&plan, 0, readingClauses) + appendExpansionSuffixPushdownDecisions(&plan, 0, readingClauses, nil) require.Empty(t, plan.ExpansionSuffixPushdown) }) } @@ -719,12 +820,13 @@ func TestLoweringPlanSkipsPathRelationshipPredicateAfterWithProjection(t *testin require.Empty(t, plan.LoweringPlan.PathRelationshipPredicate) } +// TestLoweringPlanReportsExpansionSuffixPushdown verifies that an eligible fixed suffix produces a supplemental-search decision. func TestLoweringPlanReportsExpansionSuffixPushdown(t *testing.T) { t.Parallel() regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` - MATCH p = (n:Group)-[:MemberOf*0..]->(m)-[:Enroll]->(ca:EnterpriseCA) - RETURN p + MATCH path = (root:ExpansionRoot)-[:Expand*0..16]->(boundary:ExpansionNode)-[:EnterSuffix]->(head:SuffixHead) + RETURN path `) require.NoError(t, err) @@ -738,22 +840,399 @@ func TestLoweringPlanReportsExpansionSuffixPushdown(t *testing.T) { PatternIndex: 0, StepIndex: 0, }, - SuffixLength: 1, - SuffixStartStep: 1, - SuffixEndStep: 1, + SuffixLength: 1, + SuffixStartStep: 1, + SuffixEndStep: 1, + ApplySupplemental: true, + Reason: "supplemental suffix prefilter retained for unobserved continuation", }}, plan.LoweringPlan.ExpansionSuffixPushdown) } -func TestLoweringPlanIncludesConstrainedBoundEndpointInExpansionSuffix(t *testing.T) { +// TestLoweringPlanReportsConservativeFixedSuffixSearchStrategy verifies that eligible suffix topology remains on the incumbent strategy unless qualified. +func TestLoweringPlanReportsConservativeFixedSuffixSearchStrategy(t *testing.T) { t.Parallel() regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` - MATCH (ca) - MATCH p = (n:Group)-[:MemberOf*0..]->(m)-[:Enroll]->(ct:CertTemplate)-[:PublishedTo]->(ca:EnterpriseCA) + MATCH (root:ExpansionRoot) + WHERE root.root_key = $root_key + MATCH path = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) + RETURN path + `) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Contains(t, plan.LoweringPlan.Decisions(), LoweringDecision{Name: LoweringExpansionSearchStrategy}) + require.Len(t, plan.LoweringPlan.ExpansionSearchStrategy, 1) + decision := plan.LoweringPlan.ExpansionSearchStrategy[0] + require.Equal(t, "fixed_suffix_expansion", decision.Family) + require.Equal(t, ExpansionSearchPolicyOrientationProbeV1, decision.PlannedPolicy) + require.Empty(t, decision.EmittedPolicy) + require.Equal(t, "incumbent_default", decision.SelectionMode) + require.Equal(t, "fixed-suffix-static-v1", decision.SelectorVersion) + require.Equal(t, []ExpansionSearchStrategy{ + ExpansionSearchStepwiseForward, + ExpansionSearchLateHydratedForward, + ExpansionSearchFactoredSuffixForward, + ExpansionSearchSuffixSeededReverse, + ExpansionSearchBackwardViabilityForward, + }, decision.PlannedCandidates) + require.Equal(t, []ExpansionSearchStrategy{ExpansionSearchStepwiseForward}, decision.EmittedCandidates) + require.Equal(t, ExpansionSearchExecutionBoundaryInlineStatement, decision.ExecutionBoundary) + require.Equal(t, ExpansionSearchProbeCaps{ + RootRowLimit: ExpansionSearchOrientationRootRowLimit, + ReverseSeedRowLimit: ExpansionSearchOrientationReverseSeedRowLimit, + DirectionalDegreeRowLimit: ExpansionSearchOrientationDirectionalDegreeRowLimit, + }, decision.ProbeCaps) + require.Equal(t, ExpansionSearchAdmission{ + StateLimit: ExpansionSearchOrientationStateLimit, + RequiresCompleteProbes: true, + FallbackStrategy: ExpansionSearchStepwiseForward, + }, decision.Admission) + require.True(t, decision.StructurallyEligible) + require.Contains(t, decision.EligibilityFacts, ExpansionSearchEligibilityFact{ + Name: "qualified_fixed_suffix_topology", + Eligible: true, + }) + require.Equal(t, ExpansionSearchStepwiseForward, decision.SelectedStrategy) + require.Equal(t, ExpansionSearchStepwiseForward, decision.FallbackStrategy) + require.Equal(t, ExpansionSearchFallbackTournamentUnqualified, decision.FallbackReason) + require.Equal(t, ExpansionSearchObservationFullPath, decision.ObservationMode) + require.Equal(t, int64(0), decision.MinimumDepth) + require.Equal(t, int64(16), decision.MaximumDepth) + require.Equal(t, 3, decision.SuffixLength) + require.Equal(t, "outbound", decision.LogicalDirection) +} + +// TestLoweringPlanSelectsGuardedEndpointSeededExpansion verifies guarded endpoint seeding for one statement-wide variable expansion. +func TestLoweringPlanSelectsGuardedEndpointSeededExpansion(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = (c:Computer)-[:HasSession]->(:User)-[:MemberOf*1..]->(g:Group) + WHERE g.objectid ENDS WITH $suffix + RETURN p + LIMIT 1000 + `) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ExpansionSearchStrategy, 1) + decision := plan.LoweringPlan.ExpansionSearchStrategy[0] + require.Equal(t, "fixed_prefix_terminal_expansion", decision.Family) + require.Equal(t, ExpansionSearchPolicyEndpointGuardV1, decision.PlannedPolicy) + require.Equal(t, ExpansionSearchPolicyEndpointGuardV1, decision.EmittedPolicy) + require.Equal(t, []ExpansionSearchStrategy{ExpansionSearchStepwiseForward, ExpansionSearchEndpointSeededReverse}, decision.EmittedCandidates) + require.Equal(t, ExpansionSearchExecutionBoundaryGuardedDualArm, decision.ExecutionBoundary) + require.Equal(t, ExpansionSearchProbeCaps{ReverseSeedRowLimit: 32}, decision.ProbeCaps) + require.Equal(t, ExpansionSearchAdmission{ + StateLimit: 4096, + RequiresCompleteProbes: true, + FallbackStrategy: ExpansionSearchStepwiseForward, + }, decision.Admission) + require.True(t, decision.StructurallyEligible) + require.True(t, decision.StaticallyEligible) + require.Equal(t, ExpansionSearchEndpointSeededReverse, decision.SelectedStrategy) + require.Equal(t, ExpansionSearchStepwiseForward, decision.FallbackStrategy) + require.Equal(t, "static_guarded", decision.SelectionMode) + require.Equal(t, "endpoint-seeded-guarded-v1", decision.SelectorVersion) + require.Equal(t, "property_ends_with", decision.SeedPredicateClass) + require.Equal(t, int64(32), decision.EndpointLimit) + require.Equal(t, int64(4096), decision.StateLimit) + require.Equal(t, 1, decision.PrefixLength) + require.Equal(t, int64(1), decision.MinimumDepth) + require.Equal(t, int64(15), decision.MaximumDepth) + require.True(t, decision.HasFinalLimit) + require.Empty(t, decision.FallbackReason) + require.Contains(t, decision.EligibilityFacts, ExpansionSearchEligibilityFact{Name: "single_variable_expansion_in_region", Eligible: true}) +} + +// TestEndpointSeededExpansionKeepsIndependentMultipartRegionQualified verifies +// that an earlier traversal separated by WITH does not invalidate the existing +// guarded fixed-prefix region. +func TestEndpointSeededExpansionKeepsIndependentMultipartRegionQualified(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH (s)-[:MemberOf*0..]->(excluded:Group) + WHERE excluded.objectid ENDS WITH '-516' + WITH collect(s) AS exclude + MATCH p = (c:Computer)-[:HasSession]->(:User)-[:MemberOf*1..]->(g:Group) + WHERE g.objectid ENDS WITH $suffix AND NOT c IN exclude RETURN p `) require.NoError(t, err) + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ExpansionSearchStrategy, 2) + decision := plan.LoweringPlan.ExpansionSearchStrategy[1] + require.Equal(t, "fixed_prefix_terminal_expansion", decision.Family) + require.Contains(t, decision.EligibilityFacts, ExpansionSearchEligibilityFact{Name: "single_variable_expansion_in_region", Eligible: true}) + require.True(t, decision.StructurallyEligible) + require.Equal(t, ExpansionSearchEndpointSeededReverse, decision.SelectedStrategy) + require.Equal(t, ExpansionSearchPolicyEndpointGuardV1, decision.EmittedPolicy) + require.Equal(t, []ExpansionSearchStrategy{ExpansionSearchStepwiseForward, ExpansionSearchEndpointSeededReverse}, decision.EmittedCandidates) + require.Equal(t, ExpansionSearchExecutionBoundaryGuardedDualArm, decision.ExecutionBoundary) + require.Empty(t, decision.FallbackReason) +} + +// TestGuardedEndpointSeededExpansionFallbackReasons verifies stable rejection reasons for unsafe endpoint-seeded shapes. +func TestGuardedEndpointSeededExpansionFallbackReasons(t *testing.T) { + for _, testCase := range []struct { + // name labels the structural rejection case. + name string + // query produces the endpoint-seeding candidate under test. + query string + // reason is the expected stable fallback code. + reason string + }{ + {name: "terminal not selective", query: `MATCH p = (c:Computer)-[:HasSession]->(:User)-[:MemberOf*1..]->(g:Group) RETURN p`, reason: ExpansionSearchFallbackTerminalNotSelective}, + {name: "zero depth", query: `MATCH p = (c:Computer)-[:HasSession]->(:User)-[:MemberOf*0..]->(g:Group) WHERE g.objectid ENDS WITH '-512' RETURN p`, reason: ExpansionSearchFallbackZeroDepth}, + {name: "directionless prefix", query: `MATCH p = (c:Computer)-[:HasSession]-(:User)-[:MemberOf*1..]->(g:Group) WHERE g.objectid ENDS WITH '-512' RETURN p`, reason: ExpansionSearchFallbackDirectionlessPrefix}, + {name: "correlated terminal", query: `MATCH (g:Group) MATCH p = (c:Computer)-[:HasSession]->(:User)-[:MemberOf*1..]->(g) WHERE g.objectid ENDS WITH '-512' RETURN p`, reason: ExpansionSearchFallbackCorrelatedTerminal}, + {name: "correlated terminal predicate", query: `MATCH p = (c:Computer)-[:HasSession]->(:User)-[:MemberOf*1..]->(g:Group) WHERE g.objectid ENDS WITH '-512' AND g.tenant = c.tenant RETURN p`, reason: ExpansionSearchFallbackCorrelatedTerminal}, + {name: "nonterminal expansion", query: `MATCH p = (c:Computer)-[:HasSession]->(:User)-[:MemberOf*1..]->(g:Group)-[:AdminTo]->() WHERE g.objectid ENDS WITH '-512' RETURN p`, reason: ExpansionSearchFallbackExpansionNotTerminal}, + {name: "mutation", query: `MATCH (c:Computer)-[:HasSession]->(:User)-[:MemberOf*1..]->(g:Group) WHERE g.objectid ENDS WITH '-512' CREATE (:Computer) RETURN g`, reason: ExpansionSearchFallbackMutation}, + } { + t.Run(testCase.name, func(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), testCase.query) + require.NoError(t, err) + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.NotEmpty(t, plan.LoweringPlan.ExpansionSearchStrategy) + decision := plan.LoweringPlan.ExpansionSearchStrategy[0] + require.Equal(t, "fixed_prefix_terminal_expansion", decision.Family) + require.False(t, decision.StructurallyEligible) + require.Equal(t, ExpansionSearchStepwiseForward, decision.SelectedStrategy) + require.Equal(t, testCase.reason, decision.FallbackReason) + }) + } +} + +// TestGuardedEndpointSeededExpansionAcceptsTerminalIDEquality verifies that a singleton terminal ID is a selective reverse-search seed. +func TestGuardedEndpointSeededExpansionAcceptsTerminalIDEquality(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH (c:Computer)-[:HasSession]->(:User)-[:MemberOf*1..8]->(g) + WHERE id(g) = $terminal_id + RETURN id(c), id(g) + `) + require.NoError(t, err) + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ExpansionSearchStrategy, 1) + decision := plan.LoweringPlan.ExpansionSearchStrategy[0] + require.True(t, decision.StructurallyEligible) + require.Equal(t, "id_equality", decision.SeedPredicateClass) + require.Equal(t, ExpansionSearchEndpointSeededReverse, decision.SelectedStrategy) +} + +// TestFixedSuffixSearchRejectsPredicateFunctionReevaluation verifies that reordered function evaluation disqualifies suffix search. +func TestFixedSuffixSearchRejectsPredicateFunctionReevaluation(t *testing.T) { + t.Parallel() + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH (root:ExpansionRoot) + WHERE root.root_key = 'root' + MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(:SuffixTerminal) + WHERE root.marker = toString(1) + RETURN root + `) + require.NoError(t, err) + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ExpansionSearchStrategy, 1) + decision := plan.LoweringPlan.ExpansionSearchStrategy[0] + require.False(t, decision.StructurallyEligible) + require.Equal(t, ExpansionSearchFallbackNonDeterministicPredicate, decision.FallbackReason) + require.Contains(t, decision.EligibilityFacts, ExpansionSearchEligibilityFact{ + Name: "deterministic_predicates", + Eligible: false, + }) +} + +// TestExpansionSearchObservationUsesExternalFieldRequirements verifies that downstream field requirements select the search observation mode. +func TestExpansionSearchObservationUsesExternalFieldRequirements(t *testing.T) { + for _, testCase := range []struct { + // name labels the downstream observation form. + name string + // projection contains the downstream expression being classified. + projection string + // observation is the expected search-state representation. + observation ExpansionSearchObservationMode + }{ + { + name: "endpoint IDs", + projection: "id(head), id(terminal)", + observation: ExpansionSearchObservationEndpointIDs, + }, + { + name: "ordered IDs", + projection: "length(path)", + observation: ExpansionSearchObservationOrderedPathIDs, + }, + { + name: "full path", + projection: "path", + observation: ExpansionSearchObservationFullPath, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH path = (root:ExpansionRoot)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) + RETURN `+testCase.projection) + require.NoError(t, err) + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ExpansionSearchStrategy, 1) + require.Equal(t, testCase.observation, plan.LoweringPlan.ExpansionSearchStrategy[0].ObservationMode) + }) + } +} + +// TestExpansionSearchFinalizationRejectsVariableExpansionAcrossWith verifies that multiple statement-wide expansions prevent specialized search. +func TestExpansionSearchFinalizationRejectsVariableExpansionAcrossWith(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH (root:ExpansionRoot)-[:Expand*0..16]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) + WITH root, terminal + MATCH (root)-[:Expand*0..4]->(other) + RETURN id(terminal), id(other) + `) + require.NoError(t, err) + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ExpansionSearchStrategy, 2) + require.Equal(t, ExpansionSearchFallbackMultipleVariableExpansions, plan.LoweringPlan.ExpansionSearchStrategy[0].FallbackReason) + require.False(t, plan.LoweringPlan.ExpansionSearchStrategy[0].StructurallyEligible) +} + +// TestLoweringPlanReportsStableFixedSuffixSearchFallbackCodes verifies diagnostic codes for structurally unsafe suffix searches. +func TestLoweringPlanReportsStableFixedSuffixSearchFallbackCodes(t *testing.T) { + t.Parallel() + + for _, testCase := range []struct { + // name labels the structural rejection case. + name string + // query produces the fixed-suffix candidate under test. + query string + // reason is the expected stable fallback code. + reason string + }{ + { + name: "no fixed suffix", + query: `MATCH (root)-[:Expand*0..16]->(head) RETURN id(head)`, + reason: ExpansionSearchFallbackNoFixedSuffix, + }, + { + name: "unbounded", + query: `MATCH (root)-[:Expand*0..]->()-[:EnterSuffix]->(head) RETURN id(head)`, + reason: ExpansionSearchFallbackUnboundedDepth, + }, + { + name: "short suffix", + query: `MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head) RETURN id(head)`, + reason: ExpansionSearchFallbackSuffixTooShort, + }, + { + name: "directionless", + query: `MATCH (root)-[:Expand*0..16]-()-[:EnterSuffix]->(head)-[:ContinueSuffix]->()-[:CompleteSuffix]->(terminal) RETURN id(head)`, + reason: ExpansionSearchFallbackDirectionlessExpansion, + }, + { + name: "directionless suffix", + query: `MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]-(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head)`, + reason: ExpansionSearchFallbackDirectionlessSuffix, + }, + { + name: "optional", + query: `OPTIONAL MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head)`, + reason: ExpansionSearchFallbackOptionalMatch, + }, + { + name: "shortest path", + query: `MATCH path = shortestPath((root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal)) RETURN path`, + reason: ExpansionSearchFallbackShortestPath, + }, + { + name: "all shortest paths", + query: `MATCH path = allShortestPaths((root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal)) RETURN path`, + reason: ExpansionSearchFallbackAllShortestPaths, + }, + { + name: "unbound root", + query: `MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)`, + reason: ExpansionSearchFallbackUnboundRoot, + }, + { + name: "unsupported depth", + query: `MATCH (root)-[:Expand*0..65]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head)`, + reason: ExpansionSearchFallbackUnsupportedDepth, + }, + { + name: "relationship variable", + query: `MATCH (root)-[edges:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head)`, + reason: ExpansionSearchFallbackRelationshipVariable, + }, + { + name: "relationship predicate", + query: `MATCH (root)-[edges:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) WHERE edges.enabled = true RETURN id(head)`, + reason: ExpansionSearchFallbackRelationshipPredicate, + }, + { + name: "correlated suffix", + query: `MATCH (head:SuffixHead) MATCH path = (root:ExpansionRoot)-[:Expand*0..16]->()-[:EnterSuffix]->(head)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN path`, + reason: ExpansionSearchFallbackCorrelatedSuffix, + }, + { + name: "cross-region predicate", + query: `MATCH path = (root:ExpansionRoot)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) WHERE root.partition = head.partition RETURN path`, + reason: ExpansionSearchFallbackCrossRegionPredicate, + }, + { + name: "path predicate", + query: `MATCH path = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) WHERE length(path) > 0 RETURN path`, + reason: ExpansionSearchFallbackPathDependentPredicate, + }, + { + name: "unsupported observation", + query: `MATCH path = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(path)`, + reason: ExpansionSearchFallbackUnsupportedObservation, + }, + { + name: "mutation", + query: `MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) CREATE (created) RETURN id(head)`, + reason: ExpansionSearchFallbackMutation, + }, + { + name: "limit pushdown conflict", + query: `MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head) LIMIT 10`, + reason: ExpansionSearchFallbackLimitPushdownConflict, + }, + { + name: "tournament unqualified", + query: `MATCH (root)-[:Other|Alternate*0..16]->()-[:A]->(head:X)-[:B]->(:Y)-[:C]->(terminal:Z) RETURN id(head)`, + reason: ExpansionSearchFallbackTournamentUnqualified, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), testCase.query) + require.NoError(t, err) + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ExpansionSearchStrategy, 1) + require.Equal(t, testCase.reason, plan.LoweringPlan.ExpansionSearchStrategy[0].FallbackReason) + require.False(t, plan.LoweringPlan.ExpansionSearchStrategy[0].StructurallyEligible) + }) + } +} + +// TestLoweringPlanIncludesConstrainedBoundEndpointInExpansionSuffix verifies that a pre-bound terminal remains part of suffix metadata. +func TestLoweringPlanIncludesConstrainedBoundEndpointInExpansionSuffix(t *testing.T) { + t.Parallel() + + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH (terminal) + MATCH path = (root:ExpansionRoot)-[:Expand*0..16]->(boundary:ExpansionNode)-[:EnterSuffix]->(middle:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) + RETURN path + `) + require.NoError(t, err) + plan, err := Optimize(regularQuery) require.NoError(t, err) require.Contains(t, plan.LoweringPlan.Decisions(), LoweringDecision{Name: LoweringExpansionSuffixPushdown}) @@ -764,9 +1243,11 @@ func TestLoweringPlanIncludesConstrainedBoundEndpointInExpansionSuffix(t *testin PatternIndex: 0, StepIndex: 0, }, - SuffixLength: 2, - SuffixStartStep: 1, - SuffixEndStep: 2, + SuffixLength: 2, + SuffixStartStep: 1, + SuffixEndStep: 2, + ApplySupplemental: true, + Reason: "supplemental suffix prefilter retained for unobserved continuation", }) } @@ -907,6 +1388,54 @@ func TestLoweringPlanReportsExpandInto(t *testing.T) { }}, plan.LoweringPlan.ExpandInto) } +func TestLoweringPlanReportsExpandIntoForEndpointsCarriedAcrossWithAndUnwind(t *testing.T) { + t.Parallel() + + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH (a:Group), (b:Group) + WITH a, b, [1, 2] AS copies + UNWIND copies AS copy + MATCH (a)-[:MemberOf]->(b) + RETURN copy + `) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Contains(t, plan.LoweringPlan.ExpandInto, ExpandIntoDecision{ + Target: TraversalStepTarget{ + QueryPartIndex: 1, + ClauseIndex: 1, + PatternIndex: 0, + StepIndex: 0, + }, + }) +} + +func TestLoweringPlanReportsExpandIntoForNodeIntroducedByUnwind(t *testing.T) { + t.Parallel() + + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH (a:Group), (b:Group) + WITH b, [a] AS nodes + UNWIND nodes AS source + MATCH (source)-[:MemberOf]->(b) + RETURN source + `) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Contains(t, plan.LoweringPlan.ExpandInto, ExpandIntoDecision{ + Target: TraversalStepTarget{ + QueryPartIndex: 1, + ClauseIndex: 1, + PatternIndex: 0, + StepIndex: 0, + }, + }) +} + func TestLoweringPlanReportsExpandIntoForAnonymousContinuationEndpoint(t *testing.T) { t.Parallel() @@ -1373,6 +1902,496 @@ func TestLoweringPlanReportsShortestPathStrategyForEndpointPredicates(t *testing }}, plan.LoweringPlan.ShortestPathFilter) } +// TestLoweringPlanSelectsQualifiedSingletonDistanceExecutor verifies scalar-distance selection for a statically bound endpoint pair. +func TestLoweringPlanSelectsQualifiedSingletonDistanceExecutor(t *testing.T) { + t.Parallel() + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..16]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ShortestPathExecutor, 1) + decision := plan.LoweringPlan.ShortestPathExecutor[0] + require.Equal(t, "SP", decision.Family) + require.Equal(t, "static", decision.SelectionMode) + require.Equal(t, "sp-static-v3", decision.SelectorVersion) + require.Equal(t, []ShortestPathExecutor{ + ShortestPathExecutorIncumbentWorkspace, + ShortestPathExecutorS0Direct, + ShortestPathExecutorS1ArrayBFS, + ShortestPathExecutorS2TraceRelation, + ShortestPathExecutorS3Unidirectional, + ShortestPathExecutorS3EdgeM0, + ShortestPathExecutorS4CanonicalDistance, + ShortestPathExecutorS4CanonicalWitness, + ShortestPathExecutorI1CanonicalDistance, + ShortestPathExecutorI1CanonicalWitness, + ShortestPathExecutorI1CanonicalPredecessorWitness, + ShortestPathExecutorB1AlternatingNodeDistance, + ShortestPathExecutorB1AlternatingNodeWitness, + ShortestPathExecutorB2SmallerCurrentLevelDistance, + ShortestPathExecutorB2SmallerCurrentLevelWitness, + }, decision.PlannedCandidates) + require.Equal(t, ShortestPathExecutorS3Unidirectional, decision.SelectedExecutor) + require.Equal(t, ShortestPathSchedulerSingleEndedLevel, decision.Scheduler) + require.Equal(t, ShortestPathExecutorIncumbentWorkspace, decision.FallbackExecutor) + require.Empty(t, decision.FallbackReason) + require.Equal(t, ShortestPathObservationDistance, decision.ObservationMode) + require.True(t, decision.StructurallyEligible) + require.Equal(t, int64(1), decision.MinimumDepth) + require.Equal(t, int64(16), decision.MaximumDepth) + require.True(t, decision.ExperimentalWinner) + require.Contains(t, plan.LoweringPlan.Decisions(), LoweringDecision{Name: LoweringShortestPathExecutor}) +} + +// TestLoweringPlanSelectsBoundPairAllShortestDAGExecutor verifies predecessor-DAG selection for bound all-shortest-path endpoints. +func TestLoweringPlanSelectsBoundPairAllShortestDAGExecutor(t *testing.T) { + t.Parallel() + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = allShortestPaths((s)-[*1..]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ShortestPathExecutor, 1) + decision := plan.LoweringPlan.ShortestPathExecutor[0] + require.Equal(t, "ASP", decision.Family) + require.Equal(t, ShortestPathObservationAllPaths, decision.ObservationMode) + require.Equal(t, ShortestPathExecutorASPA1DAG, decision.SelectedExecutor) + require.Equal(t, []ShortestPathExecutor{ + ShortestPathExecutorIncumbentWorkspace, + ShortestPathExecutorASPA1DAG, + ShortestPathExecutorASPI1DAG, + ShortestPathExecutorASPB1AlternatingNodeDAG, + ShortestPathExecutorASPB2SmallerCurrentLevelDAG, + }, decision.PlannedCandidates) + require.Equal(t, ShortestPathSchedulerSingleEndedLevel, decision.Scheduler) + require.Equal(t, "asp-static-v1", decision.SelectorVersion) + require.Equal(t, "static", decision.SelectionMode) + require.True(t, decision.StructurallyEligible) + require.True(t, decision.StaticallyEligible) + require.Equal(t, int64(1), decision.MinimumDepth) + require.Equal(t, defaultShortestPathExpansionDepth, decision.MaximumDepth) + require.Equal(t, defaultShortestPathStateLimit, decision.StateLimit) + require.Equal(t, defaultShortestPathFrontierLimit, decision.FrontierLimit) + require.Equal(t, defaultShortestPathPredecessorLimit, decision.PredecessorLimit) + require.Empty(t, decision.FallbackReason) +} + +// TestShortestPathExecutorSchedulersFreezesTournamentSchedulerMetadata verifies +// production controls and reserved bidirectional arms retain distinct policies. +func TestShortestPathExecutorSchedulersFreezesTournamentSchedulerMetadata(t *testing.T) { + t.Parallel() + tests := map[ShortestPathExecutor]ShortestPathScheduler{ + ShortestPathExecutorS3Unidirectional: ShortestPathSchedulerSingleEndedLevel, + ShortestPathExecutorS3EdgeM0: ShortestPathSchedulerSingleEndedLevel, + ShortestPathExecutorS4CanonicalDistance: ShortestPathSchedulerSingleEndedLevel, + ShortestPathExecutorS4CanonicalWitness: ShortestPathSchedulerSingleEndedLevel, + ShortestPathExecutorASPA1DAG: ShortestPathSchedulerSingleEndedLevel, + ShortestPathExecutorB1AlternatingNodeDistance: ShortestPathSchedulerStrictAlternatingNode, + ShortestPathExecutorB1AlternatingNodeWitness: ShortestPathSchedulerStrictAlternatingNode, + ShortestPathExecutorASPB1AlternatingNodeDAG: ShortestPathSchedulerStrictAlternatingNode, + ShortestPathExecutorB2SmallerCurrentLevelDistance: ShortestPathSchedulerSmallerCurrentLevel, + ShortestPathExecutorB2SmallerCurrentLevelWitness: ShortestPathSchedulerSmallerCurrentLevel, + ShortestPathExecutorASPB2SmallerCurrentLevelDAG: ShortestPathSchedulerSmallerCurrentLevel, + } + for executor, scheduler := range tests { + require.Equal(t, scheduler, executor.Scheduler(), executor) + } + require.Empty(t, ShortestPathExecutorIncumbentWorkspace.Scheduler()) +} + +// TestLoweringPlanShortestExecutorV4SelectionMatrix verifies executor selection across direction, depth, kind, and observation combinations. +func TestLoweringPlanShortestExecutorV4SelectionMatrix(t *testing.T) { + t.Parallel() + tests := []struct { + // name labels the executor-selection case. + name string + // pattern is the relationship pattern supplied to shortestPath. + pattern string + // observation is the return expression that consumes the path. + observation string + // executor is the physical implementation expected from selection. + executor ShortestPathExecutor + // reason is the expected fallback code when selection is ineligible. + reason string + // direction is the logical traversal direction recorded in diagnostics. + direction graph.Direction + // physicalExpansion is the edge endpoint used to advance recursive search. + physicalExpansion ShortestPathPhysicalExpansion + // topology is the expected physical topology classification. + topology ShortestPathTopologyClassification + // kindCount is the expected number of statically resolved relationship kinds. + kindCount int + // untyped reports whether the pattern is expected to omit relationship kinds. + untyped bool + // staticEligible is the expected static qualification result. + staticEligible bool + // selector identifies the policy version expected to make the decision. + selector string + }{ + { + name: "outbound distance depth 64 two kinds", + pattern: `(s)-[:MemberOf|Contains*1..64]->(e)`, + observation: `length(p)`, + executor: ShortestPathExecutorS3Unidirectional, + direction: graph.DirectionOutbound, + physicalExpansion: ShortestPathPhysicalExpansionStartID, + topology: ShortestPathTopologyPhysicalOutbound, + kindCount: 2, + staticEligible: true, + selector: "sp-static-v3", + }, + { + name: "outbound one path one kind", + pattern: `(s)-[:MemberOf*1..16]->(e)`, + observation: `p`, + executor: ShortestPathExecutorS3EdgeM0, + direction: graph.DirectionOutbound, + physicalExpansion: ShortestPathPhysicalExpansionStartID, + topology: ShortestPathTopologyPhysicalOutbound, + kindCount: 1, + staticEligible: true, + selector: "sp-static-v5-contained", + }, + { + name: "outbound one path two kinds", + pattern: `(s)-[:MemberOf|Contains*1..16]->(e)`, + observation: `p`, + executor: ShortestPathExecutorS4CanonicalWitness, + direction: graph.DirectionOutbound, + physicalExpansion: ShortestPathPhysicalExpansionStartID, + topology: ShortestPathTopologyPhysicalOutbound, + kindCount: 2, + staticEligible: true, + selector: "sp-static-v5-contained", + }, + { + name: "outbound one path wildcard", + pattern: `(s)-[*1..16]->(e)`, + observation: `p`, + executor: ShortestPathExecutorS4CanonicalWitness, + direction: graph.DirectionOutbound, + physicalExpansion: ShortestPathPhysicalExpansionStartID, + topology: ShortestPathTopologyPhysicalOutbound, + untyped: true, + staticEligible: true, + selector: "sp-static-v5-contained", + }, + { + name: "inbound distance depth one", + pattern: `(s)<-[:MemberOf*0..1]-(e)`, + observation: `length(p)`, + executor: ShortestPathExecutorS3Unidirectional, + direction: graph.DirectionInbound, + physicalExpansion: ShortestPathPhysicalExpansionEndID, + topology: ShortestPathTopologyPhysicalInboundShallow, + kindCount: 1, + staticEligible: true, + selector: "sp-static-v3", + }, + { + name: "inbound path depth one", + pattern: `(s)<-[:MemberOf*1..1]-(e)`, + observation: `p`, + executor: ShortestPathExecutorS3EdgeM0, + direction: graph.DirectionInbound, + physicalExpansion: ShortestPathPhysicalExpansionEndID, + topology: ShortestPathTopologyPhysicalInboundShallow, + kindCount: 1, + staticEligible: true, + selector: "sp-static-v5-contained", + }, + { + name: "inbound distance depth two", + pattern: `(s)<-[:MemberOf*1..2]-(e)`, + observation: `length(p)`, + executor: ShortestPathExecutorS4CanonicalDistance, + direction: graph.DirectionInbound, + physicalExpansion: ShortestPathPhysicalExpansionEndID, + topology: ShortestPathTopologyPhysicalInboundDeep, + kindCount: 1, + staticEligible: true, + selector: "sp-static-v5-contained", + }, + { + name: "inbound path depth 64 two kinds", + pattern: `(s)<-[:MemberOf|Contains*1..64]-(e)`, + observation: `p`, + executor: ShortestPathExecutorS4CanonicalWitness, + direction: graph.DirectionInbound, + physicalExpansion: ShortestPathPhysicalExpansionEndID, + topology: ShortestPathTopologyPhysicalInboundDeep, + kindCount: 2, + staticEligible: true, + selector: "sp-static-v5-contained", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), fmt.Sprintf(` + MATCH p = shortestPath(%s) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN %s + `, test.pattern, test.observation)) + require.NoError(t, err) + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ShortestPathExecutor, 1) + decision := plan.LoweringPlan.ShortestPathExecutor[0] + require.Equal(t, test.selector, decision.SelectorVersion) + require.True(t, decision.StructurallyEligible) + require.Equal(t, test.staticEligible, decision.StaticallyEligible) + require.Equal(t, test.executor, decision.SelectedExecutor) + require.Equal(t, test.reason, decision.FallbackReason) + require.Equal(t, test.direction, decision.Direction) + require.Equal(t, test.physicalExpansion, decision.PhysicalExpansion) + require.Equal(t, test.topology, decision.TopologyClassification) + require.Equal(t, test.kindCount, decision.RelationshipKindCount) + require.Equal(t, test.untyped, decision.UntypedRelationship) + }) + } +} + +// TestLoweringPlanShortestExecutorV3PreservesStructuralReasonPrecedence verifies that directionless topology wins over later static failures. +func TestLoweringPlanShortestExecutorV3PreservesStructuralReasonPrecedence(t *testing.T) { + t.Parallel() + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf|Contains*1..64]-(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + plan, err := Optimize(regularQuery) + require.NoError(t, err) + decision := plan.LoweringPlan.ShortestPathExecutor[0] + require.False(t, decision.StructurallyEligible) + require.False(t, decision.StaticallyEligible) + require.Equal(t, ShortestPathFallbackDirectionless, decision.FallbackReason) +} + +// TestLoweringPlanShortestExecutorRejectsUnsupportedMinimumDepth verifies rejection of a minimum depth greater than one. +func TestLoweringPlanShortestExecutorRejectsUnsupportedMinimumDepth(t *testing.T) { + t.Parallel() + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*2..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ShortestPathExecutor, 1) + decision := plan.LoweringPlan.ShortestPathExecutor[0] + require.False(t, decision.StructurallyEligible) + require.Equal(t, int64(2), decision.MinimumDepth) + require.Equal(t, int64(4), decision.MaximumDepth) + require.Equal(t, ShortestPathFallbackUnsupportedDepth, decision.FallbackReason) +} + +// TestLoweringPlanShortestExecutorRetainsZeroMaximumDepthInDiagnostics verifies that an explicit zero maximum is not omitted from JSON. +func TestLoweringPlanShortestExecutorRetainsZeroMaximumDepthInDiagnostics(t *testing.T) { + t.Parallel() + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*0..0]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ShortestPathExecutor, 1) + decision := plan.LoweringPlan.ShortestPathExecutor[0] + require.True(t, decision.StructurallyEligible) + require.Zero(t, decision.MinimumDepth) + require.Zero(t, decision.MaximumDepth) + + diagnostic, err := json.Marshal(decision) + require.NoError(t, err) + require.Contains(t, string(diagnostic), `"maximum_depth":0`) +} + +// TestLoweringPlanShortestExecutorUsesStatementWideCallCount verifies that multiple path calls across query parts disqualify static execution. +func TestLoweringPlanShortestExecutorUsesStatementWideCallCount(t *testing.T) { + t.Parallel() + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + WITH p + MATCH q = shortestPath((x)-[:MemberOf*1..4]->(y)) + WHERE id(x) = $other_start_id AND id(y) = $other_end_id + RETURN length(p), length(q) + `) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ShortestPathExecutor, 2) + for _, decision := range plan.LoweringPlan.ShortestPathExecutor { + require.False(t, decision.StructurallyEligible) + require.Equal(t, ShortestPathFallbackMultiplePathCalls, decision.FallbackReason) + } +} + +// TestLoweringPlanShortestExecutorUsesStatementWideReadOnlyFact verifies that a later mutation disqualifies an earlier shortest-path candidate. +func TestLoweringPlanShortestExecutorUsesStatementWideReadOnlyFact(t *testing.T) { + t.Parallel() + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + WITH p + CREATE (:Group {name: 'updated'}) + RETURN length(p) + `) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ShortestPathExecutor, 1) + decision := plan.LoweringPlan.ShortestPathExecutor[0] + require.False(t, decision.StructurallyEligible) + require.Equal(t, ShortestPathFallbackMutation, decision.FallbackReason) +} + +// TestLoweringPlanShortestExecutorObservationModeRequiresPathForNodes verifies that nodes(path) requires a path witness. +func TestLoweringPlanShortestExecutorObservationModeRequiresPathForNodes(t *testing.T) { + t.Parallel() + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + RETURN nodes(p) + `) + require.NoError(t, err) + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Equal(t, ShortestPathObservationOnePath, plan.LoweringPlan.ShortestPathExecutor[0].ObservationMode) +} + +// TestLoweringPlanShortestExecutorRequiresKnownObservationMode verifies that an unbound path result prevents static executor selection. +func TestLoweringPlanShortestExecutorRequiresKnownObservationMode(t *testing.T) { + t.Parallel() + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN s + `) + require.NoError(t, err) + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ShortestPathExecutor, 1) + decision := plan.LoweringPlan.ShortestPathExecutor[0] + require.Equal(t, ShortestPathObservationUnknown, decision.ObservationMode) + require.False(t, decision.StructurallyEligible) +} + +// TestLoweringPlanShortestExecutorRejectsAdditionalRowSources verifies fallback classification for correlated or ambiguous endpoint sources. +func TestLoweringPlanShortestExecutorRejectsAdditionalRowSources(t *testing.T) { + t.Parallel() + tests := []struct { + // name labels the additional-row-source case. + name string + // query produces the shortest-path candidate under test. + query string + // reason is the expected stable fallback code. + reason string + }{ + { + name: "unwind source", + query: ` + UNWIND [1, 2] AS source + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `, + reason: ShortestPathFallbackCorrelatedEndpoints, + }, + { + name: "additional match pattern", + query: ` + MATCH (source), p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `, + reason: ShortestPathFallbackMultipleEndpointPairs, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), test.query) + require.NoError(t, err) + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ShortestPathExecutor, 1) + decision := plan.LoweringPlan.ShortestPathExecutor[0] + require.False(t, decision.StructurallyEligible) + require.Equal(t, test.reason, decision.FallbackReason) + }) + } +} + +// TestLoweringPlanRecordsStableShortestExecutorFallbackCodes verifies diagnostic codes for unsupported shortest-path shapes. +func TestLoweringPlanRecordsStableShortestExecutorFallbackCodes(t *testing.T) { + t.Parallel() + tests := []struct { + // name labels the unsupported shortest-path shape. + name string + // query produces the shortest-path candidate under test. + query string + // reason is the expected stable fallback code. + reason string + }{ + { + name: "all shortest", + query: `MATCH p = allShortestPaths((s)-[:MemberOf*1..4]->(e)) RETURN p`, + reason: ShortestPathFallbackAllShortestPaths, + }, + { + name: "directionless", + query: `MATCH p = shortestPath((s)-[:MemberOf*1..4]-(e)) RETURN p`, + reason: ShortestPathFallbackDirectionless, + }, + { + name: "relationship variable", + query: `MATCH p = shortestPath((s)-[r:MemberOf*1..4]->(e)) RETURN p`, + reason: ShortestPathFallbackRelationshipVariable, + }, + { + name: "open depth", + query: `MATCH p = shortestPath((s)-[:MemberOf*1..]->(e)) RETURN p`, + reason: ShortestPathFallbackUnsupportedDepth, + }, + { + name: "non singleton", + query: `MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) RETURN p`, + reason: ShortestPathFallbackNonSingletonID, + }, + { + name: "multiple id equalities", + query: `MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) WHERE id(s) = 1 AND id(s) = 2 AND id(e) = 3 RETURN p`, + reason: ShortestPathFallbackMultipleIDEqualities, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), test.query) + require.NoError(t, err) + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.NotEmpty(t, plan.LoweringPlan.ShortestPathExecutor) + require.Equal(t, test.reason, plan.LoweringPlan.ShortestPathExecutor[0].FallbackReason) + }) + } +} + func TestLoweringPlanReportsShortestPathStrategyForBoundEndpointPairs(t *testing.T) { t.Parallel() @@ -1725,10 +2744,11 @@ func TestLoweringPlanSkipsDirectionlessExpansionSuffixPushdown(t *testing.T) { require.Empty(t, plan.LoweringPlan.ExpansionSuffixPushdown) } +// TestPredicateAttachmentRuleAssignsSingleBindingPredicates verifies that single-symbol predicates attach to their binding scopes. func TestPredicateAttachmentRuleAssignsSingleBindingPredicates(t *testing.T) { t.Parallel() - regularQuery, err := frontend.ParseCypher(frontend.NewContext(), adcsQuery) + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), fixedSuffixExpansionQuery) require.NoError(t, err) plan, err := Optimize(regularQuery) @@ -1741,8 +2761,8 @@ func TestPredicateAttachmentRuleAssignsSingleBindingPredicates(t *testing.T) { ClauseIndex: 0, ExpressionIndex: 0, Scope: PredicateAttachmentScopeBinding, - BindingSymbols: []string{"n"}, - Dependencies: []string{"n"}, + BindingSymbols: []string{"root"}, + Dependencies: []string{"root"}, }, plan.PredicateAttachments[0]) require.Equal(t, PredicateAttachment{ @@ -1751,8 +2771,8 @@ func TestPredicateAttachmentRuleAssignsSingleBindingPredicates(t *testing.T) { ClauseIndex: 2, ExpressionIndex: 0, Scope: PredicateAttachmentScopeBinding, - BindingSymbols: []string{"ct"}, - Dependencies: []string{"ct"}, + BindingSymbols: []string{"predicate"}, + Dependencies: []string{"predicate"}, }, plan.PredicateAttachments[1]) } @@ -1781,6 +2801,7 @@ func TestPredicateAttachmentRuleKeepsMultiBindingPredicatesAtRegionScope(t *test }, plan.PredicateAttachments[0]) } +// firstNodeSymbol returns the first node variable encountered during a structural query walk. func firstNodeSymbol(readingClause *cypher.ReadingClause) string { if readingClause == nil || readingClause.Match == nil || len(readingClause.Match.Pattern) == 0 { return "" diff --git a/cypher/models/pgsql/optimize/scalar_continuation_test.go b/cypher/models/pgsql/optimize/scalar_continuation_test.go new file mode 100644 index 00000000..8d1ce3fc --- /dev/null +++ b/cypher/models/pgsql/optimize/scalar_continuation_test.go @@ -0,0 +1,59 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// +// SPDX-License-Identifier: Apache-2.0 + +package optimize + +import ( + "testing" + + "github.com/specterops/dawgs/cypher/frontend" + "github.com/stretchr/testify/require" +) + +// fieldRequirementForSymbol optimizes cypherQuery and returns the field-requirement decision for symbol. +func fieldRequirementForSymbol(t *testing.T, cypherQuery, symbol string) FieldRequirementDecision { + t.Helper() + + query, err := frontend.ParseCypher(frontend.NewContext(), cypherQuery) + require.NoError(t, err) + + plan, err := Optimize(query) + require.NoError(t, err) + + for _, decision := range plan.LoweringPlan.FieldRequirements { + if decision.Symbol == symbol { + return decision + } + } + + require.FailNow(t, "field requirement decision not found", symbol) + return FieldRequirementDecision{} +} + +// TestScalarContinuationFieldRequirementAllowsIDOnlyObservation verifies that an ID consumer permits scalar continuation state. +func TestScalarContinuationFieldRequirementAllowsIDOnlyObservation(t *testing.T) { + t.Parallel() + + decision := fieldRequirementForSymbol(t, + `MATCH (s)-[*1..]->(mid)-[]->(e) RETURN id(mid), id(e)`, + "mid", + ) + + require.Contains(t, decision.Fields, FieldRequirementEntityID) + require.NotContains(t, decision.Fields, FieldRequirementFullEntity) +} + +// TestScalarContinuationFieldRequirementRetainsFullEntityForMutation verifies that mutation prevents scalar-only continuation state. +func TestScalarContinuationFieldRequirementRetainsFullEntityForMutation(t *testing.T) { + t.Parallel() + + decision := fieldRequirementForSymbol(t, + `MATCH (s)-[*1..]->(mid)-[]->(e) DELETE mid`, + "mid", + ) + + require.Contains(t, decision.Fields, FieldRequirementFullEntity) +} diff --git a/cypher/models/pgsql/optimize/source_references.go b/cypher/models/pgsql/optimize/source_references.go index 01dde537..ca966cca 100644 --- a/cypher/models/pgsql/optimize/source_references.go +++ b/cypher/models/pgsql/optimize/source_references.go @@ -1,19 +1,256 @@ package optimize import ( + "sort" + "strings" + "github.com/specterops/dawgs/cypher/models/cypher" "github.com/specterops/dawgs/cypher/models/walk" ) +// sourceReferenceCollector tracks referenced identifiers and repeated pattern declarations during syntax walking. type sourceReferenceCollector struct { + // VisitorHandler supplies cancellation and error propagation for the syntax walk. walk.VisitorHandler - referencedIdentifiers map[string]struct{} - matchPatternDeclarationRefs map[string]int - matchPatternDeclarations map[*cypher.PatternPart]struct{} + // referencedIdentifiers contains bindings consumed outside their defining pattern declarations. + referencedIdentifiers map[string]struct{} + // matchPatternDeclarationRefs counts match-pattern declarations by binding symbol. + matchPatternDeclarationRefs map[string]int + // matchPatternDeclarations identifies pattern parts whose variables are declarations rather than reads. + matchPatternDeclarations map[*cypher.PatternPart]struct{} + // matchPatternDeclarationDepth tracks nesting beneath the declaration currently being visited. matchPatternDeclarationDepth int } +// fieldRequirementCollector accumulates ordered representation requirements for each Cypher binding. +type fieldRequirementCollector struct { + // VisitorHandler supplies cancellation and error propagation for the syntax walk. + walk.VisitorHandler + + // queryPartIndex identifies the query part whose binding uses are being collected. + queryPartIndex int + // ordinal orders binding uses in traversal order. + ordinal int + // patternDepth tracks whether the visitor is currently inside a pattern declaration. + patternDepth int + // propertyDepth tracks nested property lookups so their base binding is classified once. + propertyDepth int + // functionStack identifies the function consuming a visited expression. + functionStack []*cypher.FunctionInvocation + // bindingKinds maps each symbol to its path, relationship, or node representation. + bindingKinds map[string]string + // patternUses counts pattern occurrences of each binding. + patternUses map[string]int + // decisions accumulates representation requirements by binding symbol. + decisions map[string]*FieldRequirementDecision +} + +// newFieldRequirementCollector initializes requirement tracking for one query part. +func newFieldRequirementCollector(queryPartIndex int) *fieldRequirementCollector { + return &fieldRequirementCollector{ + VisitorHandler: walk.NewCancelableErrorHandler(), + queryPartIndex: queryPartIndex, + bindingKinds: map[string]string{}, + patternUses: map[string]int{}, + decisions: map[string]*FieldRequirementDecision{}, + } +} + +// add records one ordered use and merges its required fields into the binding decision. +func (s *fieldRequirementCollector) add(symbol string, internal bool, fields ...FieldRequirement) { + if symbol == "" { + return + } + + s.ordinal++ + decision, found := s.decisions[symbol] + if !found { + decision = &FieldRequirementDecision{ + QueryPartIndex: s.queryPartIndex, + Symbol: symbol, + } + s.decisions[symbol] = decision + } + + useFields := append([]FieldRequirement(nil), fields...) + decision.Uses = append(decision.Uses, FieldRequirementUse{ + Ordinal: s.ordinal, + Fields: useFields, + Internal: internal, + }) + decision.LastUse = s.ordinal + + present := make(map[FieldRequirement]struct{}, len(decision.Fields)) + for _, field := range decision.Fields { + present[field] = struct{}{} + } + for _, field := range fields { + if _, found := present[field]; !found { + decision.Fields = append(decision.Fields, field) + present[field] = struct{}{} + } + } +} + +// patternVariableSymbol returns a pattern variable's symbol or an empty string when no variable is present. +func patternVariableSymbol(variable *cypher.Variable) string { + if variable == nil { + return "" + } + return variable.Symbol +} + +// addFullBinding records the complete representation required for a path, relationship, or node binding. +func (s *fieldRequirementCollector) addFullBinding(symbol, kind string) { + switch kind { + case "path": + s.add(symbol, false, FieldRequirementFullPath) + case "relationship": + s.add(symbol, false, FieldRequirementFullEntity, FieldRequirementRelationshipIDs) + default: + s.add(symbol, false, FieldRequirementFullEntity) + } +} + +// addGreedyProjectionBindings marks every visible binding for full materialization in deterministic symbol order. +func (s *fieldRequirementCollector) addGreedyProjectionBindings() { + symbols := make([]string, 0, len(s.bindingKinds)) + for symbol := range s.bindingKinds { + symbols = append(symbols, symbol) + } + sort.Strings(symbols) + + for _, symbol := range symbols { + s.addFullBinding(symbol, s.bindingKinds[symbol]) + } +} + +// Enter records representation requirements before visiting a syntax node's children. +func (s *fieldRequirementCollector) Enter(node cypher.SyntaxNode) { + switch typedNode := node.(type) { + case *cypher.PatternPart: + s.patternDepth++ + if symbol := patternVariableSymbol(typedNode.Variable); symbol != "" { + s.bindingKinds[symbol] = "path" + s.add(symbol, true, FieldRequirementOrderedPathEdgeIDs) + } + + case *cypher.NodePattern: + if symbol := patternVariableSymbol(typedNode.Variable); symbol != "" { + s.bindingKinds[symbol] = "node" + s.patternUses[symbol]++ + if s.patternUses[symbol] > 1 { + // Reused pattern bindings are consumed by bound-endpoint joins. + // Those joins still expect the entity representation; scalar-ID + // rehydration is a separate lowering capability. + s.add(symbol, true, FieldRequirementFullEntity) + } + if len(typedNode.Kinds) > 0 { + s.add(symbol, true, FieldRequirementEntityID, FieldRequirementKinds) + } + if typedNode.Properties != nil { + s.add(symbol, true, FieldRequirementEntityID, FieldRequirementProperties) + } + } + + case *cypher.RelationshipPattern: + if symbol := patternVariableSymbol(typedNode.Variable); symbol != "" { + s.bindingKinds[symbol] = "relationship" + s.patternUses[symbol]++ + if s.patternUses[symbol] > 1 { + s.add(symbol, true, FieldRequirementFullEntity) + } + s.add(symbol, true, FieldRequirementRelationshipIDs) + if len(typedNode.Kinds) > 0 { + s.add(symbol, true, FieldRequirementKinds) + } + if typedNode.Properties != nil { + s.add(symbol, true, FieldRequirementProperties) + } + } + + case *cypher.PropertyLookup: + s.propertyDepth++ + + case *cypher.FunctionInvocation: + s.functionStack = append(s.functionStack, typedNode) + + case *cypher.Variable: + if s.patternDepth > 0 { + return + } + if typedNode.Symbol == cypher.TokenLiteralAsterisk { + s.addGreedyProjectionBindings() + return + } + + if s.propertyDepth > 0 { + s.add(typedNode.Symbol, false, FieldRequirementEntityID, FieldRequirementProperties) + return + } + + if len(s.functionStack) > 0 { + switch strings.ToLower(s.functionStack[len(s.functionStack)-1].Name) { + case cypher.IdentityFunction: + s.add(typedNode.Symbol, false, FieldRequirementEntityID) + return + case cypher.NodeLabelsFunction, cypher.EdgeTypeFunction: + s.add(typedNode.Symbol, false, FieldRequirementKinds) + return + case cypher.PathLengthFunction: + s.add(typedNode.Symbol, false, FieldRequirementOrderedPathEdgeIDs) + return + case cypher.NodesFunction, cypher.RelationshipsFunction: + s.add(typedNode.Symbol, false, FieldRequirementFullPath) + return + } + } + + s.addFullBinding(typedNode.Symbol, s.bindingKinds[typedNode.Symbol]) + } +} + +// Visit performs no leaf-specific work because Enter classifies every relevant node. +func (s *fieldRequirementCollector) Visit(cypher.SyntaxNode) {} + +// Exit unwinds pattern, property, and function nesting after visiting a syntax node's children. +func (s *fieldRequirementCollector) Exit(node cypher.SyntaxNode) { + switch node.(type) { + case *cypher.PatternPart: + s.patternDepth-- + case *cypher.PropertyLookup: + s.propertyDepth-- + case *cypher.FunctionInvocation: + s.functionStack = s.functionStack[:len(s.functionStack)-1] + } +} + +// collectFieldRequirements walks root and returns normalized representation needs for its bindings. +func collectFieldRequirements(queryPartIndex int, root cypher.SyntaxNode) ([]FieldRequirementDecision, error) { + if root == nil { + return nil, nil + } + + collector := newFieldRequirementCollector(queryPartIndex) + if err := walk.Cypher(root, collector); err != nil { + return nil, err + } + + symbols := make([]string, 0, len(collector.decisions)) + for symbol := range collector.decisions { + symbols = append(symbols, symbol) + } + sort.Strings(symbols) + + decisions := make([]FieldRequirementDecision, 0, len(symbols)) + for _, symbol := range symbols { + decisions = append(decisions, *collector.decisions[symbol]) + } + return decisions, nil +} + +// newSourceReferenceCollector initializes empty reference and match-declaration tracking for a syntax walk. func newSourceReferenceCollector() *sourceReferenceCollector { return &sourceReferenceCollector{ VisitorHandler: walk.NewCancelableErrorHandler(), @@ -23,18 +260,21 @@ func newSourceReferenceCollector() *sourceReferenceCollector { } } +// addVariable records a referenced variable unless it is part of the declaration currently being traversed. func (s *sourceReferenceCollector) addVariable(variable *cypher.Variable) { if variable != nil && variable.Symbol != "" { s.referencedIdentifiers[variable.Symbol] = struct{}{} } } +// addMatchPatternDeclaration counts a non-empty variable declared inside a pattern expression so repeated declarations can be retained as references. func (s *sourceReferenceCollector) addMatchPatternDeclaration(variable *cypher.Variable) { if variable != nil && variable.Symbol != "" { s.matchPatternDeclarationRefs[variable.Symbol] += 1 } } +// collectRepeatedMatchPatternDeclarations marks multiply declared match symbols as source references. func (s *sourceReferenceCollector) collectRepeatedMatchPatternDeclarations() { for identifier, numDeclarations := range s.matchPatternDeclarationRefs { if numDeclarations > 1 { @@ -43,6 +283,7 @@ func (s *sourceReferenceCollector) collectRepeatedMatchPatternDeclarations() { } } +// isMatchPatternDeclaration reports whether node is a variable declaration belonging to a match pattern. func (s *sourceReferenceCollector) isMatchPatternDeclaration(patternPart *cypher.PatternPart) bool { _, isDeclaration := s.matchPatternDeclarations[patternPart] return isDeclaration @@ -97,6 +338,7 @@ func (s *sourceReferenceCollector) Exit(node cypher.SyntaxNode) { } } +// collectReferencedSourceIdentifiers returns identifiers used outside a declaring match pattern or declared repeatedly within one. func collectReferencedSourceIdentifiers(root cypher.SyntaxNode) (map[string]struct{}, error) { if root == nil { return map[string]struct{}{}, nil @@ -104,13 +346,14 @@ func collectReferencedSourceIdentifiers(root cypher.SyntaxNode) (map[string]stru collector := newSourceReferenceCollector() if err := walk.Cypher(root, collector); err != nil { - return collector.referencedIdentifiers, err + return nil, err } collector.collectRepeatedMatchPatternDeclarations() return collector.referencedIdentifiers, nil } +// referencesSourceIdentifier reports whether references contains symbol or the wildcard source marker. func referencesSourceIdentifier(references map[string]struct{}, symbol string) bool { if _, referencesAll := references[cypher.TokenLiteralAsterisk]; referencesAll { return true diff --git a/cypher/models/pgsql/optimize/traversal_envelope.go b/cypher/models/pgsql/optimize/traversal_envelope.go new file mode 100644 index 00000000..f381d655 --- /dev/null +++ b/cypher/models/pgsql/optimize/traversal_envelope.go @@ -0,0 +1,154 @@ +package optimize + +// Endpoint-resolution limits are immutable analysis metadata for the first +// bounded-resolution envelope. Each runtime limit has an explicit cap+1 +// sentinel; this slice records the contract without changing execution. +const ( + EndpointResolutionSingletonLimit int64 = 1 + EndpointResolutionSingletonSentinel int64 = 2 + EndpointResolutionSmallSetLimit int64 = 32 + EndpointResolutionSmallSetSentinel int64 = 33 +) + +// EndpointResolutionClass identifies how one traversal endpoint, or a +// correlated endpoint pair, could be resolved before traversal. +type EndpointResolutionClass string + +const ( + EndpointResolutionClassIDEquality EndpointResolutionClass = "id_equality" + EndpointResolutionClassUniquePropertyEquality EndpointResolutionClass = "unique_property_equality" + EndpointResolutionClassNonUniquePropertyEquality EndpointResolutionClass = "nonunique_property_equality" + EndpointResolutionClassExplicitSmallSet EndpointResolutionClass = "explicit_small_set" + EndpointResolutionClassCorrelatedPair EndpointResolutionClass = "correlated_pair" + EndpointResolutionClassUnsupported EndpointResolutionClass = "unsupported" +) + +// EndpointResolutionPlan identifies the exact incumbent and the planned-only +// bounded resolver independently of any shortest-path executor. +type EndpointResolutionPlan string + +const ( + EndpointResolutionPlanIncumbent EndpointResolutionPlan = "ENDPOINT-RESOLUTION-INCUMBENT" + EndpointResolutionPlanBounded EndpointResolutionPlan = "ENDPOINT-RESOLUTION-BOUNDED" +) + +const ( + EndpointResolutionFallbackPlannedOnly = "planned_only" + EndpointResolutionFallbackMutation = "mutation" + EndpointResolutionFallbackOptionalMatch = "optional_match" + EndpointResolutionFallbackCorrelatedPair = "correlated_pair" + EndpointResolutionFallbackUnsupported = "unsupported_endpoint_class" + EndpointResolutionFallbackSmallSetOverflow = "explicit_small_set_overflow" +) + +// EndpointResolutionCaps serializes both admitted cardinalities and their +// overflow sentinels so future SQL cannot silently reinterpret the contract. +type EndpointResolutionCaps struct { + SingletonLimit int64 `json:"singleton_limit"` + SingletonSentinel int64 `json:"singleton_sentinel"` + SmallSetLimit int64 `json:"small_set_limit"` + SmallSetSentinel int64 `json:"small_set_sentinel"` +} + +// EndpointResolutionInput records one endpoint's statically recognizable +// resolution shape. Cardinality remains runtime evidence. +type EndpointResolutionInput struct { + Symbol string `json:"symbol"` + Class EndpointResolutionClass `json:"class"` + Property string `json:"property,omitempty"` + StaticValueCount int `json:"static_value_count,omitempty"` + ParameterizedSet bool `json:"parameterized_set,omitempty"` + Limit int64 `json:"limit,omitempty"` + Sentinel int64 `json:"sentinel,omitempty"` +} + +// EndpointResolutionEligibilityFact records one conservative qualification +// check for bounded endpoint materialization. +type EndpointResolutionEligibilityFact struct { + Name string `json:"name"` + Eligible bool `json:"eligible"` +} + +// EndpointResolutionDecision is analysis-only metadata for one SP/ASP +// traversal. The exact existing resolver remains selected in this milestone. +type EndpointResolutionDecision struct { + Target TraversalStepTarget `json:"target"` + Family string `json:"family"` + Root EndpointResolutionInput `json:"root"` + Terminal EndpointResolutionInput `json:"terminal"` + PairClass EndpointResolutionClass `json:"pair_class,omitempty"` + PlannedClasses []EndpointResolutionClass `json:"planned_classes"` + Caps EndpointResolutionCaps `json:"caps"` + PlannedCandidates []EndpointResolutionPlan `json:"planned_candidates"` + CandidatePlan EndpointResolutionPlan `json:"candidate_plan"` + SelectedPlan EndpointResolutionPlan `json:"selected_plan"` + FallbackPlan EndpointResolutionPlan `json:"fallback_plan"` + EligibilityFacts []EndpointResolutionEligibilityFact `json:"eligibility_facts"` + StructurallyEligible bool `json:"structurally_eligible"` + StaticallyEligible bool `json:"statically_eligible"` + SelectionMode string `json:"selection_mode"` + SelectorVersion string `json:"selector_version"` + FallbackReason string `json:"fallback_reason"` +} + +// TraversalPredicateClass identifies the strongest safe placement property +// proven from syntax. Unsupported path forms deliberately remain conservative. +type TraversalPredicateClass string + +const ( + TraversalPredicateClassStepLocalNode TraversalPredicateClass = "step_local_node" + TraversalPredicateClassStepLocalRelationship TraversalPredicateClass = "step_local_relationship" + TraversalPredicateClassUniversalAllNodes TraversalPredicateClass = "universal_all_nodes" + TraversalPredicateClassUniversalNoneNodes TraversalPredicateClass = "universal_none_nodes" + TraversalPredicateClassUniversalAllRelationships TraversalPredicateClass = "universal_all_relationships" + TraversalPredicateClassUniversalNoneRelationships TraversalPredicateClass = "universal_none_relationships" + TraversalPredicateClassWholePath TraversalPredicateClass = "whole_path" + TraversalPredicateClassUnsupported TraversalPredicateClass = "unsupported" +) + +// TraversalPredicatePlan separates planned step evaluation from the exact +// incumbent predicate placement that remains selected. +type TraversalPredicatePlan string + +const ( + TraversalPredicatePlanIncumbent TraversalPredicatePlan = "TRAVERSAL-PREDICATE-INCUMBENT" + TraversalPredicatePlanStep TraversalPredicatePlan = "TRAVERSAL-PREDICATE-STEP" +) + +const ( + TraversalPredicateFallbackPlannedOnly = "planned_only" + TraversalPredicateFallbackMutation = "mutation" + TraversalPredicateFallbackOptional = "optional_match" + TraversalPredicateFallbackCorrelation = "correlated_predicate" + TraversalPredicateFallbackWholePath = "whole_path" + TraversalPredicateFallbackUnsupported = "unsupported_predicate" +) + +// TraversalPredicateEligibilityFact records one conservative classification +// or placement qualification. +type TraversalPredicateEligibilityFact struct { + Name string `json:"name"` + Eligible bool `json:"eligible"` +} + +// TraversalPredicateDecision records one predicate relevant to a variable +// traversal. It never authorizes placement by itself. +type TraversalPredicateDecision struct { + Target TraversalStepTarget `json:"target"` + PredicateIndex int `json:"predicate_index"` + Source string `json:"source"` + Class TraversalPredicateClass `json:"class"` + PathSymbol string `json:"path_symbol,omitempty"` + BindingSymbol string `json:"binding_symbol,omitempty"` + ReferencedSymbols []string `json:"referenced_symbols,omitempty"` + PlannedCandidates []TraversalPredicatePlan `json:"planned_candidates"` + CandidatePlan TraversalPredicatePlan `json:"candidate_plan,omitempty"` + SelectedPlan TraversalPredicatePlan `json:"selected_plan"` + FallbackPlan TraversalPredicatePlan `json:"fallback_plan"` + EligibilityFacts []TraversalPredicateEligibilityFact `json:"eligibility_facts"` + StructurallyEligible bool `json:"structurally_eligible"` + StaticallyEligible bool `json:"statically_eligible"` + SelectionMode string `json:"selection_mode"` + ClassifierVersion string `json:"classifier_version"` + FallbackReason string `json:"fallback_reason"` +} diff --git a/cypher/models/pgsql/optimize/traversal_envelope_plan.go b/cypher/models/pgsql/optimize/traversal_envelope_plan.go new file mode 100644 index 00000000..abdcebfa --- /dev/null +++ b/cypher/models/pgsql/optimize/traversal_envelope_plan.go @@ -0,0 +1,669 @@ +package optimize + +import ( + "strings" + + "github.com/specterops/dawgs/cypher/models/cypher" + "github.com/specterops/dawgs/cypher/models/walk" +) + +type endpointResolutionCandidate struct { + class EndpointResolutionClass + property string + staticValueCount int + parameterizedSet bool + rank int +} + +func endpointResolutionCaps() EndpointResolutionCaps { + return EndpointResolutionCaps{ + SingletonLimit: EndpointResolutionSingletonLimit, + SingletonSentinel: EndpointResolutionSingletonSentinel, + SmallSetLimit: EndpointResolutionSmallSetLimit, + SmallSetSentinel: EndpointResolutionSmallSetSentinel, + } +} + +func endpointResolutionInput(symbol string, node *cypher.NodePattern, where *cypher.Where) EndpointResolutionInput { + input := EndpointResolutionInput{ + Symbol: symbol, + Class: EndpointResolutionClassUnsupported, + } + if symbol == "" { + return input + } + + var candidates []endpointResolutionCandidate + if where != nil { + for _, expression := range where.Expressions { + for _, term := range cypherConjunctionTerms(expression) { + if candidate, found := endpointResolutionCandidateForTerm(term, symbol); found { + candidates = append(candidates, candidate) + } + } + } + } + candidates = append(candidates, inlineEndpointResolutionCandidates(node, symbol)...) + if len(candidates) == 0 { + return input + } + + best := candidates[0] + for _, candidate := range candidates[1:] { + if candidate.rank > best.rank { + best = candidate + } + } + input.Class = best.class + input.Property = best.property + input.StaticValueCount = best.staticValueCount + input.ParameterizedSet = best.parameterizedSet + switch best.class { + case EndpointResolutionClassIDEquality, EndpointResolutionClassUniquePropertyEquality: + input.Limit = EndpointResolutionSingletonLimit + input.Sentinel = EndpointResolutionSingletonSentinel + case EndpointResolutionClassNonUniquePropertyEquality, EndpointResolutionClassExplicitSmallSet: + input.Limit = EndpointResolutionSmallSetLimit + input.Sentinel = EndpointResolutionSmallSetSentinel + } + + return input +} + +func endpointResolutionCandidateForTerm(expression cypher.Expression, symbol string) (endpointResolutionCandidate, bool) { + expression = unwrapCypherParenthetical(expression) + comparison, ok := expression.(*cypher.Comparison) + if !ok || comparison == nil || len(comparison.Partials) != 1 || comparison.Partials[0] == nil { + return endpointResolutionCandidate{}, false + } + partial := comparison.Partials[0] + switch partial.Operator { + case cypher.OperatorEquals: + if identitySymbol, found := identityFunctionSymbol(comparison.Left); found && identitySymbol == symbol && expressionIsConstant(partial.Right) { + return endpointResolutionCandidate{class: EndpointResolutionClassIDEquality, staticValueCount: 1, rank: 5}, true + } + if identitySymbol, found := identityFunctionSymbol(partial.Right); found && identitySymbol == symbol && expressionIsConstant(comparison.Left) { + return endpointResolutionCandidate{class: EndpointResolutionClassIDEquality, staticValueCount: 1, rank: 5}, true + } + if propertySymbol, property, found := propertyLookupSymbol(comparison.Left); found && propertySymbol == symbol && expressionIsConstant(partial.Right) { + return propertyEndpointResolutionCandidate(property), true + } + if propertySymbol, property, found := propertyLookupSymbol(partial.Right); found && propertySymbol == symbol && expressionIsConstant(comparison.Left) { + return propertyEndpointResolutionCandidate(property), true + } + + case cypher.OperatorIn: + values, parameterized, recognized := explicitSetCardinality(partial.Right) + if !recognized { + return endpointResolutionCandidate{}, false + } + if identitySymbol, found := identityFunctionSymbol(comparison.Left); found && identitySymbol == symbol { + return endpointResolutionCandidate{class: EndpointResolutionClassExplicitSmallSet, staticValueCount: values, parameterizedSet: parameterized, rank: 4}, true + } + if propertySymbol, property, found := propertyLookupSymbol(comparison.Left); found && propertySymbol == symbol { + return endpointResolutionCandidate{class: EndpointResolutionClassExplicitSmallSet, property: property, staticValueCount: values, parameterizedSet: parameterized, rank: 4}, true + } + } + + return endpointResolutionCandidate{}, false +} + +func propertyEndpointResolutionCandidate(property string) endpointResolutionCandidate { + // A property name is not a uniqueness proof. Until graph-schema metadata is + // available to the optimizer, every property equality uses the bounded + // non-unique envelope and its cap+1 runtime sentinel. + return endpointResolutionCandidate{ + class: EndpointResolutionClassNonUniquePropertyEquality, + property: property, + staticValueCount: 1, + rank: 2, + } +} + +func inlineEndpointResolutionCandidates(node *cypher.NodePattern, symbol string) []endpointResolutionCandidate { + if node == nil || variableSymbol(node.Variable) != symbol { + return nil + } + properties, ok := node.Properties.(*cypher.Properties) + if !ok || properties == nil || properties.Parameter != nil { + return nil + } + + candidates := make([]endpointResolutionCandidate, 0, len(properties.Map)) + for property, value := range properties.Map { + if expressionIsConstant(value) { + candidates = append(candidates, propertyEndpointResolutionCandidate(property)) + } + } + return candidates +} + +func constantListCardinality(expression cypher.Expression) (int, bool) { + literal, ok := unwrapCypherParenthetical(expression).(*cypher.ListLiteral) + if !ok || literal == nil || len(*literal) == 0 { + return 0, false + } + for _, value := range *literal { + if !expressionIsConstant(value) { + return 0, false + } + } + return len(*literal), true +} + +// explicitSetCardinality recognizes both statically enumerable list literals +// and parameterized sets. Parameter contents remain runtime evidence and must +// pass the same 32/33 bounded-resolution sentinel as a literal set. +func explicitSetCardinality(expression cypher.Expression) (values int, parameterized, recognized bool) { + expression = unwrapCypherParenthetical(expression) + if _, ok := expression.(*cypher.Parameter); ok { + return 0, true, true + } + values, recognized = constantListCardinality(expression) + return values, false, recognized +} + +func endpointInputWithinStaticCap(input EndpointResolutionInput) bool { + return input.Class != EndpointResolutionClassExplicitSmallSet || input.StaticValueCount <= int(EndpointResolutionSmallSetLimit) +} + +func endpointResolutionClassSupported(class EndpointResolutionClass) bool { + return class != "" && class != EndpointResolutionClassUnsupported && class != EndpointResolutionClassCorrelatedPair +} + +func endpointPairPredicateCorrelated(where *cypher.Where, leftSymbol, rightSymbol string) bool { + if where == nil || leftSymbol == "" || rightSymbol == "" { + return false + } + for _, expression := range where.Expressions { + for _, term := range cypherConjunctionTerms(expression) { + dependencies := sortedDependencies(term) + if stringSliceContains(dependencies, leftSymbol) && stringSliceContains(dependencies, rightSymbol) { + return true + } + } + } + return false +} + +func appendEndpointResolutionDecisions( + plan *LoweringPlan, + queryPartIndex int, + queryPart cypher.SyntaxNode, + readingClauses []*cypher.ReadingClause, + initialDeclaredSymbols map[string]struct{}, +) { + _, updatingClauses := queryPartProjection(queryPart) + declaredSymbols := copyStringSet(initialDeclaredSymbols) + hasUnwind := false + for _, readingClause := range readingClauses { + if readingClause != nil && readingClause.Unwind != nil { + hasUnwind = true + } + } + + for clauseIndex, readingClause := range readingClauses { + if readingClause == nil || readingClause.Match == nil { + continue + } + for patternIndex, patternPart := range readingClause.Match.Pattern { + if patternPart == nil || (!patternPart.ShortestPathPattern && !patternPart.AllShortestPathsPattern) { + declarePatternSymbols(declaredSymbols, patternPart) + continue + } + steps := traversalStepsForPattern(patternPart) + for stepIndex, step := range steps { + if step.Relationship == nil || step.Relationship.Range == nil || step.LeftNode == nil || step.RightNode == nil { + continue + } + leftSymbol := variableSymbol(step.LeftNode.Variable) + rightSymbol := variableSymbol(step.RightNode.Variable) + root := endpointResolutionInput(leftSymbol, step.LeftNode, readingClause.Match.Where) + terminal := endpointResolutionInput(rightSymbol, step.RightNode, readingClause.Match.Where) + _, leftPreviouslyBound := declaredSymbols[leftSymbol] + _, rightPreviouslyBound := declaredSymbols[rightSymbol] + correlated := queryPartIndex > 0 || hasUnwind || len(readingClause.Match.Pattern) != 1 || leftPreviouslyBound || rightPreviouslyBound || endpointPairPredicateCorrelated(readingClause.Match.Where, leftSymbol, rightSymbol) + classesSupported := endpointResolutionClassSupported(root.Class) && endpointResolutionClassSupported(terminal.Class) + withinCaps := endpointInputWithinStaticCap(root) && endpointInputWithinStaticCap(terminal) + facts := []EndpointResolutionEligibilityFact{ + {Name: "supported_shortest_path_mode", Eligible: patternPart.ShortestPathPattern || patternPart.AllShortestPathsPattern}, + {Name: "single_traversal_step", Eligible: len(steps) == 1 && len(patternPart.PatternElements) == 3}, + {Name: "read_only", Eligible: updatingClauses == 0}, + {Name: "non_optional", Eligible: !readingClause.Match.Optional}, + {Name: "bounded_endpoint_classes", Eligible: classesSupported}, + {Name: "within_static_endpoint_caps", Eligible: withinCaps}, + {Name: "uncorrelated_pair", Eligible: !correlated}, + } + eligible := endpointResolutionFactsEligible(facts) + fallbackReason := EndpointResolutionFallbackPlannedOnly + switch { + case updatingClauses != 0: + fallbackReason = EndpointResolutionFallbackMutation + case readingClause.Match.Optional: + fallbackReason = EndpointResolutionFallbackOptionalMatch + case correlated: + fallbackReason = EndpointResolutionFallbackCorrelatedPair + case !withinCaps: + fallbackReason = EndpointResolutionFallbackSmallSetOverflow + case !classesSupported || len(steps) != 1 || len(patternPart.PatternElements) != 3: + fallbackReason = EndpointResolutionFallbackUnsupported + } + + plannedClasses := []EndpointResolutionClass{root.Class, terminal.Class} + pairClass := EndpointResolutionClass("") + if correlated { + pairClass = EndpointResolutionClassCorrelatedPair + plannedClasses = append(plannedClasses, pairClass) + } + family := "SP" + if patternPart.AllShortestPathsPattern { + family = "ASP" + } + plan.EndpointResolution = append(plan.EndpointResolution, EndpointResolutionDecision{ + Target: PatternTarget{ + QueryPartIndex: queryPartIndex, + ClauseIndex: clauseIndex, + PatternIndex: patternIndex, + }.TraversalStep(stepIndex), + Family: family, + Root: root, + Terminal: terminal, + PairClass: pairClass, + PlannedClasses: plannedClasses, + Caps: endpointResolutionCaps(), + PlannedCandidates: []EndpointResolutionPlan{EndpointResolutionPlanIncumbent, EndpointResolutionPlanBounded}, + CandidatePlan: EndpointResolutionPlanBounded, + SelectedPlan: EndpointResolutionPlanIncumbent, + FallbackPlan: EndpointResolutionPlanIncumbent, + EligibilityFacts: facts, + StructurallyEligible: eligible, + StaticallyEligible: false, + SelectionMode: "analysis_only", + SelectorVersion: "endpoint-resolution-v1", + FallbackReason: fallbackReason, + }) + } + declarePatternSymbols(declaredSymbols, patternPart) + } + declareWhereSymbols(declaredSymbols, readingClause.Match) + } +} + +func endpointResolutionFactsEligible(facts []EndpointResolutionEligibilityFact) bool { + for _, fact := range facts { + if !fact.Eligible { + return false + } + } + return true +} + +func setEndpointResolutionFact(decision *EndpointResolutionDecision, name string, eligible bool) { + for idx := range decision.EligibilityFacts { + if decision.EligibilityFacts[idx].Name == name { + decision.EligibilityFacts[idx].Eligible = eligible + return + } + } +} + +type traversalPredicateClassification struct { + class TraversalPredicateClass + bindingSymbol string + relevant bool + correlated bool +} + +func classifyTraversalPredicate( + expression cypher.Expression, + pathSymbol string, + nodeSymbols map[string]struct{}, + relationshipSymbols map[string]struct{}, +) traversalPredicateClassification { + expression = unwrapCypherParenthetical(expression) + if quantifier, ok := expression.(*cypher.Quantifier); ok { + return classifyTraversalQuantifier(quantifier, pathSymbol) + } + + dependencies := sortedDependencies(expression) + if pathSymbol != "" && stringSliceContains(dependencies, pathSymbol) { + return traversalPredicateClassification{ + class: TraversalPredicateClassWholePath, + relevant: true, + correlated: len(dependencies) > 1, + } + } + + var nodeDependencies, relationshipDependencies int + for _, dependency := range dependencies { + if _, found := nodeSymbols[dependency]; found { + nodeDependencies++ + } + if _, found := relationshipSymbols[dependency]; found { + relationshipDependencies++ + } + } + relevantDependencies := nodeDependencies + relationshipDependencies + if relevantDependencies == 0 { + return traversalPredicateClassification{} + } + correlated := len(dependencies) != 1 || relevantDependencies != 1 + // A WHERE reference to an endpoint symbol is a boundary predicate, and a + // variable-length relationship binding can be list/path-valued. Neither + // syntax proves evaluation against every recursive step. Only explicit + // path quantifiers and inline relationship properties are classified as + // step-evaluable below. + return traversalPredicateClassification{ + class: TraversalPredicateClassUnsupported, + relevant: true, + correlated: correlated, + } +} + +func classifyTraversalQuantifier(quantifier *cypher.Quantifier, pathSymbol string) traversalPredicateClassification { + if quantifier == nil || quantifier.Filter == nil || quantifier.Filter.Specifier == nil || quantifier.Filter.Specifier.Variable == nil { + return traversalPredicateClassification{class: TraversalPredicateClassUnsupported, relevant: true} + } + function, ok := quantifier.Filter.Specifier.Expression.(*cypher.FunctionInvocation) + if !ok || function == nil || function.NumArguments() != 1 { + return traversalPredicateClassification{class: TraversalPredicateClassUnsupported, relevant: true} + } + pathVariable, ok := function.Arguments[0].(*cypher.Variable) + if !ok || pathVariable == nil || pathSymbol == "" || pathVariable.Symbol != pathSymbol { + return traversalPredicateClassification{} + } + bindingSymbol := quantifier.Filter.Specifier.Variable.Symbol + bodyDependencies := sortedDependencies(quantifier.Filter.Where) + correlated := false + for _, dependency := range bodyDependencies { + if dependency != bindingSymbol { + correlated = true + } + } + collectionNodes := strings.EqualFold(function.Name, cypher.NodesFunction) + collectionRelationships := strings.EqualFold(function.Name, cypher.RelationshipsFunction) + if !collectionNodes && !collectionRelationships { + return traversalPredicateClassification{class: TraversalPredicateClassWholePath, bindingSymbol: bindingSymbol, relevant: true, correlated: correlated} + } + if correlated || quantifier.Filter.Where == nil || !traversalPredicateUsesOnlySafeFunctions(quantifier.Filter.Where, collectionRelationships) { + return traversalPredicateClassification{class: TraversalPredicateClassWholePath, bindingSymbol: bindingSymbol, relevant: true, correlated: correlated} + } + + classification := traversalPredicateClassification{bindingSymbol: bindingSymbol, relevant: true} + switch { + case collectionNodes && quantifier.Type == cypher.QuantifierTypeAll: + classification.class = TraversalPredicateClassUniversalAllNodes + case collectionNodes && quantifier.Type == cypher.QuantifierTypeNone: + classification.class = TraversalPredicateClassUniversalNoneNodes + case collectionRelationships && quantifier.Type == cypher.QuantifierTypeAll: + classification.class = TraversalPredicateClassUniversalAllRelationships + case collectionRelationships && quantifier.Type == cypher.QuantifierTypeNone: + classification.class = TraversalPredicateClassUniversalNoneRelationships + default: + classification.class = TraversalPredicateClassWholePath + } + return classification +} + +func traversalPredicateUsesOnlySafeFunctions(node cypher.SyntaxNode, relationshipBinding bool) bool { + safe := true + _ = walk.Cypher(node, walk.NewSimpleVisitor[cypher.SyntaxNode](func(node cypher.SyntaxNode, _ walk.VisitorHandler) { + function, ok := node.(*cypher.FunctionInvocation) + if !ok || function == nil { + return + } + if strings.EqualFold(function.Name, cypher.IdentityFunction) { + return + } + if relationshipBinding && strings.EqualFold(function.Name, cypher.EdgeTypeFunction) { + return + } + safe = false + })) + return safe +} + +func traversalPredicateClassStepEvaluable(class TraversalPredicateClass) bool { + switch class { + case TraversalPredicateClassStepLocalNode, + TraversalPredicateClassStepLocalRelationship, + TraversalPredicateClassUniversalAllNodes, + TraversalPredicateClassUniversalNoneNodes, + TraversalPredicateClassUniversalAllRelationships, + TraversalPredicateClassUniversalNoneRelationships: + return true + default: + return false + } +} + +func appendTraversalPredicateDecisions( + plan *LoweringPlan, + queryPartIndex int, + queryPart cypher.SyntaxNode, + readingClauses []*cypher.ReadingClause, +) { + _, updatingClauses := queryPartProjection(queryPart) + for clauseIndex, readingClause := range readingClauses { + if readingClause == nil || readingClause.Match == nil { + continue + } + for patternIndex, patternPart := range readingClause.Match.Pattern { + if patternPart == nil { + continue + } + pathSymbol := variableSymbol(patternPart.Variable) + steps := traversalStepsForPattern(patternPart) + for stepIndex, step := range steps { + if step.Relationship == nil || step.Relationship.Range == nil { + continue + } + target := PatternTarget{ + QueryPartIndex: queryPartIndex, + ClauseIndex: clauseIndex, + PatternIndex: patternIndex, + }.TraversalStep(stepIndex) + nodeSymbols := map[string]struct{}{} + if symbol := variableSymbol(step.LeftNode.Variable); symbol != "" { + nodeSymbols[symbol] = struct{}{} + } + if symbol := variableSymbol(step.RightNode.Variable); symbol != "" { + nodeSymbols[symbol] = struct{}{} + } + relationshipSymbols := map[string]struct{}{} + if symbol := variableSymbol(step.Relationship.Variable); symbol != "" { + relationshipSymbols[symbol] = struct{}{} + } + + predicateIndex := 0 + if readingClause.Match.Where != nil { + for _, whereExpression := range readingClause.Match.Where.Expressions { + for _, term := range cypherConjunctionTerms(whereExpression) { + classification := classifyTraversalPredicate(term, pathSymbol, nodeSymbols, relationshipSymbols) + if !classification.relevant { + continue + } + appendTraversalPredicateDecision(plan, target, predicateIndex, "where", pathSymbol, classification, sortedDependencies(term), updatingClauses == 0, !readingClause.Match.Optional) + predicateIndex++ + } + } + } + if step.Relationship.Properties != nil { + classification := traversalPredicateClassification{class: TraversalPredicateClassStepLocalRelationship, relevant: true} + if !inlinePropertiesStepLocal(step.Relationship.Properties) { + classification.class = TraversalPredicateClassUnsupported + classification.correlated = len(sortedDependencies(step.Relationship.Properties)) > 0 + } + appendTraversalPredicateDecision(plan, target, predicateIndex, "relationship_pattern", pathSymbol, classification, sortedDependencies(step.Relationship.Properties), updatingClauses == 0, !readingClause.Match.Optional) + predicateIndex++ + } + for _, node := range []*cypher.NodePattern{step.LeftNode, step.RightNode} { + if node == nil || node.Properties == nil { + continue + } + // Node-pattern properties constrain the pattern boundary; they + // are not predicates over every node visited by a variable range. + classification := traversalPredicateClassification{ + class: TraversalPredicateClassUnsupported, + relevant: true, + correlated: len(sortedDependencies(node.Properties)) > 0, + } + appendTraversalPredicateDecision(plan, target, predicateIndex, "node_pattern", pathSymbol, classification, sortedDependencies(node.Properties), updatingClauses == 0, !readingClause.Match.Optional) + predicateIndex++ + } + } + } + } +} + +func inlinePropertiesStepLocal(expression cypher.Expression) bool { + properties, ok := expression.(*cypher.Properties) + if !ok || properties == nil || properties.Parameter != nil { + return false + } + for _, value := range properties.Map { + if !expressionIsConstant(value) { + return false + } + } + return true +} + +func appendTraversalPredicateDecision( + plan *LoweringPlan, + target TraversalStepTarget, + predicateIndex int, + source, pathSymbol string, + classification traversalPredicateClassification, + referencedSymbols []string, + readOnly, nonOptional bool, +) { + stepEvaluable := traversalPredicateClassStepEvaluable(classification.class) + facts := []TraversalPredicateEligibilityFact{ + {Name: "read_only", Eligible: readOnly}, + {Name: "non_optional", Eligible: nonOptional}, + {Name: "step_evaluable", Eligible: stepEvaluable}, + {Name: "uncorrelated", Eligible: !classification.correlated}, + } + eligible := traversalPredicateFactsEligible(facts) + fallbackReason := TraversalPredicateFallbackPlannedOnly + switch { + case !readOnly: + fallbackReason = TraversalPredicateFallbackMutation + case !nonOptional: + fallbackReason = TraversalPredicateFallbackOptional + case classification.correlated: + fallbackReason = TraversalPredicateFallbackCorrelation + case classification.class == TraversalPredicateClassWholePath: + fallbackReason = TraversalPredicateFallbackWholePath + case !stepEvaluable: + fallbackReason = TraversalPredicateFallbackUnsupported + } + plannedCandidates := []TraversalPredicatePlan{TraversalPredicatePlanIncumbent} + candidatePlan := TraversalPredicatePlan("") + if stepEvaluable { + candidatePlan = TraversalPredicatePlanStep + plannedCandidates = append(plannedCandidates, candidatePlan) + } + plan.TraversalPredicate = append(plan.TraversalPredicate, TraversalPredicateDecision{ + Target: target, + PredicateIndex: predicateIndex, + Source: source, + Class: classification.class, + PathSymbol: pathSymbol, + BindingSymbol: classification.bindingSymbol, + ReferencedSymbols: referencedSymbols, + PlannedCandidates: plannedCandidates, + CandidatePlan: candidatePlan, + SelectedPlan: TraversalPredicatePlanIncumbent, + FallbackPlan: TraversalPredicatePlanIncumbent, + EligibilityFacts: facts, + StructurallyEligible: eligible, + StaticallyEligible: false, + SelectionMode: "analysis_only", + ClassifierVersion: "traversal-predicate-v1", + FallbackReason: fallbackReason, + }) +} + +func traversalPredicateFactsEligible(facts []TraversalPredicateEligibilityFact) bool { + for _, fact := range facts { + if !fact.Eligible { + return false + } + } + return true +} + +func setTraversalPredicateFact(decision *TraversalPredicateDecision, name string, eligible bool) { + for idx := range decision.EligibilityFacts { + if decision.EligibilityFacts[idx].Name == name { + decision.EligibilityFacts[idx].Eligible = eligible + return + } + } +} + +func finalizeTraversalEnvelopeDecisions(plan *LoweringPlan, query *cypher.RegularQuery) { + if plan == nil || query == nil || query.SingleQuery == nil { + return + } + readOnly := statementUpdatingClauseCount(query) == 0 + for idx := range plan.EndpointResolution { + decision := &plan.EndpointResolution[idx] + setEndpointResolutionFact(decision, "read_only", readOnly) + decision.StructurallyEligible = endpointResolutionFactsEligible(decision.EligibilityFacts) + decision.StaticallyEligible = false + if !readOnly { + decision.FallbackReason = EndpointResolutionFallbackMutation + } + } + for idx := range plan.TraversalPredicate { + decision := &plan.TraversalPredicate[idx] + setTraversalPredicateFact(decision, "read_only", readOnly) + decision.StructurallyEligible = traversalPredicateFactsEligible(decision.EligibilityFacts) + decision.StaticallyEligible = false + if !readOnly { + decision.FallbackReason = TraversalPredicateFallbackMutation + } + } +} + +func statementUpdatingClauseCount(query *cypher.RegularQuery) int { + if query == nil || query.SingleQuery == nil { + return 0 + } + count := 0 + if multiPart := query.SingleQuery.MultiPartQuery; multiPart != nil { + for _, part := range multiPart.Parts { + if part != nil { + count += len(part.UpdatingClauses) + } + } + if finalPart := multiPart.SinglePartQuery; finalPart != nil { + count += len(finalPart.UpdatingClauses) + } + } else if singlePart := query.SingleQuery.SinglePartQuery; singlePart != nil { + count += len(singlePart.UpdatingClauses) + } + return count +} + +func unwrapCypherParenthetical(expression cypher.Expression) cypher.Expression { + for { + parenthetical, ok := expression.(*cypher.Parenthetical) + if !ok || parenthetical == nil { + return expression + } + expression = parenthetical.Expression + } +} + +func stringSliceContains(values []string, expected string) bool { + for _, value := range values { + if value == expected { + return true + } + } + return false +} diff --git a/cypher/models/pgsql/optimize/traversal_envelope_test.go b/cypher/models/pgsql/optimize/traversal_envelope_test.go new file mode 100644 index 00000000..179c94a2 --- /dev/null +++ b/cypher/models/pgsql/optimize/traversal_envelope_test.go @@ -0,0 +1,431 @@ +package optimize + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/specterops/dawgs/cypher/frontend" + "github.com/stretchr/testify/require" +) + +func optimizeTraversalEnvelope(t *testing.T, query string) LoweringPlan { + t.Helper() + + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), query) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + return plan.LoweringPlan +} + +func TestEndpointResolutionClassifiesBoundedInputsWithoutSelectingThem(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + where string + rootClass EndpointResolutionClass + terminalClass EndpointResolutionClass + valueCount int + runtimeCount bool + }{ + { + name: "ID equality", + where: "id(s) = $source_id AND id(e) = $terminal_id", + rootClass: EndpointResolutionClassIDEquality, + terminalClass: EndpointResolutionClassIDEquality, + valueCount: 1, + }, + { + name: "property name is not uniqueness proof", + where: "s.objectid = $source_id AND e.objectid = $terminal_id", + rootClass: EndpointResolutionClassNonUniquePropertyEquality, + terminalClass: EndpointResolutionClassNonUniquePropertyEquality, + valueCount: 1, + }, + { + name: "nonunique property equality", + where: "s.name = $source_name AND e.name = $terminal_name", + rootClass: EndpointResolutionClassNonUniquePropertyEquality, + terminalClass: EndpointResolutionClassNonUniquePropertyEquality, + valueCount: 1, + }, + { + name: "explicit small set", + where: "id(s) IN [1, 2] AND id(e) IN [3, 4]", + rootClass: EndpointResolutionClassExplicitSmallSet, + terminalClass: EndpointResolutionClassExplicitSmallSet, + valueCount: 2, + }, + { + name: "parameterized explicit small set", + where: "id(s) IN $source_ids AND e.name IN $terminal_names", + rootClass: EndpointResolutionClassExplicitSmallSet, + terminalClass: EndpointResolutionClassExplicitSmallSet, + runtimeCount: true, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + plan := optimizeTraversalEnvelope(t, fmt.Sprintf(` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE %s + RETURN length(p) + `, testCase.where)) + + require.Len(t, plan.EndpointResolution, 1) + decision := plan.EndpointResolution[0] + require.Equal(t, testCase.rootClass, decision.Root.Class) + require.Equal(t, testCase.terminalClass, decision.Terminal.Class) + require.Equal(t, testCase.valueCount, decision.Root.StaticValueCount) + require.Equal(t, testCase.valueCount, decision.Terminal.StaticValueCount) + require.Equal(t, testCase.runtimeCount, decision.Root.ParameterizedSet) + require.Equal(t, testCase.runtimeCount, decision.Terminal.ParameterizedSet) + if testCase.runtimeCount { + require.Equal(t, EndpointResolutionSmallSetLimit, decision.Root.Limit) + require.Equal(t, EndpointResolutionSmallSetSentinel, decision.Root.Sentinel) + } + require.Equal(t, EndpointResolutionPlanBounded, decision.CandidatePlan) + require.Equal(t, EndpointResolutionPlanIncumbent, decision.SelectedPlan) + require.Equal(t, EndpointResolutionPlanIncumbent, decision.FallbackPlan) + require.True(t, decision.StructurallyEligible) + require.False(t, decision.StaticallyEligible) + require.Equal(t, "analysis_only", decision.SelectionMode) + require.Equal(t, EndpointResolutionFallbackPlannedOnly, decision.FallbackReason) + require.Contains(t, plan.Decisions(), LoweringDecision{Name: LoweringEndpointResolution}) + }) + } +} + +func TestEndpointResolutionRecordsCapsAndSentinelsInJSON(t *testing.T) { + t.Parallel() + + plan := optimizeTraversalEnvelope(t, ` + MATCH p = allShortestPaths((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) IN [1, 2] AND id(e) IN [3, 4] + RETURN p + `) + require.Len(t, plan.EndpointResolution, 1) + + diagnostic, err := json.Marshal(plan.EndpointResolution[0]) + require.NoError(t, err) + require.JSONEq(t, `{ + "target":{"query_part_index":0,"clause_index":0,"pattern_index":0,"step_index":0}, + "family":"ASP", + "root":{"symbol":"s","class":"explicit_small_set","static_value_count":2,"limit":32,"sentinel":33}, + "terminal":{"symbol":"e","class":"explicit_small_set","static_value_count":2,"limit":32,"sentinel":33}, + "planned_classes":["explicit_small_set","explicit_small_set"], + "caps":{"singleton_limit":1,"singleton_sentinel":2,"small_set_limit":32,"small_set_sentinel":33}, + "planned_candidates":["ENDPOINT-RESOLUTION-INCUMBENT","ENDPOINT-RESOLUTION-BOUNDED"], + "candidate_plan":"ENDPOINT-RESOLUTION-BOUNDED", + "selected_plan":"ENDPOINT-RESOLUTION-INCUMBENT", + "fallback_plan":"ENDPOINT-RESOLUTION-INCUMBENT", + "eligibility_facts":[ + {"name":"supported_shortest_path_mode","eligible":true}, + {"name":"single_traversal_step","eligible":true}, + {"name":"read_only","eligible":true}, + {"name":"non_optional","eligible":true}, + {"name":"bounded_endpoint_classes","eligible":true}, + {"name":"within_static_endpoint_caps","eligible":true}, + {"name":"uncorrelated_pair","eligible":true} + ], + "structurally_eligible":true, + "statically_eligible":false, + "selection_mode":"analysis_only", + "selector_version":"endpoint-resolution-v1", + "fallback_reason":"planned_only" + }`, string(diagnostic)) +} + +func TestEndpointResolutionReportsConservativeFallbackReasons(t *testing.T) { + t.Parallel() + + values := make([]string, EndpointResolutionSmallSetSentinel) + for index := range values { + values[index] = fmt.Sprint(index + 1) + } + + testCases := []struct { + name string + query string + reason string + pairClass EndpointResolutionClass + structural bool + }{ + { + name: "read only remains planned", + query: ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = 1 AND id(e) = 2 + RETURN p + `, + reason: EndpointResolutionFallbackPlannedOnly, + structural: true, + }, + { + name: "mutation", + query: ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = 1 AND id(e) = 2 + CREATE (:Audit) + RETURN p + `, + reason: EndpointResolutionFallbackMutation, + }, + { + name: "optional match", + query: ` + OPTIONAL MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = 1 AND id(e) = 2 + RETURN p + `, + reason: EndpointResolutionFallbackOptionalMatch, + }, + { + name: "correlated pair", + query: ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = 1 AND id(e) = 2 AND s.tenant = e.tenant + RETURN p + `, + reason: EndpointResolutionFallbackCorrelatedPair, + pairClass: EndpointResolutionClassCorrelatedPair, + }, + { + name: "small set cap plus one", + query: fmt.Sprintf(` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) IN [%s] AND id(e) IN [100] + RETURN p + `, strings.Join(values, ",")), + reason: EndpointResolutionFallbackSmallSetOverflow, + }, + { + name: "unsupported endpoint syntax", + query: ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE s.name STARTS WITH 'source' AND e.name STARTS WITH 'terminal' + RETURN p + `, + reason: EndpointResolutionFallbackUnsupported, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + plan := optimizeTraversalEnvelope(t, testCase.query) + require.Len(t, plan.EndpointResolution, 1) + decision := plan.EndpointResolution[0] + require.Equal(t, testCase.reason, decision.FallbackReason) + require.Equal(t, testCase.pairClass, decision.PairClass) + require.Equal(t, testCase.structural, decision.StructurallyEligible) + require.False(t, decision.StaticallyEligible) + require.Equal(t, EndpointResolutionPlanIncumbent, decision.SelectedPlan) + }) + } +} + +func TestTraversalPredicateClassifiesLocalUniversalAndWholePathForms(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + predicate string + class TraversalPredicateClass + bindingSymbol string + fallback string + structural bool + }{ + { + name: "endpoint WHERE predicate is not step local", + predicate: "s.enabled = true", + class: TraversalPredicateClassUnsupported, + fallback: TraversalPredicateFallbackUnsupported, + structural: false, + }, + { + name: "range binding WHERE predicate is not step local", + predicate: "rels.enabled = true", + class: TraversalPredicateClassUnsupported, + fallback: TraversalPredicateFallbackUnsupported, + structural: false, + }, + { + name: "all nodes", + predicate: "all(n IN nodes(p) WHERE n.enabled = true)", + class: TraversalPredicateClassUniversalAllNodes, + bindingSymbol: "n", + fallback: TraversalPredicateFallbackPlannedOnly, + structural: true, + }, + { + name: "none nodes", + predicate: "none(n IN nodes(p) WHERE n.disabled = true)", + class: TraversalPredicateClassUniversalNoneNodes, + bindingSymbol: "n", + fallback: TraversalPredicateFallbackPlannedOnly, + structural: true, + }, + { + name: "all relationships", + predicate: "all(r IN relationships(p) WHERE type(r) = 'MemberOf')", + class: TraversalPredicateClassUniversalAllRelationships, + bindingSymbol: "r", + fallback: TraversalPredicateFallbackPlannedOnly, + structural: true, + }, + { + name: "none relationships", + predicate: "none(r IN relationships(p) WHERE type(r) = 'AdminTo')", + class: TraversalPredicateClassUniversalNoneRelationships, + bindingSymbol: "r", + fallback: TraversalPredicateFallbackPlannedOnly, + structural: true, + }, + { + name: "whole path", + predicate: "length(p) > 2", + class: TraversalPredicateClassWholePath, + fallback: TraversalPredicateFallbackWholePath, + structural: false, + }, + { + name: "correlated endpoints", + predicate: "s.tenant = e.tenant", + class: TraversalPredicateClassUnsupported, + fallback: TraversalPredicateFallbackCorrelation, + structural: false, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + plan := optimizeTraversalEnvelope(t, fmt.Sprintf(` + MATCH p = shortestPath((s)-[rels:MemberOf*1..4]->(e)) + WHERE %s + RETURN p + `, testCase.predicate)) + + require.Len(t, plan.TraversalPredicate, 1) + decision := plan.TraversalPredicate[0] + require.Equal(t, testCase.class, decision.Class) + require.Equal(t, testCase.bindingSymbol, decision.BindingSymbol) + require.Equal(t, testCase.fallback, decision.FallbackReason) + require.Equal(t, testCase.structural, decision.StructurallyEligible) + require.False(t, decision.StaticallyEligible) + require.Equal(t, TraversalPredicatePlanIncumbent, decision.SelectedPlan) + require.Equal(t, TraversalPredicatePlanIncumbent, decision.FallbackPlan) + require.Equal(t, "analysis_only", decision.SelectionMode) + require.Contains(t, plan.Decisions(), LoweringDecision{Name: LoweringTraversalPredicateClassification}) + }) + } +} + +func TestTraversalPredicateOnlyClaimsInlineRelationshipPropertiesAsStepLocal(t *testing.T) { + t.Parallel() + + plan := optimizeTraversalEnvelope(t, ` + MATCH p = shortestPath((s {enabled: true})-[rels:MemberOf*1..4{active: true}]->(e)) + RETURN p + `) + require.Len(t, plan.TraversalPredicate, 2) + + require.Equal(t, "relationship_pattern", plan.TraversalPredicate[0].Source) + require.Equal(t, TraversalPredicateClassStepLocalRelationship, plan.TraversalPredicate[0].Class) + require.True(t, plan.TraversalPredicate[0].StructurallyEligible) + require.Equal(t, TraversalPredicateFallbackPlannedOnly, plan.TraversalPredicate[0].FallbackReason) + + require.Equal(t, "node_pattern", plan.TraversalPredicate[1].Source) + require.Equal(t, TraversalPredicateClassUnsupported, plan.TraversalPredicate[1].Class) + require.False(t, plan.TraversalPredicate[1].StructurallyEligible) + require.Equal(t, TraversalPredicateFallbackUnsupported, plan.TraversalPredicate[1].FallbackReason) +} + +func TestTraversalPredicateReportsMutationOptionalAndCorrelationFallbacks(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + query string + reason string + structural bool + }{ + { + name: "read only remains planned", + query: ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE all(n IN nodes(p) WHERE n.enabled = true) + RETURN p + `, + reason: TraversalPredicateFallbackPlannedOnly, + structural: true, + }, + { + name: "mutation", + query: ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE all(n IN nodes(p) WHERE n.enabled = true) + CREATE (:Audit) + RETURN p + `, + reason: TraversalPredicateFallbackMutation, + }, + { + name: "optional match", + query: ` + OPTIONAL MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE all(n IN nodes(p) WHERE n.enabled = true) + RETURN p + `, + reason: TraversalPredicateFallbackOptional, + }, + { + name: "correlated predicate", + query: ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE s.tenant = e.tenant + RETURN p + `, + reason: TraversalPredicateFallbackCorrelation, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + plan := optimizeTraversalEnvelope(t, testCase.query) + require.Len(t, plan.TraversalPredicate, 1) + decision := plan.TraversalPredicate[0] + require.Equal(t, testCase.reason, decision.FallbackReason) + require.Equal(t, testCase.structural, decision.StructurallyEligible) + require.False(t, decision.StaticallyEligible) + require.Equal(t, TraversalPredicatePlanIncumbent, decision.SelectedPlan) + }) + } +} + +func TestTraversalPredicateJSONKeepsCandidatePlannedOnly(t *testing.T) { + t.Parallel() + + plan := optimizeTraversalEnvelope(t, ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE all(n IN nodes(p) WHERE n.enabled = true) + RETURN p + `) + require.Len(t, plan.TraversalPredicate, 1) + + diagnostic, err := json.Marshal(plan.TraversalPredicate[0]) + require.NoError(t, err) + require.Contains(t, string(diagnostic), `"class":"universal_all_nodes"`) + require.Contains(t, string(diagnostic), `"planned_candidates":["TRAVERSAL-PREDICATE-INCUMBENT","TRAVERSAL-PREDICATE-STEP"]`) + require.Contains(t, string(diagnostic), `"selected_plan":"TRAVERSAL-PREDICATE-INCUMBENT"`) + require.Contains(t, string(diagnostic), `"statically_eligible":false`) + require.Contains(t, string(diagnostic), `"fallback_reason":"planned_only"`) +} diff --git a/cypher/models/pgsql/test/logical_forms_legacy_builder_test.go b/cypher/models/pgsql/test/logical_forms_legacy_builder_test.go new file mode 100644 index 00000000..ca6f3fdc --- /dev/null +++ b/cypher/models/pgsql/test/logical_forms_legacy_builder_test.go @@ -0,0 +1,163 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package test + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/query" + "github.com/stretchr/testify/require" +) + +// translateLegacyQuery builds legacy criteria, translates the resulting Cypher, and returns formatted SQL and metadata. +func translateLegacyQuery(t *testing.T, criteria ...graph.Criteria) (string, translate.Result) { + t.Helper() + + builder := query.NewBuilderWithCriteria(criteria...) + regularQuery, err := builder.Build(false) + require.NoError(t, err) + + translation, err := translate.Translate(context.Background(), regularQuery, newKindMapper(), nil, translate.DefaultGraphID) + require.NoError(t, err) + + formatted, err := translate.Translated(translation) + require.NoError(t, err) + return formatted, translation +} + +// TestLegacyBuilderPostgreSQL_LogicalForms verifies boolean grouping, typed thresholds, and binding-local predicates in migrated builder queries. +func TestLegacyBuilderPostgreSQL_LogicalForms(t *testing.T) { + t.Run("LOGIC-01 branch-local relationship kinds", func(t *testing.T) { + formatted, _ := translateLegacyQuery(t, + query.Where(query.Or( + query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Equals(query.EndID(), graph.ID(202)), + query.KindIn(query.Relationship(), graph.StringKind("RegressionKind01")), + ), + query.And( + query.Equals(query.StartID(), graph.ID(202)), + query.Equals(query.EndID(), graph.ID(101)), + query.KindIn(query.Relationship(), graph.StringKind("RegressionKind02")), + ), + )), + query.Returning(query.RelationshipID()), + ) + + require.Contains(t, formatted, " or ") + require.Contains(t, formatted, "n0.id = @pi0") + require.Contains(t, formatted, "n1.id = @pi1") + require.Contains(t, formatted, "n0.id = @pi2") + require.Contains(t, formatted, "n1.id = @pi3") + require.Contains(t, formatted, "e0.kind_id = any") + }) + + t.Run("LOGIC-02 cross-binding temporal disjunction", func(t *testing.T) { + formatted, _ := translateLegacyQuery(t, + query.Where(query.Or( + query.BeforeGraphQuery(query.RelationshipProperty("lastseen"), query.StartProperty("lastcollected")), + query.BeforeGraphQuery(query.RelationshipProperty("lastseen"), query.EndProperty("lastcollected")), + )), + query.Returning(query.RelationshipID()), + ) + + require.Contains(t, formatted, "e0.properties -> 'lastseen'") + require.Contains(t, formatted, "n0.properties -> 'lastcollected'") + require.Contains(t, formatted, "n1.properties -> 'lastcollected'") + require.Contains(t, formatted, " or ") + }) + + t.Run("LOGIC-03 typed threshold and scoped negation", func(t *testing.T) { + threshold := time.Date(2026, time.January, 2, 3, 4, 5, 0, time.UTC) + formatted, translation := translateLegacyQuery(t, + query.Where(query.And( + query.Not(query.KindIn(query.Node(), graph.StringKind("RegressionKind03"))), + query.Or( + query.Not(query.Exists(query.NodeProperty("lastseen"))), + query.Before(query.NodeProperty("lastseen"), threshold), + ), + )), + query.Returning(query.NodeID()), + ) + + require.Contains(t, formatted, "not") + require.Contains(t, formatted, " or ") + require.Contains(t, formatted, "n0.properties -> 'lastseen'") + require.Contains(t, formatted, "@pi0") + require.Equal(t, map[string]any{"pi0": threshold}, translation.Parameters) + }) +} + +// TestLegacyBuilderPostgreSQL_LOGIC05ProjectionOrder verifies that migrated projections preserve caller-specified column order. +func TestLegacyBuilderPostgreSQL_LOGIC05ProjectionOrder(t *testing.T) { + testCases := map[string]struct { + // projection supplies the legacy graph criteria for the case. + projection *graphProjection + // columns lists the SQL fragments in their required projection order. + columns []string + }{ + "full opposite node plus relationship": { + projection: projectionOf(query.Relationship(), query.End()), + columns: []string{"select s0.e0 as r", "s0.n1 as e"}, + }, + "opposite ID and kinds plus relationship ID and kind": { + projection: projectionOf(query.EndID(), query.KindsOf(query.End()), query.RelationshipID(), query.KindsOf(query.Relationship())), + columns: []string{"select (s0.n1).id", "(s0.n1).kind_ids", "(s0.e0).id", "kind_name((s0.e0).kind_id)"}, + }, + "start relationship end triple": { + projection: projectionOf(query.Start(), query.Relationship(), query.End()), + columns: []string{"select s0.n0 as s", "s0.e0 as r", "s0.n1 as e"}, + }, + "relationship ID only": { + projection: projectionOf(query.RelationshipID()), + columns: []string{"select (s0.e0).id"}, + }, + "full relationship": { + projection: projectionOf(query.Relationship()), + columns: []string{"select s0.e0 as r"}, + }, + } + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + formatted, _ := translateLegacyQuery(t, testCase.projection.criteria) + cursor := 0 + for _, column := range testCase.columns { + next := strings.Index(formatted[cursor:], column) + require.NotEqualf(t, -1, next, "missing projection column %q in %s", column, formatted) + cursor += next + len(column) + } + }) + } +} + +// graphProjection keeps the table-driven projection cases strongly typed +// without obscuring that they are legacy query criteria. +type graphProjection struct { + // criteria is the legacy returning criterion represented by this projection. + criteria graph.Criteria +} + +// projectionOf wraps returning criteria in the strongly typed projection used by table-driven cases. +func projectionOf(criteria ...graph.Criteria) *graphProjection { + return &graphProjection{criteria: query.Returning(criteria...)} +} diff --git a/cypher/models/pgsql/test/reconciliation_forms_legacy_builder_test.go b/cypher/models/pgsql/test/reconciliation_forms_legacy_builder_test.go new file mode 100644 index 00000000..5b9a42bc --- /dev/null +++ b/cypher/models/pgsql/test/reconciliation_forms_legacy_builder_test.go @@ -0,0 +1,207 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package test + +import ( + "fmt" + "strings" + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/query" + "github.com/stretchr/testify/require" +) + +// TestLegacyBuilderPostgreSQL_ReconciliationForms verifies migrated relationship reconciliation reads and deletes across kind-set sizes. +func TestLegacyBuilderPostgreSQL_ReconciliationForms(t *testing.T) { + reconciliationKinds := func(count int) graph.Kinds { + kinds := make(graph.Kinds, count) + for idx := range count { + kinds[idx] = graph.StringKind(fmt.Sprintf("RegressionKind%02d", idx+1)) + } + return kinds + } + + assertRelationshipDelete := func(t *testing.T, formatted string) { + t.Helper() + selection := strings.Index(formatted, "select ") + deletion := strings.Index(formatted, "delete from edge e1 using s0") + require.NotEqual(t, -1, selection) + require.Greater(t, deletion, selection, "selection must precede mutation: %s", formatted) + require.Contains(t, formatted, "where (s0.e0).id = e1.id") + } + + for _, count := range []int{1, 2, 9, 30} { + kinds := reconciliationKinds(count) + + t.Run(fmt.Sprintf("REC-01 inbound %d kinds", count), func(t *testing.T) { + formatted, translation := translateLegacyQuery(t, + query.Where(query.And( + query.Kind(query.End(), graph.StringKind("RegressionKind31")), + query.Equals(query.EndProperty("objectid"), "target-id"), + query.KindIn(query.Relationship(), kinds...), + )), + query.Delete(query.Relationship()), + ) + + assertRelationshipDelete(t, formatted) + require.Contains(t, formatted, "n1.id = e0.end_id") + require.Contains(t, formatted, "n1.properties -> 'objectid'") + require.Contains(t, formatted, fmt.Sprintf("array [%s]::int2[]", sequentialKindIDs(33, count))) + require.Equal(t, map[string]any{"pi0": "target-id"}, translation.Parameters) + }) + + t.Run(fmt.Sprintf("REC-02 outbound %d kinds", count), func(t *testing.T) { + formatted, translation := translateLegacyQuery(t, + query.Where(query.And( + query.Kind(query.Start(), graph.StringKind("RegressionKind31")), + query.Equals(query.StartProperty("objectid"), "target-id"), + query.KindIn(query.Relationship(), kinds...), + )), + query.Delete(query.Relationship()), + ) + + assertRelationshipDelete(t, formatted) + require.Contains(t, formatted, "n0.id = e0.start_id") + require.Contains(t, formatted, "n0.properties -> 'objectid'") + require.Contains(t, formatted, fmt.Sprintf("array [%s]::int2[]", sequentialKindIDs(33, count))) + require.Equal(t, map[string]any{"pi0": "target-id"}, translation.Parameters) + }) + } + + testCases := map[string]struct { + // criteria contains the legacy query-builder inputs for the case. + criteria []graph.Criteria + // fragments lists SQL fragments that the translation must contain. + fragments []string + // parameters is the exact parameter map expected from translation. + parameters map[string]any + // read reports whether the case reads rather than deletes a relationship. + read bool + }{ + "REC-03 inbound primary group": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.Kind(query.End(), graph.StringKind("RegressionKind31")), + query.Equals(query.EndProperty("objectid"), "group-id"), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind32")), + query.Equals(query.RelationshipProperty("isprimarygroup"), false), + )), + query.Delete(query.Relationship()), + }, + fragments: []string{"n1.id = e0.end_id", "e0.properties -> 'isprimarygroup'", "delete from edge e1 using s0"}, + parameters: map[string]any{"pi0": "group-id", "pi1": false}, + }, + "REC-03 outbound primary group": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.Kind(query.Start(), graph.StringKind("RegressionKind31")), + query.Equals(query.StartProperty("objectid"), "computer-id"), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind32")), + query.Equals(query.RelationshipProperty("isprimarygroup"), true), + )), + query.Delete(query.Relationship()), + }, + fragments: []string{"n0.id = e0.start_id", "e0.properties -> 'isprimarygroup'", "delete from edge e1 using s0"}, + parameters: map[string]any{"pi0": "computer-id", "pi1": true}, + }, + "REC-04 object ID list relationship delete": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.Kind(query.Relationship(), graph.StringKind("RegressionKind32")), + query.Kind(query.End(), graph.StringKind("RegressionKind31")), + query.In(query.EndProperty("objectid"), []string{"target-1", "target-2"}), + )), + query.Delete(query.Relationship()), + }, + fragments: []string{"n1.id = e0.end_id", "n1.properties ->> 'objectid'", "delete from edge e1 using s0"}, + parameters: map[string]any{"pi0": []string{"target-1", "target-2"}}, + }, + "REC-05 delegated enrollment discovery": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.In(query.EndProperty("objectid"), []string{"ca-1", "ca-2"}), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind32")), + query.Kind(query.Start(), graph.StringKind("RegressionKind31")), + )), + query.Returning(query.Relationship(), query.Start()), + }, + fragments: []string{"select s0.e0 as r, s0.n0 as s", "n1.properties ->> 'objectid'"}, + parameters: map[string]any{"pi0": []string{"ca-1", "ca-2"}}, + read: true, + }, + "REC-06 delegated enrollment delete by IDs": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.Kind(query.End(), graph.StringKind("RegressionKind31")), + query.InIDs(query.EndID(), graph.ID(101), graph.ID(202)), + query.KindIn(query.Relationship(), graph.StringKind("RegressionKind32")), + )), + query.Delete(query.Relationship()), + }, + fragments: []string{"n1.id = any", "delete from edge e1 using s0"}, + parameters: map[string]any{"pi0": []uint64{101, 202}}, + }, + "REC-07 HostsCAService relationship delete": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.Kind(query.End(), graph.StringKind("RegressionKind31")), + query.Equals(query.EndProperty("objectid"), "ca-id"), + query.KindIn(query.Relationship(), graph.StringKind("RegressionKind32")), + )), + query.Delete(query.Relationship()), + }, + fragments: []string{"n1.id = e0.end_id", "delete from edge e1 using s0"}, + parameters: map[string]any{"pi0": "ca-id"}, + }, + "REC-08 AD entity detach delete": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.Kind(query.Node(), graph.StringKind("RegressionKind31")), + query.In(query.NodeProperty("objectid"), []string{"target-1", "target-2"}), + )), + query.Delete(query.Node()), + }, + fragments: []string{"n0.properties ->> 'objectid'", "delete from node n1 using s0", "where (s0.n0).id = n1.id"}, + parameters: map[string]any{"pi0": []string{"target-1", "target-2"}}, + }, + } + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + formatted, translation := translateLegacyQuery(t, testCase.criteria...) + for _, fragment := range testCase.fragments { + require.Contains(t, formatted, fragment) + } + if !testCase.read { + selection := strings.Index(formatted, "select ") + deletion := strings.Index(formatted, "delete from ") + require.Greater(t, deletion, selection, "selection must precede mutation: %s", formatted) + } + require.Equal(t, testCase.parameters, translation.Parameters) + }) + } +} + +// sequentialKindIDs formats count consecutive kind IDs beginning at start for SQL-fragment assertions. +func sequentialKindIDs(first, count int) string { + ids := make([]string, count) + for idx := range count { + ids[idx] = fmt.Sprint(first + idx) + } + return strings.Join(ids, ", ") +} diff --git a/cypher/models/pgsql/test/relationship_scans_node_lookups_legacy_builder_test.go b/cypher/models/pgsql/test/relationship_scans_node_lookups_legacy_builder_test.go new file mode 100644 index 00000000..ed5971c3 --- /dev/null +++ b/cypher/models/pgsql/test/relationship_scans_node_lookups_legacy_builder_test.go @@ -0,0 +1,365 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package test + +import ( + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/query" + "github.com/stretchr/testify/require" +) + +// scanLookupRegressionKinds converts numeric fixture suffixes to their RegressionKind names. +func scanLookupRegressionKinds(numbers ...int) graph.Kinds { + kinds := make(graph.Kinds, len(numbers)) + for idx, number := range numbers { + kinds[idx] = graph.StringKind("RegressionKind" + twoDigitKindSuffix(number)) + } + return kinds +} + +// twoDigitKindSuffix formats a fixture kind number as two decimal digits. +func twoDigitKindSuffix(value int) string { + if value < 10 { + return "0" + string(rune('0'+value)) + } + return string(rune('0'+value/10)) + string(rune('0'+value%10)) +} + +// assertScanLookupTranslation translates criteria and requires every expected SQL fragment to be present. +func assertScanLookupTranslation(t *testing.T, criteria []graph.Criteria, fragments ...string) { + t.Helper() + formatted, _ := translateLegacyQuery(t, criteria...) + for _, fragment := range fragments { + require.Contains(t, formatted, fragment) + } +} + +// TestLegacyBuilderPostgreSQL_RelationshipScans verifies migrated relationship scans preserve endpoint, kind, property, and projection semantics. +func TestLegacyBuilderPostgreSQL_RelationshipScans(t *testing.T) { + t.Run("SCAN-01 base endpoints and relationship ID", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.KindIn(query.Start(), scanLookupRegressionKinds(61, 62)...), + query.Kind(query.Relationship(), scanLookupRegressionKinds(63)[0]), + query.KindIn(query.End(), scanLookupRegressionKinds(61, 62)...), + )), + query.Returning(query.RelationshipID()), + }, "n0.kind_ids", "n1.kind_ids", "array [93, 94]::int2[]", "e0.kind_id = any (array [95]::int2[])", "select (s0.e0).id") + }) + + t.Run("SCAN-02 excludes Meta endpoints and hydrates relationships", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.Not(query.KindIn(query.Start(), scanLookupRegressionKinds(64, 65)...)), + query.KindIn(query.Relationship(), scanLookupRegressionKinds(66, 67)...), + query.Not(query.KindIn(query.End(), scanLookupRegressionKinds(64, 65)...)), + )), + query.Returning(query.Relationship()), + }, "not", "array [96, 97]::int2[]", "array [98, 99]::int2[]", "select s0.e0 as r") + }) + + t.Run("SCAN-03 exists relationship property and ID", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.Not(query.KindIn(query.Start(), scanLookupRegressionKinds(64, 65)...)), + query.Kind(query.Relationship(), scanLookupRegressionKinds(68)[0]), + query.Exists(query.RelationshipProperty("lastseen")), + query.Not(query.KindIn(query.End(), scanLookupRegressionKinds(64, 65)...)), + )), + query.Returning(query.RelationshipID()), + }, "e0.properties ? 'lastseen'", "not (e0.properties -> 'lastseen')", "array [100]::int2[]", "select (s0.e0).id") + }) + + for _, relationshipKind := range []int{70, 71} { + t.Run("SCAN-04 raw ownership representative "+twoDigitKindSuffix(relationshipKind), func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.Kind(query.Relationship(), scanLookupRegressionKinds(relationshipKind)[0]), + query.Kind(query.Start(), scanLookupRegressionKinds(69)[0]), + )), + query.Returning(query.Relationship()), + }, "n0.kind_ids", "array [101]::int2[]", "select s0.e0 as r") + }) + } + + nineKinds := scanLookupRegressionKinds(72, 73, 74, 75, 76, 77, 78, 79, 80) + t.Run("SCAN-05 nine relationship kinds bound end", func(t *testing.T) { + formatted, translation := translateLegacyQuery(t, + query.Where(query.And( + query.Kind(query.Start(), scanLookupRegressionKinds(69)[0]), + query.KindIn(query.Relationship(), nineKinds...), + query.Equals(query.EndID(), graph.ID(202)), + )), + query.Returning(query.Relationship(), query.Start()), + ) + require.Contains(t, formatted, "array [104, 105, 106, 107, 108, 109, 110, 111, 112]::int2[]") + require.Contains(t, formatted, "select s0.e0 as r, s0.n0 as s") + require.Equal(t, map[string]any{"pi0": uint64(202)}, translation.Parameters) + }) + + t.Run("SCAN-06 FetchKinds column order", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.Kind(query.Relationship(), scanLookupRegressionKinds(82)[0]), + query.Kind(query.End(), scanLookupRegressionKinds(81)[0]), + )), + query.Returning(query.StartID(), query.RelationshipID(), query.KindsOf(query.Relationship()), query.EndID()), + }, "select s0.n0 as \"id(s)\", (s0.e0).id as \"id(r)\", kind_name((s0.e0).kind_id)::text as \"type(r)\", (s0.n1).id as \"id(e)\"") + }) + + t.Run("SCAN-07 directed start and end IDs", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.KindIn(query.Relationship(), scanLookupRegressionKinds(83, 84)...)), + query.Returning(query.StartID(), query.EndID()), + }, "array [115, 116]::int2[]", "select s0.n0 as \"id(s)\", s0.n1 as \"id(e)\"") + }) + + t.Run("SCAN-08 scenario A and B", func(t *testing.T) { + for name, testCase := range map[string]struct { + // endKinds optionally constrains the terminal node kinds. + endKinds graph.Kinds + // relKinds constrains the relationship kinds admitted by the scan. + relKinds graph.Kinds + }{ + "scenario A": {relKinds: scanLookupRegressionKinds(87, 88, 89, 90, 91, 92)}, + "scenario B": { + endKinds: scanLookupRegressionKinds(81), + relKinds: scanLookupRegressionKinds(87, 88, 89, 90, 91), + }, + } { + t.Run(name, func(t *testing.T) { + criteria := []graph.Criteria{ + query.KindIn(query.Start(), scanLookupRegressionKinds(85, 86, 81)...), + query.InIDs(query.EndID(), graph.ID(202), graph.ID(303)), + query.KindIn(query.Relationship(), testCase.relKinds...), + } + if len(testCase.endKinds) > 0 { + criteria = append(criteria, query.KindIn(query.End(), testCase.endKinds...)) + } + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And(criteria...)), + query.Returning(query.StartID()), + }, "n0.kind_ids", "n1.id = any", "select (s0.n0).id") + }) + } + }) +} + +// TestLegacyBuilderPostgreSQL_NodeLookups verifies migrated node lookups preserve ID, kind, property, projection, and limit semantics. +func TestLegacyBuilderPostgreSQL_NodeLookups(t *testing.T) { + t.Run("LOOKUP-01 ID and full-node projections", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.KindIn(query.Node(), scanLookupRegressionKinds(85, 86)...)), + query.Returning(query.NodeID()), + }, "array [117, 118]::int2[]", "select (s0.n0).id") + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.Kind(query.Node(), scanLookupRegressionKinds(93)[0])), + query.Returning(query.Node()), + }, "array [125]::int2[]", "select s0.n0 as n") + }) + + t.Run("LOOKUP-02 equalities and limit", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.Kind(query.Node(), scanLookupRegressionKinds(81)[0]), + query.Equals(query.NodeProperty("objectid"), "S-1-5-21"), + )), + query.Returning(query.Node()), + query.Limit(1), + }, "n0.properties -> 'objectid'", "select s0.n0 as n", "limit 1") + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.Equals(query.NodeProperty("name"), "dc.example.test"), + query.Equals(query.NodeProperty("enabled"), true), + )), + query.Returning(query.NodeID()), + }, "n0.properties -> 'name'", "n0.properties -> 'enabled'", "select (s0.n0).id") + }) + + t.Run("LOOKUP-03 boolean two-column projection", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.Kind(query.Node(), scanLookupRegressionKinds(81)[0]), + query.Equals(query.NodeProperty("hasura"), true), + )), + query.Returning(query.NodeID(), query.NodeProperty("hasura")), + }, "select (s0.n0).id as \"id(n)\", ((s0.n0).properties -> 'hasura') as \"n.hasura\"") + }) + + t.Run("LOOKUP-04 prefix suffix and equality", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.Kind(query.Node(), scanLookupRegressionKinds(94)[0]), + query.StringStartsWith(query.NodeProperty("distinguishedname"), "CN=ADMINSDHOLDER,"), + query.Equals(query.NodeProperty("domainsid"), "S-1-5-21"), + )), + query.Returning(query.Node()), + }, "cypher_starts_with", "n0.properties -> 'domainsid'") + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.Or( + query.StringEndsWith(query.NodeProperty("objectid"), "-S-1"), + query.StringEndsWith(query.NodeProperty("objectid"), "-S-2"), + )), + query.Returning(query.NodeID()), + }, "cypher_ends_with", " or ") + }) + + t.Run("LOOKUP-05 case-insensitive strings preserve literals", func(t *testing.T) { + formatted, translation := translateLegacyQuery(t, + query.Where(query.CaseInsensitiveStringStartsWith(query.NodeProperty("name"), "Remote Desktop Users%_")), + query.Returning(query.NodeID()), + ) + require.Contains(t, formatted, "lower") + require.Contains(t, formatted, "cypher_starts_with") + require.Equal(t, map[string]any{"pi0": "remote desktop users%_"}, translation.Parameters) + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.CaseInsensitiveStringContains(query.NodeProperty("objectid"), "Approver_GUID")), + query.Returning(query.Node()), + }, "lower", "cypher_contains") + }) + + t.Run("LOOKUP-06 required and excluded kind groups", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.KindIn(query.Node(), scanLookupRegressionKinds(85, 86)...), + query.Kind(query.Node(), scanLookupRegressionKinds(69)[0]), + query.StringEndsWith(query.NodeProperty("objectid"), "-512"), + query.Equals(query.NodeProperty("domainsid"), "S-1-5-21"), + )), + query.Returning(query.Node()), + }, "array [117, 118]::int2[]", "array [101]::int2[]", "cypher_ends_with") + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.Kind(query.Node(), scanLookupRegressionKinds(69)[0]), + query.Not(query.KindIn(query.Node(), scanLookupRegressionKinds(85, 98)...)), + query.StringEndsWith(query.NodeProperty("objectid"), "-512"), + )), + query.Returning(query.Node()), + }, "not", "array [117, 130]::int2[]") + }) + + t.Run("LOOKUP-07 missing property", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.Not(query.Exists(query.NodeProperty("name")))), + query.Returning(query.Node()), + }, "n0.properties ? 'name'", "not (n0.properties -> 'name')", "not") + }) + + t.Run("LOOKUP-08 nullable approver disjunction", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.Kind(query.Node(), scanLookupRegressionKinds(95)[0]), + query.Equals(query.NodeProperty("tenantid"), "tenant-1"), + query.Equals(query.NodeProperty("approvalrequired"), true), + query.Or( + query.IsNotNull(query.NodeProperty("userapprovers")), + query.IsNotNull(query.NodeProperty("groupapprovers")), + ), + )), + query.Returning(query.Node()), + }, "n0.properties ? 'userapprovers'", "n0.properties ? 'groupapprovers'", " or ") + }) + + t.Run("LOOKUP-09 duplicate ID list hydration", func(t *testing.T) { + formatted, translation := translateLegacyQuery(t, + query.Where(query.InIDs(query.NodeID(), graph.ID(101), graph.ID(202), graph.ID(101))), + query.Returning(query.Node()), + ) + require.Contains(t, formatted, "n0.id = any") + require.Contains(t, formatted, "select s0.n0 as n") + require.Equal(t, map[string]any{"pi0": []uint64{101, 202, 101}}, translation.Parameters) + }) + + t.Run("LOOKUP-10 nested negated flags", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.Kind(query.Node(), scanLookupRegressionKinds(86)[0]), + query.Not(query.And(query.Exists(query.NodeProperty("gmsa")), query.Equals(query.NodeProperty("gmsa"), true))), + query.Not(query.And(query.Exists(query.NodeProperty("msa")), query.Equals(query.NodeProperty("msa"), true))), + query.InIDs(query.NodeID(), graph.ID(101), graph.ID(202)), + )), + query.Returning(query.Node()), + }, "not", "n0.properties -> 'gmsa'", "n0.properties -> 'msa'", "n0.id = any") + }) + + t.Run("LOOKUP-11 tenant adjacency and endpoint property list", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), scanLookupRegressionKinds(97)[0]), + query.KindIn(query.End(), scanLookupRegressionKinds(95, 96)...), + query.In(query.EndProperty("roletemplateid"), []string{"role-a", "role-b"}), + )), + query.Returning(query.End()), + }, "n0.id = @pi0", "array [127, 128]::int2[]", "n1.properties ->> 'roletemplateid'", "select s0.n1 as e") + }) + + t.Run("LOOKUP-12 exact edge key and First", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Equals(query.EndID(), graph.ID(202)), + query.Kind(query.Relationship(), scanLookupRegressionKinds(83)[0]), + )), + query.Returning(query.Relationship()), + query.Limit(1), + }, "n0.id = @pi0", "n1.id = @pi1", "array [115]::int2[]", "select s0.e0 as r", "limit 1") + }) + + t.Run("LOOKUP-13 suffix with bound opposite endpoint projections", func(t *testing.T) { + for _, projection := range []graph.Criteria{query.Returning(query.Start()), query.Returning(query.StartID())} { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.StringEndsWith(query.StartProperty("objectid"), "-555"), + query.Kind(query.Relationship(), scanLookupRegressionKinds(82)[0]), + query.Equals(query.EndID(), graph.ID(202)), + )), + projection, + }, "cypher_ends_with", "n1.id = @pi1") + } + }) + + t.Run("LOOKUP-14 descending property order", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.Kind(query.Node(), scanLookupRegressionKinds(99)[0])), + query.Returning(query.Node()), + query.OrderBy(query.Order(query.NodeProperty("name"), query.Descending())), + }, "select s0.n0 as n", "order by", "desc") + }) + + t.Run("LOOKUP-16 typed and untyped four-property equalities", func(t *testing.T) { + for name, kindCriteria := range map[string]graph.Criteria{ + "typed": query.Kind(query.Node(), scanLookupRegressionKinds(81)[0]), + "untyped": query.And(), + } { + t.Run(name, func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + kindCriteria, + query.Equals(query.NodeProperty("domainsid"), "S-1-5-21"), + query.Equals(query.NodeProperty("isdc"), true), + query.Equals(query.NodeProperty("ldapavailable"), true), + query.Equals(query.NodeProperty("ldapsigning"), false), + )), + query.Returning(query.NodeID()), + }, "n0.properties -> 'domainsid'", "n0.properties -> 'isdc'", "n0.properties -> 'ldapavailable'", "n0.properties -> 'ldapsigning'", "select (s0.n0).id") + }) + } + }) +} diff --git a/cypher/models/pgsql/test/standalone_hop_forms_legacy_builder_test.go b/cypher/models/pgsql/test/standalone_hop_forms_legacy_builder_test.go new file mode 100644 index 00000000..879adef8 --- /dev/null +++ b/cypher/models/pgsql/test/standalone_hop_forms_legacy_builder_test.go @@ -0,0 +1,277 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package test + +import ( + "fmt" + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/query" + "github.com/stretchr/testify/require" +) + +// TestLegacyBuilderPostgreSQL_StandaloneHopForms verifies migrated one-hop queries preserve anchors, direction, kinds, and projections. +func TestLegacyBuilderPostgreSQL_StandaloneHopForms(t *testing.T) { + hopKinds := func(count int) graph.Kinds { + kinds := make(graph.Kinds, count) + for idx := range count { + kinds[idx] = graph.StringKind(fmt.Sprintf("RegressionKind%02d", idx+1)) + } + return kinds + } + + t.Run("HOP-01 exact and one-element IN start anchors", func(t *testing.T) { + for name, anchor := range map[string]graph.Criteria{ + "exact": query.Equals(query.StartID(), graph.ID(101)), + "in": query.InIDs(query.StartID(), graph.ID(101)), + } { + t.Run(name, func(t *testing.T) { + formatted, translation := translateLegacyQuery(t, + query.Where(query.And(anchor, query.Kind(query.Relationship(), graph.StringKind("RegressionKind01")))), + query.Returning(query.Relationship(), query.End()), + ) + require.Contains(t, formatted, "n0.id = e0.start_id") + require.Contains(t, formatted, "e0.kind_id = any (array [33]::int2[])") + require.Contains(t, formatted, "select s0.e0 as r, s0.n1 as e") + if name == "exact" { + require.Equal(t, map[string]any{"pi0": uint64(101)}, translation.Parameters) + } else { + require.Equal(t, map[string]any{"pi0": []uint64{101}}, translation.Parameters) + } + }) + } + }) + + t.Run("HOP-02 end anchor and inbound projection", func(t *testing.T) { + formatted, translation := translateLegacyQuery(t, + query.Where(query.And( + query.Equals(query.EndID(), graph.ID(202)), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind01")), + )), + query.Returning(query.Relationship(), query.Start()), + ) + require.Contains(t, formatted, "n1.id = e0.end_id") + require.Contains(t, formatted, "select s0.e0 as r, s0.n0 as s") + require.Equal(t, map[string]any{"pi0": uint64(202)}, translation.Parameters) + }) + + for _, count := range []int{2, 5, 9, 30} { + kinds := hopKinds(count) + kindIDs := sequentialKindIDs(33, count) + + t.Run(fmt.Sprintf("HOP-03 outbound %d kinds", count), func(t *testing.T) { + formatted, translation := translateLegacyQuery(t, + query.Where(query.And( + query.InIDs(query.StartID(), graph.ID(101)), + query.KindIn(query.Relationship(), kinds...), + )), + query.Returning(query.Relationship(), query.End()), + ) + require.Contains(t, formatted, fmt.Sprintf("array [%s]::int2[]", kindIDs)) + require.Contains(t, formatted, "n0.id = any") + require.Contains(t, formatted, "select s0.e0 as r, s0.n1 as e") + require.Equal(t, map[string]any{"pi0": []uint64{101}}, translation.Parameters) + }) + + t.Run(fmt.Sprintf("HOP-03 inbound %d kinds", count), func(t *testing.T) { + formatted, translation := translateLegacyQuery(t, + query.Where(query.And( + query.InIDs(query.EndID(), graph.ID(202)), + query.KindIn(query.Relationship(), kinds...), + )), + query.Returning(query.Relationship(), query.Start()), + ) + require.Contains(t, formatted, fmt.Sprintf("array [%s]::int2[]", kindIDs)) + require.Contains(t, formatted, "n1.id = any") + require.Contains(t, formatted, "select s0.e0 as r, s0.n0 as s") + require.Equal(t, map[string]any{"pi0": []uint64{202}}, translation.Parameters) + }) + } + + testCases := map[string]struct { + // criteria contains the legacy query-builder inputs for the case. + criteria []graph.Criteria + // fragments lists SQL fragments that the translation must contain. + fragments []string + // parameters is the exact parameter map expected from translation. + parameters map[string]any + }{ + "HOP-04 endpoint kind disjunction": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.InIDs(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind51")), + query.KindIn(query.End(), graph.StringKind("RegressionKind52"), graph.StringKind("RegressionKind53")), + )), + query.Returning(query.Relationship(), query.End()), + }, + fragments: []string{"n1.kind_ids operator (pg_catalog.&&) array [84, 85]::int2[]", "e0.kind_id = any (array [83]::int2[])", "select s0.e0 as r, s0.n1 as e"}, + parameters: map[string]any{"pi0": []uint64{101}}, + }, + "HOP-05 endpoint IDs through variable spelling": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind54")), + query.InIDs(query.End(), graph.ID(202), graph.ID(303)), + )), + query.Returning(query.Relationship(), query.End()), + }, + fragments: []string{"n0.id = @pi0", "n1.id = any", "e0.kind_id = any (array [86]::int2[])", "select s0.e0 as r, s0.n1 as e"}, + parameters: map[string]any{"pi0": uint64(101), "pi1": []uint64{202, 303}}, + }, + "HOP-05 endpoint IDs through identity-function spelling": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.InIDs(query.Start(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind54")), + query.InIDs(query.EndID(), graph.ID(202), graph.ID(303)), + )), + query.Returning(query.Relationship(), query.End()), + }, + fragments: []string{"n0.id = any", "n1.id = any", "e0.kind_id = any (array [86]::int2[])", "select s0.e0 as r, s0.n1 as e"}, + parameters: map[string]any{"pi0": []uint64{101}, "pi1": []uint64{202, 303}}, + }, + "HOP-06 scalar endpoint properties": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind55")), + query.Equals(query.EndProperty("enabled"), true), + query.Equals(query.EndProperty("score"), 7), + query.Equals(query.EndProperty("name"), "target"), + query.Equals(query.EndProperty("isassignabletorole"), "true"), + )), + query.Returning(query.Relationship(), query.End()), + }, + fragments: []string{"n1.properties -> 'enabled'", "n1.properties -> 'score'", "n1.properties -> 'name'", "n1.properties -> 'isassignabletorole'", "e0.kind_id = any (array [87]::int2[])"}, + parameters: map[string]any{"pi0": uint64(101), "pi1": true, "pi2": 7, "pi3": "target", "pi4": "true"}, + }, + "HOP-07 nested production branches": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind56")), + query.Kind(query.End(), graph.StringKind("RegressionKind57")), + query.Or( + query.And( + query.Equals(query.EndProperty("requiresmanagerapproval"), false), + query.GreaterThan(query.EndProperty("schemaversion"), 1), + query.Equals(query.EndProperty("authorizedsignatures"), 0), + query.Equals(query.EndProperty("authenticationenabled"), true), + ), + query.And( + query.Equals(query.EndProperty("requiresmanagerapproval"), false), + query.Equals(query.EndProperty("schemaversion"), 1), + query.Equals(query.EndProperty("authenticationenabled"), true), + ), + ), + )), + query.Returning(query.Relationship(), query.End()), + }, + fragments: []string{" or ", "n1.kind_ids operator (pg_catalog.&&) array [89]::int2[]", "n1.properties -> 'schemaversion'", "n1.properties -> 'authorizedsignatures'", "e0.kind_id = any (array [88]::int2[])"}, + parameters: map[string]any{"pi0": uint64(101), "pi1": false, "pi2": 1, "pi3": 0, "pi4": true, "pi5": false, "pi6": 1, "pi7": true}, + }, + "HOP-08 collection and scalar OR": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind58")), + query.Or( + query.Equals(query.EndProperty("schannelauthenticationenabled"), true), + query.Equals(query.Size(query.EndProperty("effectiveekus")), 0), + query.InInverted(query.EndProperty("effectiveekus"), "1.3.6.1.5.5.7.3.2"), + ), + )), + query.Returning(query.Relationship(), query.End()), + }, + fragments: []string{" or ", "jsonb_array_length", "jsonb_to_text_array", "e0.kind_id = any (array [90]::int2[])"}, + parameters: map[string]any{"pi0": uint64(101), "pi1": true, "pi2": 0, "pi3": "1.3.6.1.5.5.7.3.2"}, + }, + "HOP-09 two-sided ID lists": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.InIDs(query.StartID(), graph.ID(101), graph.ID(202)), + query.InIDs(query.EndID(), graph.ID(303), graph.ID(404)), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind59")), + )), + query.Returning(query.Relationship(), query.End()), + }, + fragments: []string{"n0.id = any", "n1.id = any", "e0.kind_id = any (array [91]::int2[])", "select s0.e0 as r, s0.n1 as e"}, + parameters: map[string]any{"pi0": []uint64{101, 202}, "pi1": []uint64{303, 404}}, + }, + "HOP-10 outbound full direction": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.InIDs(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind60")), + query.Kind(query.End(), graph.StringKind("RegressionKind52")), + query.Equals(query.EndProperty("active"), true), + )), + query.Returning(query.Relationship(), query.End()), + }, + fragments: []string{"n1.kind_ids operator (pg_catalog.&&) array [84]::int2[]", "n1.properties -> 'active'", "select s0.e0 as r, s0.n1 as e"}, + parameters: map[string]any{"pi0": []uint64{101}, "pi1": true}, + }, + "HOP-10 inbound full direction": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.InIDs(query.EndID(), graph.ID(202)), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind60")), + query.Kind(query.Start(), graph.StringKind("RegressionKind51")), + query.Equals(query.StartProperty("active"), true), + )), + query.Returning(query.Relationship(), query.Start()), + }, + fragments: []string{"n0.kind_ids operator (pg_catalog.&&) array [83]::int2[]", "n0.properties -> 'active'", "select s0.e0 as r, s0.n0 as s"}, + parameters: map[string]any{"pi0": []uint64{202}, "pi1": true}, + }, + "HOP-10 start node projection": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.InIDs(query.EndID(), graph.ID(202)), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind60")), + )), + query.Returning(query.Start()), + }, + fragments: []string{"select s0.n0 as s"}, + parameters: map[string]any{"pi0": []uint64{202}}, + }, + "HOP-10 end ID relationship projection": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.InIDs(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind60")), + )), + query.Returning(query.EndID(), query.Relationship()), + }, + fragments: []string{"select s0.n1 as \"id(e)\", s0.e0 as r"}, + parameters: map[string]any{"pi0": []uint64{101}}, + }, + } + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + formatted, translation := translateLegacyQuery(t, testCase.criteria...) + for _, fragment := range testCase.fragments { + require.Contains(t, formatted, fragment) + } + require.Equal(t, testCase.parameters, translation.Parameters) + }) + } +} diff --git a/cypher/models/pgsql/test/testcase.go b/cypher/models/pgsql/test/testcase.go index 65dcf571..69297e7f 100644 --- a/cypher/models/pgsql/test/testcase.go +++ b/cypher/models/pgsql/test/testcase.go @@ -1,6 +1,7 @@ package test import ( + "bytes" "context" "embed" "encoding/json" @@ -25,12 +26,21 @@ import ( ) const ( - prefixCase = "case:" + // prefixCase introduces a named translation case in a fixture file. + prefixCase = "case:" + + // prefixExclusiveTest marks a fixture case that must run without the other cases. prefixExclusiveTest = "exclusive:" - prefixCypherParams = "cypher_params:" - prefixPgSQLParams = "pgsql_params:" + + // prefixCypherParams introduces the JSON parameter map supplied to Cypher translation. + prefixCypherParams = "cypher_params:" + + // prefixPgSQLParams introduces the JSON parameter map expected in rendered PostgreSQL. + prefixPgSQLParams = "pgsql_params:" ) +// testCaseFiles embeds the translation fixtures consumed by the package test runner. +// //go:embed translation_cases/* var testCaseFiles embed.FS @@ -61,6 +71,7 @@ func (s *TranslationTestCase) Copy() *TranslationTestCase { } } +// writeStrings writes each string to writer in order and returns the first write failure. func writeStrings(output io.Writer, strs ...string) error { for _, str := range strs { if _, err := output.Write([]byte(str)); err != nil { @@ -71,6 +82,7 @@ func writeStrings(output io.Writer, strs ...string) error { return nil } +// licenseHeader is the exact header required at the start of every generated fixture file. var licenseHeader = `-- Copyright %d Specter Ops, Inc. -- -- Licensed under the Apache License, Version 2.0 @@ -147,6 +159,7 @@ func (s *TranslationTestCase) WriteTo(output io.Writer, kindMapper pgsql.KindMap return nil } +// Assert translates the case and compares normalized SQL and parameters with the golden expectations. func (s *TranslationTestCase) Assert(t *testing.T, expectedSQL string, kindMapper pgsql.KindMapper) { if regularQuery, err := frontend.ParseCypher(frontend.NewContext(), s.Cypher); err != nil { t.Fatalf("Failed to compile cypher query: %s - %v", s.Cypher, err) @@ -177,7 +190,15 @@ func (s *TranslationTestCase) Assert(t *testing.T, expectedSQL string, kindMappe require.Equalf(t, expectedSQL, normalizedActual, "Test case for cypher query: '%s' failed to match.", s.Cypher) if s.PgSQLParams != nil { - require.Equal(t, s.PgSQLParams, translation.Parameters) + // Golden parameters are stored as JSON, whose decoder represents + // numbers as float64. Compare the translated bag through the same + // serialization boundary so typed integer parameters do not create a + // false mismatch while their values and emitted casts remain exact. + var normalizedParameters map[string]any + encodedParameters, err := json.Marshal(translation.Parameters) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(encodedParameters, &normalizedParameters)) + require.Equal(t, s.PgSQLParams, normalizedParameters) } } } @@ -323,6 +344,7 @@ func ReadTranslationTestCaseFile(path string, fin fs.File) (TranslationTestCaseF }, err } +// updatedCasesDir returns the configured fixture update directory or an isolated temporary directory. func updatedCasesDir() (string, error) { if workingDir, err := os.Getwd(); err != nil { return "", err @@ -337,6 +359,7 @@ func updatedCasesDir() (string, error) { } } +// UpdateTranslationTestCases regenerates SQL golden files from their embedded Cypher cases. func UpdateTranslationTestCases(mapper pgsql.KindMapper) error { if updatedCasesPath, err := updatedCasesDir(); err != nil { return err @@ -357,20 +380,28 @@ func UpdateTranslationTestCases(mapper pgsql.KindMapper) error { return err } else if nextCases, _, err := caseFile.Load(); err != nil { return err - } else if output, err := os.OpenFile(updatedCaseFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644); err != nil { - return err } else { + var output strings.Builder formattedLicenseHeader := fmt.Sprintf(licenseHeader, time.Now().Year()) - if _, err := io.WriteString(output, formattedLicenseHeader); err != nil { + if _, err := io.WriteString(&output, formattedLicenseHeader); err != nil { return err } for _, nextCase := range nextCases { - nextCase.WriteTo(output, mapper) + if err := nextCase.WriteTo(&output, mapper); err != nil { + return err + } } - output.Close() + trailingNewlines := "\n" + if bytes.HasSuffix(caseFile.content, []byte("\n\n")) { + trailingNewlines = "\n\n" + } + content := strings.TrimRight(output.String(), "\n") + trailingNewlines + if err := os.WriteFile(updatedCaseFilePath, []byte(content), 0644); err != nil { + return err + } } } } diff --git a/cypher/models/pgsql/test/translation_cases/delete.sql b/cypher/models/pgsql/test/translation_cases/delete.sql index c6695b5d..db750976 100644 --- a/cypher/models/pgsql/test/translation_cases/delete.sql +++ b/cypher/models/pgsql/test/translation_cases/delete.sql @@ -21,5 +21,7 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; -- case: match ()-[]->()-[r:EdgeKind1]->() delete r -with s0 as (select e0.id as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, (e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties)::edgecomposite as e1, s0.n1 as n1 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.kind_id = any (array [3]::int2[]) and e1.id != s0.e0), s2 as (delete from edge e2 using s1 where (s1.e1).id = e2.id) select 1; +with s0 as (select e0.id as e0, n1.id as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, (e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties)::edgecomposite as e1, s0.n1 as n1 from s0 join edge e1 on s0.n1 = e1.start_id join node n2 on n2.id = e1.end_id where e1.kind_id = any (array [3]::int2[]) and e1.id != s0.e0), s2 as (delete from edge e2 using s1 where (s1.e1).id = e2.id) select 1; +-- case: match (s)-[*1..]->(mid)-[]->(e) delete mid +with s0 as (with recursive s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, true, false, array [e0.id] from edge e0 join node n1 on n1.id = e0.end_id union all select s1.root_id, e0.end_id, s1.depth + 1, true, false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied and exists (select 1 from edge e1 join node n2 on n2.id = e1.end_id where n1.id = e1.start_id)), s2 as (select s0.ep0 as ep0, s0.n0 as n0, s0.n1 as n1 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != all (s0.ep0)), s3 as (delete from node n3 using s2 where (s2.n1).id = n3.id) select 1; diff --git a/cypher/models/pgsql/test/translation_cases/multipart.sql b/cypher/models/pgsql/test/translation_cases/multipart.sql index e7995068..3229c44e 100644 --- a/cypher/models/pgsql/test/translation_cases/multipart.sql +++ b/cypher/models/pgsql/test/translation_cases/multipart.sql @@ -24,13 +24,13 @@ with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposit with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((n0.properties -> 'value'))::jsonb = to_jsonb((1)::int8)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s1.n0 as n0 from s1), s2 as (with s3 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'me'))) select s3.n1 as n1 from s3), s4 as (select s2.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s2, node n2 where (n2.id = (s2.n1).id)) select s4.n2 as b from s4; -- case: match (n:NodeKind1)-[:EdgeKind1*1..]->(:NodeKind2)-[:EdgeKind2]->(m:NodeKind1) where (n:NodeKind1 or n:NodeKind2) and n.enabled = true with m, collect(distinct(n)) as p where size(p) >= 10 return m -with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [2]::int2[]) and ((n0.properties -> 'enabled'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], e0.start_id = e0.end_id, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) union all select s2.root_id, e0.end_id, s2.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 15 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and exists (select 1 from edge e1 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where n1.id = e1.start_id and e1.kind_id = any (array [4]::int2[]))), s3 as (select s1.ep0 as ep0, s1.n0 as n0, s1.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s1 join edge e1 on (s1.n1).id = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) and e1.id != all (s1.ep0)) select s3.n2 as n2, array_remove(coalesce(array_agg(distinct (s3.n0))::nodecomposite[], array []::nodecomposite[])::nodecomposite[], null)::nodecomposite[] as i0 from s3 group by n2) select s0.n2 as m from s0 where (cardinality(s0.i0)::int >= 10); +with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [2]::int2[]) and ((n0.properties -> 'enabled'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) union all select s2.root_id, e0.end_id, s2.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 15 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and exists (select 1 from edge e1 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where n1.id = e1.start_id and e1.kind_id = any (array [4]::int2[]))), s3 as (select s1.ep0 as ep0, s1.n0 as n0, s1.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s1 join edge e1 on s1.n1 = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) and e1.id != all (s1.ep0)) select s3.n2 as n2, array_remove(coalesce(array_agg(distinct (s3.n0))::nodecomposite[], array []::nodecomposite[])::nodecomposite[], null)::nodecomposite[] as i0 from s3 group by n2) select s0.n2 as m from s0 where (cardinality(s0.i0)::int >= 10); -- case: match (n:NodeKind1)-[:EdgeKind1*1..]->(:NodeKind2)-[:EdgeKind2]->(m:NodeKind1) where (n:NodeKind1 or n:NodeKind2) and n.enabled = true with m, count(distinct(n)) as p where p >= 10 return m -with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [2]::int2[]) and ((n0.properties -> 'enabled'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], e0.start_id = e0.end_id, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) union all select s2.root_id, e0.end_id, s2.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 15 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and exists (select 1 from edge e1 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where n1.id = e1.start_id and e1.kind_id = any (array [4]::int2[]))), s3 as (select s1.ep0 as ep0, s1.n0 as n0, s1.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s1 join edge e1 on (s1.n1).id = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) and e1.id != all (s1.ep0)) select s3.n2 as n2, count(distinct (s3.n0))::int8 as i0 from s3 group by n2) select s0.n2 as m from s0 where (s0.i0 >= 10); +with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [2]::int2[]) and ((n0.properties -> 'enabled'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) union all select s2.root_id, e0.end_id, s2.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 15 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and exists (select 1 from edge e1 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where n1.id = e1.start_id and e1.kind_id = any (array [4]::int2[]))), s3 as (select s1.ep0 as ep0, s1.n0 as n0, s1.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s1 join edge e1 on s1.n1 = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) and e1.id != all (s1.ep0)) select s3.n2 as n2, count(distinct (s3.n0))::int8 as i0 from s3 group by n2) select s0.n2 as m from s0 where (s0.i0 >= 10); -- case: match (n:NodeKind1)-[:EdgeKind1*1..]->(:NodeKind2)-[:EdgeKind2]->(m:NodeKind1) where (n:NodeKind1 or n:NodeKind2) and n.enabled = true with m, count(distinct(n)) as p where p >= 10 return m -with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [2]::int2[]) and ((n0.properties -> 'enabled'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], e0.start_id = e0.end_id, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) union all select s2.root_id, e0.end_id, s2.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 15 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and exists (select 1 from edge e1 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where n1.id = e1.start_id and e1.kind_id = any (array [4]::int2[]))), s3 as (select s1.ep0 as ep0, s1.n0 as n0, s1.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s1 join edge e1 on (s1.n1).id = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) and e1.id != all (s1.ep0)) select s3.n2 as n2, count(distinct (s3.n0))::int8 as i0 from s3 group by n2) select s0.n2 as m from s0 where (s0.i0 >= 10); +with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [2]::int2[]) and ((n0.properties -> 'enabled'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) union all select s2.root_id, e0.end_id, s2.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 15 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and exists (select 1 from edge e1 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where n1.id = e1.start_id and e1.kind_id = any (array [4]::int2[]))), s3 as (select s1.ep0 as ep0, s1.n0 as n0, s1.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s1 join edge e1 on s1.n1 = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) and e1.id != all (s1.ep0)) select s3.n2 as n2, count(distinct (s3.n0))::int8 as i0 from s3 group by n2) select s0.n2 as m from s0 where (s0.i0 >= 10); -- case: with 365 as max_days match (n:NodeKind1) where n.pwdlastset < (datetime().epochseconds - (max_days * 86400)) and not n.pwdlastset IN [-1.0, 0.0] return n limit 100 with s0 as (select 365 as i0), s1 as (select s0.i0 as i0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from s0, node n0 where (not ((n0.properties ->> 'pwdlastset'))::float8 = any (array [- 1, 0]::float8[]) and ((n0.properties ->> 'pwdlastset'))::numeric < (extract(epoch from now()::timestamp with time zone)::numeric - (s0.i0 * 86400))) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s1.n0 as n from s1 limit 100; @@ -39,10 +39,10 @@ with s0 as (select 365 as i0), s1 as (select s0.i0 as i0, (n0.id, n0.kind_ids, n with recursive candidate_sources(root_id) as (select source_node.id as root_id from node source_node where (((source_node.properties -> 'hasspn'))::jsonb = to_jsonb((true)::bool)::jsonb and ((source_node.properties -> 'enabled'))::jsonb = to_jsonb((true)::bool)::jsonb and not coalesce((source_node.properties ->> 'objectid'), '')::text like '%-502' and not coalesce(((source_node.properties ->> 'gmsa'))::bool, false)::bool = true and not coalesce(((source_node.properties ->> 'msa'))::bool, false)::bool = true) and source_node.kind_ids operator (pg_catalog.@>) array [1]::int2[]), traversal(root_id, next_id, depth, path) as (select candidate_sources.root_id, e.end_id, 1, array [e.id]::int8[] from candidate_sources join edge e on e.start_id = candidate_sources.root_id where e.kind_id = any (array [3, 4]::int2[]) union all select traversal.root_id, e.end_id, traversal.depth + 1, traversal.path || e.id from traversal join lateral (select e.id, e.start_id, e.end_id from edge e where e.start_id = traversal.next_id and e.id != all (traversal.path) and e.kind_id = any (array [3, 4]::int2[]) offset 0) e on true where traversal.depth < 15), terminal_nodes(id) as materialized (select terminal_node.id from node terminal_node where terminal_node.kind_ids operator (pg_catalog.@>) array [2]::int2[]), terminal_hits(root_id) as (select traversal.root_id from traversal join terminal_nodes on terminal_nodes.id = traversal.next_id), ranked(root_id, adminCount) as (select terminal_hits.root_id, count(*)::int8 as adminCount from terminal_hits group by terminal_hits.root_id order by adminCount desc limit 100) select (source_node.id, source_node.kind_ids, source_node.properties)::nodecomposite as n from ranked join node source_node on source_node.id = ranked.root_id order by ranked.adminCount desc; -- case: match (n:NodeKind1) where n.objectid = 'S-1-5-21-1260426776-3623580948-1897206385-23225' match p = (n)-[:EdgeKind1|EdgeKind2*1..]->(c:NodeKind2) return p -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'objectid')) = 'string' and (n0.properties ->> 'objectid') = 'S-1-5-21-1260426776-3623580948-1897206385-23225')) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (with recursive s2_seed(root_id) as not materialized (select distinct (s0.n0).id as root_id from s0), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], e0.start_id = e0.end_id, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s2.root_id, e0.end_id, s2.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 15 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and (s0.n0).id = s2.root_id) select case when (s1.n0).id is null or s1.ep0 is null or (s1.n1).id is null then null else ordered_edges_to_path(s1.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s1.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as p from s1; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'objectid')) = 'string' and (n0.properties ->> 'objectid') = 'S-1-5-21-1260426776-3623580948-1897206385-23225')) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (with recursive s2_seed(root_id) as not materialized (select distinct (s0.n0).id as root_id from s0), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s2.root_id, e0.end_id, s2.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 15 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and (s0.n0).id = s2.root_id) select case when (s1.n0).id is null or s1.ep0 is null or (s1.n1).id is null then null else ordered_edge_ids_to_path(0, s1.n0, s1.ep0, array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as p from s1; -- case: match (a) with a match (b) with a, b match (a)-[]-(b) return a -with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s1.n0 as n0 from s1), s2 as (with s3 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1) select s3.n0 as n0, s3.n1 as n1 from s3), s4 as (select s2.n0 as n0, s2.n1 as n1 from s2 join edge e0 on (((s2.n0).id = e0.start_id and (s2.n1).id = e0.end_id) or ((s2.n1).id = e0.start_id and (s2.n0).id = e0.end_id)) where ((s2.n0).id <> (s2.n1).id)) select s4.n0 as a from s4; +with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s1.n0 as n0 from s1), s2 as (with s3 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1) select s3.n0 as n0, s3.n1 as n1 from s3), s4 as (select s2.n0 as n0, s2.n1 as n1 from s2 join edge e0 on (((s2.n0).id = e0.start_id and (s2.n1).id = e0.end_id) or ((s2.n1).id = e0.start_id and (s2.n0).id = e0.end_id))) select s4.n0 as a from s4; -- case: match (g1:NodeKind1) where g1.name starts with 'test' with collect (g1.domain) as excludes match (d:NodeKind2) where d.name starts with 'other' and not d.name in excludes return d with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((n0.properties ->> 'name') like 'test%') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select array_remove(coalesce(array_agg(((s1.n0).properties ->> 'domain'))::anyarray, array []::text[])::anyarray, null)::anyarray as i0 from s1), s2 as (select s0.i0 as i0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where (not (n1.properties ->> 'name') = any (s0.i0) and (n1.properties ->> 'name') like 'other%') and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]) select s2.n1 as d from s2; @@ -51,25 +51,25 @@ with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposit with s0 as (select 'a' as i0), s1 as (select s0.i0 as i0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from s0, node n0 where ((jsonb_typeof((n0.properties -> 'domain')) = 'string' and (n0.properties ->> 'domain') = ' ') and cypher_starts_with((n0.properties ->> 'name'), (i0)::text)::bool) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s1.n0 as o from s1; -- case: match (dc)-[r:EdgeKind1*0..]->(g:NodeKind1) where g.objectid ends with '-516' with collect(dc) as exclude match p = (c:NodeKind2)-[n:EdgeKind2]->(u:NodeKind2)-[:EdgeKind2*1..]->(g:NodeKind1) where g.objectid ends with '-512' and not c in exclude return p limit 100 -with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ((n1.properties ->> 'objectid') like '%-516') and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select s2_seed.root_id, s2_seed.root_id, 0, false, false, array []::int8[] from s2_seed union all select e0.end_id, e0.start_id, 1, false, e0.end_id = e0.start_id, array [e0.id] from s2_seed join edge e0 on e0.end_id = s2_seed.root_id where e0.kind_id = any (array [3]::int2[]) union all select s2.root_id, e0.start_id, s2.depth + 1, false, false, e0.id || s2.path from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true where s2.depth < 15 and not s2.is_cycle and s2.depth > 0) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.root_id offset 0) n1 on true join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.next_id offset 0) n0 on true) select array_remove(coalesce(array_agg((n0).id)::int8[], array []::int8[])::int8[], null)::int8[] as i0 from s1), s3 as (select e1.id as e1, s0.i0 as i0, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3 from s0, edge e1 join node n3 on n3.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n3.id = e1.end_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e1.start_id where (not n2.id = any (s0.i0)) and e1.kind_id = any (array [4]::int2[])), s4 as (with recursive s5_seed(root_id) as not materialized (select distinct (s3.n3).id as root_id from s3), s5(root_id, next_id, depth, satisfied, is_cycle, path) as (select e2.start_id, e2.end_id, 1, ((n4.properties ->> 'objectid') like '%-512') and n4.kind_ids operator (pg_catalog.@>) array [1]::int2[], e2.start_id = e2.end_id, array [e2.id] from s5_seed join edge e2 on e2.start_id = s5_seed.root_id join node n4 on n4.id = e2.end_id where e2.kind_id = any (array [4]::int2[]) union all select s5.root_id, e2.end_id, s5.depth + 1, ((n4.properties ->> 'objectid') like '%-512') and n4.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s5.path || e2.id from s5 join lateral (select e2.id, e2.start_id, e2.end_id, e2.kind_id, e2.properties from edge e2 where e2.start_id = s5.next_id and e2.id != all (s5.path) and e2.kind_id = any (array [4]::int2[]) offset 0) e2 on true join node n4 on n4.id = e2.end_id where s5.depth < 15 and not s5.is_cycle) select s3.e1 as e1, s5.path as ep1, s3.i0 as i0, s3.n2 as n2, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3, (n4.id, n4.kind_ids, n4.properties)::nodecomposite as n4 from s3, s5 join lateral (select n3.id, n3.kind_ids, n3.properties from node n3 where n3.id = s5.root_id offset 0) n3 on true join lateral (select n4.id, n4.kind_ids, n4.properties from node n4 where n4.id = s5.next_id offset 0) n4 on true where s5.satisfied and (s3.n3).id = s5.root_id limit 100) select case when (s4.n2).id is null or s4.e1 is null or (s4.n3).id is null or s4.ep1 is null or (s4.n4).id is null then null else ordered_edges_to_path(s4.n2, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s4.e1]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id) || (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s4.ep1) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s4.n2, s4.n3, s4.n4]::nodecomposite[])::pathcomposite end as p from s4 limit 100; +with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ((n1.properties ->> 'objectid') like '%-516') and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select s2_seed.root_id, s2_seed.root_id, 0, false, false, array []::int8[] from s2_seed union all select e0.end_id, e0.start_id, 1, false, false, array [e0.id] from s2_seed join edge e0 on e0.end_id = s2_seed.root_id where e0.kind_id = any (array [3]::int2[]) union all select s2.root_id, e0.start_id, s2.depth + 1, false, false, e0.id || s2.path from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true where s2.depth < 15 and not s2.is_cycle and s2.depth > 0) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.root_id offset 0) n1 on true join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.next_id offset 0) n0 on true) select array_remove(coalesce(array_agg((n0).id)::int8[], array []::int8[])::int8[], null)::int8[] as i0 from s1), s3 as (select e1.id as e1, s0.i0 as i0, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3 from s0, edge e1 join node n3 on n3.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n3.id = e1.end_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e1.start_id where (not n2.id = any (s0.i0)) and e1.kind_id = any (array [4]::int2[])), s4 as (with recursive s4_endpoint_seeded_endpoints as materialized (select n4.id as id, (n4.id, n4.kind_ids, n4.properties)::nodecomposite as n4 from node n4 where ((n4.properties ->> 'objectid') like '%-512') and n4.kind_ids operator (pg_catalog.@>) array [1]::int2[] limit 33), s4_endpoint_seeded_reverse(root_id, next_id, depth, path) as (select s4_endpoint_seeded_endpoints.id, s4_endpoint_seeded_endpoints.id, 0, array []::int8[] from s4_endpoint_seeded_endpoints union all select s4_endpoint_seeded_reverse.root_id, e2.start_id, s4_endpoint_seeded_reverse.depth + 1, array_prepend(e2.id, s4_endpoint_seeded_reverse.path)::int8[] from s4_endpoint_seeded_reverse join edge e2 on e2.end_id = s4_endpoint_seeded_reverse.next_id where s4_endpoint_seeded_reverse.depth < 15 and e2.id != all (s4_endpoint_seeded_reverse.path) and e2.kind_id = any (array [4]::int2[])), s4_endpoint_seeded_states as materialized (select s4_endpoint_seeded_reverse.root_id, s4_endpoint_seeded_reverse.next_id, s4_endpoint_seeded_reverse.depth, s4_endpoint_seeded_reverse.path from s4_endpoint_seeded_reverse limit 4097), s4_endpoint_seeded_incumbent as materialized (with recursive s5_seed(root_id) as not materialized (select distinct (s3.n3).id as root_id from s3), s5(root_id, next_id, depth, satisfied, is_cycle, path) as (select e2.start_id, e2.end_id, 1, ((n4.properties ->> 'objectid') like '%-512') and n4.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e2.id] from s5_seed join edge e2 on e2.start_id = s5_seed.root_id join node n4 on n4.id = e2.end_id where e2.kind_id = any (array [4]::int2[]) union all select s5.root_id, e2.end_id, s5.depth + 1, ((n4.properties ->> 'objectid') like '%-512') and n4.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s5.path || e2.id from s5 join lateral (select e2.id, e2.start_id, e2.end_id, e2.kind_id, e2.properties from edge e2 where e2.start_id = s5.next_id and e2.id != all (s5.path) and e2.kind_id = any (array [4]::int2[]) offset 0) e2 on true join node n4 on n4.id = e2.end_id where s5.depth < 15 and not s5.is_cycle) select s3.e1 as e1, s5.path as ep1, s3.i0 as i0, s3.n2 as n2, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3, (n4.id, n4.kind_ids, n4.properties)::nodecomposite as n4 from s3, s5 join lateral (select n3.id, n3.kind_ids, n3.properties from node n3 where n3.id = s5.root_id offset 0) n3 on true join lateral (select n4.id, n4.kind_ids, n4.properties from node n4 where n4.id = s5.next_id offset 0) n4 on true where s5.satisfied and (s3.n3).id = s5.root_id and not s5.path && array [s3.e1]::int8[]) select s3.e1 as e1, s4_endpoint_seeded_states.path as ep1, s3.i0 as i0, s3.n2 as n2, s3.n3 as n3, s4_endpoint_seeded_endpoints.n4 as n4 from s3 join s4_endpoint_seeded_states on (s3.n3).id = s4_endpoint_seeded_states.next_id join s4_endpoint_seeded_endpoints on s4_endpoint_seeded_endpoints.id = s4_endpoint_seeded_states.root_id where not exists (select 1 from s4_endpoint_seeded_endpoints offset 32 limit 1) and not exists (select 1 from s4_endpoint_seeded_states offset 4096 limit 1) and s4_endpoint_seeded_states.depth >= 1 and not s4_endpoint_seeded_states.path && array [s3.e1]::int8[] union all select s4_endpoint_seeded_incumbent.e1 as e1, s4_endpoint_seeded_incumbent.ep1 as ep1, s4_endpoint_seeded_incumbent.i0 as i0, s4_endpoint_seeded_incumbent.n2 as n2, s4_endpoint_seeded_incumbent.n3 as n3, s4_endpoint_seeded_incumbent.n4 as n4 from s4_endpoint_seeded_incumbent where exists (select 1 from s4_endpoint_seeded_endpoints offset 32 limit 1) or exists (select 1 from s4_endpoint_seeded_states offset 4096 limit 1) limit 100) select case when (s4.n2).id is null or s4.e1 is null or (s4.n3).id is null or s4.ep1 is null or (s4.n4).id is null then null else ordered_edge_ids_to_path(0, s4.n2, array [s4.e1]::int8[] || s4.ep1, array [s4.n2, s4.n3, s4.n4]::nodecomposite[])::pathcomposite end as p from s4 limit 100; -- case: match (n:NodeKind1)<-[:EdgeKind1]-(:NodeKind2) where n.objectid ends with '-516' with n, count(n) as dc_count where dc_count = 1 return n with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from edge e0 join node n0 on ((n0.properties ->> 'objectid') like '%-516') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.end_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[])) select s1.n0 as n0, count(s1.n0)::int8 as i0 from s1 group by n0) select s0.n0 as n from s0 where (s0.i0 = 1); -- case: match (n:NodeKind1)-[:EdgeKind1]->(m:NodeKind2) where n.enabled = true with n, collect(distinct(n)) as p where size(p) >= 100 match p = (n)-[:EdgeKind1]->(m) return p limit 10 -with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from edge e0 join node n0 on (((n0.properties -> 'enabled'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[])) select s1.n0 as n0, array_remove(coalesce(array_agg(distinct (s1.n0))::nodecomposite[], array []::nodecomposite[])::nodecomposite[], null)::nodecomposite[] as i0 from s1 group by n0), s2 as (select e1.id as e1, s0.i0 as i0, s0.n0 as n0, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (cardinality(s0.i0)::int >= 100) and (s0.n0).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.kind_id = any (array [3]::int2[]) limit 10) select case when (s2.n0).id is null or s2.e1 is null or (s2.n2).id is null then null else ordered_edges_to_path(s2.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s2.e1]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s2.n0, s2.n2]::nodecomposite[])::pathcomposite end as p from s2 limit 10; +with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from edge e0 join node n0 on (((n0.properties -> 'enabled'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[])) select s1.n0 as n0, array_remove(coalesce(array_agg(distinct (s1.n0))::nodecomposite[], array []::nodecomposite[])::nodecomposite[], null)::nodecomposite[] as i0 from s1 group by n0), s2 as (select e1.id as e1, s0.i0 as i0, s0.n0 as n0, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (cardinality(s0.i0)::int >= 100) and (s0.n0).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.kind_id = any (array [3]::int2[]) limit 10) select case when (s2.n0).id is null or s2.e1 is null or (s2.n2).id is null then null else ordered_edge_ids_to_path(0, s2.n0, array [s2.e1]::int8[], array [s2.n0, s2.n2]::nodecomposite[])::pathcomposite end as p from s2 limit 10; -- case: with "a" as check, "b" as ref match p = (u)-[:EdgeKind1]->(g:NodeKind1) where u.name starts with check and u.domain = ref with collect(tolower(g.samaccountname)) as refmembership, tolower(u.samaccountname) as samname return refmembership, samname with s0 as (select 'a' as i0, 'b' as i1), s1 as (with s2 as (select s0.i0 as i0, s0.i1 as i1, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) and ((n0.properties ->> 'domain') = s0.i1 and cypher_starts_with((n0.properties ->> 'name'), (i0)::text)::bool)) select array_remove(coalesce(array_agg(lower(((s2.n1).properties ->> 'samaccountname'))::text)::text[], array []::text[])::text[], null)::text[] as i2, lower(((s2.n0).properties ->> 'samaccountname'))::text as i3 from s2 group by lower(((s2.n0).properties ->> 'samaccountname'))::text) select s1.i2 as refmembership, s1.i3 as samname from s1; -- case: with "a" as check, "b" as ref match p = (u)-[:EdgeKind1]->(g:NodeKind1) where u.name starts with check and u.domain = ref with collect(tolower(g.samaccountname)) as refmembership, tolower(u.samaccountname) as samname match (u)-[:EdgeKind2]-(g:NodeKind1) where tolower(u.samaccountname) = samname and not tolower(g.samaccountname) IN refmembership return g -with s0 as (select 'a' as i0, 'b' as i1), s1 as (with s2 as (select s0.i0 as i0, s0.i1 as i1, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) and ((n0.properties ->> 'domain') = s0.i1 and cypher_starts_with((n0.properties ->> 'name'), (i0)::text)::bool)) select array_remove(coalesce(array_agg(lower(((s2.n1).properties ->> 'samaccountname'))::text)::text[], array []::text[])::text[], null)::text[] as i2, lower(((s2.n0).properties ->> 'samaccountname'))::text as i3 from s2 group by lower(((s2.n0).properties ->> 'samaccountname'))::text), s3 as (select s1.i2 as i2, s1.i3 as i3, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3 from s1, edge e1 join node n2 on (n2.id = e1.end_id or n2.id = e1.start_id) join node n3 on n3.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (n3.id = e1.end_id or n3.id = e1.start_id) where (n2.id <> n3.id) and (not lower((n3.properties ->> 'samaccountname'))::text = any (s1.i2)) and e1.kind_id = any (array [4]::int2[]) and (lower((n2.properties ->> 'samaccountname'))::text = s1.i3)) select s3.n3 as g from s3; +with s0 as (select 'a' as i0, 'b' as i1), s1 as (with s2 as (select s0.i0 as i0, s0.i1 as i1, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) and ((n0.properties ->> 'domain') = s0.i1 and cypher_starts_with((n0.properties ->> 'name'), (i0)::text)::bool)) select array_remove(coalesce(array_agg(lower(((s2.n1).properties ->> 'samaccountname'))::text)::text[], array []::text[])::text[], null)::text[] as i2, lower(((s2.n0).properties ->> 'samaccountname'))::text as i3 from s2 group by lower(((s2.n0).properties ->> 'samaccountname'))::text), s3 as (select s1.i2 as i2, s1.i3 as i3, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3 from s1, edge e1 join node n2 on (n2.id = e1.end_id or n2.id = e1.start_id) join node n3 on n3.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (n3.id = e1.end_id or n3.id = e1.start_id) where ((n2.id = e1.start_id and n3.id = e1.end_id) or (n3.id = e1.start_id and n2.id = e1.end_id)) and (not lower((n3.properties ->> 'samaccountname'))::text = any (s1.i2)) and e1.kind_id = any (array [4]::int2[]) and (lower((n2.properties ->> 'samaccountname'))::text = s1.i3)) select s3.n3 as g from s3; -- case: with "a" as check, "b" as ref match p = (u)-[:EdgeKind1]->(g:NodeKind1) where u.name starts with check and u.domain = ref with collect(tolower(g.samaccountname)) as refmembership, tolower(u.samaccountname) as samname match (u)-[:EdgeKind2]->(g:NodeKind1) where tolower(u.samaccountname) = samname and not tolower(g.samaccountname) IN refmembership return g with s0 as (select 'a' as i0, 'b' as i1), s1 as (with s2 as (select s0.i0 as i0, s0.i1 as i1, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) and ((n0.properties ->> 'domain') = s0.i1 and cypher_starts_with((n0.properties ->> 'name'), (i0)::text)::bool)) select array_remove(coalesce(array_agg(lower(((s2.n1).properties ->> 'samaccountname'))::text)::text[], array []::text[])::text[], null)::text[] as i2, lower(((s2.n0).properties ->> 'samaccountname'))::text as i3 from s2 group by lower(((s2.n0).properties ->> 'samaccountname'))::text), s3 as (select s1.i2 as i2, s1.i3 as i3, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3 from s1, edge e1 join node n2 on n2.id = e1.start_id join node n3 on n3.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n3.id = e1.end_id where (not lower((n3.properties ->> 'samaccountname'))::text = any (s1.i2)) and e1.kind_id = any (array [4]::int2[]) and (lower((n2.properties ->> 'samaccountname'))::text = s1.i3)) select s3.n3 as g from s3; -- case: match p =(n:NodeKind1)<-[r:EdgeKind1|EdgeKind2*..3]-(u:NodeKind1) where n.domain = 'test' with n, count(r) as incomingCount where incomingCount > 90 with collect(n) as lotsOfAdmins match p =(n:NodeKind1)<-[:EdgeKind1]-() where n in lotsOfAdmins return p -with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'domain')) = 'string' and (n0.properties ->> 'domain') = 'test')) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], e0.end_id = e0.start_id, array [e0.id] from s2_seed join edge e0 on e0.end_id = s2_seed.root_id join node n1 on n1.id = e0.start_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s2.root_id, e0.start_id, s2.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.start_id where s2.depth < 3 and not s2.is_cycle) select (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s2.path) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id) as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied) select s1.n0 as n0, count(s1.e0)::int8 as i0 from s1 group by n0), s3 as (select array_remove(coalesce(array_agg((n0).id)::int8[], array []::int8[])::int8[], null)::int8[] as i1 from s0 where (s0.i0 > 90)), s4 as (select e1.id as e1, s3.i1 as i1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3 from s3, edge e1 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id join node n3 on n3.id = e1.start_id where e1.kind_id = any (array [3]::int2[]) and (n2.id = any (s3.i1))) select case when (s4.n2).id is null or s4.e1 is null or (s4.n3).id is null then null else ordered_edges_to_path(s4.n2, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s4.e1]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s4.n2, s4.n3]::nodecomposite[])::pathcomposite end as p from s4; +with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'domain')) = 'string' and (n0.properties ->> 'domain') = 'test')) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e0.id] from s2_seed join edge e0 on e0.end_id = s2_seed.root_id join node n1 on n1.id = e0.start_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s2.root_id, e0.start_id, s2.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.start_id where s2.depth < 3 and not s2.is_cycle) select (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s2.path) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id and _edge.graph_id = 0) as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied) select s1.n0 as n0, count(s1.e0)::int8 as i0 from s1 group by n0), s3 as (select array_remove(coalesce(array_agg((n0).id)::int8[], array []::int8[])::int8[], null)::int8[] as i1 from s0 where (s0.i0 > 90)), s4 as (select e1.id as e1, s3.i1 as i1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3 from s3, edge e1 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id join node n3 on n3.id = e1.start_id where e1.kind_id = any (array [3]::int2[]) and (n2.id = any (s3.i1))) select case when (s4.n2).id is null or s4.e1 is null or (s4.n3).id is null then null else ordered_edge_ids_to_path(0, s4.n2, array [s4.e1]::int8[], array [s4.n2, s4.n3]::nodecomposite[])::pathcomposite end as p from s4; -- case: match (u:NodeKind1)-[:EdgeKind1]->(g:NodeKind2) with g match (g)<-[:EdgeKind1]-(u:NodeKind1) return g with s0 as (with s1 as (select (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[])) select s1.n1 as n1 from s1), s2 as (select s0.n1 as n1 from s0 join edge e1 on (s0.n1).id = e1.end_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.start_id where e1.kind_id = any (array [3]::int2[])) select s2.n1 as g from s2; @@ -78,19 +78,19 @@ with s0 as (with s1 as (select (n1.id, n1.kind_ids, n1.properties)::nodecomposit with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((n0.properties ->> 'name') ~ '.*TT' and (jsonb_typeof((n0.properties -> 'domain')) = 'string' and (n0.properties ->> 'domain') = 'MY DOMAIN')) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select array_remove(coalesce(array_agg(((s1.n0).properties ->> 'email'))::anyarray, array []::text[])::anyarray, null)::anyarray as i0 from s1), s2 as (select s0.i0 as i0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, edge e0 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e0.end_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[]) and (not (n2.properties ->> 'email') = any (s0.i0) and (n2.properties ->> 'name') like 'blah%')) select s2.n1 as o from s2; -- case: match (e) match p = ()-[]->(e) return p limit 1 -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0), s1 as (select e0.id as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0 join edge e0 on (s0.n0).id = e0.end_id join node n1 on n1.id = e0.start_id) select case when (s1.n1).id is null or s1.e0 is null or (s1.n0).id is null then null else ordered_edges_to_path(s1.n1, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n1, s1.n0]::nodecomposite[])::pathcomposite end as p from s1 limit 1; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0), s1 as (select e0.id as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0 join edge e0 on (s0.n0).id = e0.end_id join node n1 on n1.id = e0.start_id) select case when (s1.n1).id is null or s1.e0 is null or (s1.n0).id is null then null else ordered_edge_ids_to_path(0, s1.n1, array [s1.e0]::int8[], array [s1.n1, s1.n0]::nodecomposite[])::pathcomposite end as p from s1 limit 1; -- case: match p = (a)-[]->() match q = ()-[]->(a) return p, q -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, e1.id as e1, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n0).id = e1.end_id join node n2 on n2.id = e1.start_id) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null then null else ordered_edges_to_path(s1.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as p, case when (s1.n2).id is null or s1.e1 is null or (s1.n0).id is null then null else ordered_edges_to_path(s1.n2, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e1]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n2, s1.n0]::nodecomposite[])::pathcomposite end as q from s1; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, e1.id as e1, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n0).id = e1.end_id join node n2 on n2.id = e1.start_id) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null then null else ordered_edge_ids_to_path(0, s1.n0, array [s1.e0]::int8[], array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as p, case when (s1.n2).id is null or s1.e1 is null or (s1.n0).id is null then null else ordered_edge_ids_to_path(0, s1.n2, array [s1.e1]::int8[], array [s1.n2, s1.n0]::nodecomposite[])::pathcomposite end as q from s1; -- case: match (m:NodeKind1)-[*1..]->(g:NodeKind2)-[]->(c3:NodeKind1) where not g.name in ["foo"] with collect(g.name) as bar match p=(m:NodeKind1)-[*1..]->(g:NodeKind2) where g.name in bar return p -with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, (not (n1.properties ->> 'name') = any (array ['foo']::text[])) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], e0.start_id = e0.end_id, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id union all select s2.root_id, e0.end_id, s2.depth + 1, (not (n1.properties ->> 'name') = any (array ['foo']::text[])) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 15 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and exists (select 1 from edge e1 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where n1.id = e1.start_id)), s3 as (select s1.ep0 as ep0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e1 on (s1.n1).id = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where e1.id != all (s1.ep0)) select array_remove(coalesce(array_agg(((s3.n1).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray as i0 from s3), s4 as (with recursive s5_seed(root_id) as not materialized (select n4.id as root_id from s0, node n4 where n4.kind_ids operator (pg_catalog.@>) array [2]::int2[] and ((n4.properties ->> 'name') = any (s0.i0))), s5(root_id, next_id, depth, satisfied, is_cycle, path) as (select e2.end_id, e2.start_id, 1, n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], e2.end_id = e2.start_id, array [e2.id] from s5_seed join edge e2 on e2.end_id = s5_seed.root_id join node n3 on n3.id = e2.start_id union select s5.root_id, e2.start_id, s5.depth + 1, n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, e2.id || s5.path from s5 join lateral (select e2.id, e2.start_id, e2.end_id, e2.kind_id, e2.properties from edge e2 where e2.end_id = s5.next_id and e2.id != all (s5.path) offset 0) e2 on true join node n3 on n3.id = e2.start_id where s5.depth < 15 and not s5.is_cycle) select s5.path as ep1, s0.i0 as i0, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3, (n4.id, n4.kind_ids, n4.properties)::nodecomposite as n4 from s0, s5 join lateral (select n4.id, n4.kind_ids, n4.properties from node n4 where n4.id = s5.root_id offset 0) n4 on true join lateral (select n3.id, n3.kind_ids, n3.properties from node n3 where n3.id = s5.next_id offset 0) n3 on true where s5.satisfied) select case when (s4.n3).id is null or s4.ep1 is null or (s4.n4).id is null then null else ordered_edges_to_path(s4.n3, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s4.ep1) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s4.n3, s4.n4]::nodecomposite[])::pathcomposite end as p from s4; +with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, (not (n1.properties ->> 'name') = any (array ['foo']::text[])) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id union all select s2.root_id, e0.end_id, s2.depth + 1, (not (n1.properties ->> 'name') = any (array ['foo']::text[])) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 15 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and exists (select 1 from edge e1 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where n1.id = e1.start_id)), s3 as (select s1.ep0 as ep0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e1 on (s1.n1).id = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where e1.id != all (s1.ep0)) select array_remove(coalesce(array_agg(((s3.n1).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray as i0 from s3), s4 as (with recursive s5_seed(root_id) as not materialized (select n4.id as root_id from s0, node n4 where n4.kind_ids operator (pg_catalog.@>) array [2]::int2[] and ((n4.properties ->> 'name') = any (s0.i0))), s5(root_id, next_id, depth, satisfied, is_cycle, path) as (select e2.end_id, e2.start_id, 1, n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e2.id] from s5_seed join edge e2 on e2.end_id = s5_seed.root_id join node n3 on n3.id = e2.start_id union select s5.root_id, e2.start_id, s5.depth + 1, n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, e2.id || s5.path from s5 join lateral (select e2.id, e2.start_id, e2.end_id, e2.kind_id, e2.properties from edge e2 where e2.end_id = s5.next_id and e2.id != all (s5.path) offset 0) e2 on true join node n3 on n3.id = e2.start_id where s5.depth < 15 and not s5.is_cycle) select s5.path as ep1, s0.i0 as i0, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3, (n4.id, n4.kind_ids, n4.properties)::nodecomposite as n4 from s0, s5 join lateral (select n4.id, n4.kind_ids, n4.properties from node n4 where n4.id = s5.root_id offset 0) n4 on true join lateral (select n3.id, n3.kind_ids, n3.properties from node n3 where n3.id = s5.next_id offset 0) n3 on true where s5.satisfied) select case when (s4.n3).id is null or s4.ep1 is null or (s4.n4).id is null then null else ordered_edge_ids_to_path(0, s4.n3, s4.ep1, array [s4.n3, s4.n4]::nodecomposite[])::pathcomposite end as p from s4; -- case: match (m:NodeKind1)-[:EdgeKind1*1..]->(g:NodeKind2)-[:EdgeKind2]->(c3:NodeKind1) where m.samaccountname =~ '^[A-Z]{1,3}[0-9]{1,3}$' and not m.samaccountname contains "DEX" and not g.name IN ["D"] and not m.samaccountname =~ "^.*$" with collect(g.name) as admingroups match p=(m:NodeKind1)-[:EdgeKind1*1..]->(g:NodeKind2) where m.samaccountname =~ '^[A-Z]{1,3}[0-9]{1,3}$' and g.name in admingroups and not m.samaccountname =~ "^.*$" return p -with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.properties ->> 'samaccountname') ~ '^[A-Z]{1,3}[0-9]{1,3}$' and not coalesce((n0.properties ->> 'samaccountname'), '')::text like '%DEX%' and not (n0.properties ->> 'samaccountname') ~ '^.*$') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, (not (n1.properties ->> 'name') = any (array ['D']::text[])) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], e0.start_id = e0.end_id, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) union all select s2.root_id, e0.end_id, s2.depth + 1, (not (n1.properties ->> 'name') = any (array ['D']::text[])) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 15 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and exists (select 1 from edge e1 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where n1.id = e1.start_id and e1.kind_id = any (array [4]::int2[]))), s3 as (select s1.ep0 as ep0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e1 on (s1.n1).id = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) and e1.id != all (s1.ep0)) select array_remove(coalesce(array_agg(((s3.n1).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray as i0 from s3), s4 as (with recursive s5_seed(root_id) as not materialized (select n4.id as root_id from s0, node n4 where n4.kind_ids operator (pg_catalog.@>) array [2]::int2[] and ((n4.properties ->> 'name') = any (s0.i0))), s5(root_id, next_id, depth, satisfied, is_cycle, path) as (select e2.end_id, e2.start_id, 1, ((n3.properties ->> 'samaccountname') ~ '^[A-Z]{1,3}[0-9]{1,3}$' and not (n3.properties ->> 'samaccountname') ~ '^.*$') and n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], e2.end_id = e2.start_id, array [e2.id] from s5_seed join edge e2 on e2.end_id = s5_seed.root_id join node n3 on n3.id = e2.start_id where e2.kind_id = any (array [3]::int2[]) union select s5.root_id, e2.start_id, s5.depth + 1, ((n3.properties ->> 'samaccountname') ~ '^[A-Z]{1,3}[0-9]{1,3}$' and not (n3.properties ->> 'samaccountname') ~ '^.*$') and n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, e2.id || s5.path from s5 join lateral (select e2.id, e2.start_id, e2.end_id, e2.kind_id, e2.properties from edge e2 where e2.end_id = s5.next_id and e2.id != all (s5.path) and e2.kind_id = any (array [3]::int2[]) offset 0) e2 on true join node n3 on n3.id = e2.start_id where s5.depth < 15 and not s5.is_cycle) select s5.path as ep1, s0.i0 as i0, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3, (n4.id, n4.kind_ids, n4.properties)::nodecomposite as n4 from s0, s5 join lateral (select n4.id, n4.kind_ids, n4.properties from node n4 where n4.id = s5.root_id offset 0) n4 on true join lateral (select n3.id, n3.kind_ids, n3.properties from node n3 where n3.id = s5.next_id offset 0) n3 on true where s5.satisfied) select case when (s4.n3).id is null or s4.ep1 is null or (s4.n4).id is null then null else ordered_edges_to_path(s4.n3, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s4.ep1) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s4.n3, s4.n4]::nodecomposite[])::pathcomposite end as p from s4; +with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.properties ->> 'samaccountname') ~ '^[A-Z]{1,3}[0-9]{1,3}$' and not coalesce((n0.properties ->> 'samaccountname'), '')::text like '%DEX%' and not (n0.properties ->> 'samaccountname') ~ '^.*$') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, (not (n1.properties ->> 'name') = any (array ['D']::text[])) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) union all select s2.root_id, e0.end_id, s2.depth + 1, (not (n1.properties ->> 'name') = any (array ['D']::text[])) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 15 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and exists (select 1 from edge e1 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where n1.id = e1.start_id and e1.kind_id = any (array [4]::int2[]))), s3 as (select s1.ep0 as ep0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e1 on (s1.n1).id = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) and e1.id != all (s1.ep0)) select array_remove(coalesce(array_agg(((s3.n1).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray as i0 from s3), s4 as (with recursive s5_seed(root_id) as not materialized (select n4.id as root_id from s0, node n4 where n4.kind_ids operator (pg_catalog.@>) array [2]::int2[] and ((n4.properties ->> 'name') = any (s0.i0))), s5(root_id, next_id, depth, satisfied, is_cycle, path) as (select e2.end_id, e2.start_id, 1, ((n3.properties ->> 'samaccountname') ~ '^[A-Z]{1,3}[0-9]{1,3}$' and not (n3.properties ->> 'samaccountname') ~ '^.*$') and n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e2.id] from s5_seed join edge e2 on e2.end_id = s5_seed.root_id join node n3 on n3.id = e2.start_id where e2.kind_id = any (array [3]::int2[]) union select s5.root_id, e2.start_id, s5.depth + 1, ((n3.properties ->> 'samaccountname') ~ '^[A-Z]{1,3}[0-9]{1,3}$' and not (n3.properties ->> 'samaccountname') ~ '^.*$') and n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, e2.id || s5.path from s5 join lateral (select e2.id, e2.start_id, e2.end_id, e2.kind_id, e2.properties from edge e2 where e2.end_id = s5.next_id and e2.id != all (s5.path) and e2.kind_id = any (array [3]::int2[]) offset 0) e2 on true join node n3 on n3.id = e2.start_id where s5.depth < 15 and not s5.is_cycle) select s5.path as ep1, s0.i0 as i0, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3, (n4.id, n4.kind_ids, n4.properties)::nodecomposite as n4 from s0, s5 join lateral (select n4.id, n4.kind_ids, n4.properties from node n4 where n4.id = s5.root_id offset 0) n4 on true join lateral (select n3.id, n3.kind_ids, n3.properties from node n3 where n3.id = s5.next_id offset 0) n3 on true where s5.satisfied) select case when (s4.n3).id is null or s4.ep1 is null or (s4.n4).id is null then null else ordered_edge_ids_to_path(0, s4.n3, s4.ep1, array [s4.n3, s4.n4]::nodecomposite[])::pathcomposite end as p from s4; -- case: match (a:NodeKind2)-[:EdgeKind1]->(g:NodeKind1)-[:EdgeKind2]->(s:NodeKind2) with count(a) as uc where uc > 5 match p = (a)-[:EdgeKind1]->(g)-[:EdgeKind2]->(s) return p -with s0 as (with s1 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[])), s2 as (select s1.e0 as e0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e1 on (s1.n1).id = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) and e1.id != s1.e0) select count(s2.n0)::int8 as i0 from s2), s3 as (select e2.id as e2, s0.i0 as i0, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3, (n4.id, n4.kind_ids, n4.properties)::nodecomposite as n4 from s0, edge e2 join node n3 on n3.id = e2.start_id join node n4 on n4.id = e2.end_id where e2.kind_id = any (array [3]::int2[]) and (s0.i0 > 5)), s4 as (select s3.e2 as e2, e3.id as e3, s3.i0 as i0, s3.n3 as n3, s3.n4 as n4, (n5.id, n5.kind_ids, n5.properties)::nodecomposite as n5 from s3 join edge e3 on (s3.n4).id = e3.start_id join node n5 on n5.id = e3.end_id where e3.kind_id = any (array [4]::int2[]) and e3.id != s3.e2) select case when (s4.n3).id is null or s4.e2 is null or (s4.n4).id is null or s4.e3 is null or (s4.n5).id is null then null else ordered_edges_to_path(s4.n3, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s4.e2]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id) || (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s4.e3]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s4.n3, s4.n4, s4.n5]::nodecomposite[])::pathcomposite end as p from s4; +with s0 as (with s1 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[])), s2 as (select s1.e0 as e0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e1 on s1.n1 = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) and e1.id != s1.e0) select count(s2.n0)::int8 as i0 from s2), s3 as (select e2.id as e2, s0.i0 as i0, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3, (n4.id, n4.kind_ids, n4.properties)::nodecomposite as n4 from s0, edge e2 join node n3 on n3.id = e2.start_id join node n4 on n4.id = e2.end_id where e2.kind_id = any (array [3]::int2[]) and (s0.i0 > 5)), s4 as (select s3.e2 as e2, e3.id as e3, s3.i0 as i0, s3.n3 as n3, s3.n4 as n4, (n5.id, n5.kind_ids, n5.properties)::nodecomposite as n5 from s3 join edge e3 on (s3.n4).id = e3.start_id join node n5 on n5.id = e3.end_id where e3.kind_id = any (array [4]::int2[]) and e3.id != s3.e2) select case when (s4.n3).id is null or s4.e2 is null or (s4.n4).id is null or s4.e3 is null or (s4.n5).id is null then null else ordered_edge_ids_to_path(0, s4.n3, array [s4.e2]::int8[] || array [s4.e3]::int8[], array [s4.n3, s4.n4, s4.n5]::nodecomposite[])::pathcomposite end as p from s4; -- case: match (g:NodeKind1) optional match (g)<-[r:EdgeKind1]-(m:NodeKind2) with g, count(r) as memberCount where memberCount = 0 return g with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, s1.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join edge e0 on (s1.n0).id = e0.end_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[])), s3 as (select s1.n0 as n0, s2.e0 as e0, s2.n1 as n1 from s1 left outer join s2 on (s1.n0 = s2.n0)) select s3.n0 as n0, count(s3.e0)::int8 as i0 from s3 group by n0) select s0.n0 as g from s0 where (s0.i0 = 0); diff --git a/cypher/models/pgsql/test/translation_cases/nodes.sql b/cypher/models/pgsql/test/translation_cases/nodes.sql index 7a54ce40..ac29c06e 100644 --- a/cypher/models/pgsql/test/translation_cases/nodes.sql +++ b/cypher/models/pgsql/test/translation_cases/nodes.sql @@ -15,7 +15,7 @@ -- SPDX-License-Identifier: Apache-2.0 -- case: match (n) return labels(n) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select (array(select _kind.name from generate_subscripts((s0.n0).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s0.n0).kind_ids)[_kind_idx] order by _kind_idx))::text[] from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select (array(select _kind.name from generate_subscripts((s0.n0).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s0.n0).kind_ids)[_kind_idx] order by _kind_idx))::text[] as "labels(n)" from s0; -- case: match (n) where 'NodeKind1' in labels(n) return n with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as n from s0 where ('NodeKind1' = any ((array(select _kind.name from generate_subscripts((s0.n0).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s0.n0).kind_ids)[_kind_idx] order by _kind_idx))::text[])); @@ -24,10 +24,10 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as n from s0 where ((array(select _kind.name from generate_subscripts((s0.n0).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s0.n0).kind_ids)[_kind_idx] order by _kind_idx))::text[] = array ['NodeKind1', 'NodeKind2']::text[]); -- case: match (n) where n.name = 'n3' with labels(n) as labels return labels, size(labels) -with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n3'))) select (array(select _kind.name from generate_subscripts((s1.n0).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s1.n0).kind_ids)[_kind_idx] order by _kind_idx))::text[] as i0 from s1) select s0.i0 as labels, cardinality(s0.i0)::int from s0; +with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n3'))) select (array(select _kind.name from generate_subscripts((s1.n0).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s1.n0).kind_ids)[_kind_idx] order by _kind_idx))::text[] as i0 from s1) select s0.i0 as labels, cardinality(s0.i0)::int as "size(labels)" from s0; -- case: match (n) with 1 as _kind_idx, n return labels(n), _kind_idx -with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select 1 as i0, s1.n0 as n0 from s1) select (array(select _kind.name from generate_subscripts((s0.n0).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s0.n0).kind_ids)[_kind_idx] order by _kind_idx))::text[], s0.i0 as _kind_idx from s0; +with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select 1 as i0, s1.n0 as n0 from s1) select (array(select _kind.name from generate_subscripts((s0.n0).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s0.n0).kind_ids)[_kind_idx] order by _kind_idx))::text[] as "labels(n)", s0.i0 as _kind_idx from s0; -- case: match (n:NodeKind1) return n.name as displayname order by displayname with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select ((s0.n0).properties -> 'name') as displayname from s0 order by displayname; @@ -47,9 +47,39 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from -- case: match (n) where n.name = '1234' return n with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = '1234'))) select s0.n0 as n from s0; +-- case: match (n) where n.`a-aaa` = "123" return n +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'a-aaa')) = 'string' and (n0.properties ->> 'a-aaa') = '123'))) select s0.n0 as n from s0; + +-- case: match (n) where n.`b_bbb` = "123" return n +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'b_bbb')) = 'string' and (n0.properties ->> 'b_bbb') = '123'))) select s0.n0 as n from s0; + +-- case: match (n) where n.`has``tick` = "123" return n +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'has`tick')) = 'string' and (n0.properties ->> 'has`tick') = '123'))) select s0.n0 as n from s0; + +-- case: match (n) where n.`'` = "123" return n +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> '''')) = 'string' and (n0.properties ->> '''') = '123'))) select s0.n0 as n from s0; + +-- case: match (n) where n.```starts-tick` = "123" return n +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> '`starts-tick')) = 'string' and (n0.properties ->> '`starts-tick') = '123'))) select s0.n0 as n from s0; + +-- case: match (n) where n.```super-wrapped``` = "123" return n +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> '`super-wrapped`')) = 'string' and (n0.properties ->> '`super-wrapped`') = '123'))) select s0.n0 as n from s0; + +-- case: match (n) where n.```` = "123" return n +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> '`')) = 'string' and (n0.properties ->> '`') = '123'))) select s0.n0 as n from s0; + +-- case: match (n) where (n).`a-aaa` = "123" return n +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as n from s0 where ((jsonb_typeof((((s0.n0)).properties -> 'a-aaa')) = 'string' and (((s0.n0)).properties ->> 'a-aaa') = '123')); + +-- case: match ()-[r]-() where startNode(r).`something` = "abc" return r +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on (n0.id = e0.end_id or n0.id = e0.start_id) join node n1 on (n1.id = e0.end_id or n1.id = e0.start_id) where ((n0.id = e0.start_id and n1.id = e0.end_id) or (n1.id = e0.start_id and n0.id = e0.end_id))) select s0.e0 as r from s0 where ((jsonb_typeof(((start_node(((s0.e0).id, (s0.e0).start_id, (s0.e0).end_id, (s0.e0).kind_id, (s0.e0).properties)::edgecomposite)::nodecomposite).properties -> 'something')) = 'string' and ((start_node(((s0.e0).id, (s0.e0).start_id, (s0.e0).end_id, (s0.e0).kind_id, (s0.e0).properties)::edgecomposite)::nodecomposite).properties ->> 'something') = 'abc')); + -- case: match (n:NodeKind1 {name: "SOME NAME"}) return n with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'SOME NAME')) select s0.n0 as n from s0; +-- case: match (n:NodeKind1 {`'`: 'value'}) return n +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (jsonb_typeof((n0.properties -> '''')) = 'string' and (n0.properties ->> '''') = 'value')) select s0.n0 as n from s0; + -- case: match (n) where n.objectid in $p return n -- cypher_params: {"p":["1","2","3"]} -- pgsql_params:{"pi0":["1","2","3"]} @@ -70,7 +100,7 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [2]::int2[]))) select s0.n0 as s from s0; -- case: match (n:NodeKind1), (e) where n.name = e.name return n -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where (((s0.n0).properties -> 'name') = (n1.properties -> 'name'))) select s1.n0 as n from s1; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where (nullif(((s0.n0).properties -> 'name'), ('null')::jsonb)::jsonb = nullif((n1.properties -> 'name'), ('null')::jsonb)::jsonb)) select s1.n0 as n from s1; -- case: match (s), (e) where id(s) in e.captured_ids return s, e with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where ((s0.n0).id = any (jsonb_to_text_array((n1.properties -> 'captured_ids'))::int8[]))) select s1.n0 as s, s1.n1 as e from s1; @@ -82,7 +112,7 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = '1234'))) select s0.n0 as s from s0; -- case: match (s:NodeKind1), (e:NodeKind2) where s.selected or s.tid = e.tid and e.enabled return s, e -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where ((((s0.n0).properties ->> 'selected'))::bool or ((s0.n0).properties -> 'tid') = (n1.properties -> 'tid') and ((n1.properties ->> 'enabled'))::bool) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]) select s1.n0 as s, s1.n1 as e from s1; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where ((((s0.n0).properties ->> 'selected'))::bool or nullif(((s0.n0).properties -> 'tid'), ('null')::jsonb)::jsonb = nullif((n1.properties -> 'tid'), ('null')::jsonb)::jsonb and ((n1.properties ->> 'enabled'))::bool) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]) select s1.n0 as s, s1.n1 as e from s1; -- case: match (s) where s.value + 2 / 3 > 10 return s with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((n0.properties ->> 'value'))::int8 + 2 / 3 > 10)) select s0.n0 as s from s0; @@ -97,10 +127,10 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (lower((n0.properties ->> 'name'))::text = '1234')) select distinct s0.n0 as s from s0; -- case: match (s:NodeKind1), (e:NodeKind2) where s.name = e.name return s, e -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where (((s0.n0).properties -> 'name') = (n1.properties -> 'name')) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]) select s1.n0 as s, s1.n1 as e from s1; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where (nullif(((s0.n0).properties -> 'name'), ('null')::jsonb)::jsonb = nullif((n1.properties -> 'name'), ('null')::jsonb)::jsonb) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]) select s1.n0 as s, s1.n1 as e from s1; -- case: match (n) where n.system_tags is not null and not (n:NodeKind1 or n:NodeKind2) return id(n) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((n0.properties ? 'system_tags' and not (n0.properties -> 'system_tags') = ('null')::jsonb) and not (n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [2]::int2[]))) select (s0.n0).id from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((n0.properties ? 'system_tags' and not (n0.properties -> 'system_tags') = ('null')::jsonb) and not (n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [2]::int2[]))) select (s0.n0).id as "id(n)" from s0; -- case: match (s), (e) where s.name = '1234' and e.other = 1234 return s with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = '1234'))), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where (((n1.properties -> 'other'))::jsonb = to_jsonb((1234)::int8)::jsonb)) select s1.n0 as s from s1; @@ -109,7 +139,7 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where ((jsonb_typeof(((s0.n0).properties -> 'name')) = 'string' and ((s0.n0).properties ->> 'name') = '1234') or ((n1.properties -> 'other'))::jsonb = to_jsonb((1234)::int8)::jsonb)) select s1.n0 as s from s1; -- case: match (n), (k) where n.name = '1234' and k.name = '1234' match (e) where e.name = n.name return k, e -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = '1234'))), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = '1234'))), s2 as (select s1.n0 as n0, s1.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s1, node n2 where ((n2.properties -> 'name') = ((s1.n0).properties -> 'name'))) select s2.n1 as k, s2.n2 as e from s2; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = '1234'))), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = '1234'))), s2 as (select s1.n0 as n0, s1.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s1, node n2 where (nullif((n2.properties -> 'name'), ('null')::jsonb)::jsonb = nullif(((s1.n0).properties -> 'name'), ('null')::jsonb)::jsonb)) select s2.n1 as k, s2.n2 as e from s2; -- case: match (n) return n skip 5 limit 10 with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as n from s0 offset 5 limit 10; @@ -160,10 +190,10 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((n0.properties -> 'isassignabletorole'))::jsonb = to_jsonb((true)::bool)::jsonb)) select s0.n0 as s from s0; -- case: match (s) return s.value + 1 -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select (((s0.n0).properties ->> 'value'))::int8 + 1 from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select (((s0.n0).properties ->> 'value'))::int8 + 1 as "s.value + 1" from s0; -- case: match (s) return (s.value + 1) / 3 -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select ((((s0.n0).properties ->> 'value'))::int8 + 1) / 3 from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select ((((s0.n0).properties ->> 'value'))::int8 + 1) / 3 as "(s.value + 1) / 3" from s0; -- case: match (s) where id(s) in [1, 2, 3, 4] return s with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (n0.id = any (array [1, 2, 3, 4]::int8[]))) select s0.n0 as s from s0; @@ -211,16 +241,16 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where (not exists (select 1 from edge e0 where (e0.start_id = (s0.n0).id or e0.end_id = (s0.n0).id))); -- case: match (s) where not (s)-[]->()-[]->() return s -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where (not (with s1 as (select e0.id as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on n1.id = e0.end_id where (s0.n0).id = e0.start_id), s2 as (select s1.e0 as e0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e1 on (s1.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != s1.e0) select count(*) > 0 from s2)); +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where (not (with s1 as (select e0.id as e0, s0.n0 as n0, n1.id as n1 from edge e0 join node n1 on n1.id = e0.end_id where (s0.n0).id = e0.start_id), s2 as (select s1.e0 as e0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e1 on s1.n1 = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != s1.e0) select count(*) > 0 from s2)); -- case: match (s) where ()-[]->()-[]->(s) return s -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where ((with s1 as (select e0.id as e0, s0.n0 as n0, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from edge e0 join node n1 on n1.id = e0.start_id join node n2 on n2.id = e0.end_id), s2 as (select s1.e0 as e0, s1.n0 as n0, s1.n2 as n2 from s1 join edge e1 on (s1.n2).id = e1.start_id join node n0 on (s1.n0).id = e1.end_id where e1.id != s1.e0) select count(*) > 0 from s2)); +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where ((with s1 as (select e0.id as e0, s0.n0 as n0, n2.id as n2 from edge e0 join node n1 on n1.id = e0.start_id join node n2 on n2.id = e0.end_id), s2 as (select s1.e0 as e0, s1.n0 as n0, s1.n2 as n2 from s1 join edge e1 on s1.n2 = e1.start_id and (s1.n0).id = e1.end_id where e1.id != s1.e0) select count(*) > 0 from s2)); -- case: match (g:Group) where (:User)-[:MemberOf]->(:Group)-[:MemberOf]->(g) return count(g) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [13]::int2[]) select count(s0.n0)::int8 from s0 where ((with s1 as (select e0.id as e0, s0.n0 as n0, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from edge e0 join node n1 on n1.kind_ids operator (pg_catalog.@>) array [6]::int2[] and n1.id = e0.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [13]::int2[] and n2.id = e0.end_id where e0.kind_id = any (array [25]::int2[])), s2 as (select s1.e0 as e0, s1.n0 as n0, s1.n2 as n2 from s1 join edge e1 on (s1.n2).id = e1.start_id join node n0 on (s1.n0).id = e1.end_id where e1.kind_id = any (array [25]::int2[]) and e1.id != s1.e0) select count(*) > 0 from s2)); +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [13]::int2[]) select count(s0.n0)::int8 as "count(g)" from s0 where ((with s1 as (select e0.id as e0, s0.n0 as n0, n2.id as n2 from edge e0 join node n1 on n1.kind_ids operator (pg_catalog.@>) array [6]::int2[] and n1.id = e0.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [13]::int2[] and n2.id = e0.end_id where e0.kind_id = any (array [25]::int2[])), s2 as (select s1.e0 as e0, s1.n0 as n0, s1.n2 as n2 from s1 join edge e1 on s1.n2 = e1.start_id and (s1.n0).id = e1.end_id where e1.kind_id = any (array [25]::int2[]) and e1.id != s1.e0) select count(*) > 0 from s2)); -- case: match (s) where not (s)-[{prop: 'a'}]-({name: 'n3'}) return s -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where (not (with s1 as (select s0.n0 as n0 from edge e0 join node n1 on (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'n3') and (n1.id = e0.end_id or n1.id = e0.start_id) where ((s0.n0).id <> n1.id) and (jsonb_typeof((e0.properties -> 'prop')) = 'string' and (e0.properties ->> 'prop') = 'a') and ((s0.n0).id = e0.end_id or (s0.n0).id = e0.start_id)) select count(*) > 0 from s1)); +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where (not (with s1 as (select s0.n0 as n0 from edge e0 join node n1 on (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'n3') and (n1.id = e0.end_id or n1.id = e0.start_id) where (((s0.n0).id = e0.start_id and n1.id = e0.end_id) or (n1.id = e0.start_id and (s0.n0).id = e0.end_id)) and (jsonb_typeof((e0.properties -> 'prop')) = 'string' and (e0.properties ->> 'prop') = 'a') and ((s0.n0).id = e0.end_id or (s0.n0).id = e0.start_id)) select count(*) > 0 from s1)); -- case: match (s) where not (s)<-[{prop: 'a'}]-({name: 'n3'}) return s with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where (not (with s1 as (select s0.n0 as n0 from edge e0 join node n1 on (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'n3') and n1.id = e0.start_id where (jsonb_typeof((e0.properties -> 'prop')) = 'string' and (e0.properties ->> 'prop') = 'a') and (s0.n0).id = e0.end_id) select count(*) > 0 from s1)); @@ -241,7 +271,7 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where (not (with s1 as (select s0.n0 as n0 from edge e0 join node n1 on (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'n3') and n1.id = e0.end_id where (jsonb_typeof((e0.properties -> 'prop')) = 'string' and (e0.properties ->> 'prop') = 'a') and (s0.n0).id = e0.start_id) select count(*) > 0 from s1)); -- case: match (s) where not (s)-[]-() return id(s) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select (s0.n0).id from s0 where (not exists (select 1 from edge e0 where (e0.start_id = (s0.n0).id or e0.end_id = (s0.n0).id))); +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select (s0.n0).id as "id(s)" from s0 where (not exists (select 1 from edge e0 where (e0.start_id = (s0.n0).id or e0.end_id = (s0.n0).id))); -- case: match (s) where ()--(s) return s with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where (exists (select 1 from edge e0 where (e0.start_id = (s0.n0).id or e0.end_id = (s0.n0).id))); @@ -253,19 +283,19 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where (exists (select 1 from edge e0)); -- case: match (g) where ({name: 'n3'})-[{prop: 'a'}]-(g) return g -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as g from s0 where ((with s1 as (select s0.n0 as n0 from edge e0 join node n1 on (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'n3') and (n1.id = e0.end_id or n1.id = e0.start_id) where ((s0.n0).id <> n1.id) and (jsonb_typeof((e0.properties -> 'prop')) = 'string' and (e0.properties ->> 'prop') = 'a') and ((s0.n0).id = e0.end_id or (s0.n0).id = e0.start_id)) select count(*) > 0 from s1)); +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as g from s0 where ((with s1 as (select s0.n0 as n0 from edge e0 join node n1 on (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'n3') and (n1.id = e0.end_id or n1.id = e0.start_id) where (((s0.n0).id = e0.start_id and n1.id = e0.end_id) or (n1.id = e0.start_id and (s0.n0).id = e0.end_id)) and (jsonb_typeof((e0.properties -> 'prop')) = 'string' and (e0.properties ->> 'prop') = 'a') and ((s0.n0).id = e0.end_id or (s0.n0).id = e0.start_id)) select count(*) > 0 from s1)); -- case: match (a:NodeKind1), (b:NodeKind2) where (a:NodeKind1)-[]-(b:NodeKind2) return a -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]) select s1.n0 as a from s1 where ((with s2 as (select s1.n0 as n0, s1.n1 as n1 from edge e0 where ((s1.n0).id <> (s1.n1).id) and (((s1.n0).id = e0.start_id and (s1.n1).id = e0.end_id) or ((s1.n1).id = e0.start_id and (s1.n0).id = e0.end_id))) select count(*) > 0 from s2)); +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]) select s1.n0 as a from s1 where ((with s2 as (select s1.n0 as n0, s1.n1 as n1 from edge e0 where (((s1.n0).id = e0.start_id and (s1.n1).id = e0.end_id) or ((s1.n1).id = e0.start_id and (s1.n0).id = e0.end_id))) select count(*) > 0 from s2)); -- case: match (x:NodeKind1{name:'foo'}) match (x)-[]-(y:NodeKind2{name:'bar'}) return x -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'foo')), s1 as (select s0.n0 as n0 from s0 join edge e0 on ((s0.n0).id = e0.end_id or (s0.n0).id = e0.start_id) join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'bar') and (n1.id = e0.end_id or n1.id = e0.start_id) where ((s0.n0).id <> n1.id)) select s1.n0 as x from s1; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'foo')), s1 as (select s0.n0 as n0 from s0 join edge e0 on ((s0.n0).id = e0.end_id or (s0.n0).id = e0.start_id) join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'bar') and (n1.id = e0.end_id or n1.id = e0.start_id) where (((s0.n0).id = e0.start_id and n1.id = e0.end_id) or (n1.id = e0.start_id and (s0.n0).id = e0.end_id))) select s1.n0 as x from s1; -- case: match (y:NodeKind2{name:'bar'}) match ()-[]-(y) return y -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [2]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'bar')), s1 as (select s0.n0 as n0 from s0 join edge e0 on ((s0.n0).id = e0.end_id or (s0.n0).id = e0.start_id) join node n1 on (n1.id = e0.end_id or n1.id = e0.start_id) where ((s0.n0).id <> n1.id)) select s1.n0 as y from s1; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [2]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'bar')), s1 as (select s0.n0 as n0 from s0 join edge e0 on ((s0.n0).id = e0.end_id or (s0.n0).id = e0.start_id) join node n1 on (n1.id = e0.end_id or n1.id = e0.start_id) where (((s0.n0).id = e0.start_id and n1.id = e0.end_id) or (n1.id = e0.start_id and (s0.n0).id = e0.end_id))) select s1.n0 as y from s1; -- case: match (x:NodeKind1{name:'foo'}) match (y:NodeKind2{name:'bar'}) match (x)-[]-(y) return x -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'foo')), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'bar')), s2 as (select s1.n0 as n0, s1.n1 as n1 from s1 join edge e0 on ((s1.n0).id = e0.start_id or (s1.n0).id = e0.end_id) and ((s1.n1).id = e0.end_id or (s1.n1).id = e0.start_id) where ((s1.n0).id <> (s1.n1).id)) select s2.n0 as x from s2; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'foo')), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'bar')), s2 as (select s1.n0 as n0, s1.n1 as n1 from s1 join edge e0 on (((s1.n0).id = e0.start_id and (s1.n1).id = e0.end_id) or ((s1.n1).id = e0.start_id and (s1.n0).id = e0.end_id))) select s2.n0 as x from s2; -- case: match (n) where n.system_tags contains ($param) return n -- pgsql_params:{"pi0":null} @@ -294,7 +324,7 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((n0.properties ->> 'pwdlastset'))::numeric < (extract(epoch from now()::timestamp with time zone)::numeric * 1000 - 86400000) and not ((n0.properties ->> 'pwdlastset'))::float8 = any (array [- 1, 0]::float8[])) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s0.n0 as u from s0 limit 100; -- case: match (n:NodeKind1) where size(n.array_value) > 0 return n -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (jsonb_array_length((n0.properties -> 'array_value'))::int > 0) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s0.n0 as n from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (case when jsonb_typeof((n0.properties -> 'array_value')) = 'array' then jsonb_array_length((n0.properties -> 'array_value'))::int else null end > 0) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s0.n0 as n from s0; -- case: match (n) where 1 in n.array return n with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (1 = any (jsonb_to_text_array((n0.properties -> 'array'))::int8[]))) select s0.n0 as n from s0; @@ -371,7 +401,7 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s0.n0 as n from s0; -- case: match (n:NodeKind1) optional match (m:NodeKind2) where m.distinguishedname = n.unknown + m.unknown optional match (o:NodeKind2) where o.distinguishedname <> n.otherunknown return n, m, o -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where ((n1.properties ->> 'distinguishedname') = ((s0.n0).properties ->> 'unknown') || (n1.properties ->> 'unknown')) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s2 as (select s0.n0 as n0, s1.n1 as n1 from s0 left outer join s1 on (s0.n0 = s1.n0)), s3 as (select s2.n0 as n0, s2.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s2, node n2 where ((n2.properties -> 'distinguishedname') <> ((s2.n0).properties -> 'otherunknown')) and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s4 as (select s2.n0 as n0, s2.n1 as n1, s3.n2 as n2 from s2 left outer join s3 on (s2.n1 = s3.n1) and (s2.n0 = s3.n0)) select s4.n0 as n, s4.n1 as m, s4.n2 as o from s4; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where ((n1.properties ->> 'distinguishedname') = ((s0.n0).properties ->> 'unknown') || (n1.properties ->> 'unknown')) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s2 as (select s0.n0 as n0, s1.n1 as n1 from s0 left outer join s1 on (s0.n0 = s1.n0)), s3 as (select s2.n0 as n0, s2.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s2, node n2 where (nullif((n2.properties -> 'distinguishedname'), ('null')::jsonb)::jsonb <> nullif(((s2.n0).properties -> 'otherunknown'), ('null')::jsonb)::jsonb) and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s4 as (select s2.n0 as n0, s2.n1 as n1, s3.n2 as n2 from s2 left outer join s3 on (s2.n1 = s3.n1) and (s2.n0 = s3.n0)) select s4.n0 as n, s4.n1 as m, s4.n2 as o from s4; -- case: match (n) where n.name = "alpha' || (SELECT inet_server_addr()::text::int) || '" return n with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'alpha'' || (SELECT inet_server_addr()::text::int) || '''))) select s0.n0 as n from s0; diff --git a/cypher/models/pgsql/test/translation_cases/pattern_binding.sql b/cypher/models/pgsql/test/translation_cases/pattern_binding.sql index c32946cb..888cb03c 100644 --- a/cypher/models/pgsql/test/translation_cases/pattern_binding.sql +++ b/cypher/models/pgsql/test/translation_cases/pattern_binding.sql @@ -21,85 +21,85 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((n0.properties ->> 'name') like '%test%') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select case when (s0.n0).id is null then null else (array [s0.n0]::nodecomposite[], array []::edgecomposite[])::pathcomposite end as p from s0; -- case: match p = ()-[]->() return p -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id) select case when (s0.n0).id is null or s0.e0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s0.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id) select case when (s0.n0).id is null or s0.e0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, array [s0.e0]::int8[], array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; -- case: match p = ()-[]->() return nodes(p) -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id) select ((case when (s0.n0).id is null or s0.e0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s0.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end).nodes)::nodecomposite[] from s0; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id) select ((case when (s0.n0).id is null or s0.e0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, array [s0.e0]::int8[], array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end).nodes)::nodecomposite[] as "nodes(p)" from s0; -- case: match p = (:NodeKind1)-[:EdgeKind1|EdgeKind2*1..1]->(:NodeKind2) where any(r in relationships(p) where type(r) STARTS WITH 'EdgeKind') return p -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[])) select case when (s0.n0).id is null or s0.e0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s0.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((exists (select 1 from edge i0 where (kind_name(i0.kind_id)::text like 'EdgeKind%') and i0.id = any (array [s0.e0]::int8[])))::bool); +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[])) select case when (s0.n0).id is null or s0.e0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, array [s0.e0]::int8[], array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((exists (select 1 from edge i0 where (kind_name(i0.kind_id)::text like 'EdgeKind%') and i0.id = any (array [s0.e0]::int8[])))::bool); -- case: match (a)-[*2..2]->(b)-[]->(c) return a -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, e1.id as e1, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != s0.e0), s2 as (select s1.e0 as e0, s1.e1 as e1, s1.n0 as n0, s1.n1 as n1, s1.n2 as n2 from s1 join edge e2 on (s1.n2).id = e2.start_id join node n3 on n3.id = e2.end_id where e2.id != s1.e0 and e2.id != s1.e1) select s2.n0 as a from s2; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, e1.id as e1, s0.n0 as n0, s0.n1 as n1, n2.id as n2 from s0 join edge e1 on s0.n1 = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != s0.e0), s2 as (select s1.e0 as e0, s1.e1 as e1, s1.n0 as n0, s1.n1 as n1, s1.n2 as n2 from s1 join edge e2 on s1.n2 = e2.start_id join node n3 on n3.id = e2.end_id where e2.id != s1.e0 and e2.id != s1.e1) select s2.n0 as a from s2; -- case: match p=(:NodeKind1)-[r]->(:NodeKind1) where r.isacl return p limit 100 with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.end_id where (((e0.properties ->> 'isacl'))::bool) limit 100) select case when (s0.n0).id is null or (s0.e0).id is null or (s0.n1).id is null then null else (array [s0.n0, s0.n1]::nodecomposite[], array [s0.e0]::edgecomposite[])::pathcomposite end as p from s0 limit 100; -- case: match p = ()-[r1]->()-[r2]->(e) return e -with s0 as (select e0.id as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != s0.e0) select s1.n2 as e from s1; +with s0 as (select e0.id as e0, n1.id as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on s0.n1 = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != s0.e0) select s1.n2 as e from s1; -- case: match ()-[r1]->()-[r2]->()-[]->() where r1.name = 'a' and r2.name = 'b' return r1 -with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where ((jsonb_typeof((e0.properties -> 'name')) = 'string' and (e0.properties ->> 'name') = 'a'))), s1 as (select s0.e0 as e0, (e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties)::edgecomposite as e1, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where ((jsonb_typeof((e1.properties -> 'name')) = 'string' and (e1.properties ->> 'name') = 'b')) and e1.id != (s0.e0).id), s2 as (select s1.e0 as e0, s1.e1 as e1, s1.n1 as n1, s1.n2 as n2 from s1 join edge e2 on (s1.n2).id = e2.start_id join node n3 on n3.id = e2.end_id where e2.id != (s1.e0).id and e2.id != (s1.e1).id) select s2.e0 as r1 from s2; +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n1.id as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where ((jsonb_typeof((e0.properties -> 'name')) = 'string' and (e0.properties ->> 'name') = 'a'))), s1 as (select s0.e0 as e0, (e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties)::edgecomposite as e1, s0.n1 as n1, n2.id as n2 from s0 join edge e1 on s0.n1 = e1.start_id join node n2 on n2.id = e1.end_id where ((jsonb_typeof((e1.properties -> 'name')) = 'string' and (e1.properties ->> 'name') = 'b')) and e1.id != (s0.e0).id), s2 as (select s1.e0 as e0, s1.e1 as e1, s1.n1 as n1, s1.n2 as n2 from s1 join edge e2 on s1.n2 = e2.start_id join node n3 on n3.id = e2.end_id where e2.id != (s1.e0).id and e2.id != (s1.e1).id) select s2.e0 as r1 from s2; -- case: match p = (a)-[]->()<-[]-(f) where a.name = 'value' and f.is_target return p -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'value')) and n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, e1.id as e1, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.end_id join node n2 on (((n2.properties ->> 'is_target'))::bool) and n2.id = e1.start_id where e1.id != s0.e0) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null or s1.e1 is null or (s1.n2).id is null then null else ordered_edges_to_path(s1.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id) || (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e1]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n0, s1.n1, s1.n2]::nodecomposite[])::pathcomposite end as p from s1; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'value')) and n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, e1.id as e1, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.end_id join node n2 on (((n2.properties ->> 'is_target'))::bool) and n2.id = e1.start_id where e1.id != s0.e0) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null or s1.e1 is null or (s1.n2).id is null then null else ordered_edge_ids_to_path(0, s1.n0, array [s1.e0]::int8[] || array [s1.e1]::int8[], array [s1.n0, s1.n1, s1.n2]::nodecomposite[])::pathcomposite end as p from s1; -- case: match p = ()-[*..]->() return p limit 1 -with s0 as (with recursive s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, false, e0.start_id = e0.end_id, array [e0.id] from edge e0 union all select s1.root_id, e0.end_id, s1.depth + 1, false, false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true limit 1) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 1; +with s0 as (with recursive s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, false, false, array [e0.id] from edge e0 union all select s1.root_id, e0.end_id, s1.depth + 1, false, false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true limit 1) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 1; -- case: match p = (s)-[*..]->(i)-[]->() where id(s) = 1 and i.name = 'n3' return p limit 1 -with s0 as (with recursive s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'n3'))), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, (n0.id = 1), e0.end_id = e0.start_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n0 on n0.id = e0.start_id union all select s1.root_id, e0.start_id, s1.depth + 1, (n0.id = 1), false, e0.id || s1.path from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n0 on n0.id = e0.start_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.root_id offset 0) n1 on true join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.next_id offset 0) n0 on true where s1.satisfied), s2 as (select e1.id as e1, s0.ep0 as ep0, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != all (s0.ep0) limit 1) select case when (s2.n0).id is null or s2.ep0 is null or (s2.n1).id is null or s2.e1 is null or (s2.n2).id is null then null else ordered_edges_to_path(s2.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s2.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id) || (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s2.e1]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s2.n0, s2.n1, s2.n2]::nodecomposite[])::pathcomposite end as p from s2 limit 1; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'n3'))), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, (n0.id = 1), false, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n0 on n0.id = e0.start_id union all select s1.root_id, e0.start_id, s1.depth + 1, (n0.id = 1), false, e0.id || s1.path from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n0 on n0.id = e0.start_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.root_id offset 0) n1 on true join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.next_id offset 0) n0 on true where s1.satisfied), s2 as (select e1.id as e1, s0.ep0 as ep0, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != all (s0.ep0) limit 1) select case when (s2.n0).id is null or s2.ep0 is null or (s2.n1).id is null or s2.e1 is null or (s2.n2).id is null then null else ordered_edge_ids_to_path(0, s2.n0, s2.ep0 || array [s2.e1]::int8[], array [s2.n0, s2.n1, s2.n2]::nodecomposite[])::pathcomposite end as p from s2 limit 1; -- case: match p = ()-[e:EdgeKind1]->()-[:EdgeKind1*..]->() return e, p -with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[])), s1 as (with recursive s2_seed(root_id) as not materialized (select distinct (s0.n1).id as root_id from s0), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e1.start_id, e1.end_id, 1, false, e1.start_id = e1.end_id, array [e1.id] from s2_seed join edge e1 on e1.start_id = s2_seed.root_id where e1.kind_id = any (array [3]::int2[]) union all select s2.root_id, e1.end_id, s2.depth + 1, false, false, s2.path || e1.id from s2 join lateral (select e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties from edge e1 where e1.start_id = s2.next_id and e1.id != all (s2.path) and e1.kind_id = any (array [3]::int2[]) offset 0) e1 on true where s2.depth < 15 and not s2.is_cycle) select s0.e0 as e0, s2.path as ep0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, s2 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.root_id offset 0) n1 on true join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s2.next_id offset 0) n2 on true where (s0.n1).id = s2.root_id) select s1.e0 as e, case when (s1.n0).id is null or (s1.e0).id is null or (s1.n1).id is null or s1.ep0 is null or (s1.n2).id is null then null else ordered_edges_to_path(s1.n0, array [s1.e0]::edgecomposite[] || (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s1.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n0, s1.n1, s1.n2]::nodecomposite[])::pathcomposite end as p from s1; +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[])), s1 as (with recursive s2_seed(root_id) as not materialized (select distinct (s0.n1).id as root_id from s0), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e1.start_id, e1.end_id, 1, false, false, array [e1.id] from s2_seed join edge e1 on e1.start_id = s2_seed.root_id where e1.kind_id = any (array [3]::int2[]) union all select s2.root_id, e1.end_id, s2.depth + 1, false, false, s2.path || e1.id from s2 join lateral (select e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties from edge e1 where e1.start_id = s2.next_id and e1.id != all (s2.path) and e1.kind_id = any (array [3]::int2[]) offset 0) e1 on true where s2.depth < 15 and not s2.is_cycle) select s0.e0 as e0, s2.path as ep0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, s2 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.root_id offset 0) n1 on true join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s2.next_id offset 0) n2 on true where (s0.n1).id = s2.root_id) select s1.e0 as e, case when (s1.n0).id is null or (s1.e0).id is null or (s1.n1).id is null or s1.ep0 is null or (s1.n2).id is null then null else ordered_edges_to_path(0, s1.n0, array [s1.e0]::edgecomposite[] || (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s1.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id and _edge.graph_id = 0), array [s1.n0, s1.n1, s1.n2]::nodecomposite[])::pathcomposite end as p from s1; -- case: match p = (m:NodeKind1)-[:EdgeKind1]->(c:NodeKind2) where m.objectid ends with "-513" and not toUpper(c.operatingsystem) contains "SERVER" return p limit 1000 -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on ((n0.properties ->> 'objectid') like '%-513') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on (not upper((n1.properties ->> 'operatingsystem'))::text like '%SERVER%') and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) limit 1000) select case when (s0.n0).id is null or s0.e0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s0.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 1000; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on ((n0.properties ->> 'objectid') like '%-513') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on (not upper((n1.properties ->> 'operatingsystem'))::text like '%SERVER%') and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) limit 1000) select case when (s0.n0).id is null or s0.e0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, array [s0.e0]::int8[], array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 1000; -- case: match p = (:NodeKind1)-[:EdgeKind1|EdgeKind2]->(e:NodeKind2)-[:EdgeKind2]->(:NodeKind1) where 'a' in e.values or 'b' in e.values or size(e.values) = 0 return p -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on ('a' = any (jsonb_to_text_array((n1.properties -> 'values'))::text[]) or 'b' = any (jsonb_to_text_array((n1.properties -> 'values'))::text[]) or jsonb_array_length((n1.properties -> 'values'))::int = 0) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[])), s1 as (select s0.e0 as e0, e1.id as e1, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) and e1.id != s0.e0) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null or s1.e1 is null or (s1.n2).id is null then null else ordered_edges_to_path(s1.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id) || (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e1]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n0, s1.n1, s1.n2]::nodecomposite[])::pathcomposite end as p from s1; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on ('a' = any (jsonb_to_text_array((n1.properties -> 'values'))::text[]) or 'b' = any (jsonb_to_text_array((n1.properties -> 'values'))::text[]) or case when jsonb_typeof((n1.properties -> 'values')) = 'array' then jsonb_array_length((n1.properties -> 'values'))::int else null end = 0) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[])), s1 as (select s0.e0 as e0, e1.id as e1, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) and e1.id != s0.e0) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null or s1.e1 is null or (s1.n2).id is null then null else ordered_edge_ids_to_path(0, s1.n0, array [s1.e0]::int8[] || array [s1.e1]::int8[], array [s1.n0, s1.n1, s1.n2]::nodecomposite[])::pathcomposite end as p from s1; -- case: match p = (n:NodeKind1)-[r]-(m:NodeKind1) return p -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (n0.id = e0.end_id or n0.id = e0.start_id) join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (n1.id = e0.end_id or n1.id = e0.start_id) where (n0.id <> n1.id)) select case when (s0.n0).id is null or s0.e0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s0.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (n0.id = e0.end_id or n0.id = e0.start_id) join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (n1.id = e0.end_id or n1.id = e0.start_id) where ((n0.id = e0.start_id and n1.id = e0.end_id) or (n1.id = e0.start_id and n0.id = e0.end_id))) select case when (s0.n0).id is null or s0.e0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, array [s0.e0]::int8[], array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; -- case: match p = (:NodeKind1)-[:EdgeKind1]->(:NodeKind2)-[:EdgeKind2*1..]->(t:NodeKind2) where coalesce(t.system_tags, '') contains 'admin_tier_0' return p limit 1000 -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[])), s1 as (with recursive s2_seed(root_id) as not materialized (select distinct (s0.n1).id as root_id from s0), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e1.start_id, e1.end_id, 1, (coalesce((n2.properties ->> 'system_tags'), '')::text like '%admin_tier_0%') and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[], e1.start_id = e1.end_id, array [e1.id] from s2_seed join edge e1 on e1.start_id = s2_seed.root_id join node n2 on n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) union all select s2.root_id, e1.end_id, s2.depth + 1, (coalesce((n2.properties ->> 'system_tags'), '')::text like '%admin_tier_0%') and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e1.id from s2 join lateral (select e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties from edge e1 where e1.start_id = s2.next_id and e1.id != all (s2.path) and e1.kind_id = any (array [4]::int2[]) offset 0) e1 on true join node n2 on n2.id = e1.end_id where s2.depth < 15 and not s2.is_cycle) select s0.e0 as e0, s2.path as ep0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, s2 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.root_id offset 0) n1 on true join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s2.next_id offset 0) n2 on true where s2.satisfied and (s0.n1).id = s2.root_id limit 1000) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null or s1.ep0 is null or (s1.n2).id is null then null else ordered_edges_to_path(s1.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id) || (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s1.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n0, s1.n1, s1.n2]::nodecomposite[])::pathcomposite end as p from s1 limit 1000; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[])), s1 as (with recursive s2_seed(root_id) as not materialized (select distinct (s0.n1).id as root_id from s0), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e1.start_id, e1.end_id, 1, (coalesce((n2.properties ->> 'system_tags'), '')::text like '%admin_tier_0%') and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, array [e1.id] from s2_seed join edge e1 on e1.start_id = s2_seed.root_id join node n2 on n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) union all select s2.root_id, e1.end_id, s2.depth + 1, (coalesce((n2.properties ->> 'system_tags'), '')::text like '%admin_tier_0%') and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e1.id from s2 join lateral (select e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties from edge e1 where e1.start_id = s2.next_id and e1.id != all (s2.path) and e1.kind_id = any (array [4]::int2[]) offset 0) e1 on true join node n2 on n2.id = e1.end_id where s2.depth < 15 and not s2.is_cycle) select s0.e0 as e0, s2.path as ep0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, s2 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.root_id offset 0) n1 on true join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s2.next_id offset 0) n2 on true where s2.satisfied and (s0.n1).id = s2.root_id limit 1000) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null or s1.ep0 is null or (s1.n2).id is null then null else ordered_edge_ids_to_path(0, s1.n0, array [s1.e0]::int8[] || s1.ep0, array [s1.n0, s1.n1, s1.n2]::nodecomposite[])::pathcomposite end as p from s1 limit 1000; -- case: match (u:NodeKind1) where u.samaccountname in ["foo", "bar"] match p = (u)-[:EdgeKind1|EdgeKind2*1..3]->(t) where coalesce(t.system_tags, '') contains 'admin_tier_0' return p limit 1000 -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((n0.properties ->> 'samaccountname') = any (array ['foo', 'bar']::text[])) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (with recursive s2_seed(root_id) as not materialized (select distinct (s0.n0).id as root_id from s0), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, (coalesce((n1.properties ->> 'system_tags'), '')::text like '%admin_tier_0%'), e0.start_id = e0.end_id, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s2.root_id, e0.end_id, s2.depth + 1, (coalesce((n1.properties ->> 'system_tags'), '')::text like '%admin_tier_0%'), false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 3 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and (s0.n0).id = s2.root_id) select case when (s1.n0).id is null or s1.ep0 is null or (s1.n1).id is null then null else ordered_edges_to_path(s1.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s1.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as p from s1 limit 1000; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((n0.properties ->> 'samaccountname') = any (array ['foo', 'bar']::text[])) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (with recursive s2_seed(root_id) as not materialized (select distinct (s0.n0).id as root_id from s0), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, (coalesce((n1.properties ->> 'system_tags'), '')::text like '%admin_tier_0%'), false, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s2.root_id, e0.end_id, s2.depth + 1, (coalesce((n1.properties ->> 'system_tags'), '')::text like '%admin_tier_0%'), false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 3 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and (s0.n0).id = s2.root_id) select case when (s1.n0).id is null or s1.ep0 is null or (s1.n1).id is null then null else ordered_edge_ids_to_path(0, s1.n0, s1.ep0, array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as p from s1 limit 1000; -- case: match (x:NodeKind1) where x.name = 'foo' match (y:NodeKind2) where y.name = 'bar' match p=(x)-[:EdgeKind1]->(y) return p -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'foo')) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'bar')) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s2 as (select e0.id as e0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e0 on (s1.n0).id = e0.start_id and (s1.n1).id = e0.end_id where e0.kind_id = any (array [3]::int2[])) select case when (s2.n0).id is null or s2.e0 is null or (s2.n1).id is null then null else ordered_edges_to_path(s2.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s2.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s2.n0, s2.n1]::nodecomposite[])::pathcomposite end as p from s2; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'foo')) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'bar')) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s2 as (select e0.id as e0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e0 on (s1.n0).id = e0.start_id and (s1.n1).id = e0.end_id where e0.kind_id = any (array [3]::int2[])) select case when (s2.n0).id is null or s2.e0 is null or (s2.n1).id is null then null else ordered_edge_ids_to_path(0, s2.n0, array [s2.e0]::int8[], array [s2.n0, s2.n1]::nodecomposite[])::pathcomposite end as p from s2; -- case: match (x:NodeKind1{name:'foo'}) match (y:NodeKind2{name:'bar'}) match p=(x)-[:EdgeKind1]->(y) return p -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'foo')), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'bar')), s2 as (select e0.id as e0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e0 on (s1.n0).id = e0.start_id and (s1.n1).id = e0.end_id where e0.kind_id = any (array [3]::int2[])) select case when (s2.n0).id is null or s2.e0 is null or (s2.n1).id is null then null else ordered_edges_to_path(s2.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s2.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s2.n0, s2.n1]::nodecomposite[])::pathcomposite end as p from s2; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'foo')), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'bar')), s2 as (select e0.id as e0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e0 on (s1.n0).id = e0.start_id and (s1.n1).id = e0.end_id where e0.kind_id = any (array [3]::int2[])) select case when (s2.n0).id is null or s2.e0 is null or (s2.n1).id is null then null else ordered_edge_ids_to_path(0, s2.n0, array [s2.e0]::int8[], array [s2.n0, s2.n1]::nodecomposite[])::pathcomposite end as p from s2; -- case: match (x:NodeKind1{name:'foo'}) match p=(x)-[:EdgeKind1]->(y:NodeKind2{name:'bar'}) return p -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'foo')), s1 as (select e0.id as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0 join edge e0 on (s0.n0).id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'bar') and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[])) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null then null else ordered_edges_to_path(s1.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as p from s1; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'foo')), s1 as (select e0.id as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0 join edge e0 on (s0.n0).id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'bar') and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[])) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null then null else ordered_edge_ids_to_path(0, s1.n0, array [s1.e0]::int8[], array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as p from s1; -- case: match (x:NodeKind1{name:'foo'}) match p=(x)-[]-(y:NodeKind2{name:'bar'}) return p -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'foo')), s1 as (select e0.id as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0 join edge e0 on ((s0.n0).id = e0.end_id or (s0.n0).id = e0.start_id) join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'bar') and (n1.id = e0.end_id or n1.id = e0.start_id) where ((s0.n0).id <> n1.id)) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null then null else ordered_edges_to_path(s1.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as p from s1; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'foo')), s1 as (select e0.id as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0 join edge e0 on ((s0.n0).id = e0.end_id or (s0.n0).id = e0.start_id) join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'bar') and (n1.id = e0.end_id or n1.id = e0.start_id) where (((s0.n0).id = e0.start_id and n1.id = e0.end_id) or (n1.id = e0.start_id and (s0.n0).id = e0.end_id))) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null then null else ordered_edge_ids_to_path(0, s1.n0, array [s1.e0]::int8[], array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as p from s1; -- case: match (e) match p = ()-[]-(e) return p limit 1 -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0), s1 as (select e0.id as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0 join edge e0 on ((s0.n0).id = e0.end_id or (s0.n0).id = e0.start_id) join node n1 on (n1.id = e0.end_id or n1.id = e0.start_id) where ((s0.n0).id <> n1.id)) select case when (s1.n1).id is null or s1.e0 is null or (s1.n0).id is null then null else ordered_edges_to_path(s1.n1, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n1, s1.n0]::nodecomposite[])::pathcomposite end as p from s1 limit 1; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0), s1 as (select e0.id as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0 join edge e0 on ((s0.n0).id = e0.end_id or (s0.n0).id = e0.start_id) join node n1 on (n1.id = e0.end_id or n1.id = e0.start_id) where (((s0.n0).id = e0.start_id and n1.id = e0.end_id) or (n1.id = e0.start_id and (s0.n0).id = e0.end_id))) select case when (s1.n1).id is null or s1.e0 is null or (s1.n0).id is null then null else ordered_edge_ids_to_path(0, s1.n1, array [s1.e0]::int8[], array [s1.n1, s1.n0]::nodecomposite[])::pathcomposite end as p from s1 limit 1; -- case: match (x:NodeKind1{name:'foo'}) match (y:NodeKind2{name:'bar'}) match p=(x)-[]-(y) return p -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'foo')), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'bar')), s2 as (select e0.id as e0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e0 on ((s1.n0).id = e0.start_id or (s1.n0).id = e0.end_id) and ((s1.n1).id = e0.end_id or (s1.n1).id = e0.start_id) where ((s1.n0).id <> (s1.n1).id)) select case when (s2.n0).id is null or s2.e0 is null or (s2.n1).id is null then null else ordered_edges_to_path(s2.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s2.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s2.n0, s2.n1]::nodecomposite[])::pathcomposite end as p from s2; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'foo')), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'bar')), s2 as (select e0.id as e0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e0 on (((s1.n0).id = e0.start_id and (s1.n1).id = e0.end_id) or ((s1.n1).id = e0.start_id and (s1.n0).id = e0.end_id))) select case when (s2.n0).id is null or s2.e0 is null or (s2.n1).id is null then null else ordered_edge_ids_to_path(0, s2.n0, array [s2.e0]::int8[], array [s2.n0, s2.n1]::nodecomposite[])::pathcomposite end as p from s2; -- case: match (e) match p = ()-[]->(e) return p limit 1 -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0), s1 as (select e0.id as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0 join edge e0 on (s0.n0).id = e0.end_id join node n1 on n1.id = e0.start_id) select case when (s1.n1).id is null or s1.e0 is null or (s1.n0).id is null then null else ordered_edges_to_path(s1.n1, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n1, s1.n0]::nodecomposite[])::pathcomposite end as p from s1 limit 1; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0), s1 as (select e0.id as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0 join edge e0 on (s0.n0).id = e0.end_id join node n1 on n1.id = e0.start_id) select case when (s1.n1).id is null or s1.e0 is null or (s1.n0).id is null then null else ordered_edge_ids_to_path(0, s1.n1, array [s1.e0]::int8[], array [s1.n1, s1.n0]::nodecomposite[])::pathcomposite end as p from s1 limit 1; -- case: match p = (a)-[]->() match q = ()-[]->(a) return p, q -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, e1.id as e1, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n0).id = e1.end_id join node n2 on n2.id = e1.start_id) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null then null else ordered_edges_to_path(s1.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as p, case when (s1.n2).id is null or s1.e1 is null or (s1.n0).id is null then null else ordered_edges_to_path(s1.n2, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e1]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n2, s1.n0]::nodecomposite[])::pathcomposite end as q from s1; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, e1.id as e1, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n0).id = e1.end_id join node n2 on n2.id = e1.start_id) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null then null else ordered_edge_ids_to_path(0, s1.n0, array [s1.e0]::int8[], array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as p, case when (s1.n2).id is null or s1.e1 is null or (s1.n0).id is null then null else ordered_edge_ids_to_path(0, s1.n2, array [s1.e1]::int8[], array [s1.n2, s1.n0]::nodecomposite[])::pathcomposite end as q from s1; -- case: match (m:NodeKind1)-[*1..]->(g:NodeKind2)-[]->(c3:NodeKind1) where not g.name in ["foo"] with collect(g.name) as bar match p=(m:NodeKind1)-[*1..]->(g:NodeKind2) where g.name in bar return p -with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, (not (n1.properties ->> 'name') = any (array ['foo']::text[])) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], e0.start_id = e0.end_id, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id union all select s2.root_id, e0.end_id, s2.depth + 1, (not (n1.properties ->> 'name') = any (array ['foo']::text[])) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 15 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and exists (select 1 from edge e1 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where n1.id = e1.start_id)), s3 as (select s1.ep0 as ep0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e1 on (s1.n1).id = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where e1.id != all (s1.ep0)) select array_remove(coalesce(array_agg(((s3.n1).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray as i0 from s3), s4 as (with recursive s5_seed(root_id) as not materialized (select n4.id as root_id from s0, node n4 where n4.kind_ids operator (pg_catalog.@>) array [2]::int2[] and ((n4.properties ->> 'name') = any (s0.i0))), s5(root_id, next_id, depth, satisfied, is_cycle, path) as (select e2.end_id, e2.start_id, 1, n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], e2.end_id = e2.start_id, array [e2.id] from s5_seed join edge e2 on e2.end_id = s5_seed.root_id join node n3 on n3.id = e2.start_id union select s5.root_id, e2.start_id, s5.depth + 1, n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, e2.id || s5.path from s5 join lateral (select e2.id, e2.start_id, e2.end_id, e2.kind_id, e2.properties from edge e2 where e2.end_id = s5.next_id and e2.id != all (s5.path) offset 0) e2 on true join node n3 on n3.id = e2.start_id where s5.depth < 15 and not s5.is_cycle) select s5.path as ep1, s0.i0 as i0, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3, (n4.id, n4.kind_ids, n4.properties)::nodecomposite as n4 from s0, s5 join lateral (select n4.id, n4.kind_ids, n4.properties from node n4 where n4.id = s5.root_id offset 0) n4 on true join lateral (select n3.id, n3.kind_ids, n3.properties from node n3 where n3.id = s5.next_id offset 0) n3 on true where s5.satisfied) select case when (s4.n3).id is null or s4.ep1 is null or (s4.n4).id is null then null else ordered_edges_to_path(s4.n3, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s4.ep1) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s4.n3, s4.n4]::nodecomposite[])::pathcomposite end as p from s4; +with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, (not (n1.properties ->> 'name') = any (array ['foo']::text[])) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id union all select s2.root_id, e0.end_id, s2.depth + 1, (not (n1.properties ->> 'name') = any (array ['foo']::text[])) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 15 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and exists (select 1 from edge e1 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where n1.id = e1.start_id)), s3 as (select s1.ep0 as ep0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e1 on (s1.n1).id = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where e1.id != all (s1.ep0)) select array_remove(coalesce(array_agg(((s3.n1).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray as i0 from s3), s4 as (with recursive s5_seed(root_id) as not materialized (select n4.id as root_id from s0, node n4 where n4.kind_ids operator (pg_catalog.@>) array [2]::int2[] and ((n4.properties ->> 'name') = any (s0.i0))), s5(root_id, next_id, depth, satisfied, is_cycle, path) as (select e2.end_id, e2.start_id, 1, n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e2.id] from s5_seed join edge e2 on e2.end_id = s5_seed.root_id join node n3 on n3.id = e2.start_id union select s5.root_id, e2.start_id, s5.depth + 1, n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, e2.id || s5.path from s5 join lateral (select e2.id, e2.start_id, e2.end_id, e2.kind_id, e2.properties from edge e2 where e2.end_id = s5.next_id and e2.id != all (s5.path) offset 0) e2 on true join node n3 on n3.id = e2.start_id where s5.depth < 15 and not s5.is_cycle) select s5.path as ep1, s0.i0 as i0, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3, (n4.id, n4.kind_ids, n4.properties)::nodecomposite as n4 from s0, s5 join lateral (select n4.id, n4.kind_ids, n4.properties from node n4 where n4.id = s5.root_id offset 0) n4 on true join lateral (select n3.id, n3.kind_ids, n3.properties from node n3 where n3.id = s5.next_id offset 0) n3 on true where s5.satisfied) select case when (s4.n3).id is null or s4.ep1 is null or (s4.n4).id is null then null else ordered_edge_ids_to_path(0, s4.n3, s4.ep1, array [s4.n3, s4.n4]::nodecomposite[])::pathcomposite end as p from s4; -- case: MATCH p=(:Computer)-[r:HasSession]->(:User) WHERE r.lastseen >= datetime() - duration('P3D') RETURN p LIMIT 100 with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [5]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [6]::int2[] and n1.id = e0.end_id where (((e0.properties ->> 'lastseen'))::timestamp with time zone >= now()::timestamp with time zone - interval 'P3D') and e0.kind_id = any (array [7]::int2[]) limit 100) select case when (s0.n0).id is null or (s0.e0).id is null or (s0.n1).id is null then null else (array [s0.n0, s0.n1]::nodecomposite[], array [s0.e0]::edgecomposite[])::pathcomposite end as p from s0 limit 100; -- case: MATCH p=(:GPO)-[r:GPLink|Contains*1..]->(:Base) WHERE HEAD(r).enforced OR NONE(n in TAIL(TAIL(NODES(p))) WHERE (n:OU AND n.blocksinheritance)) RETURN p -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [8]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [10]::int2[], e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [11, 12]::int2[]) union all select s1.root_id, e0.end_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [10]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [11, 12]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s1.path) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id) as e0, s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied) select s2.pc0 as p from s0, lateral (select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as pc0 offset 0) s2 where (((((s0.e0)[1]).properties ->> 'enforced'))::bool or ((select count(*)::int from unnest(coalesce((coalesce((((s2.pc0).nodes)::nodecomposite[])[2:], array []::nodecomposite[])::nodecomposite[])[2:], array []::nodecomposite[])::nodecomposite[]) as i0 where ((i0.kind_ids operator (pg_catalog.@>) array [9]::int2[] and ((i0.properties ->> 'blocksinheritance'))::bool))) = 0 and coalesce((coalesce((((s2.pc0).nodes)::nodecomposite[])[2:], array []::nodecomposite[])::nodecomposite[])[2:], array []::nodecomposite[])::nodecomposite[] is not null)::bool); +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [8]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [10]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [11, 12]::int2[]) union all select s1.root_id, e0.end_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [10]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [11, 12]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s1.path) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id and _edge.graph_id = 0) as e0, s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied) select s2.pc0 as p from s0, lateral (select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as pc0 offset 0) s2 where (((((s0.e0)[1]).properties ->> 'enforced'))::bool or ((select count(*)::int from unnest(coalesce((coalesce((((s2.pc0).nodes)::nodecomposite[])[2:], array []::nodecomposite[])::nodecomposite[])[2:], array []::nodecomposite[])::nodecomposite[]) as i0 where ((i0.kind_ids operator (pg_catalog.@>) array [9]::int2[] and ((i0.properties ->> 'blocksinheritance'))::bool))) = 0 and coalesce((coalesce((((s2.pc0).nodes)::nodecomposite[])[2:], array []::nodecomposite[])::nodecomposite[])[2:], array []::nodecomposite[])::nodecomposite[] is not null)::bool); -- case: MATCH p=(:GPO)-[r:GPLink|Contains*1..]->(:Base) WHERE NONE(x in TAIL(r) WHERE NOT type(x) = 'Contains') RETURN p -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [8]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [10]::int2[], e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [11, 12]::int2[]) union all select s1.root_id, e0.end_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [10]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [11, 12]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s1.path) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id) as e0, s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where (((select count(*)::int from unnest(coalesce((s0.e0)[2:], array []::edgecomposite[])::edgecomposite[]) as i0 where (not i0.kind_id = 12)) = 0 and coalesce((s0.e0)[2:], array []::edgecomposite[])::edgecomposite[] is not null)::bool); +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [8]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [10]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [11, 12]::int2[]) union all select s1.root_id, e0.end_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [10]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [11, 12]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s1.path) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id and _edge.graph_id = 0) as e0, s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where (((select count(*)::int from unnest(coalesce((s0.e0)[2:], array []::edgecomposite[])::edgecomposite[]) as i0 where (not i0.kind_id = 12)) = 0 and coalesce((s0.e0)[2:], array []::edgecomposite[])::edgecomposite[] is not null)::bool); diff --git a/cypher/models/pgsql/test/translation_cases/pattern_expansion.sql b/cypher/models/pgsql/test/translation_cases/pattern_expansion.sql index 895d3c41..bc6e662d 100644 --- a/cypher/models/pgsql/test/translation_cases/pattern_expansion.sql +++ b/cypher/models/pgsql/test/translation_cases/pattern_expansion.sql @@ -15,71 +15,73 @@ -- SPDX-License-Identifier: Apache-2.0 -- case: match (n)-[*..]->(e) return n, e -with s0 as (with recursive s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, false, e0.start_id = e0.end_id, array [e0.id] from edge e0 union all select s1.root_id, e0.end_id, s1.depth + 1, false, false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true where s1.depth < 15 and not s1.is_cycle) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true) select s0.n0 as n, s0.n1 as e from s0; +with s0 as (with recursive s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, false, false, array [e0.id] from edge e0 union all select s1.root_id, e0.end_id, s1.depth + 1, false, false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true where s1.depth < 15 and not s1.is_cycle) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true) select s0.n0 as n, s0.n1 as e from s0; -- case: match (n)-[*1..2]->(e) return n, e -with s0 as (with recursive s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, false, e0.start_id = e0.end_id, array [e0.id] from edge e0 union all select s1.root_id, e0.end_id, s1.depth + 1, false, false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true where s1.depth < 2 and not s1.is_cycle) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true) select s0.n0 as n, s0.n1 as e from s0; +with s0 as (with recursive s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, false, false, array [e0.id] from edge e0 union all select s1.root_id, e0.end_id, s1.depth + 1, false, false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true where s1.depth < 2 and not s1.is_cycle) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true) select s0.n0 as n, s0.n1 as e from s0; -- case: match (n)-[*3..5]->(e) return n, e -with s0 as (with recursive s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, false, e0.start_id = e0.end_id, array [e0.id] from edge e0 union all select s1.root_id, e0.end_id, s1.depth + 1, false, false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true where s1.depth < 5 and not s1.is_cycle) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.depth >= 3) select s0.n0 as n, s0.n1 as e from s0; +with s0 as (with recursive s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, false, false, array [e0.id] from edge e0 union all select s1.root_id, e0.end_id, s1.depth + 1, false, false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true where s1.depth < 5 and not s1.is_cycle) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.depth >= 3) select s0.n0 as n, s0.n1 as e from s0; -- case: match (n)<-[*2..5]-(e) return n, e -with s0 as (with recursive s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, false, e0.end_id = e0.start_id, array [e0.id] from edge e0 union all select s1.root_id, e0.start_id, s1.depth + 1, false, false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true where s1.depth < 5 and not s1.is_cycle) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.depth >= 2) select s0.n0 as n, s0.n1 as e from s0; +with s0 as (with recursive s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, false, false, array [e0.id] from edge e0 union all select s1.root_id, e0.start_id, s1.depth + 1, false, false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true where s1.depth < 5 and not s1.is_cycle) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.depth >= 2) select s0.n0 as n, s0.n1 as e from s0; -- case: match p = (n)-[*..]->(e:NodeKind1) return p -with s0 as (with recursive s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where n1.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, false, e0.end_id = e0.start_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id union all select s1.root_id, e0.start_id, s1.depth + 1, false, false, e0.id || s1.path from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.root_id offset 0) n1 on true join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.next_id offset 0) n0 on true) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where n1.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, false, false, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id union all select s1.root_id, e0.start_id, s1.depth + 1, false, false, e0.id || s1.path from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.root_id offset 0) n1 on true join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.next_id offset 0) n0 on true) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; -- case: match (n)-[*..]->(e:NodeKind1) where n.name = 'n1' return e -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n1'))), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id union all select s1.root_id, e0.end_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied) select s0.n1 as e from s0; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n1'))), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id union all select s1.root_id, e0.end_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied) select s0.n1 as e from s0; -- case: match (n)-[*..]->(e:NodeKind1) where n.name = 'n2' return n -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n2'))), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id union all select s1.root_id, e0.end_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied) select s0.n0 as n from s0; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n2'))), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id union all select s1.root_id, e0.end_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied) select s0.n0 as n from s0; -- case: match (n)-[*..]->(e:NodeKind1)-[]->(l) where n.name = 'n1' return l -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n1'))), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id union all select s1.root_id, e0.end_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied), s2 as (select s0.ep0 as ep0, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != all (s0.ep0)) select s2.n2 as l from s2; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n1'))), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id union all select s1.root_id, e0.end_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied), s2 as (select s0.ep0 as ep0, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on s0.n1 = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != all (s0.ep0)) select s2.n2 as l from s2; -- case: match (n)-[*2..3]->(e:NodeKind1)-[]->(l) where n.name = 'n1' return l -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n1'))), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id union all select s1.root_id, e0.end_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 3 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.depth >= 2 and s1.satisfied), s2 as (select s0.ep0 as ep0, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != all (s0.ep0)) select s2.n2 as l from s2; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n1'))), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id union all select s1.root_id, e0.end_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 3 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.depth >= 2 and s1.satisfied), s2 as (select s0.ep0 as ep0, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on s0.n1 = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != all (s0.ep0)) select s2.n2 as l from s2; -- case: match (n)-[]->(e:NodeKind1)-[*2..3]->(l) where n.name = 'n1' return l -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n1')) and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.end_id), s1 as (with recursive s2_seed(root_id) as not materialized (select distinct (s0.n1).id as root_id from s0), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e1.start_id, e1.end_id, 1, false, e1.start_id = e1.end_id, array [e1.id] from s2_seed join edge e1 on e1.start_id = s2_seed.root_id union all select s2.root_id, e1.end_id, s2.depth + 1, false, false, s2.path || e1.id from s2 join lateral (select e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties from edge e1 where e1.start_id = s2.next_id and e1.id != all (s2.path) offset 0) e1 on true where s2.depth < 3 and not s2.is_cycle) select s0.e0 as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, s2 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.root_id offset 0) n1 on true join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s2.next_id offset 0) n2 on true where s2.depth >= 2 and (s0.n1).id = s2.root_id) select s1.n2 as l from s1; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n0 on ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n1')) and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.end_id), s1 as (with recursive s2_seed(root_id) as not materialized (select distinct s0.n1 as root_id from s0), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e1.start_id, e1.end_id, 1, false, false, array [e1.id] from s2_seed join edge e1 on e1.start_id = s2_seed.root_id union all select s2.root_id, e1.end_id, s2.depth + 1, false, false, s2.path || e1.id from s2 join lateral (select e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties from edge e1 where e1.start_id = s2.next_id and e1.id != all (s2.path) offset 0) e1 on true where s2.depth < 3 and not s2.is_cycle) select s0.e0 as e0, s0.n0 as n0, n1.id as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, s2 join lateral (select n1.id from node n1 where n1.id = s2.root_id offset 0) n1 on true join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s2.next_id offset 0) n2 on true where s2.depth >= 2 and s0.n1 = s2.root_id) select s1.n2 as l from s1; -- case: match (n)-[*..]->(e)-[:EdgeKind1|EdgeKind2]->()-[*..]->(l) where n.name = 'n1' and e.name = 'n2' return l -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n1'))), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'n2')), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id union all select s1.root_id, e0.end_id, s1.depth + 1, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'n2')), false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied and exists (select 1 from edge e1 join node n2 on n2.id = e1.end_id where n1.id = e1.start_id and e1.kind_id = any (array [3, 4]::int2[]))), s2 as (select e1.id as e1, s0.ep0 as ep0, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.kind_id = any (array [3, 4]::int2[]) and e1.id != all (s0.ep0)), s3 as (with recursive s4_seed(root_id) as not materialized (select distinct (s2.n2).id as root_id from s2), s4(root_id, next_id, depth, satisfied, is_cycle, path) as (select e2.start_id, e2.end_id, 1, false, e2.start_id = e2.end_id, array [e2.id] from s4_seed join edge e2 on e2.start_id = s4_seed.root_id union all select s4.root_id, e2.end_id, s4.depth + 1, false, false, s4.path || e2.id from s4 join lateral (select e2.id, e2.start_id, e2.end_id, e2.kind_id, e2.properties from edge e2 where e2.start_id = s4.next_id and e2.id != all (s4.path) offset 0) e2 on true where s4.depth < 15 and not s4.is_cycle) select s2.e1 as e1, s2.ep0 as ep0, s2.n0 as n0, s2.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3 from s2, s4 join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s4.root_id offset 0) n2 on true join lateral (select n3.id, n3.kind_ids, n3.properties from node n3 where n3.id = s4.next_id offset 0) n3 on true where (s2.n2).id = s4.root_id) select s3.n3 as l from s3; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n1'))), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'n2')), false, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id union all select s1.root_id, e0.end_id, s1.depth + 1, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'n2')), false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied and exists (select 1 from edge e1 join node n2 on n2.id = e1.end_id where n1.id = e1.start_id and e1.kind_id = any (array [3, 4]::int2[]))), s2 as (select e1.id as e1, s0.ep0 as ep0, s0.n0 as n0, s0.n1 as n1, n2.id as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.kind_id = any (array [3, 4]::int2[]) and e1.id != all (s0.ep0)), s3 as (with recursive s4_seed(root_id) as not materialized (select distinct s2.n2 as root_id from s2), s4(root_id, next_id, depth, satisfied, is_cycle, path) as (select e2.start_id, e2.end_id, 1, false, false, array [e2.id] from s4_seed join edge e2 on e2.start_id = s4_seed.root_id union all select s4.root_id, e2.end_id, s4.depth + 1, false, false, s4.path || e2.id from s4 join lateral (select e2.id, e2.start_id, e2.end_id, e2.kind_id, e2.properties from edge e2 where e2.start_id = s4.next_id and e2.id != all (s4.path) offset 0) e2 on true where s4.depth < 15 and not s4.is_cycle) select s2.e1 as e1, s2.ep0 as ep0, s2.n0 as n0, s2.n1 as n1, n2.id as n2, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3 from s2, s4 join lateral (select n2.id from node n2 where n2.id = s4.root_id offset 0) n2 on true join lateral (select n3.id, n3.kind_ids, n3.properties from node n3 where n3.id = s4.next_id offset 0) n3 on true where s2.n2 = s4.root_id) select s3.n3 as l from s3; -- case: match p = (:NodeKind1)-[:EdgeKind1*1..]->(n:NodeKind2) where 'admin_tier_0' in split(n.system_tags, ' ') return p limit 1000 -with s0 as (with recursive s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ('admin_tier_0' = any (string_to_array((n1.properties ->> 'system_tags'), ' ')::text[])) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, n0.kind_ids operator (pg_catalog.@>) array [1]::int2[], e0.end_id = e0.start_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [3]::int2[]) union all select s1.root_id, e0.start_id, s1.depth + 1, n0.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, e0.id || s1.path from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n0 on n0.id = e0.start_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.root_id offset 0) n1 on true join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.next_id offset 0) n0 on true where s1.satisfied limit 1000) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 1000; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ('admin_tier_0' = any (string_to_array((n1.properties ->> 'system_tags'), ' ')::text[])) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, n0.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [3]::int2[]) union all select s1.root_id, e0.start_id, s1.depth + 1, n0.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, e0.id || s1.path from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n0 on n0.id = e0.start_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.root_id offset 0) n1 on true join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.next_id offset 0) n0 on true where s1.satisfied limit 1000) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 1000; -- case: match p = (s:NodeKind1)-[*..]->(e:NodeKind2) where s <> e return p -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id union all select s1.root_id, e0.end_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied and (n0.id <> n1.id)) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id union all select s1.root_id, e0.end_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied and (n0.id <> n1.id)) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; -- case: match p = (g:NodeKind1)-[:EdgeKind1|EdgeKind2*]->(target:NodeKind1) where g.objectid ends with '1234' and target.objectid ends with '4567' return p -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.properties ->> 'objectid') like '%1234') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, ((n1.properties ->> 'objectid') like '%4567') and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s1.root_id, e0.end_id, s1.depth + 1, ((n1.properties ->> 'objectid') like '%4567') and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.properties ->> 'objectid') like '%1234') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, ((n1.properties ->> 'objectid') like '%4567') and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s1.root_id, e0.end_id, s1.depth + 1, ((n1.properties ->> 'objectid') like '%4567') and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; -- case: match p = (m:NodeKind2)-[:EdgeKind1*1..]->(n:NodeKind1) where n.objectid = '1234' return p limit 10 -with s0 as (with recursive s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ((jsonb_typeof((n1.properties -> 'objectid')) = 'string' and (n1.properties ->> 'objectid') = '1234')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, n0.kind_ids operator (pg_catalog.@>) array [2]::int2[], e0.end_id = e0.start_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [3]::int2[]) union all select s1.root_id, e0.start_id, s1.depth + 1, n0.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, e0.id || s1.path from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n0 on n0.id = e0.start_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.root_id offset 0) n1 on true join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.next_id offset 0) n0 on true where s1.satisfied limit 10) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 10; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ((jsonb_typeof((n1.properties -> 'objectid')) = 'string' and (n1.properties ->> 'objectid') = '1234')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, n0.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [3]::int2[]) union all select s1.root_id, e0.start_id, s1.depth + 1, n0.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, e0.id || s1.path from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n0 on n0.id = e0.start_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.root_id offset 0) n1 on true join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.next_id offset 0) n0 on true where s1.satisfied limit 10) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 10; -- case: match p = (:NodeKind1)<-[:EdgeKind1|EdgeKind2*..]-() return p limit 10 -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, false, e0.end_id = e0.start_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s1.root_id, e0.start_id, s1.depth + 1, false, false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true limit 10) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 10; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, false, false, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s1.root_id, e0.start_id, s1.depth + 1, false, false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true limit 10) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 10; -- case: match p = (:NodeKind1)<-[:EdgeKind1|EdgeKind2*..]-(:NodeKind2)<-[:EdgeKind1|EdgeKind2*2..]-(:NodeKind1) return p limit 10 -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], e0.end_id = e0.start_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n1 on n1.id = e0.start_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s1.root_id, e0.start_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.start_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied), s2 as (with recursive s3_seed(root_id) as not materialized (select distinct (s0.n1).id as root_id from s0), s3(root_id, next_id, depth, satisfied, is_cycle, path) as (select e1.end_id, e1.start_id, 1, n2.kind_ids operator (pg_catalog.@>) array [1]::int2[], e1.end_id = e1.start_id, array [e1.id] from s3_seed join edge e1 on e1.end_id = s3_seed.root_id join node n2 on n2.id = e1.start_id where e1.kind_id = any (array [3, 4]::int2[]) union all select s3.root_id, e1.start_id, s3.depth + 1, n2.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s3.path || e1.id from s3 join lateral (select e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties from edge e1 where e1.end_id = s3.next_id and e1.id != all (s3.path) and e1.kind_id = any (array [3, 4]::int2[]) offset 0) e1 on true join node n2 on n2.id = e1.start_id where s3.depth < 15 and not s3.is_cycle) select s0.ep0 as ep0, s3.path as ep1, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, s3 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s3.root_id offset 0) n1 on true join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s3.next_id offset 0) n2 on true where s3.depth >= 2 and s3.satisfied and (s0.n1).id = s3.root_id limit 10) select case when (s2.n0).id is null or s2.ep0 is null or (s2.n1).id is null or s2.ep1 is null or (s2.n2).id is null then null else ordered_edges_to_path(s2.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s2.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id) || (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s2.ep1) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s2.n0, s2.n1, s2.n2]::nodecomposite[])::pathcomposite end as p from s2 limit 10; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n1 on n1.id = e0.start_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s1.root_id, e0.start_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.start_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied), s2 as (with recursive s3_seed(root_id) as not materialized (select distinct (s0.n1).id as root_id from s0), s3(root_id, next_id, depth, satisfied, is_cycle, path) as (select e1.end_id, e1.start_id, 1, n2.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e1.id] from s3_seed join edge e1 on e1.end_id = s3_seed.root_id join node n2 on n2.id = e1.start_id where e1.kind_id = any (array [3, 4]::int2[]) union all select s3.root_id, e1.start_id, s3.depth + 1, n2.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s3.path || e1.id from s3 join lateral (select e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties from edge e1 where e1.end_id = s3.next_id and e1.id != all (s3.path) and e1.kind_id = any (array [3, 4]::int2[]) offset 0) e1 on true join node n2 on n2.id = e1.start_id where s3.depth < 15 and not s3.is_cycle) select s0.ep0 as ep0, s3.path as ep1, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, s3 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s3.root_id offset 0) n1 on true join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s3.next_id offset 0) n2 on true where s3.depth >= 2 and s3.satisfied and (s0.n1).id = s3.root_id limit 10) select case when (s2.n0).id is null or s2.ep0 is null or (s2.n1).id is null or s2.ep1 is null or (s2.n2).id is null then null else ordered_edge_ids_to_path(0, s2.n0, s2.ep0 || s2.ep1, array [s2.n0, s2.n1, s2.n2]::nodecomposite[])::pathcomposite end as p from s2 limit 10; -- case: match p = (:NodeKind1)<-[:EdgeKind1|EdgeKind2*..]-(:NodeKind2)<-[:EdgeKind1|EdgeKind2*..]-(:NodeKind1) return p limit 10 -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], e0.end_id = e0.start_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n1 on n1.id = e0.start_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s1.root_id, e0.start_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.start_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied), s2 as (with recursive s3_seed(root_id) as not materialized (select distinct (s0.n1).id as root_id from s0), s3(root_id, next_id, depth, satisfied, is_cycle, path) as (select e1.end_id, e1.start_id, 1, n2.kind_ids operator (pg_catalog.@>) array [1]::int2[], e1.end_id = e1.start_id, array [e1.id] from s3_seed join edge e1 on e1.end_id = s3_seed.root_id join node n2 on n2.id = e1.start_id where e1.kind_id = any (array [3, 4]::int2[]) union all select s3.root_id, e1.start_id, s3.depth + 1, n2.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s3.path || e1.id from s3 join lateral (select e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties from edge e1 where e1.end_id = s3.next_id and e1.id != all (s3.path) and e1.kind_id = any (array [3, 4]::int2[]) offset 0) e1 on true join node n2 on n2.id = e1.start_id where s3.depth < 15 and not s3.is_cycle) select s0.ep0 as ep0, s3.path as ep1, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, s3 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s3.root_id offset 0) n1 on true join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s3.next_id offset 0) n2 on true where s3.satisfied and (s0.n1).id = s3.root_id limit 10) select case when (s2.n0).id is null or s2.ep0 is null or (s2.n1).id is null or s2.ep1 is null or (s2.n2).id is null then null else ordered_edges_to_path(s2.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s2.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id) || (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s2.ep1) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s2.n0, s2.n1, s2.n2]::nodecomposite[])::pathcomposite end as p from s2 limit 10; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n1 on n1.id = e0.start_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s1.root_id, e0.start_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.start_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied), s2 as (with recursive s3_seed(root_id) as not materialized (select distinct (s0.n1).id as root_id from s0), s3(root_id, next_id, depth, satisfied, is_cycle, path) as (select e1.end_id, e1.start_id, 1, n2.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e1.id] from s3_seed join edge e1 on e1.end_id = s3_seed.root_id join node n2 on n2.id = e1.start_id where e1.kind_id = any (array [3, 4]::int2[]) union all select s3.root_id, e1.start_id, s3.depth + 1, n2.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s3.path || e1.id from s3 join lateral (select e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties from edge e1 where e1.end_id = s3.next_id and e1.id != all (s3.path) and e1.kind_id = any (array [3, 4]::int2[]) offset 0) e1 on true join node n2 on n2.id = e1.start_id where s3.depth < 15 and not s3.is_cycle) select s0.ep0 as ep0, s3.path as ep1, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, s3 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s3.root_id offset 0) n1 on true join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s3.next_id offset 0) n2 on true where s3.satisfied and (s0.n1).id = s3.root_id limit 10) select case when (s2.n0).id is null or s2.ep0 is null or (s2.n1).id is null or s2.ep1 is null or (s2.n2).id is null then null else ordered_edge_ids_to_path(0, s2.n0, s2.ep0 || s2.ep1, array [s2.n0, s2.n1, s2.n2]::nodecomposite[])::pathcomposite end as p from s2 limit 10; -- case: match p = (n:NodeKind1)-[:EdgeKind1|EdgeKind2*1..2]->(r:NodeKind2) where r.name =~ '(?i)Global Administrator.*|User Administrator.*|Cloud Application Administrator.*|Authentication Policy Administrator.*|Exchange Administrator.*|Helpdesk Administrator.*|Privileged Authentication Administrator.*' return p limit 10 -with s0 as (with recursive s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ((n1.properties ->> 'name') ~ '(?i)Global Administrator.*|User Administrator.*|Cloud Application Administrator.*|Authentication Policy Administrator.*|Exchange Administrator.*|Helpdesk Administrator.*|Privileged Authentication Administrator.*') and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, n0.kind_ids operator (pg_catalog.@>) array [1]::int2[], e0.end_id = e0.start_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s1.root_id, e0.start_id, s1.depth + 1, n0.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, e0.id || s1.path from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true join node n0 on n0.id = e0.start_id where s1.depth < 2 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.root_id offset 0) n1 on true join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.next_id offset 0) n0 on true where s1.satisfied limit 10) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 10; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ((n1.properties ->> 'name') ~ '(?i)Global Administrator.*|User Administrator.*|Cloud Application Administrator.*|Authentication Policy Administrator.*|Exchange Administrator.*|Helpdesk Administrator.*|Privileged Authentication Administrator.*') and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, n0.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s1.root_id, e0.start_id, s1.depth + 1, n0.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, e0.id || s1.path from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true join node n0 on n0.id = e0.start_id where s1.depth < 2 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.root_id offset 0) n1 on true join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.next_id offset 0) n0 on true where s1.satisfied limit 10) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 10; -- case: match p = (t:NodeKind2)<-[:EdgeKind1*1..]-(a) where (a:NodeKind1 or a:NodeKind2) and t.objectid ends with '-512' return p limit 1000 -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.properties ->> 'objectid') like '%-512') and n0.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, ((n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] or n1.kind_ids operator (pg_catalog.@>) array [2]::int2[])), e0.end_id = e0.start_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n1 on n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[]) union all select s1.root_id, e0.start_id, s1.depth + 1, ((n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] or n1.kind_ids operator (pg_catalog.@>) array [2]::int2[])), false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.start_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied limit 1000) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 1000; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.properties ->> 'objectid') like '%-512') and n0.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, ((n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] or n1.kind_ids operator (pg_catalog.@>) array [2]::int2[])), false, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n1 on n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[]) union all select s1.root_id, e0.start_id, s1.depth + 1, ((n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] or n1.kind_ids operator (pg_catalog.@>) array [2]::int2[])), false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.start_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied limit 1000) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 1000; -- case: match p=(n:NodeKind1)-[:EdgeKind1|EdgeKind2]->(g:NodeKind1)-[:EdgeKind2]->(:NodeKind2)-[:EdgeKind1*1..]->(m:NodeKind1) where n.objectid = m.objectid return p limit 100 -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[])), s1 as (select s0.e0 as e0, e1.id as e1, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) and e1.id != s0.e0), s2 as (with recursive s3_seed(root_id) as not materialized (select distinct (s1.n2).id as root_id from s1), s3(root_id, next_id, depth, satisfied, is_cycle, path) as (select e2.start_id, e2.end_id, 1, n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], e2.start_id = e2.end_id, array [e2.id] from s3_seed join edge e2 on e2.start_id = s3_seed.root_id join node n3 on n3.id = e2.end_id where e2.kind_id = any (array [3]::int2[]) union all select s3.root_id, e2.end_id, s3.depth + 1, n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s3.path || e2.id from s3 join lateral (select e2.id, e2.start_id, e2.end_id, e2.kind_id, e2.properties from edge e2 where e2.start_id = s3.next_id and e2.id != all (s3.path) and e2.kind_id = any (array [3]::int2[]) offset 0) e2 on true join node n3 on n3.id = e2.end_id where s3.depth < 15 and not s3.is_cycle) select s1.e0 as e0, s1.e1 as e1, s3.path as ep0, s1.n0 as n0, s1.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3 from s1, s3 join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s3.root_id offset 0) n2 on true join lateral (select n3.id, n3.kind_ids, n3.properties from node n3 where n3.id = s3.next_id offset 0) n3 on true where s3.satisfied and (s1.n2).id = s3.root_id and (((s1.n0).properties -> 'objectid') = (n3.properties -> 'objectid')) limit 100) select case when (s2.n0).id is null or s2.e0 is null or (s2.n1).id is null or s2.e1 is null or (s2.n2).id is null or s2.ep0 is null or (s2.n3).id is null then null else ordered_edges_to_path(s2.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s2.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id) || (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s2.e1]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id) || (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s2.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s2.n0, s2.n1, s2.n2, s2.n3]::nodecomposite[])::pathcomposite end as p from s2 limit 100; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[])), s1 as (select s0.e0 as e0, e1.id as e1, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) and e1.id != s0.e0), s2 as (with recursive s3_seed(root_id) as not materialized (select distinct (s1.n2).id as root_id from s1), s3(root_id, next_id, depth, satisfied, is_cycle, path) as (select e2.start_id, e2.end_id, 1, n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e2.id] from s3_seed join edge e2 on e2.start_id = s3_seed.root_id join node n3 on n3.id = e2.end_id where e2.kind_id = any (array [3]::int2[]) union all select s3.root_id, e2.end_id, s3.depth + 1, n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s3.path || e2.id from s3 join lateral (select e2.id, e2.start_id, e2.end_id, e2.kind_id, e2.properties from edge e2 where e2.start_id = s3.next_id and e2.id != all (s3.path) and e2.kind_id = any (array [3]::int2[]) offset 0) e2 on true join node n3 on n3.id = e2.end_id where s3.depth < 15 and not s3.is_cycle) select s1.e0 as e0, s1.e1 as e1, s3.path as ep0, s1.n0 as n0, s1.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3 from s1, s3 join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s3.root_id offset 0) n2 on true join lateral (select n3.id, n3.kind_ids, n3.properties from node n3 where n3.id = s3.next_id offset 0) n3 on true where s3.satisfied and (s1.n2).id = s3.root_id and (nullif(((s1.n0).properties -> 'objectid'), ('null')::jsonb)::jsonb = nullif((n3.properties -> 'objectid'), ('null')::jsonb)::jsonb) limit 100) select case when (s2.n0).id is null or s2.e0 is null or (s2.n1).id is null or s2.e1 is null or (s2.n2).id is null or s2.ep0 is null or (s2.n3).id is null then null else ordered_edge_ids_to_path(0, s2.n0, array [s2.e0]::int8[] || array [s2.e1]::int8[] || s2.ep0, array [s2.n0, s2.n1, s2.n2, s2.n3]::nodecomposite[])::pathcomposite end as p from s2 limit 100; -- case: match (a:NodeKind1)-[:EdgeKind1*0..]->(b:NodeKind1) where a.name = 'solo' and b.name = 'solo' return a.name, b.name -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'solo')) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select s1_seed.root_id, s1_seed.root_id, 0, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'solo')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array []::int8[] from s1_seed join node n1 on n1.id = s1_seed.root_id union all select e0.start_id, e0.end_id, 1, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'solo')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) union all select s1.root_id, e0.end_id, s1.depth + 1, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'solo')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle and s1.depth > 0) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied) select ((s0.n0).properties -> 'name'), ((s0.n1).properties -> 'name') from s0; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'solo')) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select s1_seed.root_id, s1_seed.root_id, 0, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'solo')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array []::int8[] from s1_seed join node n1 on n1.id = s1_seed.root_id union all select e0.start_id, e0.end_id, 1, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'solo')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) union all select s1.root_id, e0.end_id, s1.depth + 1, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'solo')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle and s1.depth > 0) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied) select ((s0.n0).properties -> 'name') as "a.name", ((s0.n1).properties -> 'name') as "b.name" from s0; -- case: match (a:NodeKind1)-[:EdgeKind1*0..]->(b:NodeKind1) where a.name = 'zero-source' and b.name = 'zero-target' return count(b) -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'zero-source')) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select s1_seed.root_id, s1_seed.root_id, 0, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'zero-target')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array []::int8[] from s1_seed join node n1 on n1.id = s1_seed.root_id union all select e0.start_id, e0.end_id, 1, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'zero-target')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) union all select s1.root_id, e0.end_id, s1.depth + 1, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'zero-target')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle and s1.depth > 0) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied) select count(s0.n1)::int8 from s0; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'zero-source')) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select s1_seed.root_id, s1_seed.root_id, 0, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'zero-target')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array []::int8[] from s1_seed join node n1 on n1.id = s1_seed.root_id union all select e0.start_id, e0.end_id, 1, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'zero-target')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) union all select s1.root_id, e0.end_id, s1.depth + 1, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'zero-target')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle and s1.depth > 0) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied) select count(s0.n1)::int8 as "count(b)" from s0; +-- case: match (s)-[*1..]->(mid)-[]->(e) return id(mid), id(e) +with s0 as (with recursive s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, true, false, array [e0.id] from edge e0 join node n1 on n1.id = e0.end_id union all select s1.root_id, e0.end_id, s1.depth + 1, true, false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied and exists (select 1 from edge e1 join node n2 on n2.id = e1.end_id where n1.id = e1.start_id)), s2 as (select s0.ep0 as ep0, s0.n0 as n0, s0.n1 as n1, n2.id as n2 from s0 join edge e1 on s0.n1 = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != all (s0.ep0)) select s2.n1 as "id(mid)", s2.n2 as "id(e)" from s2; diff --git a/cypher/models/pgsql/test/translation_cases/post_processing.sql b/cypher/models/pgsql/test/translation_cases/post_processing.sql new file mode 100644 index 00000000..41dcae59 --- /dev/null +++ b/cypher/models/pgsql/test/translation_cases/post_processing.sql @@ -0,0 +1,45 @@ +-- Copyright 2026 Specter Ops, Inc. +-- +-- Licensed under the Apache License, Version 2.0 +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- SPDX-License-Identifier: Apache-2.0 + +-- case: match (n) where not n:RegressionKind03 and (n.lastseen is null or n.lastseen < datetime($threshold)) return id(n) +-- cypher_params: {"threshold":"2026-01-02T03:04:05Z"} +-- pgsql_params:{"pi0":"2026-01-02T03:04:05Z"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (not n0.kind_ids operator (pg_catalog.@>) array [35]::int2[] and ((not n0.properties ? 'lastseen' or (n0.properties -> 'lastseen') = ('null')::jsonb) or ((n0.properties ->> 'lastseen'))::timestamp with time zone < (@pi0::text)::timestamp with time zone))) select (s0.n0).id as "id(n)" from s0; + +-- case: match ()-[r]->() where not r:RegressionKind45 and r.lastseen < datetime($threshold) return id(r) +-- cypher_params: {"threshold":"2026-01-03T00:00:00Z"} +-- pgsql_params:{"pi0":"2026-01-03T00:00:00Z"} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where (not e0.kind_id = any (array [77]::int2[]) and ((e0.properties ->> 'lastseen'))::timestamp with time zone < (@pi0::text)::timestamp with time zone)) select (s0.e0).id as "id(r)" from s0; + +-- case: match ()-[r]->() where not (r:RegressionKind45 or r:RegressionKind46) and r.lastseen < datetime($threshold) return id(r) +-- cypher_params: {"threshold":"2026-01-03T00:00:00Z"} +-- pgsql_params:{"pi0":"2026-01-03T00:00:00Z"} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where (not (e0.kind_id = any (array [77]::int2[]) or e0.kind_id = any (array [78]::int2[])) and ((e0.properties ->> 'lastseen'))::timestamp with time zone < (@pi0::text)::timestamp with time zone)) select (s0.e0).id as "id(r)" from s0; + +-- case: match ()-[r:HasSession]->() where r.lastseen is null or r.lastseen < datetime($threshold) return id(r) +-- cypher_params: {"threshold":"2026-01-03T00:00:00Z"} +-- pgsql_params:{"pi0":"2026-01-03T00:00:00Z"} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where ((not e0.properties ? 'lastseen' or (e0.properties -> 'lastseen') = ('null')::jsonb) or ((e0.properties ->> 'lastseen'))::timestamp with time zone < (@pi0::text)::timestamp with time zone) and e0.kind_id = any (array [7]::int2[])) select (s0.e0).id as "id(r)" from s0; + +-- case: match (n) where not (n:RegressionKind48 or n:RegressionKind49) and (n.lastseen is null or n.lastseen < datetime($threshold)) return id(n) +-- cypher_params: {"threshold":"2026-01-03T00:00:00Z"} +-- pgsql_params:{"pi0":"2026-01-03T00:00:00Z"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (not (n0.kind_ids operator (pg_catalog.@>) array [80]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [81]::int2[]) and ((not n0.properties ? 'lastseen' or (n0.properties -> 'lastseen') = ('null')::jsonb) or ((n0.properties ->> 'lastseen'))::timestamp with time zone < (@pi0::text)::timestamp with time zone))) select (s0.n0).id as "id(n)" from s0; + +-- case: match (n) where not (n:RegressionKind48 or n:RegressionKind49) and n.name is null and n.objectid starts with $sid_prefix return id(n) +-- cypher_params: {"sid_prefix":"S-1-5"} +-- pgsql_params:{"pi0":"S-1-5"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (not (n0.kind_ids operator (pg_catalog.@>) array [80]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [81]::int2[]) and (not n0.properties ? 'name' or (n0.properties -> 'name') = ('null')::jsonb) and cypher_starts_with((n0.properties ->> 'objectid'), (@pi0::text)::text)::bool)) select (s0.n0).id as "id(n)" from s0; diff --git a/cypher/models/pgsql/test/translation_cases/quantifiers.sql b/cypher/models/pgsql/test/translation_cases/quantifiers.sql index c4d2f249..5b342396 100644 --- a/cypher/models/pgsql/test/translation_cases/quantifiers.sql +++ b/cypher/models/pgsql/test/translation_cases/quantifiers.sql @@ -30,20 +30,20 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((n0.properties ->> 'usedeskeyonly'))::bool or ((select count(*)::int from unnest(jsonb_to_text_array((n0.properties -> 'supportedencryptiontypes'))) as i0 where (i0 like '%DES%')) >= 1)::bool or ((select count(*)::int from unnest(jsonb_to_text_array((n0.properties -> 'serviceprincipalnames'))) as i1 where (lower(i1)::text like '%mssqlservercluster%' or lower(i1)::text like '%mssqlserverclustermgmtapi%' or lower(i1)::text like '%msclustervirtualserver%')) >= 1)::bool) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s0.n0 as n from s0 limit 100; -- case: MATCH (m:NodeKind1) WHERE m.unconstraineddelegation = true WITH m MATCH (n:NodeKind1)-[:EdgeKind1]->(g:NodeKind2) WHERE g.objectid ENDS WITH '-516' WITH m, COLLECT(n) AS matchingNs WHERE NONE(n IN matchingNs WHERE n.objectid = m.objectid) RETURN m -with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((n0.properties -> 'unconstraineddelegation'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s1.n0 as n0 from s1), s2 as (with s3 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, edge e0 join node n2 on ((n2.properties ->> 'objectid') like '%-516') and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e0.end_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[])) select s3.n0 as n0, array_remove(coalesce(array_agg(s3.n1)::nodecomposite[], array []::nodecomposite[])::nodecomposite[], null)::nodecomposite[] as i0 from s3 group by n0) select s2.n0 as m from s2 where (((select count(*)::int from unnest(s2.i0) as i1 where ((i1.properties -> 'objectid') = ((s2.n0).properties -> 'objectid'))) = 0 and s2.i0 is not null)::bool); +with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((n0.properties -> 'unconstraineddelegation'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s1.n0 as n0 from s1), s2 as (with s3 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, edge e0 join node n2 on ((n2.properties ->> 'objectid') like '%-516') and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e0.end_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[])) select s3.n0 as n0, array_remove(coalesce(array_agg(s3.n1)::nodecomposite[], array []::nodecomposite[])::nodecomposite[], null)::nodecomposite[] as i0 from s3 group by n0) select s2.n0 as m from s2 where (((select count(*)::int from unnest(s2.i0) as i1 where (nullif((i1.properties -> 'objectid'), ('null')::jsonb)::jsonb = nullif(((s2.n0).properties -> 'objectid'), ('null')::jsonb)::jsonb)) = 0 and s2.i0 is not null)::bool); -- case: MATCH (m:NodeKind1) WHERE m.unconstraineddelegation = true WITH m MATCH (n:NodeKind1)-[:EdgeKind1]->(g:NodeKind2) WHERE g.objectid ENDS WITH '-516' WITH m, COLLECT(n) AS matchingNs WHERE ALL(n IN matchingNs WHERE n.objectid = m.objectid) RETURN m -with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((n0.properties -> 'unconstraineddelegation'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s1.n0 as n0 from s1), s2 as (with s3 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, edge e0 join node n2 on ((n2.properties ->> 'objectid') like '%-516') and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e0.end_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[])) select s3.n0 as n0, array_remove(coalesce(array_agg(s3.n1)::nodecomposite[], array []::nodecomposite[])::nodecomposite[], null)::nodecomposite[] as i0 from s3 group by n0) select s2.n0 as m from s2 where (((select count(*)::int from unnest(s2.i0) as i1 where ((i1.properties -> 'objectid') = ((s2.n0).properties -> 'objectid'))) = cardinality(s2.i0))::bool); +with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((n0.properties -> 'unconstraineddelegation'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s1.n0 as n0 from s1), s2 as (with s3 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, edge e0 join node n2 on ((n2.properties ->> 'objectid') like '%-516') and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e0.end_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[])) select s3.n0 as n0, array_remove(coalesce(array_agg(s3.n1)::nodecomposite[], array []::nodecomposite[])::nodecomposite[], null)::nodecomposite[] as i0 from s3 group by n0) select s2.n0 as m from s2 where (((select count(*)::int from unnest(s2.i0) as i1 where (nullif((i1.properties -> 'objectid'), ('null')::jsonb)::jsonb = nullif(((s2.n0).properties -> 'objectid'), ('null')::jsonb)::jsonb)) = cardinality(s2.i0))::bool); -- case: MATCH (m:NodeKind1) WHERE ANY(name in m.serviceprincipalnames WHERE name CONTAINS "PHANTOM") WITH m MATCH (n:NodeKind1)-[:EdgeKind1]->(g:NodeKind2) WHERE g.objectid ENDS WITH '-525' WITH m, COLLECT(n) AS matchingNs WHERE NONE(t IN matchingNs WHERE t.objectid = m.objectid) RETURN m -with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((select count(*)::int from unnest(jsonb_to_text_array((n0.properties -> 'serviceprincipalnames'))) as i0 where (i0 like '%PHANTOM%')) >= 1)::bool) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s1.n0 as n0 from s1), s2 as (with s3 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, edge e0 join node n2 on ((n2.properties ->> 'objectid') like '%-525') and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e0.end_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[])) select s3.n0 as n0, array_remove(coalesce(array_agg(s3.n1)::nodecomposite[], array []::nodecomposite[])::nodecomposite[], null)::nodecomposite[] as i1 from s3 group by n0) select s2.n0 as m from s2 where (((select count(*)::int from unnest(s2.i1) as i2 where ((i2.properties -> 'objectid') = ((s2.n0).properties -> 'objectid'))) = 0 and s2.i1 is not null)::bool); +with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((select count(*)::int from unnest(jsonb_to_text_array((n0.properties -> 'serviceprincipalnames'))) as i0 where (i0 like '%PHANTOM%')) >= 1)::bool) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s1.n0 as n0 from s1), s2 as (with s3 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, edge e0 join node n2 on ((n2.properties ->> 'objectid') like '%-525') and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e0.end_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[])) select s3.n0 as n0, array_remove(coalesce(array_agg(s3.n1)::nodecomposite[], array []::nodecomposite[])::nodecomposite[], null)::nodecomposite[] as i1 from s3 group by n0) select s2.n0 as m from s2 where (((select count(*)::int from unnest(s2.i1) as i2 where (nullif((i2.properties -> 'objectid'), ('null')::jsonb)::jsonb = nullif(((s2.n0).properties -> 'objectid'), ('null')::jsonb)::jsonb)) = 0 and s2.i1 is not null)::bool); -- case: WITH [1, 2] AS nums MATCH (n:NodeKind1) WHERE ANY(num IN nums + [3] WHERE num = 3) RETURN n with s0 as (select array [1, 2]::int8[] as i0), s1 as (select s0.i0 as i0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from s0, node n0 where (((select count(*)::int from unnest(s0.i0 || array [3]::int8[]) as i1 where (i1 = 3)) >= 1)::bool) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s1.n0 as n from s1; -- case: MATCH (m:NodeKind1) WHERE m.unconstraineddelegation = true WITH m MATCH (n:NodeKind1)-[:EdgeKind1]->(g:NodeKind2) WHERE g.objectid ENDS WITH '-516' WITH m, COLLECT(n) AS matchingNs WHERE ALL(n IN matchingNs WHERE n.objectid = m.objectid) RETURN m -with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((n0.properties -> 'unconstraineddelegation'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s1.n0 as n0 from s1), s2 as (with s3 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, edge e0 join node n2 on ((n2.properties ->> 'objectid') like '%-516') and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e0.end_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[])) select s3.n0 as n0, array_remove(coalesce(array_agg(s3.n1)::nodecomposite[], array []::nodecomposite[])::nodecomposite[], null)::nodecomposite[] as i0 from s3 group by n0) select s2.n0 as m from s2 where (((select count(*)::int from unnest(s2.i0) as i1 where ((i1.properties -> 'objectid') = ((s2.n0).properties -> 'objectid'))) = cardinality(s2.i0))::bool); +with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((n0.properties -> 'unconstraineddelegation'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s1.n0 as n0 from s1), s2 as (with s3 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, edge e0 join node n2 on ((n2.properties ->> 'objectid') like '%-516') and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e0.end_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[])) select s3.n0 as n0, array_remove(coalesce(array_agg(s3.n1)::nodecomposite[], array []::nodecomposite[])::nodecomposite[], null)::nodecomposite[] as i0 from s3 group by n0) select s2.n0 as m from s2 where (((select count(*)::int from unnest(s2.i0) as i1 where (nullif((i1.properties -> 'objectid'), ('null')::jsonb)::jsonb = nullif(((s2.n0).properties -> 'objectid'), ('null')::jsonb)::jsonb)) = cardinality(s2.i0))::bool); -- case: MATCH (m:NodeKind1) WHERE ANY(name in m.serviceprincipalnames WHERE name CONTAINS "PHANTOM") WITH m MATCH (n:NodeKind1)-[:EdgeKind1]->(g:NodeKind2) WHERE g.objectid ENDS WITH '-525' WITH m, COLLECT(n) AS matchingNs WHERE NONE(t IN matchingNs WHERE t.objectid = m.objectid) RETURN m -with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((select count(*)::int from unnest(jsonb_to_text_array((n0.properties -> 'serviceprincipalnames'))) as i0 where (i0 like '%PHANTOM%')) >= 1)::bool) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s1.n0 as n0 from s1), s2 as (with s3 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, edge e0 join node n2 on ((n2.properties ->> 'objectid') like '%-525') and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e0.end_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[])) select s3.n0 as n0, array_remove(coalesce(array_agg(s3.n1)::nodecomposite[], array []::nodecomposite[])::nodecomposite[], null)::nodecomposite[] as i1 from s3 group by n0) select s2.n0 as m from s2 where (((select count(*)::int from unnest(s2.i1) as i2 where ((i2.properties -> 'objectid') = ((s2.n0).properties -> 'objectid'))) = 0 and s2.i1 is not null)::bool); +with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((select count(*)::int from unnest(jsonb_to_text_array((n0.properties -> 'serviceprincipalnames'))) as i0 where (i0 like '%PHANTOM%')) >= 1)::bool) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s1.n0 as n0 from s1), s2 as (with s3 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, edge e0 join node n2 on ((n2.properties ->> 'objectid') like '%-525') and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e0.end_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[])) select s3.n0 as n0, array_remove(coalesce(array_agg(s3.n1)::nodecomposite[], array []::nodecomposite[])::nodecomposite[], null)::nodecomposite[] as i1 from s3 group by n0) select s2.n0 as m from s2 where (((select count(*)::int from unnest(s2.i1) as i2 where (nullif((i2.properties -> 'objectid'), ('null')::jsonb)::jsonb = nullif(((s2.n0).properties -> 'objectid'), ('null')::jsonb)::jsonb)) = 0 and s2.i1 is not null)::bool); diff --git a/cypher/models/pgsql/test/translation_cases/reconciliation.sql b/cypher/models/pgsql/test/translation_cases/reconciliation.sql new file mode 100644 index 00000000..3a42470a --- /dev/null +++ b/cypher/models/pgsql/test/translation_cases/reconciliation.sql @@ -0,0 +1,139 @@ +-- Copyright 2026 Specter Ops, Inc. +-- +-- Licensed under the Apache License, Version 2.0 +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- SPDX-License-Identifier: Apache-2.0 + +-- case: match (s)-[r]->(e) where (id(s) = $forward_start and id(e) = $forward_end and r:RegressionKind01) or (id(s) = $forward_end and id(e) = $forward_start and r:RegressionKind02) return id(r) +-- cypher_params: {"forward_end":202,"forward_start":101} +-- pgsql_params:{"pi0":101,"pi1":202} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, n1.id as n1 from edge e0 join node n1 on n1.id = e0.end_id join node n0 on n0.id = e0.start_id where ((n0.id = @pi0::float8 and n1.id = @pi1::float8 and e0.kind_id = any (array [33]::int2[])) or (n0.id = @pi1::float8 and n1.id = @pi0::float8 and e0.kind_id = any (array [34]::int2[])))) select (s0.e0).id as "id(r)" from s0; + +-- case: match (s:RegressionKind03)-[r:RegressionKind04]->(e:RegressionKind03) where r.lastseen < s.lastcollected or r.lastseen < e.lastcollected return id(r) +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [35]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [35]::int2[] and n1.id = e0.end_id where (nullif((e0.properties -> 'lastseen'), ('null')::jsonb)::jsonb < nullif((n0.properties -> 'lastcollected'), ('null')::jsonb)::jsonb or nullif((e0.properties -> 'lastseen'), ('null')::jsonb)::jsonb < nullif((n1.properties -> 'lastcollected'), ('null')::jsonb)::jsonb) and e0.kind_id = any (array [36]::int2[])) select (s0.e0).id as "id(r)" from s0; + +-- case: match (s:RegressionKind05)-[r:RegressionKind06]->(e:RegressionKind07) where e.objectid = $object_id and r.shoulddelete = $should_delete delete r +-- cypher_params: {"object_id":"delete-edge","should_delete":true} +-- pgsql_params:{"pi0":"delete-edge","pi1":true} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on ((jsonb_typeof((n1.properties -> 'objectid')) = 'string' and (n1.properties ->> 'objectid') = @pi0::text)) and n1.kind_ids operator (pg_catalog.@>) array [39]::int2[] and n1.id = e0.end_id join node n0 on n0.kind_ids operator (pg_catalog.@>) array [37]::int2[] and n0.id = e0.start_id where (((e0.properties -> 'shoulddelete'))::jsonb = to_jsonb((@pi1::bool)::bool)::jsonb) and e0.kind_id = any (array [38]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match (n:RegressionKind08) where n.objectid = $object_id detach delete n +-- cypher_params: {"object_id":"delete-node"} +-- pgsql_params:{"pi0":"delete-node"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'objectid')) = 'string' and (n0.properties ->> 'objectid') = @pi0::text)) and n0.kind_ids operator (pg_catalog.@>) array [40]::int2[]), s1 as (delete from node n1 using s0 where (s0.n0).id = n1.id) select 1; + +-- case: match ()-[r:RegressionKind09]->(e) return r, e +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [41]::int2[])) select s0.e0 as r, s0.n1 as e from s0; + +-- case: match ()-[r:RegressionKind09]->(e) return id(e), labels(e), id(r), type(r) +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [41]::int2[])) select (s0.n1).id as "id(e)", (array(select _kind.name from generate_subscripts((s0.n1).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s0.n1).kind_ids)[_kind_idx] order by _kind_idx))::text[] as "labels(e)", (s0.e0).id as "id(r)", kind_name((s0.e0).kind_id)::text as "type(r)" from s0; + +-- case: match (s)-[r:RegressionKind09]->(e) return s, r, e +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [41]::int2[])) select s0.n0 as s, s0.e0 as r, s0.n1 as e from s0; + +-- case: match ()-[r:RegressionKind09]->() return id(r) +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [41]::int2[])) select (s0.e0).id as "id(r)" from s0; + +-- case: match ()-[r:RegressionKind09]->() return r +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [41]::int2[])) select s0.e0 as r from s0; + +-- case: match ()-[r:RegressionKind01]->(e:RegressionKind31) where e.objectid = $object_id delete r +-- cypher_params: {"object_id":"rec-01"} +-- pgsql_params:{"pi0":"rec-01"} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on ((jsonb_typeof((n1.properties -> 'objectid')) = 'string' and (n1.properties ->> 'objectid') = @pi0::text)) and n1.kind_ids operator (pg_catalog.@>) array [63]::int2[] and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [33]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match ()-[r:RegressionKind01|RegressionKind02]->(e:RegressionKind31) where e.objectid = $object_id delete r +-- cypher_params: {"object_id":"rec-01"} +-- pgsql_params:{"pi0":"rec-01"} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on ((jsonb_typeof((n1.properties -> 'objectid')) = 'string' and (n1.properties ->> 'objectid') = @pi0::text)) and n1.kind_ids operator (pg_catalog.@>) array [63]::int2[] and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [33, 34]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match ()-[r:RegressionKind01|RegressionKind02|RegressionKind03|RegressionKind04|RegressionKind05|RegressionKind06|RegressionKind07|RegressionKind08|RegressionKind09]->(e:RegressionKind31) where e.objectid = $object_id delete r +-- cypher_params: {"object_id":"rec-01"} +-- pgsql_params:{"pi0":"rec-01"} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on ((jsonb_typeof((n1.properties -> 'objectid')) = 'string' and (n1.properties ->> 'objectid') = @pi0::text)) and n1.kind_ids operator (pg_catalog.@>) array [63]::int2[] and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [33, 34, 35, 36, 37, 38, 39, 40, 41]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match ()-[r:RegressionKind01|RegressionKind02|RegressionKind03|RegressionKind04|RegressionKind05|RegressionKind06|RegressionKind07|RegressionKind08|RegressionKind09|RegressionKind10|RegressionKind11|RegressionKind12|RegressionKind13|RegressionKind14|RegressionKind15|RegressionKind16|RegressionKind17|RegressionKind18|RegressionKind19|RegressionKind20|RegressionKind21|RegressionKind22|RegressionKind23|RegressionKind24|RegressionKind25|RegressionKind26|RegressionKind27|RegressionKind28|RegressionKind29|RegressionKind30]->(e:RegressionKind31) where e.objectid = $object_id delete r +-- cypher_params: {"object_id":"rec-01"} +-- pgsql_params:{"pi0":"rec-01"} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on ((jsonb_typeof((n1.properties -> 'objectid')) = 'string' and (n1.properties ->> 'objectid') = @pi0::text)) and n1.kind_ids operator (pg_catalog.@>) array [63]::int2[] and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match (s:RegressionKind31)-[r:RegressionKind01]->() where s.objectid = $object_id delete r +-- cypher_params: {"object_id":"rec-02"} +-- pgsql_params:{"pi0":"rec-02"} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from edge e0 join node n0 on ((jsonb_typeof((n0.properties -> 'objectid')) = 'string' and (n0.properties ->> 'objectid') = @pi0::text)) and n0.kind_ids operator (pg_catalog.@>) array [63]::int2[] and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [33]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match (s:RegressionKind31)-[r:RegressionKind01|RegressionKind02]->() where s.objectid = $object_id delete r +-- cypher_params: {"object_id":"rec-02"} +-- pgsql_params:{"pi0":"rec-02"} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from edge e0 join node n0 on ((jsonb_typeof((n0.properties -> 'objectid')) = 'string' and (n0.properties ->> 'objectid') = @pi0::text)) and n0.kind_ids operator (pg_catalog.@>) array [63]::int2[] and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [33, 34]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match (s:RegressionKind31)-[r:RegressionKind01|RegressionKind02|RegressionKind03|RegressionKind04|RegressionKind05|RegressionKind06|RegressionKind07|RegressionKind08|RegressionKind09]->() where s.objectid = $object_id delete r +-- cypher_params: {"object_id":"rec-02"} +-- pgsql_params:{"pi0":"rec-02"} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from edge e0 join node n0 on ((jsonb_typeof((n0.properties -> 'objectid')) = 'string' and (n0.properties ->> 'objectid') = @pi0::text)) and n0.kind_ids operator (pg_catalog.@>) array [63]::int2[] and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [33, 34, 35, 36, 37, 38, 39, 40, 41]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match (s:RegressionKind31)-[r:RegressionKind01|RegressionKind02|RegressionKind03|RegressionKind04|RegressionKind05|RegressionKind06|RegressionKind07|RegressionKind08|RegressionKind09|RegressionKind10|RegressionKind11|RegressionKind12|RegressionKind13|RegressionKind14|RegressionKind15|RegressionKind16|RegressionKind17|RegressionKind18|RegressionKind19|RegressionKind20|RegressionKind21|RegressionKind22|RegressionKind23|RegressionKind24|RegressionKind25|RegressionKind26|RegressionKind27|RegressionKind28|RegressionKind29|RegressionKind30]->() where s.objectid = $object_id delete r +-- cypher_params: {"object_id":"rec-02"} +-- pgsql_params:{"pi0":"rec-02"} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from edge e0 join node n0 on ((jsonb_typeof((n0.properties -> 'objectid')) = 'string' and (n0.properties ->> 'objectid') = @pi0::text)) and n0.kind_ids operator (pg_catalog.@>) array [63]::int2[] and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match ()-[r:RegressionKind32]->(e:RegressionKind31) where e.objectid = $object_id and r.isprimarygroup = $flag delete r +-- cypher_params: {"flag":false,"object_id":"rec-03-in"} +-- pgsql_params:{"pi0":"rec-03-in","pi1":false} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on ((jsonb_typeof((n1.properties -> 'objectid')) = 'string' and (n1.properties ->> 'objectid') = @pi0::text)) and n1.kind_ids operator (pg_catalog.@>) array [63]::int2[] and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where (((e0.properties -> 'isprimarygroup'))::jsonb = to_jsonb((@pi1::bool)::bool)::jsonb) and e0.kind_id = any (array [64]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match (s:RegressionKind31)-[r:RegressionKind32]->() where s.objectid = $object_id and r.isprimarygroup = $flag delete r +-- cypher_params: {"flag":true,"object_id":"rec-03-out"} +-- pgsql_params:{"pi0":"rec-03-out","pi1":true} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from edge e0 join node n0 on ((jsonb_typeof((n0.properties -> 'objectid')) = 'string' and (n0.properties ->> 'objectid') = @pi0::text)) and n0.kind_ids operator (pg_catalog.@>) array [63]::int2[] and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where (((e0.properties -> 'isprimarygroup'))::jsonb = to_jsonb((@pi1::bool)::bool)::jsonb) and e0.kind_id = any (array [64]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match ()-[r:RegressionKind32]->(e:RegressionKind31) where e.objectid in $object_ids delete r +-- cypher_params: {"object_ids":["rec-04-a","rec-04-b"]} +-- pgsql_params:{"pi0":["rec-04-a","rec-04-b"]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on ((n1.properties ->> 'objectid') = any (@pi0::text[])) and n1.kind_ids operator (pg_catalog.@>) array [63]::int2[] and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [64]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match ()-[r:RegressionKind34]->(e:RegressionKind33) where e.objectid in $object_ids delete r +-- cypher_params: {"object_ids":["rec-04-azure-a","rec-04-azure-b"]} +-- pgsql_params:{"pi0":["rec-04-azure-a","rec-04-azure-b"]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on ((n1.properties ->> 'objectid') = any (@pi0::text[])) and n1.kind_ids operator (pg_catalog.@>) array [65]::int2[] and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [66]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match (s:RegressionKind35)-[r:RegressionKind36]->(e) where e.objectid in $ca_ids return r, s +-- cypher_params: {"ca_ids":["ca-a","ca-b"]} +-- pgsql_params:{"pi0":["ca-a","ca-b"]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on ((n1.properties ->> 'objectid') = any (@pi0::text[])) and n1.id = e0.end_id join node n0 on n0.kind_ids operator (pg_catalog.@>) array [67]::int2[] and n0.id = e0.start_id where e0.kind_id = any (array [68]::int2[])) select s0.e0 as r, s0.n0 as s from s0; + +-- case: match ()-[r:RegressionKind37]->(e:RegressionKind35) where id(e) in $template_ids delete r +-- cypher_params: {"template_ids":[101,202]} +-- pgsql_params:{"pi0":[101,202]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n1.id as n1 from edge e0 join node n1 on (n1.id = any (@pi0::float8[])) and n1.kind_ids operator (pg_catalog.@>) array [67]::int2[] and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [69]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match ()-[r:RegressionKind39]->(e:RegressionKind38) where e.objectid = $object_id delete r +-- cypher_params: {"object_id":"rec-07"} +-- pgsql_params:{"pi0":"rec-07"} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on ((jsonb_typeof((n1.properties -> 'objectid')) = 'string' and (n1.properties ->> 'objectid') = @pi0::text)) and n1.kind_ids operator (pg_catalog.@>) array [70]::int2[] and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [71]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match (n:RegressionKind31) where n.objectid in $object_ids detach delete n +-- cypher_params: {"object_ids":["rec-08-a","rec-08-b"]} +-- pgsql_params:{"pi0":["rec-08-a","rec-08-b"]} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((n0.properties ->> 'objectid') = any (@pi0::text[])) and n0.kind_ids operator (pg_catalog.@>) array [63]::int2[]), s1 as (delete from node n1 using s0 where (s0.n0).id = n1.id) select 1; + +-- case: match (s:RegressionKind40)-[r:RegressionKind41]->(e:RegressionKind40) where r.lastseen < s.lastcollected or r.lastseen < e.lastcollected return id(r) +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [72]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [72]::int2[] and n1.id = e0.end_id where (nullif((e0.properties -> 'lastseen'), ('null')::jsonb)::jsonb < nullif((n0.properties -> 'lastcollected'), ('null')::jsonb)::jsonb or nullif((e0.properties -> 'lastseen'), ('null')::jsonb)::jsonb < nullif((n1.properties -> 'lastcollected'), ('null')::jsonb)::jsonb) and e0.kind_id = any (array [73]::int2[])) select (s0.e0).id as "id(r)" from s0; + +-- case: match (s:RegressionKind40)-[r:RegressionKind42]->(e:RegressionKind40) where r.lastseen < s.lastcollected or r.lastseen < e.lastcollected return r +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [72]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [72]::int2[] and n1.id = e0.end_id where (nullif((e0.properties -> 'lastseen'), ('null')::jsonb)::jsonb < nullif((n0.properties -> 'lastcollected'), ('null')::jsonb)::jsonb or nullif((e0.properties -> 'lastseen'), ('null')::jsonb)::jsonb < nullif((n1.properties -> 'lastcollected'), ('null')::jsonb)::jsonb) and e0.kind_id = any (array [74]::int2[])) select s0.e0 as r from s0; + +-- case: match (s:RegressionKind40)-[r]->(e:RegressionKind40) where (id(s) = $forward_start and id(e) = $forward_end and r:RegressionKind43) or (id(s) = $forward_end and id(e) = $forward_start and r:RegressionKind44) return id(r) +-- cypher_params: {"forward_end":202,"forward_start":101} +-- pgsql_params:{"pi0":101,"pi1":202} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, n1.id as n1 from edge e0 join node n1 on n1.kind_ids operator (pg_catalog.@>) array [72]::int2[] and n1.id = e0.end_id join node n0 on n0.kind_ids operator (pg_catalog.@>) array [72]::int2[] and n0.id = e0.start_id where ((n0.id = @pi0::float8 and n1.id = @pi1::float8 and e0.kind_id = any (array [75]::int2[])) or (n0.id = @pi1::float8 and n1.id = @pi0::float8 and e0.kind_id = any (array [76]::int2[])))) select (s0.e0).id as "id(r)" from s0; diff --git a/cypher/models/pgsql/test/translation_cases/relationship_scans_node_lookups.sql b/cypher/models/pgsql/test/translation_cases/relationship_scans_node_lookups.sql new file mode 100644 index 00000000..2dd82d4d --- /dev/null +++ b/cypher/models/pgsql/test/translation_cases/relationship_scans_node_lookups.sql @@ -0,0 +1,164 @@ +-- Copyright 2026 Specter Ops, Inc. +-- +-- Licensed under the Apache License, Version 2.0 +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- SPDX-License-Identifier: Apache-2.0 + +-- case: match (s)-[r:RegressionKind63]->(e) where (s:RegressionKind61 or s:RegressionKind62) and (e:RegressionKind61 or e:RegressionKind62) return id(r) +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on ((n0.kind_ids operator (pg_catalog.@>) array [93]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [94]::int2[])) and n0.id = e0.start_id join node n1 on ((n1.kind_ids operator (pg_catalog.@>) array [93]::int2[] or n1.kind_ids operator (pg_catalog.@>) array [94]::int2[])) and n1.id = e0.end_id where e0.kind_id = any (array [95]::int2[])) select (s0.e0).id as "id(r)" from s0; + +-- case: match (s)-[r:RegressionKind66|RegressionKind67]->(e) where not (s:RegressionKind64 or s:RegressionKind65) and not (e:RegressionKind64 or e:RegressionKind65) return r +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (not (n0.kind_ids operator (pg_catalog.@>) array [96]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [97]::int2[])) and n0.id = e0.start_id join node n1 on (not (n1.kind_ids operator (pg_catalog.@>) array [96]::int2[] or n1.kind_ids operator (pg_catalog.@>) array [97]::int2[])) and n1.id = e0.end_id where e0.kind_id = any (array [98, 99]::int2[])) select s0.e0 as r from s0; + +-- case: match (s)-[r:RegressionKind68]->(e) where not (s:RegressionKind64 or s:RegressionKind65) and r.lastseen is not null and not (e:RegressionKind64 or e:RegressionKind65) return id(r) +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (not (n0.kind_ids operator (pg_catalog.@>) array [96]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [97]::int2[])) and n0.id = e0.start_id join node n1 on (not (n1.kind_ids operator (pg_catalog.@>) array [96]::int2[] or n1.kind_ids operator (pg_catalog.@>) array [97]::int2[])) and n1.id = e0.end_id where ((e0.properties ? 'lastseen' and not (e0.properties -> 'lastseen') = ('null')::jsonb)) and e0.kind_id = any (array [100]::int2[])) select (s0.e0).id as "id(r)" from s0; + +-- case: match (s:RegressionKind69)-[r:RegressionKind70]->() return r +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [101]::int2[] and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [102]::int2[])) select s0.e0 as r from s0; + +-- case: match (s:RegressionKind69)-[r:RegressionKind71]->() return r +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [101]::int2[] and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [103]::int2[])) select s0.e0 as r from s0; + +-- case: match (s:RegressionKind69)-[r:RegressionKind72]->(e) where id(e) = $end_id return r, s +-- cypher_params: {"end_id":202} +-- pgsql_params:{"pi0":202} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n1 on (n1.id = @pi0::float8) and n1.id = e0.end_id join node n0 on n0.kind_ids operator (pg_catalog.@>) array [101]::int2[] and n0.id = e0.start_id where e0.kind_id = any (array [104]::int2[])) select s0.e0 as r, s0.n0 as s from s0; + +-- case: match (s:RegressionKind69)-[r:RegressionKind72|RegressionKind73|RegressionKind74|RegressionKind75|RegressionKind76|RegressionKind77|RegressionKind78|RegressionKind79|RegressionKind80]->(e) where id(e) = $end_id return r, s +-- cypher_params: {"end_id":202} +-- pgsql_params:{"pi0":202} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n1 on (n1.id = @pi0::float8) and n1.id = e0.end_id join node n0 on n0.kind_ids operator (pg_catalog.@>) array [101]::int2[] and n0.id = e0.start_id where e0.kind_id = any (array [104, 105, 106, 107, 108, 109, 110, 111, 112]::int2[])) select s0.e0 as r, s0.n0 as s from s0; + +-- case: match (s)-[r:RegressionKind82]->(e:RegressionKind81) return id(s), id(r), type(r), id(e) +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, n1.id as n1 from edge e0 join node n1 on n1.kind_ids operator (pg_catalog.@>) array [113]::int2[] and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [114]::int2[])) select s0.n0 as "id(s)", (s0.e0).id as "id(r)", kind_name((s0.e0).kind_id)::text as "type(r)", s0.n1 as "id(e)" from s0; + +-- case: match (s)-[r:RegressionKind83]->(e) return id(s), id(e) +with s0 as (select n0.id as n0, n1.id as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [115]::int2[])) select s0.n0 as "id(s)", s0.n1 as "id(e)" from s0; + +-- case: match (s)-[r:RegressionKind83|RegressionKind84]->(e) return id(s), id(e) +with s0 as (select n0.id as n0, n1.id as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [115, 116]::int2[])) select s0.n0 as "id(s)", s0.n1 as "id(e)" from s0; + +-- case: match (s)-[r:RegressionKind87|RegressionKind88|RegressionKind89|RegressionKind90|RegressionKind91|RegressionKind92]->(e) where (s:RegressionKind85 or s:RegressionKind86 or s:RegressionKind81) and id(e) in $end_ids return id(s) +-- cypher_params: {"end_ids":[202,303]} +-- pgsql_params:{"pi0":[202,303]} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n1 on (n1.id = any (@pi0::float8[])) and n1.id = e0.end_id join node n0 on ((n0.kind_ids operator (pg_catalog.@>) array [117]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [118]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [113]::int2[])) and n0.id = e0.start_id where e0.kind_id = any (array [119, 120, 121, 122, 123, 124]::int2[])) select (s0.n0).id as "id(s)" from s0; + +-- case: match (s)-[r:RegressionKind87|RegressionKind88|RegressionKind89|RegressionKind90|RegressionKind91]->(e:RegressionKind81) where (s:RegressionKind85 or s:RegressionKind86 or s:RegressionKind81) and id(e) in $end_ids return id(s) +-- cypher_params: {"end_ids":[202,303]} +-- pgsql_params:{"pi0":[202,303]} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n1 on (n1.id = any (@pi0::float8[])) and n1.kind_ids operator (pg_catalog.@>) array [113]::int2[] and n1.id = e0.end_id join node n0 on ((n0.kind_ids operator (pg_catalog.@>) array [117]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [118]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [113]::int2[])) and n0.id = e0.start_id where e0.kind_id = any (array [119, 120, 121, 122, 123]::int2[])) select (s0.n0).id as "id(s)" from s0; + +-- case: match (n) where n:RegressionKind85 or n:RegressionKind86 return id(n) +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (n0.kind_ids operator (pg_catalog.@>) array [117]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [118]::int2[])) select (s0.n0).id as "id(n)" from s0; + +-- case: match (n:RegressionKind93) return n +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [125]::int2[]) select s0.n0 as n from s0; + +-- case: match (n:RegressionKind81) where n.objectid = $objectid return n limit 1 +-- cypher_params: {"objectid":"S-1-5-21"} +-- pgsql_params:{"pi0":"S-1-5-21"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'objectid')) = 'string' and (n0.properties ->> 'objectid') = @pi0::text)) and n0.kind_ids operator (pg_catalog.@>) array [113]::int2[]) select s0.n0 as n from s0 limit 1; + +-- case: match (n) where n.name = $name and n.enabled = $enabled return id(n) +-- cypher_params: {"enabled":true,"name":"dc.example.test"} +-- pgsql_params:{"pi0":"dc.example.test","pi1":true} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = @pi0::text) and ((n0.properties -> 'enabled'))::jsonb = to_jsonb((@pi1::bool)::bool)::jsonb)) select (s0.n0).id as "id(n)" from s0; + +-- case: match (n:RegressionKind81) where n.hasura = true return id(n), n.hasura +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((n0.properties -> 'hasura'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [113]::int2[]) select (s0.n0).id as "id(n)", ((s0.n0).properties -> 'hasura') as "n.hasura" from s0; + +-- case: match (n:RegressionKind94) where n.distinguishedname starts with $prefix and n.domainsid = $domain return n +-- cypher_params: {"domain":"S-1-5-21","prefix":"CN=ADMINSDHOLDER,CN=SYSTEM,"} +-- pgsql_params:{"pi0":"CN=ADMINSDHOLDER,CN=SYSTEM,","pi1":"S-1-5-21"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (cypher_starts_with((n0.properties ->> 'distinguishedname'), (@pi0::text)::text)::bool and (jsonb_typeof((n0.properties -> 'domainsid')) = 'string' and (n0.properties ->> 'domainsid') = @pi1::text)) and n0.kind_ids operator (pg_catalog.@>) array [126]::int2[]) select s0.n0 as n from s0; + +-- case: match (n:RegressionKind85) where n.objectid ends with $suffix_a or n.objectid ends with $suffix_b return id(n) +-- cypher_params: {"suffix_a":"-S-1","suffix_b":"-S-2"} +-- pgsql_params:{"pi0":"-S-1","pi1":"-S-2"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (cypher_ends_with((n0.properties ->> 'objectid'), (@pi0::text)::text)::bool or cypher_ends_with((n0.properties ->> 'objectid'), (@pi1::text)::text)::bool) and n0.kind_ids operator (pg_catalog.@>) array [117]::int2[]) select (s0.n0).id as "id(n)" from s0; + +-- case: match (n) where toLower(n.name) starts with $prefix return id(n) +-- cypher_params: {"prefix":"remote desktop users%_"} +-- pgsql_params:{"pi0":"remote desktop users%_"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (cypher_starts_with((lower((n0.properties ->> 'name'))::text)::text, (@pi0::text)::text)::bool)) select (s0.n0).id as "id(n)" from s0; + +-- case: match (n) where toLower(n.objectid) contains $fragment return n +-- cypher_params: {"fragment":"approver_guid"} +-- pgsql_params:{"pi0":"approver_guid"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (cypher_contains((lower((n0.properties ->> 'objectid'))::text)::text, (@pi0::text)::text)::bool)) select s0.n0 as n from s0; + +-- case: match (n) where (n:RegressionKind85 or n:RegressionKind86) and n:RegressionKind69 and n.objectid ends with $suffix and n.domainsid = $domain return n +-- cypher_params: {"domain":"S-1-5-21","suffix":"-512"} +-- pgsql_params:{"pi0":"-512","pi1":"S-1-5-21"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((n0.kind_ids operator (pg_catalog.@>) array [117]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [118]::int2[]) and n0.kind_ids operator (pg_catalog.@>) array [101]::int2[] and cypher_ends_with((n0.properties ->> 'objectid'), (@pi0::text)::text)::bool and (jsonb_typeof((n0.properties -> 'domainsid')) = 'string' and (n0.properties ->> 'domainsid') = @pi1::text))) select s0.n0 as n from s0; + +-- case: match (n:RegressionKind69) where not (n:RegressionKind85 or n:RegressionKind98) and n.objectid ends with $suffix return n +-- cypher_params: {"suffix":"-512"} +-- pgsql_params:{"pi0":"-512"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (not (n0.kind_ids operator (pg_catalog.@>) array [117]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [130]::int2[]) and cypher_ends_with((n0.properties ->> 'objectid'), (@pi0::text)::text)::bool) and n0.kind_ids operator (pg_catalog.@>) array [101]::int2[]) select s0.n0 as n from s0; + +-- case: match (n) where n.name is null return n +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((not n0.properties ? 'name' or (n0.properties -> 'name') = ('null')::jsonb))) select s0.n0 as n from s0; + +-- case: match (n:RegressionKind95) where n.tenantid = $tenant and n.approvalrequired = true and (n.userapprovers is not null or n.groupapprovers is not null) return n +-- cypher_params: {"tenant":"tenant-1"} +-- pgsql_params:{"pi0":"tenant-1"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'tenantid')) = 'string' and (n0.properties ->> 'tenantid') = @pi0::text) and ((n0.properties -> 'approvalrequired'))::jsonb = to_jsonb((true)::bool)::jsonb and ((n0.properties ? 'userapprovers' and not (n0.properties -> 'userapprovers') = ('null')::jsonb) or (n0.properties ? 'groupapprovers' and not (n0.properties -> 'groupapprovers') = ('null')::jsonb))) and n0.kind_ids operator (pg_catalog.@>) array [127]::int2[]) select s0.n0 as n from s0; + +-- case: match (n) where id(n) in $ids return n +-- cypher_params: {"ids":[101,202,101]} +-- pgsql_params:{"pi0":[101,202,101]} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (n0.id = any (@pi0::float8[]))) select s0.n0 as n from s0; + +-- case: match (n:RegressionKind86) where not (n.gmsa is not null and n.gmsa = true) and not (n.msa is not null and n.msa = true) and id(n) in $ids return n +-- cypher_params: {"ids":[101,202]} +-- pgsql_params:{"pi0":[101,202]} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (not ((n0.properties ? 'gmsa' and not (n0.properties -> 'gmsa') = ('null')::jsonb) and ((n0.properties -> 'gmsa'))::jsonb = to_jsonb((true)::bool)::jsonb) and not ((n0.properties ? 'msa' and not (n0.properties -> 'msa') = ('null')::jsonb) and ((n0.properties -> 'msa'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.id = any (@pi0::float8[])) and n0.kind_ids operator (pg_catalog.@>) array [118]::int2[]) select s0.n0 as n from s0; + +-- case: match (s)-[:RegressionKind97]->(e) where id(s) = $tenant_id and (e:RegressionKind95 or e:RegressionKind96) and e.roletemplateid in $role_ids return e +-- cypher_params: {"role_ids":["role-a","role-b"],"tenant_id":101} +-- pgsql_params:{"pi0":101,"pi1":["role-a","role-b"]} +with s0 as (select n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = @pi0::float8) and n0.id = e0.start_id join node n1 on ((n1.kind_ids operator (pg_catalog.@>) array [127]::int2[] or n1.kind_ids operator (pg_catalog.@>) array [128]::int2[]) and (n1.properties ->> 'roletemplateid') = any (@pi1::text[])) and n1.id = e0.end_id where e0.kind_id = any (array [129]::int2[])) select s0.n1 as e from s0; + +-- case: match (s)-[:RegressionKind97]->(e:RegressionKind95) where id(s) = $tenant_id and e.enabled = true return e +-- cypher_params: {"tenant_id":101} +-- pgsql_params:{"pi0":101} +with s0 as (select n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = @pi0::float8) and n0.id = e0.start_id join node n1 on (((n1.properties -> 'enabled'))::jsonb = to_jsonb((true)::bool)::jsonb) and n1.kind_ids operator (pg_catalog.@>) array [127]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [129]::int2[])) select s0.n1 as e from s0; + +-- case: match (s)-[r:RegressionKind83]->(e) where id(s) = $start_id and id(e) = $end_id return r limit 1 +-- cypher_params: {"end_id":202,"start_id":101} +-- pgsql_params:{"pi0":101,"pi1":202} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, n1.id as n1 from edge e0 join node n0 on (n0.id = @pi0::float8) and n0.id = e0.start_id join node n1 on (n1.id = @pi1::float8) and n1.id = e0.end_id where e0.kind_id = any (array [115]::int2[]) limit 1) select s0.e0 as r from s0 limit 1; + +-- case: match (s)-[:RegressionKind82]->(e) where s.objectid ends with $suffix and id(e) = $end_id return s +-- cypher_params: {"end_id":202,"suffix":"-555"} +-- pgsql_params:{"pi0":"-555","pi1":202} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n1 on (n1.id = @pi1::float8) and n1.id = e0.end_id join node n0 on (cypher_ends_with((n0.properties ->> 'objectid'), (@pi0::text)::text)::bool) and n0.id = e0.start_id where e0.kind_id = any (array [114]::int2[])) select s0.n0 as s from s0; + +-- case: match (s)-[:RegressionKind82]->(e) where s.objectid ends with $suffix and id(e) = $end_id return id(s) +-- cypher_params: {"end_id":202,"suffix":"-555"} +-- pgsql_params:{"pi0":"-555","pi1":202} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n1 on (n1.id = @pi1::float8) and n1.id = e0.end_id join node n0 on (cypher_ends_with((n0.properties ->> 'objectid'), (@pi0::text)::text)::bool) and n0.id = e0.start_id where e0.kind_id = any (array [114]::int2[])) select (s0.n0).id as "id(s)" from s0; + +-- case: match (n:RegressionKind99) return n order by n.name desc +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [131]::int2[]) select s0.n0 as n from s0 order by ((s0.n0).properties -> 'name') desc; + +-- case: match (n:RegressionKind81) where n.domainsid = $domain and n.isdc = true and n.ldapavailable = true and n.ldapsigning = false return id(n) +-- cypher_params: {"domain":"S-1-5-21"} +-- pgsql_params:{"pi0":"S-1-5-21"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'domainsid')) = 'string' and (n0.properties ->> 'domainsid') = @pi0::text) and ((n0.properties -> 'isdc'))::jsonb = to_jsonb((true)::bool)::jsonb and ((n0.properties -> 'ldapavailable'))::jsonb = to_jsonb((true)::bool)::jsonb and ((n0.properties -> 'ldapsigning'))::jsonb = to_jsonb((false)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [113]::int2[]) select (s0.n0).id as "id(n)" from s0; + +-- case: match (n) where n.domainsid = $domain and n.isdc = true and n.ldapsavailable = true and n.epa = false return n +-- cypher_params: {"domain":"S-1-5-21"} +-- pgsql_params:{"pi0":"S-1-5-21"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'domainsid')) = 'string' and (n0.properties ->> 'domainsid') = @pi0::text) and ((n0.properties -> 'isdc'))::jsonb = to_jsonb((true)::bool)::jsonb and ((n0.properties -> 'ldapsavailable'))::jsonb = to_jsonb((true)::bool)::jsonb and ((n0.properties -> 'epa'))::jsonb = to_jsonb((false)::bool)::jsonb)) select s0.n0 as n from s0; diff --git a/cypher/models/pgsql/test/translation_cases/scalar_aggregation.sql b/cypher/models/pgsql/test/translation_cases/scalar_aggregation.sql index 21458c72..98a9830e 100644 --- a/cypher/models/pgsql/test/translation_cases/scalar_aggregation.sql +++ b/cypher/models/pgsql/test/translation_cases/scalar_aggregation.sql @@ -15,58 +15,58 @@ -- SPDX-License-Identifier: Apache-2.0 -- case: MATCH (n) RETURN sum(n.age) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select sum((((s0.n0).properties ->> 'age'))::float8)::numeric from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select sum((((s0.n0).properties ->> 'age'))::float8)::numeric as "sum(n.age)" from s0; -- case: MATCH (n) RETURN avg(n.salary) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select avg((((s0.n0).properties ->> 'salary'))::float8)::numeric from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select avg((((s0.n0).properties ->> 'salary'))::float8)::numeric as "avg(n.salary)" from s0; -- case: MATCH (n) RETURN min(n.created_date) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select cypher_min(((s0.n0).properties -> 'created_date'))::jsonb from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select cypher_min(((s0.n0).properties -> 'created_date'))::jsonb as "min(n.created_date)" from s0; -- case: MATCH (n) RETURN max(n.updated_date) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select cypher_max(((s0.n0).properties -> 'updated_date'))::jsonb from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select cypher_max(((s0.n0).properties -> 'updated_date'))::jsonb as "max(n.updated_date)" from s0; -- case: MATCH (n) RETURN min(n.name) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select cypher_min(((s0.n0).properties -> 'name'))::jsonb from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select cypher_min(((s0.n0).properties -> 'name'))::jsonb as "min(n.name)" from s0; -- case: MATCH (n) RETURN max(n.name) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select cypher_max(((s0.n0).properties -> 'name'))::jsonb from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select cypher_max(((s0.n0).properties -> 'name'))::jsonb as "max(n.name)" from s0; -- case: MATCH (n) RETURN n.department, sum(n.salary) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select ((s0.n0).properties -> 'department'), sum((((s0.n0).properties ->> 'salary'))::float8)::numeric from s0 group by ((s0.n0).properties -> 'department'); +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select ((s0.n0).properties -> 'department') as "n.department", sum((((s0.n0).properties ->> 'salary'))::float8)::numeric as "sum(n.salary)" from s0 group by ((s0.n0).properties -> 'department'); -- case: MATCH (n) RETURN n.department, avg(n.age) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select ((s0.n0).properties -> 'department'), avg((((s0.n0).properties ->> 'age'))::float8)::numeric from s0 group by ((s0.n0).properties -> 'department'); +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select ((s0.n0).properties -> 'department') as "n.department", avg((((s0.n0).properties ->> 'age'))::float8)::numeric as "avg(n.age)" from s0 group by ((s0.n0).properties -> 'department'); -- case: MATCH (n) RETURN count(n), sum(n.age), avg(n.age), min(n.age), max(n.age) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select count(s0.n0)::int8, sum((((s0.n0).properties ->> 'age'))::float8)::numeric, avg((((s0.n0).properties ->> 'age'))::float8)::numeric, cypher_min(((s0.n0).properties -> 'age'))::jsonb, cypher_max(((s0.n0).properties -> 'age'))::jsonb from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select count(s0.n0)::int8 as "count(n)", sum((((s0.n0).properties ->> 'age'))::float8)::numeric as "sum(n.age)", avg((((s0.n0).properties ->> 'age'))::float8)::numeric as "avg(n.age)", cypher_min(((s0.n0).properties -> 'age'))::jsonb as "min(n.age)", cypher_max(((s0.n0).properties -> 'age'))::jsonb as "max(n.age)" from s0; -- case: RETURN 'hello world' -select 'hello world'; +select 'hello world' as "'hello world'"; -- case: RETURN 2 + 3 -select 2 + 3; +select 2 + 3 as "2 + 3"; -- case: MATCH (n) RETURN n.department, collect(n.name) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select ((s0.n0).properties -> 'department'), array_remove(coalesce(array_agg(((s0.n0).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray from s0 group by ((s0.n0).properties -> 'department'); +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select ((s0.n0).properties -> 'department') as "n.department", array_remove(coalesce(array_agg(((s0.n0).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray as "collect(n.name)" from s0 group by ((s0.n0).properties -> 'department'); -- case: MATCH (n) RETURN collect(n.name) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select array_remove(coalesce(array_agg(((s0.n0).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select array_remove(coalesce(array_agg(((s0.n0).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray as "collect(n.name)" from s0; -- case: MATCH (n) RETURN n.department, collect(n.name), count(n) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select ((s0.n0).properties -> 'department'), array_remove(coalesce(array_agg(((s0.n0).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray, count(s0.n0)::int8 from s0 group by ((s0.n0).properties -> 'department'); +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select ((s0.n0).properties -> 'department') as "n.department", array_remove(coalesce(array_agg(((s0.n0).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray as "collect(n.name)", count(s0.n0)::int8 as "count(n)" from s0 group by ((s0.n0).properties -> 'department'); -- case: MATCH (n) RETURN size(n.tags) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select jsonb_array_length(((s0.n0).properties -> 'tags'))::int from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select case when jsonb_typeof(((s0.n0).properties -> 'tags')) = 'array' then jsonb_array_length(((s0.n0).properties -> 'tags'))::int else null end as "size(n.tags)" from s0; -- case: MATCH (n) RETURN size(collect(n.name)) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select cardinality(array_remove(coalesce(array_agg(((s0.n0).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray)::int from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select cardinality(array_remove(coalesce(array_agg(((s0.n0).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray)::int as "size(collect(n.name))" from s0; -- case: MATCH (n) WITH collect(labels(n)) as label_sets RETURN size(label_sets) -with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select array_remove(coalesce(array_agg(to_jsonb((array(select _kind.name from generate_subscripts((s1.n0).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s1.n0).kind_ids)[_kind_idx] order by _kind_idx))::text[])::jsonb)::jsonb[], array []::jsonb[])::jsonb[], null)::jsonb[] as i0 from s1) select cardinality(s0.i0)::int from s0; +with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select array_remove(coalesce(array_agg(to_jsonb((array(select _kind.name from generate_subscripts((s1.n0).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s1.n0).kind_ids)[_kind_idx] order by _kind_idx))::text[])::jsonb)::jsonb[], array []::jsonb[])::jsonb[], null)::jsonb[] as i0 from s1) select cardinality(s0.i0)::int as "size(label_sets)" from s0; -- case: MATCH (n) WHERE size(n.permissions) > 2 RETURN n -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (jsonb_array_length((n0.properties -> 'permissions'))::int > 2)) select s0.n0 as n from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (case when jsonb_typeof((n0.properties -> 'permissions')) = 'array' then jsonb_array_length((n0.properties -> 'permissions'))::int else null end > 2)) select s0.n0 as n from s0; -- case: MATCH (n) WITH n, collect(n.prop) as props WHERE size(props) > 1 RETURN n, props with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s1.n0 as n0, array_remove(coalesce(array_agg(((s1.n0).properties ->> 'prop'))::anyarray, array []::text[])::anyarray, null)::anyarray as i0 from s1 group by n0) select s0.n0 as n, s0.i0 as props from s0 where (cardinality(s0.i0)::int > 1); @@ -84,19 +84,19 @@ with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposit with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select count(s1.n0)::int8 as i0 from s1), s2 as (select s0.i0 as i0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1) select s2.n1 as o from s2; -- case: MATCH (n) RETURN count(n) + count(n) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select count(s0.n0)::int8 + count(s0.n0)::int8 from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select count(s0.n0)::int8 + count(s0.n0)::int8 as "count(n) + count(n)" from s0; -- case: MATCH (n) RETURN count(n) * 2 -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select count(s0.n0)::int8 * 2 from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select count(s0.n0)::int8 * 2 as "count(n) * 2" from s0; -- case: MATCH (n) RETURN count(n) AS total ORDER BY total DESC with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select count(s0.n0)::int8 as total from s0 order by total desc; -- case: MATCH (n) RETURN toInteger(n.value) + count(n) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select (((s0.n0).properties ->> 'value'))::int8 + count(s0.n0)::int8 from s0 group by (((s0.n0).properties ->> 'value'))::int8; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select (((s0.n0).properties ->> 'value'))::int8 + count(s0.n0)::int8 as "toInteger(n.value) + count(n)" from s0 group by (((s0.n0).properties ->> 'value'))::int8; -- case: MATCH (n) WITH toInteger(n.value) AS value, count(n) AS node_count RETURN value + node_count -with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select (((s1.n0).properties ->> 'value'))::int8 as i0, count(s1.n0)::int8 as i1 from s1 group by (((s1.n0).properties ->> 'value'))::int8) select s0.i0 + s0.i1 from s0; +with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select (((s1.n0).properties ->> 'value'))::int8 as i0, count(s1.n0)::int8 as i1 from s1 group by (((s1.n0).properties ->> 'value'))::int8) select s0.i0 + s0.i1 as "value + node_count" from s0; -- case: MATCH (n) WITH toInteger(n.value) + count(n) AS score RETURN score with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select (((s1.n0).properties ->> 'value'))::int8 + count(s1.n0)::int8 as i0 from s1 group by (((s1.n0).properties ->> 'value'))::int8) select s0.i0 as score from s0; diff --git a/cypher/models/pgsql/test/translation_cases/shortest_paths.sql b/cypher/models/pgsql/test/translation_cases/shortest_paths.sql index d2439bdd..19736c8c 100644 --- a/cypher/models/pgsql/test/translation_cases/shortest_paths.sql +++ b/cypher/models/pgsql/test/translation_cases/shortest_paths.sql @@ -16,81 +16,81 @@ -- case: match p = allShortestPaths((s:NodeKind1)-[*..]->()) return p -- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.start_id, e0.end_id, 1, exists (select 1 from edge where end_id = e0.start_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id where case when (select count(*)::int8 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id) = 0 then true else shortest_path_self_endpoint_error(e0.start_id, e0.start_id) end;","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.end_id, s1.depth + 1, exists (select 1 from edge where end_id = e0.start_id), false, s1.path || e0.id from forward_front s1 join edge e0 on e0.start_id = s1.next_id where e0.id != all (s1.path);"} -with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_asp_harness(@pi0::text, @pi1::text, 15)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id where case when s1.root_id != s1.next_id then true else shortest_path_self_endpoint_error(s1.root_id, s1.next_id) end) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; +with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_asp_harness(@pi0::text, @pi1::text, 15)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id where case when s1.root_id != s1.next_id then true else shortest_path_self_endpoint_error(s1.root_id, s1.next_id) end) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; -- case: match p = allShortestPaths((s:NodeKind1)-[*..]->({name: "123"})) return p -- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where (jsonb_typeof((n1.properties -\u003e 'name')) = 'string' and (n1.properties -\u003e\u003e 'name') = '123')) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id where case when (select count(*)::int8 from traversal_terminal_filter where traversal_terminal_filter.id = e0.end_id) = 0 then true else shortest_path_self_endpoint_error(e0.end_id, e0.end_id) end;","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.start_id, s1.depth + 1, exists (select 1 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id), false, e0.id || s1.path from forward_front s1 join edge e0 on e0.end_id = s1.next_id where e0.id != all (s1.path);"} -with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_asp_harness(@pi0::text, @pi1::text, 15, ('')::text, ('insert into traversal_terminal_filter (id) select distinct n0.id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id is not null;')::text)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n1 on n1.id = s1.root_id join node n0 on n0.id = s1.next_id where case when s1.root_id != s1.next_id then true else shortest_path_self_endpoint_error(s1.root_id, s1.next_id) end) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; +with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_asp_harness(@pi0::text, @pi1::text, 15, ('')::text, ('insert into traversal_terminal_filter (id) select distinct n0.id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id is not null;')::text)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n1 on n1.id = s1.root_id join node n0 on n0.id = s1.next_id where case when s1.root_id != s1.next_id then true else shortest_path_self_endpoint_error(s1.root_id, s1.next_id) end) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; -- case: match p = allShortestPaths((s:NodeKind1)-[*..]->(e)) where e.name = '123' return p -- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ((jsonb_typeof((n1.properties -\u003e 'name')) = 'string' and (n1.properties -\u003e\u003e 'name') = '123'))) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id where case when (select count(*)::int8 from traversal_terminal_filter where traversal_terminal_filter.id = e0.end_id) = 0 then true else shortest_path_self_endpoint_error(e0.end_id, e0.end_id) end;","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.start_id, s1.depth + 1, exists (select 1 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id), false, e0.id || s1.path from forward_front s1 join edge e0 on e0.end_id = s1.next_id where e0.id != all (s1.path);"} -with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_asp_harness(@pi0::text, @pi1::text, 15, ('')::text, ('insert into traversal_terminal_filter (id) select distinct n0.id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id is not null;')::text)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n1 on n1.id = s1.root_id join node n0 on n0.id = s1.next_id where case when s1.root_id != s1.next_id then true else shortest_path_self_endpoint_error(s1.root_id, s1.next_id) end) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; +with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_asp_harness(@pi0::text, @pi1::text, 15, ('')::text, ('insert into traversal_terminal_filter (id) select distinct n0.id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id is not null;')::text)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n1 on n1.id = s1.root_id join node n0 on n0.id = s1.next_id where case when s1.root_id != s1.next_id then true else shortest_path_self_endpoint_error(s1.root_id, s1.next_id) end) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; -- case: match p=shortestPath((n:NodeKind1)-[:EdgeKind1*1..]->(m)) where 'admin_tier_0' in split(m.system_tags, ' ') and n.objectid ends with '-513' and n<>m return p limit 1000 --- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.properties -\u003e\u003e 'objectid') like '%-513') and n0.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.end_id, s1.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = s1.root_id and traversal_pair_filter.terminal_id = e0.end_id), false, s1.path || e0.id from forward_front s1 join edge e0 on e0.start_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from forward_visited where forward_visited.root_id = s1.root_id and forward_visited.id = e0.end_id);","pi2":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ('admin_tier_0' = any (string_to_array((n1.properties -\u003e\u003e 'system_tags'), ' ')::text[]))) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi3":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.start_id, s1.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = s1.root_id), false, e0.id || s1.path from backward_front s1 join edge e0 on e0.end_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from backward_visited where backward_visited.root_id = s1.root_id and backward_visited.id = e0.start_id);"} -with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_sp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into traversal_pair_filter (root_id, terminal_id) select distinct n0.id, n1.id from node n0, node n1 where ((n0.properties ->> ''objectid'') like ''%-513'') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (''admin_tier_0'' = any (string_to_array((n1.properties ->> ''system_tags''), '' '')::text[])) and n0.id is not null and n1.id is not null;')::text, (1000)::int8)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((s0.n0).id <> (s0.n1).id) limit 1000; +-- pgsql_params:{"pi0":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.properties -\u003e\u003e 'objectid') like '%-513') and n0.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi1":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.end_id, s1.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = s1.root_id and traversal_pair_filter.terminal_id = e0.end_id), false, s1.path || e0.id from pg_temp.bsp_forward_front s1 join edge e0 on e0.start_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from pg_temp.bsp_forward_visited where pg_temp.bsp_forward_visited.root_id = s1.root_id and pg_temp.bsp_forward_visited.id = e0.end_id);","pi2":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ('admin_tier_0' = any (string_to_array((n1.properties -\u003e\u003e 'system_tags'), ' ')::text[]))) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi3":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.start_id, s1.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = s1.root_id), false, e0.id || s1.path from pg_temp.bsp_backward_front s1 join edge e0 on e0.end_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from pg_temp.bsp_backward_visited where pg_temp.bsp_backward_visited.root_id = s1.root_id and pg_temp.bsp_backward_visited.id = e0.start_id);"} +with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_sp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into pg_temp.bsp_pair_filter (root_id, terminal_id) select distinct n0.id, n1.id from node n0, node n1 where ((n0.properties ->> ''objectid'') like ''%-513'') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (''admin_tier_0'' = any (string_to_array((n1.properties ->> ''system_tags''), '' '')::text[])) and n0.id is not null and n1.id is not null;')::text, false, (1000)::int8) limit 1000) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((s0.n0).id <> (s0.n1).id) limit 1000; -- case: match p=shortestPath((n:NodeKind1)-[:EdgeKind1*1..]->(m)) where 'admin_tier_0' in split(m.system_tags, ' ') and n.objectid ends with '-513' and m<>n return p limit 1000 --- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.properties -\u003e\u003e 'objectid') like '%-513') and n0.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.end_id, s1.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = s1.root_id and traversal_pair_filter.terminal_id = e0.end_id), false, s1.path || e0.id from forward_front s1 join edge e0 on e0.start_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from forward_visited where forward_visited.root_id = s1.root_id and forward_visited.id = e0.end_id);","pi2":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ('admin_tier_0' = any (string_to_array((n1.properties -\u003e\u003e 'system_tags'), ' ')::text[]))) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi3":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.start_id, s1.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = s1.root_id), false, e0.id || s1.path from backward_front s1 join edge e0 on e0.end_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from backward_visited where backward_visited.root_id = s1.root_id and backward_visited.id = e0.start_id);"} -with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_sp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into traversal_pair_filter (root_id, terminal_id) select distinct n0.id, n1.id from node n0, node n1 where ((n0.properties ->> ''objectid'') like ''%-513'') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (''admin_tier_0'' = any (string_to_array((n1.properties ->> ''system_tags''), '' '')::text[])) and n0.id is not null and n1.id is not null;')::text, (1000)::int8)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((s0.n1).id <> (s0.n0).id) limit 1000; +-- pgsql_params:{"pi0":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.properties -\u003e\u003e 'objectid') like '%-513') and n0.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi1":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.end_id, s1.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = s1.root_id and traversal_pair_filter.terminal_id = e0.end_id), false, s1.path || e0.id from pg_temp.bsp_forward_front s1 join edge e0 on e0.start_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from pg_temp.bsp_forward_visited where pg_temp.bsp_forward_visited.root_id = s1.root_id and pg_temp.bsp_forward_visited.id = e0.end_id);","pi2":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ('admin_tier_0' = any (string_to_array((n1.properties -\u003e\u003e 'system_tags'), ' ')::text[]))) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi3":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.start_id, s1.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = s1.root_id), false, e0.id || s1.path from pg_temp.bsp_backward_front s1 join edge e0 on e0.end_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from pg_temp.bsp_backward_visited where pg_temp.bsp_backward_visited.root_id = s1.root_id and pg_temp.bsp_backward_visited.id = e0.start_id);"} +with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_sp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into pg_temp.bsp_pair_filter (root_id, terminal_id) select distinct n0.id, n1.id from node n0, node n1 where ((n0.properties ->> ''objectid'') like ''%-513'') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (''admin_tier_0'' = any (string_to_array((n1.properties ->> ''system_tags''), '' '')::text[])) and n0.id is not null and n1.id is not null;')::text, false, (1000)::int8) limit 1000) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((s0.n1).id <> (s0.n0).id) limit 1000; -- case: match p=shortestPath((t:NodeKind1)<-[:EdgeKind1|EdgeKind2*1..]-(s:NodeKind2)) where coalesce(t.system_tags, '') contains 'admin_tier_0' and t.name =~ 'name.*' and s<>t return p limit 1000 -- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where (coalesce((n0.properties -\u003e\u003e 'system_tags'), '')::text like '%admin_tier_0%' and (n0.properties -\u003e\u003e 'name') ~ 'name.*') and n0.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id where e0.kind_id = any (array [3, 4]::int2[]);","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.start_id, s1.depth + 1, exists (select 1 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id), false, s1.path || e0.id from forward_front s1 join edge e0 on e0.end_id = s1.next_id where e0.kind_id = any (array [3, 4]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from visited where visited.root_id = s1.root_id and visited.id = e0.start_id);"} -with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_sp_harness(@pi0::text, @pi1::text, 15, ('')::text, ('insert into traversal_terminal_filter (id) select distinct n1.id from node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id is not null;')::text, (1000)::int8)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((s0.n1).id <> (s0.n0).id) limit 1000; +with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_sp_harness(@pi0::text, @pi1::text, 15, ('')::text, ('insert into traversal_terminal_filter (id) select distinct n1.id from node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id is not null;')::text, (1000)::int8) limit 1000) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((s0.n1).id <> (s0.n0).id) limit 1000; -- case: match p=shortestPath((a)-[:EdgeKind1*]->(b)) where id(a) = 1 and id(b) = 2 return p --- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where (n0.id = 1)) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]) and case when (select count(*)::int8 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.start_id) = 0 then true else shortest_path_self_endpoint_error(e0.start_id, e0.start_id) end;","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.end_id, s1.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = s1.root_id and traversal_pair_filter.terminal_id = e0.end_id), false, s1.path || e0.id from forward_front s1 join edge e0 on e0.start_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from forward_visited where forward_visited.root_id = s1.root_id and forward_visited.id = e0.end_id);","pi2":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where (n1.id = 2)) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi3":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.start_id, s1.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = s1.root_id), false, e0.id || s1.path from backward_front s1 join edge e0 on e0.end_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from backward_visited where backward_visited.root_id = s1.root_id and backward_visited.id = e0.start_id);"} -with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_sp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into traversal_pair_filter (root_id, terminal_id) select distinct n0.id, n1.id from node n0, node n1 where (n0.id = 1) and (n1.id = 2) and n0.id is not null and n1.id is not null;')::text)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id where case when s1.root_id != s1.next_id then true else shortest_path_self_endpoint_error(s1.root_id, s1.next_id) end) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; +-- pgsql_params:{"pi0":1,"pi1":2,"pi2":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select distinct n0.id as root_id from unnest($1::int8[]) as s1_seed_parameter(id) join node n0 on n0.id = s1_seed_parameter.id where (n0.id = 1)) select e0.start_id, e0.end_id, 1, (n1.id = 2), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]);","pi3":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.end_id, s1.depth + 1, (n1.id = 2), false, s1.path || e0.id from pg_temp.bsp_forward_front s1 join edge e0 on e0.start_id = s1.next_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from pg_temp.bsp_forward_visited where pg_temp.bsp_forward_visited.root_id = s1.root_id and pg_temp.bsp_forward_visited.id = e0.end_id);","pi4":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select distinct n1.id as root_id from unnest($2::int8[]) as s1_seed_parameter(id) join node n1 on n1.id = s1_seed_parameter.id where (n1.id = 2)) select e0.end_id, e0.start_id, 1, (n0.id = 1), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [3]::int2[]);","pi5":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.start_id, s1.depth + 1, (n0.id = 1), false, e0.id || s1.path from pg_temp.bsp_backward_front s1 join edge e0 on e0.end_id = s1.next_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from pg_temp.bsp_backward_visited where pg_temp.bsp_backward_visited.root_id = s1.root_id and pg_temp.bsp_backward_visited.id = e0.start_id);"} +with s0 as (with singleton_endpoints as (select n0.id as root_id, n1.id as terminal_id from node n0, node n1 where (n0.id = @pi0::int8) and (n1.id = @pi1::int8)), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select bidirectional_sp_harness.* from singleton_endpoints, bidirectional_sp_harness(@pi2::text, @pi3::text, @pi4::text, @pi5::text, 15, array [singleton_endpoints.root_id]::int8[], array [singleton_endpoints.terminal_id]::int8[], false)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id where case when s1.root_id != s1.next_id then true else shortest_path_self_endpoint_error(s1.root_id, s1.next_id) end) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; -- case: match p=shortestPath((a)-[:EdgeKind1*]->(b:NodeKind1)) where a <> b return p -- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where n1.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.end_id, e0.start_id, 1, exists (select 1 from edge where end_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.start_id, s1.depth + 1, exists (select 1 from edge where end_id = e0.end_id), false, e0.id || s1.path from forward_front s1 join edge e0 on e0.end_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from visited where visited.root_id = s1.root_id and visited.id = e0.start_id);"} -with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_sp_harness(@pi0::text, @pi1::text, 15)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n1 on n1.id = s1.root_id join node n0 on n0.id = s1.next_id) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((s0.n0).id <> (s0.n1).id); +with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_sp_harness(@pi0::text, @pi1::text, 15)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n1 on n1.id = s1.root_id join node n0 on n0.id = s1.next_id) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((s0.n0).id <> (s0.n1).id); -- case: match p=shortestPath((a:NodeKind2)-[:EdgeKind1*]->(b)) where a <> b return p -- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@\u003e) array [2]::int2[]) select e0.start_id, e0.end_id, 1, exists (select 1 from edge where end_id = e0.start_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.end_id, s1.depth + 1, exists (select 1 from edge where end_id = e0.start_id), false, s1.path || e0.id from forward_front s1 join edge e0 on e0.start_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from visited where visited.root_id = s1.root_id and visited.id = e0.end_id);"} -with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_sp_harness(@pi0::text, @pi1::text, 15)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((s0.n0).id <> (s0.n1).id); +with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_sp_harness(@pi0::text, @pi1::text, 15)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((s0.n0).id <> (s0.n1).id); -- case: match p=shortestPath((b)<-[:EdgeKind1*]-(a)) where id(a) = 1 and id(b) = 2 return p --- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where (n0.id = 2)) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.end_id and traversal_pair_filter.terminal_id = e0.start_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]) and case when (select count(*)::int8 from traversal_pair_filter where traversal_pair_filter.root_id = e0.end_id and traversal_pair_filter.terminal_id = e0.end_id) = 0 then true else shortest_path_self_endpoint_error(e0.end_id, e0.end_id) end;","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.start_id, s1.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = s1.root_id and traversal_pair_filter.terminal_id = e0.start_id), false, s1.path || e0.id from forward_front s1 join edge e0 on e0.end_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from forward_visited where forward_visited.root_id = s1.root_id and forward_visited.id = e0.start_id);","pi2":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where (n1.id = 1)) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.end_id and traversal_pair_filter.terminal_id = e0.start_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi3":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.end_id, s1.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.end_id and traversal_pair_filter.terminal_id = s1.root_id), false, e0.id || s1.path from backward_front s1 join edge e0 on e0.start_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from backward_visited where backward_visited.root_id = s1.root_id and backward_visited.id = e0.end_id);"} -with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_sp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into traversal_pair_filter (root_id, terminal_id) select distinct n0.id, n1.id from node n0, node n1 where (n0.id = 2) and (n1.id = 1) and n0.id is not null and n1.id is not null;')::text)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id where case when s1.root_id != s1.next_id then true else shortest_path_self_endpoint_error(s1.root_id, s1.next_id) end) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; +-- pgsql_params:{"pi0":2,"pi1":1,"pi2":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select distinct n0.id as root_id from unnest($1::int8[]) as s1_seed_parameter(id) join node n0 on n0.id = s1_seed_parameter.id where (n0.id = 2)) select e0.end_id, e0.start_id, 1, (n1.id = 1), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n1 on n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[]);","pi3":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.start_id, s1.depth + 1, (n1.id = 1), false, s1.path || e0.id from pg_temp.bsp_forward_front s1 join edge e0 on e0.end_id = s1.next_id join node n1 on n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from pg_temp.bsp_forward_visited where pg_temp.bsp_forward_visited.root_id = s1.root_id and pg_temp.bsp_forward_visited.id = e0.start_id);","pi4":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select distinct n1.id as root_id from unnest($2::int8[]) as s1_seed_parameter(id) join node n1 on n1.id = s1_seed_parameter.id where (n1.id = 1)) select e0.start_id, e0.end_id, 1, (n0.id = 2), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n0 on n0.id = e0.end_id where e0.kind_id = any (array [3]::int2[]);","pi5":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.end_id, s1.depth + 1, (n0.id = 2), false, e0.id || s1.path from pg_temp.bsp_backward_front s1 join edge e0 on e0.start_id = s1.next_id join node n0 on n0.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from pg_temp.bsp_backward_visited where pg_temp.bsp_backward_visited.root_id = s1.root_id and pg_temp.bsp_backward_visited.id = e0.end_id);"} +with s0 as (with singleton_endpoints as (select n0.id as root_id, n1.id as terminal_id from node n0, node n1 where (n0.id = @pi0::int8) and (n1.id = @pi1::int8)), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select bidirectional_sp_harness.* from singleton_endpoints, bidirectional_sp_harness(@pi2::text, @pi3::text, @pi4::text, @pi5::text, 15, array [singleton_endpoints.root_id]::int8[], array [singleton_endpoints.terminal_id]::int8[], false)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id where case when s1.root_id != s1.next_id then true else shortest_path_self_endpoint_error(s1.root_id, s1.next_id) end) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; -- case: match p = allShortestPaths((m:NodeKind1)<-[:EdgeKind1*..]-(n)) where coalesce(m.system_tags, '') contains 'admin_tier_0' and n.name = '123' and n <> m return p -- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ((jsonb_typeof((n1.properties -\u003e 'name')) = 'string' and (n1.properties -\u003e\u003e 'name') = '123'))) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.end_id, s1.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = s1.root_id and traversal_pair_filter.terminal_id = e0.end_id), false, s1.path || e0.id from forward_front s1 join edge e0 on e0.start_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path);","pi2":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where (coalesce((n0.properties -\u003e\u003e 'system_tags'), '')::text like '%admin_tier_0%') and n0.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi3":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.start_id, s1.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = s1.root_id), false, e0.id || s1.path from backward_front s1 join edge e0 on e0.end_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path);"} -with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_asp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into traversal_pair_filter (root_id, terminal_id) select distinct n1.id, n0.id from node n1, node n0 where ((jsonb_typeof((n1.properties -> ''name'')) = ''string'' and (n1.properties ->> ''name'') = ''123'')) and (coalesce((n0.properties ->> ''system_tags''), '''')::text like ''%admin_tier_0%'') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id is not null and n0.id is not null;')::text)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n1 on n1.id = s1.root_id join node n0 on n0.id = s1.next_id) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((s0.n1).id <> (s0.n0).id); +with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_asp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into traversal_pair_filter (root_id, terminal_id) select distinct n1.id, n0.id from node n1, node n0 where ((jsonb_typeof((n1.properties -> ''name'')) = ''string'' and (n1.properties ->> ''name'') = ''123'')) and (coalesce((n0.properties ->> ''system_tags''), '''')::text like ''%admin_tier_0%'') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id is not null and n0.id is not null;')::text)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n1 on n1.id = s1.root_id join node n0 on n0.id = s1.next_id) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((s0.n1).id <> (s0.n0).id); -- case: match p=shortestPath((a)-[:EdgeKind1*]->(b:NodeKind1)) where a <> b return p -- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where n1.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.end_id, e0.start_id, 1, exists (select 1 from edge where end_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.start_id, s1.depth + 1, exists (select 1 from edge where end_id = e0.end_id), false, e0.id || s1.path from forward_front s1 join edge e0 on e0.end_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from visited where visited.root_id = s1.root_id and visited.id = e0.start_id);"} -with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_sp_harness(@pi0::text, @pi1::text, 15)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n1 on n1.id = s1.root_id join node n0 on n0.id = s1.next_id) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((s0.n0).id <> (s0.n1).id); +with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_sp_harness(@pi0::text, @pi1::text, 15)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n1 on n1.id = s1.root_id join node n0 on n0.id = s1.next_id) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((s0.n0).id <> (s0.n1).id); -- case: match p=(c:NodeKind1)-[]->(u:NodeKind2) match p2=shortestPath((u:NodeKind2)-[*1..]->(d:NodeKind1)) return p, p2 limit 500 -- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s2_seed(root_id) as not materialized (select distinct n1.id as root_id from traversal_root_filter s2_seed_filter join node n1 on n1.id = s2_seed_filter.id where n1.kind_ids operator (pg_catalog.@\u003e) array [2]::int2[]) select e1.start_id, e1.end_id, 1, exists (select 1 from traversal_terminal_filter where traversal_terminal_filter.id = e1.end_id), e1.start_id = e1.end_id, array [e1.id] from s2_seed join edge e1 on e1.start_id = s2_seed.root_id where case when (select count(*)::int8 from traversal_terminal_filter where traversal_terminal_filter.id = e1.start_id) = 0 then true else shortest_path_self_endpoint_error(e1.start_id, e1.start_id) end;","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s2.root_id, e1.end_id, s2.depth + 1, exists (select 1 from traversal_terminal_filter where traversal_terminal_filter.id = e1.end_id), false, s2.path || e1.id from forward_front s2 join edge e1 on e1.start_id = s2.next_id where e1.id != all (s2.path) and not exists (select 1 from visited where visited.root_id = s2.root_id and visited.id = e1.end_id);"} -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id), s1 as (with s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_sp_harness(@pi0::text, @pi1::text, 15, ('insert into traversal_root_filter (id) select distinct (s0.n1).id from s0 where (s0.n1).id is not null;')::text, ('insert into traversal_terminal_filter (id) select distinct n2.id from node n2 where n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id is not null;')::text)) select s0.e0 as e0, s2.path as ep0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, s2 join node n1 on n1.id = s2.root_id join node n2 on n2.id = s2.next_id where (s0.n1).id = s2.root_id and case when s2.root_id != s2.next_id then true else shortest_path_self_endpoint_error(s2.root_id, s2.next_id) end) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null then null else ordered_edges_to_path(s1.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as p, case when (s1.n1).id is null or s1.ep0 is null or (s1.n2).id is null then null else ordered_edges_to_path(s1.n1, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s1.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n1, s1.n2]::nodecomposite[])::pathcomposite end as p2 from s1 limit 500; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id), s1 as (with s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_sp_harness(@pi0::text, @pi1::text, 15, ('insert into traversal_root_filter (id) select distinct (s0.n1).id from s0 where (s0.n1).id is not null;')::text, ('insert into traversal_terminal_filter (id) select distinct n2.id from node n2 where n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id is not null;')::text)) select s0.e0 as e0, s2.path as ep0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, s2 join node n1 on n1.id = s2.root_id join node n2 on n2.id = s2.next_id where (s0.n1).id = s2.root_id and case when s2.root_id != s2.next_id then true else shortest_path_self_endpoint_error(s2.root_id, s2.next_id) end) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null then null else ordered_edge_ids_to_path(0, s1.n0, array [s1.e0]::int8[], array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as p, case when (s1.n1).id is null or s1.ep0 is null or (s1.n2).id is null then null else ordered_edge_ids_to_path(0, s1.n1, s1.ep0, array [s1.n1, s1.n2]::nodecomposite[])::pathcomposite end as p2 from s1 limit 500; -- case: match p = allShortestPaths((a)-[:EdgeKind1*..]->()) return p -- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select e0.start_id, e0.end_id, 1, exists (select 1 from edge where end_id = e0.start_id), e0.start_id = e0.end_id, array [e0.id] from edge e0 where e0.kind_id = any (array [3]::int2[]) and case when (select count(*)::int8 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id) = 0 then true else shortest_path_self_endpoint_error(e0.start_id, e0.start_id) end;","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.end_id, s1.depth + 1, exists (select 1 from edge where end_id = e0.start_id), false, s1.path || e0.id from forward_front s1 join edge e0 on e0.start_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path);"} -with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_asp_harness(@pi0::text, @pi1::text, 15)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id where case when s1.root_id != s1.next_id then true else shortest_path_self_endpoint_error(s1.root_id, s1.next_id) end) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; +with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_asp_harness(@pi0::text, @pi1::text, 15)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id where case when s1.root_id != s1.next_id then true else shortest_path_self_endpoint_error(s1.root_id, s1.next_id) end) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; -- case: match p=shortestPath((n:NodeKind1)-[:EdgeKind1*1..]->(m:NodeKind2)) return p limit 10 -- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_terminal_filter where traversal_terminal_filter.id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]) and case when (select count(*)::int8 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id) = 0 then true else shortest_path_self_endpoint_error(e0.start_id, e0.start_id) end;","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.end_id, s1.depth + 1, exists (select 1 from traversal_terminal_filter where traversal_terminal_filter.id = e0.end_id), false, s1.path || e0.id from forward_front s1 join edge e0 on e0.start_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from visited where visited.root_id = s1.root_id and visited.id = e0.end_id);"} -with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_sp_harness(@pi0::text, @pi1::text, 15, ('')::text, ('insert into traversal_terminal_filter (id) select distinct n1.id from node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id is not null;')::text, (10)::int8)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id where case when s1.root_id != s1.next_id then true else shortest_path_self_endpoint_error(s1.root_id, s1.next_id) end) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 10; +with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_sp_harness(@pi0::text, @pi1::text, 15, ('')::text, ('insert into traversal_terminal_filter (id) select distinct n1.id from node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id is not null;')::text, (10)::int8) limit 10) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id where case when s1.root_id != s1.next_id then true else shortest_path_self_endpoint_error(s1.root_id, s1.next_id) end) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 10; -- case: match (a:NodeKind1), (b:NodeKind2) match p=shortestPath((a)-[:EdgeKind1*]->(b)) return p --- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s3_seed(root_id) as not materialized (select s3_seed_filter.id as root_id from traversal_root_filter s3_seed_filter) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s3_seed join edge e0 on e0.start_id = s3_seed.root_id where e0.kind_id = any (array [3]::int2[]) and case when (select count(*)::int8 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id) = 0 then true else shortest_path_self_endpoint_error(e0.start_id, e0.start_id) end;","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s3.root_id, e0.end_id, s3.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = s3.root_id and traversal_pair_filter.terminal_id = e0.end_id), false, s3.path || e0.id from forward_front s3 join edge e0 on e0.start_id = s3.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s3.path) and not exists (select 1 from forward_visited where forward_visited.root_id = s3.root_id and forward_visited.id = e0.end_id);","pi2":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s3_seed(root_id) as not materialized (select s3_seed_filter.id as root_id from traversal_terminal_filter s3_seed_filter) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s3_seed join edge e0 on e0.end_id = s3_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi3":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s3.root_id, e0.start_id, s3.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = s3.root_id), false, e0.id || s3.path from backward_front s3 join edge e0 on e0.end_id = s3.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s3.path) and not exists (select 1 from backward_visited where backward_visited.root_id = s3.root_id and backward_visited.id = e0.start_id);"} -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s2 as (with s3(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_sp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into traversal_pair_filter (root_id, terminal_id) select distinct (s1.n0).id, (s1.n1).id from s1 where (s1.n0).id is not null and (s1.n1).id is not null;')::text)) select s3.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1, s3 join node n0 on n0.id = s3.root_id join node n1 on n1.id = s3.next_id where (s1.n0).id = s3.root_id and (s1.n1).id = s3.next_id and case when s3.root_id != s3.next_id then true else shortest_path_self_endpoint_error(s3.root_id, s3.next_id) end) select case when (s2.n0).id is null or s2.ep0 is null or (s2.n1).id is null then null else ordered_edges_to_path(s2.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s2.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s2.n0, s2.n1]::nodecomposite[])::pathcomposite end as p from s2; +-- pgsql_params:{"pi0":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s3_seed(root_id) as not materialized (select s3_seed_filter.id as root_id from traversal_root_filter s3_seed_filter) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s3_seed join edge e0 on e0.start_id = s3_seed.root_id where e0.kind_id = any (array [3]::int2[]) and case when (select count(*)::int8 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id) = 0 then true else shortest_path_self_endpoint_error(e0.start_id, e0.start_id) end;","pi1":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s3.root_id, e0.end_id, s3.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = s3.root_id and traversal_pair_filter.terminal_id = e0.end_id), false, s3.path || e0.id from pg_temp.bsp_forward_front s3 join edge e0 on e0.start_id = s3.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s3.path) and not exists (select 1 from pg_temp.bsp_forward_visited where pg_temp.bsp_forward_visited.root_id = s3.root_id and pg_temp.bsp_forward_visited.id = e0.end_id);","pi2":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s3_seed(root_id) as not materialized (select s3_seed_filter.id as root_id from traversal_terminal_filter s3_seed_filter) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s3_seed join edge e0 on e0.end_id = s3_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi3":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s3.root_id, e0.start_id, s3.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = s3.root_id), false, e0.id || s3.path from pg_temp.bsp_backward_front s3 join edge e0 on e0.end_id = s3.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s3.path) and not exists (select 1 from pg_temp.bsp_backward_visited where pg_temp.bsp_backward_visited.root_id = s3.root_id and pg_temp.bsp_backward_visited.id = e0.start_id);"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s2 as (with s3(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_sp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into pg_temp.bsp_pair_filter (root_id, terminal_id) select distinct (s1.n0).id, (s1.n1).id from s1 where (s1.n0).id is not null and (s1.n1).id is not null;')::text, false)) select s3.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1, s3 join node n0 on n0.id = s3.root_id join node n1 on n1.id = s3.next_id where (s1.n0).id = s3.root_id and (s1.n1).id = s3.next_id and case when s3.root_id != s3.next_id then true else shortest_path_self_endpoint_error(s3.root_id, s3.next_id) end) select case when (s2.n0).id is null or s2.ep0 is null or (s2.n1).id is null then null else ordered_edge_ids_to_path(0, s2.n0, s2.ep0, array [s2.n0, s2.n1]::nodecomposite[])::pathcomposite end as p from s2; -- case: match (a:NodeKind1), (b:NodeKind2) match p=allShortestPaths((a)-[:EdgeKind1*..]->(b)) return p -- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s3_seed(root_id) as not materialized (select s3_seed_filter.id as root_id from traversal_root_filter s3_seed_filter) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s3_seed join edge e0 on e0.start_id = s3_seed.root_id where e0.kind_id = any (array [3]::int2[]) and case when (select count(*)::int8 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id) = 0 then true else shortest_path_self_endpoint_error(e0.start_id, e0.start_id) end;","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s3.root_id, e0.end_id, s3.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = s3.root_id and traversal_pair_filter.terminal_id = e0.end_id), false, s3.path || e0.id from forward_front s3 join edge e0 on e0.start_id = s3.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s3.path);","pi2":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s3_seed(root_id) as not materialized (select s3_seed_filter.id as root_id from traversal_terminal_filter s3_seed_filter) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s3_seed join edge e0 on e0.end_id = s3_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi3":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s3.root_id, e0.start_id, s3.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = s3.root_id), false, e0.id || s3.path from backward_front s3 join edge e0 on e0.end_id = s3.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s3.path);"} -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s2 as (with s3(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_asp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into traversal_pair_filter (root_id, terminal_id) select distinct (s1.n0).id, (s1.n1).id from s1 where (s1.n0).id is not null and (s1.n1).id is not null;')::text)) select s3.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1, s3 join node n0 on n0.id = s3.root_id join node n1 on n1.id = s3.next_id where (s1.n0).id = s3.root_id and (s1.n1).id = s3.next_id and case when s3.root_id != s3.next_id then true else shortest_path_self_endpoint_error(s3.root_id, s3.next_id) end) select case when (s2.n0).id is null or s2.ep0 is null or (s2.n1).id is null then null else ordered_edges_to_path(s2.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s2.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s2.n0, s2.n1]::nodecomposite[])::pathcomposite end as p from s2; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s2 as (with s3(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_asp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into traversal_pair_filter (root_id, terminal_id) select distinct (s1.n0).id, (s1.n1).id from s1 where (s1.n0).id is not null and (s1.n1).id is not null;')::text)) select s3.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1, s3 join node n0 on n0.id = s3.root_id join node n1 on n1.id = s3.next_id where (s1.n0).id = s3.root_id and (s1.n1).id = s3.next_id and case when s3.root_id != s3.next_id then true else shortest_path_self_endpoint_error(s3.root_id, s3.next_id) end) select case when (s2.n0).id is null or s2.ep0 is null or (s2.n1).id is null then null else ordered_edge_ids_to_path(0, s2.n0, s2.ep0, array [s2.n0, s2.n1]::nodecomposite[])::pathcomposite end as p from s2; -- case: match p=shortestPath((u:NodeKind1)-[:EdgeKind1*1..]->(g:NodeKind2)) with distinct g as Group, count(u) as UserCount return Group.name, UserCount order by UserCount desc limit 5 -- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_terminal_filter where traversal_terminal_filter.id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id where e0.kind_id = any (array [3]::int2[]) and case when (select count(*)::int8 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id) = 0 then true else shortest_path_self_endpoint_error(e0.start_id, e0.start_id) end;","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s2.root_id, e0.end_id, s2.depth + 1, exists (select 1 from traversal_terminal_filter where traversal_terminal_filter.id = e0.end_id), false, s2.path || e0.id from forward_front s2 join edge e0 on e0.start_id = s2.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s2.path) and not exists (select 1 from visited where visited.root_id = s2.root_id and visited.id = e0.end_id);"} -with s0 as (with s1 as (with s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_sp_harness(@pi0::text, @pi1::text, 15, ('')::text, ('insert into traversal_terminal_filter (id) select distinct n1.id from node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id is not null;')::text)) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join node n0 on n0.id = s2.root_id join node n1 on n1.id = s2.next_id where case when s2.root_id != s2.next_id then true else shortest_path_self_endpoint_error(s2.root_id, s2.next_id) end) select distinct s1.n1 as n2, count(s1.n0)::int8 as i0 from s1 group by n1) select ((s0.n2).properties -> 'name'), s0.i0 as UserCount from s0 order by s0.i0 desc limit 5; +with s0 as (with s1 as (with s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_sp_harness(@pi0::text, @pi1::text, 15, ('')::text, ('insert into traversal_terminal_filter (id) select distinct n1.id from node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id is not null;')::text)) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join node n0 on n0.id = s2.root_id join node n1 on n1.id = s2.next_id where case when s2.root_id != s2.next_id then true else shortest_path_self_endpoint_error(s2.root_id, s2.next_id) end) select distinct s1.n1 as n2, count(s1.n0)::int8 as i0 from s1 group by n1) select ((s0.n2).properties -> 'name') as "Group.name", s0.i0 as UserCount from s0 order by s0.i0 desc limit 5; -- case: MATCH (g1:Group) MATCH (g2:Group) WHERE g1.name STARTS WITH 'DOMAIN USERS@' AND g2.name STARTS WITH 'DOMAIN ADMINS@' MATCH p=shortestPath((g1)-[:AddAllowedToAct|AddMember|AdminTo|AllExtendedRights|AllowedToDelegate|CanRDP|Contains|ForceChangePassword|GenericAll|GenericWrite|GetChangesAll|GetChanges|HasSession|MemberOf|Owns|ReadLAPSPassword|SQLAdmin|TrustedBy|WriteAccountRestrictions|WriteOwner*1..]->(g2)) WHERE NONE(r IN relationships(p) WHERE type(r) = 'HasSession' AND startNode(r).name = 'DF-WIN10-DEV01.DUMPSTER.FIRE') RETURN p --- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s3_seed(root_id) as not materialized (select s3_seed_filter.id as root_id from traversal_root_filter s3_seed_filter) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s3_seed join edge e0 on e0.start_id = s3_seed.root_id where e0.kind_id = any (array [14, 15, 16, 17, 18, 19, 12, 20, 21, 22, 23, 24, 7, 25, 26, 27, 28, 29, 30, 31]::int2[]) and case when (select count(*)::int8 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id) = 0 then true else shortest_path_self_endpoint_error(e0.start_id, e0.start_id) end;","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s3.root_id, e0.end_id, s3.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = s3.root_id and traversal_pair_filter.terminal_id = e0.end_id), false, s3.path || e0.id from forward_front s3 join edge e0 on e0.start_id = s3.next_id where e0.kind_id = any (array [14, 15, 16, 17, 18, 19, 12, 20, 21, 22, 23, 24, 7, 25, 26, 27, 28, 29, 30, 31]::int2[]) and e0.id != all (s3.path) and not exists (select 1 from forward_visited where forward_visited.root_id = s3.root_id and forward_visited.id = e0.end_id);","pi2":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s3_seed(root_id) as not materialized (select s3_seed_filter.id as root_id from traversal_terminal_filter s3_seed_filter) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s3_seed join edge e0 on e0.end_id = s3_seed.root_id where e0.kind_id = any (array [14, 15, 16, 17, 18, 19, 12, 20, 21, 22, 23, 24, 7, 25, 26, 27, 28, 29, 30, 31]::int2[]);","pi3":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s3.root_id, e0.start_id, s3.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = s3.root_id), false, e0.id || s3.path from backward_front s3 join edge e0 on e0.end_id = s3.next_id where e0.kind_id = any (array [14, 15, 16, 17, 18, 19, 12, 20, 21, 22, 23, 24, 7, 25, 26, 27, 28, 29, 30, 31]::int2[]) and e0.id != all (s3.path) and not exists (select 1 from backward_visited where backward_visited.root_id = s3.root_id and backward_visited.id = e0.start_id);"} -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [13]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where ((n1.properties ->> 'name') like 'DOMAIN ADMINS@%' and ((s0.n0).properties ->> 'name') like 'DOMAIN USERS@%') and n1.kind_ids operator (pg_catalog.@>) array [13]::int2[]), s2 as (with s3(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_sp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into traversal_pair_filter (root_id, terminal_id) select distinct (s1.n0).id, (s1.n1).id from s1 where (s1.n0).id is not null and (s1.n1).id is not null;')::text)) select s3.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1, s3 join node n0 on n0.id = s3.root_id join node n1 on n1.id = s3.next_id where (s1.n0).id = s3.root_id and (s1.n1).id = s3.next_id and case when s3.root_id != s3.next_id then true else shortest_path_self_endpoint_error(s3.root_id, s3.next_id) end) select case when (s2.n0).id is null or s2.ep0 is null or (s2.n1).id is null then null else ordered_edges_to_path(s2.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s2.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s2.n0, s2.n1]::nodecomposite[])::pathcomposite end as p from s2 where ((not exists (select 1 from edge i0 where ((jsonb_typeof(((start_node((i0.id, i0.start_id, i0.end_id, i0.kind_id, i0.properties)::edgecomposite)::nodecomposite).properties -> 'name')) = 'string' and ((start_node((i0.id, i0.start_id, i0.end_id, i0.kind_id, i0.properties)::edgecomposite)::nodecomposite).properties ->> 'name') = 'DF-WIN10-DEV01.DUMPSTER.FIRE') and i0.kind_id = 7) and i0.id = any (s2.ep0)) and s2.ep0 is not null)::bool); +-- pgsql_params:{"pi0":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s3_seed(root_id) as not materialized (select s3_seed_filter.id as root_id from traversal_root_filter s3_seed_filter) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s3_seed join edge e0 on e0.start_id = s3_seed.root_id where e0.kind_id = any (array [14, 15, 16, 17, 18, 19, 12, 20, 21, 22, 23, 24, 7, 25, 26, 27, 28, 29, 30, 31]::int2[]) and case when (select count(*)::int8 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id) = 0 then true else shortest_path_self_endpoint_error(e0.start_id, e0.start_id) end;","pi1":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s3.root_id, e0.end_id, s3.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = s3.root_id and traversal_pair_filter.terminal_id = e0.end_id), false, s3.path || e0.id from pg_temp.bsp_forward_front s3 join edge e0 on e0.start_id = s3.next_id where e0.kind_id = any (array [14, 15, 16, 17, 18, 19, 12, 20, 21, 22, 23, 24, 7, 25, 26, 27, 28, 29, 30, 31]::int2[]) and e0.id != all (s3.path) and not exists (select 1 from pg_temp.bsp_forward_visited where pg_temp.bsp_forward_visited.root_id = s3.root_id and pg_temp.bsp_forward_visited.id = e0.end_id);","pi2":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s3_seed(root_id) as not materialized (select s3_seed_filter.id as root_id from traversal_terminal_filter s3_seed_filter) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s3_seed join edge e0 on e0.end_id = s3_seed.root_id where e0.kind_id = any (array [14, 15, 16, 17, 18, 19, 12, 20, 21, 22, 23, 24, 7, 25, 26, 27, 28, 29, 30, 31]::int2[]);","pi3":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s3.root_id, e0.start_id, s3.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = s3.root_id), false, e0.id || s3.path from pg_temp.bsp_backward_front s3 join edge e0 on e0.end_id = s3.next_id where e0.kind_id = any (array [14, 15, 16, 17, 18, 19, 12, 20, 21, 22, 23, 24, 7, 25, 26, 27, 28, 29, 30, 31]::int2[]) and e0.id != all (s3.path) and not exists (select 1 from pg_temp.bsp_backward_visited where pg_temp.bsp_backward_visited.root_id = s3.root_id and pg_temp.bsp_backward_visited.id = e0.start_id);"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [13]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where ((n1.properties ->> 'name') like 'DOMAIN ADMINS@%' and ((s0.n0).properties ->> 'name') like 'DOMAIN USERS@%') and n1.kind_ids operator (pg_catalog.@>) array [13]::int2[]), s2 as (with s3(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_sp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into pg_temp.bsp_pair_filter (root_id, terminal_id) select distinct (s1.n0).id, (s1.n1).id from s1 where (s1.n0).id is not null and (s1.n1).id is not null;')::text, false)) select s3.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1, s3 join node n0 on n0.id = s3.root_id join node n1 on n1.id = s3.next_id where (s1.n0).id = s3.root_id and (s1.n1).id = s3.next_id and case when s3.root_id != s3.next_id then true else shortest_path_self_endpoint_error(s3.root_id, s3.next_id) end) select case when (s2.n0).id is null or s2.ep0 is null or (s2.n1).id is null then null else ordered_edge_ids_to_path(0, s2.n0, s2.ep0, array [s2.n0, s2.n1]::nodecomposite[])::pathcomposite end as p from s2 where ((not exists (select 1 from edge i0 where ((jsonb_typeof(((start_node((i0.id, i0.start_id, i0.end_id, i0.kind_id, i0.properties)::edgecomposite)::nodecomposite).properties -> 'name')) = 'string' and ((start_node((i0.id, i0.start_id, i0.end_id, i0.kind_id, i0.properties)::edgecomposite)::nodecomposite).properties ->> 'name') = 'DF-WIN10-DEV01.DUMPSTER.FIRE') and i0.kind_id = 7) and i0.id = any (s2.ep0)) and s2.ep0 is not null)::bool); -- case: match p=shortestPath((s:NodeKind1)-[:EdgeKind1|HasSession*1..]->(d:NodeKind1)) where s.name = 'path-filter-src' and d.name = 'path-filter-dst' with p where none(r in relationships(p) where type(r) = 'HasSession' and startNode(r).name = 'blocked-session-host') return p --- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -\u003e 'name')) = 'string' and (n0.properties -\u003e\u003e 'name') = 'path-filter-src')) and n0.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id where e0.kind_id = any (array [3, 7]::int2[]) and case when (select count(*)::int8 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.start_id) = 0 then true else shortest_path_self_endpoint_error(e0.start_id, e0.start_id) end;","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s2.root_id, e0.end_id, s2.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = s2.root_id and traversal_pair_filter.terminal_id = e0.end_id), false, s2.path || e0.id from forward_front s2 join edge e0 on e0.start_id = s2.next_id where e0.kind_id = any (array [3, 7]::int2[]) and e0.id != all (s2.path) and not exists (select 1 from forward_visited where forward_visited.root_id = s2.root_id and forward_visited.id = e0.end_id);","pi2":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s2_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ((jsonb_typeof((n1.properties -\u003e 'name')) = 'string' and (n1.properties -\u003e\u003e 'name') = 'path-filter-dst')) and n1.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s2_seed join edge e0 on e0.end_id = s2_seed.root_id where e0.kind_id = any (array [3, 7]::int2[]);","pi3":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s2.root_id, e0.start_id, s2.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = s2.root_id), false, e0.id || s2.path from backward_front s2 join edge e0 on e0.end_id = s2.next_id where e0.kind_id = any (array [3, 7]::int2[]) and e0.id != all (s2.path) and not exists (select 1 from backward_visited where backward_visited.root_id = s2.root_id and backward_visited.id = e0.start_id);"} -with s0 as (with s1 as (with s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_sp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into traversal_pair_filter (root_id, terminal_id) select distinct n0.id, n1.id from node n0, node n1 where ((jsonb_typeof((n0.properties -> ''name'')) = ''string'' and (n0.properties ->> ''name'') = ''path-filter-src'')) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and ((jsonb_typeof((n1.properties -> ''name'')) = ''string'' and (n1.properties ->> ''name'') = ''path-filter-dst'')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id is not null and n1.id is not null;')::text)) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join node n0 on n0.id = s2.root_id join node n1 on n1.id = s2.next_id where case when s2.root_id != s2.next_id then true else shortest_path_self_endpoint_error(s2.root_id, s2.next_id) end) select case when (s1.n0).id is null or s1.ep0 is null or (s1.n1).id is null then null else ordered_edges_to_path(s1.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s1.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as pc0 from s1) select s0.pc0 as p from s0 where (((select count(*)::int from unnest(((s0.pc0).edges)::edgecomposite[]) as i0 where ((jsonb_typeof(((start_node((i0.id, i0.start_id, i0.end_id, i0.kind_id, i0.properties)::edgecomposite)::nodecomposite).properties -> 'name')) = 'string' and ((start_node((i0.id, i0.start_id, i0.end_id, i0.kind_id, i0.properties)::edgecomposite)::nodecomposite).properties ->> 'name') = 'blocked-session-host') and i0.kind_id = 7)) = 0 and ((s0.pc0).edges)::edgecomposite[] is not null)::bool); +-- pgsql_params:{"pi0":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -\u003e 'name')) = 'string' and (n0.properties -\u003e\u003e 'name') = 'path-filter-src')) and n0.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id where e0.kind_id = any (array [3, 7]::int2[]) and case when (select count(*)::int8 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.start_id) = 0 then true else shortest_path_self_endpoint_error(e0.start_id, e0.start_id) end;","pi1":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s2.root_id, e0.end_id, s2.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = s2.root_id and traversal_pair_filter.terminal_id = e0.end_id), false, s2.path || e0.id from pg_temp.bsp_forward_front s2 join edge e0 on e0.start_id = s2.next_id where e0.kind_id = any (array [3, 7]::int2[]) and e0.id != all (s2.path) and not exists (select 1 from pg_temp.bsp_forward_visited where pg_temp.bsp_forward_visited.root_id = s2.root_id and pg_temp.bsp_forward_visited.id = e0.end_id);","pi2":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s2_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ((jsonb_typeof((n1.properties -\u003e 'name')) = 'string' and (n1.properties -\u003e\u003e 'name') = 'path-filter-dst')) and n1.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s2_seed join edge e0 on e0.end_id = s2_seed.root_id where e0.kind_id = any (array [3, 7]::int2[]);","pi3":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s2.root_id, e0.start_id, s2.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = s2.root_id), false, e0.id || s2.path from pg_temp.bsp_backward_front s2 join edge e0 on e0.end_id = s2.next_id where e0.kind_id = any (array [3, 7]::int2[]) and e0.id != all (s2.path) and not exists (select 1 from pg_temp.bsp_backward_visited where pg_temp.bsp_backward_visited.root_id = s2.root_id and pg_temp.bsp_backward_visited.id = e0.start_id);"} +with s0 as (with s1 as (with s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_sp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into pg_temp.bsp_pair_filter (root_id, terminal_id) select distinct n0.id, n1.id from node n0, node n1 where ((jsonb_typeof((n0.properties -> ''name'')) = ''string'' and (n0.properties ->> ''name'') = ''path-filter-src'')) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and ((jsonb_typeof((n1.properties -> ''name'')) = ''string'' and (n1.properties ->> ''name'') = ''path-filter-dst'')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id is not null and n1.id is not null;')::text, false)) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join node n0 on n0.id = s2.root_id join node n1 on n1.id = s2.next_id where case when s2.root_id != s2.next_id then true else shortest_path_self_endpoint_error(s2.root_id, s2.next_id) end) select case when (s1.n0).id is null or s1.ep0 is null or (s1.n1).id is null then null else ordered_edge_ids_to_path(0, s1.n0, s1.ep0, array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as pc0 from s1) select s0.pc0 as p from s0 where (((select count(*)::int from unnest(((s0.pc0).edges)::edgecomposite[]) as i0 where ((jsonb_typeof(((start_node((i0.id, i0.start_id, i0.end_id, i0.kind_id, i0.properties)::edgecomposite)::nodecomposite).properties -> 'name')) = 'string' and ((start_node((i0.id, i0.start_id, i0.end_id, i0.kind_id, i0.properties)::edgecomposite)::nodecomposite).properties ->> 'name') = 'blocked-session-host') and i0.kind_id = 7)) = 0 and ((s0.pc0).edges)::edgecomposite[] is not null)::bool); diff --git a/cypher/models/pgsql/test/translation_cases/stepwise_traversal.sql b/cypher/models/pgsql/test/translation_cases/stepwise_traversal.sql index f48a2bcf..741f5895 100644 --- a/cypher/models/pgsql/test/translation_cases/stepwise_traversal.sql +++ b/cypher/models/pgsql/test/translation_cases/stepwise_traversal.sql @@ -21,16 +21,16 @@ with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::e with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where (e0.kind_id = 3)) select s0.e0 as r from s0; -- case: match ()-[r]->() return type(r) order by type(r) -with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id) select kind_name((s0.e0).kind_id)::text from s0 order by kind_name((s0.e0).kind_id)::text; +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id) select kind_name((s0.e0).kind_id)::text as "type(r)" from s0 order by kind_name((s0.e0).kind_id)::text; -- case: match ()-[r]->() where type(r) <> 'EdgeKind1' return type(r) order by type(r) -with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where (e0.kind_id <> 3)) select kind_name((s0.e0).kind_id)::text from s0 order by kind_name((s0.e0).kind_id)::text; +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where (e0.kind_id <> 3)) select kind_name((s0.e0).kind_id)::text as "type(r)" from s0 order by kind_name((s0.e0).kind_id)::text; -- case: match ()-[r]->() where type(r) in ['EdgeKind2'] return type(r) order by type(r) -with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where (kind_name(e0.kind_id)::text = any (array ['EdgeKind2']::text[]))) select kind_name((s0.e0).kind_id)::text from s0 order by kind_name((s0.e0).kind_id)::text; +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where (kind_name(e0.kind_id)::text = any (array ['EdgeKind2']::text[]))) select kind_name((s0.e0).kind_id)::text as "type(r)" from s0 order by kind_name((s0.e0).kind_id)::text; -- case: match ()-[r]->() where type(r) STARTS WITH 'EdgeKind' return type(r) order by type(r) -with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where (kind_name(e0.kind_id)::text like 'EdgeKind%')) select kind_name((s0.e0).kind_id)::text from s0 order by kind_name((s0.e0).kind_id)::text; +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where (kind_name(e0.kind_id)::text like 'EdgeKind%')) select kind_name((s0.e0).kind_id)::text as "type(r)" from s0 order by kind_name((s0.e0).kind_id)::text; -- case: match ()-[r]->() where 'EdgeKind1' = type(r) return r with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where (3 = e0.kind_id)) select s0.e0 as r from s0; @@ -42,7 +42,7 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, (e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties)::edgecomposite as e1 from s0, edge e1 join node n2 on n2.id = e1.start_id join node n3 on n3.id = e1.end_id) select s1.e0 as r, s1.e1 as e from s1; -- case: match p = (:NodeKind1)-[:EdgeKind1|EdgeKind2]->(c:NodeKind2) where '123' in c.prop2 or '243' in c.prop2 or size(c.prop2) = 0 return p limit 10 -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on ('123' = any (jsonb_to_text_array((n1.properties -> 'prop2'))::text[]) or '243' = any (jsonb_to_text_array((n1.properties -> 'prop2'))::text[]) or jsonb_array_length((n1.properties -> 'prop2'))::int = 0) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[]) limit 10) select case when (s0.n0).id is null or s0.e0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s0.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 10; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on ('123' = any (jsonb_to_text_array((n1.properties -> 'prop2'))::text[]) or '243' = any (jsonb_to_text_array((n1.properties -> 'prop2'))::text[]) or case when jsonb_typeof((n1.properties -> 'prop2')) = 'array' then jsonb_array_length((n1.properties -> 'prop2'))::int else null end = 0) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[]) limit 10) select case when (s0.n0).id is null or s0.e0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, array [s0.e0]::int8[], array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 10; -- case: match ()-[r:EdgeKind1]->() return count(r) as the_count select count(*)::int8 as the_count from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]); @@ -53,10 +53,115 @@ with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::e -- case: match ()-[r:EdgeKind1]->({name: "123"}) return count(r) as the_count with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n1 on (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = '123') and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [3]::int2[])) select count(s0.e0)::int8 as the_count from s0; +-- case: match (s)-[r:RegressionKind01]->(e) where id(s) = $start_id return r, e +-- cypher_params: {"start_id":101} +-- pgsql_params:{"pi0":101} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = @pi0::float8) and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [33]::int2[])) select s0.e0 as r, s0.n1 as e from s0; + +-- case: match (s)-[r:RegressionKind01]->(e) where id(s) in $start_ids return r, e +-- cypher_params: {"start_ids":[101]} +-- pgsql_params:{"pi0":[101]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = any (@pi0::float8[])) and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [33]::int2[])) select s0.e0 as r, s0.n1 as e from s0; + +-- case: match (s)-[r:RegressionKind01]->(e) where id(e) = $end_id return r, s +-- cypher_params: {"end_id":202} +-- pgsql_params:{"pi0":202} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n1 on (n1.id = @pi0::float8) and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [33]::int2[])) select s0.e0 as r, s0.n0 as s from s0; + +-- case: match (s)-[r:RegressionKind01|RegressionKind02]->(e) where id(s) in $start_ids return r, e +-- cypher_params: {"start_ids":[101]} +-- pgsql_params:{"pi0":[101]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = any (@pi0::float8[])) and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [33, 34]::int2[])) select s0.e0 as r, s0.n1 as e from s0; + +-- case: match (s)-[r:RegressionKind01|RegressionKind02]->(e) where id(e) in $end_ids return r, s +-- cypher_params: {"end_ids":[202]} +-- pgsql_params:{"pi0":[202]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n1 on (n1.id = any (@pi0::float8[])) and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [33, 34]::int2[])) select s0.e0 as r, s0.n0 as s from s0; + +-- case: match (s)-[r:RegressionKind01|RegressionKind02|RegressionKind03|RegressionKind04|RegressionKind05]->(e) where id(s) in $start_ids return r, e +-- cypher_params: {"start_ids":[101]} +-- pgsql_params:{"pi0":[101]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = any (@pi0::float8[])) and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [33, 34, 35, 36, 37]::int2[])) select s0.e0 as r, s0.n1 as e from s0; + +-- case: match (s)-[r:RegressionKind01|RegressionKind02|RegressionKind03|RegressionKind04|RegressionKind05]->(e) where id(e) in $end_ids return r, s +-- cypher_params: {"end_ids":[202]} +-- pgsql_params:{"pi0":[202]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n1 on (n1.id = any (@pi0::float8[])) and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [33, 34, 35, 36, 37]::int2[])) select s0.e0 as r, s0.n0 as s from s0; + +-- case: match (s)-[r:RegressionKind01|RegressionKind02|RegressionKind03|RegressionKind04|RegressionKind05|RegressionKind06|RegressionKind07|RegressionKind08|RegressionKind09]->(e) where id(s) in $start_ids return r, e +-- cypher_params: {"start_ids":[101]} +-- pgsql_params:{"pi0":[101]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = any (@pi0::float8[])) and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [33, 34, 35, 36, 37, 38, 39, 40, 41]::int2[])) select s0.e0 as r, s0.n1 as e from s0; + +-- case: match (s)-[r:RegressionKind01|RegressionKind02|RegressionKind03|RegressionKind04|RegressionKind05|RegressionKind06|RegressionKind07|RegressionKind08|RegressionKind09]->(e) where id(e) in $end_ids return r, s +-- cypher_params: {"end_ids":[202]} +-- pgsql_params:{"pi0":[202]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n1 on (n1.id = any (@pi0::float8[])) and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [33, 34, 35, 36, 37, 38, 39, 40, 41]::int2[])) select s0.e0 as r, s0.n0 as s from s0; + +-- case: match (s)-[r:RegressionKind01|RegressionKind02|RegressionKind03|RegressionKind04|RegressionKind05|RegressionKind06|RegressionKind07|RegressionKind08|RegressionKind09|RegressionKind10|RegressionKind11|RegressionKind12|RegressionKind13|RegressionKind14|RegressionKind15|RegressionKind16|RegressionKind17|RegressionKind18|RegressionKind19|RegressionKind20|RegressionKind21|RegressionKind22|RegressionKind23|RegressionKind24|RegressionKind25|RegressionKind26|RegressionKind27|RegressionKind28|RegressionKind29|RegressionKind30]->(e) where id(s) in $start_ids return r, e +-- cypher_params: {"start_ids":[101]} +-- pgsql_params:{"pi0":[101]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = any (@pi0::float8[])) and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62]::int2[])) select s0.e0 as r, s0.n1 as e from s0; + +-- case: match (s)-[r:RegressionKind01|RegressionKind02|RegressionKind03|RegressionKind04|RegressionKind05|RegressionKind06|RegressionKind07|RegressionKind08|RegressionKind09|RegressionKind10|RegressionKind11|RegressionKind12|RegressionKind13|RegressionKind14|RegressionKind15|RegressionKind16|RegressionKind17|RegressionKind18|RegressionKind19|RegressionKind20|RegressionKind21|RegressionKind22|RegressionKind23|RegressionKind24|RegressionKind25|RegressionKind26|RegressionKind27|RegressionKind28|RegressionKind29|RegressionKind30]->(e) where id(e) in $end_ids return r, s +-- cypher_params: {"end_ids":[202]} +-- pgsql_params:{"pi0":[202]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n1 on (n1.id = any (@pi0::float8[])) and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62]::int2[])) select s0.e0 as r, s0.n0 as s from s0; + +-- case: match (s)-[r:RegressionKind51]->(e) where id(s) in $start_ids and (e:RegressionKind52 or e:RegressionKind53) return r, e +-- cypher_params: {"start_ids":[101]} +-- pgsql_params:{"pi0":[101]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = any (@pi0::float8[])) and n0.id = e0.start_id join node n1 on ((n1.kind_ids operator (pg_catalog.@>) array [84]::int2[] or n1.kind_ids operator (pg_catalog.@>) array [85]::int2[])) and n1.id = e0.end_id where e0.kind_id = any (array [83]::int2[])) select s0.e0 as r, s0.n1 as e from s0; + +-- case: match (s)-[r:RegressionKind54]->(e) where id(s) = $start_id and id(e) in $end_ids return r, e +-- cypher_params: {"end_ids":[202,303],"start_id":101} +-- pgsql_params:{"pi0":101,"pi1":[202,303]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = @pi0::float8) and n0.id = e0.start_id join node n1 on (n1.id = any (@pi1::float8[])) and n1.id = e0.end_id where e0.kind_id = any (array [86]::int2[])) select s0.e0 as r, s0.n1 as e from s0; + +-- case: match (s)-[r:RegressionKind55]->(e) where id(s) = $start_id and e.enabled = $enabled and e.score = $score and e.name = $name and e.isassignabletorole = $role_value return r, e +-- cypher_params: {"enabled":true,"name":"target","role_value":"true","score":7,"start_id":101} +-- pgsql_params:{"pi0":101,"pi1":true,"pi2":7,"pi3":"target","pi4":"true"} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on (((n1.properties -> 'enabled'))::jsonb = to_jsonb((@pi1::bool)::bool)::jsonb and ((n1.properties -> 'score'))::jsonb = to_jsonb((@pi2::float8)::float8)::jsonb and (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = @pi3::text) and (jsonb_typeof((n1.properties -> 'isassignabletorole')) = 'string' and (n1.properties ->> 'isassignabletorole') = @pi4::text)) and n1.id = e0.end_id join node n0 on (n0.id = @pi0::float8) and n0.id = e0.start_id where e0.kind_id = any (array [87]::int2[])) select s0.e0 as r, s0.n1 as e from s0; + +-- case: match (s)-[r:RegressionKind56]->(e:RegressionKind57) where id(s) = $start_id and ((e.requiresmanagerapproval = false and e.schemaversion > 1 and e.authorizedsignatures = 0 and e.authenticationenabled = true) or (e.requiresmanagerapproval = false and e.schemaversion = 1 and e.authenticationenabled = true)) return r, e +-- cypher_params: {"start_id":101} +-- pgsql_params:{"pi0":101} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = @pi0::float8) and n0.id = e0.start_id join node n1 on (((((n1.properties -> 'requiresmanagerapproval'))::jsonb = to_jsonb((false)::bool)::jsonb and ((n1.properties ->> 'schemaversion'))::int8 > 1 and ((n1.properties -> 'authorizedsignatures'))::jsonb = to_jsonb((0)::int8)::jsonb and ((n1.properties -> 'authenticationenabled'))::jsonb = to_jsonb((true)::bool)::jsonb) or (((n1.properties -> 'requiresmanagerapproval'))::jsonb = to_jsonb((false)::bool)::jsonb and ((n1.properties -> 'schemaversion'))::jsonb = to_jsonb((1)::int8)::jsonb and ((n1.properties -> 'authenticationenabled'))::jsonb = to_jsonb((true)::bool)::jsonb))) and n1.kind_ids operator (pg_catalog.@>) array [89]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [88]::int2[])) select s0.e0 as r, s0.n1 as e from s0; + +-- case: match (s)-[r:RegressionKind58]->(e) where id(s) = $start_id and (e.schannelauthenticationenabled = true or size(e.effectiveekus) = 0 or $eku in e.effectiveekus) return r, e +-- cypher_params: {"eku":"1.3.6.1.5.5.7.3.2","start_id":101} +-- pgsql_params:{"pi0":101,"pi1":"1.3.6.1.5.5.7.3.2"} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = @pi0::float8) and n0.id = e0.start_id join node n1 on ((((n1.properties -> 'schannelauthenticationenabled'))::jsonb = to_jsonb((true)::bool)::jsonb or case when jsonb_typeof((n1.properties -> 'effectiveekus')) = 'array' then jsonb_array_length((n1.properties -> 'effectiveekus'))::int else null end = 0 or @pi1::text = any (jsonb_to_text_array((n1.properties -> 'effectiveekus'))::text[]))) and n1.id = e0.end_id where e0.kind_id = any (array [90]::int2[])) select s0.e0 as r, s0.n1 as e from s0; + +-- case: match (s)-[r:RegressionKind59]->(e) where id(s) in $start_ids and id(e) in $end_ids return r, e +-- cypher_params: {"end_ids":[303,404],"start_ids":[101,202]} +-- pgsql_params:{"pi0":[101,202],"pi1":[303,404]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = any (@pi0::float8[])) and n0.id = e0.start_id join node n1 on (n1.id = any (@pi1::float8[])) and n1.id = e0.end_id where e0.kind_id = any (array [91]::int2[])) select s0.e0 as r, s0.n1 as e from s0; + +-- case: match (s)-[r:RegressionKind60]->(e:RegressionKind52) where id(s) in $start_ids and e.active = true return r, e +-- cypher_params: {"start_ids":[101]} +-- pgsql_params:{"pi0":[101]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = any (@pi0::float8[])) and n0.id = e0.start_id join node n1 on (((n1.properties -> 'active'))::jsonb = to_jsonb((true)::bool)::jsonb) and n1.kind_ids operator (pg_catalog.@>) array [84]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [92]::int2[])) select s0.e0 as r, s0.n1 as e from s0; + +-- case: match (s:RegressionKind51)-[r:RegressionKind60]->(e) where id(e) in $end_ids and s.active = true return r, s +-- cypher_params: {"end_ids":[202]} +-- pgsql_params:{"pi0":[202]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n1 on (n1.id = any (@pi0::float8[])) and n1.id = e0.end_id join node n0 on (((n0.properties -> 'active'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [83]::int2[] and n0.id = e0.start_id where e0.kind_id = any (array [92]::int2[])) select s0.e0 as r, s0.n0 as s from s0; + +-- case: match (s)-[r:RegressionKind60]->(e) where id(e) in $end_ids return s +-- cypher_params: {"end_ids":[202]} +-- pgsql_params:{"pi0":[202]} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n1 on (n1.id = any (@pi0::float8[])) and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [92]::int2[])) select s0.n0 as s from s0; + +-- case: match (s)-[r:RegressionKind60]->(e) where id(s) in $start_ids return id(e), r +-- cypher_params: {"start_ids":[101]} +-- pgsql_params:{"pi0":[101]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, n1.id as n1 from edge e0 join node n0 on (n0.id = any (@pi0::float8[])) and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [92]::int2[])) select s0.n1 as "id(e)", s0.e0 as r from s0; + -- case: match (s)-[r]->(e) where id(e) = $a and not (id(s) = $b) and (r:EdgeKind1 or r:EdgeKind2) and not (s.objectid ends with $c or e.objectid ends with $d) return distinct id(s), id(r), id(e) -- cypher_params: {"a":1,"b":2,"c":"123","d":"456"} -- pgsql_params:{"pi0":1,"pi1":2,"pi2":"123","pi3":"456"} -with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on n1.id = e0.end_id join node n0 on (not (n0.id = @pi1::float8)) and n0.id = e0.start_id where ((e0.kind_id = any (array [3]::int2[]) or e0.kind_id = any (array [4]::int2[]))) and (not (cypher_ends_with((n0.properties ->> 'objectid'), (@pi2::text)::text)::bool or cypher_ends_with((n1.properties ->> 'objectid'), (@pi3::text)::text)::bool) and n1.id = @pi0::float8)) select distinct (s0.n0).id, (s0.e0).id, (s0.n1).id from s0; +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on n1.id = e0.end_id join node n0 on (not (n0.id = @pi1::float8)) and n0.id = e0.start_id where ((e0.kind_id = any (array [3]::int2[]) or e0.kind_id = any (array [4]::int2[]))) and (not (cypher_ends_with((n0.properties ->> 'objectid'), (@pi2::text)::text)::bool or cypher_ends_with((n1.properties ->> 'objectid'), (@pi3::text)::text)::bool) and n1.id = @pi0::float8)) select distinct (s0.n0).id as "id(s)", (s0.e0).id as "id(r)", (s0.n1).id as "id(e)" from s0; -- case: match (s)-[r]->(e) where s.name = '123' and e:NodeKind1 and not r.property return s, r, e with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = '123')) and n0.id = e0.start_id join node n1 on (n1.kind_ids operator (pg_catalog.@>) array [1]::int2[]) and n1.id = e0.end_id where (not ((e0.properties ->> 'property'))::bool)) select s0.n0 as s, s0.e0 as r, s0.n1 as e from s0; @@ -89,22 +194,22 @@ with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::e with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.end_id join node n1 on n1.id = e0.start_id), s1 as (select s0.e0 as e0, (e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties)::edgecomposite as e1, s0.n1 as n1 from s0 join edge e1 on (s0.n1).id = e1.end_id join node n2 on n2.id = e1.start_id where e1.id != (s0.e0).id) select s1.e0 as e0, s1.n1 as n, s1.e1 as e1 from s1; -- case: match (s)<-[r:EdgeKind1|EdgeKind2]-(e) return s.name, e.name -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.end_id join node n1 on n1.id = e0.start_id where e0.kind_id = any (array [3, 4]::int2[])) select ((s0.n0).properties -> 'name'), ((s0.n1).properties -> 'name') from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.end_id join node n1 on n1.id = e0.start_id where e0.kind_id = any (array [3, 4]::int2[])) select ((s0.n0).properties -> 'name') as "s.name", ((s0.n1).properties -> 'name') as "e.name" from s0; -- case: match (s)-[:EdgeKind1|EdgeKind2]->(e)-[:EdgeKind1]->() return s.name as s_name, e.name as e_name with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[])), s1 as (select s0.e0 as e0, s0.n0 as n0, s0.n1 as n1 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.kind_id = any (array [3]::int2[]) and e1.id != s0.e0) select ((s1.n0).properties -> 'name') as s_name, ((s1.n1).properties -> 'name') as e_name from s1; -- case: match (s:NodeKind1)-[r:EdgeKind1|EdgeKind2]->(e:NodeKind2) return s.name, e.name -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[])) select ((s0.n0).properties -> 'name'), ((s0.n1).properties -> 'name') from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[])) select ((s0.n0).properties -> 'name') as "s.name", ((s0.n1).properties -> 'name') as "e.name" from s0; -- case: match (s)-[r:EdgeKind1]->() where (s)-[r {prop: 'a'}]->() return s with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where (jsonb_typeof((e0.properties -> 'prop')) = 'string' and (e0.properties ->> 'prop') = 'a') and e0.kind_id = any (array [3]::int2[])) select s0.n0 as s from s0 where ((with s1 as (select s0.e0 as e0, s0.n0 as n0 from edge e0 join node n2 on n2.id = (s0.e0).end_id where (s0.n0).id = (s0.e0).start_id) select count(*) > 0 from s1)); -- case: match (s)-[r:EdgeKind1]->(e) where not (s.system_tags contains 'admin_tier_0') and id(e) = 1 return id(s), labels(s), id(r), type(r) -with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on (n1.id = 1) and n1.id = e0.end_id join node n0 on (not (coalesce((n0.properties ->> 'system_tags'), '')::text like '%admin\_tier\_0%')) and n0.id = e0.start_id where e0.kind_id = any (array [3]::int2[])) select (s0.n0).id, (array(select _kind.name from generate_subscripts((s0.n0).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s0.n0).kind_ids)[_kind_idx] order by _kind_idx))::text[], (s0.e0).id, kind_name((s0.e0).kind_id)::text from s0; +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n1 on (n1.id = 1) and n1.id = e0.end_id join node n0 on (not (coalesce((n0.properties ->> 'system_tags'), '')::text like '%admin\_tier\_0%')) and n0.id = e0.start_id where e0.kind_id = any (array [3]::int2[])) select (s0.n0).id as "id(s)", (array(select _kind.name from generate_subscripts((s0.n0).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s0.n0).kind_ids)[_kind_idx] order by _kind_idx))::text[] as "labels(s)", (s0.e0).id as "id(r)", kind_name((s0.e0).kind_id)::text as "type(r)" from s0; -- case: match (s)-[r]->(e) where s:NodeKind1 and toLower(s.name) starts with 'test' and r:EdgeKind1 and id(e) in [1, 2] return r limit 1 -with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and lower((n0.properties ->> 'name'))::text like 'test%') and n0.id = e0.start_id join node n1 on (n1.id = any (array [1, 2]::int8[])) and n1.id = e0.end_id where (e0.kind_id = any (array [3]::int2[])) limit 1) select s0.e0 as r from s0 limit 1; +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n0 on (n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and lower((n0.properties ->> 'name'))::text like 'test%') and n0.id = e0.start_id join node n1 on (n1.id = any (array [1, 2]::int8[])) and n1.id = e0.end_id where (e0.kind_id = any (array [3]::int2[])) limit 1) select s0.e0 as r from s0 limit 1; -- case: match (n1)-[]->(n2) where n1 <> n2 return n2 with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where (n0.id <> n1.id)) select s0.n1 as n2 from s0; @@ -113,8 +218,8 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1 with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where (n1.id <> n0.id)) select s0.n1 as n2 from s0; -- case: match ()-[r]->()-[e]->(n) where r <> e return n -with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, (e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties)::edgecomposite as e1, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where ((s0.e0).id <> e1.id) and e1.id != (s0.e0).id) select s1.n2 as n from s1; +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n1.id as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, (e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties)::edgecomposite as e1, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on s0.n1 = e1.start_id join node n2 on n2.id = e1.end_id where ((s0.e0).id <> e1.id) and e1.id != (s0.e0).id) select s1.n2 as n from s1; -- case: match (s:NodeKind1:NodeKind2)-[r:EdgeKind1|EdgeKind2]->(e:NodeKind2:NodeKind1) return s.name, e.name -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1, 2]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2, 1]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[])) select ((s0.n0).properties -> 'name'), ((s0.n1).properties -> 'name') from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1, 2]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2, 1]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[])) select ((s0.n0).properties -> 'name') as "s.name", ((s0.n1).properties -> 'name') as "e.name" from s0; diff --git a/cypher/models/pgsql/test/translation_cases/unwind.sql b/cypher/models/pgsql/test/translation_cases/unwind.sql index 4c00ab6e..c7cccd4f 100644 --- a/cypher/models/pgsql/test/translation_cases/unwind.sql +++ b/cypher/models/pgsql/test/translation_cases/unwind.sql @@ -33,7 +33,7 @@ with s0 as (select array [1, 2, 3]::int8[] as i0) select i1 as x from s0, unnest with s0 as (select array [1, 2, 3, 1, 2]::int8[] as i0) select distinct i1 as x from s0, unnest(i0) as i1; -- case: with [1, 2, 3] as ids unwind ids as x return count(x) -with s0 as (select array [1, 2, 3]::int8[] as i0) select count(i1)::int8 from s0, unnest(i0) as i1; +with s0 as (select array [1, 2, 3]::int8[] as i0) select count(i1)::int8 as "count(x)" from s0, unnest(i0) as i1; -- case: match (n:NodeKind1) with collect(n.name) as names unwind names as name match (m:NodeKind2) where m.name = name return m with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select array_remove(coalesce(array_agg(((s1.n0).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray as i0 from s1), s2 as (select s0.i0 as i0, i1 as i1, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, unnest(i0) as i1, node n1 where ((n1.properties ->> 'name') = i1) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]) select s2.n1 as m from s2; diff --git a/cypher/models/pgsql/test/translation_cases/update.sql b/cypher/models/pgsql/test/translation_cases/update.sql index 7663e030..28ff09dc 100644 --- a/cypher/models/pgsql/test/translation_cases/update.sql +++ b/cypher/models/pgsql/test/translation_cases/update.sql @@ -39,7 +39,7 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = '1234'))), s1 as (update node n1 set properties = n1.properties || jsonb_build_object('is_target', true)::jsonb from s0 where (s0.n0).id = n1.id returning (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n0) select 1; -- case: match (n) where n.name = '1234' match (e) where e.tag = n.tag_id set e.is_target = true -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = '1234'))), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where ((n1.properties -> 'tag') = ((s0.n0).properties -> 'tag_id'))), s2 as (update node n2 set properties = n2.properties || jsonb_build_object('is_target', true)::jsonb from s1 where (s1.n1).id = n2.id returning s1.n0 as n0, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n1) select 1; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = '1234'))), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where (nullif((n1.properties -> 'tag'), ('null')::jsonb)::jsonb = nullif(((s0.n0).properties -> 'tag_id'), ('null')::jsonb)::jsonb)), s2 as (update node n2 set properties = n2.properties || jsonb_build_object('is_target', true)::jsonb from s1 where (s1.n1).id = n2.id returning s1.n0 as n0, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n1) select 1; -- case: match (n1), (n3) set n1.target = true set n3.target = true return n1, n3 with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1), s2 as (update node n2 set properties = n2.properties || jsonb_build_object('target', true)::jsonb from s1 where (s1.n0).id = n2.id returning (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n0, s1.n1 as n1), s3 as (update node n3 set properties = n3.properties || jsonb_build_object('target', true)::jsonb from s2 where (s2.n1).id = n3.id returning s2.n0 as n0, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n1) select s3.n0 as n1, s3.n1 as n3 from s3; @@ -69,5 +69,5 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from edge e0 join node n0 on (n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[])), s1 as (update edge e1 set properties = e1.properties || jsonb_build_object('visited', true)::jsonb from s0 where (s0.e0).id = e1.id returning (e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties)::edgecomposite as e0, s0.n0 as n0) select s1.e0 as r from s1; -- case: match (n)-[]->()-[r]->() where n.name = 'n1' set r.visited = true return r.name -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n1')) and n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, (e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties)::edgecomposite as e1, s0.n0 as n0, s0.n1 as n1 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != s0.e0), s2 as (update edge e2 set properties = e2.properties || jsonb_build_object('visited', true)::jsonb from s1 where (s1.e1).id = e2.id returning s1.e0 as e0, (e2.id, e2.start_id, e2.end_id, e2.kind_id, e2.properties)::edgecomposite as e1, s1.n0 as n0, s1.n1 as n1) select ((s2.e1).properties -> 'name') from s2; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n0 on ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n1')) and n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, (e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties)::edgecomposite as e1, s0.n0 as n0, s0.n1 as n1 from s0 join edge e1 on s0.n1 = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != s0.e0), s2 as (update edge e2 set properties = e2.properties || jsonb_build_object('visited', true)::jsonb from s1 where (s1.e1).id = e2.id returning s1.e0 as e0, (e2.id, e2.start_id, e2.end_id, e2.kind_id, e2.properties)::edgecomposite as e1, s1.n0 as n0, s1.n1 as n1) select ((s2.e1).properties -> 'name') as "r.name" from s2; diff --git a/cypher/models/pgsql/test/translation_test.go b/cypher/models/pgsql/test/translation_test.go index 12033919..0f7f33c2 100644 --- a/cypher/models/pgsql/test/translation_test.go +++ b/cypher/models/pgsql/test/translation_test.go @@ -12,6 +12,7 @@ import ( "github.com/specterops/dawgs/graph" ) +// translationTestKinds returns the stable kind set and numeric IDs used by translation fixtures. func translationTestKinds() graph.Kinds { // Keep this order stable. Translation case SQL fixtures depend on these IDs. return graph.Kinds{ @@ -48,9 +49,113 @@ func translationTestKinds() graph.Kinds { "WriteAccountRestrictions", "WriteOwner", "AZUser", + // Synthetic reconciliation kinds are append-only. The first 9 and all 30 + // are used by cardinality-sensitive golden cases without renumbering any + // established kind IDs above. + "RegressionKind01", + "RegressionKind02", + "RegressionKind03", + "RegressionKind04", + "RegressionKind05", + "RegressionKind06", + "RegressionKind07", + "RegressionKind08", + "RegressionKind09", + "RegressionKind10", + "RegressionKind11", + "RegressionKind12", + "RegressionKind13", + "RegressionKind14", + "RegressionKind15", + "RegressionKind16", + "RegressionKind17", + "RegressionKind18", + "RegressionKind19", + "RegressionKind20", + "RegressionKind21", + "RegressionKind22", + "RegressionKind23", + "RegressionKind24", + "RegressionKind25", + "RegressionKind26", + "RegressionKind27", + "RegressionKind28", + "RegressionKind29", + "RegressionKind30", + "RegressionKind31", + "RegressionKind32", + "RegressionKind33", + "RegressionKind34", + "RegressionKind35", + "RegressionKind36", + "RegressionKind37", + "RegressionKind38", + "RegressionKind39", + "RegressionKind40", + "RegressionKind41", + "RegressionKind42", + "RegressionKind43", + "RegressionKind44", + "RegressionKind45", + "RegressionKind46", + "RegressionKind47", + "RegressionKind48", + "RegressionKind49", + "RegressionKind50", + "RegressionKind51", + "RegressionKind52", + "RegressionKind53", + "RegressionKind54", + "RegressionKind55", + "RegressionKind56", + "RegressionKind57", + "RegressionKind58", + "RegressionKind59", + "RegressionKind60", + "RegressionKind61", + "RegressionKind62", + "RegressionKind63", + "RegressionKind64", + "RegressionKind65", + "RegressionKind66", + "RegressionKind67", + "RegressionKind68", + "RegressionKind69", + "RegressionKind70", + "RegressionKind71", + "RegressionKind72", + "RegressionKind73", + "RegressionKind74", + "RegressionKind75", + "RegressionKind76", + "RegressionKind77", + "RegressionKind78", + "RegressionKind79", + "RegressionKind80", + "RegressionKind81", + "RegressionKind82", + "RegressionKind83", + "RegressionKind84", + "RegressionKind85", + "RegressionKind86", + "RegressionKind87", + "RegressionKind88", + "RegressionKind89", + "RegressionKind90", + "RegressionKind91", + "RegressionKind92", + "RegressionKind93", + "RegressionKind94", + "RegressionKind95", + "RegressionKind96", + "RegressionKind97", + "RegressionKind98", + "RegressionKind99", + "RegressionKind100", })...) } +// newKindMapper returns a mapper populated with the translation fixture's deterministic kind IDs. func newKindMapper() pgsql.KindMapper { mapper := pgutil.NewInMemoryKindMapper() diff --git a/cypher/models/pgsql/test/trust_pruning_forms_legacy_builder_test.go b/cypher/models/pgsql/test/trust_pruning_forms_legacy_builder_test.go new file mode 100644 index 00000000..e8a7a705 --- /dev/null +++ b/cypher/models/pgsql/test/trust_pruning_forms_legacy_builder_test.go @@ -0,0 +1,170 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package test + +import ( + "testing" + "time" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/query" + "github.com/stretchr/testify/require" +) + +// TestLegacyBuilderPostgreSQL_TrustAndPruningForms verifies migrated trust filters and pruning projections retain their SQL contracts. +func TestLegacyBuilderPostgreSQL_TrustAndPruningForms(t *testing.T) { + threshold := time.Date(2026, time.January, 3, 0, 0, 0, 0, time.UTC) + + testCases := map[string]struct { + // criteria contains the legacy query-builder inputs for the case. + criteria []graph.Criteria + // fragments lists SQL fragments that the translation must contain. + fragments []string + // parameters is the exact parameter map expected from translation. + parameters map[string]any + }{ + "TRUST-01 SameForestTrust ID projection": { + criteria: trustPruningCriteria("RegressionKind40", "RegressionKind41", query.RelationshipID()), + fragments: []string{ + "n0.kind_ids operator (pg_catalog.&&) array [72]::int2[]", + "n1.kind_ids operator (pg_catalog.&&) array [72]::int2[]", + "e0.kind_id = any (array [73]::int2[])", + "e0.properties -> 'lastseen'", + "n0.properties -> 'lastcollected'", + "n1.properties -> 'lastcollected'", + "select (s0.e0).id", + }, + parameters: map[string]any{}, + }, + "TRUST-02 CrossForestTrust full projection": { + criteria: trustPruningCriteria("RegressionKind40", "RegressionKind42", query.Relationship()), + fragments: []string{ + "e0.kind_id = any (array [74]::int2[])", + "select s0.e0 as r", + }, + parameters: map[string]any{}, + }, + "TRUST-03 branch-local derived trust kinds": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.Kind(query.Start(), graph.StringKind("RegressionKind40")), + query.Kind(query.End(), graph.StringKind("RegressionKind40")), + query.Or( + query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Equals(query.EndID(), graph.ID(202)), + query.KindIn(query.Relationship(), graph.StringKind("RegressionKind43")), + ), + query.And( + query.Equals(query.StartID(), graph.ID(202)), + query.Equals(query.EndID(), graph.ID(101)), + query.KindIn(query.Relationship(), graph.StringKind("RegressionKind44")), + ), + ), + )), + query.Returning(query.RelationshipID()), + }, + fragments: []string{ + " or ", + "n0.id = @pi0", + "n1.id = @pi1", + "n0.id = @pi2", + "n1.id = @pi3", + "e0.kind_id = any (array [75]::int2[])", + "e0.kind_id = any (array [76]::int2[])", + }, + parameters: map[string]any{"pi0": uint64(101), "pi1": uint64(202), "pi2": uint64(202), "pi3": uint64(101)}, + }, + "PRUNE-01 relationship TTL": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.Not(query.KindIn(query.Relationship(), graph.StringKind("RegressionKind45"), graph.StringKind("RegressionKind46"))), + query.Before(query.RelationshipProperty("lastseen"), threshold), + )), + query.Returning(query.RelationshipID()), + }, + fragments: []string{"not (e0.kind_id = any (array [77, 78]::int2[]))", "e0.properties ->> 'lastseen'", "select (s0.e0).id"}, + parameters: map[string]any{"pi0": threshold}, + }, + "PRUNE-02 HasSession TTL": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.KindIn(query.Relationship(), graph.StringKind("HasSession")), + query.Or( + query.Not(query.Exists(query.RelationshipProperty("lastseen"))), + query.Before(query.RelationshipProperty("lastseen"), threshold), + ), + )), + query.Returning(query.RelationshipID()), + }, + fragments: []string{"not ((e0.properties ? 'lastseen'", " or ", "e0.kind_id = any (array [7]::int2[])"}, + parameters: map[string]any{"pi0": threshold}, + }, + "PRUNE-03 node TTL": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.Not(query.KindIn(query.Node(), graph.StringKind("RegressionKind48"), graph.StringKind("RegressionKind49"))), + query.Or( + query.Not(query.Exists(query.NodeProperty("lastseen"))), + query.Before(query.NodeProperty("lastseen"), threshold), + ), + )), + query.Returning(query.NodeID()), + }, + fragments: []string{"not (n0.kind_ids operator (pg_catalog.&&) array [80, 81]::int2[])", "not ((n0.properties ? 'lastseen'", "select (s0.n0).id"}, + parameters: map[string]any{"pi0": threshold}, + }, + "PRUNE-04 orphan SID prefix": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.Not(query.KindIn(query.Node(), graph.StringKind("RegressionKind48"), graph.StringKind("RegressionKind49"))), + query.Not(query.Exists(query.NodeProperty("name"))), + query.StringStartsWith(query.NodeProperty("objectid"), "S-1-5"), + )), + query.Returning(query.NodeID()), + }, + fragments: []string{"not ((n0.properties ? 'name'", "cypher_starts_with", "select (s0.n0).id"}, + parameters: map[string]any{"pi0": "S-1-5"}, + }, + } + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + formatted, translation := translateLegacyQuery(t, testCase.criteria...) + for _, fragment := range testCase.fragments { + require.Contains(t, formatted, fragment) + } + require.Equal(t, testCase.parameters, translation.Parameters) + }) + } +} + +// trustPruningCriteria builds the shared trust-kind and timestamp predicate used by pruning regression cases. +func trustPruningCriteria(domainKind, relationshipKind string, projection graph.Criteria) []graph.Criteria { + return []graph.Criteria{ + query.Where(query.And( + query.Kind(query.Start(), graph.StringKind(domainKind)), + query.Kind(query.End(), graph.StringKind(domainKind)), + query.Kind(query.Relationship(), graph.StringKind(relationshipKind)), + query.Or( + query.BeforeGraphQuery(query.RelationshipProperty("lastseen"), query.StartProperty("lastcollected")), + query.BeforeGraphQuery(query.RelationshipProperty("lastseen"), query.EndProperty("lastcollected")), + ), + )), + query.Returning(projection), + } +} diff --git a/cypher/models/pgsql/translate/expansion.go b/cypher/models/pgsql/translate/expansion.go index c7d27587..1e50d578 100644 --- a/cypher/models/pgsql/translate/expansion.go +++ b/cypher/models/pgsql/translate/expansion.go @@ -3,6 +3,7 @@ package translate import ( "errors" "fmt" + "strings" "github.com/specterops/dawgs/cypher/models" "github.com/specterops/dawgs/cypher/models/pgsql" @@ -12,18 +13,33 @@ import ( "github.com/specterops/dawgs/graph" ) +// translateDefaultMaxTraversalDepth caps unbounded recursive traversals to prevent runaway expansion. const translateDefaultMaxTraversalDepth int64 = 15 var ( - expansionRootFilter = pgsql.Identifier("traversal_root_filter") - expansionTerminalFilter = pgsql.Identifier("traversal_terminal_filter") - expansionPairFilter = pgsql.Identifier("traversal_pair_filter") - expansionTerminalID = pgsql.Identifier("terminal_id") - expansionVisited = pgsql.Identifier("visited") - expansionForwardVisited = pgsql.Identifier("forward_visited") + // expansionRootFilter names the CTE that materializes admissible traversal roots. + expansionRootFilter = pgsql.Identifier("traversal_root_filter") + + // expansionTerminalFilter names the CTE that materializes admissible traversal terminals. + expansionTerminalFilter = pgsql.Identifier("traversal_terminal_filter") + + // expansionPairFilter names the CTE that materializes admissible root-terminal pairs. + expansionPairFilter = pgsql.Identifier("traversal_pair_filter") + + // expansionTerminalID names the filtered terminal-ID column. + expansionTerminalID = pgsql.Identifier("terminal_id") + + // expansionVisited names the relation that records states visited by shortest-path search. + expansionVisited = pgsql.Identifier("visited") + + // expansionForwardVisited names states visited from the root side of bidirectional search. + expansionForwardVisited = pgsql.Identifier("forward_visited") + + // expansionBackwardVisited names states visited from the terminal side of bidirectional search. expansionBackwardVisited = pgsql.Identifier("backward_visited") ) +// expansionEdgeJoinCondition matches the current node to the start of the next directed edge. func expansionEdgeJoinCondition(traversalStep *TraversalStep) (pgsql.Expression, error) { return pgd.Equals( pgd.EntityID(traversalStep.LeftNode.Identifier), @@ -31,6 +47,7 @@ func expansionEdgeJoinCondition(traversalStep *TraversalStep) (pgsql.Expression, ), nil } +// expansionConstraints limits recursion by maximum depth and rejects cyclic expansion states. func expansionConstraints(traversalStep *TraversalStep) pgsql.Expression { expansionModel := traversalStep.Expansion @@ -45,41 +62,66 @@ func expansionConstraints(traversalStep *TraversalStep) pgsql.Expression { ) } -var ( - ErrUnsupportedExpansionDirection = errors.New("unsupported expansion direction") -) +// ErrUnsupportedExpansionDirection reports a traversal direction that cannot be lowered to SQL. +var ErrUnsupportedExpansionDirection = errors.New("unsupported expansion direction") +// ExpansionBuilder assembles the seed, recursive, and projection statements for one traversal expansion. type ExpansionBuilder struct { - PrimerStatement pgsql.Select - RecursiveStatement pgsql.Select + // PrimerStatement produces the first traversal edge for each root. + PrimerStatement pgsql.Select + + // RecursiveStatement advances each eligible expansion state by one edge. + RecursiveStatement pgsql.Select + + // ProjectionStatement converts internal expansion state into the requested result shape. ProjectionStatement pgsql.Select - ZeroDepthStatement *pgsql.Select - UseUnionAll bool + // ZeroDepthStatement produces empty-path rows when the traversal admits depth zero. + ZeroDepthStatement *pgsql.Select + + // UseUnionAll controls whether recursive branches retain duplicate states. + UseUnionAll bool + + // queryParameters contains literal values lifted while constructing harness calls. queryParameters map[string]any - traversalStep *TraversalStep - model *Expansion - unwindClauses []UnwindClause - unwindSources []pgsql.FromClause + + // graphID identifies the graph partitions referenced by generated traversal SQL. + graphID int32 + + // traversalStep describes the edge, endpoints, direction, and constraints being expanded. + traversalStep *TraversalStep + + // model contains the frame and search options shared by the generated statements. + model *Expansion + + // unwindClauses contains active UNWIND bindings that expansion predicates may reference. + unwindClauses []UnwindClause + + // unwindSources caches the SQL sources corresponding to unwindClauses. + unwindSources []pgsql.FromClause } -func NewExpansionBuilder(queryParameters map[string]any, traversalStep *TraversalStep) (*ExpansionBuilder, error) { +// NewExpansionBuilder validates traversal expansion state and constructs its SQL builder. +func NewExpansionBuilder(queryParameters map[string]any, traversalStep *TraversalStep, graphID int32) (*ExpansionBuilder, error) { if traversalStep.Expansion == nil { return nil, errors.New("traversal step must have expansion set") } return &ExpansionBuilder{ queryParameters: queryParameters, + graphID: graphID, traversalStep: traversalStep, model: traversalStep.Expansion, }, nil } +// SetUnwindClauses registers the active UNWIND bindings and their SQL sources for expansion queries. func (s *ExpansionBuilder) SetUnwindClauses(clauses []UnwindClause) { s.unwindClauses = clauses s.unwindSources = unwindFromClauses(clauses) } +// nextFrontInsert wraps a frontier-producing expression in an insert into the next-front workspace. func nextFrontInsert(body pgsql.SetExpression) pgsql.Insert { return pgsql.Insert{ Table: pgsql.TableReference{ @@ -92,6 +134,7 @@ func nextFrontInsert(body pgsql.SetExpression) pgsql.Insert { } } +// expansionNodeTableReference aliases the graph node table for an expansion binding. func expansionNodeTableReference(binding pgsql.Identifier) pgsql.TableReference { return pgsql.TableReference{ Name: pgsql.TableNode.AsCompoundIdentifier(), @@ -99,6 +142,7 @@ func expansionNodeTableReference(binding pgsql.Identifier) pgsql.TableReference } } +// expansionEdgeTableReference aliases the graph edge table for an expansion binding. func expansionEdgeTableReference(binding pgsql.Identifier) pgsql.TableReference { return pgsql.TableReference{ Name: pgsql.TableEdge.AsCompoundIdentifier(), @@ -106,21 +150,28 @@ func expansionEdgeTableReference(binding pgsql.Identifier) pgsql.TableReference } } +// expansionSeed describes the query and record shape that supply traversal root identifiers. type expansionSeed struct { + // identifier names the seed common table expression. identifier pgsql.Identifier - query pgsql.Select + + // query selects the root identifiers supplied to the expansion. + query pgsql.Select } +// expansionSeedIdentifier derives the CTE name reserved for an expansion's seed rows. func expansionSeedIdentifier(expansionIdentifier pgsql.Identifier) pgsql.Identifier { return pgsql.Identifier(string(expansionIdentifier) + "_seed") } +// expansionSeedColumns returns the single root-identifier column emitted by every seed query. func expansionSeedColumns() *pgsql.RecordShape { return pgsql.NewRecordShape([]pgsql.Identifier{ expansionRootID, }) } +// newExpansionSeed builds a seed query that projects a root expression from the supplied sources and predicate. func newExpansionSeed(identifier pgsql.Identifier, rootExpression pgsql.Expression, from []pgsql.FromClause, where pgsql.Expression) expansionSeed { return expansionSeed{ identifier: identifier, @@ -137,12 +188,14 @@ func newExpansionSeed(identifier pgsql.Identifier, rootExpression pgsql.Expressi } } +// newExpansionNodeSeed builds a seed by scanning candidate root nodes under the supplied constraints. func newExpansionNodeSeed(identifier, nodeIdentifier pgsql.Identifier, constraints pgsql.Expression) expansionSeed { return newExpansionSeed(identifier, pgd.EntityID(nodeIdentifier), []pgsql.FromClause{{ Source: expansionNodeTableReference(nodeIdentifier), }}, constraints) } +// newExpansionNodeFilterSeed reads root identifiers from a materialized filter and joins nodes when constraints require hydration. func newExpansionNodeFilterSeed(identifier, filterIdentifier, nodeIdentifier pgsql.Identifier, constraints pgsql.Expression) expansionSeed { var ( filterAlias = pgsql.Identifier(string(identifier) + "_filter") @@ -179,14 +232,9 @@ func newExpansionNodeFilterSeed(identifier, filterIdentifier, nodeIdentifier pgs return seed } -func newExpansionBoundNodeSeed(identifier pgsql.Identifier, previousFrame *Frame, nodeIdentifier pgsql.Identifier, constraints pgsql.Expression) expansionSeed { - seed := newExpansionSeed(identifier, pgsql.RowColumnReference{ - Identifier: pgsql.CompoundIdentifier{ - previousFrame.Binding.Identifier, - nodeIdentifier, - }, - Column: pgsql.ColumnID, - }, []pgsql.FromClause{{ +// newExpansionBoundNodeSeed projects distinct bound-node identifiers from the preceding frame. +func newExpansionBoundNodeSeed(identifier pgsql.Identifier, previousFrame *Frame, binding *BoundIdentifier, constraints pgsql.Expression) expansionSeed { + seed := newExpansionSeed(identifier, boundEndpointIDReference(previousFrame, binding), []pgsql.FromClause{{ Source: pgsql.TableReference{ Name: pgsql.CompoundIdentifier{previousFrame.Binding.Identifier}, }, @@ -196,6 +244,7 @@ func newExpansionBoundNodeSeed(identifier pgsql.Identifier, previousFrame *Frame return seed } +// fromClausesContainSource reports whether a FROM list directly names the requested table source. func fromClausesContainSource(fromClauses []pgsql.FromClause, identifier pgsql.Identifier) bool { for _, fromClause := range fromClauses { if tableReference, isTableReference := fromClause.Source.(pgsql.TableReference); isTableReference && @@ -208,6 +257,7 @@ func fromClausesContainSource(fromClauses []pgsql.FromClause, identifier pgsql.I return false } +// prependFrameSourceIfMissing ensures the preceding frame is the first source in a FROM list. func prependFrameSourceIfMissing(fromClauses []pgsql.FromClause, frame *Frame) []pgsql.FromClause { if frame == nil || fromClausesContainSource(fromClauses, frame.Binding.Identifier) { return fromClauses @@ -220,6 +270,7 @@ func prependFrameSourceIfMissing(fromClauses []pgsql.FromClause, frame *Frame) [ }}, fromClauses...) } +// expressionReferencesUnwindBinding reports whether an expression depends on any active UNWIND binding. func expressionReferencesUnwindBinding(expression pgsql.Expression, unwindClauses []UnwindClause) (bool, error) { if expression == nil || len(unwindClauses) == 0 { return false, nil @@ -239,6 +290,7 @@ func expressionReferencesUnwindBinding(expression pgsql.Expression, unwindClause return false, nil } +// seedEndpointConstraintSplit rewrites bound endpoint references for the seed and separates local predicates from deferred ones. func (s *ExpansionBuilder) seedEndpointConstraintSplit(expression pgsql.Expression, nodeIdentifier pgsql.Identifier, previousFrameIdentifier pgsql.Identifier) (pgsql.Expression, pgsql.Expression) { var ( seedExpression = rewriteBoundEndpointSeedReference(expression, previousFrameIdentifier, nodeIdentifier) @@ -254,6 +306,7 @@ func (s *ExpansionBuilder) seedEndpointConstraintSplit(expression pgsql.Expressi return partitionConstraintByLocality(seedExpression, localScope) } +// appendUnwindSourcesIfReferenced adds frame and UNWIND sources only when the supplied expressions use an UNWIND binding. func (s *ExpansionBuilder) appendUnwindSourcesIfReferenced(selectBody *pgsql.Select, expressions ...pgsql.Expression) error { for _, expression := range expressions { if referencesUnwind, err := expressionReferencesUnwindBinding(expression, s.unwindClauses); err != nil { @@ -273,18 +326,47 @@ func (s *ExpansionBuilder) appendUnwindSourcesIfReferenced(selectBody *pgsql.Sel return nil } +// appendUnwindSources appends every active UNWIND source to a select body. func (s *ExpansionBuilder) appendUnwindSources(selectBody *pgsql.Select) { selectBody.From = append(selectBody.From, s.unwindSources...) } +// newExpansionRootIDsParameterSeed builds a root seed from the materialized root-identifier parameter. func newExpansionRootIDsParameterSeed(identifier, nodeIdentifier pgsql.Identifier, constraints pgsql.Expression) expansionSeed { return newExpansionNodeFilterSeed(identifier, expansionRootFilter, nodeIdentifier, constraints) } +// newExpansionTerminalIDsParameterSeed builds a root seed from the materialized terminal-identifier parameter. func newExpansionTerminalIDsParameterSeed(identifier, nodeIdentifier pgsql.Identifier, constraints pgsql.Expression) expansionSeed { return newExpansionNodeFilterSeed(identifier, expansionTerminalFilter, nodeIdentifier, constraints) } +// newExpansionArrayParameterSeed unnests an identifier-array parameter and filters the corresponding nodes. +func newExpansionArrayParameterSeed(identifier, nodeIdentifier pgsql.Identifier, constraints pgsql.Expression, parameterPosition int) expansionSeed { + parameterAlias := pgsql.Identifier(string(identifier) + "_parameter") + parameterID := pgsql.CompoundIdentifier{parameterAlias, pgsql.ColumnID} + seed := newExpansionSeed(identifier, pgd.EntityID(nodeIdentifier), []pgsql.FromClause{{ + Source: pgsql.FormattingLiteral(fmt.Sprintf( + "unnest($%d::int8[]) as %s(id)", + parameterPosition, + parameterAlias, + )), + Joins: []pgsql.Join{{ + Table: expansionNodeTableReference(nodeIdentifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgd.Equals( + pgd.EntityID(nodeIdentifier), + parameterID, + ), + }, + }}, + }}, constraints) + seed.query.Distinct = true + return seed +} + +// CTE exposes the seed query as a non-materialized common table expression. func (s expansionSeed) CTE() pgsql.CommonTableExpression { return pgsql.CommonTableExpression{ Alias: pgsql.TableAlias{ @@ -298,10 +380,12 @@ func (s expansionSeed) CTE() pgsql.CommonTableExpression { } } +// rootID returns the qualified root-identifier column of the seed CTE. func (s expansionSeed) rootID() pgsql.CompoundIdentifier { return pgsql.CompoundIdentifier{s.identifier, expansionRootID} } +// fromClause references the seed CTE and attaches the supplied joins. func (s expansionSeed) fromClause(joins ...pgsql.Join) pgsql.FromClause { return pgsql.FromClause{ Source: pgsql.TableReference{ @@ -311,6 +395,7 @@ func (s expansionSeed) fromClause(joins ...pgsql.Join) pgsql.FromClause { } } +// edgeJoin joins a seed root identifier to the starting endpoint of an edge binding. func (s expansionSeed) edgeJoin(edgeIdentifier pgsql.Identifier, edgeStartColumn pgsql.CompoundIdentifier) pgsql.Join { return pgsql.Join{ Table: expansionEdgeTableReference(edgeIdentifier), @@ -321,6 +406,7 @@ func (s expansionSeed) edgeJoin(edgeIdentifier pgsql.Identifier, edgeStartColumn } } +// expansionEdgeFromClause references the graph edge table with the joins needed by an expansion query. func expansionEdgeFromClause(edgeIdentifier pgsql.Identifier, joins ...pgsql.Join) pgsql.FromClause { return pgsql.FromClause{ Source: expansionEdgeTableReference(edgeIdentifier), @@ -328,6 +414,7 @@ func expansionEdgeFromClause(edgeIdentifier pgsql.Identifier, joins ...pgsql.Joi } } +// recursiveExpansionEdgeProjection projects every stored column of the recursively selected edge. func recursiveExpansionEdgeProjection(edgeIdentifier pgsql.Identifier) pgsql.Projection { projection := make(pgsql.Projection, len(pgsql.EdgeTableColumns)) @@ -338,6 +425,7 @@ func recursiveExpansionEdgeProjection(edgeIdentifier pgsql.Identifier) pgsql.Pro return projection } +// expansionEdgeNotInPath rejects an edge identifier already present in the accumulated path. func expansionEdgeNotInPath(edgeIdentifier, frameIdentifier pgsql.Identifier) *pgsql.BinaryExpression { return pgsql.NewBinaryExpression( pgd.EntityID(edgeIdentifier), @@ -348,6 +436,7 @@ func expansionEdgeNotInPath(edgeIdentifier, frameIdentifier pgsql.Identifier) *p ) } +// recursiveExpansionEdgeLookupJoin builds the correlated lateral lookup for unused edges leaving the current frontier node. func recursiveExpansionEdgeLookupJoin(traversalStep *TraversalStep) pgsql.Join { var ( expansionModel = traversalStep.Expansion @@ -386,24 +475,30 @@ func recursiveExpansionEdgeLookupJoin(traversalStep *TraversalStep) pgsql.Join { } } -func expansionNodeProjection(nodeIdentifier pgsql.Identifier) pgsql.Projection { +// expansionNodeProjection projects either a node identifier or the complete node record required by its binding. +func expansionNodeProjection(binding *BoundIdentifier) pgsql.Projection { + if binding.IDOnly { + return pgsql.Projection{pgsql.CompoundIdentifier{binding.Identifier, pgsql.ColumnID}} + } + projection := make(pgsql.Projection, len(pgsql.NodeTableColumns)) for idx, column := range pgsql.NodeTableColumns { - projection[idx] = pgsql.CompoundIdentifier{nodeIdentifier, column} + projection[idx] = pgsql.CompoundIdentifier{binding.Identifier, column} } return projection } -func expansionNodeLookupJoin(nodeIdentifier pgsql.Identifier, nodeID pgsql.Expression) pgsql.Join { +// expansionNodeLookupJoin builds a correlated lateral lookup that hydrates a node by identifier. +func expansionNodeLookupJoin(binding *BoundIdentifier, nodeID pgsql.Expression) pgsql.Join { nodeLookup := pgsql.Select{ - Projection: expansionNodeProjection(nodeIdentifier), + Projection: expansionNodeProjection(binding), From: []pgsql.FromClause{{ - Source: expansionNodeTableReference(nodeIdentifier), + Source: expansionNodeTableReference(binding.Identifier), }}, Where: pgd.Equals( - pgsql.CompoundIdentifier{nodeIdentifier, pgsql.ColumnID}, + pgsql.CompoundIdentifier{binding.Identifier, pgsql.ColumnID}, nodeID, ), } @@ -415,7 +510,7 @@ func expansionNodeLookupJoin(nodeIdentifier pgsql.Identifier, nodeID pgsql.Expre // OFFSET 0 keeps PostgreSQL from flattening this correlated lookup into a full-table hash join. Offset: pgsql.NewLiteral(0, pgsql.Int), }, - Binding: models.OptionalValue(nodeIdentifier), + Binding: models.OptionalValue(binding.Identifier), }, JoinOperator: pgsql.JoinOperator{ JoinType: pgsql.JoinTypeInner, @@ -489,6 +584,7 @@ func rewriteBoundEndpointSeedReference(expression pgsql.Expression, previousFram Distinct: typedExpression.Distinct, Function: typedExpression.Function, Parameters: parameters, + OrderBy: typedExpression.OrderBy, Over: typedExpression.Over, CastType: typedExpression.CastType, } @@ -624,6 +720,7 @@ func rewriteBoundEndpointSeedReference(expression pgsql.Expression, previousFram } } +// seededFrontPrimerQuery places a seed CTE in front of the query that initializes a search frontier. func seededFrontPrimerQuery(seed expansionSeed, primer pgsql.Select) pgsql.Query { return pgsql.Query{ CommonTableExpressions: &pgsql.With{ @@ -633,6 +730,7 @@ func seededFrontPrimerQuery(seed expansionSeed, primer pgsql.Select) pgsql.Query } } +// frontPrimerQuery returns a frontier primer with its optional seed CTE attached. func frontPrimerQuery(seed *expansionSeed, primer pgsql.Select) pgsql.Query { if seed == nil { return pgsql.Query{Body: primer} @@ -641,10 +739,12 @@ func frontPrimerQuery(seed *expansionSeed, primer pgsql.Select) pgsql.Query { return seededFrontPrimerQuery(*seed, primer) } +// expansionAllowsZeroDepth reports whether the traversal's lower bound explicitly admits an empty path. func expansionAllowsZeroDepth(expansionModel *Expansion) bool { return expansionModel.Options.MinDepth.Set && expansionModel.Options.MinDepth.Value == 0 } +// zeroDepthNodeJoin joins a node binding to the identifier representing an empty path's endpoint. func zeroDepthNodeJoin(nodeIdentifier pgsql.Identifier, nodeID pgsql.Expression) pgsql.Join { return pgsql.Join{ Table: expansionNodeTableReference(nodeIdentifier), @@ -655,6 +755,7 @@ func zeroDepthNodeJoin(nodeIdentifier pgsql.Identifier, nodeID pgsql.Expression) } } +// zeroDepthTerminalSatisfaction returns the terminal predicate that can be evaluated without traversing an edge. func zeroDepthTerminalSatisfaction(traversalStep *TraversalStep) pgsql.Expression { localSatisfaction, _ := expansionTerminalSatisfactionLocality(traversalStep) if localSatisfaction == nil { @@ -670,6 +771,7 @@ func zeroDepthTerminalSatisfaction(traversalStep *TraversalStep) pgsql.Expressio return localSatisfaction } +// buildZeroDepthExpansionSelect emits the depth-zero expansion state for roots that already satisfy the terminal predicate. func (s *ExpansionBuilder) buildZeroDepthExpansionSelect(seed *expansionSeed) (pgsql.Select, error) { var ( expansionModel = s.traversalStep.Expansion @@ -724,18 +826,22 @@ func (s *ExpansionBuilder) buildZeroDepthExpansionSelect(seed *expansionSeed) (p }, nil } +// usesBoundRootIDs reports whether roots must be read from a binding in the preceding frame. func (s *ExpansionBuilder) usesBoundRootIDs() bool { return s.traversalStep.LeftNodeBound && s.traversalStep.Frame != nil && s.traversalStep.Frame.Previous != nil } +// usesBoundTerminalIDs reports whether terminals must be read from a binding in the preceding frame. func (s *ExpansionBuilder) usesBoundTerminalIDs() bool { return s.traversalStep.RightNodeBound && s.traversalStep.Frame != nil && s.traversalStep.Frame.Previous != nil } +// usesBoundEndpointPairs reports whether both endpoints are paired bindings from the preceding frame. func (s *ExpansionBuilder) usesBoundEndpointPairs() bool { return s.usesBoundRootIDs() && s.usesBoundTerminalIDs() } +// boundNodeIDsFilterStatement inserts distinct non-null bound node identifiers into a filter table. func (s *ExpansionBuilder) boundNodeIDsFilterStatement(filterIdentifier pgsql.Identifier, nodeIdentifier pgsql.Identifier) pgsql.Insert { var ( previousFrameIdentifier = s.traversalStep.Frame.Previous.Binding.Identifier @@ -771,6 +877,7 @@ func (s *ExpansionBuilder) boundNodeIDsFilterStatement(filterIdentifier pgsql.Id } } +// boundRootIDsFilterStatement builds the root-filter insert when the traversal has a bound root. func (s *ExpansionBuilder) boundRootIDsFilterStatement() (pgsql.Insert, bool) { if !s.usesBoundRootIDs() { return pgsql.Insert{}, false @@ -779,6 +886,7 @@ func (s *ExpansionBuilder) boundRootIDsFilterStatement() (pgsql.Insert, bool) { return s.boundNodeIDsFilterStatement(expansionRootFilter, s.traversalStep.LeftNode.Identifier), true } +// boundTerminalIDsFilterStatement builds the terminal-filter insert when the traversal has a bound terminal. func (s *ExpansionBuilder) boundTerminalIDsFilterStatement() (pgsql.Insert, bool) { if !s.usesBoundTerminalIDs() { return pgsql.Insert{}, false @@ -787,6 +895,7 @@ func (s *ExpansionBuilder) boundTerminalIDsFilterStatement() (pgsql.Insert, bool return s.boundNodeIDsFilterStatement(expansionTerminalFilter, s.traversalStep.RightNode.Identifier), true } +// unboundTerminalIDsFilterStatement materializes terminal node identifiers selected by terminal constraints. func (s *ExpansionBuilder) unboundTerminalIDsFilterStatement() (pgsql.Insert, bool) { expansionModel := s.traversalStep.Expansion if !expansionModel.UseMaterializedTerminalFilter { @@ -796,6 +905,7 @@ func (s *ExpansionBuilder) unboundTerminalIDsFilterStatement() (pgsql.Insert, bo return s.nodeIDsFilterStatement(expansionTerminalFilter, s.traversalStep.RightNode.Identifier, expansionModel.TerminalNodeConstraints), true } +// nodeIDsFilterStatement inserts distinct constrained node identifiers into a filter table. func (s *ExpansionBuilder) nodeIDsFilterStatement(filterIdentifier pgsql.Identifier, nodeIdentifier pgsql.Identifier, constraints pgsql.Expression) pgsql.Insert { nodeIDExpression := pgsql.CompoundIdentifier{nodeIdentifier, pgsql.ColumnID} @@ -826,6 +936,7 @@ func (s *ExpansionBuilder) nodeIDsFilterStatement(filterIdentifier pgsql.Identif } } +// boundEndpointPairFilterStatement inserts distinct non-null bound root and terminal pairs from the preceding frame. func (s *ExpansionBuilder) boundEndpointPairFilterStatement() (pgsql.Insert, bool) { if !s.usesBoundEndpointPairs() { return pgsql.Insert{}, false @@ -877,6 +988,7 @@ func (s *ExpansionBuilder) boundEndpointPairFilterStatement() (pgsql.Insert, boo }, true } +// materializedEndpointPairFilterStatement inserts root and terminal pairs selected independently by endpoint constraints. func (s *ExpansionBuilder) materializedEndpointPairFilterStatement() (pgsql.Insert, bool) { expansionModel := s.traversalStep.Expansion if !expansionModel.UseMaterializedEndpointPairFilter { @@ -923,6 +1035,7 @@ func (s *ExpansionBuilder) materializedEndpointPairFilterStatement() (pgsql.Inse }, true } +// boundTerminalFilterSatisfaction tests whether an expansion endpoint occurs in the materialized terminal filter. func boundTerminalFilterSatisfaction(expansionModel *Expansion) pgsql.Expression { return pgsql.ExistsExpression{ Subquery: pgsql.Subquery{ @@ -947,6 +1060,7 @@ func boundTerminalFilterSatisfaction(expansionModel *Expansion) pgsql.Expression } } +// boundTerminalPairFilterSatisfaction tests whether a root and terminal form a materialized endpoint pair. func boundTerminalPairFilterSatisfaction(rootIDExpression pgsql.Expression, terminalIDExpression pgsql.Expression) pgsql.Expression { return pgsql.ExistsExpression{ Subquery: pgsql.Subquery{ @@ -977,6 +1091,7 @@ func boundTerminalPairFilterSatisfaction(rootIDExpression pgsql.Expression, term } } +// boundRootFilterSatisfaction tests whether an expansion root occurs in the materialized root filter. func boundRootFilterSatisfaction(expansionModel *Expansion) pgsql.Expression { return pgsql.ExistsExpression{ Subquery: pgsql.Subquery{ @@ -1001,6 +1116,7 @@ func boundRootFilterSatisfaction(expansionModel *Expansion) pgsql.Expression { } } +// shortestPathVisitedPruningCondition rejects a root and frontier-node pair already recorded by the search. func shortestPathVisitedPruningCondition(visitedTable pgsql.Identifier, rootIDExpression pgsql.Expression, nextIDExpression pgsql.Expression) pgsql.Expression { return pgsql.ExistsExpression{ Subquery: pgsql.Subquery{ @@ -1031,6 +1147,7 @@ func shortestPathVisitedPruningCondition(visitedTable pgsql.Identifier, rootIDEx } } +// forwardContinuationSatisfaction tests whether another eligible edge leaves the forward frontier endpoint. func forwardContinuationSatisfaction(expansionModel *Expansion) pgsql.Expression { return pgsql.ExistsExpression{ Subquery: pgsql.Subquery{ @@ -1055,6 +1172,7 @@ func forwardContinuationSatisfaction(expansionModel *Expansion) pgsql.Expression } } +// forwardTerminalSatisfaction selects the cheapest available test that marks a forward frontier row terminal. func (s *ExpansionBuilder) forwardTerminalSatisfaction(expansionModel *Expansion, rootIDExpression pgsql.Expression) pgsql.SelectItem { var satisfied pgsql.Expression @@ -1076,6 +1194,7 @@ func (s *ExpansionBuilder) forwardTerminalSatisfaction(expansionModel *Expansion return satisfiedSelectItem } +// forwardTerminalSatisfactionProjection returns a local terminal predicate when no materialized filter supplies it. func forwardTerminalSatisfactionProjection(expansionModel *Expansion) pgsql.Expression { if expansionModel.TerminalNodeSatisfactionProjection != nil && !expansionModel.UseMaterializedTerminalFilter && @@ -1086,6 +1205,7 @@ func forwardTerminalSatisfactionProjection(expansionModel *Expansion) pgsql.Expr return nil } +// backwardContinuationSatisfaction tests whether another eligible edge enters the backward frontier endpoint. func backwardContinuationSatisfaction(expansionModel *Expansion) pgsql.Expression { return pgsql.ExistsExpression{ Subquery: pgsql.Subquery{ @@ -1110,6 +1230,7 @@ func backwardContinuationSatisfaction(expansionModel *Expansion) pgsql.Expressio } } +// backwardTerminalSatisfaction selects the cheapest available test that marks a backward frontier row terminal. func (s *ExpansionBuilder) backwardTerminalSatisfaction(expansionModel *Expansion, terminalIDExpression pgsql.Expression) pgsql.SelectItem { var satisfied pgsql.Expression @@ -1129,6 +1250,7 @@ func (s *ExpansionBuilder) backwardTerminalSatisfaction(expansionModel *Expansio return satisfiedSelectItem } +// backwardTerminalSatisfactionProjection returns a local root predicate when no materialized filter supplies it. func backwardTerminalSatisfactionProjection(expansionModel *Expansion) pgsql.Expression { if expansionModel.PrimerNodeSatisfactionProjection != nil && !expansionModel.UseMaterializedEndpointPairFilter { return pgsql.Expression(expansionModel.PrimerNodeSatisfactionProjection) @@ -1137,6 +1259,7 @@ func backwardTerminalSatisfactionProjection(expansionModel *Expansion) pgsql.Exp return nil } +// prepareForwardFrontPrimerQuery builds the first-edge query and deferred predicate for the forward search frontier. func (s *ExpansionBuilder) prepareForwardFrontPrimerQuery(expansionModel *Expansion) (pgsql.Query, pgsql.Expression, error) { var ( primerSeedConstraints pgsql.Expression @@ -1158,7 +1281,15 @@ func (s *ExpansionBuilder) prepareForwardFrontPrimerQuery(expansionModel *Expans previousFrameIdentifier, ) - if s.usesBoundRootIDs() { + if expansionModel.UsesSingletonEndpointPair() { + rootIDsSeed := newExpansionArrayParameterSeed( + expansionSeedIdentifier(expansionModel.Frame.Binding.Identifier), + s.traversalStep.LeftNode.Identifier, + primerSeedConstraints, + 1, + ) + seed = &rootIDsSeed + } else if s.usesBoundRootIDs() { rootIDsSeed := newExpansionRootIDsParameterSeed( expansionSeedIdentifier(expansionModel.Frame.Binding.Identifier), s.traversalStep.LeftNode.Identifier, @@ -1226,7 +1357,9 @@ func (s *ExpansionBuilder) prepareForwardFrontPrimerQuery(expansionModel *Expans return pgsql.Query{}, nil, err } - if !expansionModel.HasExplicitEndpointInequality { + if !expansionModel.HasExplicitEndpointInequality && + !expansionModel.UsesSingletonEndpointPair() && + !expansionAllowsZeroDepth(expansionModel) { nextQuery.Where = pgsql.OptionalAnd( nextQuery.Where, shortestPathSeedSelfEndpointGuard(s.model.EdgeStartColumn, expansionModel.UseMaterializedEndpointPairFilter), @@ -1236,6 +1369,7 @@ func (s *ExpansionBuilder) prepareForwardFrontPrimerQuery(expansionModel *Expans return frontPrimerQuery(seed, nextQuery), primerProjectionPredicate, nil } +// prepareForwardFrontRecursiveQuery builds the query that advances the forward frontier by one unused edge. func (s *ExpansionBuilder) prepareForwardFrontRecursiveQuery(expansionModel *Expansion) (pgsql.Select, error) { nextQuery := pgsql.Select{ Where: expansionModel.EdgeConstraints, @@ -1323,6 +1457,7 @@ func (s *ExpansionBuilder) prepareForwardFrontRecursiveQuery(expansionModel *Exp return nextQuery, nil } +// prepareBackwardFrontPrimerQuery builds the first-edge query and deferred predicate for the backward search frontier. func (s *ExpansionBuilder) prepareBackwardFrontPrimerQuery(expansionModel *Expansion) (pgsql.Query, pgsql.Expression, error) { var ( terminalSeedConstraints pgsql.Expression @@ -1344,7 +1479,15 @@ func (s *ExpansionBuilder) prepareBackwardFrontPrimerQuery(expansionModel *Expan previousFrameIdentifier, ) - if s.usesBoundTerminalIDs() { + if expansionModel.UsesSingletonEndpointPair() { + terminalIDsSeed := newExpansionArrayParameterSeed( + expansionSeedIdentifier(expansionModel.Frame.Binding.Identifier), + s.traversalStep.RightNode.Identifier, + terminalSeedConstraints, + 2, + ) + seed = &terminalIDsSeed + } else if s.usesBoundTerminalIDs() { terminalIDsSeed := newExpansionTerminalIDsParameterSeed( expansionSeedIdentifier(expansionModel.Frame.Binding.Identifier), s.traversalStep.RightNode.Identifier, @@ -1412,6 +1555,7 @@ func (s *ExpansionBuilder) prepareBackwardFrontPrimerQuery(expansionModel *Expan return frontPrimerQuery(seed, nextQuery), terminalProjectionPredicate, nil } +// prepareBackwardFrontRecursiveQuery builds the query that advances the backward frontier by one unused edge. func (s *ExpansionBuilder) prepareBackwardFrontRecursiveQuery(expansionModel *Expansion) (pgsql.Select, error) { nextQuery := pgsql.Select{ Where: expansionModel.EdgeConstraints, @@ -1484,7 +1628,32 @@ func (s *ExpansionBuilder) prepareBackwardFrontRecursiveQuery(expansionModel *Ex return nextQuery, nil } +// shortestPathSearchCTE invokes a shortest-path harness and exposes its rows through the standard search CTE. func shortestPathSearchCTE(functionName pgsql.Identifier, expansionModel *Expansion, harnessParameters []pgsql.Expression) pgsql.CommonTableExpression { + return shortestPathSearchCTEFrom(functionName, expansionModel, harnessParameters, "singleton_endpoints", expansionModel.Frame.Binding.Identifier) +} + +// shortestPathSearchCTEFrom builds the search CTE, substituting validated singleton endpoint identifiers when present. +func shortestPathSearchCTEFrom(functionName pgsql.Identifier, expansionModel *Expansion, harnessParameters []pgsql.Expression, validatedEndpoints, searchAlias pgsql.Identifier) pgsql.CommonTableExpression { + + if expansionModel.UsesSingletonEndpointPair() { + harnessParameters = append([]pgsql.Expression(nil), harnessParameters...) + rootArrayIndex := len(harnessParameters) - 3 + terminalArrayIndex := len(harnessParameters) - 2 + harnessParameters[rootArrayIndex] = pgsql.ArrayLiteral{ + Values: []pgsql.Expression{ + pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}, + }, + CastType: pgsql.Int8Array, + } + harnessParameters[terminalArrayIndex] = pgsql.ArrayLiteral{ + Values: []pgsql.Expression{ + pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}, + }, + CastType: pgsql.Int8Array, + } + } + var ( innerQuery = pgsql.Query{ Body: pgsql.Select{ @@ -1500,27 +1669,64 @@ func shortestPathSearchCTE(functionName pgsql.Identifier, expansionModel *Expans }, } ) + if expansionModel.UsesSingletonEndpointPair() { + selectBody := innerQuery.Body.(pgsql.Select) + selectBody.Projection = []pgsql.SelectItem{ + pgsql.CompoundIdentifier{functionName, pgsql.WildcardIdentifier}, + } + selectBody.From = append([]pgsql.FromClause{{ + Source: pgsql.TableReference{Name: validatedEndpoints.AsCompoundIdentifier()}, + }}, selectBody.From...) + innerQuery.Body = selectBody + } return pgsql.CommonTableExpression{ Alias: pgsql.TableAlias{ - Name: expansionModel.Frame.Binding.Identifier, + Name: searchAlias, Shape: expansionColumns(), }, Query: innerQuery, } } -func boundEndpointProjectionConstraint(prevFrameID, nodeIdentifier, expansionFrameID, expansionColumn pgsql.Identifier) pgsql.Expression { - return pgsql.NewBinaryExpression( - pgsql.RowColumnReference{ - Identifier: pgsql.CompoundIdentifier{prevFrameID, nodeIdentifier}, - Column: pgsql.ColumnID, +// singletonEndpointValidationCTE validates a single root and terminal pair against both endpoint predicates. +func singletonEndpointValidationCTE(traversalStep *TraversalStep, expansionModel *Expansion) pgsql.CommonTableExpression { + const validatedEndpoints pgsql.Identifier = "singleton_endpoints" + + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: validatedEndpoints}, + Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: []pgsql.SelectItem{ + &pgsql.AliasedExpression{ + Expression: pgd.EntityID(traversalStep.LeftNode.Identifier), + Alias: models.OptionalValue(expansionRootID), + }, + &pgsql.AliasedExpression{ + Expression: pgd.EntityID(traversalStep.RightNode.Identifier), + Alias: models.OptionalValue(expansionTerminalID), + }, + }, + From: []pgsql.FromClause{ + {Source: expansionNodeTableReference(traversalStep.LeftNode.Identifier)}, + {Source: expansionNodeTableReference(traversalStep.RightNode.Identifier)}, + }, + Where: pgsql.OptionalAnd(expansionModel.PrimerNodeConstraints, expansionModel.TerminalNodeConstraints), + }, }, + } +} + +// boundEndpointProjectionConstraint equates a projected expansion endpoint with its binding in the preceding frame. +func boundEndpointProjectionConstraint(prevFrameID pgsql.Identifier, binding *BoundIdentifier, expansionFrameID, expansionColumn pgsql.Identifier) pgsql.Expression { + return pgsql.NewBinaryExpression( + projectedNodeIDReference(prevFrameID, binding), pgsql.OperatorEquals, pgsql.CompoundIdentifier{expansionFrameID, expansionColumn}, ) } +// applyBoundEndpointProjectionConstraints attaches preceding-frame sources and equalities for bound expansion endpoints. func (s *ExpansionBuilder) applyBoundEndpointProjectionConstraints(projectionQuery *pgsql.Select, expansionModel *Expansion) { if s.traversalStep.Frame == nil || s.traversalStep.Frame.Previous == nil { return @@ -1538,7 +1744,7 @@ func (s *ExpansionBuilder) applyBoundEndpointProjectionConstraints(projectionQue projectionQuery.Where = pgsql.OptionalAnd(projectionQuery.Where, boundEndpointProjectionConstraint( prevFrameID, - s.traversalStep.LeftNode.Identifier, + s.traversalStep.LeftNode, expansionModel.Frame.Binding.Identifier, expansionRootID, ), @@ -1549,7 +1755,7 @@ func (s *ExpansionBuilder) applyBoundEndpointProjectionConstraints(projectionQue projectionQuery.Where = pgsql.OptionalAnd(projectionQuery.Where, boundEndpointProjectionConstraint( prevFrameID, - s.traversalStep.RightNode.Identifier, + s.traversalStep.RightNode, expansionModel.Frame.Binding.Identifier, expansionNextID, ), @@ -1557,6 +1763,7 @@ func (s *ExpansionBuilder) applyBoundEndpointProjectionConstraints(projectionQue } } +// ensureProjectionFrameSource ensures a projection query reads from the requested frame. func ensureProjectionFrameSource(projectionQuery *pgsql.Select, frameIdentifier pgsql.Identifier) { for _, from := range projectionQuery.From { if tableReference, ok := from.Source.(pgsql.TableReference); ok && len(tableReference.Name) == 1 && tableReference.Name[0] == frameIdentifier { @@ -1571,6 +1778,7 @@ func ensureProjectionFrameSource(projectionQuery *pgsql.Select, frameIdentifier }}, projectionQuery.From...) } +// applyShortestPathSeedProjectionConstraints adds deferred seed predicates and any frame source they reference. func (s *ExpansionBuilder) applyShortestPathSeedProjectionConstraints(projectionQuery *pgsql.Select, projectionConstraints pgsql.Expression) { if projectionConstraints == nil { return @@ -1586,6 +1794,7 @@ func (s *ExpansionBuilder) applyShortestPathSeedProjectionConstraints(projection projectionQuery.Where = pgsql.OptionalAnd(projectionQuery.Where, projectionConstraints) } +// shortestPathSelfEndpointGuard rejects a shortest-path request whose root and terminal are identical. // Match Neo4j's shortest-path behavior by surfacing an error for result rows // where the resolved root and terminal endpoints are the same node. func shortestPathSelfEndpointGuard(expansionFrame pgsql.Identifier) pgsql.Expression { @@ -1597,6 +1806,7 @@ func shortestPathSelfEndpointGuard(expansionFrame pgsql.Identifier) pgsql.Expres return shortestPathSelfEndpointGuardCase(rootID, terminalID) } +// shortestPathSelfEndpointGuardCase emits the conditional expression that raises the self-endpoint error. func shortestPathSelfEndpointGuardCase(rootID, terminalID pgsql.Expression) pgsql.Expression { return shortestPathSelfEndpointConditionGuard( pgsql.NewBinaryExpression(rootID, pgsql.OperatorNotEquals, terminalID), @@ -1605,6 +1815,7 @@ func shortestPathSelfEndpointGuardCase(rootID, terminalID pgsql.Expression) pgsq ) } +// shortestPathSelfEndpointConditionGuard applies the self-endpoint check only to rows matching a predicate. func shortestPathSelfEndpointConditionGuard(condition pgsql.Expression, rootID, terminalID pgsql.Expression) pgsql.Expression { return &pgsql.Case{ Conditions: []pgsql.Expression{ @@ -1623,6 +1834,7 @@ func shortestPathSelfEndpointConditionGuard(condition pgsql.Expression, rootID, } } +// shortestPathTerminalFilterSelfEndpointGuard rejects a root present in a singleton terminal filter. // PostgreSQL has no portable expression-level RAISE. Keep the normal path // visible in generated SQL and call the schema helper only for the error path. func shortestPathTerminalFilterSelfEndpointGuard(rootID pgsql.Expression) pgsql.Expression { @@ -1673,6 +1885,7 @@ func shortestPathTerminalFilterSelfEndpointGuard(rootID pgsql.Expression) pgsql. } } +// shortestPathEndpointPairFilterSelfEndpointGuard rejects a self-pair present in the endpoint-pair filter. func shortestPathEndpointPairFilterSelfEndpointGuard(rootID pgsql.Expression) pgsql.Expression { matchingEndpointPairCount := pgsql.Subquery{ Query: pgsql.Query{ @@ -1718,6 +1931,7 @@ func shortestPathEndpointPairFilterSelfEndpointGuard(rootID pgsql.Expression) pg ) } +// shortestPathSeedSelfEndpointGuard selects the appropriate self-endpoint check for the active seed filters. func shortestPathSeedSelfEndpointGuard(rootID pgsql.Expression, useEndpointPairFilter bool) pgsql.Expression { if useEndpointPairFilter { return shortestPathEndpointPairFilterSelfEndpointGuard(rootID) @@ -1726,8 +1940,9 @@ func shortestPathSeedSelfEndpointGuard(rootID pgsql.Expression, useEndpointPairF return shortestPathTerminalFilterSelfEndpointGuard(rootID) } +// applyShortestPathSelfEndpointGuard adds self-endpoint validation unless an existing inequality already excludes it. func (s *ExpansionBuilder) applyShortestPathSelfEndpointGuard(projectionQuery *pgsql.Select, expansionModel *Expansion) { - if expansionModel.HasExplicitEndpointInequality { + if expansionModel.HasExplicitEndpointInequality || expansionAllowsZeroDepth(expansionModel) { return } @@ -1737,6 +1952,7 @@ func (s *ExpansionBuilder) applyShortestPathSelfEndpointGuard(projectionQuery *p ) } +// buildShortestPathsHarnessCall assembles the seeded search, harness invocation, and final shortest-path projection. func (s *ExpansionBuilder) buildShortestPathsHarnessCall(harnessFunctionName pgsql.Identifier) (pgsql.Query, error) { var ( expansionModel = s.traversalStep.Expansion @@ -1799,125 +2015,1107 @@ func (s *ExpansionBuilder) buildShortestPathsHarnessCall(harnessFunctionName pgs Body: projectionQuery, } + if expansionModel.UsesSingletonEndpointPair() { + query.AddCTE(singletonEndpointValidationCTE(s.traversalStep, expansionModel)) + } query.AddCTE(shortestPathSearchCTE(harnessFunctionName, expansionModel, harnessParameters)) return query, nil } } +// BuildShortestPathsRoot builds a unidirectional single-shortest-path harness query. func (s *ExpansionBuilder) BuildShortestPathsRoot() (pgsql.Query, error) { return s.buildShortestPathsHarnessCall(pgsql.FunctionUnidirectionalSPHarness) } -func (s *ExpansionBuilder) BuildAllShortestPathsRoot() (pgsql.Query, error) { - return s.buildShortestPathsHarnessCall(pgsql.FunctionUnidirectionalASPHarness) +// shortestDistanceColumns returns the harness result shape for identifier-only or rooted distance searches. +func shortestDistanceColumns(idOnly bool) *pgsql.RecordShape { + if idOnly { + return pgsql.NewRecordShape([]pgsql.Identifier{expansionNextID, expansionDepth}) + } + return pgsql.NewRecordShape([]pgsql.Identifier{expansionRootID, expansionNextID, expansionDepth}) } -func (s *ExpansionBuilder) canMaterializeTerminalFilter(expansionModel *Expansion) bool { - return canMaterializeTerminalFilterForStep(s.traversalStep, expansionModel) +// shortestDistanceEndpointID reads a validated singleton endpoint identifier through a scalar subquery. +func shortestDistanceEndpointID(validatedEndpoints, endpointID pgsql.Identifier) pgsql.Subquery { + return pgsql.Subquery{ + Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.CompoundIdentifier{validatedEndpoints, endpointID}}, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{ + Name: validatedEndpoints.AsCompoundIdentifier(), + }, + }}, + }, + }, + } } -func (s *ExpansionBuilder) canMaterializeEndpointPairFilter(expansionModel *Expansion) bool { - return canMaterializeEndpointPairFilterForStep(s.traversalStep, expansionModel) +// shortestDistanceIDProjection rewrites endpoint identifier projections to use validated search-state columns. +func shortestDistanceIDProjection(projection pgsql.Projection, traversalStep *TraversalStep, stateID, validatedEndpoints pgsql.Identifier) pgsql.Projection { + result := append(pgsql.Projection(nil), projection...) + for idx, item := range result { + aliased, ok := item.(*pgsql.AliasedExpression) + if !ok { + continue + } + identifier, ok := aliased.Expression.(pgsql.CompoundIdentifier) + if !ok || len(identifier) != 2 || identifier[1] != pgsql.ColumnID { + continue + } + var replacement pgsql.Expression + switch identifier[0] { + case traversalStep.LeftNode.Identifier: + replacement = shortestDistanceEndpointID(validatedEndpoints, expansionRootID) + case traversalStep.RightNode.Identifier: + replacement = pgsql.CompoundIdentifier{stateID, expansionNextID} + default: + continue + } + copy := *aliased + copy.Expression = replacement + result[idx] = © + } + return result } -func (s *ExpansionBuilder) buildBiDirectionalShortestPathsHarnessCall(harnessFunctionName pgsql.Identifier) (pgsql.Query, error) { - var ( - expansionModel = s.traversalStep.Expansion - projectionQuery pgsql.Select - ) - - expansionModel.UseMaterializedEndpointPairFilter = s.canMaterializeEndpointPairFilter(expansionModel) +// BuildShortestDistanceRoot emits the bounded, distance-only SP-S3-U-D +// recursive search. ID-only endpoint projections use only next ID and depth; +// other projections retain the constant root ID. Neither shape contains path, +// predecessor, visited-edge, cycle, or materialization columns. +func (s *ExpansionBuilder) BuildShortestDistanceRoot() (pgsql.Query, error) { + const validatedEndpoints pgsql.Identifier = "singleton_endpoints" - forwardFrontPrimerQuery, forwardSeedProjectionConstraints, err := s.prepareForwardFrontPrimerQuery(expansionModel) - if err != nil { - return pgsql.Query{}, err + expansionModel := s.traversalStep.Expansion + if !expansionModel.UsesSingletonEndpointPair() { + return pgsql.Query{}, errors.New("SP-S3-U-D requires one validated endpoint pair") } - - forwardFrontRecursiveQuery, err := s.prepareForwardFrontRecursiveQuery(expansionModel) - if err != nil { - return pgsql.Query{}, err + if !expansionModel.Options.MaxDepth.Set { + return pgsql.Query{}, errors.New("SP-S3-U-D requires a bounded maximum depth") } - backwardFrontPrimerQuery, backwardSeedProjectionConstraints, err := s.prepareBackwardFrontPrimerQuery(expansionModel) - if err != nil { - return pgsql.Query{}, err + endpointCTE := singletonEndpointValidationCTE(s.traversalStep, expansionModel) + if expansionModel.Options.MinDepth.GetOr(1) > 0 { + endpointSelect := endpointCTE.Query.Body.(pgsql.Select) + endpointSelect.Where = pgsql.OptionalAnd(endpointSelect.Where, shortestPathSelfEndpointGuardCase( + pgd.EntityID(s.traversalStep.LeftNode.Identifier), + pgd.EntityID(s.traversalStep.RightNode.Identifier), + )) + endpointCTE.Query.Body = endpointSelect } - backwardFrontRecursiveQuery, err := s.prepareBackwardFrontRecursiveQuery(expansionModel) - if err != nil { - return pgsql.Query{}, err + stateID := expansionModel.Frame.Binding.Identifier + idOnly := s.traversalStep.LeftNode.IDOnly && s.traversalStep.RightNode.IDOnly + anchorProjection := pgsql.Projection{ + pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}, + pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}, + pgsql.NewLiteral(int64(0), pgsql.Int8), + } + if idOnly { + anchorProjection = pgsql.Projection{ + pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}, + pgsql.NewLiteral(int64(0), pgsql.Int8), + } + } + anchor := pgsql.Select{ + Projection: anchorProjection, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{ + Name: validatedEndpoints.AsCompoundIdentifier(), + }, + }}, } - projectionQuery.Projection = expansionModel.Projection + recursiveProjection := pgsql.Projection{ + pgsql.CompoundIdentifier{stateID, expansionRootID}, + expansionModel.EdgeEndColumn, + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{stateID, expansionDepth}, + pgsql.OperatorAdd, + pgsql.NewLiteral(int64(1), pgsql.Int8), + ), + } + if idOnly { + recursiveProjection = recursiveProjection[1:] + } + recursive := pgsql.Select{ + Projection: recursiveProjection, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: stateID.AsCompoundIdentifier()}, + Joins: []pgsql.Join{{ + Table: expansionEdgeTableReference(s.traversalStep.Edge.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + expansionModel.EdgeStartColumn, + pgsql.OperatorEquals, + pgsql.CompoundIdentifier{stateID, expansionNextID}, + ), + }, + }}, + }}, + Where: pgsql.OptionalAnd( + expansionModel.EdgeConstraints, + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{stateID, expansionDepth}, + pgsql.OperatorLessThan, + pgsql.NewLiteral(expansionModel.Options.MaxDepth.Value, pgsql.Int8), + ), + ), + } - // Select the expansion components for the projection statement - projectionQuery.From = []pgsql.FromClause{{ - Source: pgsql.TableReference{ - Name: pgsql.CompoundIdentifier{expansionModel.Frame.Binding.Identifier}, - Binding: models.EmptyOptional[pgsql.Identifier](), - }, - Joins: []pgsql.Join{{ - Table: expansionNodeTableReference(s.traversalStep.LeftNode.Identifier), - JoinOperator: pgsql.JoinOperator{ - JoinType: pgsql.JoinTypeInner, - Constraint: pgsql.NewBinaryExpression( - pgsql.CompoundIdentifier{s.traversalStep.LeftNode.Identifier, pgsql.ColumnID}, - pgsql.OperatorEquals, - pgsql.CompoundIdentifier{expansionModel.Frame.Binding.Identifier, expansionRootID}, + projectionItems := pgsql.Projection(expansionModel.Projection) + var endpointConstraint pgsql.Expression + joins := []pgsql.Join{{ + Table: pgsql.TableReference{Name: validatedEndpoints.AsCompoundIdentifier()}, + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.OptionalAnd( + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{stateID, expansionRootID}, pgsql.OperatorEquals, + pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}, ), - }, - }, { - Table: expansionNodeTableReference(s.traversalStep.RightNode.Identifier), - JoinOperator: pgsql.JoinOperator{ - JoinType: pgsql.JoinTypeInner, - Constraint: pgsql.NewBinaryExpression( - pgsql.CompoundIdentifier{s.traversalStep.RightNode.Identifier, pgsql.ColumnID}, - pgsql.OperatorEquals, - pgsql.CompoundIdentifier{expansionModel.Frame.Binding.Identifier, expansionNextID}, + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{stateID, expansionNextID}, pgsql.OperatorEquals, + pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}, ), - }, - }}, + ), + }, }} - - s.applyBoundEndpointProjectionConstraints(&projectionQuery, expansionModel) - s.applyShortestPathSeedProjectionConstraints(&projectionQuery, pgsql.OptionalAnd(forwardSeedProjectionConstraints, backwardSeedProjectionConstraints)) - s.appendUnwindSources(&projectionQuery) - s.applyShortestPathSelfEndpointGuard(&projectionQuery, expansionModel) - - if harnessParameters, err := s.bidirectionalAllShortestPathsParameters(expansionModel, forwardFrontPrimerQuery, forwardFrontRecursiveQuery, backwardFrontPrimerQuery, backwardFrontRecursiveQuery); err != nil { - return pgsql.Query{}, err + if idOnly { + projectionItems = shortestDistanceIDProjection(projectionItems, s.traversalStep, stateID, validatedEndpoints) + joins = nil + endpointConstraint = pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{stateID, expansionNextID}, + pgsql.OperatorEquals, + shortestDistanceEndpointID(validatedEndpoints, expansionTerminalID), + ) } else { - query := pgsql.Query{ - CommonTableExpressions: &pgsql.With{}, - Body: projectionQuery, - } + joins = append(joins, + pgsql.Join{ + Table: expansionNodeTableReference(s.traversalStep.LeftNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{s.traversalStep.LeftNode.Identifier, pgsql.ColumnID}, pgsql.OperatorEquals, + pgsql.CompoundIdentifier{stateID, expansionRootID}, + ), + }, + }, + pgsql.Join{ + Table: expansionNodeTableReference(s.traversalStep.RightNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{s.traversalStep.RightNode.Identifier, pgsql.ColumnID}, pgsql.OperatorEquals, + pgsql.CompoundIdentifier{stateID, expansionNextID}, + ), + }, + }, + ) + } - query.AddCTE(shortestPathSearchCTE(harnessFunctionName, expansionModel, harnessParameters)) - return query, nil + projection := pgsql.Select{ + Projection: projectionItems, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: stateID.AsCompoundIdentifier()}, + Joins: joins, + }}, + Where: pgsql.OptionalAnd( + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{stateID, expansionDepth}, + pgsql.OperatorGreaterThanOrEqualTo, + pgsql.NewLiteral(expansionModel.Options.MinDepth.GetOr(1), pgsql.Int8), + ), + endpointConstraint, + ), } -} -func (s *ExpansionBuilder) BuildBiDirectionalShortestPathsRoot() (pgsql.Query, error) { - return s.buildBiDirectionalShortestPathsHarnessCall(pgsql.FunctionBidirectionalSPHarness) + query := pgsql.Query{ + CommonTableExpressions: &pgsql.With{Recursive: true}, + Body: projection, + OrderBy: []*pgsql.OrderBy{{ + Expression: pgsql.CompoundIdentifier{stateID, expansionDepth}, + Ascending: true, + }}, + Limit: pgsql.NewLiteral(int64(1), pgsql.Int8), + } + query.AddCTE(endpointCTE) + query.AddCTE(pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: stateID, + Shape: shortestDistanceColumns(idOnly), + }, + Query: pgsql.Query{ + Body: pgsql.SetOperation{ + LOperand: anchor, + ROperand: recursive, + Operator: pgsql.OperatorUnion, + }, + }, + }) + + return query, nil } -func (s *ExpansionBuilder) BuildBiDirectionalAllShortestPathsRoot() (pgsql.Query, error) { - return s.buildBiDirectionalShortestPathsHarnessCall(pgsql.FunctionBidirectionalASPHarness) +// shortestPathNodeComposite constructs the stored composite value for a hydrated path node. +func shortestPathNodeComposite(identifier pgsql.Identifier) pgsql.CompositeValue { + value := pgsql.CompositeValue{DataType: pgsql.NodeComposite} + for _, column := range pgsql.NodeTableColumns { + value.Values = append(value.Values, pgsql.CompoundIdentifier{identifier, column}) + } + return value } -func (s *ExpansionBuilder) boundEndpointFilterParameters() ([]pgsql.Expression, error) { - var ( - rootFilterStatement, hasRootFilter = s.boundRootIDsFilterStatement() - terminalFilterStatement, hasTerminalFilter = s.boundTerminalIDsFilterStatement() - pairFilterStatement, hasPairFilter = s.boundEndpointPairFilterStatement() +// shortestPathM0Hydration expands an edge-identifier path into ordered node and edge composites. +func shortestPathM0Hydration(stateID pgsql.Identifier, direction graph.Direction) pgsql.LateralSubquery { + const ( + pathIndex pgsql.Identifier = "m0_path_index" + pathEdge pgsql.Identifier = "m0_edge" + pathTerminal pgsql.Identifier = "m0_terminal" + hydrated pgsql.Identifier = "m0_hydrated" + hydratedNodes pgsql.Identifier = "nodes" + hydratedEdges pgsql.Identifier = "edges" + hydratedCount pgsql.Identifier = "hydrated_count" ) - if !hasPairFilter { - pairFilterStatement, hasPairFilter = s.materializedEndpointPairFilterStatement() + pathIDs := pgsql.CompoundIdentifier{stateID, expansionPath} + edgeID := &pgsql.ArrayIndex{ + Expression: pgsql.NewParenthetical(pathIDs), + Indexes: []pgsql.Expression{pathIndex}, + CastType: pgsql.Int8, } + nextNodeColumn := pgsql.ColumnEndID + if direction == graph.DirectionInbound { + nextNodeColumn = pgsql.ColumnStartID + } + joins := []pgsql.Join{{ + Table: expansionEdgeTableReference(pathEdge), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{pathEdge, pgsql.ColumnID}, pgsql.OperatorEquals, edgeID, + ), + }, + }, { + Table: expansionNodeTableReference(pathTerminal), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{pathTerminal, pgsql.ColumnID}, pgsql.OperatorEquals, + pgsql.CompoundIdentifier{pathEdge, nextNodeColumn}, + ), + }, + }} - if !hasTerminalFilter { - terminalFilterStatement, hasTerminalFilter = s.unboundTerminalIDsFilterStatement() + return pgsql.LateralSubquery{ + Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{ + &pgsql.AliasedExpression{ + Expression: pgsql.FunctionCall{ + Function: pgsql.FunctionArrayAggregate, + Parameters: []pgsql.Expression{shortestPathNodeComposite(pathTerminal)}, + OrderBy: []*pgsql.OrderBy{{ + Expression: pathIndex, + Ascending: true, + }}, + CastType: pgsql.NodeCompositeArray, + }, + Alias: pgsql.AsOptionalIdentifier(hydratedNodes), + }, + &pgsql.AliasedExpression{ + Expression: pgsql.FunctionCall{ + Function: pgsql.FunctionArrayAggregate, + Parameters: []pgsql.Expression{edgeCompositeValue(pathEdge)}, + OrderBy: []*pgsql.OrderBy{{ + Expression: pathIndex, + Ascending: true, + }}, + CastType: pgsql.EdgeCompositeArray, + }, + Alias: pgsql.AsOptionalIdentifier(hydratedEdges), + }, + &pgsql.AliasedExpression{ + Expression: pgsql.FunctionCall{ + Function: pgsql.FunctionCount, + Parameters: []pgsql.Expression{pgsql.Wildcard{}}, + CastType: pgsql.Int8, + }, + Alias: pgsql.AsOptionalIdentifier(hydratedCount), + }, + }, + From: []pgsql.FromClause{{ + Source: pgsql.AliasedExpression{ + Expression: pgsql.FunctionCall{ + Function: pgsql.FunctionGenerateSubscripts, + Parameters: []pgsql.Expression{pathIDs, pgsql.NewLiteral(1, pgsql.Int)}, + }, + Alias: pgsql.AsOptionalIdentifier(pathIndex), + }, + Joins: joins, + }}, + }, + }, + Binding: pgsql.AsOptionalIdentifier(hydrated), + } +} + +// shortestPathM0Projection replaces the raw path state with its hydrated graph-path value. +func shortestPathM0Projection(projection pgsql.Projection, stateID pgsql.Identifier, path pgsql.Expression) pgsql.Projection { + result := append(pgsql.Projection(nil), projection...) + for idx, item := range result { + aliased, ok := item.(*pgsql.AliasedExpression) + if !ok { + continue + } + identifier, ok := aliased.Expression.(pgsql.CompoundIdentifier) + if !ok || len(identifier) != 2 || identifier[0] != stateID || identifier[1] != expansionPath { + continue + } + copy := *aliased + copy.Expression = path + result[idx] = © + } + return result +} + +// BuildShortestPathEdgeM0Root emits the bounded one-path SP-S3-U-E search and +// direction-aware MAT-M0 hydration. Recursive state contains only the current +// node, depth, and ordered edge IDs; node order is derived from edge endpoints. +func (s *ExpansionBuilder) BuildShortestPathEdgeM0Root() (pgsql.Query, error) { + const ( + validatedEndpoints pgsql.Identifier = "singleton_endpoints" + hydrated pgsql.Identifier = "m0_hydrated" + hydratedNodes pgsql.Identifier = "nodes" + hydratedEdges pgsql.Identifier = "edges" + hydratedCount pgsql.Identifier = "hydrated_count" + ) + + expansionModel := s.traversalStep.Expansion + if !expansionModel.UsesSingletonEndpointPair() { + return pgsql.Query{}, errors.New("SP-S3-U-E+MAT-M0 requires one validated endpoint pair") + } + if !expansionModel.Options.MaxDepth.Set { + return pgsql.Query{}, errors.New("SP-S3-U-E+MAT-M0 requires a bounded maximum depth") + } + + endpointCTE := singletonEndpointValidationCTE(s.traversalStep, expansionModel) + if expansionModel.Options.MinDepth.GetOr(1) > 0 { + endpointSelect := endpointCTE.Query.Body.(pgsql.Select) + endpointSelect.Where = pgsql.OptionalAnd(endpointSelect.Where, shortestPathSelfEndpointGuardCase( + pgd.EntityID(s.traversalStep.LeftNode.Identifier), + pgd.EntityID(s.traversalStep.RightNode.Identifier), + )) + endpointCTE.Query.Body = endpointSelect + } + + stateID := expansionModel.Frame.Binding.Identifier + pathIDs := pgsql.CompoundIdentifier{stateID, expansionPath} + anchor := pgsql.Select{ + Projection: pgsql.Projection{ + pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}, + pgsql.NewLiteral(int64(0), pgsql.Int8), + pgsql.ArrayLiteral{CastType: pgsql.Int8Array}, + }, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{ + Name: validatedEndpoints.AsCompoundIdentifier(), + }, + }}, + } + recursive := pgsql.Select{ + Projection: pgsql.Projection{ + expansionModel.EdgeEndColumn, + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{stateID, expansionDepth}, pgsql.OperatorAdd, pgsql.NewLiteral(int64(1), pgsql.Int8)), + pgsql.NewBinaryExpression(pathIDs, pgsql.OperatorConcatenate, pgsql.ArrayLiteral{ + Values: []pgsql.Expression{pgsql.CompoundIdentifier{s.traversalStep.Edge.Identifier, pgsql.ColumnID}}, + CastType: pgsql.Int8Array, + }), + }, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: stateID.AsCompoundIdentifier()}, + Joins: []pgsql.Join{{ + Table: expansionEdgeTableReference(s.traversalStep.Edge.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + expansionModel.EdgeStartColumn, pgsql.OperatorEquals, pgsql.CompoundIdentifier{stateID, expansionNextID}, + ), + }, + }}, + }}, + Where: pgsql.OptionalAnd( + expansionModel.EdgeConstraints, + pgsql.OptionalAnd( + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{stateID, expansionDepth}, pgsql.OperatorLessThan, pgsql.NewLiteral(expansionModel.Options.MaxDepth.Value, pgsql.Int8)), + relationshipIDNotInPath(pgsql.CompoundIdentifier{s.traversalStep.Edge.Identifier, pgsql.ColumnID}, pathIDs), + ), + ), + } + + hydration := shortestPathM0Hydration(stateID, s.traversalStep.Direction) + rootArray := pgsql.ArrayLiteral{ + Values: []pgsql.Expression{shortestPathNodeComposite(s.traversalStep.LeftNode.Identifier)}, + CastType: pgsql.NodeCompositeArray, + } + nodes := pgsql.FunctionCall{ + Function: pgsql.FunctionCoalesce, + Parameters: []pgsql.Expression{ + pgsql.CompoundIdentifier{hydrated, hydratedNodes}, pgsql.ArrayLiteral{ + CastType: pgsql.NodeCompositeArray, + }, + }, + } + edges := pgsql.FunctionCall{ + Function: pgsql.FunctionCoalesce, + Parameters: []pgsql.Expression{ + pgsql.CompoundIdentifier{hydrated, hydratedEdges}, pgsql.ArrayLiteral{ + CastType: pgsql.EdgeCompositeArray, + }, + }, + } + path := pgsql.CompositeValue{ + DataType: pgsql.PathComposite, + Values: []pgsql.Expression{ + pgsql.NewBinaryExpression(rootArray, pgsql.OperatorConcatenate, nodes), + edges, + }, + } + + projection := pgsql.Select{ + Projection: shortestPathM0Projection(expansionModel.Projection, stateID, path), + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: stateID.AsCompoundIdentifier()}, + Joins: []pgsql.Join{ + { + Table: pgsql.TableReference{ + Name: validatedEndpoints.AsCompoundIdentifier(), + }, + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{stateID, expansionNextID}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}, + ), + }, + }, + { + Table: expansionNodeTableReference(s.traversalStep.LeftNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{s.traversalStep.LeftNode.Identifier, pgsql.ColumnID}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}, + ), + }, + }, + { + Table: expansionNodeTableReference(s.traversalStep.RightNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{s.traversalStep.RightNode.Identifier, pgsql.ColumnID}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{stateID, expansionNextID}, + ), + }, + }, + { + Table: hydration, + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewLiteral(true, pgsql.Boolean), + }, + }, + }, + }}, + Where: pgsql.OptionalAnd( + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{stateID, expansionDepth}, pgsql.OperatorGreaterThanOrEqualTo, pgsql.NewLiteral(expansionModel.Options.MinDepth.GetOr(1), pgsql.Int8)), + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{hydrated, hydratedCount}, pgsql.OperatorEquals, pgsql.FunctionCall{ + Function: pgsql.FunctionCardinality, + Parameters: []pgsql.Expression{pathIDs}, + }), + ), + } + + query := pgsql.Query{ + CommonTableExpressions: &pgsql.With{ + Recursive: true, + }, + Body: projection, + OrderBy: []*pgsql.OrderBy{{ + Expression: pgsql.CompoundIdentifier{stateID, expansionDepth}, + Ascending: true, + }, { + Expression: pathIDs, + Ascending: true, + }}, + Limit: pgsql.NewLiteral(int64(1), pgsql.Int8), + } + query.AddCTE(endpointCTE) + query.AddCTE(pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: stateID, + Shape: pgsql.NewRecordShape([]pgsql.Identifier{expansionNextID, expansionDepth, expansionPath}), + }, + Query: pgsql.Query{ + Body: pgsql.SetOperation{ + LOperand: anchor, + ROperand: recursive, + Operator: pgsql.OperatorUnion, + All: true, + }, + }, + }) + return query, nil +} + +// BuildAllShortestPathsRoot builds a unidirectional all-shortest-paths harness query. +func (s *ExpansionBuilder) BuildAllShortestPathsRoot() (pgsql.Query, error) { + return s.buildShortestPathsHarnessCall(pgsql.FunctionUnidirectionalASPHarness) +} + +// compactShortestExecutor reports whether executor emits the compact distance/witness row shape that requires legacy expansion-shape adaptation. +func compactShortestExecutor(executor optimize.ShortestPathExecutor) bool { + switch executor { + case optimize.ShortestPathExecutorASPA1DAG, + optimize.ShortestPathExecutorASPI1DAG, + optimize.ShortestPathExecutorASPB1AlternatingNodeDAG, + optimize.ShortestPathExecutorASPB2SmallerCurrentLevelDAG, + optimize.ShortestPathExecutorS4CanonicalDistance, + optimize.ShortestPathExecutorS4CanonicalWitness, + optimize.ShortestPathExecutorB1AlternatingNodeDistance, + optimize.ShortestPathExecutorB1AlternatingNodeWitness, + optimize.ShortestPathExecutorB2SmallerCurrentLevelDistance, + optimize.ShortestPathExecutorB2SmallerCurrentLevelWitness: + return true + default: + return false + } +} + +// buildCompactBoundShortestPathsRoot invokes a typed, static bound-pair +// executor and keeps the legacy expansion row shape at its boundary. That lets +// existing projection and path materialization code consume compact search +// results without carrying entity composites through discovery. +func (s *ExpansionBuilder) buildCompactBoundShortestPathsRoot(functionName pgsql.Identifier, limits ...int64) (pgsql.Query, error) { + const ( + validatedEndpoints pgsql.Identifier = "singleton_endpoints" + hydrated pgsql.Identifier = "m0_hydrated" + hydratedNodes pgsql.Identifier = "nodes" + hydratedEdges pgsql.Identifier = "edges" + hydratedCount pgsql.Identifier = "hydrated_count" + ) + + expansionModel := s.traversalStep.Expansion + if !expansionModel.UsesSingletonEndpointPair() { + return pgsql.Query{}, fmt.Errorf("%s requires one validated endpoint pair", functionName) + } + + endpointCTE := singletonEndpointValidationCTE(s.traversalStep, expansionModel) + if expansionModel.Options.MinDepth.GetOr(1) > 0 { + endpointSelect := endpointCTE.Query.Body.(pgsql.Select) + endpointSelect.Where = pgsql.OptionalAnd(endpointSelect.Where, shortestPathSelfEndpointGuardCase( + pgd.EntityID(s.traversalStep.LeftNode.Identifier), + pgd.EntityID(s.traversalStep.RightNode.Identifier), + )) + endpointCTE.Query.Body = endpointSelect + } + + maxDepth := expansionModel.Options.MaxDepth.GetOr(translateDefaultMaxTraversalDepth) + parameters := []pgsql.Expression{ + pgsql.NewLiteral(s.graphID, pgsql.Int4), + pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}, + pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}, + pgsql.NewLiteral(expansionModel.Options.MinDepth.GetOr(1), pgsql.Int4), + pgsql.NewLiteral(maxDepth, pgsql.Int4), + pgsql.NewLiteral(append([]int16(nil), expansionModel.RelationshipKindIDs...), pgsql.Int2Array), + pgsql.NewLiteral(s.traversalStep.Direction == graph.DirectionInbound, pgsql.Boolean), + } + for _, limit := range limits { + if limit <= 0 { + return pgsql.Query{}, fmt.Errorf("%s requires positive compact workspace limits", functionName) + } + parameters = append(parameters, pgsql.NewLiteral(limit, pgsql.Int8)) + } + + stateID := expansionModel.Frame.Binding.Identifier + search := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: stateID, + Shape: expansionColumns(), + }, + Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.CompoundIdentifier{functionName, pgsql.WildcardIdentifier}}, + From: []pgsql.FromClause{ + { + Source: pgsql.TableReference{ + Name: validatedEndpoints.AsCompoundIdentifier(), + }, + }, + { + Source: pgsql.FunctionCall{ + Function: functionName, + Parameters: parameters, + }, + }, + }, + }, + }, + } + + projection := pgsql.Select{ + Projection: expansionModel.Projection, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{ + Name: stateID.AsCompoundIdentifier(), + }, + Joins: []pgsql.Join{ + { + Table: expansionNodeTableReference(s.traversalStep.LeftNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{s.traversalStep.LeftNode.Identifier, pgsql.ColumnID}, pgsql.OperatorEquals, + pgsql.CompoundIdentifier{stateID, expansionRootID}, + ), + }, + }, + { + Table: expansionNodeTableReference(s.traversalStep.RightNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{s.traversalStep.RightNode.Identifier, pgsql.ColumnID}, pgsql.OperatorEquals, + pgsql.CompoundIdentifier{stateID, expansionNextID}, + ), + }, + }, + }, + }}, + } + + // S4 witness search returns only ordered edge identifiers. Hydrate those + // identifiers at the inline statement boundary, exactly as S3 M0 does, + // instead of invoking the generic ordered_edge_ids_to_path helper. Keeping + // search and hydration as separate SQL operators avoids a second stored + // helper boundary and makes S3/S4 materialization evidence comparable. + if functionName == pgsql.FunctionShortestPathCompact && expansionModel.ShortestPathExecutor == optimize.ShortestPathExecutorS4CanonicalWitness { + pathIDs := pgsql.CompoundIdentifier{stateID, expansionPath} + hydration := shortestPathM0Hydration(stateID, s.traversalStep.Direction) + rootArray := pgsql.ArrayLiteral{ + Values: []pgsql.Expression{shortestPathNodeComposite(s.traversalStep.LeftNode.Identifier)}, + CastType: pgsql.NodeCompositeArray, + } + nodes := pgsql.FunctionCall{ + Function: pgsql.FunctionCoalesce, + Parameters: []pgsql.Expression{ + pgsql.CompoundIdentifier{hydrated, hydratedNodes}, + pgsql.ArrayLiteral{CastType: pgsql.NodeCompositeArray}, + }, + } + edges := pgsql.FunctionCall{ + Function: pgsql.FunctionCoalesce, + Parameters: []pgsql.Expression{ + pgsql.CompoundIdentifier{hydrated, hydratedEdges}, + pgsql.ArrayLiteral{CastType: pgsql.EdgeCompositeArray}, + }, + } + path := pgsql.CompositeValue{ + DataType: pgsql.PathComposite, + Values: []pgsql.Expression{ + pgsql.NewBinaryExpression(rootArray, pgsql.OperatorConcatenate, nodes), + edges, + }, + } + projection.Projection = shortestPathM0Projection(projection.Projection, stateID, path) + projection.From[0].Joins = append(projection.From[0].Joins, pgsql.Join{ + Table: hydration, + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewLiteral(true, pgsql.Boolean), + }, + }) + projection.Where = pgsql.OptionalAnd(projection.Where, pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{hydrated, hydratedCount}, pgsql.OperatorEquals, + pgsql.FunctionCall{Function: pgsql.FunctionCardinality, Parameters: []pgsql.Expression{pathIDs}}, + )) + } + + query := pgsql.Query{ + CommonTableExpressions: &pgsql.With{}, + Body: projection, + } + query.AddCTE(endpointCTE) + query.AddCTE(search) + return query, nil +} + +// BuildAllShortestPathsDAGRoot builds the bound-endpoint query that enumerates all shortest paths from a predecessor DAG. +func (s *ExpansionBuilder) BuildAllShortestPathsDAGRoot() (pgsql.Query, error) { + return s.buildCompactBoundShortestPathsRoot(pgsql.FunctionAllShortestPathsDAG) +} + +// BuildB1AllShortestPathsDAGRoot builds strict node-alternating two-sided predecessor-DAG enumeration. +func (s *ExpansionBuilder) BuildB1AllShortestPathsDAGRoot() (pgsql.Query, error) { + expansion := s.traversalStep.Expansion + return s.buildCompactBoundShortestPathsRoot(pgsql.FunctionAllShortestPathsB1StrictAlternating, + expansion.ShortestPathStateLimit, expansion.ShortestPathFrontierLimit, + expansion.ShortestPathPredecessorLimit, expansion.ShortestPathEnumerationLimit, + expansion.ShortestPathOutputBytesLimit) +} + +// BuildB2AllShortestPathsDAGRoot builds smaller-current-level two-sided predecessor-DAG enumeration. +func (s *ExpansionBuilder) BuildB2AllShortestPathsDAGRoot() (pgsql.Query, error) { + expansion := s.traversalStep.Expansion + return s.buildCompactBoundShortestPathsRoot(pgsql.FunctionAllShortestPathsB2SmallerCurrentLevel, + expansion.ShortestPathStateLimit, expansion.ShortestPathFrontierLimit, + expansion.ShortestPathPredecessorLimit, expansion.ShortestPathEnumerationLimit, + expansion.ShortestPathOutputBytesLimit) +} + +// BuildCompactShortestPathRoot builds the bound-endpoint query that returns one compact shortest-path witness. +func (s *ExpansionBuilder) BuildCompactShortestPathRoot() (pgsql.Query, error) { + return s.buildCompactBoundShortestPathsRoot(pgsql.FunctionShortestPathCompact, s.traversalStep.Expansion.ShortestPathStateLimit) +} + +// BuildB1CompactShortestPathRoot builds strict node-alternating compact bidirectional search. +func (s *ExpansionBuilder) BuildB1CompactShortestPathRoot() (pgsql.Query, error) { + expansion := s.traversalStep.Expansion + return s.buildCompactBoundShortestPathsRoot(pgsql.FunctionShortestPathB1StrictAlternating, + expansion.ShortestPathStateLimit, expansion.ShortestPathFrontierLimit, expansion.ShortestPathPredecessorLimit) +} + +// BuildB2CompactShortestPathRoot builds smaller-current-level compact bidirectional search. +func (s *ExpansionBuilder) BuildB2CompactShortestPathRoot() (pgsql.Query, error) { + expansion := s.traversalStep.Expansion + return s.buildCompactBoundShortestPathsRoot(pgsql.FunctionShortestPathB2SmallerCurrentLevel, + expansion.ShortestPathStateLimit, expansion.ShortestPathFrontierLimit, expansion.ShortestPathPredecessorLimit) +} + +// canMaterializeTerminalFilter reports whether terminal constraints can be precomputed as an identifier filter. +func (s *ExpansionBuilder) canMaterializeTerminalFilter(expansionModel *Expansion) bool { + return canMaterializeTerminalFilterForStep(s.traversalStep, expansionModel) +} + +// canMaterializeEndpointPairFilter reports whether root and terminal constraints can be precomputed as endpoint pairs. +func (s *ExpansionBuilder) canMaterializeEndpointPairFilter(expansionModel *Expansion) bool { + return canMaterializeEndpointPairFilterForStep(s.traversalStep, expansionModel) +} + +// buildBiDirectionalShortestPathsHarnessCall assembles both search fronts, the bidirectional harness, and its final projection. +func (s *ExpansionBuilder) buildBiDirectionalShortestPathsHarnessCall(harnessFunctionName pgsql.Identifier) (pgsql.Query, error) { + var ( + expansionModel = s.traversalStep.Expansion + projectionQuery pgsql.Select + ) + + if !expansionModel.UsesSingletonEndpointPair() { + expansionModel.UseMaterializedEndpointPairFilter = s.canMaterializeEndpointPairFilter(expansionModel) + } + + forwardFrontPrimerQuery, forwardSeedProjectionConstraints, err := s.prepareForwardFrontPrimerQuery(expansionModel) + if err != nil { + return pgsql.Query{}, err + } + + forwardFrontRecursiveQuery, err := s.prepareForwardFrontRecursiveQuery(expansionModel) + if err != nil { + return pgsql.Query{}, err + } + + backwardFrontPrimerQuery, backwardSeedProjectionConstraints, err := s.prepareBackwardFrontPrimerQuery(expansionModel) + if err != nil { + return pgsql.Query{}, err + } + + backwardFrontRecursiveQuery, err := s.prepareBackwardFrontRecursiveQuery(expansionModel) + if err != nil { + return pgsql.Query{}, err + } + + projectionQuery.Projection = expansionModel.Projection + + // Select the expansion components for the projection statement + projectionQuery.From = []pgsql.FromClause{{ + Source: pgsql.TableReference{ + Name: pgsql.CompoundIdentifier{expansionModel.Frame.Binding.Identifier}, + Binding: models.EmptyOptional[pgsql.Identifier](), + }, + Joins: []pgsql.Join{{ + Table: expansionNodeTableReference(s.traversalStep.LeftNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{s.traversalStep.LeftNode.Identifier, pgsql.ColumnID}, + pgsql.OperatorEquals, + pgsql.CompoundIdentifier{expansionModel.Frame.Binding.Identifier, expansionRootID}, + ), + }, + }, { + Table: expansionNodeTableReference(s.traversalStep.RightNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{s.traversalStep.RightNode.Identifier, pgsql.ColumnID}, + pgsql.OperatorEquals, + pgsql.CompoundIdentifier{expansionModel.Frame.Binding.Identifier, expansionNextID}, + ), + }, + }}, + }} + + s.applyBoundEndpointProjectionConstraints(&projectionQuery, expansionModel) + s.applyShortestPathSeedProjectionConstraints(&projectionQuery, pgsql.OptionalAnd(forwardSeedProjectionConstraints, backwardSeedProjectionConstraints)) + s.appendUnwindSources(&projectionQuery) + s.applyShortestPathSelfEndpointGuard(&projectionQuery, expansionModel) + + if harnessParameters, err := s.bidirectionalShortestPathsParameters( + expansionModel, + forwardFrontPrimerQuery, + forwardFrontRecursiveQuery, + backwardFrontPrimerQuery, + backwardFrontRecursiveQuery, + harnessFunctionName == pgsql.FunctionBidirectionalSPHarness, + ); err != nil { + return pgsql.Query{}, err + } else { + query := pgsql.Query{ + CommonTableExpressions: &pgsql.With{}, + Body: projectionQuery, + } + + if expansionModel.UsesSingletonEndpointPair() { + query.AddCTE(singletonEndpointValidationCTE(s.traversalStep, expansionModel)) + } + query.AddCTE(shortestPathSearchCTE(harnessFunctionName, expansionModel, harnessParameters)) + return query, nil + } +} + +// BuildBiDirectionalShortestPathsRoot builds a bidirectional single-shortest-path harness query. +func (s *ExpansionBuilder) BuildBiDirectionalShortestPathsRoot() (pgsql.Query, error) { + return s.buildBiDirectionalShortestPathsHarnessCall(pgsql.FunctionBidirectionalSPHarness) +} + +// BuildBiDirectionalShortestPathsRootWithDirectPreflight emits the tool-only +// SP-S0-DIRECT arm. A materialized one-edge probe returns immediately when it +// finds a valid bound-endpoint witness. The workspace-backed incumbent is +// dependent on a zero-or-one-row fallback endpoint CTE, so PostgreSQL cannot +// invoke it on a direct hit. Both branches execute in one statement snapshot. +func (s *ExpansionBuilder) BuildBiDirectionalShortestPathsRootWithDirectPreflight() (pgsql.Query, error) { + const ( + validatedEndpoints pgsql.Identifier = "singleton_endpoints" + directHit pgsql.Identifier = "direct_shortest" + fallbackEndpoints pgsql.Identifier = "fallback_endpoints" + workspaceSearch pgsql.Identifier = "workspace_shortest" + ) + + expansionModel := s.traversalStep.Expansion + if !expansionModel.UsesSingletonEndpointPair() { + return pgsql.Query{}, errors.New("SP-S0-DIRECT requires one validated endpoint pair") + } + if expansionModel.Options.MinDepth.GetOr(1) != 1 || expansionModel.Options.MaxDepth.GetOr(0) < 1 { + return pgsql.Query{}, errors.New("SP-S0-DIRECT requires minimum depth one and a positive bounded maximum depth") + } + + forwardFrontPrimerQuery, forwardSeedProjectionConstraints, err := s.prepareForwardFrontPrimerQuery(expansionModel) + if err != nil { + return pgsql.Query{}, err + } + forwardFrontRecursiveQuery, err := s.prepareForwardFrontRecursiveQuery(expansionModel) + if err != nil { + return pgsql.Query{}, err + } + backwardFrontPrimerQuery, backwardSeedProjectionConstraints, err := s.prepareBackwardFrontPrimerQuery(expansionModel) + if err != nil { + return pgsql.Query{}, err + } + backwardFrontRecursiveQuery, err := s.prepareBackwardFrontRecursiveQuery(expansionModel) + if err != nil { + return pgsql.Query{}, err + } + + harnessParameters, err := s.bidirectionalShortestPathsParameters( + expansionModel, + forwardFrontPrimerQuery, + forwardFrontRecursiveQuery, + backwardFrontPrimerQuery, + backwardFrontRecursiveQuery, + true, + ) + if err != nil { + return pgsql.Query{}, err + } + + projectionQuery := pgsql.Select{ + Projection: expansionModel.Projection, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{ + Name: expansionModel.Frame.Binding.Identifier.AsCompoundIdentifier(), + }, + Joins: []pgsql.Join{ + { + Table: expansionNodeTableReference(s.traversalStep.LeftNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{s.traversalStep.LeftNode.Identifier, pgsql.ColumnID}, pgsql.OperatorEquals, + pgsql.CompoundIdentifier{expansionModel.Frame.Binding.Identifier, expansionRootID}, + ), + }, + }, + { + Table: expansionNodeTableReference(s.traversalStep.RightNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{s.traversalStep.RightNode.Identifier, pgsql.ColumnID}, pgsql.OperatorEquals, + pgsql.CompoundIdentifier{expansionModel.Frame.Binding.Identifier, expansionNextID}, + ), + }, + }, + }, + }}, + } + s.applyShortestPathSeedProjectionConstraints(&projectionQuery, pgsql.OptionalAnd(forwardSeedProjectionConstraints, backwardSeedProjectionConstraints)) + s.appendUnwindSources(&projectionQuery) + s.applyShortestPathSelfEndpointGuard(&projectionQuery, expansionModel) + + directQuery := pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{ + pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}, + pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}, + pgsql.NewLiteral(int64(1), pgsql.Int8), + pgsql.NewLiteral(true, pgsql.Boolean), + pgd.Equals(pgd.StartID(s.traversalStep.Edge.Identifier), pgd.EndID(s.traversalStep.Edge.Identifier)), + pgd.ExpressionArrayLiteral(pgd.EntityID(s.traversalStep.Edge.Identifier)), + }, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{ + Name: validatedEndpoints.AsCompoundIdentifier(), + }, + Joins: []pgsql.Join{{ + Table: expansionEdgeTableReference(s.traversalStep.Edge.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.OptionalAnd( + pgd.Equals(expansionModel.EdgeStartColumn, pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}), + pgd.Equals(expansionModel.EdgeEndColumn, pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}), + ), + }, + }}, + }}, + Where: expansionModel.EdgeConstraints, + }, + OrderBy: []*pgsql.OrderBy{{ + Expression: pgd.EntityID(s.traversalStep.Edge.Identifier), + Ascending: true, + }}, + Limit: pgsql.NewLiteral(int64(1), pgsql.Int8), + } + + fallbackEndpointQuery := pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.Wildcard{}}, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{ + Name: validatedEndpoints.AsCompoundIdentifier(), + }, + }}, + Where: pgsql.ExistsExpression{ + Negated: true, + Subquery: pgsql.Subquery{ + Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.NewLiteral(int64(1), pgsql.Int8)}, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{ + Name: directHit.AsCompoundIdentifier(), + }, + }}, + }, + }, + }, + }, + }, + } + + stateQuery := pgsql.Query{ + Body: pgsql.SetOperation{ + LOperand: pgsql.Select{ + Projection: pgsql.Projection{pgsql.Wildcard{}}, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{ + Name: directHit.AsCompoundIdentifier(), + }, + }}, + }, + ROperand: pgsql.Select{ + Projection: pgsql.Projection{pgsql.Wildcard{}}, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{ + Name: workspaceSearch.AsCompoundIdentifier(), + }, + }}, + }, + Operator: pgsql.OperatorUnion, + All: true, + }, + } + + query := pgsql.Query{ + CommonTableExpressions: &pgsql.With{}, + Body: projectionQuery, + } + query.AddCTE(singletonEndpointValidationCTE(s.traversalStep, expansionModel)) + query.AddCTE(pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: directHit, + Shape: expansionColumns(), + }, + Materialized: &pgsql.Materialized{ + Materialized: true, + }, + Query: directQuery, + }) + query.AddCTE(pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: fallbackEndpoints, + }, + Query: fallbackEndpointQuery, + }) + query.AddCTE(shortestPathSearchCTEFrom(pgsql.FunctionBidirectionalSPHarness, expansionModel, harnessParameters, fallbackEndpoints, workspaceSearch)) + query.AddCTE(pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: expansionModel.Frame.Binding.Identifier, + Shape: expansionColumns(), + }, + Query: stateQuery, + }) + + return query, nil +} + +// BuildBiDirectionalAllShortestPathsRoot builds a bidirectional all-shortest-paths harness query. +func (s *ExpansionBuilder) BuildBiDirectionalAllShortestPathsRoot() (pgsql.Query, error) { + return s.buildBiDirectionalShortestPathsHarnessCall(pgsql.FunctionBidirectionalASPHarness) +} + +// boundEndpointFilterParameters renders the available bound endpoint inserts as harness SQL parameters. +func (s *ExpansionBuilder) boundEndpointFilterParameters() ([]pgsql.Expression, error) { + var ( + rootFilterStatement, hasRootFilter = s.boundRootIDsFilterStatement() + terminalFilterStatement, hasTerminalFilter = s.boundTerminalIDsFilterStatement() + pairFilterStatement, hasPairFilter = s.boundEndpointPairFilterStatement() + ) + + if !hasPairFilter { + pairFilterStatement, hasPairFilter = s.materializedEndpointPairFilterStatement() + } + + if !hasTerminalFilter { + terminalFilterStatement, hasTerminalFilter = s.unboundTerminalIDsFilterStatement() } if !hasRootFilter && !hasTerminalFilter && !hasPairFilter { @@ -1933,13 +3131,13 @@ func (s *ExpansionBuilder) boundEndpointFilterParameters() ([]pgsql.Expression, ) if hasPairFilter { - if formattedFilter, err := format.Statement(pairFilterStatement, format.NewOutputBuilder().WithMaterializedParameters(s.queryParameters)); err != nil { + if formattedFilter, err := format.Statement(pairFilterStatement, format.NewOutputBuilder().WithTargetGraph(s.graphID).WithMaterializedParameters(s.queryParameters)); err != nil { return nil, err } else { pairFilter = formattedFilter } } else if hasRootFilter { - if formattedFilter, err := format.Statement(rootFilterStatement, format.NewOutputBuilder().WithMaterializedParameters(s.queryParameters)); err != nil { + if formattedFilter, err := format.Statement(rootFilterStatement, format.NewOutputBuilder().WithTargetGraph(s.graphID).WithMaterializedParameters(s.queryParameters)); err != nil { return nil, err } else { rootFilter = formattedFilter @@ -1947,7 +3145,7 @@ func (s *ExpansionBuilder) boundEndpointFilterParameters() ([]pgsql.Expression, } if !hasPairFilter && hasTerminalFilter { - if formattedFilter, err := format.Statement(terminalFilterStatement, format.NewOutputBuilder().WithMaterializedParameters(s.queryParameters)); err != nil { + if formattedFilter, err := format.Statement(terminalFilterStatement, format.NewOutputBuilder().WithTargetGraph(s.graphID).WithMaterializedParameters(s.queryParameters)); err != nil { return nil, err } else { terminalFilter = formattedFilter @@ -1966,13 +3164,14 @@ func (s *ExpansionBuilder) boundEndpointFilterParameters() ([]pgsql.Expression, return filterParameters, nil } +// shortestPathsParameters renders a forward search's query fragments, depth limit, and filter inserts as harness parameters. func (s *ExpansionBuilder) shortestPathsParameters(expansionModel *Expansion, forwardFrontPrimerQuery pgsql.SetExpression, forwardFrontRecursiveQuery pgsql.SetExpression) ([]pgsql.Expression, error) { var ( harnessParameters []pgsql.Expression formatFragment = func(query pgsql.SetExpression) (string, error) { return format.Statement( nextFrontInsert(query), - format.NewOutputBuilder().WithMaterializedParameters(s.queryParameters)) + format.NewOutputBuilder().WithTargetGraph(s.graphID).WithMaterializedParameters(s.queryParameters)) } ) @@ -2010,13 +3209,34 @@ func (s *ExpansionBuilder) shortestPathsParameters(expansionModel *Expansion, fo return harnessParameters, nil } -func (s *ExpansionBuilder) bidirectionalAllShortestPathsParameters(expansionModel *Expansion, forwardFrontPrimerQuery pgsql.SetExpression, forwardFrontRecursiveQuery pgsql.SetExpression, backwardFrontPrimerQuery pgsql.SetExpression, backwardFrontRecursiveQuery pgsql.SetExpression) ([]pgsql.Expression, error) { +// shortestPathWorkspaceFragment rewrites generic workspace identifiers to the reusable bidirectional-search namespace. +func shortestPathWorkspaceFragment(fragment string) string { + return strings.NewReplacer( + "on conflict on constraint forward_visited_pkey", "on conflict on constraint bsp_forward_visited_pkey", + "on conflict on constraint backward_visited_pkey", "on conflict on constraint bsp_backward_visited_pkey", + "forward_visited", "pg_temp.bsp_forward_visited", + "backward_visited", "pg_temp.bsp_backward_visited", + "forward_front", "pg_temp.bsp_forward_front", + "backward_front", "pg_temp.bsp_backward_front", + "next_front", "pg_temp.bsp_next_front", + ).Replace(fragment) +} + +// bidirectionalShortestPathsParameters renders both search fronts and endpoint inputs for the bidirectional harness. +func (s *ExpansionBuilder) bidirectionalShortestPathsParameters(expansionModel *Expansion, forwardFrontPrimerQuery pgsql.SetExpression, forwardFrontRecursiveQuery pgsql.SetExpression, backwardFrontPrimerQuery pgsql.SetExpression, backwardFrontRecursiveQuery pgsql.SetExpression, useReusableWorkspace bool) ([]pgsql.Expression, error) { var ( harnessParameters []pgsql.Expression formatFragment = func(query pgsql.SetExpression) (string, error) { - return format.Statement( + fragment, err := format.Statement( nextFrontInsert(query), - format.NewOutputBuilder().WithMaterializedParameters(s.queryParameters)) + format.NewOutputBuilder().WithTargetGraph(s.graphID).WithMaterializedParameters(s.queryParameters)) + if err != nil { + return "", err + } + if useReusableWorkspace { + fragment = shortestPathWorkspaceFragment(fragment) + } + return fragment, nil } ) @@ -2064,16 +3284,57 @@ func (s *ExpansionBuilder) bidirectionalAllShortestPathsParameters(expansionMode } harnessParameters = append(harnessParameters, pgsql.NewLiteral(expansionModel.Options.MaxDepth.GetOr(translateDefaultMaxTraversalDepth), pgsql.Int)) + if expansionModel.UsesSingletonEndpointPair() { + harnessParameters = append(harnessParameters, + pgsql.ArrayLiteral{ + Values: []pgsql.Expression{expansionModel.SingletonRootID}, + CastType: pgsql.Int8Array, + }, + pgsql.ArrayLiteral{ + Values: []pgsql.Expression{expansionModel.SingletonTerminalID}, + CastType: pgsql.Int8Array, + }, + ) + if useReusableWorkspace { + harnessParameters = append(harnessParameters, pgsql.NewLiteral(expansionAllowsZeroDepth(expansionModel), pgsql.Boolean)) + } + return harnessParameters, nil + } if filterParameters, err := s.boundEndpointFilterParameters(); err != nil { return nil, err } else { + if useReusableWorkspace { + for idx, filterParameter := range filterParameters { + typeCast, isTypeCast := filterParameter.(pgsql.TypeCast) + if !isTypeCast { + continue + } + literal, isLiteral := typeCast.Expression.(pgsql.Literal) + if !isLiteral { + continue + } + if value, isString := literal.Value.(string); isString { + literal.Value = strings.NewReplacer( + "traversal_root_filter", "pg_temp.bsp_root_filter", + "traversal_terminal_filter", "pg_temp.bsp_terminal_filter", + "traversal_pair_filter", "pg_temp.bsp_pair_filter", + ).Replace(value) + typeCast.Expression = literal + filterParameters[idx] = typeCast + } + } + } harnessParameters = append(harnessParameters, filterParameters...) } + if useReusableWorkspace { + harnessParameters = append(harnessParameters, pgsql.NewLiteral(expansionAllowsZeroDepth(expansionModel), pgsql.Boolean)) + } return harnessParameters, nil } +// Build combines the configured expansion stages into a recursive CTE and final projection query. func (s *ExpansionBuilder) Build(expansionIdentifier pgsql.Identifier, commonTableExpressions ...pgsql.CommonTableExpression) pgsql.Query { expansionBody := pgsql.SetExpression(pgsql.SetOperation{ LOperand: s.PrimerStatement, @@ -2130,6 +3391,7 @@ func (s *ExpansionBuilder) Build(expansionIdentifier pgsql.Identifier, commonTab return query } +// projectionAliasExpressions indexes each projected alias or identifier by its underlying expression. func projectionAliasExpressions(projection pgsql.Projection) map[pgsql.Identifier]pgsql.Expression { aliases := make(map[pgsql.Identifier]pgsql.Expression) @@ -2158,6 +3420,7 @@ func projectionAliasExpressions(projection pgsql.Projection) map[pgsql.Identifie return aliases } +// rewriteCurrentFrameProjectionSetExpression substitutes current-frame aliases throughout a set expression. func rewriteCurrentFrameProjectionSetExpression(setExpression pgsql.SetExpression, frameID pgsql.Identifier, aliases map[pgsql.Identifier]pgsql.Expression) pgsql.SetExpression { switch typedSetExpression := setExpression.(type) { case pgsql.Select: @@ -2173,6 +3436,7 @@ func rewriteCurrentFrameProjectionSetExpression(setExpression pgsql.SetExpressio } } +// rewriteCurrentFrameProjectionQuery substitutes current-frame aliases throughout a query and its CTEs. func rewriteCurrentFrameProjectionQuery(query pgsql.Query, frameID pgsql.Identifier, aliases map[pgsql.Identifier]pgsql.Expression) pgsql.Query { query.Body = rewriteCurrentFrameProjectionSetExpression(query.Body, frameID, aliases) @@ -2188,6 +3452,7 @@ func rewriteCurrentFrameProjectionQuery(query pgsql.Query, frameID pgsql.Identif return query } +// rewriteCurrentFrameProjectionSelect substitutes current-frame aliases in every expression-bearing select clause. func rewriteCurrentFrameProjectionSelect(selectBody pgsql.Select, frameID pgsql.Identifier, aliases map[pgsql.Identifier]pgsql.Expression) pgsql.Select { for idx, selectItem := range selectBody.Projection { if rewritten, isSelectItem := rewriteCurrentFrameProjectionReferences(selectItem, frameID, aliases).(pgsql.SelectItem); isSelectItem { @@ -2215,6 +3480,7 @@ func rewriteCurrentFrameProjectionSelect(selectBody pgsql.Select, frameID pgsql. return selectBody } +// rewriteCurrentFrameProjectionReferences replaces qualified current-frame references with their projected expressions. func rewriteCurrentFrameProjectionReferences(expression pgsql.Expression, frameID pgsql.Identifier, aliases map[pgsql.Identifier]pgsql.Expression) pgsql.Expression { if expression == nil { return nil @@ -2256,12 +3522,22 @@ func rewriteCurrentFrameProjectionReferences(expression pgsql.Expression, frameI for idx, parameter := range typedExpression.Parameters { typedExpression.Parameters[idx] = rewriteCurrentFrameProjectionReferences(parameter, frameID, aliases) } + for _, orderBy := range typedExpression.OrderBy { + if orderBy != nil { + orderBy.Expression = rewriteCurrentFrameProjectionReferences(orderBy.Expression, frameID, aliases) + } + } return typedExpression case *pgsql.FunctionCall: for idx, parameter := range typedExpression.Parameters { typedExpression.Parameters[idx] = rewriteCurrentFrameProjectionReferences(parameter, frameID, aliases) } + for _, orderBy := range typedExpression.OrderBy { + if orderBy != nil { + orderBy.Expression = rewriteCurrentFrameProjectionReferences(orderBy.Expression, frameID, aliases) + } + } return typedExpression case pgsql.TypeCast: @@ -2280,6 +3556,7 @@ func rewriteCurrentFrameProjectionReferences(expression pgsql.Expression, frameI case *pgsql.EdgeArrayFromPathIDs: typedExpression.PathIDs = rewriteCurrentFrameProjectionReferences(typedExpression.PathIDs, frameID, aliases) + typedExpression.GraphID = rewriteCurrentFrameProjectionReferences(typedExpression.GraphID, frameID, aliases) return typedExpression case pgsql.ArrayLiteral: @@ -2410,6 +3687,7 @@ func rewriteCurrentFrameProjectionReferences(expression pgsql.Expression, frameI } } +// buildExpansionPatternRoot builds the seed and recursive query for a variable-length traversal that starts a pattern. func (s *Translator) buildExpansionPatternRoot(traversalStepContext TraversalStepContext, expansion *ExpansionBuilder) (pgsql.Query, error) { var ( traversalStep = traversalStepContext.CurrentStep @@ -2439,7 +3717,7 @@ func (s *Translator) buildExpansionPatternRoot(traversalStepContext TraversalSte return pgsql.Query{}, fmt.Errorf("left node is marked as bound but there is no previous frame to reference") } - boundSeed := newExpansionBoundNodeSeed(seedIdentifier, traversalStep.Frame.Previous, traversalStep.LeftNode.Identifier, seedConstraints) + boundSeed := newExpansionBoundNodeSeed(seedIdentifier, traversalStep.Frame.Previous, traversalStep.LeftNode, seedConstraints) seed = &boundSeed expansion.UseUnionAll = true } else if seedConstraints != nil { @@ -2567,11 +3845,11 @@ func (s *Translator) buildExpansionPatternRoot(traversalStepContext TraversalSte }, Joins: []pgsql.Join{ expansionNodeLookupJoin( - traversalStep.LeftNode.Identifier, + traversalStep.LeftNode, pgsql.CompoundIdentifier{expansionModel.Frame.Binding.Identifier, expansionRootID}, ), expansionNodeLookupJoin( - traversalStep.RightNode.Identifier, + traversalStep.RightNode, pgsql.CompoundIdentifier{expansionModel.Frame.Binding.Identifier, expansionNextID}, ), }, @@ -2585,7 +3863,7 @@ func (s *Translator) buildExpansionPatternRoot(traversalStepContext TraversalSte projectionConstraints, boundEndpointProjectionConstraint( previousProjectionFrameID, - traversalStep.LeftNode.Identifier, + traversalStep.LeftNode, expansionModel.Frame.Binding.Identifier, expansionRootID, ), @@ -2596,7 +3874,7 @@ func (s *Translator) buildExpansionPatternRoot(traversalStepContext TraversalSte projectionConstraints, boundEndpointProjectionConstraint( previousProjectionFrameID, - traversalStep.RightNode.Identifier, + traversalStep.RightNode, expansionModel.Frame.Binding.Identifier, expansionNextID, ), @@ -2618,6 +3896,7 @@ func (s *Translator) buildExpansionPatternRoot(traversalStepContext TraversalSte return expansion.Build(expansionModel.Frame.Binding.Identifier), nil } +// buildExpansionPatternStep builds the seed and recursive query for a variable-length traversal after an existing pattern step. func (s *Translator) buildExpansionPatternStep(traversalStepContext TraversalStepContext, expansion *ExpansionBuilder) (pgsql.Query, error) { var ( traversalStep = traversalStepContext.CurrentStep @@ -2625,7 +3904,7 @@ func (s *Translator) buildExpansionPatternStep(traversalStepContext TraversalSte seed = newExpansionBoundNodeSeed( expansionSeedIdentifier(expansionModel.Frame.Binding.Identifier), traversalStep.Frame.Previous, - traversalStep.LeftNode.Identifier, + traversalStep.LeftNode, expansionModel.PrimerNodeConstraints, ) ) @@ -2709,11 +3988,11 @@ func (s *Translator) buildExpansionPatternStep(traversalStepContext TraversalSte }, Joins: []pgsql.Join{ expansionNodeLookupJoin( - traversalStep.LeftNode.Identifier, + traversalStep.LeftNode, pgsql.CompoundIdentifier{expansionModel.Frame.Binding.Identifier, expansionRootID}, ), expansionNodeLookupJoin( - traversalStep.RightNode.Identifier, + traversalStep.RightNode, pgsql.CompoundIdentifier{expansionModel.Frame.Binding.Identifier, expansionNextID}, ), }, @@ -2733,6 +4012,7 @@ func (s *Translator) buildExpansionPatternStep(traversalStepContext TraversalSte return expansion.Build(expansionModel.Frame.Binding.Identifier, seed.CTE()), nil } +// expansionTerminalSatisfactionLocality partitions terminal predicates into traversal-local and deferred expressions. func expansionTerminalSatisfactionLocality(traversalStep *TraversalStep) (pgsql.Expression, pgsql.Expression) { return partitionConstraintByLocality( pgsql.Expression(traversalStep.Expansion.TerminalNodeSatisfactionProjection), @@ -2744,6 +4024,7 @@ func expansionTerminalSatisfactionLocality(traversalStep *TraversalStep) (pgsql. ) } +// applyExpansionSuffixPushdown pushes an eligible fixed-length suffix into the preceding variable expansion's terminal test. func applyExpansionSuffixPushdown(part *PatternPart) (int, error) { var applied int @@ -2754,7 +4035,7 @@ func applyExpansionSuffixPushdown(part *PatternPart) (int, error) { ) if candidateApplied, err := applyExpansionSuffixPushdownCandidate(currentStep, suffixSteps); err != nil { - return applied, err + return 0, err } else if candidateApplied { applied++ } @@ -2763,6 +4044,7 @@ func applyExpansionSuffixPushdown(part *PatternPart) (int, error) { return applied, nil } +// applyExpansionSuffixPushdownCandidate attaches a suffix-existence predicate when all suffix steps can be evaluated locally. func applyExpansionSuffixPushdownCandidate(currentStep *TraversalStep, suffixSteps []*TraversalStep) (bool, error) { if suffixSatisfaction, satisfied := expansionSuffixTerminalSatisfaction(currentStep, suffixSteps); satisfied { currentStep.Expansion.TerminalNodeConstraints = pgsql.OptionalAnd( @@ -2782,6 +4064,7 @@ func applyExpansionSuffixPushdownCandidate(currentStep *TraversalStep, suffixSte return false, nil } +// suffixEdgeLeftEndpoint returns the edge endpoint connected to a suffix step's left node for its direction. func suffixEdgeLeftEndpoint(edgeIdentifier pgsql.Identifier, direction graph.Direction) (pgsql.Expression, bool) { switch direction { case graph.DirectionOutbound: @@ -2793,6 +4076,7 @@ func suffixEdgeLeftEndpoint(edgeIdentifier pgsql.Identifier, direction graph.Dir } } +// suffixEdgeRightEndpoint returns the edge endpoint connected to a suffix step's right node for its direction. func suffixEdgeRightEndpoint(edgeIdentifier pgsql.Identifier, direction graph.Direction) (pgsql.Expression, bool) { switch direction { case graph.DirectionOutbound: @@ -2804,6 +4088,7 @@ func suffixEdgeRightEndpoint(edgeIdentifier pgsql.Identifier, direction graph.Di } } +// suffixBoundNodeIDReference resolves a suffix node to its identifier projection in the preceding frame. func suffixBoundNodeIDReference(currentStep *TraversalStep, node *BoundIdentifier) (pgsql.Expression, bool) { if currentStep == nil || currentStep.Frame == nil || @@ -2814,12 +4099,10 @@ func suffixBoundNodeIDReference(currentStep *TraversalStep, node *BoundIdentifie return nil, false } - return pgsql.RowColumnReference{ - Identifier: pgsql.CompoundIdentifier{currentStep.Frame.Previous.Binding.Identifier, node.Identifier}, - Column: pgsql.ColumnID, - }, true + return projectedNodeIDReference(currentStep.Frame.Previous.Binding.Identifier, node), true } +// suffixStepEdgeConstraints returns only the predicates local to a suffix step's edge binding. func suffixStepEdgeConstraints(step *TraversalStep) pgsql.Expression { if step == nil || step.EdgeConstraints == nil { return nil @@ -2833,6 +4116,7 @@ func suffixStepEdgeConstraints(step *TraversalStep) pgsql.Expression { return localConstraints } +// expansionSuffixTerminalSatisfaction builds an existence test proving that a fixed suffix continues from an expansion endpoint. func expansionSuffixTerminalSatisfaction(currentStep *TraversalStep, suffixSteps []*TraversalStep) (pgsql.Expression, bool) { if currentStep == nil || currentStep.Expansion == nil || @@ -2932,6 +4216,7 @@ func expansionSuffixTerminalSatisfaction(currentStep *TraversalStep, suffixSteps }, true } +// expansionLocalTerminalSatisfactionProjection projects the local terminal predicate, defaulting to true when none exists. func expansionLocalTerminalSatisfactionProjection(traversalStep *TraversalStep) (pgsql.SelectItem, error) { localSatisfiedConstraint, _ := expansionTerminalSatisfactionLocality(traversalStep) @@ -2942,8 +4227,17 @@ func expansionLocalTerminalSatisfactionProjection(traversalStep *TraversalStep) return pgsql.As[pgsql.SelectItem](localSatisfiedConstraint) } +// buildExpansionPrimerProjection constructs the root, endpoint, depth, satisfaction, cycle, and path columns for the first edge. func (s *Translator) buildExpansionPrimerProjection(traversalStep *TraversalStep) ([]pgsql.SelectItem, error) { expansionModel := traversalStep.Expansion + isCycleProjection := pgsql.SelectItem(pgsql.NewLiteral(false, pgsql.Boolean)) + if expansionModel.Options.FindShortestPath || expansionModel.Options.FindAllShortestPaths { + isCycleProjection = pgsql.NewBinaryExpression( + expansionModel.EdgeStartColumn, + pgsql.OperatorEquals, + expansionModel.EdgeEndColumn, + ) + } if expansionModel.TerminalNodeSatisfactionProjection != nil { satisfiedProjection, err := expansionLocalTerminalSatisfactionProjection(traversalStep) @@ -2956,11 +4250,7 @@ func (s *Translator) buildExpansionPrimerProjection(traversalStep *TraversalStep expansionModel.EdgeEndColumn, pgsql.NewLiteral(1, pgsql.Int), satisfiedProjection, - pgsql.NewBinaryExpression( - expansionModel.EdgeStartColumn, - pgsql.OperatorEquals, - expansionModel.EdgeEndColumn, - ), + isCycleProjection, pgsql.ArrayLiteral{ Values: []pgsql.Expression{ pgsql.CompoundIdentifier{traversalStep.Edge.Identifier, pgsql.ColumnID}, @@ -2973,11 +4263,7 @@ func (s *Translator) buildExpansionPrimerProjection(traversalStep *TraversalStep expansionModel.EdgeEndColumn, pgsql.NewLiteral(1, pgsql.Int), pgsql.NewLiteral(false, pgsql.Boolean), - pgsql.NewBinaryExpression( - expansionModel.EdgeStartColumn, - pgsql.OperatorEquals, - expansionModel.EdgeEndColumn, - ), + isCycleProjection, pgsql.ArrayLiteral{ Values: []pgsql.Expression{ pgsql.CompoundIdentifier{traversalStep.Edge.Identifier, pgsql.ColumnID}, @@ -2987,6 +4273,7 @@ func (s *Translator) buildExpansionPrimerProjection(traversalStep *TraversalStep } } +// expansionRecursivePathExpression appends or prepends the next edge identifier according to traversal direction. func expansionRecursivePathExpression(traversalStep *TraversalStep) *pgsql.BinaryExpression { var ( expansionModel = traversalStep.Expansion @@ -3001,6 +4288,7 @@ func expansionRecursivePathExpression(traversalStep *TraversalStep) *pgsql.Binar return pgsql.NewBinaryExpression(path, pgsql.OperatorConcatenate, edgeID) } +// buildExpansionRecursiveProjection advances the expansion state and accumulated path by one edge. func (s *Translator) buildExpansionRecursiveProjection(traversalStep *TraversalStep) ([]pgsql.SelectItem, error) { expansionModel := traversalStep.Expansion @@ -3048,6 +4336,7 @@ func (s *Translator) buildExpansionRecursiveProjection(traversalStep *TraversalS } } +// buildExpansionProjectionConstraints combines join, depth, satisfaction, and deferred predicates for projected expansion rows. func (s *Translator) buildExpansionProjectionConstraints(traversalStepContext TraversalStepContext) (pgsql.Expression, error) { var ( currentStep = traversalStepContext.CurrentStep @@ -3061,16 +4350,13 @@ func (s *Translator) buildExpansionProjectionConstraints(traversalStepContext Tr if previousStep != nil { joinCondition = pgd.Equals( - pgsql.RowColumnReference{ - Identifier: pgsql.CompoundIdentifier{previousStep.Frame.Binding.Identifier, currentStep.LeftNode.Identifier}, - Column: pgsql.ColumnID, - }, + projectedNodeIDReference(previousStep.Frame.Binding.Identifier, currentStep.LeftNode), pgd.Column(expansionModel.Frame.Binding.Identifier, expansionRootID), ) } if constraints, err = s.treeTranslator.ConsumeConstraintsFromVisibleSet(expansionModel.Frame.Visible); err != nil { - return projectionConstraints, err + return nil, err } else { // Constraints that target the terminal node may crop up here where it's finally in scope. Additionally, // only accept paths that are marked satisfied from the recursive descent CTE @@ -3082,7 +4368,7 @@ func (s *Translator) buildExpansionProjectionConstraints(traversalStepContext Tr } if projectionConstraints, err = ConjoinExpressions(s.kindMapper, expressions); err != nil { - return projectionConstraints, err + return nil, err } // Append any deferred (non-local) constraints onto the projection constraints @@ -3091,7 +4377,7 @@ func (s *Translator) buildExpansionProjectionConstraints(traversalStepContext Tr } } else { if projectionConstraints, err = ConjoinExpressions(s.kindMapper, []pgsql.Expression{constraints.Expression, joinCondition}); err != nil { - return projectionConstraints, err + return nil, err } } } @@ -3111,6 +4397,7 @@ func (s *Translator) buildExpansionProjectionConstraints(traversalStepContext Tr return projectionConstraints, nil } +// translateTraversalPatternPartWithExpansion lowers a variable-length pattern step and updates frame bindings for its projected state. func (s *Translator) translateTraversalPatternPartWithExpansion(part *PatternPart, stepIndex int, isFirstTraversalStep bool, traversalStep *TraversalStep, allowProjectionPruning bool) error { expansionModel := traversalStep.Expansion @@ -3119,6 +4406,30 @@ func (s *Translator) translateTraversalPatternPartWithExpansion(part *PatternPar if err := s.translateExpansionConstraints(part, stepIndex, isFirstTraversalStep, traversalStep, expansionModel); err != nil { return err } + if decision, selected := s.shortestPathExecutorDecision(part, stepIndex); selected { + expansionModel.ShortestPathExecutor = decision.SelectedExecutor + expansionModel.ShortestPathTarget = decision.Target + expansionModel.ShortestPathStateLimit = decision.StateLimit + expansionModel.ShortestPathFrontierLimit = decision.FrontierLimit + expansionModel.ShortestPathPredecessorLimit = decision.PredecessorLimit + expansionModel.ShortestPathEnumerationLimit = decision.EnumerationLimit + expansionModel.ShortestPathOutputBytesLimit = decision.OutputBytesLimit + if !expansionModel.Options.MaxDepth.Set && decision.MaximumDepth > 0 { + expansionModel.Options.MaxDepth = models.OptionalValue(decision.MaximumDepth) + } + if decision.SelectedExecutor == optimize.ShortestPathExecutorS3Unidirectional || + decision.SelectedExecutor == optimize.ShortestPathExecutorI1CanonicalDistance || + decision.SelectedExecutor == optimize.ShortestPathExecutorS4CanonicalDistance || + decision.SelectedExecutor == optimize.ShortestPathExecutorB1AlternatingNodeDistance || + decision.SelectedExecutor == optimize.ShortestPathExecutorB2SmallerCurrentLevelDistance { + expansionModel.PathBinding.DistanceOnly = true + expansionModel.PathBinding.DataType = pgsql.Int + if part.PatternBinding != nil { + part.PatternBinding.DistanceOnly = true + part.PatternBinding.DataType = pgsql.Int + } + } + } // Export the path from the traversal's scope traversalStep.Frame.Export(expansionModel.PathBinding.Identifier) @@ -3159,6 +4470,11 @@ func (s *Translator) translateTraversalPatternPartWithExpansion(part *PatternPar // Remove the previous projections of the root and terminal node to reproject them after expansion traversalStep.LeftNode.Dematerialize() traversalStep.RightNode.Dematerialize() + leftNodeIDOnly := s.applyIDOnlyNodeProjection(part, stepIndex, traversalStep.LeftNode) + rightNodeIDOnly := s.applyIDOnlyNodeProjection(part, stepIndex, traversalStep.RightNode) + if leftNodeIDOnly || rightNodeIDOnly { + s.recordLowering(optimize.LoweringFieldRequirements) + } if boundProjections, err := buildVisibleProjections(s.scope); err != nil { return err @@ -3185,6 +4501,12 @@ func (s *Translator) translateTraversalPatternPartWithExpansion(part *PatternPar traversalStep.Projection = boundProjections.Items } + if expansionModel.ShortestPathExecutor == optimize.ShortestPathExecutorS3EdgeM0 || expansionModel.ShortestPathExecutor == optimize.ShortestPathExecutorS4CanonicalWitness || expansionModel.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalWitness || expansionModel.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness { + expansionModel.PathBinding.DataType = pgsql.PathComposite + if part.PatternBinding != nil { + part.PatternBinding.DataType = pgsql.PathComposite + } + } if expansionModel.Options.FindShortestPath || expansionModel.Options.FindAllShortestPaths { if err := s.translateShortestPathTraversal(part, stepIndex, traversalStep, expansionModel); err != nil { @@ -3195,6 +4517,7 @@ func (s *Translator) translateTraversalPatternPartWithExpansion(part *PatternPar return nil } +// translateExpansionConstraints consumes applicable constraints and partitions them among expansion bindings and outer frames. func (s *Translator) translateExpansionConstraints(part *PatternPart, stepIndex int, isFirstTraversalStep bool, step *TraversalStep, expansionModel *Expansion) error { if constraints, err := consumePatternConstraints(isFirstTraversalStep, recursivePattern, step, s.treeTranslator); err != nil { return err @@ -3270,6 +4593,7 @@ func (s *Translator) translateExpansionConstraints(part *PatternPart, stepIndex return nil } +// translateShortestPathTraversal selects and parameterizes the physical shortest-path harness for a traversal step. func (s *Translator) translateShortestPathTraversal(part *PatternPart, stepIndex int, traversalStep *TraversalStep, expansionModel *Expansion) error { var ( useBidirectionalSearch bool @@ -3282,12 +4606,49 @@ func (s *Translator) translateShortestPathTraversal(part *PatternPart, stepIndex return err } - expansionModel.UseBidirectionalSearch = useBidirectionalSearch + inlineShortest := expansionModel.ShortestPathExecutor == optimize.ShortestPathExecutorS3Unidirectional || + expansionModel.ShortestPathExecutor == optimize.ShortestPathExecutorS3EdgeM0 || + expansionModel.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalDistance || + expansionModel.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalWitness || + expansionModel.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness || + expansionModel.ShortestPathExecutor == optimize.ShortestPathExecutorASPI1DAG + expansionModel.UseBidirectionalSearch = useBidirectionalSearch && !inlineShortest && !compactShortestExecutor(expansionModel.ShortestPathExecutor) expansionModel.HasExplicitEndpointInequality = s.treeTranslator.HasEndpointInequality( traversalStep.LeftNode.Identifier, traversalStep.RightNode.Identifier, ) s.applyShortestPathFilterMaterialization(part, stepIndex, traversalStep, expansionModel) + if (compactShortestExecutor(expansionModel.ShortestPathExecutor) || expansionModel.UseBidirectionalSearch || inlineShortest) && + !traversalStep.LeftNodeBound && + !traversalStep.RightNodeBound && + (!expansionModel.Options.MinDepth.Set || expansionModel.Options.MinDepth.Value > 0 || inlineShortest || compactShortestExecutor(expansionModel.ShortestPathExecutor)) { + rootAnchor, hasRootAnchor := singletonIDAnchor(expansionModel.PrimerNodeConstraints, traversalStep.LeftNode.Identifier) + terminalAnchor, hasTerminalAnchor := singletonIDAnchor(expansionModel.TerminalNodeConstraints, traversalStep.RightNode.Identifier) + if hasRootAnchor && hasTerminalAnchor { + var err error + if expansionModel.SingletonRootID, err = s.liftSingletonIDAnchor(rootAnchor); err != nil { + return err + } + expansionModel.PrimerNodeConstraints = replaceSingletonIDAnchor( + expansionModel.PrimerNodeConstraints, + traversalStep.LeftNode.Identifier, + expansionModel.SingletonRootID, + ) + if expansionModel.SingletonTerminalID, err = s.liftSingletonIDAnchor(terminalAnchor); err != nil { + return err + } + expansionModel.TerminalNodeConstraints = replaceSingletonIDAnchor( + expansionModel.TerminalNodeConstraints, + traversalStep.RightNode.Identifier, + expansionModel.SingletonTerminalID, + ) + expansionModel.UseMaterializedEndpointPairFilter = false + } + } + + if inlineShortest || compactShortestExecutor(expansionModel.ShortestPathExecutor) { + return nil + } // If this query is a shortest-path look up, the translator will have to use a function harness for // traversal. As such, query fragments for the traversal harness will have to be passed by the parameters @@ -3323,6 +4684,38 @@ func (s *Translator) translateShortestPathTraversal(part *PatternPart, stepIndex return nil } +// liftSingletonIDAnchor converts a literal or parameter singleton identifier into a typed harness parameter. +func (s *Translator) liftSingletonIDAnchor(expression pgsql.Expression) (pgsql.Expression, error) { + switch typedExpression := unwrapParenthetical(expression).(type) { + case pgsql.Literal: + parameterBinding, err := s.scope.DefineNew(pgsql.ParameterIdentifier) + if err != nil { + return nil, err + } + parameter, err := pgsql.AsParameter(parameterBinding.Identifier, typedExpression.Value) + if err != nil { + return nil, err + } + parameter.CastType = pgsql.Int8 + parameterBinding.Parameter = parameter + s.translation.Parameters[parameterBinding.Identifier.String()] = typedExpression.Value + return parameter, nil + + case pgsql.Parameter: + typedExpression.CastType = pgsql.Int8 + return typedExpression, nil + case *pgsql.Parameter: + copy := *typedExpression + copy.CastType = pgsql.Int8 + return ©, nil + case pgsql.TypeCast: + return s.liftSingletonIDAnchor(typedExpression.Expression) + default: + return nil, fmt.Errorf("unsupported singleton endpoint expression: %T", expression) + } +} + +// translateNonTraversalPatternPart lowers a fixed-length pattern part into a new frame and materialized projection. func (s *Translator) translateNonTraversalPatternPart(part *PatternPart) error { if nextFrame, err := s.scope.PushFrame(); err != nil { return err diff --git a/cypher/models/pgsql/translate/expansion_all_shortest_inline.go b/cypher/models/pgsql/translate/expansion_all_shortest_inline.go new file mode 100644 index 00000000..034520f3 --- /dev/null +++ b/cypher/models/pgsql/translate/expansion_all_shortest_inline.go @@ -0,0 +1,699 @@ +package translate + +import ( + "errors" + + "github.com/specterops/dawgs/cypher/models" + "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/pgd" + "github.com/specterops/dawgs/graph" +) + +const ( + aspI1Distance pgsql.Identifier = "asp_i1_distance" + aspI1Direct pgsql.Identifier = "asp_i1_direct" + aspI1Preflight pgsql.Identifier = "asp_i1_preflight" + aspI1PreflightBounded pgsql.Identifier = "asp_i1_preflight_bounded" + aspI1DistanceBounded pgsql.Identifier = "asp_i1_distance_bounded" + aspI1Target pgsql.Identifier = "asp_i1_target" + aspI1Predecessor pgsql.Identifier = "asp_i1_predecessor" + aspI1PredecessorBounded pgsql.Identifier = "asp_i1_predecessor_bounded" + aspI1Paths pgsql.Identifier = "asp_i1_paths" + aspI1PathsBounded pgsql.Identifier = "asp_i1_paths_bounded" + aspI1Shortest pgsql.Identifier = "asp_i1_shortest" + aspI1Admission pgsql.Identifier = "asp_i1_admission" + aspI1Decision pgsql.Identifier = "asp_i1_decision" + aspI1CandidateMarker pgsql.Identifier = "asp_i1_candidate_marker" + aspI1FallbackMarker pgsql.Identifier = "asp_i1_fallback_marker" + aspI1CandidateBody pgsql.Identifier = "asp_i1_candidate_body" + aspI1FallbackBody pgsql.Identifier = "asp_i1_fallback_body" + aspI1CandidateRows pgsql.Identifier = "asp_i1_candidate_rows" + aspI1FallbackRows pgsql.Identifier = "asp_i1_fallback_rows" + aspI1NodeID pgsql.Identifier = "node_id" + aspI1PredecessorID pgsql.Identifier = "predecessor_id" + aspI1EdgeID pgsql.Identifier = "edge_id" + aspI1UseCandidate pgsql.Identifier = "use_candidate" + aspI1UseFallback pgsql.Identifier = "use_fallback" + aspI1Overflow pgsql.Identifier = "overflow" + aspI1NoPath pgsql.Identifier = "no_path" + aspI1RuntimeReceipt pgsql.Identifier = "runtime_receipt" + aspI1RuntimeAttestationFn pgsql.Identifier = "record_requested_traversal_runtime_attestation_v1" + aspI1ColumnSizeFn pgsql.Identifier = "pg_column_size" +) + +func aspI1Aliased(expression pgsql.Expression, alias pgsql.Identifier) pgsql.SelectItem { + return &pgsql.AliasedExpression{Expression: expression, Alias: models.OptionalValue(alias)} +} + +func aspI1Table(alias, binding pgsql.Identifier) pgsql.TableReference { + return pgsql.TableReference{Name: alias.AsCompoundIdentifier(), Binding: models.OptionalValue(binding)} +} + +func aspI1CanonicalProjection(source pgsql.Identifier) pgsql.Projection { + return pgsql.Projection{ + aspI1Aliased(pgsql.CompoundIdentifier{source, expansionRootID}, expansionRootID), + aspI1Aliased(pgsql.CompoundIdentifier{source, expansionNextID}, expansionNextID), + aspI1Aliased(pgsql.CompoundIdentifier{source, expansionDepth}, expansionDepth), + aspI1Aliased(pgsql.CompoundIdentifier{source, expansionSatisfied}, expansionSatisfied), + aspI1Aliased(pgsql.CompoundIdentifier{source, expansionIsCycle}, expansionIsCycle), + aspI1Aliased(pgsql.CompoundIdentifier{source, expansionPath}, expansionPath), + } +} + +func aspI1OverflowAny(overflows ...pgsql.Expression) pgsql.Expression { + var result pgsql.Expression + for _, overflow := range overflows { + if result == nil { + result = overflow + } else { + result = pgsql.NewBinaryExpression(result, pgsql.OperatorOr, overflow) + } + } + return result +} + +func aspI1OutputBytes(source pgsql.Identifier) pgsql.Subquery { + return pgsql.Subquery{Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.FunctionCall{ + Function: pgsql.FunctionCoalesce, + Parameters: []pgsql.Expression{ + pgsql.FunctionCall{ + Function: pgsql.FunctionSum, + Parameters: []pgsql.Expression{pgsql.FunctionCall{ + Function: aspI1ColumnSizeFn, + Parameters: []pgsql.Expression{pgsql.CompoundIdentifier{source, expansionPath}}, + }}, + }, + pgsql.NewLiteral(int64(0), pgsql.Int8), + }, + CastType: pgsql.Int8, + }}, + From: []pgsql.FromClause{tableFrom(source)}, + }}} +} + +func aspI1Marker(alias pgsql.Identifier, selected pgsql.Identifier) pgsql.CommonTableExpression { + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: alias}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{aspI1Aliased(pgsql.NewLiteral(true, pgsql.Boolean), orientationArmExecuted)}, + From: []pgsql.FromClause{tableFrom(aspI1Decision)}, + Where: pgsql.CompoundIdentifier{aspI1Decision, selected}, + }}, + } +} + +type inlinePredecessorDAGMode struct { + identity optimize.ShortestPathExecutor + fallback optimize.ShortestPathExecutor + oneWitness bool +} + +// BuildInlineAllShortestPathsDAGRoot emits the guarded ASP-I1 predecessor-DAG statement. +func (s *ExpansionBuilder) BuildInlineAllShortestPathsDAGRoot() (pgsql.Query, error) { + return s.buildInlinePredecessorDAGRoot(inlinePredecessorDAGMode{ + identity: optimize.ShortestPathExecutorASPI1DAG, + fallback: optimize.ShortestPathExecutorASPA1DAG, + }) +} + +// BuildInlineCanonicalShortestPathRoot emits one guarded canonical witness and +// invokes compact S4 exactly once if any candidate resource sentinel overflows. +func (s *ExpansionBuilder) BuildInlineCanonicalShortestPathRoot() (pgsql.Query, error) { + return s.buildInlinePredecessorDAGRoot(inlinePredecessorDAGMode{ + identity: optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, + fallback: optimize.ShortestPathExecutorS4CanonicalWitness, + oneWitness: true, + }) +} + +// buildInlinePredecessorDAGRoot shares guarded minimum-distance and predecessor +// primitives between the ASP enumerator and the singleton canonical witness. +// Recursive producers are consumed only through materialized cap+1 relations; +// complementary markers prevent candidate/fallback row mixing. +func (s *ExpansionBuilder) buildInlinePredecessorDAGRoot(mode inlinePredecessorDAGMode) (pgsql.Query, error) { + const validatedEndpoints pgsql.Identifier = "singleton_endpoints" + + expansionModel := s.traversalStep.Expansion + if !expansionModel.UsesSingletonEndpointPair() { + return pgsql.Query{}, errors.New(string(mode.identity) + " requires one validated endpoint pair") + } + if expansionModel.Options.MinDepth.GetOr(1) != 1 || !expansionModel.Options.MaxDepth.Set || expansionModel.Options.MaxDepth.Value < 1 || expansionModel.Options.MaxDepth.Value > 64 { + return pgsql.Query{}, errors.New(string(mode.identity) + " requires min depth 1 and bounded max depth <= 64") + } + if s.traversalStep.Direction != graph.DirectionOutbound && s.traversalStep.Direction != graph.DirectionInbound { + return pgsql.Query{}, errors.New(string(mode.identity) + " requires a directed traversal") + } + for _, limit := range []int64{ + expansionModel.ShortestPathStateLimit, + expansionModel.ShortestPathPredecessorLimit, + expansionModel.ShortestPathEnumerationLimit, + expansionModel.ShortestPathOutputBytesLimit, + } { + if limit <= 0 { + return pgsql.Query{}, errors.New(string(mode.identity) + " requires positive bounded limits") + } + } + + endpointCTE := singletonEndpointValidationCTE(s.traversalStep, expansionModel) + endpointSelect := endpointCTE.Query.Body.(pgsql.Select) + endpointSelect.Where = pgsql.OptionalAnd(endpointSelect.Where, shortestPathSelfEndpointGuardCase( + pgd.EntityID(s.traversalStep.LeftNode.Identifier), + pgd.EntityID(s.traversalStep.RightNode.Identifier), + )) + endpointCTE.Query.Body = endpointSelect + + // Exact one/two-hop preflights prevent the recursive distance producer from + // exploring an irrelevant tail when the target is already shallow. The + // preflight itself is consumed through the enumeration cap+1 sentinel so a + // large parallel-edge result falls back before exposing partial rows. + firstEdge := s.traversalStep.Edge.Identifier + secondEdge := pgsql.Identifier("asp_i1_preflight_edge_2") + edgeScope := func(alias pgsql.Identifier) pgsql.Expression { + var scope pgsql.Expression = pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{alias, pgsql.ColumnGraphID}, pgsql.OperatorEquals, pgsql.NewLiteral(s.graphID, pgsql.Int4), + ) + if len(expansionModel.RelationshipKindIDs) > 0 { + scope = pgsql.OptionalAnd(scope, pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{alias, pgsql.ColumnKindID}, pgsql.OperatorEquals, + pgsql.NewAnyExpressionHinted(pgsql.NewLiteral(append([]int16(nil), expansionModel.RelationshipKindIDs...), pgsql.Int2Array)), + )) + } + return scope + } + startColumn, endColumn := pgsql.ColumnStartID, pgsql.ColumnEndID + if s.traversalStep.Direction == graph.DirectionInbound { + startColumn, endColumn = endColumn, startColumn + } + directWhere := pgsql.OptionalAnd(edgeScope(firstEdge), pgsql.OptionalAnd( + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{firstEdge, startColumn}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}), + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{firstEdge, endColumn}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}), + )) + direct := pgsql.Select{ + Projection: pgsql.Projection{ + aspI1Aliased(pgsql.NewLiteral(int64(1), pgsql.Int8), expansionDepth), + aspI1Aliased(pgsql.ArrayLiteral{Values: []pgsql.Expression{pgsql.CompoundIdentifier{firstEdge, pgsql.ColumnID}}, CastType: pgsql.Int8Array}, expansionPath), + }, + From: []pgsql.FromClause{tableFrom(validatedEndpoints), {Source: expansionEdgeTableReference(firstEdge)}}, + Where: directWhere, + } + directCTE := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: aspI1Direct, Shape: pgsql.NewRecordShape([]pgsql.Identifier{expansionDepth, expansionPath})}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: direct}, + } + directExists := pgsql.ExistsExpression{Subquery: pgsql.Subquery{Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.NewLiteral(int64(1), pgsql.Int8)}, From: []pgsql.FromClause{tableFrom(aspI1Direct)}, + }, Limit: pgsql.NewLiteral(int64(1), pgsql.Int8)}}} + secondJoin := pgsql.OptionalAnd(edgeScope(secondEdge), pgsql.OptionalAnd( + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{secondEdge, startColumn}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{firstEdge, endColumn}), + pgsql.NewLiteral(true, pgsql.Boolean), + )) + twoHop := pgsql.Select{ + Projection: pgsql.Projection{ + aspI1Aliased(pgsql.NewLiteral(int64(2), pgsql.Int8), expansionDepth), + aspI1Aliased(pgsql.ArrayLiteral{Values: []pgsql.Expression{ + pgsql.CompoundIdentifier{firstEdge, pgsql.ColumnID}, pgsql.CompoundIdentifier{secondEdge, pgsql.ColumnID}, + }, CastType: pgsql.Int8Array}, expansionPath), + }, + From: []pgsql.FromClause{tableFrom(validatedEndpoints), {Source: expansionEdgeTableReference(firstEdge), Joins: []pgsql.Join{{ + Table: expansionEdgeTableReference(secondEdge), JoinOperator: pgsql.JoinOperator{JoinType: pgsql.JoinTypeInner, Constraint: secondJoin}, + }}}}, + Where: pgsql.OptionalAnd( + pgd.Not(directExists), + pgsql.OptionalAnd(edgeScope(firstEdge), pgsql.OptionalAnd( + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{firstEdge, startColumn}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}), + pgsql.OptionalAnd( + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{secondEdge, endColumn}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}), + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{firstEdge, pgsql.ColumnID}, pgsql.OperatorNotEquals, pgsql.CompoundIdentifier{secondEdge, pgsql.ColumnID}), + ), + )), + ), + } + preflight := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: aspI1Preflight, Shape: pgsql.NewRecordShape([]pgsql.Identifier{expansionDepth, expansionPath})}, + Query: pgsql.Query{Body: pgsql.SetOperation{ + LOperand: pgsql.Select{ + Projection: pgsql.Projection{ + pgsql.CompoundIdentifier{aspI1Direct, expansionDepth}, + pgsql.CompoundIdentifier{aspI1Direct, expansionPath}, + }, + From: []pgsql.FromClause{tableFrom(aspI1Direct)}, + }, + ROperand: twoHop, Operator: pgsql.OperatorUnion, All: true, + }}, + } + preflightBounded := boundedTraversalStateProbe( + aspI1PreflightBounded, aspI1Preflight, []pgsql.Identifier{expansionDepth, expansionPath}, expansionModel.ShortestPathEnumerationLimit, + ) + preflightOverflow := boundedProbeOverflow(aspI1PreflightBounded, expansionModel.ShortestPathEnumerationLimit) + preflightExists := pgsql.ExistsExpression{Subquery: pgsql.Subquery{Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.NewLiteral(int64(1), pgsql.Int8)}, From: []pgsql.FromClause{tableFrom(aspI1PreflightBounded)}, + }, Limit: pgsql.NewLiteral(int64(1), pgsql.Int8)}}} + + anchor := pgsql.Select{ + Projection: pgsql.Projection{ + pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}, + pgsql.NewLiteral(int64(0), pgsql.Int8), + }, + From: []pgsql.FromClause{tableFrom(validatedEndpoints)}, + Where: pgd.Not(preflightExists), + } + recursive := pgsql.Select{ + Projection: pgsql.Projection{ + expansionModel.EdgeEndColumn, + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{aspI1Distance, expansionDepth}, + pgsql.OperatorAdd, + pgsql.NewLiteral(int64(1), pgsql.Int8), + ), + }, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: aspI1Distance.AsCompoundIdentifier()}, + Joins: []pgsql.Join{{ + Table: expansionEdgeTableReference(s.traversalStep.Edge.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + expansionModel.EdgeStartColumn, + pgsql.OperatorEquals, + pgsql.CompoundIdentifier{aspI1Distance, aspI1NodeID}, + ), + }, + }}, + }}, + Where: pgsql.OptionalAnd( + expansionModel.EdgeConstraints, + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{aspI1Distance, expansionDepth}, + pgsql.OperatorLessThan, + pgsql.NewLiteral(expansionModel.Options.MaxDepth.Value, pgsql.Int8), + ), + ), + } + + distance := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: aspI1Distance, Shape: pgsql.NewRecordShape([]pgsql.Identifier{aspI1NodeID, expansionDepth})}, + Query: pgsql.Query{Body: pgsql.SetOperation{ + LOperand: anchor, + ROperand: recursive, + Operator: pgsql.OperatorUnion, + }}, + } + distanceBounded := boundedTraversalStateProbe( + aspI1DistanceBounded, + aspI1Distance, + []pgsql.Identifier{aspI1NodeID, expansionDepth}, + expansionModel.ShortestPathStateLimit, + ) + stateOverflow := boundedProbeOverflow(aspI1DistanceBounded, expansionModel.ShortestPathStateLimit) + + target := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: aspI1Target, Shape: pgsql.NewRecordShape([]pgsql.Identifier{expansionDepth})}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.CompoundIdentifier{aspI1DistanceBounded, expansionDepth}}, + From: []pgsql.FromClause{tableFrom(aspI1DistanceBounded)}, + Where: pgsql.OptionalAnd( + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{aspI1DistanceBounded, aspI1NodeID}, + pgsql.OperatorEquals, + pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}, + ), + pgd.Not(stateOverflow), + ), + }, + OrderBy: []*pgsql.OrderBy{{Expression: pgsql.CompoundIdentifier{aspI1DistanceBounded, expansionDepth}, Ascending: true}}, + Limit: pgsql.NewLiteral(int64(1), pgsql.Int8), + }, + } + // The endpoint relation is correlated through a scalar subquery so target + // retains a single FROM source and a stable materialization shape. + targetSelect := target.Query.Body.(pgsql.Select) + targetTerminal := pgsql.Subquery{Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}}, + From: []pgsql.FromClause{tableFrom(validatedEndpoints)}, + }}} + targetSelect.Where = pgsql.OptionalAnd( + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{aspI1DistanceBounded, aspI1NodeID}, + pgsql.OperatorEquals, + targetTerminal, + ), + pgd.Not(stateOverflow), + ) + target.Query.Body = targetSelect + + child, prior := pgsql.Identifier("asp_i1_child"), pgsql.Identifier("asp_i1_prior") + predecessorEdgeConstraint := pgsql.OptionalAnd( + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{prior, expansionDepth}, + pgsql.OperatorEquals, + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{child, expansionDepth}, pgsql.OperatorSubtract, pgsql.NewLiteral(int64(1), pgsql.Int8)), + ), + expansionModel.EdgeConstraints, + ) + if s.traversalStep.Direction == graph.DirectionOutbound { + predecessorEdgeConstraint = pgsql.OptionalAnd(predecessorEdgeConstraint, + pgsql.OptionalAnd( + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{s.traversalStep.Edge.Identifier, pgsql.ColumnStartID}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{prior, aspI1NodeID}), + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{s.traversalStep.Edge.Identifier, pgsql.ColumnEndID}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{child, aspI1NodeID}), + ), + ) + } else { + predecessorEdgeConstraint = pgsql.OptionalAnd(predecessorEdgeConstraint, + pgsql.OptionalAnd( + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{s.traversalStep.Edge.Identifier, pgsql.ColumnEndID}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{prior, aspI1NodeID}), + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{s.traversalStep.Edge.Identifier, pgsql.ColumnStartID}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{child, aspI1NodeID}), + ), + ) + } + + predecessor := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: aspI1Predecessor, Shape: pgsql.NewRecordShape([]pgsql.Identifier{ + aspI1NodeID, expansionDepth, aspI1PredecessorID, aspI1EdgeID, + })}, + Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{ + pgsql.CompoundIdentifier{child, aspI1NodeID}, + pgsql.CompoundIdentifier{child, expansionDepth}, + pgsql.CompoundIdentifier{prior, aspI1NodeID}, + pgsql.CompoundIdentifier{s.traversalStep.Edge.Identifier, pgsql.ColumnID}, + }, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: aspI1Target.AsCompoundIdentifier()}, + Joins: []pgsql.Join{ + {Table: aspI1Table(aspI1DistanceBounded, child), JoinOperator: pgsql.JoinOperator{JoinType: pgsql.JoinTypeInner, Constraint: pgsql.OptionalAnd( + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{child, expansionDepth}, pgsql.OperatorGreaterThan, pgsql.NewLiteral(int64(0), pgsql.Int8)), + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{child, expansionDepth}, pgsql.OperatorLessThanOrEqualTo, pgsql.CompoundIdentifier{aspI1Target, expansionDepth}), + )}}, + {Table: aspI1Table(aspI1DistanceBounded, prior), JoinOperator: pgsql.JoinOperator{JoinType: pgsql.JoinTypeInner, Constraint: pgsql.NewLiteral(true, pgsql.Boolean)}}, + {Table: expansionEdgeTableReference(s.traversalStep.Edge.Identifier), JoinOperator: pgsql.JoinOperator{JoinType: pgsql.JoinTypeInner, Constraint: predecessorEdgeConstraint}}, + }, + }}, + }}, + } + predecessorBounded := boundedTraversalStateProbe( + aspI1PredecessorBounded, + aspI1Predecessor, + []pgsql.Identifier{aspI1NodeID, expansionDepth, aspI1PredecessorID, aspI1EdgeID}, + expansionModel.ShortestPathPredecessorLimit, + ) + predecessorOverflow := boundedProbeOverflow(aspI1PredecessorBounded, expansionModel.ShortestPathPredecessorLimit) + + pathAnchor := pgsql.Select{ + Projection: pgsql.Projection{ + pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}, + pgsql.CompoundIdentifier{aspI1Target, expansionDepth}, + pgsql.ArrayLiteral{CastType: pgsql.Int8Array}, + }, + From: []pgsql.FromClause{ + tableFrom(aspI1Target), + tableFrom(validatedEndpoints), + }, + Where: pgd.Not(pgsql.NewParenthetical(aspI1OverflowAny(stateOverflow, predecessorOverflow))), + } + pathRecursive := pgsql.Select{ + Projection: pgsql.Projection{ + pgsql.CompoundIdentifier{aspI1PredecessorBounded, aspI1PredecessorID}, + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{aspI1Paths, expansionDepth}, pgsql.OperatorSubtract, pgsql.NewLiteral(int64(1), pgsql.Int8)), + pgsql.NewBinaryExpression( + pgsql.ArrayLiteral{Values: []pgsql.Expression{pgsql.CompoundIdentifier{aspI1PredecessorBounded, aspI1EdgeID}}, CastType: pgsql.Int8Array}, + pgsql.OperatorConcatenate, + pgsql.CompoundIdentifier{aspI1Paths, expansionPath}, + ), + }, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: aspI1Paths.AsCompoundIdentifier()}, + Joins: []pgsql.Join{{ + Table: pgsql.TableReference{Name: aspI1PredecessorBounded.AsCompoundIdentifier()}, + JoinOperator: pgsql.JoinOperator{JoinType: pgsql.JoinTypeInner, Constraint: pgsql.OptionalAnd( + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{aspI1PredecessorBounded, aspI1NodeID}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{aspI1Paths, aspI1NodeID}), + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{aspI1PredecessorBounded, expansionDepth}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{aspI1Paths, expansionDepth}), + )}, + }}, + }}, + } + paths := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: aspI1Paths, Shape: pgsql.NewRecordShape([]pgsql.Identifier{aspI1NodeID, expansionDepth, expansionPath})}, + Query: pgsql.Query{Body: pgsql.SetOperation{ + LOperand: pathAnchor, + ROperand: pathRecursive, + Operator: pgsql.OperatorUnion, + All: true, + }}, + } + pathsBounded := boundedTraversalStateProbe( + aspI1PathsBounded, + aspI1Paths, + []pgsql.Identifier{aspI1NodeID, expansionDepth, expansionPath}, + expansionModel.ShortestPathEnumerationLimit, + ) + enumerationOverflow := boundedProbeOverflow(aspI1PathsBounded, expansionModel.ShortestPathEnumerationLimit) + + shortest := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: aspI1Shortest, Shape: pgsql.NewRecordShape([]pgsql.Identifier{expansionDepth, expansionPath})}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{ + pgsql.CompoundIdentifier{aspI1Target, expansionDepth}, + pgsql.CompoundIdentifier{aspI1PathsBounded, expansionPath}, + }, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: aspI1PathsBounded.AsCompoundIdentifier()}, + Joins: []pgsql.Join{{Table: pgsql.TableReference{Name: aspI1Target.AsCompoundIdentifier()}, JoinOperator: pgsql.JoinOperator{JoinType: pgsql.JoinTypeInner, Constraint: pgsql.NewLiteral(true, pgsql.Boolean)}}}, + }}, + Where: pgsql.OptionalAnd( + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{aspI1PathsBounded, aspI1NodeID}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}), + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{aspI1PathsBounded, expansionDepth}, pgsql.OperatorEquals, pgsql.NewLiteral(int64(0), pgsql.Int8)), + ), + }}, + } + if mode.oneWitness { + shortestSelect := shortest.Query.Body.(pgsql.Select) + // ORDER BY at a UNION boundary may reference only the set output name, + // not a source relation that belongs to one operand. + shortest.Query.OrderBy = []*pgsql.OrderBy{{Expression: pgsql.CompoundIdentifier{expansionPath}, Ascending: true}} + shortest.Query.Limit = pgsql.NewLiteral(int64(1), pgsql.Int8) + shortest.Query.Body = shortestSelect + } + shortestSelect := shortest.Query.Body.(pgsql.Select) + shortestSelect.From = append(shortestSelect.From, tableFrom(validatedEndpoints)) + shortest.Query.Body = shortestSelect + shortest.Query.Body = pgsql.SetOperation{ + LOperand: pgsql.Select{ + Projection: pgsql.Projection{ + pgsql.CompoundIdentifier{aspI1PreflightBounded, expansionDepth}, + pgsql.CompoundIdentifier{aspI1PreflightBounded, expansionPath}, + }, + From: []pgsql.FromClause{tableFrom(aspI1PreflightBounded)}, + }, + ROperand: shortest.Query.Body.(pgsql.Select), Operator: pgsql.OperatorUnion, All: true, + } + + bytesOverflow := pgsql.NewBinaryExpression( + aspI1OutputBytes(aspI1Shortest), + pgsql.OperatorGreaterThan, + pgsql.NewLiteral(expansionModel.ShortestPathOutputBytesLimit, pgsql.Int8), + ) + overflow := aspI1OverflowAny(preflightOverflow, stateOverflow, predecessorOverflow, enumerationOverflow, bytesOverflow) + useCandidate := pgd.Not(pgsql.NewParenthetical(overflow)) + noPath := pgd.Not(pgsql.ExistsExpression{Subquery: pgsql.Subquery{Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.NewLiteral(int64(1), pgsql.Int8)}, + From: []pgsql.FromClause{tableFrom(aspI1Shortest)}, + }, + Limit: pgsql.NewLiteral(int64(1), pgsql.Int8), + }}}) + admission := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: aspI1Admission}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{Projection: pgsql.Projection{ + aspI1Aliased(overflow, aspI1Overflow), + aspI1Aliased(noPath, aspI1NoPath), + }}}, + } + admissionOverflow := pgsql.CompoundIdentifier{aspI1Admission, aspI1Overflow} + admissionNoPath := pgsql.CompoundIdentifier{aspI1Admission, aspI1NoPath} + useCandidate = pgd.Not(admissionOverflow) + candidateBranch := "inline_predecessor_dag" + noPathBranch := "inline_no_path" + fallbackBranch := "exact_a1_fallback" + if mode.oneWitness { + candidateBranch = "inline_canonical_witness" + noPathBranch = "inline_canonical_no_path" + fallbackBranch = "exact_s4_fallback" + } + branch := pgsql.Case{ + Conditions: []pgsql.Expression{admissionOverflow, admissionNoPath}, + Then: []pgsql.Expression{ + pgsql.NewLiteral(fallbackBranch, pgsql.Text), + pgsql.NewLiteral(noPathBranch, pgsql.Text), + }, + Else: pgsql.NewLiteral(candidateBranch, pgsql.Text), + } + runtimeExecutor := pgsql.Case{ + Conditions: []pgsql.Expression{admissionOverflow}, + Then: []pgsql.Expression{pgsql.NewLiteral(string(mode.fallback), pgsql.Text)}, + Else: pgsql.NewLiteral(string(mode.identity), pgsql.Text), + } + decision := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: aspI1Decision}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{Projection: pgsql.Projection{ + aspI1Aliased(useCandidate, aspI1UseCandidate), + aspI1Aliased(admissionOverflow, aspI1UseFallback), + aspI1Aliased(pgsql.FunctionCall{ + Function: aspI1RuntimeAttestationFn, + Parameters: []pgsql.Expression{ + branch, + admissionOverflow, + runtimeExecutor, + }, + }, aspI1RuntimeReceipt), + }, From: []pgsql.FromClause{tableFrom(aspI1Admission)}}}, + } + + candidateProjection := pgsql.Projection{ + aspI1Aliased(pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}, expansionRootID), + aspI1Aliased(pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}, expansionNextID), + aspI1Aliased(pgsql.CompoundIdentifier{aspI1Shortest, expansionDepth}, expansionDepth), + aspI1Aliased(pgsql.NewLiteral(true, pgsql.Boolean), expansionSatisfied), + aspI1Aliased(pgsql.NewLiteral(false, pgsql.Boolean), expansionIsCycle), + aspI1Aliased(pgsql.CompoundIdentifier{aspI1Shortest, expansionPath}, expansionPath), + } + candidateQuery := pgsql.Query{Body: pgsql.Select{ + Projection: candidateProjection, + From: []pgsql.FromClause{ + tableFrom(validatedEndpoints), + tableFrom(aspI1Shortest), + }, + }} + candidateBody, err := gateQueryBehindMarker(aspI1CandidateMarker, aspI1CandidateBody, candidateQuery, candidateProjection) + if err != nil { + return pgsql.Query{}, err + } + candidateRows := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: aspI1CandidateRows, Shape: expansionColumns()}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: candidateBody}, + } + + fallbackFunction := pgsql.FunctionAllShortestPathsDAG + if mode.oneWitness { + fallbackFunction = pgsql.FunctionShortestPathCompact + } + fallbackProjection := aspI1CanonicalProjection(fallbackFunction) + fallbackParameters := []pgsql.Expression{ + pgsql.NewLiteral(s.graphID, pgsql.Int4), + pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}, + pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}, + pgsql.NewLiteral(int64(1), pgsql.Int4), + pgsql.NewLiteral(expansionModel.Options.MaxDepth.Value, pgsql.Int4), + pgsql.NewLiteral(append([]int16(nil), expansionModel.RelationshipKindIDs...), pgsql.Int2Array), + pgsql.NewLiteral(s.traversalStep.Direction == graph.DirectionInbound, pgsql.Boolean), + } + if mode.oneWitness { + fallbackParameters = append(fallbackParameters, pgsql.NewLiteral(expansionModel.ShortestPathStateLimit, pgsql.Int8)) + } + fallbackQuery := pgsql.Query{Body: pgsql.Select{ + Projection: fallbackProjection, + From: []pgsql.FromClause{ + tableFrom(validatedEndpoints), + {Source: pgsql.FunctionCall{ + Function: fallbackFunction, + Parameters: fallbackParameters, + }}, + }, + }} + fallbackBody, err := gateQueryBehindMarker(aspI1FallbackMarker, aspI1FallbackBody, fallbackQuery, fallbackProjection) + if err != nil { + return pgsql.Query{}, err + } + fallbackRows := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: aspI1FallbackRows, Shape: expansionColumns()}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: fallbackBody}, + } + + stateID := expansionModel.Frame.Binding.Identifier + search := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: stateID, Shape: expansionColumns()}, + Query: pgsql.Query{Body: pgsql.SetOperation{ + LOperand: pgsql.Select{Projection: aspI1CanonicalProjection(aspI1CandidateRows), From: []pgsql.FromClause{tableFrom(aspI1CandidateRows)}}, + ROperand: pgsql.Select{Projection: aspI1CanonicalProjection(aspI1FallbackRows), From: []pgsql.FromClause{tableFrom(aspI1FallbackRows)}}, + Operator: pgsql.OperatorUnion, + All: true, + }}, + } + + projection := pgsql.Select{ + Projection: expansionModel.Projection, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: stateID.AsCompoundIdentifier()}, + Joins: []pgsql.Join{ + {Table: expansionNodeTableReference(s.traversalStep.LeftNode.Identifier), JoinOperator: pgsql.JoinOperator{JoinType: pgsql.JoinTypeInner, Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{s.traversalStep.LeftNode.Identifier, pgsql.ColumnID}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{stateID, expansionRootID}, + )}}, + {Table: expansionNodeTableReference(s.traversalStep.RightNode.Identifier), JoinOperator: pgsql.JoinOperator{JoinType: pgsql.JoinTypeInner, Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{s.traversalStep.RightNode.Identifier, pgsql.ColumnID}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{stateID, expansionNextID}, + )}}, + }, + }}, + } + if mode.oneWitness { + const ( + hydrated pgsql.Identifier = "m0_hydrated" + hydratedNodes pgsql.Identifier = "nodes" + hydratedEdges pgsql.Identifier = "edges" + hydratedCount pgsql.Identifier = "hydrated_count" + ) + pathIDs := pgsql.CompoundIdentifier{stateID, expansionPath} + hydration := shortestPathM0Hydration(stateID, s.traversalStep.Direction) + path := pgsql.CompositeValue{DataType: pgsql.PathComposite, Values: []pgsql.Expression{ + pgsql.NewBinaryExpression( + pgsql.ArrayLiteral{Values: []pgsql.Expression{shortestPathNodeComposite(s.traversalStep.LeftNode.Identifier)}, CastType: pgsql.NodeCompositeArray}, + pgsql.OperatorConcatenate, + pgsql.FunctionCall{Function: pgsql.FunctionCoalesce, Parameters: []pgsql.Expression{pgsql.CompoundIdentifier{hydrated, hydratedNodes}, pgsql.ArrayLiteral{CastType: pgsql.NodeCompositeArray}}}, + ), + pgsql.FunctionCall{Function: pgsql.FunctionCoalesce, Parameters: []pgsql.Expression{pgsql.CompoundIdentifier{hydrated, hydratedEdges}, pgsql.ArrayLiteral{CastType: pgsql.EdgeCompositeArray}}}, + }} + projection.Projection = shortestPathM0Projection(projection.Projection, stateID, path) + projection.From[0].Joins = append(projection.From[0].Joins, pgsql.Join{Table: hydration, JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, Constraint: pgsql.NewLiteral(true, pgsql.Boolean), + }}) + projection.Where = pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{hydrated, hydratedCount}, pgsql.OperatorEquals, + pgsql.FunctionCall{Function: pgsql.FunctionCardinality, Parameters: []pgsql.Expression{pathIDs}}, + ) + } + + query := pgsql.Query{CommonTableExpressions: &pgsql.With{Recursive: true}, Body: projection} + for _, cte := range []pgsql.CommonTableExpression{ + endpointCTE, + directCTE, + preflight, + preflightBounded, + distance, + distanceBounded, + target, + predecessor, + predecessorBounded, + paths, + pathsBounded, + shortest, + admission, + decision, + aspI1Marker(aspI1CandidateMarker, aspI1UseCandidate), + aspI1Marker(aspI1FallbackMarker, aspI1UseFallback), + candidateRows, + fallbackRows, + search, + } { + query.AddCTE(cte) + } + return query, nil +} diff --git a/cypher/models/pgsql/translate/expansion_endpoint_seeded.go b/cypher/models/pgsql/translate/expansion_endpoint_seeded.go new file mode 100644 index 00000000..1e0e37bd --- /dev/null +++ b/cypher/models/pgsql/translate/expansion_endpoint_seeded.go @@ -0,0 +1,290 @@ +package translate + +import ( + "fmt" + + "github.com/specterops/dawgs/cypher/models" + "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/pgd" +) + +// endpointSeededIdentifiers names the seed, reverse-state, admitted-state, and fallback CTEs for one rewrite. +type endpointSeededIdentifiers struct { + // endpoints names the materialized terminal-endpoint seed relation. + endpoints pgsql.Identifier + // reverse names the recursive reverse-search relation. + reverse pgsql.Identifier + // states names the deduplicated reverse states admitted for candidate matching. + states pgsql.Identifier + // incumbent names the original forward plan retained as an overflow fallback. + incumbent pgsql.Identifier +} + +// newEndpointSeededIdentifiers derives collision-resistant CTE names from the incumbent final frame. +func newEndpointSeededIdentifiers(finalFrame pgsql.Identifier) endpointSeededIdentifiers { + prefix := string(finalFrame) + "_endpoint_seeded_" + return endpointSeededIdentifiers{ + endpoints: pgsql.Identifier(prefix + "endpoints"), + reverse: pgsql.Identifier(prefix + "reverse"), + states: pgsql.Identifier(prefix + "states"), + incumbent: pgsql.Identifier(prefix + "incumbent"), + } +} + +// selectedEndpointSeededDecision returns the first traversal decision that selected endpoint-seeded reverse search. +func selectedEndpointSeededDecision(part *PatternPart, decisions map[optimize.TraversalStepTarget]optimize.ExpansionSearchStrategyDecision) (optimize.ExpansionSearchStrategyDecision, bool) { + for _, step := range part.TraversalSteps { + if step == nil || !step.HasSourceTarget { + continue + } + if decision, found := decisions[step.SourceTarget]; found && decision.SelectedStrategy == optimize.ExpansionSearchEndpointSeededReverse { + return decision, true + } + } + return optimize.ExpansionSearchStrategyDecision{}, false +} + +// rewriteTraversalPatternAsEndpointSeededReverse replaces a qualified two-step incumbent chain with guarded reverse search and fallback. +func (s *Translator) rewriteTraversalPatternAsEndpointSeededReverse(part *PatternPart, decision optimize.ExpansionSearchStrategyDecision, firstCTE int) error { + if decision.PrefixLength != 1 || decision.Target.StepIndex != 1 || len(part.TraversalSteps) != 2 { + return fmt.Errorf("endpoint-seeded reverse target requires exactly one fixed prefix step and one terminal expansion") + } + + prefixStep := part.TraversalSteps[0] + expansionStep := part.TraversalSteps[1] + if prefixStep == nil || prefixStep.Edge == nil || prefixStep.Frame == nil || expansionStep == nil || expansionStep.Expansion == nil || expansionStep.Frame == nil || expansionStep.RightNode == nil || expansionStep.LeftNode == nil || expansionStep.Edge == nil { + return fmt.Errorf("endpoint-seeded reverse target has an incomplete traversal step") + } + + ctes := s.query.CurrentPart().Model.CommonTableExpressions.Expressions + if firstCTE < 0 || firstCTE >= len(ctes) { + return fmt.Errorf("endpoint-seeded reverse target did not emit an incumbent frame chain") + } + + incumbentFinal := ctes[len(ctes)-1] + if incumbentFinal.Alias.Name != expansionStep.Frame.Binding.Identifier { + return fmt.Errorf("endpoint-seeded reverse final frame mismatch: expected %s but found %s", expansionStep.Frame.Binding.Identifier, incumbentFinal.Alias.Name) + } + incumbentSelect, ok := incumbentFinal.Query.Body.(pgsql.Select) + if !ok { + return fmt.Errorf("endpoint-seeded reverse final frame must be a select") + } + + prefixEdgeIDs := pgsql.ArrayLiteral{ + Values: []pgsql.Expression{pgsql.CompoundIdentifier{prefixStep.Frame.Binding.Identifier, prefixStep.Edge.Identifier}}, + CastType: pgsql.Int8Array, + } + incumbentSelect.Where = pgsql.OptionalAnd(incumbentSelect.Where, pgd.Not(pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{expansionStep.Expansion.Frame.Binding.Identifier, expansionPath}, + pgsql.OperatorArrayOverlap, + prefixEdgeIDs, + ))) + incumbentQuery := incumbentFinal.Query + incumbentQuery.Body = incumbentSelect + + ids := newEndpointSeededIdentifiers(incumbentFinal.Alias.Name) + query, err := s.buildGuardedEndpointSeededQuery(decision, prefixStep, expansionStep, ids, incumbentQuery, incumbentSelect.Projection) + if err != nil { + return err + } + + s.query.CurrentPart().Model.CommonTableExpressions.Expressions = append(ctes[:len(ctes)-1], pgsql.CommonTableExpression{ + Alias: incumbentFinal.Alias, + Query: query, + }) + s.recordExpansionSearchStrategy(decision.Target, optimize.ExpansionSearchEndpointSeededReverse) + return nil +} + +// buildGuardedEndpointSeededQuery unions bounded endpoint-seeded candidates with the incumbent overflow fallback. +func (s *Translator) buildGuardedEndpointSeededQuery( + decision optimize.ExpansionSearchStrategyDecision, + prefixStep *TraversalStep, + expansionStep *TraversalStep, + ids endpointSeededIdentifiers, + incumbent pgsql.Query, + incumbentProjection pgsql.Projection, +) (pgsql.Query, error) { + endpointCTE, err := buildEndpointSeedCTE(decision, expansionStep, ids) + if err != nil { + return pgsql.Query{}, err + } + + reverseCTE, err := buildEndpointReverseCTE(decision, expansionStep, ids) + if err != nil { + return pgsql.Query{}, err + } + + statesCTE := buildEndpointStateProbeCTE(decision, ids) + incumbentCTE := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.incumbent}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: incumbent, + } + + candidateProjection, fallbackProjection, err := endpointSeededProjections(prefixStep, expansionStep, ids, incumbentProjection) + if err != nil { + return pgsql.Query{}, err + } + + prefixFrame := prefixStep.Frame.Binding.Identifier + admitted, fallbackGate := boundedAdmissionGates( + boundedProbeLimit{source: ids.endpoints, limit: decision.EndpointLimit}, + boundedProbeLimit{source: ids.states, limit: decision.StateLimit}, + ) + + prefixEdgeIDs := pgsql.ArrayLiteral{ + Values: []pgsql.Expression{pgsql.CompoundIdentifier{prefixFrame, prefixStep.Edge.Identifier}}, + CastType: pgsql.Int8Array, + } + candidateWhere := pgsql.OptionalAnd( + admitted, + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{ids.states, expansionDepth}, pgsql.OperatorGreaterThanOrEqualTo, pgsql.NewLiteral(decision.MinimumDepth, pgsql.Int8)), + ) + candidateWhere = pgsql.OptionalAnd(candidateWhere, pgd.Not(pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{ids.states, expansionPath}, pgsql.OperatorArrayOverlap, prefixEdgeIDs, + ))) + + candidate := pgsql.Select{ + Projection: candidateProjection, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: prefixFrame.AsCompoundIdentifier()}, + Joins: []pgsql.Join{ + { + Table: pgsql.TableReference{Name: ids.states.AsCompoundIdentifier()}, + JoinOperator: pgsql.JoinOperator{JoinType: pgsql.JoinTypeInner, Constraint: pgsql.NewBinaryExpression( + projectedNodeIDReference(prefixFrame, expansionStep.LeftNode), pgsql.OperatorEquals, pgsql.CompoundIdentifier{ids.states, expansionNextID}, + )}, + }, + { + Table: pgsql.TableReference{Name: ids.endpoints.AsCompoundIdentifier()}, + JoinOperator: pgsql.JoinOperator{JoinType: pgsql.JoinTypeInner, Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{ids.endpoints, pgsql.ColumnID}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{ids.states, expansionRootID}, + )}, + }, + }, + }}, + Where: candidateWhere, + } + + fallback := pgsql.Select{ + Projection: fallbackProjection, + From: []pgsql.FromClause{tableFrom(ids.incumbent)}, + Where: fallbackGate, + } + + return pgsql.Query{ + CommonTableExpressions: &pgsql.With{ + Recursive: true, + Expressions: []pgsql.CommonTableExpression{endpointCTE, reverseCTE, statesCTE, incumbentCTE}, + }, + Body: pgsql.SetOperation{Operator: pgsql.OperatorUnion, All: true, LOperand: candidate, ROperand: fallback}, + }, nil +} + +// buildEndpointSeedCTE materializes locally constrained terminal IDs up to the endpoint guard limit. +func buildEndpointSeedCTE(decision optimize.ExpansionSearchStrategyDecision, expansionStep *TraversalStep, ids endpointSeededIdentifiers) (pgsql.CommonTableExpression, error) { + local, external := partitionConstraintByLocality(expansionStep.Expansion.TerminalNodeConstraints, pgsql.AsIdentifierSet(expansionStep.RightNode.Identifier)) + if external != nil { + return pgsql.CommonTableExpression{}, fmt.Errorf("endpoint-seeded reverse terminal predicate is not local") + } + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.endpoints}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: []pgsql.SelectItem{ + &pgsql.AliasedExpression{Expression: pgd.EntityID(expansionStep.RightNode.Identifier), Alias: models.OptionalValue(pgsql.ColumnID)}, + &pgsql.AliasedExpression{Expression: suffixSeededNodeValue(expansionStep.RightNode), Alias: models.OptionalValue(expansionStep.RightNode.Identifier)}, + }, + From: []pgsql.FromClause{{Source: expansionNodeTableReference(expansionStep.RightNode.Identifier)}}, + Where: local, + }, + Limit: pgsql.NewLiteral(decision.EndpointLimit+1, pgsql.Int8), + }, + }, nil +} + +// buildEndpointReverseCTE builds recursive reverse traversal from terminal seeds while preserving edge uniqueness. +func buildEndpointReverseCTE(decision optimize.ExpansionSearchStrategyDecision, expansionStep *TraversalStep, ids endpointSeededIdentifiers) (pgsql.CommonTableExpression, error) { + localEdgeConstraint, external := partitionConstraintByLocality(expansionStep.Expansion.EdgeConstraints, pgsql.AsIdentifierSet(expansionStep.Edge.Identifier)) + if external != nil { + return pgsql.CommonTableExpression{}, fmt.Errorf("endpoint-seeded reverse relationship predicate is not local") + } + emptyPath := pgsql.ArrayLiteral{CastType: pgsql.Int8Array} + seed := pgsql.Select{ + Projection: []pgsql.SelectItem{ + pgsql.CompoundIdentifier{ids.endpoints, pgsql.ColumnID}, + pgsql.CompoundIdentifier{ids.endpoints, pgsql.ColumnID}, + pgsql.NewLiteral(int64(0), pgsql.Int8), + emptyPath, + }, + From: []pgsql.FromClause{tableFrom(ids.endpoints)}, + } + path := pgsql.CompoundIdentifier{ids.reverse, expansionPath} + recursiveWhere := pgsql.OptionalAnd( + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{ids.reverse, expansionDepth}, pgsql.OperatorLessThan, pgsql.NewLiteral(decision.MaximumDepth, pgsql.Int8)), + pgsql.NewBinaryExpression(pgd.EntityID(expansionStep.Edge.Identifier), pgsql.OperatorNotEquals, pgsql.NewAllExpression(path)), + ) + recursiveWhere = pgsql.OptionalAnd(recursiveWhere, localEdgeConstraint) + recursive := pgsql.Select{ + Projection: []pgsql.SelectItem{ + pgsql.CompoundIdentifier{ids.reverse, expansionRootID}, + pgsql.CompoundIdentifier{expansionStep.Edge.Identifier, expansionStep.Expansion.EdgeStartIdentifier}, + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{ids.reverse, expansionDepth}, pgsql.OperatorAdd, pgsql.NewLiteral(int64(1), pgsql.Int8)), + pgsql.FunctionCall{Function: pgsql.Identifier("array_prepend"), Parameters: []pgsql.Expression{pgd.EntityID(expansionStep.Edge.Identifier), path}, CastType: pgsql.Int8Array}, + }, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: ids.reverse.AsCompoundIdentifier()}, + Joins: []pgsql.Join{{ + Table: expansionEdgeTableReference(expansionStep.Edge.Identifier), + JoinOperator: pgsql.JoinOperator{JoinType: pgsql.JoinTypeInner, Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{expansionStep.Edge.Identifier, expansionStep.Expansion.EdgeEndIdentifier}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{ids.reverse, expansionNextID}, + )}, + }}, + }}, + Where: recursiveWhere, + } + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.reverse, Shape: pgsql.NewRecordShape([]pgsql.Identifier{expansionRootID, expansionNextID, expansionDepth, expansionPath})}, + Query: pgsql.Query{Body: pgsql.SetOperation{Operator: pgsql.OperatorUnion, All: true, LOperand: seed, ROperand: recursive}}, + }, nil +} + +// buildEndpointStateProbeCTE materializes at most the guarded number of reverse states for candidate matching. +func buildEndpointStateProbeCTE(decision optimize.ExpansionSearchStrategyDecision, ids endpointSeededIdentifiers) pgsql.CommonTableExpression { + return boundedTraversalStateProbe(ids.states, ids.reverse, []pgsql.Identifier{ + expansionRootID, + expansionNextID, + expansionDepth, + expansionPath, + }, decision.StateLimit) +} + +// endpointSeededProjections aligns reverse-search results and incumbent rows to the original projection shape. +func endpointSeededProjections(prefixStep, expansionStep *TraversalStep, ids endpointSeededIdentifiers, incumbent pgsql.Projection) (pgsql.Projection, pgsql.Projection, error) { + prefixFrame := prefixStep.Frame.Binding.Identifier + candidate := make(pgsql.Projection, 0, len(incumbent)) + fallback := make(pgsql.Projection, 0, len(incumbent)) + for _, item := range incumbent { + alias, ok := selectItemAlias(item) + if !ok { + return nil, nil, fmt.Errorf("endpoint-seeded reverse final projection contains an unaliased item %T", item) + } + var expression pgsql.Expression + switch { + case expansionStep.Expansion.PathBinding != nil && alias == expansionStep.Expansion.PathBinding.Identifier: + expression = pgsql.CompoundIdentifier{ids.states, expansionPath} + case alias == expansionStep.LeftNode.Identifier: + expression = pgsql.CompoundIdentifier{prefixFrame, alias} + case alias == expansionStep.RightNode.Identifier: + expression = pgsql.CompoundIdentifier{ids.endpoints, alias} + default: + expression = pgsql.CompoundIdentifier{prefixFrame, alias} + } + candidate = append(candidate, &pgsql.AliasedExpression{Expression: expression, Alias: models.OptionalValue(alias)}) + fallback = append(fallback, &pgsql.AliasedExpression{Expression: pgsql.CompoundIdentifier{ids.incumbent, alias}, Alias: models.OptionalValue(alias)}) + } + return candidate, fallback, nil +} diff --git a/cypher/models/pgsql/translate/expansion_orientation.go b/cypher/models/pgsql/translate/expansion_orientation.go new file mode 100644 index 00000000..6ef0fcc4 --- /dev/null +++ b/cypher/models/pgsql/translate/expansion_orientation.go @@ -0,0 +1,626 @@ +package translate + +import ( + "fmt" + + "github.com/specterops/dawgs/cypher/models" + "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/pgd" +) + +const ( + orientationRootID pgsql.Identifier = "root_id" + orientationDegreeSample pgsql.Identifier = "sampled" + orientationRootRows pgsql.Identifier = "root_rows" + orientationSuffixRows pgsql.Identifier = "suffix_rows" + orientationBoundaryRows pgsql.Identifier = "boundary_rows" + orientationForwardDegreeRows pgsql.Identifier = "forward_degree_rows" + orientationReverseDegreeRows pgsql.Identifier = "reverse_degree_rows" + orientationProbesComplete pgsql.Identifier = "probes_complete" + orientationForwardScore pgsql.Identifier = "forward_score" + orientationReverseScore pgsql.Identifier = "reverse_score" + orientationUseReverse pgsql.Identifier = "use_reverse" + orientationWouldSelectReverse pgsql.Identifier = "would_select_reverse" + orientationShadowSelected pgsql.Identifier = "selected" + orientationArmExecuted pgsql.Identifier = "executed" +) + +// expansionOrientationIdentifiers gives every probe, decision, candidate, +// and fallback relation a stable suffix suitable for plan and telemetry +// attribution. +type expansionOrientationIdentifiers struct { + rootProbe pgsql.Identifier + rootPresence pgsql.Identifier + suffixProbe pgsql.Identifier + boundaries pgsql.Identifier + forwardDegreeProbe pgsql.Identifier + reverseDegreeProbe pgsql.Identifier + metrics pgsql.Identifier + decision pgsql.Identifier + admission pgsql.Identifier + shadowForward pgsql.Identifier + shadowReverse pgsql.Identifier + shadowSelection pgsql.Identifier + reverseGate pgsql.Identifier + reverseSeed pgsql.Identifier + reverseSeedRows pgsql.Identifier + executedCandidate pgsql.Identifier + executedIncumbent pgsql.Identifier + candidateBody pgsql.Identifier + incumbentBody pgsql.Identifier + reverse pgsql.Identifier + states pgsql.Identifier + incumbent pgsql.Identifier +} + +func newExpansionOrientationIdentifiers(finalFrame pgsql.Identifier) expansionOrientationIdentifiers { + prefix := string(finalFrame) + "_orientation_" + return expansionOrientationIdentifiers{ + rootProbe: pgsql.Identifier(prefix + "root_probe"), + rootPresence: pgsql.Identifier(prefix + "root_presence"), + suffixProbe: pgsql.Identifier(prefix + "suffix_probe"), + boundaries: pgsql.Identifier(prefix + "boundaries"), + forwardDegreeProbe: pgsql.Identifier(prefix + "forward_degree_probe"), + reverseDegreeProbe: pgsql.Identifier(prefix + "reverse_degree_probe"), + metrics: pgsql.Identifier(prefix + "metrics"), + decision: pgsql.Identifier(prefix + "decision"), + admission: pgsql.Identifier(prefix + "admission"), + shadowForward: pgsql.Identifier(prefix + "shadow_forward"), + shadowReverse: pgsql.Identifier(prefix + "shadow_reverse"), + shadowSelection: pgsql.Identifier(prefix + "shadow_selection"), + reverseGate: pgsql.Identifier(prefix + "reverse_gate"), + reverseSeed: pgsql.Identifier(prefix + "reverse_seed"), + reverseSeedRows: pgsql.Identifier(prefix + "reverse_seed_rows"), + executedCandidate: pgsql.Identifier(prefix + "executed_candidate"), + executedIncumbent: pgsql.Identifier(prefix + "executed_incumbent"), + candidateBody: pgsql.Identifier(prefix + "candidate_body"), + incumbentBody: pgsql.Identifier(prefix + "incumbent_body"), + reverse: pgsql.Identifier(prefix + "reverse"), + states: pgsql.Identifier(prefix + "states"), + incumbent: pgsql.Identifier(prefix + "incumbent"), + } +} + +// pairwiseRelationshipIDUniqueness excludes every repeated relationship in a +// fixed orientation region. This is intentionally explicit: constraints +// attached while translating the incumbent traversal may be partitioned away +// when the region is rebuilt as an independent seed relation. +func pairwiseRelationshipIDUniqueness(relationships []pgsql.Identifier) pgsql.Expression { + var constraint pgsql.Expression + for right := 1; right < len(relationships); right++ { + for left := 0; left < right; left++ { + constraint = pgsql.OptionalAnd( + constraint, + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{relationships[right], pgsql.ColumnID}, + pgsql.OperatorNotEquals, + pgsql.CompoundIdentifier{relationships[left], pgsql.ColumnID}, + ), + ) + } + } + return constraint +} + +// expansionOrientationReverseDominates mirrors orientation-probe-v1's SQL +// hysteresis rule: reverse evidence must be strictly below 75 percent of +// forward evidence, so equality and ties keep the incumbent. +func expansionOrientationReverseDominates(forwardScore, reverseScore int64) bool { + return reverseScore*optimize.ExpansionSearchOrientationReverseScoreMultiplier < forwardScore*optimize.ExpansionSearchOrientationForwardScoreMultiplier +} + +// boundedProbeOverflow detects the cap+1 sentinel row of a bounded relation. +func boundedProbeOverflow(source pgsql.Identifier, limit int64) pgsql.ExistsExpression { + return pgsql.ExistsExpression{Subquery: pgsql.Subquery{Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: []pgsql.SelectItem{pgsql.NewLiteral(int64(1), pgsql.Int8)}, + From: []pgsql.FromClause{tableFrom(source)}, + }, + Offset: pgsql.NewLiteral(limit, pgsql.Int8), + Limit: pgsql.NewLiteral(int64(1), pgsql.Int8), + }}} +} + +// boundedTraversalStateProbe materializes a cap+1 view over recursive state. +// Orientation families provide their own state columns and retain their +// existing candidate/fallback semantics around this common admission boundary. +func boundedTraversalStateProbe( + alias, source pgsql.Identifier, + columns []pgsql.Identifier, + limit int64, + executionMarker ...pgsql.Identifier, +) pgsql.CommonTableExpression { + projection := make(pgsql.Projection, 0, len(columns)) + for _, column := range columns { + projection = append(projection, pgsql.CompoundIdentifier{source, column}) + } + query := pgsql.Query{ + Body: pgsql.Select{ + Projection: projection, + From: []pgsql.FromClause{tableFrom(source)}, + }, + Limit: pgsql.NewLiteral(limit+1, pgsql.Int8), + } + if len(executionMarker) > 0 && executionMarker[0] != "" { + marker := executionMarker[0] + bodyAlias := pgsql.Identifier(string(alias) + "_body") + body := query.Body.(pgsql.Select) + body.Where = pgsql.CompoundIdentifier{marker, orientationArmExecuted} + query.Body = body + query.Offset = pgsql.NewLiteral(int64(0), pgsql.Int8) + + outerProjection := make(pgsql.Projection, 0, len(columns)) + for _, column := range columns { + outerProjection = append(outerProjection, &pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{bodyAlias, column}, + Alias: models.OptionalValue(column), + }) + } + query = pgsql.Query{Body: pgsql.Select{ + Projection: outerProjection, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: marker.AsCompoundIdentifier()}, + Joins: []pgsql.Join{{ + Table: pgsql.LateralSubquery{Query: query, Binding: models.OptionalValue(bodyAlias)}, + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewLiteral(true, pgsql.Boolean), + }, + }}, + }}, + }} + } + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: alias}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: query, + } +} + +// boundedAdmissionGates returns exact complementary candidate and incumbent +// gates for independent bounded probes. Empty input admits the candidate and +// suppresses fallback; every ordinary orientation supplies at least one gate. +type boundedProbeLimit struct { + source pgsql.Identifier + limit int64 +} + +func boundedAdmissionGates(probes ...boundedProbeLimit) (candidate, fallback pgsql.Expression) { + for _, probe := range probes { + overflow := boundedProbeOverflow(probe.source, probe.limit) + candidate = pgsql.OptionalAnd(candidate, pgd.Not(overflow)) + if fallback == nil { + fallback = overflow + } else { + fallback = pgsql.NewBinaryExpression(fallback, pgsql.OperatorOr, overflow) + } + } + if candidate == nil { + candidate = pgsql.NewLiteral(true, pgsql.Boolean) + } + if fallback == nil { + fallback = pgsql.NewLiteral(false, pgsql.Boolean) + } + return candidate, fallback +} + +func orientationCount(source pgsql.Identifier) pgsql.Subquery { + return pgsql.Subquery{Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.FunctionCall{ + Function: pgsql.FunctionCount, + Parameters: []pgsql.Expression{pgsql.Wildcard{}}, + CastType: pgsql.Int8, + }}, + From: []pgsql.FromClause{tableFrom(source)}, + }}} +} + +// buildExpansionOrientationRootProbe materializes duplicate-preserving root +// evidence. It is evidence only; candidate and fallback continue to read the +// exact root relation. +func buildExpansionOrientationRootProbe(rootFrame pgsql.Identifier, root *BoundIdentifier, ids expansionOrientationIdentifiers, cap int64) pgsql.CommonTableExpression { + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.rootProbe}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{&pgsql.AliasedExpression{ + Expression: projectedNodeIDReference(rootFrame, root), + Alias: models.OptionalValue(orientationRootID), + }}, + From: []pgsql.FromClause{tableFrom(rootFrame)}, + }, + Limit: pgsql.NewLiteral(cap+1, pgsql.Int8), + }, + } +} + +func buildExpansionOrientationRootPresence(ids expansionOrientationIdentifiers) pgsql.CommonTableExpression { + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.rootPresence}, + Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.NewLiteral(int64(1), pgsql.Int8)}, + From: []pgsql.FromClause{tableFrom(ids.rootProbe)}, + }, + Limit: pgsql.NewLiteral(int64(1), pgsql.Int8), + }, + } +} + +// buildExpansionOrientationDegreeProbe materializes one evidence row per typed +// adjacency. Each seed row is retained, so duplicate forward roots contribute +// their real work multiplier while reverse boundaries remain distinct. +func buildExpansionOrientationDegreeProbe( + alias, seedSource, seedColumn pgsql.Identifier, + edgeAlias, edgeSeedColumn pgsql.Identifier, + edgeConstraint pgsql.Expression, + cap int64, +) pgsql.CommonTableExpression { + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: alias}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{&pgsql.AliasedExpression{ + Expression: pgsql.NewLiteral(true, pgsql.Boolean), + Alias: models.OptionalValue(orientationDegreeSample), + }}, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: seedSource.AsCompoundIdentifier()}, + Joins: []pgsql.Join{{ + Table: expansionEdgeTableReference(edgeAlias), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{edgeAlias, edgeSeedColumn}, + pgsql.OperatorEquals, + pgsql.CompoundIdentifier{seedSource, seedColumn}, + ), + }, + }}, + }}, + Where: edgeConstraint, + }, + Limit: pgsql.NewLiteral(cap+1, pgsql.Int8), + }, + } +} + +func buildExpansionOrientationMetrics(ids expansionOrientationIdentifiers, caps optimize.ExpansionSearchProbeCaps) pgsql.CommonTableExpression { + complete := pgsql.OptionalAnd( + pgd.Not(boundedProbeOverflow(ids.rootProbe, caps.RootRowLimit)), + pgd.Not(boundedProbeOverflow(ids.suffixProbe, caps.ReverseSeedRowLimit)), + ) + complete = pgsql.OptionalAnd(complete, pgd.Not(boundedProbeOverflow(ids.forwardDegreeProbe, caps.DirectionalDegreeRowLimit))) + complete = pgsql.OptionalAnd(complete, pgd.Not(boundedProbeOverflow(ids.reverseDegreeProbe, caps.DirectionalDegreeRowLimit))) + + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.metrics}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{Projection: pgsql.Projection{ + &pgsql.AliasedExpression{Expression: orientationCount(ids.rootProbe), Alias: models.OptionalValue(orientationRootRows)}, + &pgsql.AliasedExpression{Expression: orientationCount(ids.suffixProbe), Alias: models.OptionalValue(orientationSuffixRows)}, + &pgsql.AliasedExpression{Expression: orientationCount(ids.boundaries), Alias: models.OptionalValue(orientationBoundaryRows)}, + &pgsql.AliasedExpression{Expression: orientationCount(ids.forwardDegreeProbe), Alias: models.OptionalValue(orientationForwardDegreeRows)}, + &pgsql.AliasedExpression{Expression: orientationCount(ids.reverseDegreeProbe), Alias: models.OptionalValue(orientationReverseDegreeRows)}, + &pgsql.AliasedExpression{Expression: complete, Alias: models.OptionalValue(orientationProbesComplete)}, + }}}, + } +} + +// buildExpansionOrientationDecision renders the immutable score formula for +// the requested policy identity. V1 counts one forward-degree sample per root; +// v2 weights those samples by the traversal's inclusive maximum depth. +func buildExpansionOrientationDecision(ids expansionOrientationIdentifiers, policy optimize.ExpansionSearchPolicy, maximumDepth int64) (pgsql.CommonTableExpression, error) { + var ( + forwardWork pgsql.Expression = pgsql.CompoundIdentifier{ids.metrics, orientationForwardDegreeRows} + reverseMultiplier = optimize.ExpansionSearchOrientationReverseScoreMultiplier + forwardMultiplier = optimize.ExpansionSearchOrientationForwardScoreMultiplier + ) + switch policy { + case optimize.ExpansionSearchPolicyOrientationProbeV1: + case optimize.ExpansionSearchPolicyOrientationProbeV2: + if maximumDepth <= 0 { + return pgsql.CommonTableExpression{}, fmt.Errorf("%s requires a positive maximum depth", policy) + } + forwardWork = pgsql.NewBinaryExpression( + pgsql.NewLiteral(maximumDepth, pgsql.Int8), + pgsql.OperatorMultiply, + forwardWork, + ) + reverseMultiplier = optimize.ExpansionSearchOrientationV2ReverseScoreMultiplier + forwardMultiplier = optimize.ExpansionSearchOrientationV2ForwardScoreMultiplier + default: + return pgsql.CommonTableExpression{}, fmt.Errorf("unsupported expansion orientation policy %q", policy) + } + forwardScore := pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{ids.metrics, orientationRootRows}, + pgsql.OperatorAdd, + forwardWork, + ) + reverseScore := pgsql.NewBinaryExpression( + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{ids.metrics, orientationSuffixRows}, + pgsql.OperatorAdd, + pgsql.CompoundIdentifier{ids.metrics, orientationBoundaryRows}, + ), + pgsql.OperatorAdd, + pgsql.CompoundIdentifier{ids.metrics, orientationReverseDegreeRows}, + ) + dominates := pgsql.NewBinaryExpression( + pgsql.NewBinaryExpression(pgsql.NewParenthetical(reverseScore), pgsql.OperatorMultiply, pgsql.NewLiteral(reverseMultiplier, pgsql.Int8)), + pgsql.OperatorLessThan, + pgsql.NewBinaryExpression(pgsql.NewParenthetical(forwardScore), pgsql.OperatorMultiply, pgsql.NewLiteral(forwardMultiplier, pgsql.Int8)), + ) + useReverse := pgsql.OptionalAnd(pgsql.CompoundIdentifier{ids.metrics, orientationProbesComplete}, dominates) + + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.decision}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{ + &pgsql.AliasedExpression{Expression: forwardScore, Alias: models.OptionalValue(orientationForwardScore)}, + &pgsql.AliasedExpression{Expression: reverseScore, Alias: models.OptionalValue(orientationReverseScore)}, + &pgsql.AliasedExpression{Expression: pgsql.CompoundIdentifier{ids.metrics, orientationProbesComplete}, Alias: models.OptionalValue(orientationProbesComplete)}, + &pgsql.AliasedExpression{Expression: useReverse, Alias: models.OptionalValue(orientationUseReverse)}, + &pgsql.AliasedExpression{Expression: useReverse, Alias: models.OptionalValue(orientationWouldSelectReverse)}, + }, + From: []pgsql.FromClause{tableFrom(ids.metrics)}, + }}, + }, nil +} + +// buildExpansionOrientationShadowMarkers turns the SQL-visible policy result +// into two mutually exclusive, named plan branches. The final one-row relation +// preserves would_select_reverse without adding a column to the public query +// result. JSON EXPLAIN can therefore attribute the shadow choice while the +// incumbent remains the only executable traversal arm. +func buildExpansionOrientationShadowMarkers(ids expansionOrientationIdentifiers) []pgsql.CommonTableExpression { + shadowMarker := func(alias pgsql.Identifier, selected bool, predicate pgsql.Expression) pgsql.CommonTableExpression { + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: alias}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{&pgsql.AliasedExpression{ + Expression: pgsql.NewLiteral(selected, pgsql.Boolean), + Alias: models.OptionalValue(orientationShadowSelected), + }}, + From: []pgsql.FromClause{tableFrom(ids.decision)}, + Where: predicate, + }}, + } + } + + forward := shadowMarker( + ids.shadowForward, + false, + pgd.Not(pgsql.CompoundIdentifier{ids.decision, orientationWouldSelectReverse}), + ) + reverse := shadowMarker( + ids.shadowReverse, + true, + pgsql.CompoundIdentifier{ids.decision, orientationWouldSelectReverse}, + ) + selection := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.shadowSelection}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.SetOperation{ + Operator: pgsql.OperatorUnion, + All: true, + LOperand: pgsql.Select{ + Projection: pgsql.Projection{&pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{ids.shadowForward, orientationShadowSelected}, + Alias: models.OptionalValue(orientationWouldSelectReverse), + }}, + From: []pgsql.FromClause{tableFrom(ids.shadowForward)}, + }, + ROperand: pgsql.Select{ + Projection: pgsql.Projection{&pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{ids.shadowReverse, orientationShadowSelected}, + Alias: models.OptionalValue(orientationWouldSelectReverse), + }}, + From: []pgsql.FromClause{tableFrom(ids.shadowReverse)}, + }, + }}, + } + incumbent := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.executedIncumbent}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{&pgsql.AliasedExpression{ + Expression: pgsql.FunctionCall{ + Function: pgsql.Identifier("record_traversal_runtime_attestation_v1"), + Parameters: []pgsql.Expression{ + pgsql.NewLiteral(string(optimize.ExpansionSearchStepwiseForward), pgsql.Text), + pgsql.NewLiteral("shadow_incumbent", pgsql.Text), + pgsql.NewLiteral(false, pgsql.Boolean), + }, + CastType: pgsql.Boolean, + }, + Alias: models.OptionalValue(orientationArmExecuted), + }}, + From: []pgsql.FromClause{tableFrom(ids.shadowSelection)}, + }}, + } + + return []pgsql.CommonTableExpression{forward, reverse, selection, incumbent} +} + +// buildExpansionOrientationAdmission materializes the recursive-state +// sentinel once. Both execution markers consume this one decision row so the +// cap+1 state relation is not rescanned independently by each gate and receipt. +func buildExpansionOrientationAdmission(ids expansionOrientationIdentifiers, stateLimit int64) pgsql.CommonTableExpression { + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.admission}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{ + &pgsql.AliasedExpression{Expression: pgsql.CompoundIdentifier{ids.decision, orientationUseReverse}, Alias: models.OptionalValue(orientationUseReverse)}, + &pgsql.AliasedExpression{Expression: pgsql.CompoundIdentifier{ids.decision, orientationProbesComplete}, Alias: models.OptionalValue(orientationProbesComplete)}, + &pgsql.AliasedExpression{Expression: boundedProbeOverflow(ids.states, stateLimit), Alias: models.OptionalValue[pgsql.Identifier]("state_overflow")}, + }, + From: []pgsql.FromClause{tableFrom(ids.decision)}, + }}, + } +} + +// buildExpansionOrientationExecutionMarkers materializes exactly one named +// marker for the arm admitted by the tournament. Unlike recursive-loop row +// counts, these relations remain unambiguous when a selected arm legitimately +// produces no traversal rows. Candidate admission requires both the policy +// choice and a complete state probe; state overflow selects the incumbent. +func buildExpansionOrientationExecutionMarkers(ids expansionOrientationIdentifiers) []pgsql.CommonTableExpression { + stateOverflow := pgsql.CompoundIdentifier{ids.admission, pgsql.Identifier("state_overflow")} + stateAdmitted := pgd.Not(stateOverflow) + useReverse := pgsql.CompoundIdentifier{ids.admission, orientationUseReverse} + probeOverflow := pgd.Not(pgsql.CompoundIdentifier{ids.admission, orientationProbesComplete}) + candidateGate := pgsql.OptionalAnd(useReverse, stateAdmitted) + incumbentGate := pgsql.NewBinaryExpression(pgd.Not(useReverse), pgsql.OperatorOr, stateOverflow) + fallbackExecuted := pgsql.NewBinaryExpression(probeOverflow, pgsql.OperatorOr, stateOverflow) + + marker := func(alias pgsql.Identifier, gate pgsql.Expression, runtimeIdentity, runtimeBranch string, fallback pgsql.Expression) pgsql.CommonTableExpression { + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: alias}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{&pgsql.AliasedExpression{ + Expression: pgsql.FunctionCall{ + Function: pgsql.Identifier("record_traversal_runtime_attestation_v1"), + Parameters: []pgsql.Expression{ + pgsql.NewLiteral(runtimeIdentity, pgsql.Text), + pgsql.NewLiteral(runtimeBranch, pgsql.Text), + fallback, + }, + CastType: pgsql.Boolean, + }, + Alias: models.OptionalValue(orientationArmExecuted), + }}, + From: []pgsql.FromClause{tableFrom(ids.admission)}, + Where: gate, + }}, + } + } + + return []pgsql.CommonTableExpression{ + marker(ids.executedCandidate, candidateGate, string(optimize.ExpansionSearchSuffixSeededReverse), "suffix_seeded_reverse", pgsql.NewLiteral(false, pgsql.Boolean)), + marker(ids.executedIncumbent, incumbentGate, string(optimize.ExpansionSearchStepwiseForward), "exact_forward_incumbent", fallbackExecuted), + } +} + +// buildExpansionOrientationReverseSeed puts the policy marker on the outer +// side of a correlated LATERAL boundary scan. PostgreSQL therefore cannot +// initialize the reverse recursion's seed scan when the policy keeps the +// incumbent; the lateral subquery has no invocation row in that case. +func buildExpansionOrientationReverseSeed(ids expansionOrientationIdentifiers) []pgsql.CommonTableExpression { + gate := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.reverseGate}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{&pgsql.AliasedExpression{ + Expression: pgsql.NewLiteral(true, pgsql.Boolean), + Alias: models.OptionalValue(orientationArmExecuted), + }}, + From: []pgsql.FromClause{tableFrom(ids.decision)}, + Where: pgsql.CompoundIdentifier{ids.decision, orientationUseReverse}, + }}, + } + seedRows := pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{&pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{ids.boundaries, fixedSuffixBoundaryID}, + Alias: models.OptionalValue(fixedSuffixBoundaryID), + }}, + From: []pgsql.FromClause{tableFrom(ids.boundaries)}, + Where: pgsql.CompoundIdentifier{ids.reverseGate, orientationArmExecuted}, + }, + // OFFSET 0 is a deliberate planner boundary for this correlated gate. + Offset: pgsql.NewLiteral(int64(0), pgsql.Int8), + } + + seed := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.reverseSeed}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{&pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{ids.reverseSeedRows, fixedSuffixBoundaryID}, + Alias: models.OptionalValue(fixedSuffixBoundaryID), + }}, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: ids.reverseGate.AsCompoundIdentifier()}, + Joins: []pgsql.Join{{ + Table: pgsql.LateralSubquery{ + Query: seedRows, + Binding: models.OptionalValue(ids.reverseSeedRows), + }, + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewLiteral(true, pgsql.Boolean), + }, + }}, + }}, + }}, + } + return []pgsql.CommonTableExpression{gate, seed} +} + +// gateQueryBehindMarker makes an execution marker the outer relation of a +// correlated LATERAL query. Merely listing a marker after a materialized CTE +// does not prove PostgreSQL avoids initializing that CTE; this dependency does. +func gateQueryBehindMarker( + marker, bodyAlias pgsql.Identifier, + query pgsql.Query, + exposedProjection pgsql.Projection, +) (pgsql.Select, error) { + body, ok := query.Body.(pgsql.Select) + if !ok { + return pgsql.Select{}, fmt.Errorf("gated orientation body must be a select, found %T", query.Body) + } + body.Where = pgsql.OptionalAnd( + body.Where, + pgsql.CompoundIdentifier{marker, orientationArmExecuted}, + ) + query.Body = body + // The correlated reference and OFFSET 0 keep the expensive inner query + // below the marker-driven LATERAL invocation boundary. + query.Offset = pgsql.NewLiteral(int64(0), pgsql.Int8) + + projection := make(pgsql.Projection, 0, len(exposedProjection)) + for _, item := range exposedProjection { + alias, ok := selectItemAlias(item) + if !ok { + return pgsql.Select{}, fmt.Errorf("gated orientation projection contains an unaliased item %T", item) + } + projection = append(projection, &pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{bodyAlias, alias}, + Alias: models.OptionalValue(alias), + }) + } + + return pgsql.Select{ + Projection: projection, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: marker.AsCompoundIdentifier()}, + Joins: []pgsql.Join{{ + Table: pgsql.LateralSubquery{ + Query: query, + Binding: models.OptionalValue(bodyAlias), + }, + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewLiteral(true, pgsql.Boolean), + }, + }}, + }}, + }, nil +} + +func expansionOrientationStateProbe(decision optimize.ExpansionSearchStrategyDecision, ids expansionOrientationIdentifiers) pgsql.CommonTableExpression { + return boundedTraversalStateProbe(ids.states, ids.reverse, []pgsql.Identifier{ + fixedSuffixBoundaryID, + expansionNextID, + expansionDepth, + expansionPath, + }, decision.Admission.StateLimit, ids.reverseGate) +} diff --git a/cypher/models/pgsql/translate/expansion_orientation_test.go b/cypher/models/pgsql/translate/expansion_orientation_test.go new file mode 100644 index 00000000..08d812ca --- /dev/null +++ b/cypher/models/pgsql/translate/expansion_orientation_test.go @@ -0,0 +1,563 @@ +package translate + +import ( + "context" + "regexp" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/cypher/models" + "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/specterops/dawgs/cypher/models/pgsql/format" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" +) + +const guardedSuffixOrientationQuery = ` + MATCH (root:ExpansionRoot) + WHERE root.root_key = $root_key + MATCH path = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) + RETURN path +` + +func TestExpansionOrientationReverseDominanceHasStrictHysteresis(t *testing.T) { + require.False(t, expansionOrientationReverseDominates(0, 0)) + require.False(t, expansionOrientationReverseDominates(100, 75)) + require.False(t, expansionOrientationReverseDominates(4, 3)) + require.True(t, expansionOrientationReverseDominates(100, 74)) + require.True(t, expansionOrientationReverseDominates(4, 2)) +} + +func TestExpansionOrientationBooleanModesRemainV1ByDefault(t *testing.T) { + for _, testCase := range []struct { + name string + options ToolOptions + }{ + {name: "guarded", options: ToolOptions{EnableExpansionOrientationTournament: true}}, + {name: "shadow", options: ToolOptions{EnableExpansionOrientationShadow: true}}, + } { + t.Run(testCase.name, func(t *testing.T) { + translate := func(options ToolOptions) (Result, string) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": "v1-default-root", + }, DefaultGraphID, options) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + return translation, formatted + } + + implicit, implicitSQL := translate(testCase.options) + explicitOptions := testCase.options + explicitOptions.ExpansionOrientationPolicy = optimize.ExpansionSearchPolicyOrientationProbeV1 + explicit, explicitSQL := translate(explicitOptions) + + require.Equal(t, implicitSQL, explicitSQL) + require.Contains(t, implicitSQL, "(s5_orientation_metrics.suffix_rows + s5_orientation_metrics.boundary_rows + s5_orientation_metrics.reverse_degree_rows) * 4 < (s5_orientation_metrics.root_rows + s5_orientation_metrics.forward_degree_rows) * 3") + require.NotContains(t, implicitSQL, "16 * s5_orientation_metrics.forward_degree_rows") + require.Equal(t, implicit.Optimization.LoweringPlan.ExpansionSearchStrategy, explicit.Optimization.LoweringPlan.ExpansionSearchStrategy) + decision := implicit.Optimization.LoweringPlan.ExpansionSearchStrategy[0] + require.Equal(t, optimize.ExpansionSearchPolicyOrientationProbeV1, decision.PlannedPolicy) + require.Equal(t, optimize.ExpansionSearchPolicyOrientationProbeV1, decision.EmittedPolicy) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV1), decision.SelectorVersion) + outcome := requireTraversalTargetOutcome(t, implicit.Optimization, optimize.LoweringExpansionSearchStrategy, decision.Target) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV1), outcome.PlannedPolicy) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV1), outcome.EmittedPolicy) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV1), outcome.SelectorVersion) + }) + } +} + +func TestExpansionOrientationProbeV2IsExplicitAndDepthWeighted(t *testing.T) { + for _, testCase := range []struct { + name string + options ToolOptions + expectedMode string + expectedBoundary string + expectedCandidates []optimize.ExpansionSearchStrategy + }{ + { + name: "guarded", + options: ToolOptions{EnableExpansionOrientationTournament: true}, + expectedMode: "guarded_tool", + expectedBoundary: optimize.ExpansionSearchExecutionBoundaryGuardedDualArm, + expectedCandidates: []optimize.ExpansionSearchStrategy{optimize.ExpansionSearchStepwiseForward, optimize.ExpansionSearchSuffixSeededReverse}, + }, + { + name: "shadow", + options: ToolOptions{EnableExpansionOrientationShadow: true}, + expectedMode: "shadow_tool", + expectedBoundary: optimize.ExpansionSearchExecutionBoundaryInlineStatement, + expectedCandidates: []optimize.ExpansionSearchStrategy{optimize.ExpansionSearchStepwiseForward}, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + options := testCase.options + options.ExpansionOrientationPolicy = optimize.ExpansionSearchPolicyOrientationProbeV2 + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": "v2-depth-root", + }, DefaultGraphID, options) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "(s5_orientation_metrics.suffix_rows + s5_orientation_metrics.boundary_rows + s5_orientation_metrics.reverse_degree_rows) * 4 < (s5_orientation_metrics.root_rows + 16 * s5_orientation_metrics.forward_degree_rows) * 3") + require.Contains(t, formatted, "s5_orientation_metrics.probes_complete and (s5_orientation_metrics.suffix_rows + s5_orientation_metrics.boundary_rows + s5_orientation_metrics.reverse_degree_rows) * 4 <") + require.NotContains(t, formatted, "(s5_orientation_metrics.root_rows + s5_orientation_metrics.forward_degree_rows) * 3") + + decision := translation.Optimization.LoweringPlan.ExpansionSearchStrategy[0] + require.Equal(t, int64(16), decision.MaximumDepth) + require.Equal(t, optimize.ExpansionSearchPolicyOrientationProbeV2, decision.PlannedPolicy) + require.Equal(t, optimize.ExpansionSearchPolicyOrientationProbeV2, decision.EmittedPolicy) + require.Equal(t, testCase.expectedCandidates, decision.EmittedCandidates) + require.Equal(t, testCase.expectedMode, decision.SelectionMode) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV2), decision.SelectorVersion) + require.Equal(t, testCase.expectedBoundary, decision.ExecutionBoundary) + + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy, decision.Target) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV2), outcome.PlannedPolicy) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV2), outcome.EmittedPolicy) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV2), outcome.SelectorVersion) + require.Equal(t, testCase.expectedBoundary, outcome.ExecutionBoundary) + }) + } +} + +func TestBoundedAdmissionGatesAreStrictComplements(t *testing.T) { + admitted, fallback := boundedAdmissionGates( + boundedProbeLimit{source: "endpoint_probe", limit: 32}, + boundedProbeLimit{source: "state_probe", limit: 4096}, + ) + require.NotNil(t, admitted) + require.NotNil(t, fallback) + + query := pgsql.Query{Body: pgsql.Select{Projection: pgsql.Projection{ + &pgsql.AliasedExpression{Expression: admitted, Alias: models.OptionalValue[pgsql.Identifier]("admitted")}, + &pgsql.AliasedExpression{Expression: fallback, Alias: models.OptionalValue[pgsql.Identifier]("fallback")}, + }}} + rendered, err := format.Statement(query, format.NewOutputBuilder()) + require.NoError(t, err) + require.Contains(t, rendered, "not exists") + require.Contains(t, rendered, "offset 32 limit 1") + require.Contains(t, rendered, "offset 4096 limit 1") + require.Contains(t, rendered, "or exists") +} + +func TestGuardedSuffixOrientationTournamentEmitsBoundedDisjointBranches(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": "guarded-fixed-suffix-root", + }, DefaultGraphID, ToolOptions{EnableExpansionOrientationTournament: true}) + require.NoError(t, err) + + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "s5_orientation_root_probe as materialized") + require.Contains(t, formatted, "s5_orientation_suffix_probe as materialized") + require.Contains(t, formatted, "s5_orientation_boundaries as materialized") + require.Contains(t, formatted, "s5_orientation_forward_degree_probe as materialized") + require.Contains(t, formatted, "s5_orientation_reverse_degree_probe as materialized") + require.Contains(t, formatted, "select true as sampled from s5_orientation_root_probe") + require.Contains(t, formatted, "select true as sampled from s5_orientation_boundaries") + require.Contains(t, formatted, "s5_orientation_metrics as materialized") + require.Contains(t, formatted, "s5_orientation_decision as materialized") + require.Contains(t, formatted, "s5_orientation_states as materialized") + require.Contains(t, formatted, "s5_orientation_admission as materialized") + require.Contains(t, formatted, "s5_orientation_executed_candidate as materialized") + require.Contains(t, formatted, "s5_orientation_executed_incumbent as materialized") + require.Contains(t, formatted, "record_traversal_runtime_attestation_v1('EXPANSION-SUFFIX-SEEDED-REVERSE', 'suffix_seeded_reverse', false)") + require.Contains(t, formatted, "record_traversal_runtime_attestation_v1('EXPANSION-STEPWISE-FORWARD', 'exact_forward_incumbent'") + require.Contains(t, formatted, "s5_orientation_incumbent as materialized") + require.Contains(t, formatted, "limit 513") + require.Contains(t, formatted, "limit 16385") + require.Contains(t, formatted, "limit 4097") + require.Contains(t, formatted, "select (s0.n0).id as root_id from s0 limit 513") + require.NotContains(t, formatted, "select distinct (s0.n0).id as root_id from s0 limit 513") + require.Contains(t, formatted, "select distinct s5_orientation_suffix_probe.boundary_id as boundary_id") + require.Contains(t, formatted, "e3.id != e2.id limit 513") + require.Contains(t, formatted, "offset 512 limit 1") + require.Contains(t, formatted, "offset 16384 limit 1") + require.Contains(t, formatted, "offset 4096 limit 1") + require.Contains(t, formatted, "(s5_orientation_metrics.suffix_rows + s5_orientation_metrics.boundary_rows + s5_orientation_metrics.reverse_degree_rows) * 4 < (s5_orientation_metrics.root_rows + s5_orientation_metrics.forward_degree_rows) * 3") + require.Contains(t, formatted, "s5_orientation_admission.use_reverse and not s5_orientation_admission.state_overflow") + require.Contains(t, formatted, "not s5_orientation_admission.use_reverse or s5_orientation_admission.state_overflow") + require.Contains(t, formatted, "not s5_orientation_admission.probes_complete or s5_orientation_admission.state_overflow") + require.Equal(t, 1, strings.Count(formatted, "offset 4096 limit 1")) + require.Contains(t, formatted, "from s5_orientation_executed_candidate join lateral") + require.Contains(t, formatted, "s5_orientation_executed_candidate.executed offset 0") + require.Contains(t, formatted, "s5_orientation_incumbent as materialized (with") + require.Contains(t, formatted, "from s5_orientation_executed_incumbent join lateral") + require.Contains(t, formatted, "s5_orientation_executed_incumbent.executed offset 0") + require.Contains(t, formatted, "s5_orientation_reverse_gate as materialized") + require.Contains(t, formatted, "from s5_orientation_reverse_gate join lateral") + require.Contains(t, formatted, "s5_orientation_reverse_gate.executed offset 0") + require.Contains(t, formatted, "s5_orientation_states as materialized (select") + require.Contains(t, formatted, "from s5_orientation_reverse_gate join lateral (select s5_orientation_reverse.boundary_id") + guardedSuffixProjection := regexp.MustCompile(`(?s)s5_orientation_suffix_probe as materialized \(select (.*?) from s5_orientation_root_presence`).FindStringSubmatch(formatted) + require.Len(t, guardedSuffixProjection, 2) + require.Contains(t, guardedSuffixProjection[1], "n1.id as boundary_id") + require.Contains(t, guardedSuffixProjection[1], "e1.id as e1") + require.Contains(t, guardedSuffixProjection[1], "e2.id as e2") + require.Contains(t, guardedSuffixProjection[1], "e3.id as e3") + require.Contains(t, guardedSuffixProjection[1], "::nodecomposite") + require.Contains(t, formatted, "e3.id != e1.id") + require.Contains(t, formatted, "e3.id != e2.id") + require.Contains(t, formatted, "union all") + require.NotContains(t, formatted, "_orientation_shadow_") + require.NotContains(t, formatted, "s5_orientation_incumbent as materialized (with s1 as (with recursive s2_seed(root_id) as not materialized (select distinct (s0.n0).id as root_id from s0 limit") + + require.Len(t, translation.Optimization.LoweringPlan.ExpansionSearchStrategy, 1) + decision := translation.Optimization.LoweringPlan.ExpansionSearchStrategy[0] + require.Equal(t, optimize.ExpansionSearchStepwiseForward, decision.SelectedStrategy) + require.Equal(t, optimize.ExpansionSearchPolicyOrientationProbeV1, decision.EmittedPolicy) + require.Equal(t, []optimize.ExpansionSearchStrategy{ + optimize.ExpansionSearchStepwiseForward, + optimize.ExpansionSearchSuffixSeededReverse, + }, decision.EmittedCandidates) + require.Equal(t, "guarded_tool", decision.SelectionMode) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV1), decision.SelectorVersion) + require.Empty(t, decision.FallbackReason) + require.Equal(t, optimize.ExpansionSearchProbeCaps{ + RootRowLimit: optimize.ExpansionSearchOrientationRootRowLimit, + ReverseSeedRowLimit: optimize.ExpansionSearchOrientationReverseSeedRowLimit, + DirectionalDegreeRowLimit: optimize.ExpansionSearchOrientationDirectionalDegreeRowLimit, + }, decision.ProbeCaps) + require.Equal(t, optimize.ExpansionSearchAdmission{ + StateLimit: optimize.ExpansionSearchOrientationStateLimit, + RequiresCompleteProbes: true, + FallbackStrategy: optimize.ExpansionSearchStepwiseForward, + }, decision.Admission) + + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy, decision.Target) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV1), outcome.EmittedPolicy) + require.Equal(t, "guarded_dual_arm", outcome.ExecutionBoundary) + require.Empty(t, outcome.Applied) + require.Empty(t, outcome.SkipReason) + requireOptimizationLowering(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy) + requireNoSkippedOptimizationLowering(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy) +} + +func TestProductionCanaryExpansionOrientationUsesVersionedGuardedPolicy(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + translation, err := TranslateWithProductionOptions(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": "guarded-fixed-suffix-root", + }, DefaultGraphID, ProductionOptions{EnableExpansionOrientation: true, SelectorVersion: "traversal-production-g11"}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "record_traversal_runtime_attestation_v1") + require.Contains(t, formatted, "(s5_orientation_metrics.suffix_rows + s5_orientation_metrics.boundary_rows + s5_orientation_metrics.reverse_degree_rows) * 4 < (s5_orientation_metrics.root_rows + s5_orientation_metrics.forward_degree_rows) * 3") + require.NotContains(t, formatted, "16 * s5_orientation_metrics.forward_degree_rows") + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy, + optimize.TraversalStepTarget{QueryPartIndex: 0, ClauseIndex: 1, PatternIndex: 0, StepIndex: 0}) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV1), outcome.PlannedPolicy) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV1), outcome.EmittedPolicy) + require.Equal(t, "production_canary", outcome.SelectionMode) + require.Equal(t, "traversal-production-g11", outcome.SelectorVersion) + require.Equal(t, "guarded_dual_arm", outcome.ExecutionBoundary) +} + +func TestSuffixOrientationShadowEmitsWouldSelectMetadataAndOnlyIncumbent(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": "shadow-fixed-suffix-root", + }, DefaultGraphID, ToolOptions{EnableExpansionOrientationShadow: true}) + require.NoError(t, err) + + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "s5_orientation_root_probe as materialized") + require.Contains(t, formatted, "s5_orientation_suffix_probe as materialized") + require.Contains(t, formatted, "s5_orientation_forward_degree_probe as materialized") + require.Contains(t, formatted, "s5_orientation_reverse_degree_probe as materialized") + require.Contains(t, formatted, "select true as sampled from s5_orientation_root_probe") + require.Contains(t, formatted, "select true as sampled from s5_orientation_boundaries") + require.Contains(t, formatted, "s5_orientation_metrics as materialized") + require.Contains(t, formatted, "s5_orientation_decision as materialized") + require.Contains(t, formatted, "as would_select_reverse") + require.Contains(t, formatted, "s5_orientation_shadow_forward as materialized") + require.Contains(t, formatted, "s5_orientation_shadow_reverse as materialized") + require.Contains(t, formatted, "s5_orientation_shadow_selection as materialized") + require.Contains(t, formatted, "s5_orientation_executed_incumbent as materialized") + require.Contains(t, formatted, "record_traversal_runtime_attestation_v1('EXPANSION-STEPWISE-FORWARD', 'shadow_incumbent', false)") + require.Contains(t, formatted, "from s5_orientation_executed_incumbent join lateral") + require.Contains(t, formatted, "s5_orientation_executed_incumbent.executed offset 0") + require.Contains(t, formatted, "limit 513") + require.Contains(t, formatted, "limit 16385") + require.Contains(t, formatted, "offset 512 limit 1") + require.Contains(t, formatted, "offset 16384 limit 1") + require.NotContains(t, formatted, "s5_orientation_states") + require.NotContains(t, formatted, "s5_orientation_reverse(boundary_id") + require.NotContains(t, formatted, "limit 4097") + forwardDegreeProjection := regexp.MustCompile(`(?s)s5_orientation_forward_degree_probe as materialized \(select (.*?) from s5_orientation_root_probe`).FindStringSubmatch(formatted) + require.Len(t, forwardDegreeProjection, 2) + require.Equal(t, "true as sampled", forwardDegreeProjection[1]) + reverseDegreeProjection := regexp.MustCompile(`(?s)s5_orientation_reverse_degree_probe as materialized \(select (.*?) from s5_orientation_boundaries`).FindStringSubmatch(formatted) + require.Len(t, reverseDegreeProjection, 2) + require.Equal(t, "true as sampled", reverseDegreeProjection[1]) + suffixProjection := regexp.MustCompile(`(?s)s5_orientation_suffix_probe as materialized \(select (.*?) from s5_orientation_root_presence`).FindStringSubmatch(formatted) + require.Len(t, suffixProjection, 2) + require.Equal(t, "n1.id as boundary_id", suffixProjection[1]) + + require.Len(t, translation.Optimization.LoweringPlan.ExpansionSearchStrategy, 1) + decision := translation.Optimization.LoweringPlan.ExpansionSearchStrategy[0] + require.Equal(t, optimize.ExpansionSearchStepwiseForward, decision.SelectedStrategy) + require.Equal(t, optimize.ExpansionSearchPolicyOrientationProbeV1, decision.EmittedPolicy) + require.Equal(t, []optimize.ExpansionSearchStrategy{optimize.ExpansionSearchStepwiseForward}, decision.EmittedCandidates) + require.Equal(t, "shadow_tool", decision.SelectionMode) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV1), decision.SelectorVersion) + require.Empty(t, decision.FallbackReason) + + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy, decision.Target) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV1), outcome.EmittedPolicy) + require.Equal(t, []string{string(optimize.ExpansionSearchStepwiseForward)}, outcome.EmittedCandidates) + require.Equal(t, string(optimize.ExpansionSearchStepwiseForward), outcome.Selected) + require.Equal(t, "inline_statement", outcome.ExecutionBoundary) + require.Empty(t, outcome.Applied) + require.Empty(t, outcome.SkipReason) +} + +func TestSuffixOrientationShadowIsParameterStable(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + translate := func(rootKey string) string { + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": rootKey, + }, DefaultGraphID, ToolOptions{EnableExpansionOrientationShadow: true}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + return formatted + } + + first := translate("shadow-root-a") + second := translate("shadow-root-b") + require.Equal(t, first, second) + require.Contains(t, first, "@pi0::text") +} + +func TestGuardedSuffixOrientationSQLIsParameterStable(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + translate := func(rootKey string) string { + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": rootKey, + }, DefaultGraphID, ToolOptions{EnableExpansionOrientationTournament: true}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + return formatted + } + + first := translate("root-a") + second := translate("root-b") + require.Equal(t, first, second) + require.Contains(t, first, "@pi0::text") +} + +func TestGuardedSuffixOrientationAlignsSupportedOutputShapes(t *testing.T) { + for _, testCase := range []struct { + name string + projection string + expected string + }{ + {name: "endpoint IDs", projection: "id(head), id(terminal)", expected: `select s5.n2 as "id(head)", s5.n4 as "id(terminal)"`}, + {name: "ordered path IDs", projection: "length(path)", expected: `as "length(path)"`}, + {name: "full path", projection: "path", expected: "ordered_edge_ids_to_path"}, + } { + t.Run(testCase.name, func(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH (root:ExpansionRoot) + WHERE root.root_key = $root_key + MATCH path = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) + RETURN `+testCase.projection) + require.NoError(t, err) + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": "output-root", + }, DefaultGraphID, ToolOptions{EnableExpansionOrientationTournament: true}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, testCase.expected) + require.Equal(t, optimize.ExpansionSearchPolicyOrientationProbeV1, translation.Optimization.LoweringPlan.ExpansionSearchStrategy[0].EmittedPolicy) + }) + } +} + +func TestProductionFixedSuffixTranslationRemainsIncumbent(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + translation, err := Translate(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": "production-root", + }, DefaultGraphID) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.NotContains(t, formatted, "_orientation_") + require.Contains(t, formatted, "s2(root_id, next_id, depth, satisfied, is_cycle, path)") + + decision := translation.Optimization.LoweringPlan.ExpansionSearchStrategy[0] + require.Equal(t, optimize.ExpansionSearchStepwiseForward, decision.SelectedStrategy) + require.Empty(t, decision.EmittedPolicy) + require.Equal(t, []optimize.ExpansionSearchStrategy{optimize.ExpansionSearchStepwiseForward}, decision.EmittedCandidates) + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy, decision.Target) + require.Equal(t, "inline_statement", outcome.ExecutionBoundary) +} + +func TestGuardedSuffixOrientationUsesOnlyTargetGraphRelations(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": "graph-scoped-root", + }, 42, ToolOptions{EnableExpansionOrientationTournament: true}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "node_42") + require.Contains(t, formatted, "edge_42") + require.Contains(t, formatted, "ordered_edge_ids_to_path(42,") + require.NotRegexp(t, regexp.MustCompile(`(?i)(from|join) (node|edge)(?:\s|;)`), formatted) +} + +func TestExpansionOrientationTournamentRejectsConflictingForceWithoutMutation(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + plan, err := optimize.Optimize(regularQuery) + require.NoError(t, err) + before := append([]optimize.ExpansionSearchStrategyDecision(nil), plan.LoweringPlan.ExpansionSearchStrategy...) + + err = applyToolOptions(&plan, ToolOptions{ + EnableExpansionOrientationTournament: true, + ForceExpansionSearchStrategy: optimize.ExpansionSearchSuffixSeededReverse, + }) + require.ErrorContains(t, err, "mutually exclusive") + require.Equal(t, before, plan.LoweringPlan.ExpansionSearchStrategy) +} + +func TestExpansionOrientationShadowRejectsConflictingModesWithoutMutation(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + plan, err := optimize.Optimize(regularQuery) + require.NoError(t, err) + before := append([]optimize.ExpansionSearchStrategyDecision(nil), plan.LoweringPlan.ExpansionSearchStrategy...) + + for _, options := range []ToolOptions{ + { + EnableExpansionOrientationTournament: true, + EnableExpansionOrientationShadow: true, + }, + { + EnableExpansionOrientationShadow: true, + ForceExpansionSearchStrategy: optimize.ExpansionSearchSuffixSeededReverse, + }, + } { + err := applyToolOptions(&plan, options) + require.ErrorContains(t, err, "mutually exclusive") + require.Equal(t, before, plan.LoweringPlan.ExpansionSearchStrategy) + } +} + +func TestExpansionOrientationPolicyRequiresSupportedEnabledMode(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + plan, err := optimize.Optimize(regularQuery) + require.NoError(t, err) + before := append([]optimize.ExpansionSearchStrategyDecision(nil), plan.LoweringPlan.ExpansionSearchStrategy...) + + err = applyToolOptions(&plan, ToolOptions{ExpansionOrientationPolicy: optimize.ExpansionSearchPolicyOrientationProbeV2}) + require.ErrorContains(t, err, "requires tournament or shadow mode") + require.Equal(t, before, plan.LoweringPlan.ExpansionSearchStrategy) + + err = applyToolOptions(&plan, ToolOptions{ + ExpansionOrientationPolicy: optimize.ExpansionSearchPolicy("orientation-probe-v3"), + EnableExpansionOrientationTournament: true, + }) + require.ErrorContains(t, err, "unsupported expansion orientation policy") + require.Equal(t, before, plan.LoweringPlan.ExpansionSearchStrategy) +} + +func TestExpansionOrientationShadowRequiresExactlyOneEligibleTarget(t *testing.T) { + plan := optimize.Plan{LoweringPlan: optimize.LoweringPlan{ + ExpansionSearchStrategy: []optimize.ExpansionSearchStrategyDecision{ + { + Family: "fixed_suffix_expansion", + CandidateStrategy: optimize.ExpansionSearchSuffixSeededReverse, + SelectedStrategy: optimize.ExpansionSearchStepwiseForward, + StructurallyEligible: true, + StaticallyEligible: true, + EmittedCandidates: []optimize.ExpansionSearchStrategy{optimize.ExpansionSearchStepwiseForward}, + }, + { + Family: "fixed_suffix_expansion", + CandidateStrategy: optimize.ExpansionSearchSuffixSeededReverse, + SelectedStrategy: optimize.ExpansionSearchStepwiseForward, + StructurallyEligible: true, + StaticallyEligible: true, + EmittedCandidates: []optimize.ExpansionSearchStrategy{optimize.ExpansionSearchStepwiseForward}, + }, + }, + }} + before := append([]optimize.ExpansionSearchStrategyDecision(nil), plan.LoweringPlan.ExpansionSearchStrategy...) + + err := applyExpansionOrientationShadow(&plan) + require.ErrorContains(t, err, "matched 2 structurally eligible fixed-suffix targets; expected exactly one") + require.Equal(t, before, plan.LoweringPlan.ExpansionSearchStrategy) +} + +func TestExpansionOrientationTournamentRequiresExactlyOneEligibleTarget(t *testing.T) { + plan := optimize.Plan{LoweringPlan: optimize.LoweringPlan{ + ExpansionSearchStrategy: []optimize.ExpansionSearchStrategyDecision{ + { + Family: "fixed_suffix_expansion", + CandidateStrategy: optimize.ExpansionSearchSuffixSeededReverse, + SelectedStrategy: optimize.ExpansionSearchStepwiseForward, + StructurallyEligible: true, + StaticallyEligible: true, + EmittedCandidates: []optimize.ExpansionSearchStrategy{optimize.ExpansionSearchStepwiseForward}, + FallbackReason: optimize.ExpansionSearchFallbackTournamentUnqualified, + }, + { + Family: "fixed_suffix_expansion", + CandidateStrategy: optimize.ExpansionSearchSuffixSeededReverse, + SelectedStrategy: optimize.ExpansionSearchStepwiseForward, + StructurallyEligible: true, + StaticallyEligible: true, + EmittedCandidates: []optimize.ExpansionSearchStrategy{optimize.ExpansionSearchStepwiseForward}, + FallbackReason: optimize.ExpansionSearchFallbackTournamentUnqualified, + }, + }, + }} + before := append([]optimize.ExpansionSearchStrategyDecision(nil), plan.LoweringPlan.ExpansionSearchStrategy...) + + err := applyExpansionOrientationTournament(&plan) + require.ErrorContains(t, err, "matched 2 structurally eligible fixed-suffix targets; expected exactly one") + require.Equal(t, before, plan.LoweringPlan.ExpansionSearchStrategy) +} + +func TestExpansionOrientationTournamentRejectsNonInitialVariableRegion(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH (root:ExpansionRoot) + WHERE root.root_key = $root_key + MATCH ()-[:Prefix]->(root)-[:Expand*0..16]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(:SuffixTerminal) + RETURN id(root) + `) + require.NoError(t, err) + plan, err := optimize.Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ExpansionSearchStrategy, 1) + require.False(t, plan.LoweringPlan.ExpansionSearchStrategy[0].StructurallyEligible) + require.NotEqual(t, "fixed_suffix_expansion", plan.LoweringPlan.ExpansionSearchStrategy[0].Family) + require.ErrorContains(t, applyExpansionOrientationTournament(&plan), "has no structurally eligible fixed-suffix target") +} diff --git a/cypher/models/pgsql/translate/expansion_suffix_seeded.go b/cypher/models/pgsql/translate/expansion_suffix_seeded.go new file mode 100644 index 00000000..3dca77f6 --- /dev/null +++ b/cypher/models/pgsql/translate/expansion_suffix_seeded.go @@ -0,0 +1,988 @@ +package translate + +import ( + "fmt" + + "github.com/specterops/dawgs/cypher/models" + "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/pgd" +) + +const ( + // fixedSuffixBoundaryID names the column containing the node where reverse search enters the fixed suffix. + fixedSuffixBoundaryID pgsql.Identifier = "boundary_id" +) + +// suffixSeededIdentifiers names the root-presence, suffix, boundary, and reverse-search CTEs for one rewrite. +type suffixSeededIdentifiers struct { + // rootPresence names the relation that records whether the bound root produced rows. + rootPresence pgsql.Identifier + // suffix names the materialized matches for the fixed terminal suffix. + suffix pgsql.Identifier + // boundaries names the distinct suffix-boundary nodes used to seed reverse search. + boundaries pgsql.Identifier + // reverse names the recursive relation that searches from each boundary toward the root. + reverse pgsql.Identifier +} + +// newSuffixSeededIdentifiers derives collision-resistant CTE names from the incumbent final frame. +func newSuffixSeededIdentifiers(finalFrame pgsql.Identifier) suffixSeededIdentifiers { + prefix := string(finalFrame) + "_suffix_seeded_" + return suffixSeededIdentifiers{ + rootPresence: pgsql.Identifier(prefix + "root_presence"), + suffix: pgsql.Identifier(prefix + "suffix"), + boundaries: pgsql.Identifier(prefix + "boundaries"), + reverse: pgsql.Identifier(prefix + "reverse"), + } +} + +// selectedFixedSuffixDecision returns the first traversal decision that selected suffix-seeded reverse search. +func selectedFixedSuffixDecision(part *PatternPart, decisions map[optimize.TraversalStepTarget]optimize.ExpansionSearchStrategyDecision) (optimize.ExpansionSearchStrategyDecision, bool) { + for _, step := range part.TraversalSteps { + if step == nil || !step.HasSourceTarget { + continue + } + if decision, found := decisions[step.SourceTarget]; found && decision.SelectedStrategy == optimize.ExpansionSearchSuffixSeededReverse { + return decision, true + } + } + + return optimize.ExpansionSearchStrategyDecision{}, false +} + +// selectedGuardedFixedSuffixDecision returns a tool-enabled suffix policy +// without treating its runtime decision as a compile-time selected arm. The +// decision's selection mode distinguishes guarded execution from true shadow. +func selectedGuardedFixedSuffixDecision(part *PatternPart, decisions map[optimize.TraversalStepTarget]optimize.ExpansionSearchStrategyDecision) (optimize.ExpansionSearchStrategyDecision, bool) { + for _, step := range part.TraversalSteps { + if step == nil || !step.HasSourceTarget { + continue + } + if decision, found := decisions[step.SourceTarget]; found && + decision.Family == "fixed_suffix_expansion" && + decision.CandidateStrategy == optimize.ExpansionSearchSuffixSeededReverse && + supportedExpansionOrientationPolicy(decision.EmittedPolicy) { + return decision, true + } + } + + return optimize.ExpansionSearchStrategyDecision{}, false +} + +// rewriteTraversalPatternAsSuffixSeededReverse replaces a qualified incumbent frame chain with fixed-suffix reverse search. +func (s *Translator) rewriteTraversalPatternAsSuffixSeededReverse(part *PatternPart, decision optimize.ExpansionSearchStrategyDecision, firstCTE int) error { + if len(part.TraversalSteps) != decision.SuffixEndStep+1 || decision.SuffixLength != 3 || decision.Target.StepIndex != 0 { + return fmt.Errorf("forced suffix-seeded reverse target requires one expansion followed by exactly three terminal suffix steps") + } + + expansionStep := part.TraversalSteps[decision.Target.StepIndex] + if expansionStep == nil || expansionStep.Expansion == nil || expansionStep.Frame == nil || expansionStep.Frame.Previous == nil || !expansionStep.LeftNodeBound { + return fmt.Errorf("forced suffix-seeded reverse target requires a bound root materialized by a previous frame") + } + + suffix := part.TraversalSteps[decision.SuffixStartStep : decision.SuffixEndStep+1] + for _, step := range suffix { + if step == nil || step.Frame == nil || step.Edge == nil || step.LeftNode == nil || step.RightNode == nil { + return fmt.Errorf("forced suffix-seeded reverse target has an incomplete fixed suffix step") + } + } + + ctes := s.query.CurrentPart().Model.CommonTableExpressions.Expressions + if firstCTE < 0 || firstCTE >= len(ctes) { + return fmt.Errorf("forced suffix-seeded reverse target did not emit an incumbent frame chain") + } + incumbentFinal := ctes[len(ctes)-1] + if incumbentFinal.Alias.Name != suffix[len(suffix)-1].Frame.Binding.Identifier { + return fmt.Errorf("forced suffix-seeded reverse final frame mismatch: expected %s but found %s", suffix[len(suffix)-1].Frame.Binding.Identifier, incumbentFinal.Alias.Name) + } + + finalSelect, ok := incumbentFinal.Query.Body.(pgsql.Select) + if !ok { + return fmt.Errorf("forced suffix-seeded reverse final frame must be a select") + } + + ids := newSuffixSeededIdentifiers(incumbentFinal.Alias.Name) + rootFrame := expansionStep.Frame.Previous.Binding.Identifier + suffixSeededQuery, err := s.buildSuffixSeededReverseQuery(part, decision, expansionStep, suffix, rootFrame, ids, finalSelect.Projection) + if err != nil { + return err + } + + replacement := pgsql.CommonTableExpression{ + Alias: incumbentFinal.Alias, + Query: suffixSeededQuery, + } + s.query.CurrentPart().Model.CommonTableExpressions.Expressions = append(ctes[:firstCTE], replacement) + s.recordExpansionSearchStrategy(decision.Target, optimize.ExpansionSearchSuffixSeededReverse) + return nil +} + +// rewriteTraversalPatternAsGuardedSuffixOrientation emits a tool-selected, +// versioned orientation policy. Guarded mode wraps the incumbent and reverse +// arm in disjoint runtime gates; shadow mode executes the same bounded probes +// but leaves the incumbent as the only traversal arm. +func (s *Translator) rewriteTraversalPatternAsGuardedSuffixOrientation(part *PatternPart, decision optimize.ExpansionSearchStrategyDecision, firstCTE int) error { + if len(part.TraversalSteps) != decision.SuffixEndStep+1 || decision.SuffixLength != 3 || decision.Target.StepIndex != 0 { + return fmt.Errorf("guarded suffix orientation requires one expansion followed by exactly three terminal suffix steps") + } + + expansionStep := part.TraversalSteps[decision.Target.StepIndex] + if expansionStep == nil || expansionStep.Expansion == nil || expansionStep.Frame == nil || expansionStep.Frame.Previous == nil || !expansionStep.LeftNodeBound || expansionStep.Edge == nil || expansionStep.LeftNode == nil { + return fmt.Errorf("guarded suffix orientation requires a complete expansion and bound root") + } + + suffix := part.TraversalSteps[decision.SuffixStartStep : decision.SuffixEndStep+1] + for _, step := range suffix { + if step == nil || step.Frame == nil || step.Edge == nil || step.LeftNode == nil || step.RightNode == nil { + return fmt.Errorf("guarded suffix orientation has an incomplete fixed suffix step") + } + } + + ctes := s.query.CurrentPart().Model.CommonTableExpressions.Expressions + if firstCTE < 0 || firstCTE >= len(ctes) { + return fmt.Errorf("guarded suffix orientation did not emit an incumbent frame chain") + } + incumbentChain := append([]pgsql.CommonTableExpression(nil), ctes[firstCTE:]...) + incumbentFinal := incumbentChain[len(incumbentChain)-1] + if incumbentFinal.Alias.Name != suffix[len(suffix)-1].Frame.Binding.Identifier { + return fmt.Errorf("guarded suffix orientation final frame mismatch: expected %s but found %s", suffix[len(suffix)-1].Frame.Binding.Identifier, incumbentFinal.Alias.Name) + } + incumbentSelect, ok := incumbentFinal.Query.Body.(pgsql.Select) + if !ok { + return fmt.Errorf("guarded suffix orientation final frame must be a select") + } + + ids := newExpansionOrientationIdentifiers(incumbentFinal.Alias.Name) + rootFrame := expansionStep.Frame.Previous.Binding.Identifier + var ( + query pgsql.Query + err error + ) + if decision.SelectionMode == "shadow_tool" { + query, err = s.buildShadowSuffixOrientationQuery( + decision, + expansionStep, + suffix, + rootFrame, + ids, + incumbentChain, + incumbentFinal.Alias.Name, + incumbentSelect.Projection, + ) + } else { + query, err = s.buildGuardedSuffixOrientationQuery( + part, + decision, + expansionStep, + suffix, + rootFrame, + ids, + incumbentChain, + incumbentFinal.Alias.Name, + incumbentSelect.Projection, + ) + } + if err != nil { + return err + } + + s.query.CurrentPart().Model.CommonTableExpressions.Expressions = append(ctes[:firstCTE], pgsql.CommonTableExpression{ + Alias: incumbentFinal.Alias, + Query: query, + }) + s.recordExpansionSearchPolicy(decision.Target, decision.EmittedPolicy) + return nil +} + +// buildShadowSuffixOrientationQuery executes only bounded policy probes and +// the exact incumbent. Named, mutually exclusive marker CTEs preserve the +// policy's would_select_reverse result for plan-derived diagnostic metadata; +// they never dispatch the reverse traversal candidate. +func (s *Translator) buildShadowSuffixOrientationQuery( + decision optimize.ExpansionSearchStrategyDecision, + expansionStep *TraversalStep, + suffix []*TraversalStep, + rootFrame pgsql.Identifier, + ids expansionOrientationIdentifiers, + incumbentChain []pgsql.CommonTableExpression, + incumbentFinal pgsql.Identifier, + incumbentProjection pgsql.Projection, +) (pgsql.Query, error) { + if decision.ProbeCaps.RootRowLimit <= 0 || decision.ProbeCaps.ReverseSeedRowLimit <= 0 || decision.ProbeCaps.DirectionalDegreeRowLimit <= 0 { + return pgsql.Query{}, fmt.Errorf("shadow suffix orientation requires positive immutable probe caps") + } + + localEdgeConstraint, externalEdgeConstraint := partitionConstraintByLocality( + expansionStep.Expansion.EdgeConstraints, + pgsql.AsIdentifierSet(expansionStep.Edge.Identifier), + ) + if externalEdgeConstraint != nil { + return pgsql.Query{}, fmt.Errorf("shadow suffix orientation relationship predicate is not local") + } + + suffixIDs := suffixSeededIdentifiers{ + rootPresence: ids.rootPresence, + suffix: ids.suffixProbe, + boundaries: ids.boundaries, + } + rootProbe := buildExpansionOrientationRootProbe(rootFrame, expansionStep.LeftNode, ids, decision.ProbeCaps.RootRowLimit) + rootPresence := buildExpansionOrientationRootPresence(ids) + suffixProbe, err := s.buildFixedSuffixEvidenceProbeCTE(expansionStep, suffix, suffixIDs, decision.ProbeCaps.ReverseSeedRowLimit) + if err != nil { + return pgsql.Query{}, err + } + boundaries := buildFixedSuffixBoundariesCTE(suffixIDs) + forwardDegree := buildExpansionOrientationDegreeProbe( + ids.forwardDegreeProbe, + ids.rootProbe, + orientationRootID, + expansionStep.Edge.Identifier, + expansionStep.Expansion.EdgeStartIdentifier, + localEdgeConstraint, + decision.ProbeCaps.DirectionalDegreeRowLimit, + ) + reverseDegree := buildExpansionOrientationDegreeProbe( + ids.reverseDegreeProbe, + ids.boundaries, + fixedSuffixBoundaryID, + expansionStep.Edge.Identifier, + expansionStep.Expansion.EdgeEndIdentifier, + localEdgeConstraint, + decision.ProbeCaps.DirectionalDegreeRowLimit, + ) + metrics := buildExpansionOrientationMetrics(ids, decision.ProbeCaps) + policyDecision, err := buildExpansionOrientationDecision(ids, decision.EmittedPolicy, decision.MaximumDepth) + if err != nil { + return pgsql.Query{}, err + } + shadowMarkers := buildExpansionOrientationShadowMarkers(ids) + incumbent, incumbentOutput, err := buildExpansionOrientationIncumbentCTE(ids, incumbentChain, incumbentFinal, incumbentProjection) + if err != nil { + return pgsql.Query{}, err + } + gatedIncumbent, err := gateQueryBehindMarker( + ids.executedIncumbent, + ids.incumbentBody, + pgsql.Query{Body: pgsql.Select{ + Projection: incumbentOutput, + From: []pgsql.FromClause{tableFrom(ids.incumbent)}, + }}, + incumbentOutput, + ) + if err != nil { + return pgsql.Query{}, err + } + + expressions := []pgsql.CommonTableExpression{ + rootProbe, + rootPresence, + suffixProbe, + boundaries, + forwardDegree, + reverseDegree, + metrics, + policyDecision, + } + expressions = append(expressions, shadowMarkers...) + expressions = append(expressions, incumbent) + + return pgsql.Query{ + CommonTableExpressions: &pgsql.With{ + Recursive: true, + Expressions: expressions, + }, + Body: gatedIncumbent, + }, nil +} + +// buildGuardedSuffixOrientationQuery emits bounded evidence, a versioned +// decision, reverse-state admission, and strictly complementary candidate and +// incumbent branches. No candidate row can pass until every evidence and +// state sentinel proves completeness. +func (s *Translator) buildGuardedSuffixOrientationQuery( + part *PatternPart, + decision optimize.ExpansionSearchStrategyDecision, + expansionStep *TraversalStep, + suffix []*TraversalStep, + rootFrame pgsql.Identifier, + ids expansionOrientationIdentifiers, + incumbentChain []pgsql.CommonTableExpression, + incumbentFinal pgsql.Identifier, + incumbentProjection pgsql.Projection, +) (pgsql.Query, error) { + if decision.ProbeCaps.RootRowLimit <= 0 || decision.ProbeCaps.ReverseSeedRowLimit <= 0 || decision.ProbeCaps.DirectionalDegreeRowLimit <= 0 || decision.Admission.StateLimit <= 0 { + return pgsql.Query{}, fmt.Errorf("guarded suffix orientation requires positive immutable probe and admission caps") + } + + localEdgeConstraint, externalEdgeConstraint := partitionConstraintByLocality( + expansionStep.Expansion.EdgeConstraints, + pgsql.AsIdentifierSet(expansionStep.Edge.Identifier), + ) + if externalEdgeConstraint != nil { + return pgsql.Query{}, fmt.Errorf("guarded suffix orientation relationship predicate is not local") + } + + suffixIDs := suffixSeededIdentifiers{ + rootPresence: ids.rootPresence, + suffix: ids.suffixProbe, + boundaries: ids.boundaries, + reverse: ids.reverse, + } + rootProbe := buildExpansionOrientationRootProbe(rootFrame, expansionStep.LeftNode, ids, decision.ProbeCaps.RootRowLimit) + rootPresence := buildExpansionOrientationRootPresence(ids) + suffixProbe, err := s.buildFixedSuffixProbeCTE(expansionStep, suffix, suffixIDs, decision.ProbeCaps.ReverseSeedRowLimit) + if err != nil { + return pgsql.Query{}, err + } + boundaries := buildFixedSuffixBoundariesCTE(suffixIDs) + forwardDegree := buildExpansionOrientationDegreeProbe( + ids.forwardDegreeProbe, + ids.rootProbe, + orientationRootID, + expansionStep.Edge.Identifier, + expansionStep.Expansion.EdgeStartIdentifier, + localEdgeConstraint, + decision.ProbeCaps.DirectionalDegreeRowLimit, + ) + reverseDegree := buildExpansionOrientationDegreeProbe( + ids.reverseDegreeProbe, + ids.boundaries, + fixedSuffixBoundaryID, + expansionStep.Edge.Identifier, + expansionStep.Expansion.EdgeEndIdentifier, + localEdgeConstraint, + decision.ProbeCaps.DirectionalDegreeRowLimit, + ) + metrics := buildExpansionOrientationMetrics(ids, decision.ProbeCaps) + policyDecision, err := buildExpansionOrientationDecision(ids, decision.EmittedPolicy, decision.MaximumDepth) + if err != nil { + return pgsql.Query{}, err + } + reverseSeed := buildExpansionOrientationReverseSeed(ids) + reverseIDs := suffixIDs + reverseIDs.boundaries = ids.reverseSeed + reverse, err := buildSuffixSeededReverseCTE(expansionStep, decision, reverseIDs, "", "") + if err != nil { + return pgsql.Query{}, err + } + states := expansionOrientationStateProbe(decision, ids) + admission := buildExpansionOrientationAdmission(ids, decision.Admission.StateLimit) + executionMarkers := buildExpansionOrientationExecutionMarkers(ids) + incumbent, fallbackProjection, err := buildExpansionOrientationIncumbentCTE(ids, incumbentChain, incumbentFinal, incumbentProjection) + if err != nil { + return pgsql.Query{}, err + } + candidateProjection, err := suffixSeededFinalProjection(part, expansionStep, suffix, rootFrame, suffixIDs, ids.states, incumbentProjection, nil) + if err != nil { + return pgsql.Query{}, err + } + + suffixEdgeIDs := pgsql.ArrayLiteral{CastType: pgsql.Int8Array} + for _, step := range suffix { + suffixEdgeIDs.Values = append(suffixEdgeIDs.Values, pgsql.CompoundIdentifier{ids.suffixProbe, step.Edge.Identifier}) + } + var candidateWhere pgsql.Expression = pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{ids.states, expansionDepth}, + pgsql.OperatorGreaterThanOrEqualTo, + pgsql.NewLiteral(decision.MinimumDepth, pgsql.Int8), + ) + candidateWhere = pgsql.OptionalAnd(candidateWhere, pgd.Not(pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{ids.states, expansionPath}, + pgsql.OperatorArrayOverlap, + suffixEdgeIDs, + ))) + + candidate := pgsql.Select{ + Projection: candidateProjection, + From: []pgsql.FromClause{ + { + Source: pgsql.TableReference{Name: rootFrame.AsCompoundIdentifier()}, + Joins: []pgsql.Join{ + { + Table: pgsql.TableReference{Name: ids.states.AsCompoundIdentifier()}, + JoinOperator: pgsql.JoinOperator{JoinType: pgsql.JoinTypeInner, Constraint: pgsql.NewBinaryExpression( + projectedNodeIDReference(rootFrame, expansionStep.LeftNode), + pgsql.OperatorEquals, + pgsql.CompoundIdentifier{ids.states, expansionNextID}, + )}, + }, + { + Table: pgsql.TableReference{Name: ids.suffixProbe.AsCompoundIdentifier()}, + JoinOperator: pgsql.JoinOperator{JoinType: pgsql.JoinTypeInner, Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{ids.suffixProbe, fixedSuffixBoundaryID}, + pgsql.OperatorEquals, + pgsql.CompoundIdentifier{ids.states, fixedSuffixBoundaryID}, + )}, + }, + }, + }, + }, + Where: candidateWhere, + } + candidate, err = gateQueryBehindMarker( + ids.executedCandidate, + ids.candidateBody, + pgsql.Query{Body: candidate}, + candidateProjection, + ) + if err != nil { + return pgsql.Query{}, err + } + + fallback := pgsql.Select{ + Projection: fallbackProjection, + From: []pgsql.FromClause{tableFrom(ids.incumbent)}, + } + fallback, err = gateQueryBehindMarker( + ids.executedIncumbent, + ids.incumbentBody, + pgsql.Query{Body: fallback}, + fallbackProjection, + ) + if err != nil { + return pgsql.Query{}, err + } + expressions := []pgsql.CommonTableExpression{ + rootProbe, + rootPresence, + suffixProbe, + boundaries, + forwardDegree, + reverseDegree, + metrics, + policyDecision, + } + expressions = append(expressions, reverseSeed...) + expressions = append(expressions, reverse, states, admission) + expressions = append(expressions, executionMarkers...) + expressions = append(expressions, incumbent) + + return pgsql.Query{ + CommonTableExpressions: &pgsql.With{ + Recursive: true, + Expressions: expressions, + }, + Body: pgsql.SetOperation{ + Operator: pgsql.OperatorUnion, + All: true, + LOperand: candidate, + ROperand: fallback, + }, + }, nil +} + +func buildFixedSuffixBoundariesCTE(ids suffixSeededIdentifiers) pgsql.CommonTableExpression { + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.boundaries}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{ + Distinct: true, + Projection: pgsql.Projection{&pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{ids.suffix, fixedSuffixBoundaryID}, + Alias: models.OptionalValue(fixedSuffixBoundaryID), + }}, + From: []pgsql.FromClause{tableFrom(ids.suffix)}, + }}, + } +} + +// buildExpansionOrientationIncumbentCTE nests the original unmodified frame +// chain as the exact fallback. It has no tournament cap and preserves the +// incumbent's projection and bag semantics. +func buildExpansionOrientationIncumbentCTE( + ids expansionOrientationIdentifiers, + incumbentChain []pgsql.CommonTableExpression, + incumbentFinal pgsql.Identifier, + incumbentProjection pgsql.Projection, +) (pgsql.CommonTableExpression, pgsql.Projection, error) { + projection := make(pgsql.Projection, 0, len(incumbentProjection)) + fallback := make(pgsql.Projection, 0, len(incumbentProjection)) + for _, item := range incumbentProjection { + alias, ok := selectItemAlias(item) + if !ok { + return pgsql.CommonTableExpression{}, nil, fmt.Errorf("guarded suffix orientation incumbent projection contains an unaliased item %T", item) + } + projection = append(projection, &pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{incumbentFinal, alias}, + Alias: models.OptionalValue(alias), + }) + fallback = append(fallback, &pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{ids.incumbent, alias}, + Alias: models.OptionalValue(alias), + }) + } + + incumbentQuery := pgsql.Query{ + CommonTableExpressions: &pgsql.With{Expressions: incumbentChain}, + Body: pgsql.Select{ + Projection: projection, + From: []pgsql.FromClause{tableFrom(incumbentFinal)}, + }, + } + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.incumbent}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: incumbentQuery, + }, fallback, nil +} + +// buildSuffixSeededReverseQuery joins bound roots to reverse states seeded by materialized fixed-suffix matches. +func (s *Translator) buildSuffixSeededReverseQuery( + part *PatternPart, + decision optimize.ExpansionSearchStrategyDecision, + expansionStep *TraversalStep, + suffix []*TraversalStep, + rootFrame pgsql.Identifier, + ids suffixSeededIdentifiers, + incumbentProjection pgsql.Projection, +) (pgsql.Query, error) { + rootPresence := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: ids.rootPresence, + }, + Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: []pgsql.SelectItem{pgsql.NewLiteral(int64(1), pgsql.Int8)}, + From: []pgsql.FromClause{tableFrom(rootFrame)}, + }, + Limit: pgsql.NewLiteral(int64(1), pgsql.Int8), + }, + } + + suffixCTE, err := s.buildFixedSuffixCTE(expansionStep, suffix, ids) + if err != nil { + return pgsql.Query{}, err + } + + boundaries := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: ids.boundaries, + }, + Materialized: &pgsql.Materialized{ + Materialized: true, + }, + Query: pgsql.Query{ + Body: pgsql.Select{ + Distinct: true, + Projection: []pgsql.SelectItem{&pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{ids.suffix, fixedSuffixBoundaryID}, + Alias: models.OptionalValue(fixedSuffixBoundaryID), + }}, + From: []pgsql.FromClause{tableFrom(ids.suffix)}, + }, + }, + } + reverse, err := buildSuffixSeededReverseCTE(expansionStep, decision, ids, "", "") + if err != nil { + return pgsql.Query{}, err + } + + projection, err := suffixSeededFinalProjection(part, expansionStep, suffix, rootFrame, ids, ids.reverse, incumbentProjection, nil) + if err != nil { + return pgsql.Query{}, err + } + + suffixEdgeIDs := pgsql.ArrayLiteral{ + CastType: pgsql.Int8Array, + } + for _, step := range suffix { + suffixEdgeIDs.Values = append(suffixEdgeIDs.Values, pgsql.CompoundIdentifier{ids.suffix, step.Edge.Identifier}) + } + + reversePath := pgsql.CompoundIdentifier{ids.reverse, expansionPath} + finalWhere := pgsql.OptionalAnd( + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{ids.reverse, expansionDepth}, + pgsql.OperatorGreaterThanOrEqualTo, + pgsql.NewLiteral(decision.MinimumDepth, pgsql.Int8), + ), + pgd.Not(pgsql.NewBinaryExpression(reversePath, pgsql.OperatorArrayOverlap, suffixEdgeIDs)), + ) + + return pgsql.Query{ + CommonTableExpressions: &pgsql.With{ + Recursive: true, + Expressions: []pgsql.CommonTableExpression{ + rootPresence, + suffixCTE, + boundaries, + reverse, + }, + }, + Body: pgsql.Select{ + Projection: projection, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{ + Name: rootFrame.AsCompoundIdentifier(), + }, + Joins: []pgsql.Join{ + { + Table: pgsql.TableReference{ + Name: ids.reverse.AsCompoundIdentifier(), + }, + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + projectedNodeIDReference(rootFrame, expansionStep.LeftNode), + pgsql.OperatorEquals, + pgsql.CompoundIdentifier{ids.reverse, expansionNextID}, + ), + }, + }, + { + Table: pgsql.TableReference{ + Name: ids.suffix.AsCompoundIdentifier(), + }, + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{ids.suffix, fixedSuffixBoundaryID}, + pgsql.OperatorEquals, + pgsql.CompoundIdentifier{ids.reverse, fixedSuffixBoundaryID}, + ), + }, + }, + }, + }}, + Where: finalWhere, + }, + }, nil +} + +// buildFixedSuffixCTE materializes every locally valid fixed-suffix path and its boundary node. +func (s *Translator) buildFixedSuffixCTE(expansionStep *TraversalStep, suffix []*TraversalStep, ids suffixSeededIdentifiers) (pgsql.CommonTableExpression, error) { + return s.buildFixedSuffixCTEWithOptions(expansionStep, suffix, ids, false, false, 0) +} + +// buildFixedSuffixProbeCTE builds a bounded suffix probe used to guard the specialized branch. +func (s *Translator) buildFixedSuffixProbeCTE(expansionStep *TraversalStep, suffix []*TraversalStep, ids suffixSeededIdentifiers, rowLimit int64) (pgsql.CommonTableExpression, error) { + return s.buildFixedSuffixCTEWithOptions(expansionStep, suffix, ids, false, false, rowLimit) +} + +// buildFixedSuffixEvidenceProbeCTE preserves the suffix join and row +// multiplicity used by orientation scoring while projecting only the boundary +// ID needed by the shadow policy. Candidate execution is impossible in shadow +// mode, so materializing edge IDs and node composites would be pure overhead. +func (s *Translator) buildFixedSuffixEvidenceProbeCTE(expansionStep *TraversalStep, suffix []*TraversalStep, ids suffixSeededIdentifiers, rowLimit int64) (pgsql.CommonTableExpression, error) { + return s.buildFixedSuffixCTEWithOptions(expansionStep, suffix, ids, false, true, rowLimit) +} + +// buildFixedSuffixCTEWithOptions builds the fixed-suffix join chain with an +// optional evidence-only projection and row limit. +func (s *Translator) buildFixedSuffixCTEWithOptions(expansionStep *TraversalStep, suffix []*TraversalStep, ids suffixSeededIdentifiers, projectNodeIDs, evidenceOnly bool, rowLimit int64) (pgsql.CommonTableExpression, error) { + localScope := pgsql.NewIdentifierSet() + for _, step := range suffix { + localScope.Add(step.Edge.Identifier) + localScope.Add(step.LeftNode.Identifier) + localScope.Add(step.RightNode.Identifier) + } + + projection := pgsql.Projection{&pgsql.AliasedExpression{ + Expression: pgd.EntityID(suffix[0].LeftNode.Identifier), + Alias: models.OptionalValue(fixedSuffixBoundaryID), + }} + if !evidenceOnly { + for _, step := range suffix { + projection = append(projection, &pgsql.AliasedExpression{ + Expression: pgd.EntityID(step.Edge.Identifier), + Alias: models.OptionalValue(step.Edge.Identifier), + }) + } + for idx, step := range suffix { + binding := step.RightNode + expression := suffixSeededNodeValue(binding) + if projectNodeIDs { + expression = pgd.EntityID(binding.Identifier) + } + projection = append(projection, &pgsql.AliasedExpression{ + Expression: expression, + Alias: models.OptionalValue(binding.Identifier), + }) + if idx == 0 { + leftExpression := suffixSeededNodeValue(step.LeftNode) + if projectNodeIDs { + leftExpression = pgd.EntityID(step.LeftNode.Identifier) + } + projection = append(projection, &pgsql.AliasedExpression{ + Expression: leftExpression, + Alias: models.OptionalValue(step.LeftNode.Identifier), + }) + } + } + } + + first := suffix[0] + from := pgsql.FromClause{ + Source: pgsql.TableReference{ + Name: ids.rootPresence.AsCompoundIdentifier(), + }, + Joins: []pgsql.Join{ + { + Table: expansionEdgeTableReference(first.Edge.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewLiteral(true, pgsql.Boolean), + }, + }, + { + Table: expansionNodeTableReference(first.LeftNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgd.EntityID(first.LeftNode.Identifier), pgsql.OperatorEquals, pgsql.CompoundIdentifier{first.Edge.Identifier, pgsql.ColumnStartID}, + ), + }, + }, + { + Table: expansionNodeTableReference(first.RightNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgd.EntityID(first.RightNode.Identifier), pgsql.OperatorEquals, pgsql.CompoundIdentifier{first.Edge.Identifier, pgsql.ColumnEndID}, + ), + }, + }, + }, + } + for _, step := range suffix[1:] { + from.Joins = append(from.Joins, + pgsql.Join{ + Table: expansionEdgeTableReference(step.Edge.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{step.Edge.Identifier, pgsql.ColumnStartID}, pgsql.OperatorEquals, pgd.EntityID(step.LeftNode.Identifier), + ), + }, + }, + pgsql.Join{ + Table: expansionNodeTableReference(step.RightNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgd.EntityID(step.RightNode.Identifier), pgsql.OperatorEquals, pgsql.CompoundIdentifier{step.Edge.Identifier, pgsql.ColumnEndID}, + ), + }, + }, + ) + } + + var boundaryConstraint pgsql.Expression + if expansionStep.Expansion != nil { + boundaryConstraint = expansionStep.Expansion.TerminalNodeConstraints + } + localBoundaryConstraint, _ := partitionConstraintByLocality(boundaryConstraint, localScope) + where := localBoundaryConstraint + suffixRelationships := make([]pgsql.Identifier, 0, len(suffix)) + for _, step := range suffix { + suffixRelationships = append(suffixRelationships, step.Edge.Identifier) + localLeftConstraint, _ := partitionConstraintByLocality(step.LeftNodeConstraints, localScope) + localEdgeConstraint, _ := partitionConstraintByLocality(step.EdgeConstraints.Expression, localScope) + localRightConstraint, _ := partitionConstraintByLocality(step.RightNodeConstraints, localScope) + where = pgsql.OptionalAnd(where, localLeftConstraint) + where = pgsql.OptionalAnd(where, localEdgeConstraint) + where = pgsql.OptionalAnd(where, localRightConstraint) + } + where = pgsql.OptionalAnd(where, pairwiseRelationshipIDUniqueness(suffixRelationships)) + + query := pgsql.Query{ + Body: pgsql.Select{ + Projection: projection, + From: []pgsql.FromClause{from}, + Where: where, + }, + } + if rowLimit > 0 { + query.Limit = pgsql.NewLiteral(rowLimit+1, pgsql.Int8) + } + + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: ids.suffix, + }, + Materialized: &pgsql.Materialized{ + Materialized: true, + }, + Query: query, + }, nil +} + +// buildSuffixSeededReverseCTE recursively walks from suffix boundaries back toward bound roots without reusing edges. +func buildSuffixSeededReverseCTE(expansionStep *TraversalStep, decision optimize.ExpansionSearchStrategyDecision, ids suffixSeededIdentifiers, gateSource, gateColumn pgsql.Identifier) (pgsql.CommonTableExpression, error) { + if expansionStep.Edge == nil || expansionStep.RightNode == nil { + return pgsql.CommonTableExpression{}, fmt.Errorf("forced suffix-seeded reverse expansion step is incomplete") + } + + emptyPath := pgsql.ArrayLiteral{ + CastType: pgsql.Int8Array, + } + seed := pgsql.Select{ + Projection: []pgsql.SelectItem{ + pgsql.CompoundIdentifier{ids.boundaries, fixedSuffixBoundaryID}, + pgsql.CompoundIdentifier{ids.boundaries, fixedSuffixBoundaryID}, + pgsql.NewLiteral(int64(0), pgsql.Int8), + emptyPath, + }, + From: []pgsql.FromClause{tableFrom(ids.boundaries)}, + } + if gateSource != "" && gateColumn != "" { + seed.From = append(seed.From, tableFrom(gateSource)) + seed.Where = pgsql.CompoundIdentifier{gateSource, gateColumn} + } + + path := pgsql.CompoundIdentifier{ids.reverse, expansionPath} + localEdgeConstraint, _ := partitionConstraintByLocality( + expansionStep.Expansion.EdgeConstraints, + pgsql.AsIdentifierSet(expansionStep.Edge.Identifier), + ) + recursiveWhere := pgsql.OptionalAnd( + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{ids.reverse, expansionDepth}, + pgsql.OperatorLessThan, + pgsql.NewLiteral(decision.MaximumDepth, pgsql.Int8), + ), + pgsql.NewBinaryExpression( + pgd.EntityID(expansionStep.Edge.Identifier), + pgsql.OperatorNotEquals, + pgsql.NewAllExpression(path), + ), + ) + recursiveWhere = pgsql.OptionalAnd(recursiveWhere, localEdgeConstraint) + + recursive := pgsql.Select{ + Projection: []pgsql.SelectItem{ + pgsql.CompoundIdentifier{ids.reverse, fixedSuffixBoundaryID}, + pgsql.CompoundIdentifier{expansionStep.Edge.Identifier, pgsql.ColumnStartID}, + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{ids.reverse, expansionDepth}, pgsql.OperatorAdd, pgsql.NewLiteral(int64(1), pgsql.Int8)), + pgsql.FunctionCall{ + Function: pgsql.Identifier("array_prepend"), + Parameters: []pgsql.Expression{ + pgd.EntityID(expansionStep.Edge.Identifier), path, + }, + CastType: pgsql.Int8Array, + }, + }, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{ + Name: ids.reverse.AsCompoundIdentifier(), + }, + Joins: []pgsql.Join{ + { + Table: expansionEdgeTableReference(expansionStep.Edge.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{expansionStep.Edge.Identifier, pgsql.ColumnEndID}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{ids.reverse, expansionNextID}, + ), + }, + }, + { + Table: expansionNodeTableReference(expansionStep.LeftNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgd.EntityID(expansionStep.LeftNode.Identifier), pgsql.OperatorEquals, pgsql.CompoundIdentifier{expansionStep.Edge.Identifier, pgsql.ColumnStartID}, + ), + }, + }, + }, + }}, + Where: recursiveWhere, + } + + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: ids.reverse, + Shape: pgsql.NewRecordShape([]pgsql.Identifier{ + fixedSuffixBoundaryID, expansionNextID, expansionDepth, expansionPath, + }), + }, + Query: pgsql.Query{ + Body: pgsql.SetOperation{ + Operator: pgsql.OperatorUnion, + All: true, + LOperand: seed, + ROperand: recursive, + }, + }, + }, nil +} + +// suffixSeededFinalProjection reconstructs the incumbent projection from root, reverse-state, and suffix columns. +func suffixSeededFinalProjection( + part *PatternPart, + expansionStep *TraversalStep, + suffix []*TraversalStep, + rootFrame pgsql.Identifier, + ids suffixSeededIdentifiers, + reverseStateSource pgsql.Identifier, + incumbent pgsql.Projection, + suffixOverrides map[pgsql.Identifier]pgsql.Expression, +) (pgsql.Projection, error) { + suffixBindings := map[pgsql.Identifier]struct{}{} + for _, step := range suffix { + suffixBindings[step.Edge.Identifier] = struct{}{} + suffixBindings[step.LeftNode.Identifier] = struct{}{} + suffixBindings[step.RightNode.Identifier] = struct{}{} + } + + projection := make(pgsql.Projection, 0, len(incumbent)) + for _, item := range incumbent { + alias, ok := selectItemAlias(item) + if !ok { + return nil, fmt.Errorf("forced suffix-seeded reverse final projection contains an unaliased item %T", item) + } + + var expression pgsql.Expression + switch { + case expansionStep.Expansion != nil && expansionStep.Expansion.PathBinding != nil && alias == expansionStep.Expansion.PathBinding.Identifier: + expression = pgsql.CompoundIdentifier{reverseStateSource, expansionPath} + case alias == expansionStep.LeftNode.Identifier: + expression = pgsql.CompoundIdentifier{rootFrame, alias} + case suffixOverrides[alias] != nil: + expression = suffixOverrides[alias] + default: + if _, found := suffixBindings[alias]; found { + expression = pgsql.CompoundIdentifier{ids.suffix, alias} + } else { + expression = pgsql.CompoundIdentifier{rootFrame, alias} + } + } + projection = append(projection, &pgsql.AliasedExpression{ + Expression: expression, + Alias: models.OptionalValue(alias), + }) + } + + return projection, nil +} + +// selectItemAlias returns an explicit alias or the identifier naturally exposed by a select item. +func selectItemAlias(item pgsql.SelectItem) (pgsql.Identifier, bool) { + switch typed := item.(type) { + case *pgsql.AliasedExpression: + return typed.Alias.Value, typed.Alias.Set + case pgsql.AliasedExpression: + return typed.Alias.Value, typed.Alias.Set + default: + return "", false + } +} + +// suffixSeededNodeValue returns a node's scalar ID or composite value according to its projection representation. +func suffixSeededNodeValue(binding *BoundIdentifier) pgsql.Expression { + if binding.IDOnly { + return pgd.EntityID(binding.Identifier) + } + return aggregateNodeComposite(binding.Identifier) +} + +// tableFrom wraps a relation name as a single PostgreSQL FROM clause. +func tableFrom(identifier pgsql.Identifier) pgsql.FromClause { + return pgsql.FromClause{ + Source: pgsql.TableReference{ + Name: identifier.AsCompoundIdentifier(), + }, + } +} diff --git a/cypher/models/pgsql/translate/expansion_test.go b/cypher/models/pgsql/translate/expansion_test.go index 9075d55e..89b9ffbf 100644 --- a/cypher/models/pgsql/translate/expansion_test.go +++ b/cypher/models/pgsql/translate/expansion_test.go @@ -4,6 +4,7 @@ import ( "strings" "testing" + "github.com/specterops/dawgs/cypher/models" "github.com/specterops/dawgs/cypher/models/pgsql" "github.com/specterops/dawgs/cypher/models/pgsql/format" "github.com/specterops/dawgs/cypher/models/pgsql/pgd" @@ -11,14 +12,38 @@ import ( ) const ( + // shortestPathSeedTestPreviousFrame identifies the frame that supplies bound endpoint values in seed tests. shortestPathSeedTestPreviousFrame pgsql.Identifier = "s0" - shortestPathSeedTestFrame pgsql.Identifier = "s1" - shortestPathSeedTestRoot pgsql.Identifier = "n0" - shortestPathSeedTestTerminal pgsql.Identifier = "n1" - shortestPathSeedTestOther pgsql.Identifier = "x" - shortestPathSeedTestEdge pgsql.Identifier = "e0" + + // shortestPathSeedTestFrame identifies the generated shortest-path frame in seed tests. + shortestPathSeedTestFrame pgsql.Identifier = "s1" + + // shortestPathSeedTestRoot identifies the root-node binding in seed tests. + shortestPathSeedTestRoot pgsql.Identifier = "n0" + + // shortestPathSeedTestTerminal identifies the terminal-node binding in seed tests. + shortestPathSeedTestTerminal pgsql.Identifier = "n1" + + // shortestPathSeedTestOther identifies an unrelated binding used to test locality rejection. + shortestPathSeedTestOther pgsql.Identifier = "x" + + // shortestPathSeedTestEdge identifies the relationship binding in seed tests. + shortestPathSeedTestEdge pgsql.Identifier = "e0" ) +// TestShortestDistanceColumnsCompactsOnlyIDOnlyState verifies that compact state omits root ID only when endpoint identity is already carried. +func TestShortestDistanceColumnsCompactsOnlyIDOnlyState(t *testing.T) { + require.Equal(t, + []pgsql.Identifier{expansionNextID, expansionDepth}, + shortestDistanceColumns(true).Columns, + ) + require.Equal(t, + []pgsql.Identifier{expansionRootID, expansionNextID, expansionDepth}, + shortestDistanceColumns(false).Columns, + ) +} + +// shortestPathSeedTestBoundColumn references a composite field from the fixture's preceding frame. func shortestPathSeedTestBoundColumn(nodeIdentifier pgsql.Identifier, column pgsql.Identifier) pgsql.RowColumnReference { return pgsql.RowColumnReference{ Identifier: pgsql.CompoundIdentifier{shortestPathSeedTestPreviousFrame, nodeIdentifier}, @@ -26,6 +51,7 @@ func shortestPathSeedTestBoundColumn(nodeIdentifier pgsql.Identifier, column pgs } } +// shortestPathSeedTestLocalFunctionPredicate builds a deterministic predicate that depends only on the selected node. func shortestPathSeedTestLocalFunctionPredicate(nodeIdentifier pgsql.Identifier, value string) pgsql.Expression { return pgsql.NewBinaryExpression( pgsql.FunctionCall{ @@ -40,6 +66,7 @@ func shortestPathSeedTestLocalFunctionPredicate(nodeIdentifier pgsql.Identifier, ) } +// shortestPathSeedTestExternalPredicate builds a predicate that deliberately depends on an unrelated binding. func shortestPathSeedTestExternalPredicate(nodeIdentifier pgsql.Identifier) pgsql.Expression { return pgsql.NewBinaryExpression( shortestPathSeedTestBoundColumn(nodeIdentifier, pgsql.ColumnID), @@ -48,6 +75,7 @@ func shortestPathSeedTestExternalPredicate(nodeIdentifier pgsql.Identifier) pgsq ) } +// newShortestPathSeedTestBuilder creates a shortest-path builder with deterministic fixture bindings and parameters. func newShortestPathSeedTestBuilder(leftBound, rightBound bool) (*ExpansionBuilder, *Expansion) { previousFrame := &Frame{ Binding: &BoundIdentifier{Identifier: shortestPathSeedTestPreviousFrame}, @@ -115,6 +143,26 @@ func TestShortestPathSelfEndpointGuardsUseCaseErrorHelper(t *testing.T) { require.NotContains(t, endpointPairFilterGuard, " / ") } +// TestForwardPrimerSkipsSelfEndpointGuardWhenZeroDepthIsAllowed verifies that a zero-length path may use the same root and terminal. +func TestForwardPrimerSkipsSelfEndpointGuardWhenZeroDepthIsAllowed(t *testing.T) { + builder, expansionModel := newShortestPathSeedTestBuilder(false, false) + expansionModel.UseMaterializedEndpointPairFilter = true + expansionModel.Options.MinDepth = models.OptionalValue[int64](0) + + query, _, err := builder.prepareForwardFrontPrimerQuery(expansionModel) + require.NoError(t, err) + formatted, err := format.SyntaxNode(query) + require.NoError(t, err) + require.NotContains(t, formatted, "shortest_path_self_endpoint_error") + + expansionModel.Options.MinDepth = models.OptionalValue[int64](1) + query, _, err = builder.prepareForwardFrontPrimerQuery(expansionModel) + require.NoError(t, err) + formatted, err = format.SyntaxNode(query) + require.NoError(t, err) + require.Contains(t, formatted, "shortest_path_self_endpoint_error") +} + func TestBoundRootShortestPathPrimerKeepsOnlySeedLocalConstraints(t *testing.T) { builder, expansionModel := newShortestPathSeedTestBuilder(true, false) expansionModel.PrimerNodeConstraints = pgsql.NewBinaryExpression( diff --git a/cypher/models/pgsql/translate/expression.go b/cypher/models/pgsql/translate/expression.go index 6123f8a5..bd1ae197 100644 --- a/cypher/models/pgsql/translate/expression.go +++ b/cypher/models/pgsql/translate/expression.go @@ -10,6 +10,7 @@ import ( "github.com/specterops/dawgs/cypher/models/walk" ) +// unwrapParenthetical removes every enclosing parenthetical expression and returns the innermost operand. func unwrapParenthetical(parenthetical pgsql.Expression) pgsql.Expression { next := parenthetical @@ -26,6 +27,7 @@ func unwrapParenthetical(parenthetical pgsql.Expression) pgsql.Expression { return parenthetical } +// expressionHasCompositeProperties reports whether a data type exposes an entity properties field. func expressionHasCompositeProperties(expressionType pgsql.DataType) bool { switch expressionType { case pgsql.NodeComposite, pgsql.EdgeComposite, pgsql.ExpansionRootNode, pgsql.ExpansionEdge, pgsql.ExpansionTerminalNode: @@ -36,10 +38,12 @@ func expressionHasCompositeProperties(expressionType pgsql.DataType) bool { } } +// isCompositePropertyLookupTarget reports whether a type-hinted expression exposes composite properties. func isCompositePropertyLookupTarget(expression pgsql.TypeHinted) bool { return expressionHasCompositeProperties(expression.TypeHint()) } +// translateCompositePropertyLookup pushes a lookup of the properties field from a composite expression. func (s *Translator) translateCompositePropertyLookup(target pgsql.Expression, lookup *cypher.PropertyLookup) error { if fieldIdentifierLiteral, err := pgsql.AsLiteral(lookup.Symbol); err != nil { return err @@ -54,7 +58,12 @@ func (s *Translator) translateCompositePropertyLookup(target pgsql.Expression, l } } +// translatePropertyLookup lowers a validated Cypher property access according to its translated atom type. func (s *Translator) translatePropertyLookup(lookup *cypher.PropertyLookup) error { + if err := cypher.ValidatePropertyKeyName(lookup.Symbol); err != nil { + return err + } + if translatedAtom, err := s.treeTranslator.PopOperand(); err != nil { return err } else { @@ -154,6 +163,7 @@ func (s *Translator) translatePropertyLookup(lookup *cypher.PropertyLookup) erro return nil } +// translateCypherAssignmentOperator maps supported Cypher assignment operators to their PostgreSQL AST equivalents. func translateCypherAssignmentOperator(operator cypher.AssignmentOperator) (pgsql.Operator, error) { switch operator { case cypher.OperatorAssignment: @@ -188,6 +198,7 @@ func ExtractSyntaxNodeReferences(root pgsql.SyntaxNode) (*pgsql.IdentifierSet, e )) } +// rewriteStringWildCardLiteral escapes LIKE metacharacters in a literal string operand. func rewriteStringWildCardLiteral(expression pgsql.Expression) (pgsql.Expression, error) { switch typedExpression := expression.(type) { case pgsql.Literal: @@ -207,6 +218,7 @@ func rewriteStringWildCardLiteral(expression pgsql.Expression) (pgsql.Expression } } +// rewritePropertyLookupOperator selects JSON text extraction, JSON extraction, and casts for the requested result type. func rewritePropertyLookupOperator(propertyLookup *pgsql.BinaryExpression, dataType pgsql.DataType) pgsql.Expression { if dataType.IsArrayType() { // Ensure that array conversions use JSONB @@ -238,6 +250,7 @@ func rewritePropertyLookupOperator(propertyLookup *pgsql.BinaryExpression, dataT } } +// isJSONScalarEqualityType reports whether a scalar can be normalized to JSONB for Cypher equality. func isJSONScalarEqualityType(dataType pgsql.DataType) bool { switch dataType { case pgsql.Boolean, pgsql.Float4, pgsql.Float8, pgsql.Int, pgsql.Int2, pgsql.Int4, pgsql.Int8, pgsql.Numeric: @@ -248,6 +261,7 @@ func isJSONScalarEqualityType(dataType pgsql.DataType) bool { } } +// rewriteJSONScalarEqualityOperand converts a non-null supported scalar to comparable JSONB. func rewriteJSONScalarEqualityOperand(expression pgsql.Expression) (pgsql.Expression, bool) { if literal, isLiteral := expression.(pgsql.Literal); isLiteral && literal.Null { return nil, false @@ -268,6 +282,7 @@ func rewriteJSONScalarEqualityOperand(expression pgsql.Expression) (pgsql.Expres } } +// rewriteStringEqualityOperand accepts a non-null text expression for string-specific equality handling. func rewriteStringEqualityOperand(expression pgsql.Expression) (pgsql.Expression, bool) { if literal, isLiteral := expression.(pgsql.Literal); isLiteral && literal.Null { return nil, false @@ -282,6 +297,7 @@ func rewriteStringEqualityOperand(expression pgsql.Expression) (pgsql.Expression return expression, true } +// lookupRequiresElementType reports whether an array comparison expects a property's element type rather than its array type. func lookupRequiresElementType(typeHint pgsql.DataType, operator pgsql.Operator, otherOperand pgsql.SyntaxNode) bool { if typeHint.IsArrayType() { switch operator { @@ -298,9 +314,10 @@ func lookupRequiresElementType(typeHint pgsql.DataType, operator pgsql.Operator, return false } +// TypeCastExpression applies a type hint, rewriting property comparisons when the operator requires element typing. func TypeCastExpression(expression pgsql.Expression, dataType pgsql.DataType) (pgsql.Expression, error) { if propertyLookup, isPropertyLookup := expressionToPropertyLookupBinaryExpression(expression); isPropertyLookup { - var lookupTypeHint = dataType + lookupTypeHint := dataType if lookupRequiresElementType(dataType, propertyLookup.Operator, propertyLookup.ROperand) { // Take the base type of the array type hint: in @@ -313,6 +330,24 @@ func TypeCastExpression(expression pgsql.Expression, dataType pgsql.DataType) (p return pgsql.NewTypeCast(expression, dataType), nil } +// jsonNullLiteral returns the JSONB representation of a JSON null value. +func jsonNullLiteral() pgsql.Expression { + return pgsql.NewTypeCast(pgsql.NewLiteral(pgsql.StringLiteralNull, pgsql.Text), pgsql.JSONB) +} + +// nullifyJSONPropertyLookup converts a JSON null property value to SQL NULL with NULLIF. +func nullifyJSONPropertyLookup(propertyLookup *pgsql.BinaryExpression) pgsql.Expression { + return pgsql.FunctionCall{ + Function: pgsql.FunctionNullIf, + Parameters: []pgsql.Expression{ + propertyLookup, + jsonNullLiteral(), + }, + CastType: pgsql.JSONB, + } +} + +// rewritePropertyLookupOperands assigns extraction operators and casts using the comparison's opposite operand. func rewritePropertyLookupOperands(kindMapper *contextAwareKindMapper, expression *pgsql.BinaryExpression) error { var ( leftPropertyLookup, hasLeftPropertyLookup = expressionToPropertyLookupBinaryExpression(expression.LOperand) @@ -325,6 +360,8 @@ func rewritePropertyLookupOperands(kindMapper *contextAwareKindMapper, expressio (pgsql.OperatorIsComparator(expression.Operator) || expression.Operator == pgsql.OperatorCypherNotEquals) { leftPropertyLookup.Operator = pgsql.OperatorJSONField rightPropertyLookup.Operator = pgsql.OperatorJSONField + expression.LOperand = nullifyJSONPropertyLookup(leftPropertyLookup) + expression.ROperand = nullifyJSONPropertyLookup(rightPropertyLookup) return nil } @@ -411,6 +448,7 @@ func rewritePropertyLookupOperands(kindMapper *contextAwareKindMapper, expressio return nil } +// newFunctionCallComparatorError returns a focused type-mismatch error for function comparisons with special Cypher semantics. func newFunctionCallComparatorError(functionCall pgsql.FunctionCall, operator pgsql.Operator, comparisonType pgsql.DataType) error { switch functionCall.Function { case pgsql.FunctionCoalesce: @@ -513,6 +551,7 @@ func NewExpressionTreeTranslator(kindMapper *contextAwareKindMapper) *Expression } } +// mergeUserAndTranslationConstraints combines user predicates with translator-added safety constraints. func mergeUserAndTranslationConstraints(userConstraints, translationConstraints *Constraint) *Constraint { if userConstraints.Expression != nil { // Fold the user constraints into the translation constraints wrapped in a parenthetical @@ -525,10 +564,14 @@ func mergeUserAndTranslationConstraints(userConstraints, translationConstraints return translationConstraints } +// HasAnyConstraints reports whether the supplied scope can evaluate any satisfiable user or translator constraint. func (s *ExpressionTreeTranslator) HasAnyConstraints(scope *pgsql.IdentifierSet) (bool, error) { - if hasUser, err := s.UserConstraints.HasConstraints(scope); err != nil || hasUser { - return hasUser, err + if hasUser, err := s.UserConstraints.HasConstraints(scope); err != nil { + return false, err + } else if hasUser { + return true, nil } + return s.TranslationConstraints.HasConstraints(scope) } @@ -577,6 +620,7 @@ func (s *ExpressionTreeTranslator) PopOperand() (pgsql.Expression, error) { return s.treeBuilder.PopOperand(s.kindMapper) } +// popOperandAsUserConstraint removes the next operand, normalizes bare property truth tests, and records its dependencies. func (s *ExpressionTreeTranslator) popOperandAsUserConstraint() error { if nextExpression, err := s.PopOperand(); err != nil { return err @@ -659,6 +703,7 @@ func (s *ExpressionTreeTranslator) PopBinaryExpression(operator pgsql.Operator) } } +// rewriteIdentityOperands replaces entity comparisons with comparisons of their scalar identity fields. func rewriteIdentityOperands(scope *Scope, newExpression *pgsql.BinaryExpression) error { switch typedLOperand := newExpression.LOperand.(type) { case pgsql.Identifier: @@ -750,13 +795,13 @@ func rewriteIdentityOperands(scope *Scope, newExpression *pgsql.BinaryExpression } } } - } } return nil } +// isPropertyLookup reports whether expression is a property-lookup binary expression, including wrapped forms. func isPropertyLookup(expression pgsql.Expression) bool { _, isPropertyLookup := expressionToPropertyLookupBinaryExpression(expression) return isPropertyLookup @@ -791,6 +836,7 @@ func isConcatenationOperation(lOperand, rOperand pgsql.Expression, lOperandType, return false } +// isEmptyArrayLiteralPropertyComparison finds a property lookup paired with an untyped empty array literal. func isEmptyArrayLiteralPropertyComparison(expression *pgsql.BinaryExpression) (*pgsql.BinaryExpression, bool) { var ( hasPropertyLookup bool @@ -819,11 +865,13 @@ func isEmptyArrayLiteralPropertyComparison(expression *pgsql.BinaryExpression) ( return propertyLookup, hasPropertyLookup && hasEmptyArrayLiteral } +// isEmptyAnyArrayLiteral reports whether expression is an empty array with no inferred element type. func isEmptyAnyArrayLiteral(expression pgsql.Expression) bool { arrayLiteral, isArrayLiteral := expression.(pgsql.ArrayLiteral) return isArrayLiteral && arrayLiteral.CastType == pgsql.AnyArray && len(arrayLiteral.Values) == 0 } +// isKnownEmptyArrayExpression reports whether expression is an untyped empty array or a parameter statically typed as NULL. func isKnownEmptyArrayExpression(expression pgsql.Expression) bool { if isEmptyAnyArrayLiteral(expression) { return true @@ -839,14 +887,12 @@ func isKnownEmptyArrayExpression(expression pgsql.Expression) bool { } } -func jsonNullLiteral() pgsql.Expression { - return pgsql.NewTypeCast(pgsql.NewLiteral(pgsql.StringLiteralNull, pgsql.Text), pgsql.JSONB) -} - +// jsonEmptyArrayLiteral returns the JSONB representation of an empty array. func jsonEmptyArrayLiteral() pgsql.Expression { return pgsql.NewTypeCast(pgsql.NewLiteral(pgsql.StringLiteralEmptyArray, pgsql.Text), pgsql.JSONB) } +// rewritePropertyLookupNullCheck preserves Cypher null semantics for missing keys and explicit JSON null values. func rewritePropertyLookupNullCheck(propertyLookup *pgsql.BinaryExpression, isNotNull bool) pgsql.Expression { propertyLookup.Operator = pgsql.OperatorJSONField @@ -878,14 +924,17 @@ func rewritePropertyLookupNullCheck(propertyLookup *pgsql.BinaryExpression, isNo )) } +// jsonFieldPropertyLookup copies a property lookup using JSONB field extraction. func jsonFieldPropertyLookup(propertyLookup *pgsql.BinaryExpression) *pgsql.BinaryExpression { return pgsql.NewBinaryExpression(propertyLookup.LOperand, pgsql.OperatorJSONField, propertyLookup.ROperand) } +// jsonTextPropertyLookup copies a property lookup using text field extraction. func jsonTextPropertyLookup(propertyLookup *pgsql.BinaryExpression) *pgsql.BinaryExpression { return pgsql.NewBinaryExpression(propertyLookup.LOperand, pgsql.OperatorJSONTextField, propertyLookup.ROperand) } +// jsonbTypeof returns a call that inspects an expression's JSONB value type. func jsonbTypeof(expression pgsql.Expression) pgsql.Expression { return pgsql.FunctionCall{ Function: pgsql.FunctionJSONBTypeof, @@ -893,6 +942,7 @@ func jsonbTypeof(expression pgsql.Expression) pgsql.Expression { } } +// jsonbStringTypeCheck reports at SQL runtime whether a property contains a JSON string. func jsonbStringTypeCheck(propertyLookup *pgsql.BinaryExpression) pgsql.Expression { return pgsql.NewBinaryExpression( jsonbTypeof(jsonFieldPropertyLookup(propertyLookup)), @@ -901,6 +951,7 @@ func jsonbStringTypeCheck(propertyLookup *pgsql.BinaryExpression) pgsql.Expressi ) } +// toJSONBTextOperand converts expression through text to a JSONB scalar for type-safe comparison. func toJSONBTextOperand(expression pgsql.Expression) pgsql.Expression { return pgsql.FunctionCall{ Function: pgsql.FunctionToJSONB, @@ -911,6 +962,7 @@ func toJSONBTextOperand(expression pgsql.Expression) pgsql.Expression { } } +// buildStringPropertyEqualityComparison compares a property's text extraction with a text operand in the original operand order. func buildStringPropertyEqualityComparison(propertyLookup *pgsql.BinaryExpression, textOperand pgsql.Expression, propertyOnLeft bool, operator pgsql.Operator) pgsql.Expression { textPropertyLookup := jsonTextPropertyLookup(propertyLookup) @@ -921,6 +973,7 @@ func buildStringPropertyEqualityComparison(propertyLookup *pgsql.BinaryExpressio return pgsql.NewBinaryExpression(textOperand, operator, textPropertyLookup) } +// buildStringPropertyEqualityPredicate recognizes string/property equality and builds its type-aware predicate. func buildStringPropertyEqualityPredicate(expression *pgsql.BinaryExpression) (pgsql.Expression, bool) { if !expression.Operator.IsIn(pgsql.OperatorEquals, pgsql.OperatorCypherNotEquals) { return nil, false @@ -946,6 +999,7 @@ func buildStringPropertyEqualityPredicate(expression *pgsql.BinaryExpression) (p return nil, false } +// buildStringPropertyComparisonPredicate guards text comparison by JSON type while preserving inequality for non-string values. func buildStringPropertyComparisonPredicate(propertyLookup *pgsql.BinaryExpression, textOperand pgsql.Expression, propertyOnLeft bool, operator pgsql.Operator) pgsql.Expression { stringComparison := buildStringPropertyEqualityComparison(propertyLookup, textOperand, propertyOnLeft, operator) @@ -981,6 +1035,7 @@ func buildStringPropertyComparisonPredicate(propertyLookup *pgsql.BinaryExpressi )) } +// buildEmptyArrayPropertyComparison compares a property with [] while retaining null taint and optional negation. func buildEmptyArrayPropertyComparison(propertyLookup *pgsql.BinaryExpression, negated bool) *pgsql.BinaryExpression { var ( emptyArrayExpression = pgsql.NewBinaryExpression( @@ -1027,6 +1082,7 @@ func buildEmptyArrayPropertyComparison(propertyLookup *pgsql.BinaryExpression, n ) } +// cypherStringPredicateTextOperand converts a predicate operand to text while retaining null propagation. func cypherStringPredicateTextOperand(operand pgsql.Expression) (pgsql.Expression, error) { if propertyLookup, isPropertyLookup := expressionToPropertyLookupBinaryExpression(operand); isPropertyLookup { propertyLookup.Operator = pgsql.OperatorJSONTextField @@ -1040,6 +1096,7 @@ func cypherStringPredicateTextOperand(operand pgsql.Expression) (pgsql.Expressio return pgsql.NewTypeCast(operand, pgsql.Text), nil } +// cypherStringPredicateFunction maps a Cypher string predicate operator to its PostgreSQL helper function. func cypherStringPredicateFunction(function pgsql.Identifier, lOperand, rOperand pgsql.Expression) (pgsql.Expression, error) { leftText, err := cypherStringPredicateTextOperand(lOperand) if err != nil { @@ -1061,6 +1118,7 @@ func cypherStringPredicateFunction(function pgsql.Identifier, lOperand, rOperand }, nil } +// rewriteBinaryExpression applies operator-specific casts, wildcard escaping, and Cypher null semantics before pushing the result. func (s *ExpressionTreeTranslator) rewriteBinaryExpression(newExpression *pgsql.BinaryExpression) error { switch newExpression.Operator { case pgsql.OperatorAdd: diff --git a/cypher/models/pgsql/translate/expression_test.go b/cypher/models/pgsql/translate/expression_test.go index 9c9df618..393b94af 100644 --- a/cypher/models/pgsql/translate/expression_test.go +++ b/cypher/models/pgsql/translate/expression_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/require" ) +// mustAsLiteral converts value to a PostgreSQL literal and panics when the value type is unsupported. func mustAsLiteral(value any) pgsql.Literal { if literal, err := pgsql.AsLiteral(value); err != nil { panic(fmt.Sprintf("%v", err)) @@ -225,6 +226,7 @@ func TestInferUnaryExpressionType(t *testing.T) { } } +// TestInferWrappedExpressionType verifies that wrappers preserve or derive the data type of their enclosed expressions. func TestInferWrappedExpressionType(t *testing.T) { testCases := []struct { Name string @@ -284,6 +286,17 @@ func TestInferWrappedExpressionType(t *testing.T) { Name: "all expression over scalar", ExpectedType: pgsql.UnknownDataType, Expression: pgsql.NewAllExpression(mustAsLiteral(int64(1))), + }, { + Name: "case expression ignores null branch during inference", + ExpectedType: pgsql.Int, + Expression: pgsql.Case{ + Conditions: []pgsql.Expression{mustAsLiteral(true)}, + Then: []pgsql.Expression{pgsql.FunctionCall{ + Function: pgsql.FunctionJSONBArrayLength, + CastType: pgsql.Int, + }}, + Else: pgsql.NullLiteral(), + }, }} for _, nextCase := range testCases { @@ -296,6 +309,7 @@ func TestInferWrappedExpressionType(t *testing.T) { } } +// TestPropertyLookupEqualityScalarRewrites verifies scalar equality operators receive type-aware property extraction. func TestPropertyLookupEqualityScalarRewrites(t *testing.T) { var ( propertyLookup = func(property string) *pgsql.BinaryExpression { @@ -390,7 +404,13 @@ func TestPropertyLookupEqualityScalarRewrites(t *testing.T) { LOperand: propertyLookup("left"), Operator: pgsql.OperatorEquals, ROperand: propertyLookup("right"), - Expected: "(n.properties -> 'left') = (n.properties -> 'right')", + Expected: "nullif((n.properties -> 'left'), ('null')::jsonb)::jsonb = nullif((n.properties -> 'right'), ('null')::jsonb)::jsonb", + }, { + Name: "property ordering treats JSON null as SQL null", + LOperand: propertyLookup("left"), + Operator: pgsql.OperatorLessThan, + ROperand: propertyLookup("right"), + Expected: "nullif((n.properties -> 'left'), ('null')::jsonb)::jsonb < nullif((n.properties -> 'right'), ('null')::jsonb)::jsonb", }} ) @@ -500,6 +520,7 @@ func TestExpressionTreeTranslator(t *testing.T) { validateConstraints(t, treeTranslator, idents, expectedTranslation) } +// validateConstraints requires the generated constraint collection to contain exactly the expected SQL expressions. func validateConstraints(t *testing.T, constraintTracker *translate.ExpressionTreeTranslator, idents *pgsql.IdentifierSet, expectedTranslation string) { constraint, err := constraintTracker.ConsumeConstraintsFromVisibleSet(idents) diff --git a/cypher/models/pgsql/translate/format.go b/cypher/models/pgsql/translate/format.go index 1bd0e696..3d0cfdda 100644 --- a/cypher/models/pgsql/translate/format.go +++ b/cypher/models/pgsql/translate/format.go @@ -3,6 +3,7 @@ package translate import ( "bytes" "context" + "strings" "github.com/specterops/dawgs/cypher/models/cypher" cypherFormat "github.com/specterops/dawgs/cypher/models/cypher/format" @@ -10,27 +11,51 @@ import ( "github.com/specterops/dawgs/cypher/models/pgsql/format" ) +// Translated renders a translation result as PostgreSQL for its target graph. func Translated(translation Result) (string, error) { - return format.Statement(translation.Statement, format.NewOutputBuilder()) + return format.Statement(translation.Statement, format.NewOutputBuilder().WithTargetGraph(translation.GraphID)) } +// newlineToCommentReplacer prefixes every PostgreSQL line-comment continuation after \r, \n, or both, per the scanner source: +// https://github.com/postgres/postgres/blob/824d5f6241ea7a0a85c9d2b3d27beb78e42a36ab/src/backend/parser/scan.l#L186-L211 +var newlineToCommentReplacer = strings.NewReplacer( + "\r\n", "\n-- ", + "\r", "\n-- ", + "\n", "\n-- ", +) + +// FromCypher renders a Cypher query as a SQL comment followed by its PostgreSQL translation. func FromCypher(ctx context.Context, regularQuery *cypher.RegularQuery, kindMapper pgsql.KindMapper, stripLiterals bool, graphID int32) (format.Formatted, error) { var ( output = &bytes.Buffer{} emitter = cypherFormat.NewCypherEmitter(stripLiterals) ) - output.WriteString("-- ") + // 1. write cypher to output if err := emitter.Write(regularQuery, output); err != nil { return format.Formatted{}, err } + // 2. save copy of cypher and reset output for commented cypher + + raw := strings.TrimSpace(output.String()) + output.Reset() + + // 3. write commented cypher + + output.WriteString("-- ") // opening comment + if _, err := newlineToCommentReplacer.WriteString(output, raw); err != nil { + return format.Formatted{}, err + } + + // 4. continue with SQL + output.WriteString("\n") if translation, err := Translate(ctx, regularQuery, kindMapper, nil, graphID); err != nil { return format.Formatted{}, err - } else if sqlQuery, err := format.Statement(translation.Statement, format.NewOutputBuilder()); err != nil { + } else if sqlQuery, err := format.Statement(translation.Statement, format.NewOutputBuilder().WithTargetGraph(translation.GraphID)); err != nil { return format.Formatted{}, err } else { output.WriteString(sqlQuery) diff --git a/cypher/models/pgsql/translate/format_test.go b/cypher/models/pgsql/translate/format_test.go new file mode 100644 index 00000000..f163fc60 --- /dev/null +++ b/cypher/models/pgsql/translate/format_test.go @@ -0,0 +1,42 @@ +package translate + +import ( + "context" + "strings" + "testing" + + "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/drivers/pg/pgutil" + "github.com/stretchr/testify/require" +) + +// TestFromCypherProperlyEscapesDebugComment verifies that every source-query line remains inside the emitted PostgreSQL comment. +func TestFromCypherProperlyEscapesDebugComment(t *testing.T) { + t.Parallel() + + kindMapper := pgutil.NewInMemoryKindMapper() + + query, err := frontend.ParseCypher( + frontend.NewContext(), + "MATCH (n) WHERE n.`begin\nfail1\rfail2\r\nfail3\n\rfail4` = 1 RETURN n", + ) + require.NoError(t, err) + + formatted, err := FromCypher(context.Background(), query, kindMapper, false, DefaultGraphID) + require.NoError(t, err) + + IsPGNewline := func(r rune) bool { + return r == '\n' || r == '\r' + } + for line := range strings.FieldsFuncSeq(formatted.Statement, IsPGNewline) { + if strings.HasPrefix(line, "with s0") { + break + } + is_commented := strings.HasPrefix(strings.TrimSpace(line), "--") + require.True(t, is_commented, "cypher line '%v' does not start with '--'", line) + if is_commented { + continue + } + require.NotContains(t, line, "fail") + } +} diff --git a/cypher/models/pgsql/translate/function.go b/cypher/models/pgsql/translate/function.go index 319ac897..1f8290a5 100644 --- a/cypher/models/pgsql/translate/function.go +++ b/cypher/models/pgsql/translate/function.go @@ -11,6 +11,7 @@ import ( "github.com/specterops/dawgs/cypher/models/pgsql/optimize" ) +// legacyToIntegerFunction identifies the legacy spelling normalized to Cypher's toInteger function. const legacyToIntegerFunction = "toint" func SymbolsFor(node pgsql.SyntaxNode) (*pgsql.SymbolTable, error) { @@ -27,6 +28,7 @@ func SymbolsFor(node pgsql.SyntaxNode) (*pgsql.SymbolTable, error) { })) } +// asFunctionCall unwraps parentheses and returns a PostgreSQL function call when expression contains one. func asFunctionCall(node pgsql.SyntaxNode) (pgsql.FunctionCall, bool) { switch typedNode := node.(type) { case pgsql.FunctionCall: @@ -110,6 +112,7 @@ func ContainsAggregateFunction(node pgsql.SyntaxNode) (bool, error) { })) } +// appendIfReferencedGroupByExpression appends expression to GROUP BY only when it contains a binding reference. func appendIfReferencedGroupByExpression(groupByExpressions []pgsql.Expression, expression pgsql.Expression) ([]pgsql.Expression, error) { if references, err := ExtractSyntaxNodeReferences(expression); err != nil { return nil, err @@ -120,6 +123,7 @@ func appendIfReferencedGroupByExpression(groupByExpressions []pgsql.Expression, } } +// appendNonAggregateGroupByExpressions adds non-aggregate projection expressions required by PostgreSQL grouping rules. func appendNonAggregateGroupByExpressions(groupByExpressions []pgsql.Expression, expressions ...pgsql.Expression) ([]pgsql.Expression, error) { for _, expression := range expressions { nextGroupByExpressions, err := NonAggregateGroupByExpressions(expression) @@ -272,6 +276,7 @@ func NonAggregateGroupByExpressions(expression pgsql.Expression) ([]pgsql.Expres } } +// bindingExpressionType returns the effective data type of a bound identifier expression. func bindingExpressionType(binding *BoundIdentifier) pgsql.DataType { switch binding.DataType { case pgsql.ExpansionEdge: @@ -288,6 +293,7 @@ func bindingExpressionType(binding *BoundIdentifier) pgsql.DataType { } } +// inferRowColumnReferenceType infers a composite field's data type from the referenced binding and column. func inferRowColumnReferenceType(expression pgsql.RowColumnReference) pgsql.DataType { switch expression.Column { case pgsql.ColumnGraphID, pgsql.ColumnID, pgsql.ColumnStartID, pgsql.ColumnEndID: @@ -313,6 +319,7 @@ func inferRowColumnReferenceType(expression pgsql.RowColumnReference) pgsql.Data } } +// inferExpressionType resolves an expression's SQL data type using scope information when needed. func (s *Translator) inferExpressionType(expression pgsql.Expression) (pgsql.DataType, error) { switch typedExpression := unwrapParenthetical(expression).(type) { case pgsql.Identifier: @@ -339,6 +346,7 @@ func (s *Translator) inferExpressionType(expression pgsql.Expression) (pgsql.Dat return InferExpressionType(expression) } +// inferArrayExpressionType resolves an array expression's element-derived PostgreSQL array type. func (s *Translator) inferArrayExpressionType(expression pgsql.Expression) (pgsql.DataType, error) { if expressionType, err := s.inferExpressionType(expression); err != nil { return pgsql.UnsetDataType, err @@ -351,6 +359,7 @@ func (s *Translator) inferArrayExpressionType(expression pgsql.Expression) (pgsq } } +// expressionForPath returns the path representation carried by binding or reports that it cannot satisfy the requested use. func (s *Translator) expressionForPath(expression pgsql.Expression) (pgsql.Expression, error) { switch typedExpression := unwrapParenthetical(expression).(type) { case pgsql.Identifier: @@ -378,6 +387,7 @@ func (s *Translator) expressionForPath(expression pgsql.Expression) (pgsql.Expre } } +// translateHeadFunction lowers head(list) to safe PostgreSQL array indexing. func (s *Translator) translateHeadFunction(functionInvocation *cypher.FunctionInvocation) error { if functionInvocation.NumArguments() != 1 { return fmt.Errorf("expected only one argument for cypher function: %s", functionInvocation.Name) @@ -400,6 +410,7 @@ func (s *Translator) translateHeadFunction(functionInvocation *cypher.FunctionIn return nil } +// translateTailFunction lowers tail(list) to a PostgreSQL array slice that excludes the first element. func (s *Translator) translateTailFunction(functionInvocation *cypher.FunctionInvocation) error { if functionInvocation.NumArguments() != 1 { return fmt.Errorf("expected only one argument for cypher function: %s", functionInvocation.Name) @@ -429,6 +440,7 @@ func (s *Translator) translateTailFunction(functionInvocation *cypher.FunctionIn return nil } +// cypherMinMaxFunction selects the Cypher-aware minimum or maximum SQL aggregate for the invocation name. func cypherMinMaxFunction(function pgsql.Identifier, argument pgsql.Expression) pgsql.FunctionCall { if propertyLookup, isPropertyLookup := expressionToPropertyLookupBinaryExpression(argument); isPropertyLookup { propertyLookup.Operator = pgsql.OperatorJSONField @@ -456,6 +468,7 @@ func cypherMinMaxFunction(function pgsql.Identifier, argument pgsql.Expression) } } +// translatePathComponentFunction lowers nodes(path) or relationships(path) from the binding's carried path representation. func (s *Translator) translatePathComponentFunction(functionInvocation *cypher.FunctionInvocation, column pgsql.Identifier, castType pgsql.DataType) error { if functionInvocation.NumArguments() != 1 { return fmt.Errorf("expected only one argument for cypher function: %s", functionInvocation.Name) @@ -498,6 +511,86 @@ func (s *Translator) translatePathComponentFunction(functionInvocation *cypher.F return nil } +// translatePathLengthFunction lowers length(path) to the cardinality of carried ordered edge IDs when possible. +func (s *Translator) translatePathLengthFunction(functionInvocation *cypher.FunctionInvocation) error { + if functionInvocation.NumArguments() != 1 { + return fmt.Errorf("expected only one argument for cypher function: %s", functionInvocation.Name) + } + + argument, err := s.treeTranslator.PopOperand() + if err != nil { + return err + } + + if literal, isLiteral := argument.(pgsql.Literal); isLiteral && literal.Null { + s.treeTranslator.PushOperand(pgsql.NewTypeCast(literal, pgsql.Int)) + return nil + } + + if identifier, isIdentifier := unwrapParenthetical(argument).(pgsql.Identifier); isIdentifier { + binding, bound := s.scope.Lookup(identifier) + if !bound { + binding, bound = s.scope.AliasedLookup(identifier) + } + if !bound { + return fmt.Errorf("unable to resolve path identifier %s", identifier) + } + if binding.DistanceOnly { + var distance pgsql.Expression = binding.Identifier + if binding.LastProjection != nil { + distance = pgsql.CompoundIdentifier{binding.LastProjection.Binding.Identifier, binding.Identifier} + } else { + for _, dependency := range binding.Dependencies { + if dependency.DistanceOnly && dependency.LastProjection != nil { + distance = pgsql.CompoundIdentifier{dependency.LastProjection.Binding.Identifier, dependency.Identifier} + break + } + } + } + s.treeTranslator.PushOperand(pgsql.NewTypeCast(distance, pgsql.Int)) + return nil + } + if binding.DataType != pgsql.PathComposite { + return fmt.Errorf("expected path expression but received %s", binding.DataType) + } + + var edges pgsql.Expression + if binding.LastProjection == nil { + edges, err = pathCompositeEdgeIDArrayExpression(s.scope, binding) + } else { + edges = pgsql.RowColumnReference{ + Identifier: pgsql.CompoundIdentifier{binding.LastProjection.Binding.Identifier, binding.Identifier}, + Column: pgsql.ColumnEdges, + } + } + if err != nil { + return err + } + + s.treeTranslator.PushOperand(pgsql.FunctionCall{ + Function: pgsql.FunctionCardinality, + Parameters: []pgsql.Expression{edges}, + CastType: pgsql.Int, + }) + return nil + } + + pathExpression, err := s.expressionForPath(argument) + if err != nil { + return err + } + s.treeTranslator.PushOperand(pgsql.FunctionCall{ + Function: pgsql.FunctionCardinality, + Parameters: []pgsql.Expression{pgsql.RowColumnReference{ + Identifier: pathExpression, + Column: pgsql.ColumnEdges, + }}, + CastType: pgsql.Int, + }) + return nil +} + +// prepareCollectExpression prepares a value and result type for PostgreSQL array aggregation. func prepareCollectExpression(scope *Scope, collectedExpression pgsql.Expression, functionName string) (pgsql.Expression, pgsql.DataType, error) { castType := pgsql.AnyArray @@ -529,6 +622,7 @@ func prepareCollectExpression(scope *Scope, collectedExpression pgsql.Expression return collectedExpression, castType, nil } +// prepareCollectIDExpression extracts a scalar entity ID before collection and records the ID-only alias. func prepareCollectIDExpression(scope *Scope, collectedExpression pgsql.Expression) (pgsql.Expression, bool) { identifier, isIdentifier := unwrapParenthetical(collectedExpression).(pgsql.Identifier) if !isIdentifier { @@ -551,6 +645,7 @@ func prepareCollectIDExpression(scope *Scope, collectedExpression pgsql.Expressi } } +// translateNodeLabelsExpression lowers labels(node) to kind-name lookup over the node's kind IDs. func translateNodeLabelsExpression(identifier pgsql.Identifier) pgsql.TypeHinted { const ( kindAlias pgsql.Identifier = "_kind" @@ -602,6 +697,7 @@ func translateNodeLabelsExpression(identifier pgsql.Identifier) pgsql.TypeHinted }, pgsql.TextArray) } +// relationshipEndpointFunctionArgument expands a bound edge identifier to the composite accepted by startNode or endNode. func (s *Translator) relationshipEndpointFunctionArgument(argument pgsql.Expression) pgsql.Expression { identifier, isIdentifier := unwrapParenthetical(argument).(pgsql.Identifier) if !isIdentifier { @@ -619,6 +715,7 @@ func (s *Translator) relationshipEndpointFunctionArgument(argument pgsql.Express return argument } +// translateRelationshipEndpointFunction lowers startNode or endNode with graph-scoped entity hydration. func (s *Translator) translateRelationshipEndpointFunction(function pgsql.Identifier, functionInvocation *cypher.FunctionInvocation) error { if functionInvocation.NumArguments() != 1 { return fmt.Errorf("expected only one argument for cypher function: %s", functionInvocation.Name) @@ -637,6 +734,7 @@ func (s *Translator) translateRelationshipEndpointFunction(function pgsql.Identi return nil } +// translateFunction dispatches a Cypher function invocation to its function-specific PostgreSQL lowering. func (s *Translator) translateFunction(typedExpression *cypher.FunctionInvocation) { switch formattedName := strings.ToLower(typedExpression.Name); formattedName { case cypher.DurationFunction: @@ -662,6 +760,8 @@ func (s *Translator) translateFunction(typedExpression *cypher.FunctionInvocatio s.SetError(err) } else if referenceArgument, typeOK := argument.(pgsql.Identifier); !typeOK { s.SetErrorf("expected an identifier for the cypher function: %s but received %T", typedExpression.Name, argument) + } else if binding, bound := s.scope.Lookup(referenceArgument); bound && binding.IDOnly && binding.LastProjection != nil { + s.treeTranslator.PushOperand(referenceArgument) } else { s.treeTranslator.PushOperand(pgsql.CompoundIdentifier{referenceArgument, pgsql.ColumnID}) } @@ -790,29 +890,37 @@ func (s *Translator) translateFunction(typedExpression *cypher.FunctionInvocatio } else if argument, err := s.treeTranslator.PopOperand(); err != nil { s.SetError(err) } else { - var functionCall pgsql.FunctionCall + var sizeExpression pgsql.Expression if propertyLookup, isPropertyLookup := expressionToPropertyLookupBinaryExpression(argument); isPropertyLookup { // Ensure that the JSONB array length function receives the JSONB type propertyLookup.Operator = pgsql.OperatorJSONField - functionCall = pgsql.FunctionCall{ - Function: pgsql.FunctionJSONBArrayLength, - Parameters: []pgsql.Expression{argument}, - CastType: pgsql.Int, + sizeExpression = pgsql.Case{ + Conditions: []pgsql.Expression{pgsql.NewBinaryExpression( + jsonbTypeof(argument), + pgsql.OperatorEquals, + pgsql.NewLiteral("array", pgsql.Text), + )}, + Then: []pgsql.Expression{pgsql.FunctionCall{ + Function: pgsql.FunctionJSONBArrayLength, + Parameters: []pgsql.Expression{argument}, + CastType: pgsql.Int, + }}, + Else: pgsql.NullLiteral(), } } else if isKnownEmptyArrayExpression(argument) { s.treeTranslator.PushOperand(pgsql.NewLiteral(0, pgsql.Int)) return } else { - functionCall = pgsql.FunctionCall{ + sizeExpression = pgsql.FunctionCall{ Function: pgsql.FunctionCardinality, Parameters: []pgsql.Expression{argument}, CastType: pgsql.Int, } } - s.treeTranslator.PushOperand(functionCall) + s.treeTranslator.PushOperand(sizeExpression) } case cypher.HeadFunction: @@ -835,6 +943,11 @@ func (s *Translator) translateFunction(typedExpression *cypher.FunctionInvocatio s.SetError(err) } + case cypher.PathLengthFunction: + if err := s.translatePathLengthFunction(typedExpression); err != nil { + s.SetError(err) + } + case cypher.ToUpperFunction: if typedExpression.NumArguments() != 1 { s.SetError(fmt.Errorf("expected only one argument for cypher function: %s", typedExpression.Name)) @@ -1001,6 +1114,7 @@ func functionWrapCollectToArray(distinct bool, collectedExpression pgsql.Express } } +// translateDateTimeFunctionCall lowers Cypher temporal constructors and validates supported argument forms. func (s *Translator) translateDateTimeFunctionCall(cypherFunc *cypher.FunctionInvocation, dataType pgsql.DataType) error { // Ensure the local date time function uses the default precision const defaultTimestampPrecision = 6 @@ -1068,6 +1182,7 @@ func (s *Translator) translateDateTimeFunctionCall(cypherFunc *cypher.FunctionIn return nil } +// translateCoalesceFunction lowers coalesce after reconciling every argument to one compatible result type. func (s *Translator) translateCoalesceFunction(functionInvocation *cypher.FunctionInvocation) error { if numArgs := functionInvocation.NumArguments(); numArgs == 0 { s.SetError(fmt.Errorf("expected at least one argument for cypher function: %s", functionInvocation.Name)) diff --git a/cypher/models/pgsql/translate/function_test.go b/cypher/models/pgsql/translate/function_test.go index 066f0cf0..cfbe3eea 100644 --- a/cypher/models/pgsql/translate/function_test.go +++ b/cypher/models/pgsql/translate/function_test.go @@ -9,6 +9,7 @@ import ( "github.com/specterops/dawgs/cypher/models/cypher" "github.com/specterops/dawgs/cypher/models/pgsql" "github.com/specterops/dawgs/drivers/pg/pgutil" + "github.com/specterops/dawgs/graph" "github.com/stretchr/testify/require" ) @@ -57,6 +58,25 @@ func TestPathComponentFunctionsTranslateNullArguments(t *testing.T) { require.Contains(t, formatted, "(null)::edgecomposite[]") } +// TestListSizeGuardsDynamicJSONPropertiesByType verifies that size() distinguishes JSON strings and arrays at runtime. +func TestListSizeGuardsDynamicJSONPropertiesByType(t *testing.T) { + kindMapper := pgutil.NewInMemoryKindMapper() + kindMapper.Put(graph.StringKind("TestNode")) + + query, err := frontend.ParseCypher(frontend.NewContext(), `MATCH (n:TestNode) RETURN size(n.values)`) + require.NoError(t, err) + + translation, err := Translate(context.Background(), query, kindMapper, nil, DefaultGraphID) + require.NoError(t, err) + + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "case when jsonb_typeof") + require.Contains(t, formatted, "= 'array' then jsonb_array_length") + require.Contains(t, formatted, "else null end") +} + +// TestTailFunctionDoesNotDuplicatePathComponentExpression verifies nested tail calls hydrate path components only once. func TestTailFunctionDoesNotDuplicatePathComponentExpression(t *testing.T) { kindMapper := pgutil.NewInMemoryKindMapper() @@ -68,10 +88,12 @@ func TestTailFunctionDoesNotDuplicatePathComponentExpression(t *testing.T) { formatted, err := Translated(translation) require.NoError(t, err) - require.Equal(t, 1, strings.Count(formatted, "ordered_edges_to_path"), formatted) + require.Equal(t, 1, strings.Count(formatted, "ordered_edge_ids_to_path"), formatted) + require.NotContains(t, formatted, "ordered_edges_to_path") require.NotContains(t, formatted, "cardinality(((case when") } +// TestTailPredicateStagesPathComponentExpression verifies predicates reuse a staged path-component projection. func TestTailPredicateStagesPathComponentExpression(t *testing.T) { kindMapper := pgutil.NewInMemoryKindMapper() @@ -83,11 +105,13 @@ func TestTailPredicateStagesPathComponentExpression(t *testing.T) { formatted, err := Translated(translation) require.NoError(t, err) - require.Equal(t, 1, strings.Count(formatted, "ordered_edges_to_path")) + require.Equal(t, 1, strings.Count(formatted, "ordered_edge_ids_to_path")) + require.NotContains(t, formatted, "ordered_edges_to_path") require.Contains(t, formatted, "lateral (select") require.Contains(t, formatted, ".nodes") } +// TestProjectionStagesPathBeforeReadingComponents verifies path hydration is staged before node and edge access. func TestProjectionStagesPathBeforeReadingComponents(t *testing.T) { kindMapper := pgutil.NewInMemoryKindMapper() @@ -100,11 +124,13 @@ func TestProjectionStagesPathBeforeReadingComponents(t *testing.T) { formatted, err := Translated(translation) require.NoError(t, err) require.Contains(t, formatted, "lateral (select") - require.Equal(t, 1, strings.Count(formatted, "ordered_edges_to_path"), formatted) + require.Equal(t, 1, strings.Count(formatted, "ordered_edge_ids_to_path"), formatted) + require.NotContains(t, formatted, "ordered_edges_to_path") require.Contains(t, formatted, ".nodes") require.Contains(t, formatted, ".edges") } +// TestProjectionStagesRepeatedPathComponents verifies repeated component access shares one staged path hydration. func TestProjectionStagesRepeatedPathComponents(t *testing.T) { kindMapper := pgutil.NewInMemoryKindMapper() @@ -117,12 +143,238 @@ func TestProjectionStagesRepeatedPathComponents(t *testing.T) { formatted, err := Translated(translation) require.NoError(t, err) require.Contains(t, formatted, "lateral (select") - require.Equal(t, 1, strings.Count(formatted, "ordered_edges_to_path"), formatted) - require.Equal(t, 1, strings.Count(formatted, "from unnest"), formatted) + require.Equal(t, 1, strings.Count(formatted, "ordered_edge_ids_to_path"), formatted) + require.NotContains(t, formatted, "ordered_edges_to_path") + require.NotContains(t, formatted, "from unnest") require.Contains(t, formatted, ".nodes") require.Contains(t, formatted, ".edges") } +// TestPathLengthUsesOrderedEdgeIDsWithoutHydration verifies that length(path) counts carried edge IDs without hydrating a path. +func TestPathLengthUsesOrderedEdgeIDsWithoutHydration(t *testing.T) { + kindMapper := pgutil.NewInMemoryKindMapper() + + query, err := frontend.ParseCypher(frontend.NewContext(), `MATCH p = shortestPath((s)-[*1..]->(e)) WHERE id(s) = 1 AND id(e) = 2 RETURN length(p)`) + require.NoError(t, err) + + translation, err := Translate(context.Background(), query, kindMapper, nil, DefaultGraphID) + require.NoError(t, err) + + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "cardinality(s0.ep0)") + require.NotContains(t, formatted, "ordered_edge_ids_to_path") + require.NotContains(t, formatted, "ordered_edges_to_path") + require.NotContains(t, formatted, "from unnest") +} + +// TestIDOnlyTerminalProjectionCarriesScalarID verifies that an ID-only terminal consumer receives scalar state. +func TestIDOnlyTerminalProjectionCarriesScalarID(t *testing.T) { + kindMapper := pgutil.NewInMemoryKindMapper() + kindMapper.Put(graph.StringKind("TestNode")) + + query, err := frontend.ParseCypher(frontend.NewContext(), `MATCH ()-[]->(e:TestNode) RETURN id(e)`) + require.NoError(t, err) + + translation, err := Translate(context.Background(), query, kindMapper, nil, DefaultGraphID) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "n1.id as n1") + require.Contains(t, formatted, "select s0.n1 as \"id(e)\"") + require.NotContains(t, formatted, "(n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1") + require.Contains(t, formatted, "n1.kind_ids operator") +} + +// TestIDOnlyTerminalProjectionRetainsCompositeForMixedUse verifies that mixed ID and property consumers retain the terminal composite. +func TestIDOnlyTerminalProjectionRetainsCompositeForMixedUse(t *testing.T) { + kindMapper := pgutil.NewInMemoryKindMapper() + + query, err := frontend.ParseCypher(frontend.NewContext(), `MATCH ()-[]->(e) RETURN id(e), e.name`) + require.NoError(t, err) + + translation, err := Translate(context.Background(), query, kindMapper, nil, DefaultGraphID) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "(n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1") + require.Contains(t, formatted, "(s0.n1).id") + require.Contains(t, formatted, "(s0.n1).properties") +} + +// TestIDOnlyTerminalProjectionRetainsCompositeForLaterPatternReuse verifies that a reused terminal remains a complete entity binding. +func TestIDOnlyTerminalProjectionRetainsCompositeForLaterPatternReuse(t *testing.T) { + kindMapper := pgutil.NewInMemoryKindMapper() + + query, err := frontend.ParseCypher(frontend.NewContext(), `MATCH ()-[]->(e) MATCH (e)-[]->() RETURN id(e)`) + require.NoError(t, err) + + translation, err := Translate(context.Background(), query, kindMapper, nil, DefaultGraphID) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "(n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1") + require.NotContains(t, formatted, "n1.id as n1") +} + +// TestIDOnlyTerminalProjectionRetainsCompositeForObservedPath verifies that observing the path retains complete terminal state. +func TestIDOnlyTerminalProjectionRetainsCompositeForObservedPath(t *testing.T) { + kindMapper := pgutil.NewInMemoryKindMapper() + + query, err := frontend.ParseCypher(frontend.NewContext(), `MATCH p = ()-[*1..]->(e) WHERE id(e) = 2 RETURN p`) + require.NoError(t, err) + + translation, err := Translate(context.Background(), query, kindMapper, nil, DefaultGraphID) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "(n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1") + require.Contains(t, formatted, "ordered_edge_ids_to_path") +} + +// TestIDOnlyExpansionContinuationCarriesScalarID verifies that an ID-only intermediate binding continues as scalar state. +func TestIDOnlyExpansionContinuationCarriesScalarID(t *testing.T) { + kindMapper := pgutil.NewInMemoryKindMapper() + + query, err := frontend.ParseCypher(frontend.NewContext(), `MATCH (s)-[*1..]->(mid)-[]->(e) RETURN id(mid), id(e)`) + require.NoError(t, err) + + translation, err := Translate(context.Background(), query, kindMapper, nil, DefaultGraphID) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "join lateral (select n1.id from node n1 where n1.id = s1.next_id offset 0) n1 on true") + require.Contains(t, formatted, "s0.n1 = e1.start_id") + require.Contains(t, formatted, "s0.n1 as n1") + require.NotContains(t, formatted, "(n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1") + require.NotContains(t, formatted, "(s0.n1).id = e1.start_id") +} + +// TestIDOnlyExpansionContinuationRetainsCompositeForPropertyUse verifies that a property consumer prevents scalar-only continuation state. +func TestIDOnlyExpansionContinuationRetainsCompositeForPropertyUse(t *testing.T) { + kindMapper := pgutil.NewInMemoryKindMapper() + + query, err := frontend.ParseCypher(frontend.NewContext(), `MATCH (s)-[*1..]->(mid)-[]->(e) RETURN mid.name`) + require.NoError(t, err) + + translation, err := Translate(context.Background(), query, kindMapper, nil, DefaultGraphID) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "(n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1") + require.Contains(t, formatted, "(s0.n1).id = e1.start_id") +} + +// TestIDOnlyExpansionContinuationSeedsFollowingExpansionFromScalarID verifies that a following traversal can join from a scalar intermediate ID. +func TestIDOnlyExpansionContinuationSeedsFollowingExpansionFromScalarID(t *testing.T) { + kindMapper := pgutil.NewInMemoryKindMapper() + + query, err := frontend.ParseCypher(frontend.NewContext(), `MATCH (s)-[*1..]->(mid)-[*1..]->(e) RETURN id(mid), id(e)`) + require.NoError(t, err) + + translation, err := Translate(context.Background(), query, kindMapper, nil, DefaultGraphID) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "select distinct s0.n1 as root_id from s0") + require.Contains(t, formatted, "s0.n1 = s3.root_id") + require.NotContains(t, formatted, "select distinct (s0.n1).id as root_id from s0") +} + +// TestIDOnlyExpansionContinuationRetainsCompositeForObservedPath verifies that observing the path prevents scalar-only intermediate state. +func TestIDOnlyExpansionContinuationRetainsCompositeForObservedPath(t *testing.T) { + kindMapper := pgutil.NewInMemoryKindMapper() + + query, err := frontend.ParseCypher(frontend.NewContext(), `MATCH p = (s)-[*1..]->(mid)-[]->(e) RETURN p`) + require.NoError(t, err) + + translation, err := Translate(context.Background(), query, kindMapper, nil, DefaultGraphID) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "(n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1") + require.Contains(t, formatted, "(s0.n1).id = e1.start_id") + require.Contains(t, formatted, "ordered_edge_ids_to_path") +} + +// TestIDOnlyExpansionContinuationRetainsCompositeForMutation verifies that mutating an intermediate node retains its complete entity value. +func TestIDOnlyExpansionContinuationRetainsCompositeForMutation(t *testing.T) { + kindMapper := pgutil.NewInMemoryKindMapper() + + query, err := frontend.ParseCypher(frontend.NewContext(), `MATCH (s)-[*1..]->(mid)-[]->(e) DELETE mid`) + require.NoError(t, err) + + translation, err := Translate(context.Background(), query, kindMapper, nil, DefaultGraphID) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "(n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1") + require.Contains(t, formatted, "(s0.n1).id = e1.start_id") + require.Contains(t, formatted, "delete from node") +} + +// TestBoundPairShortestPathUsesStableSingletonArrays verifies deterministic singleton endpoint-array construction for bound shortest paths. +func TestBoundPairShortestPathUsesStableSingletonArrays(t *testing.T) { + kindMapper := pgutil.NewInMemoryKindMapper() + translateQuery := func(cypherQuery string) (Result, string) { + query, err := frontend.ParseCypher(frontend.NewContext(), cypherQuery) + require.NoError(t, err) + translation, err := Translate(context.Background(), query, kindMapper, nil, DefaultGraphID) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + return translation, formatted + } + + first, firstSQL := translateQuery(`MATCH p = shortestPath((s)-[*1..]->(e)) WHERE id(s) = 1 AND id(e) = 2 RETURN p LIMIT 1`) + second, secondSQL := translateQuery(`MATCH p = shortestPath((s)-[*1..]->(e)) WHERE id(s) = 41 AND id(e) = 42 RETURN p LIMIT 1`) + + require.Equal(t, firstSQL, secondSQL) + require.Contains(t, firstSQL, "::int8[]") + require.NotContains(t, firstSQL, "insert into pg_temp.bsp_pair_filter") + require.NotContains(t, firstSQL, "traversal_pair_filter") + require.Contains(t, firstSQL, "limit 1") + require.Contains(t, firstSQL, "with singleton_endpoints as") + require.Contains(t, firstSQL, "array [singleton_endpoints.root_id]::int8[]") + require.Contains(t, firstSQL, "array [singleton_endpoints.terminal_id]::int8[]") + require.NotContains(t, firstSQL, "n0.id = 1") + require.NotContains(t, secondSQL, "n0.id = 41") + var firstEndpointValues, secondEndpointValues []any + for _, value := range first.Parameters { + if _, isString := value.(string); !isString { + firstEndpointValues = append(firstEndpointValues, value) + } + } + for _, value := range second.Parameters { + if _, isString := value.(string); !isString { + secondEndpointValues = append(secondEndpointValues, value) + } + } + require.ElementsMatch(t, []any{int64(1), int64(2)}, firstEndpointValues) + require.ElementsMatch(t, []any{int64(41), int64(42)}, secondEndpointValues) + + var hasRootArraySeed, hasTerminalArraySeed bool + for _, value := range first.Parameters { + fragment, isString := value.(string) + if !isString { + continue + } + hasRootArraySeed = hasRootArraySeed || strings.Contains(fragment, "unnest($1::int8[])") + hasTerminalArraySeed = hasTerminalArraySeed || strings.Contains(fragment, "unnest($2::int8[])") + } + require.True(t, hasRootArraySeed) + require.True(t, hasTerminalArraySeed) +} + func TestRelationshipEndpointFunctionsUseEdgeCompositeArguments(t *testing.T) { t.Parallel() diff --git a/cypher/models/pgsql/translate/graph_scope_test.go b/cypher/models/pgsql/translate/graph_scope_test.go new file mode 100644 index 00000000..97fe5c45 --- /dev/null +++ b/cypher/models/pgsql/translate/graph_scope_test.go @@ -0,0 +1,62 @@ +package translate + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/specterops/dawgs/cypher/frontend" + "github.com/stretchr/testify/require" +) + +// TestTargetGraphUsesConcreteRelationsInOuterAndHarnessSQL verifies graph partitioning in both the outer query and shortest-path harness. +func TestTargetGraphUsesConcreteRelationsInOuterAndHarnessSQL(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s:Group)-[:MemberOf*1..]->(e:Domain)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + LIMIT 1 + `) + require.NoError(t, err) + + translation, err := Translate(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), + "end_id": int64(2), + }, 42) + require.NoError(t, err) + + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "node_42") + require.Contains(t, formatted, "ordered_edge_ids_to_path(42,") + require.NotRegexp(t, `(?i)(from|join) (node|edge)(?:\s|;)`, formatted) + + var fragments []string + for _, value := range translation.Parameters { + if fragment, ok := value.(string); ok && strings.Contains(fragment, "pg_temp.bsp_") { + fragments = append(fragments, fragment) + } + } + require.NotEmpty(t, fragments) + for _, fragment := range fragments { + require.Contains(t, fragment, "edge_42", fmt.Sprintf("unscoped harness fragment: %s", fragment)) + require.NotRegexp(t, `(?i)(from|join) (node|edge)(?:\s|;)`, fragment) + } +} + +// TestFixedSuffixTargetGraphUsesOnlyConcreteRelations verifies that suffix-seeded translation never falls back to unpartitioned graph tables. +func TestFixedSuffixTargetGraphUsesOnlyConcreteRelations(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), optimizerFixedSuffixQuery) + require.NoError(t, err) + + translation, err := Translate(context.Background(), regularQuery, optimizerSafetyKindMapper(), nil, 42) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "node_42") + require.Contains(t, formatted, "edge_42") + require.NotRegexp(t, `(?i)(from|join) (node|edge)(?:\s|;)`, formatted) + require.Equal(t, 2, strings.Count(formatted, "ordered_edge_ids_to_path(42,")) +} diff --git a/cypher/models/pgsql/translate/hinting.go b/cypher/models/pgsql/translate/hinting.go index 5d837c4e..afd0944b 100644 --- a/cypher/models/pgsql/translate/hinting.go +++ b/cypher/models/pgsql/translate/hinting.go @@ -18,6 +18,7 @@ func GetTypeHint(expression pgsql.Expression) (pgsql.DataType, bool) { return pgsql.UnsetDataType, false } +// applyUnaryExpressionTypeHints casts a unary operand to the type required by its operator. func applyUnaryExpressionTypeHints(expression *pgsql.UnaryExpression) error { if propertyLookup, isPropertyLookup := expressionToPropertyLookupBinaryExpression(expression.Operand); isPropertyLookup { expression.Operand = rewritePropertyLookupOperator(propertyLookup, pgsql.Boolean) @@ -26,6 +27,7 @@ func applyUnaryExpressionTypeHints(expression *pgsql.UnaryExpression) error { return nil } +// inferBinaryExpressionType returns the result type implied by a binary operator and its operand hints. func inferBinaryExpressionType(expression *pgsql.BinaryExpression) (pgsql.DataType, error) { var ( leftHint, isLeftHinted = GetTypeHint(expression.LOperand) @@ -93,6 +95,7 @@ func inferBinaryExpressionType(expression *pgsql.BinaryExpression) (pgsql.DataTy } } +// inferUnaryExpressionType returns the result type implied by a unary operator and operand hint. func inferUnaryExpressionType(expression pgsql.UnaryExpression) (pgsql.DataType, error) { switch expression.Operator { case pgsql.OperatorNot, pgsql.OperatorIs, pgsql.OperatorIsNot: @@ -112,6 +115,7 @@ func inferUnaryExpressionType(expression pgsql.UnaryExpression) (pgsql.DataType, } } +// inferAllExpressionType returns the boolean type of a valid ALL predicate after checking its operands. func inferAllExpressionType(expression pgsql.AllExpression) (pgsql.DataType, error) { if expressionType, err := InferExpressionType(expression.Expression); err != nil { return pgsql.UnsetDataType, err @@ -122,6 +126,46 @@ func inferAllExpressionType(expression pgsql.AllExpression) (pgsql.DataType, err } } +// inferCaseExpressionType finds the common result type of a CASE expression's branches. +func inferCaseExpressionType(expression pgsql.Case) (pgsql.DataType, error) { + var ( + resultType = pgsql.UnknownDataType + branches = append(append([]pgsql.Expression(nil), expression.Then...), expression.Else) + ) + + for _, branch := range branches { + if branch == nil { + continue + } + + branchType, err := InferExpressionType(branch) + if err != nil { + return pgsql.UnsetDataType, err + } + if branchType == pgsql.Null || !branchType.IsKnown() { + continue + } + + if !resultType.IsKnown() { + resultType = branchType + continue + } + + if resultType == branchType { + continue + } + + if supertype, valid := resultType.CoerceToSupertype(branchType); valid { + resultType = supertype + } else { + return pgsql.UnknownDataType, nil + } + } + + return resultType, nil +} + +// InferExpressionType derives the PostgreSQL data type produced by an expression when it can be determined statically. func InferExpressionType(expression pgsql.Expression) (pgsql.DataType, error) { switch typedExpression := expression.(type) { case pgsql.Identifier, pgsql.RowColumnReference: @@ -193,6 +237,16 @@ func InferExpressionType(expression pgsql.Expression) (pgsql.DataType, error) { case pgsql.AllExpression: return inferAllExpressionType(typedExpression) + case *pgsql.Case: + if typedExpression == nil { + return pgsql.UnknownDataType, nil + } + + return inferCaseExpressionType(*typedExpression) + + case pgsql.Case: + return inferCaseExpressionType(typedExpression) + case *pgsql.AliasedExpression: if typedExpression == nil { return pgsql.UnknownDataType, nil @@ -218,12 +272,17 @@ func InferExpressionType(expression pgsql.Expression) (pgsql.DataType, error) { } } +// contextAwareKindMapper adapts request-scoped kind resolution to translation helpers that do not accept a context. type contextAwareKindMapper struct { - ctx context.Context + // ctx carries cancellation and deadlines into kind-name lookups. + ctx context.Context + // kindMapper performs the underlying graph kind-name resolution. kindMapper pgsql.KindMapper + // parameters is the translation parameter map shared with the owning translator. parameters map[string]any } +// newContextAwareKindMapper wraps a mapper with request context and retains the associated translation parameter map. func newContextAwareKindMapper(ctx context.Context, kindMapper pgsql.KindMapper, parameters map[string]any) *contextAwareKindMapper { return &contextAwareKindMapper{ ctx: ctx, @@ -240,6 +299,7 @@ func (s *contextAwareKindMapper) AssertKinds(kinds graph.Kinds) ([]int16, error) return s.kindMapper.AssertKinds(s.ctx, kinds) } +// relationshipTypeKindIDExpression returns the scalar kind-ID field used to implement type(relationship). func relationshipTypeKindIDExpression(expression pgsql.Expression) (pgsql.Expression, bool) { functionCall, isFunctionCall := unwrapParenthetical(expression).(pgsql.FunctionCall) if !isFunctionCall || functionCall.Function != pgsql.FunctionKindName || len(functionCall.Parameters) != 1 { @@ -249,6 +309,7 @@ func relationshipTypeKindIDExpression(expression pgsql.Expression) (pgsql.Expres return functionCall.Parameters[0], true } +// literalKindID resolves a string kind name through the context-bound mapper and returns its numeric PostgreSQL literal. func literalKindID(kindMapper *contextAwareKindMapper, literal pgsql.Literal) (pgsql.Literal, bool, error) { if literal.CastType != pgsql.Text { return pgsql.Literal{}, false, nil @@ -270,10 +331,12 @@ func literalKindID(kindMapper *contextAwareKindMapper, literal pgsql.Literal) (p return pgsql.NewLiteral(kindIDs[0], pgsql.Int2), true, nil } +// mapsRelationshipTypeLiteralToKindID reports whether operator permits mapping a relationship type name to its kind ID. func mapsRelationshipTypeLiteralToKindID(operator pgsql.Operator) bool { return operator.IsIn(pgsql.OperatorEquals, pgsql.OperatorNotEquals, pgsql.OperatorCypherNotEquals) } +// applyTypeFunctionLikeTypeHints normalizes operands for equality involving type(relationship). func applyTypeFunctionLikeTypeHints(kindMapper *contextAwareKindMapper, expression *pgsql.BinaryExpression) error { mapTypeLiteralToKindID := mapsRelationshipTypeLiteralToKindID(expression.Operator) @@ -431,6 +494,7 @@ func applyTypeFunctionLikeTypeHints(kindMapper *contextAwareKindMapper, expressi return nil } +// applyBinaryExpressionTypeHints rewrites property lookups and casts operands to types compatible with the binary operator. func applyBinaryExpressionTypeHints(kindMapper *contextAwareKindMapper, expression *pgsql.BinaryExpression) error { switch expression.Operator { case pgsql.OperatorPropertyLookup: diff --git a/cypher/models/pgsql/translate/limit_pushdown_test.go b/cypher/models/pgsql/translate/limit_pushdown_test.go index e7edcb5f..bc87c832 100644 --- a/cypher/models/pgsql/translate/limit_pushdown_test.go +++ b/cypher/models/pgsql/translate/limit_pushdown_test.go @@ -9,13 +9,23 @@ import ( ) const ( - limitPushdownTestSourceFrame pgsql.Identifier = "s0" - limitPushdownTestHarnessFrame pgsql.Identifier = "s1" + // limitPushdownTestSourceFrame identifies the source frame referenced by limit-pushdown fixtures. + limitPushdownTestSourceFrame pgsql.Identifier = "s0" + + // limitPushdownTestHarnessFrame identifies the shortest-path harness frame in limit-pushdown fixtures. + limitPushdownTestHarnessFrame pgsql.Identifier = "s1" + + // limitPushdownTestPreviousFrame identifies the frame that supplies bound endpoints in fixtures. limitPushdownTestPreviousFrame pgsql.Identifier = "s2" - limitPushdownTestRootAlias pgsql.Identifier = "n0" + + // limitPushdownTestRootAlias identifies the root-node binding in limit-pushdown fixtures. + limitPushdownTestRootAlias pgsql.Identifier = "n0" + + // limitPushdownTestTerminalAlias identifies the terminal-node binding in limit-pushdown fixtures. limitPushdownTestTerminalAlias pgsql.Identifier = "n1" ) +// limitPushdownTestEndpointRef references an endpoint ID projected by the fixture source frame. func limitPushdownTestEndpointRef(alias pgsql.Identifier) pgsql.RowColumnReference { return pgsql.RowColumnReference{ Identifier: pgsql.CompoundIdentifier{limitPushdownTestSourceFrame, alias}, @@ -23,6 +33,7 @@ func limitPushdownTestEndpointRef(alias pgsql.Identifier) pgsql.RowColumnReferen } } +// limitPushdownTestEndpointInequality builds the Cypher inequality used to exclude identical endpoints. func limitPushdownTestEndpointInequality(leftAlias, rightAlias pgsql.Identifier) pgsql.Expression { return pgsql.NewBinaryExpression( limitPushdownTestEndpointRef(leftAlias), @@ -31,6 +42,7 @@ func limitPushdownTestEndpointInequality(leftAlias, rightAlias pgsql.Identifier) ) } +// limitPushdownTestBoundEndpointConstraint equates a previous-frame endpoint ID with a harness expansion column. func limitPushdownTestBoundEndpointConstraint(endpointAlias, expansionColumn pgsql.Identifier) pgsql.Expression { return pgsql.NewBinaryExpression( pgsql.RowColumnReference{ @@ -42,6 +54,7 @@ func limitPushdownTestBoundEndpointConstraint(endpointAlias, expansionColumn pgs ) } +// limitPushdownTestSourceWhere combines the fixture's root, terminal, and endpoint-pair constraints. func limitPushdownTestSourceWhere(t *testing.T, part *QueryPart, where pgsql.Expression) { t.Helper() @@ -55,6 +68,7 @@ func limitPushdownTestSourceWhere(t *testing.T, part *QueryPart, where pgsql.Exp sourceCTE.Query.Body = selectBody } +// limitPushdownTestJoin joins one bound endpoint from the previous frame to the shortest-path harness. func limitPushdownTestJoin(nodeAlias, expansionColumn pgsql.Identifier) pgsql.Join { return pgsql.Join{ Table: pgsql.TableReference{ @@ -72,6 +86,7 @@ func limitPushdownTestJoin(nodeAlias, expansionColumn pgsql.Identifier) pgsql.Jo } } +// limitPushdownTestPart constructs a query part containing a bounded shortest-path harness and final projection. func limitPushdownTestPart(harnessFunction pgsql.Identifier) *QueryPart { part := NewQueryPart(1, 0) part.Limit = pgsql.NewLiteral(10, pgsql.Int) @@ -100,6 +115,7 @@ func limitPushdownTestPart(harnessFunction pgsql.Identifier) *QueryPart { return part } +// limitPushdownTestTail returns the terminal query part used to determine whether a limit may be pushed down. func limitPushdownTestTail(where pgsql.Expression) pgsql.Select { return pgsql.Select{ From: []pgsql.FromClause{{ @@ -228,6 +244,7 @@ func TestLimitPushdownTailSourceAllowsBidirectionalShortestPathEndpointInequalit require.Equal(t, limitPushdownTestSourceFrame, sourceFrame) } +// TestPushDownShortestPathLimitAppendsHarnessLimitWithEndpointInequality verifies endpoint filtering does not displace the harness limit. func TestPushDownShortestPathLimitAppendsHarnessLimitWithEndpointInequality(t *testing.T) { var ( part = limitPushdownTestPart(pgsql.FunctionUnidirectionalSPHarness) @@ -245,6 +262,7 @@ func TestPushDownShortestPathLimitAppendsHarnessLimitWithEndpointInequality(t *t require.Len(t, sourceCTE.Query.CommonTableExpressions.Expressions, 1) harnessCTE := sourceCTE.Query.CommonTableExpressions.Expressions[0] + require.Equal(t, part.Limit, harnessCTE.Query.Limit) selectBody, isSelect := harnessCTE.Query.Body.(pgsql.Select) require.True(t, isSelect) require.Len(t, selectBody.From, 1) diff --git a/cypher/models/pgsql/translate/model.go b/cypher/models/pgsql/translate/model.go index b29e8c14..15b1c23b 100644 --- a/cypher/models/pgsql/translate/model.go +++ b/cypher/models/pgsql/translate/model.go @@ -12,17 +12,35 @@ import ( ) const ( - expansionRootID pgsql.Identifier = "root_id" - expansionNextID pgsql.Identifier = "next_id" - expansionDepth pgsql.Identifier = "depth" - expansionSatisfied pgsql.Identifier = "satisfied" - expansionIsCycle pgsql.Identifier = "is_cycle" - expansionPath pgsql.Identifier = "path" - expansionForwardFront pgsql.Identifier = "forward_front" + // expansionRootID names the recursive-state column containing the traversal's initial node ID. + expansionRootID pgsql.Identifier = "root_id" + + // expansionNextID names the recursive-state column containing the current frontier node ID. + expansionNextID pgsql.Identifier = "next_id" + + // expansionDepth names the recursive-state column containing the number of traversed edges. + expansionDepth pgsql.Identifier = "depth" + + // expansionSatisfied names the recursive-state column that marks a satisfied terminal predicate. + expansionSatisfied pgsql.Identifier = "satisfied" + + // expansionIsCycle names the recursive-state column that marks an edge-reusing path. + expansionIsCycle pgsql.Identifier = "is_cycle" + + // expansionPath names the recursive-state column containing ordered traversed edge IDs. + expansionPath pgsql.Identifier = "path" + + // expansionForwardFront names the current forward frontier in bidirectional search. + expansionForwardFront pgsql.Identifier = "forward_front" + + // expansionBackwardFront names the current backward frontier in bidirectional search. expansionBackwardFront pgsql.Identifier = "backward_front" - expansionNextFront pgsql.Identifier = "next_front" + + // expansionNextFront names the staging relation for the next bidirectional-search frontier. + expansionNextFront pgsql.Identifier = "next_front" ) +// expansionColumns returns the canonical root, frontier, depth, satisfaction, cycle, and path state shape. func expansionColumns() *pgsql.RecordShape { return pgsql.NewRecordShape([]pgsql.Identifier{ expansionRootID, @@ -48,6 +66,7 @@ type ExpansionOptions struct { MaxDepth models.Optional[int64] } +// newExpansionOptions derives shortest-path and depth options from a pattern part and relationship range. func newExpansionOptions(part *PatternPart, relationshipPattern *cypher.RelationshipPattern) ExpansionOptions { return ExpansionOptions{ FindShortestPath: part.ShortestPath, @@ -57,40 +76,92 @@ func newExpansionOptions(part *PatternPart, relationshipPattern *cypher.Relation } } +// Expansion contains the bindings, constraints, and execution choices for one variable-length traversal. type Expansion struct { - Frame *Frame + // Frame is the scope frame that materializes the expansion result. + Frame *Frame + // PathBinding is the optional Cypher path variable backed by recursive path state. PathBinding *BoundIdentifier - Options ExpansionOptions - - PrimerNodeConstraints pgsql.Expression - PrimerNodeSatisfactionProjection pgsql.SelectItem - PrimerNodeJoinCondition pgsql.Expression - EdgeConstraints pgsql.Expression - EdgeJoinCondition pgsql.Expression - RecursiveConstraints pgsql.Expression - ExpansionNodeJoinCondition pgsql.Expression - TerminalNodeConstraints pgsql.Expression + // Options records shortest-path mode and traversal depth bounds. + Options ExpansionOptions + + // PrimerNodeConstraints restricts root nodes used to seed recursive traversal. + PrimerNodeConstraints pgsql.Expression + // PrimerNodeSatisfactionProjection evaluates terminal satisfaction at the seed node. + PrimerNodeSatisfactionProjection pgsql.SelectItem + // PrimerNodeJoinCondition joins the expansion seed to its root node. + PrimerNodeJoinCondition pgsql.Expression + // EdgeConstraints restricts relationships admitted into the expansion. + EdgeConstraints pgsql.Expression + // EdgeJoinCondition joins a relationship to the current traversal frontier. + EdgeJoinCondition pgsql.Expression + // RecursiveConstraints restricts recursive states independently of edge and node predicates. + RecursiveConstraints pgsql.Expression + // ExpansionNodeJoinCondition joins the traversed relationship to its next node. + ExpansionNodeJoinCondition pgsql.Expression + // TerminalNodeConstraints restricts nodes considered valid expansion terminals. + TerminalNodeConstraints pgsql.Expression + // TerminalNodeSatisfactionProjection computes whether a recursive state satisfies terminal predicates. TerminalNodeSatisfactionProjection pgsql.SelectItem + // DeferredNodeSatisfactionConstraint retains terminal predicates that require outer bindings. DeferredNodeSatisfactionConstraint pgsql.Expression - UseMaterializedTerminalFilter bool - UseMaterializedEndpointPairFilter bool - HasExplicitEndpointInequality bool - - PrimerQueryParameter *BoundIdentifier - BackwardPrimerQueryParameter *BoundIdentifier - RecursiveQueryParameter *BoundIdentifier + // UseMaterializedTerminalFilter enables lookup against precomputed terminal node IDs. + UseMaterializedTerminalFilter bool + // UseMaterializedEndpointPairFilter enables lookup against precomputed root-terminal ID pairs. + UseMaterializedEndpointPairFilter bool + // HasExplicitEndpointInequality reports whether the source query already excludes identical endpoints. + HasExplicitEndpointInequality bool + + // PrimerQueryParameter identifies the harness parameter containing the forward primer query. + PrimerQueryParameter *BoundIdentifier + // BackwardPrimerQueryParameter identifies the harness parameter containing the backward primer query. + BackwardPrimerQueryParameter *BoundIdentifier + // RecursiveQueryParameter identifies the harness parameter containing the forward recursive query. + RecursiveQueryParameter *BoundIdentifier + // BackwardRecursiveQueryParameter identifies the harness parameter containing the backward recursive query. BackwardRecursiveQueryParameter *BoundIdentifier + // UseBidirectionalSearch reports whether shortest-path traversal expands from both endpoints. UseBidirectionalSearch bool - + // ShortestPathExecutor selects the physical implementation for this shortest-path expansion. + ShortestPathExecutor optimize.ShortestPathExecutor + // ShortestPathTarget locates this expansion in the optimizer's lowering plan. + ShortestPathTarget optimize.TraversalStepTarget + // ShortestPathStateLimit caps distinct seen state for compact executors. + ShortestPathStateLimit int64 + // ShortestPathFrontierLimit caps current and queued frontier state. + ShortestPathFrontierLimit int64 + // ShortestPathPredecessorLimit caps retained witness predecessors. + ShortestPathPredecessorLimit int64 + // ShortestPathEnumerationLimit caps staged all-shortest-path arrays. + ShortestPathEnumerationLimit int64 + // ShortestPathOutputBytesLimit caps staged all-shortest-path array bytes. + ShortestPathOutputBytesLimit int64 + // SingletonRootID holds the statically resolved root ID when exactly one root is known. + SingletonRootID pgsql.Expression + // SingletonTerminalID holds the statically resolved terminal ID when exactly one terminal is known. + SingletonTerminalID pgsql.Expression + // RelationshipKindIDs contains the statically resolved relationship kinds admitted by the expansion. + RelationshipKindIDs []int16 + + // EdgeStartIdentifier is the unqualified edge endpoint column from which the chosen direction advances. EdgeStartIdentifier pgsql.Identifier - EdgeStartColumn pgsql.CompoundIdentifier - EdgeEndIdentifier pgsql.Identifier - EdgeEndColumn pgsql.CompoundIdentifier - + // EdgeStartColumn is the qualified edge endpoint expression from which the chosen direction advances. + EdgeStartColumn pgsql.CompoundIdentifier + // EdgeEndIdentifier is the unqualified edge endpoint column reached by the chosen direction. + EdgeEndIdentifier pgsql.Identifier + // EdgeEndColumn is the qualified edge endpoint expression reached by the chosen direction. + EdgeEndColumn pgsql.CompoundIdentifier + + // Projection contains the select items exposed by the completed expansion frame. Projection []pgsql.SelectItem } +// UsesSingletonEndpointPair reports whether both expansion endpoints are statically singleton IDs. +func (s *Expansion) UsesSingletonEndpointPair() bool { + return s != nil && s.SingletonRootID != nil && s.SingletonTerminalID != nil +} + func NewExpansionModel(part *PatternPart, relationshipPattern *cypher.RelationshipPattern) *Expansion { return &Expansion{ Options: newExpansionOptions(part, relationshipPattern), @@ -137,18 +208,22 @@ func (s *TraversalStep) CanExecuteBidirectionalSearch() bool { (s.LeftNodeBound && s.RightNodeBound && s.Frame != nil && s.Frame.Previous != nil) } +// hasPreviousFrameBinding reports whether the step can reference bindings materialized by a prior frame. func (s *TraversalStep) hasPreviousFrameBinding() bool { return s.Frame != nil && s.Frame.Previous != nil } +// usesBoundEndpointPairs reports whether both endpoints come from a previous frame. func (s *TraversalStep) usesBoundEndpointPairs() bool { return s.LeftNodeBound && s.RightNodeBound && s.hasPreviousFrameBinding() } +// usesBoundTerminalIDs reports whether the terminal endpoint comes from a previous frame. func (s *TraversalStep) usesBoundTerminalIDs() bool { return s.RightNodeBound && s.hasPreviousFrameBinding() } +// canMaterializeTerminalFilterForStep reports whether terminal constraints are local and useful as an independent filter. func canMaterializeTerminalFilterForStep(traversalStep *TraversalStep, expansionModel *Expansion) bool { if traversalStep == nil || expansionModel == nil || traversalStep.RightNode == nil || expansionModel.TerminalNodeConstraints == nil || @@ -167,6 +242,7 @@ func canMaterializeTerminalFilterForStep(traversalStep *TraversalStep, expansion return externalConstraints == nil } +// canMaterializeEndpointPairFilterForStep reports whether both local endpoint constraints restrict harness search columns. func canMaterializeEndpointPairFilterForStep(traversalStep *TraversalStep, expansionModel *Expansion) bool { // Pair filters enumerate the exact root/terminal combinations the // bidirectional harness must resolve. Kind-only endpoint predicates are not @@ -185,14 +261,17 @@ func canMaterializeEndpointPairFilterForStep(traversalStep *TraversalStep, expan return true } +// endpointSelectivity scores an endpoint expression using binding and previous-frame context. func (s *TraversalStep) endpointSelectivity(scope *Scope, expression pgsql.Expression, bound bool) (int, error) { return optimize.NewSelectivityModel(scope).EndpointSelectivity(expression, bound, s.hasPreviousFrameBinding()) } +// isBidirectionalSearchAnchor reports whether a selectivity score is strong enough to seed bidirectional search. func isBidirectionalSearchAnchor(selectivity int) bool { return optimize.IsBidirectionalSearchAnchor(selectivity) } +// hasIDEqualityConstraint reports whether identifier's ID equals a row-independent value in a conjunction. func hasIDEqualityConstraint(expression pgsql.Expression, identifier pgsql.Identifier) bool { for _, term := range flattenConjunction(expression) { binaryExpression, isBinaryExpression := unwrapParenthetical(term).(*pgsql.BinaryExpression) @@ -217,6 +296,7 @@ func hasIDEqualityConstraint(expression pgsql.Expression, identifier pgsql.Ident return false } +// hasLocalIDEqualityConstraint reports whether an ID equality depends only on identifier and static values. func hasLocalIDEqualityConstraint(expression pgsql.Expression, identifier pgsql.Identifier) bool { if !hasIDEqualityConstraint(expression, identifier) { return false @@ -225,6 +305,7 @@ func hasLocalIDEqualityConstraint(expression pgsql.Expression, identifier pgsql. return hasLocalEndpointConstraint(expression, identifier) } +// hasLocalEndpointConstraint reports whether expression references identifier without any external binding. func hasLocalEndpointConstraint(expression pgsql.Expression, identifier pgsql.Identifier) bool { if expression == nil || !referencesIdentifier(expression, identifier) { return false @@ -234,6 +315,7 @@ func hasLocalEndpointConstraint(expression pgsql.Expression, identifier pgsql.Id return externalConstraints == nil } +// referencesIdentifier reports whether expression contains a direct, compound, or row-column reference rooted at identifier. func referencesIdentifier(expression pgsql.Expression, identifier pgsql.Identifier) bool { references := false @@ -266,11 +348,13 @@ func referencesIdentifier(expression pgsql.Expression, identifier pgsql.Identifi return references } +// hasPairAwareEndpointConstraint reports whether a local constraint restricts endpoint values beyond node kinds. func hasPairAwareEndpointConstraint(expression pgsql.Expression, identifier pgsql.Identifier) bool { return hasLocalEndpointConstraint(expression, identifier) && referencesEndpointSearchColumn(expression, identifier) } +// referencesEndpointSearchColumn reports whether expression reads a non-kind field used to restrict endpoint search. func referencesEndpointSearchColumn(expression pgsql.Expression, identifier pgsql.Identifier) bool { references := false @@ -291,6 +375,7 @@ func referencesEndpointSearchColumn(expression pgsql.Expression, identifier pgsq return references } +// isStaticIDEqualityOperand reports whether expression contains no row or identifier references. func isStaticIDEqualityOperand(expression pgsql.Expression) bool { if expression == nil { return false @@ -311,6 +396,7 @@ func isStaticIDEqualityOperand(expression pgsql.Expression) bool { return isStatic } +// isIdentifierIDReference reports whether expression is exactly identifier.id. func isIdentifierIDReference(expression pgsql.Expression, identifier pgsql.Identifier) bool { compoundIdentifier, isCompoundIdentifier := unwrapParenthetical(expression).(pgsql.CompoundIdentifier) return isCompoundIdentifier && len(compoundIdentifier) == 2 && @@ -318,6 +404,84 @@ func isIdentifierIDReference(expression pgsql.Expression, identifier pgsql.Ident compoundIdentifier[1] == pgsql.ColumnID } +// isSingletonIDOperand reports whether expression denotes one non-null integer ID literal or parameter. +func isSingletonIDOperand(expression pgsql.Expression) bool { + switch typedExpression := unwrapParenthetical(expression).(type) { + case pgsql.Literal: + return !typedExpression.Null + case pgsql.Parameter, *pgsql.Parameter: + return true + case pgsql.TypeCast: + switch typedExpression.CastType { + case pgsql.Int, pgsql.Int2, pgsql.Int4, pgsql.Int8: + return isSingletonIDOperand(typedExpression.Expression) + default: + return false + } + default: + return false + } +} + +// singletonIDAnchor returns the sole static value equated with identifier.id, rejecting ambiguous multiple equalities. +func singletonIDAnchor(expression pgsql.Expression, identifier pgsql.Identifier) (pgsql.Expression, bool) { + var anchor pgsql.Expression + + for _, term := range flattenConjunction(expression) { + binaryExpression, isBinaryExpression := unwrapParenthetical(term).(*pgsql.BinaryExpression) + if !isBinaryExpression || binaryExpression.Operator != pgsql.OperatorEquals { + continue + } + + var candidate pgsql.Expression + switch { + case isIdentifierIDReference(binaryExpression.LOperand, identifier) && isSingletonIDOperand(binaryExpression.ROperand): + candidate = binaryExpression.ROperand + case isIdentifierIDReference(binaryExpression.ROperand, identifier) && isSingletonIDOperand(binaryExpression.LOperand): + candidate = binaryExpression.LOperand + default: + continue + } + + if anchor != nil { + // Multiple ID equalities may be contradictory and require the generic + // validation path until the singleton validator can retain every term. + return nil, false + } + anchor = candidate + } + + return anchor, anchor != nil +} + +// replaceSingletonIDAnchor substitutes replacement for the static side of identifier's singleton ID equality. +func replaceSingletonIDAnchor(expression pgsql.Expression, identifier pgsql.Identifier, replacement pgsql.Expression) pgsql.Expression { + switch typedExpression := expression.(type) { + case *pgsql.Parenthetical: + typedExpression.Expression = replaceSingletonIDAnchor(typedExpression.Expression, identifier, replacement) + return typedExpression + + case *pgsql.BinaryExpression: + if typedExpression.Operator == pgsql.OperatorEquals { + switch { + case isIdentifierIDReference(typedExpression.LOperand, identifier) && isSingletonIDOperand(typedExpression.ROperand): + typedExpression.ROperand = replacement + return typedExpression + case isIdentifierIDReference(typedExpression.ROperand, identifier) && isSingletonIDOperand(typedExpression.LOperand): + typedExpression.LOperand = replacement + return typedExpression + } + } + + typedExpression.LOperand = replaceSingletonIDAnchor(typedExpression.LOperand, identifier, replacement) + typedExpression.ROperand = replaceSingletonIDAnchor(typedExpression.ROperand, identifier, replacement) + return typedExpression + + default: + return expression + } +} + func (s *TraversalStep) CanExecuteSelectiveBidirectionalSearch(scope *Scope) (bool, error) { if s.Expansion == nil { return false, nil @@ -375,42 +539,52 @@ func (s *TraversalStep) CanExecutePairAwareBidirectionalSearch(scope *Scope) (bo } } +// flattenConjunction returns the independent terms of a nested PostgreSQL AND expression. func flattenConjunction(expr pgsql.Expression) []pgsql.Expression { return optimize.FlattenConjunction(expr) } +// expressionReferencesOnlyLocalIdentifiers reports whether every binding referenced by expression belongs to localScope. func expressionReferencesOnlyLocalIdentifiers(expression pgsql.Expression, localScope *pgsql.IdentifierSet) bool { return optimize.ExpressionReferencesOnlyLocalIdentifiers(expression, localScope) } +// subqueryReferencesOnlyLocalIdentifiers reports whether a subquery has no dependencies outside localScope. func subqueryReferencesOnlyLocalIdentifiers(subquery pgsql.Subquery, localScope *pgsql.IdentifierSet) bool { return optimize.SubqueryReferencesOnlyLocalIdentifiers(subquery, localScope) } +// queryReferencesOnlyLocalIdentifiers reports whether a query has no dependencies outside localScope. func queryReferencesOnlyLocalIdentifiers(query pgsql.Query, localScope *pgsql.IdentifierSet) bool { return optimize.QueryReferencesOnlyLocalIdentifiers(query, localScope) } +// addFromClauseBindings adds every alias introduced by fromClauses to localScope. func addFromClauseBindings(localScope *pgsql.IdentifierSet, fromClauses []pgsql.FromClause) { optimize.AddFromClauseBindings(localScope, fromClauses) } +// addFromExpressionBinding adds the alias introduced by a FROM expression to localScope. func addFromExpressionBinding(localScope *pgsql.IdentifierSet, expression pgsql.Expression) { optimize.AddFromExpressionBinding(localScope, expression) } +// selectReferencesOnlyLocalIdentifiers reports whether a SELECT body has no dependencies outside localScope. func selectReferencesOnlyLocalIdentifiers(selectBody pgsql.Select, localScope *pgsql.IdentifierSet) bool { return optimize.SelectReferencesOnlyLocalIdentifiers(selectBody, localScope) } +// fromExpressionReferencesOnlyLocalIdentifiers reports whether a FROM expression has no dependencies outside localScope. func fromExpressionReferencesOnlyLocalIdentifiers(expression pgsql.Expression, localScope *pgsql.IdentifierSet) bool { return optimize.FromExpressionReferencesOnlyLocalIdentifiers(expression, localScope) } +// isLocalToScope reports whether expression can be evaluated using only identifiers in localScope. func isLocalToScope(expression pgsql.Expression, localScope *pgsql.IdentifierSet) bool { return optimize.IsLocalToScope(expression, localScope) } +// partitionConstraintByLocality separates conjuncts evaluable in localScope from those requiring outer bindings. func partitionConstraintByLocality(expression pgsql.Expression, localScope *pgsql.IdentifierSet) (pgsql.Expression, pgsql.Expression) { return optimize.PartitionConstraintByLocality(expression, localScope) } @@ -514,6 +688,7 @@ type PatternPart struct { nextSourceStep int } +// nextSourceTarget returns the optimizer coordinates for the next traversal step and advances the step cursor. func (s *PatternPart) nextSourceTarget() (optimize.TraversalStepTarget, bool) { if s == nil { return optimize.TraversalStepTarget{}, false @@ -854,6 +1029,7 @@ func (s *Mutations) AddDeletion(scope *Scope, targetIdentifier pgsql.Identifier, } } +// newIdentifierAssignment allocates a distinct update binding and empty assignment collections for targetBinding. func (s *Mutations) newIdentifierAssignment(scope *Scope, targetBinding *BoundIdentifier) (*Update, error) { if updateBinding, err := scope.DefineNew(targetBinding.DataType); err != nil { return nil, err @@ -872,6 +1048,7 @@ func (s *Mutations) newIdentifierAssignment(scope *Scope, targetBinding *BoundId } } +// getIdentifierMutation returns the existing update for targetIdentifier or creates its first assignment state. func (s *Mutations) getIdentifierMutation(scope *Scope, targetIdentifier pgsql.Identifier) (*Update, error) { if targetBinding, bound := scope.Lookup(targetIdentifier); !bound { return nil, fmt.Errorf("invalid identifier: %s", targetIdentifier) @@ -946,6 +1123,7 @@ func (s *Projections) Current() *Projection { return s.Items[len(s.Items)-1] } +// extractIdentifierFromCypherExpression returns the variable or alias directly declared by a supported Cypher expression. func extractIdentifierFromCypherExpression(expression cypher.Expression) (pgsql.Identifier, bool, error) { if expression == nil { return "", false, nil diff --git a/cypher/models/pgsql/translate/optimizer_safety_test.go b/cypher/models/pgsql/translate/optimizer_safety_test.go index 5e1786a2..3d594a8b 100644 --- a/cypher/models/pgsql/translate/optimizer_safety_test.go +++ b/cypher/models/pgsql/translate/optimizer_safety_test.go @@ -2,6 +2,7 @@ package translate import ( "context" + "fmt" "strings" "testing" @@ -12,18 +13,20 @@ import ( "github.com/stretchr/testify/require" ) -const optimizerADCSQuery = ` -MATCH (n:Group) -WHERE n.objectid = 'S-1-5-21-2643190041-1319121918-239771340-513' -MATCH p1 = (n)-[:MemberOf*0..]->()-[:Enroll]->(ca:EnterpriseCA)-[:TrustedForNTAuth]->(:NTAuthStore)-[:NTAuthStoreFor]->(d:Domain) -MATCH p2 = (n)-[:MemberOf*0..]->()-[:GenericAll|Enroll|AllExtendedRights]->(ct:CertTemplate)-[:PublishedTo]->(ca)-[:IssuedSignedBy|EnterpriseCAFor*1..]->(:RootCA)-[:RootCAFor]->(d) -WHERE ct.authenticationenabled = true -AND ct.requiresmanagerapproval = false -AND ct.enrolleesuppliessubject = true -AND (ct.schemaversion = 1 OR ct.authorizedsignatures = 0) +// optimizerFixedSuffixQuery exercises a bounded variable expansion followed by a selective three-edge suffix. +const optimizerFixedSuffixQuery = ` +MATCH (root:ExpansionRoot) +WHERE root.root_key = 'root' +MATCH p1 = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) +MATCH p2 = (root)-[:Expand*0..16]->()-[:OptionA|OptionB|OptionC]->(predicate:PredicateNode)-[:JoinSuffix]->(head)-[:HeadToBridge|HeadToAlternateBridge*1..16]->(:BridgeNode)-[:ReachTerminal]->(terminal) +WHERE predicate.eligible = true +AND predicate.requires_review = false +AND predicate.allows_direct = true +AND (predicate.version = 1 OR predicate.required_approvals = 0) RETURN p1, p2 ` +// optimizerSafetyKindMapper returns deterministic numeric IDs for the kinds used by optimizer-safety fixtures. func optimizerSafetyKindMapper() *pgutil.InMemoryKindMapper { mapper := pgutil.NewInMemoryKindMapper() @@ -31,23 +34,41 @@ func optimizerSafetyKindMapper() *pgutil.InMemoryKindMapper { "AllExtendedRights", "CertTemplate", "Domain", - "Enroll", - "EnterpriseCA", - "EnterpriseCAFor", + "SuffixEdgeOne", + "SuffixNodeOne", + "SuffixNodeOneFor", "GenericAll", "Group", "IssuedSignedBy", "MemberOf", - "NTAuthStore", - "NTAuthStoreFor", + "SuffixNodeTwo", + "SuffixEdgeThree", "PublishedTo", "RootCA", "RootCAFor", - "TrustedForNTAuth", + "SuffixEdgeTwo", "AdminTo", "Computer", "Tag_Tier_Zero", "User", + "ExpansionRoot", + "ExpansionNode", + "Expand", + "SuffixHead", + "EnterSuffix", + "SuffixMiddle", + "ContinueSuffix", + "SuffixTerminal", + "CompleteSuffix", + "OptionA", + "OptionB", + "OptionC", + "PredicateNode", + "JoinSuffix", + "HeadToBridge", + "HeadToAlternateBridge", + "BridgeNode", + "ReachTerminal", }) { mapper.Put(kind) } @@ -55,6 +76,7 @@ func optimizerSafetyKindMapper() *pgutil.InMemoryKindMapper { return mapper } +// optimizerSafetySQL translates cypherQuery and returns its rendered PostgreSQL text. func optimizerSafetySQL(t *testing.T, cypherQuery string) string { t.Helper() @@ -66,12 +88,14 @@ func optimizerSafetySQL(t *testing.T, cypherQuery string) string { return strings.Join(strings.Fields(formattedQuery), " ") } +// optimizerSafetyTranslation parses and translates cypherQuery with the optimizer-safety kind mapper. func optimizerSafetyTranslation(t *testing.T, cypherQuery string) Result { t.Helper() return optimizerSafetyTranslationWithParameters(t, cypherQuery, nil) } +// optimizerSafetyTranslationWithParameters parses and translates cypherQuery with the supplied parameter values. func optimizerSafetyTranslationWithParameters(t *testing.T, cypherQuery string, parameters map[string]any) Result { t.Helper() @@ -84,6 +108,7 @@ func optimizerSafetyTranslationWithParameters(t *testing.T, cypherQuery string, return translation } +// requireOptimizationLowering requires name to appear among the lowerings applied during translation. func requireOptimizationLowering(t *testing.T, summary OptimizationSummary, name string) { t.Helper() @@ -96,6 +121,7 @@ func requireOptimizationLowering(t *testing.T, summary OptimizationSummary, name require.Failf(t, "missing optimization lowering", "expected lowering %q in %#v", name, summary.Lowerings) } +// requireNoOptimizationLowering requires name to be absent from applied lowering diagnostics. func requireNoOptimizationLowering(t *testing.T, summary OptimizationSummary, name string) { t.Helper() @@ -104,6 +130,7 @@ func requireNoOptimizationLowering(t *testing.T, summary OptimizationSummary, na } } +// requirePlannedOptimizationLowering requires name to appear in the optimizer's planned lowerings. func requirePlannedOptimizationLowering(t *testing.T, summary OptimizationSummary, name string) { t.Helper() @@ -116,6 +143,7 @@ func requirePlannedOptimizationLowering(t *testing.T, summary OptimizationSummar require.Failf(t, "missing planned optimization lowering", "expected planned lowering %q in %#v", name, summary.PlannedLowerings) } +// requireNoPlannedOptimizationLowering requires name to be absent from the optimizer's planned lowerings. func requireNoPlannedOptimizationLowering(t *testing.T, summary OptimizationSummary, name string) { t.Helper() @@ -124,6 +152,7 @@ func requireNoPlannedOptimizationLowering(t *testing.T, summary OptimizationSumm } } +// requirePlanParameterContains requires at least one translated parameter value to contain expected. func requirePlanParameterContains(t *testing.T, translation Result, expected string) { t.Helper() @@ -136,6 +165,7 @@ func requirePlanParameterContains(t *testing.T, translation Result, expected str require.Failf(t, "missing plan parameter content", "expected a plan parameter to contain %q in %#v", expected, translation.Parameters) } +// requireSkippedOptimizationLowering requires a skipped-lowering diagnostic with the expected name and reason. func requireSkippedOptimizationLowering(t *testing.T, summary OptimizationSummary, name string, reason string) { t.Helper() @@ -149,6 +179,7 @@ func requireSkippedOptimizationLowering(t *testing.T, summary OptimizationSummar require.Failf(t, "missing skipped optimization lowering", "expected skipped lowering %q in %#v", name, summary.SkippedLowerings) } +// requireSkippedOptimizationLoweringCount requires a skipped-lowering diagnostic with the expected occurrence count. func requireSkippedOptimizationLoweringCount(t *testing.T, summary OptimizationSummary, name string, count int) { t.Helper() @@ -162,6 +193,7 @@ func requireSkippedOptimizationLoweringCount(t *testing.T, summary OptimizationS require.Failf(t, "missing skipped optimization lowering", "expected skipped lowering %q in %#v", name, summary.SkippedLowerings) } +// requireNoSkippedOptimizationLowering requires name to be absent from skipped-lowering diagnostics. func requireNoSkippedOptimizationLowering(t *testing.T, summary OptimizationSummary, name string) { t.Helper() @@ -189,6 +221,1357 @@ func TestOptimizerSafetyReportsPartiallySkippedLowerings(t *testing.T) { requireSkippedOptimizationLoweringCount(t, translator.translation.Optimization, optimize.LoweringPredicatePlacement, 1) } +// TestFixedSuffixSearchStrategyIsPlannedButConservativelySkipped verifies that an unforced candidate remains diagnostic-only. +func TestFixedSuffixSearchStrategyIsPlannedButConservativelySkipped(t *testing.T) { + translation := optimizerSafetyTranslation(t, ` + MATCH (root:ExpansionRoot) + WHERE root.root_key = $root_key + MATCH path = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) + RETURN path + `) + + requirePlannedOptimizationLowering(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy) + requireNoOptimizationLowering(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy) + requireSkippedOptimizationLowering(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy, optimize.ExpansionSearchFallbackTournamentUnqualified) + require.Len(t, translation.Optimization.LoweringPlan.ExpansionSearchStrategy, 1) + require.True(t, translation.Optimization.LoweringPlan.ExpansionSearchStrategy[0].StructurallyEligible) + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 1, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, "fixed_suffix_expansion", outcome.Family) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV1), outcome.PlannedPolicy) + require.Empty(t, outcome.EmittedPolicy) + require.Equal(t, []string{"EXPANSION-STEPWISE-FORWARD", "EXPANSION-LATE-HYDRATED-FORWARD", "EXPANSION-FACTORED-SUFFIX-FORWARD", "EXPANSION-SUFFIX-SEEDED-REVERSE", "EXPANSION-BACKWARD-VIABILITY-FORWARD"}, outcome.PlannedCandidates) + require.Equal(t, []string{string(optimize.ExpansionSearchStepwiseForward)}, outcome.EmittedCandidates) + require.Equal(t, &optimize.ExpansionSearchProbeCaps{ + RootRowLimit: optimize.ExpansionSearchOrientationRootRowLimit, + ReverseSeedRowLimit: optimize.ExpansionSearchOrientationReverseSeedRowLimit, + DirectionalDegreeRowLimit: optimize.ExpansionSearchOrientationDirectionalDegreeRowLimit, + }, outcome.ProbeCaps) + require.Equal(t, &optimize.ExpansionSearchAdmission{ + StateLimit: optimize.ExpansionSearchOrientationStateLimit, + RequiresCompleteProbes: true, + FallbackStrategy: optimize.ExpansionSearchStepwiseForward, + }, outcome.Admission) + require.Contains(t, outcome.EligibilityFacts, TargetEligibilityFact{ + Name: "qualified_fixed_suffix_topology", + Eligible: true, + }) + require.Equal(t, string(optimize.ExpansionSearchObservationFullPath), outcome.ObservationMode) + require.NotNil(t, outcome.Eligible) + require.True(t, *outcome.Eligible) + require.Equal(t, "incumbent_default", outcome.SelectionMode) + require.Equal(t, "fixed-suffix-static-v1", outcome.SelectorVersion) + require.Equal(t, string(optimize.ExpansionSearchStepwiseForward), outcome.Selected) + require.Equal(t, string(optimize.ExpansionSearchStepwiseForward), outcome.Fallback) + require.Equal(t, optimize.ExpansionSearchFallbackTournamentUnqualified, outcome.SkipReason) +} + +// TestForcedSuffixSeededReverseEmitsNativeReverseTrailState verifies the reverse-search CTE and ordered edge-ID state emitted by a forced strategy. +func TestForcedSuffixSeededReverseEmitsNativeReverseTrailState(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH (root:ExpansionRoot) + WHERE root.root_key = $root_key + MATCH path = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) + RETURN path + `) + require.NoError(t, err) + + plan, err := optimize.Optimize(regularQuery) + require.NoError(t, err) + require.NoError(t, applyToolOptions(&plan, ToolOptions{ + ForceExpansionSearchStrategy: optimize.ExpansionSearchSuffixSeededReverse, + })) + require.Len(t, plan.LoweringPlan.ExpansionSearchStrategy, 1) + decision := plan.LoweringPlan.ExpansionSearchStrategy[0] + require.Equal(t, optimize.ExpansionSearchSuffixSeededReverse, decision.SelectedStrategy) + require.Empty(t, decision.EmittedPolicy) + require.Equal(t, []optimize.ExpansionSearchStrategy{optimize.ExpansionSearchSuffixSeededReverse}, decision.EmittedCandidates) + require.Equal(t, "forced_tool", decision.SelectionMode) + require.Equal(t, "suffix-seeded-reverse-tool-v1", decision.SelectorVersion) + require.Empty(t, decision.FallbackReason) + + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": "forced-fixed-suffix-root", + }, DefaultGraphID, ToolOptions{ForceExpansionSearchStrategy: optimize.ExpansionSearchSuffixSeededReverse}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "with recursive") + require.Contains(t, formatted, "_suffix_seeded_suffix as materialized") + require.Contains(t, formatted, "_suffix_seeded_reverse(boundary_id, next_id, depth, path)") + require.Contains(t, formatted, "array_prepend(e0.id") + require.Contains(t, formatted, "e0.id != all (s5_suffix_seeded_reverse.path)") + require.Contains(t, formatted, "e0.end_id = s5_suffix_seeded_reverse.next_id") + require.Contains(t, formatted, "s5_suffix_seeded_reverse.path && array [s5_suffix_seeded_suffix.e1, s5_suffix_seeded_suffix.e2, s5_suffix_seeded_suffix.e3]::int8[]") + require.Contains(t, formatted, "e2.id != e1.id") + require.Contains(t, formatted, "e3.id != e1.id") + require.Contains(t, formatted, "e3.id != e2.id") + require.NotContains(t, formatted, "s2(root_id, next_id, depth, satisfied, is_cycle, path)") + + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 1, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, string(optimize.ExpansionSearchSuffixSeededReverse), outcome.Selected) + require.Equal(t, string(optimize.ExpansionSearchSuffixSeededReverse), outcome.Applied) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV1), outcome.PlannedPolicy) + require.Empty(t, outcome.EmittedPolicy) + require.Equal(t, []string{string(optimize.ExpansionSearchSuffixSeededReverse)}, outcome.EmittedCandidates) + require.Equal(t, "inline_statement", outcome.ExecutionBoundary) + require.Equal(t, "forced_tool", outcome.SelectionMode) + require.Empty(t, outcome.SkipReason) + requireOptimizationLowering(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy) + requireNoSkippedOptimizationLowering(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy) +} + +// TestEndpointSeededReverseIsAutomaticallyGuardedAndApplied verifies that qualified endpoint seeding emits bounded probes and reports application. +func TestEndpointSeededReverseIsAutomaticallyGuardedAndApplied(t *testing.T) { + translation := optimizerSafetyTranslationWithParameters(t, ` + MATCH p = (c:Computer)-[:AdminTo]->(:User)-[:MemberOf*1..]->(g:Group) + WHERE g.objectid ENDS WITH $suffix + RETURN p + LIMIT 1000 + `, map[string]any{"suffix": "-512"}) + + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "_endpoint_seeded_endpoints as materialized") + require.Contains(t, formatted, "limit 33") + require.Contains(t, formatted, "_endpoint_seeded_states as materialized") + require.Contains(t, formatted, "_endpoint_seeded_incumbent as materialized") + require.Contains(t, formatted, "limit 4097") + require.Contains(t, formatted, "array_prepend") + require.Contains(t, formatted, "_endpoint_seeded_reverse.next_id") + require.Contains(t, formatted, "offset 32 limit 1") + require.Contains(t, formatted, "offset 4096 limit 1") + require.Contains(t, formatted, "_endpoint_seeded_incumbent") + require.Contains(t, formatted, "_endpoint_seeded_states.path && array [") + + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy, optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 1, + }) + require.Equal(t, string(optimize.ExpansionSearchEndpointSeededReverse), outcome.Selected) + require.Equal(t, string(optimize.ExpansionSearchEndpointSeededReverse), outcome.Applied) + require.Equal(t, string(optimize.ExpansionSearchPolicyEndpointGuardV1), outcome.PlannedPolicy) + require.Equal(t, string(optimize.ExpansionSearchPolicyEndpointGuardV1), outcome.EmittedPolicy) + require.Equal(t, []string{string(optimize.ExpansionSearchStepwiseForward), string(optimize.ExpansionSearchEndpointSeededReverse)}, outcome.EmittedCandidates) + require.Equal(t, "guarded_dual_arm", outcome.ExecutionBoundary) + require.Equal(t, &optimize.ExpansionSearchProbeCaps{ReverseSeedRowLimit: 32}, outcome.ProbeCaps) + require.Equal(t, &optimize.ExpansionSearchAdmission{ + StateLimit: 4096, + RequiresCompleteProbes: true, + FallbackStrategy: optimize.ExpansionSearchStepwiseForward, + }, outcome.Admission) + require.Equal(t, int64(32), outcome.EndpointLimit) + require.Equal(t, int64(4096), outcome.StateLimit) + require.Equal(t, "property_ends_with", outcome.SeedPredicateClass) + require.Equal(t, 1, outcome.PrefixLength) + require.True(t, outcome.HasFinalLimit) +} + +func TestProductionEndpointSeededKillSwitchRestoresStepwiseSQL(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = (c:Computer)-[:AdminTo]->(:User)-[:MemberOf*1..]->(g:Group) + WHERE g.objectid ENDS WITH $suffix + RETURN p LIMIT 1000 + `) + require.NoError(t, err) + translation, err := TranslateWithProductionOptions(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{"suffix": "-512"}, DefaultGraphID, ProductionOptions{ + DisableEndpointSeededReverse: true, SelectorVersion: "endpoint-seeded-kill-switch-v1", + }) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.NotContains(t, formatted, "_endpoint_seeded_endpoints") + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy, optimize.TraversalStepTarget{QueryPartIndex: 0, ClauseIndex: 0, PatternIndex: 0, StepIndex: 1}) + require.Equal(t, string(optimize.ExpansionSearchStepwiseForward), outcome.Selected) + require.Empty(t, outcome.EmittedPolicy) + require.Equal(t, []string{string(optimize.ExpansionSearchStepwiseForward)}, outcome.EmittedCandidates) + require.Equal(t, "production_kill_switch", outcome.SelectionMode) + require.Equal(t, "inline_statement", outcome.ExecutionBoundary) +} + +// TestOrdinaryExpansionMayContinueAfterSelfLoop verifies that encountering a self-loop does not stop unrelated recursive expansion. +func TestOrdinaryExpansionMayContinueAfterSelfLoop(t *testing.T) { + formatted := optimizerSafetySQL(t, `MATCH p = (s)-[:MemberOf*1..3]->(g) RETURN p`) + require.Contains(t, formatted, "1, false, false, array [e0.id]") + require.NotContains(t, formatted, "e0.start_id = e0.end_id, array [e0.id]") +} + +// TestForcedSuffixSeededReverseEndpointSQLIsParameterStable verifies deterministic parameter numbering in forced reverse-search SQL. +func TestForcedSuffixSeededReverseEndpointSQLIsParameterStable(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH (root:ExpansionRoot) + WHERE root.root_key = $root_key + MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) + RETURN id(head), id(terminal) + `) + require.NoError(t, err) + + translateForced := func(rootKey string) string { + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": rootKey, + }, DefaultGraphID, ToolOptions{ForceExpansionSearchStrategy: optimize.ExpansionSearchSuffixSeededReverse}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + return formatted + } + + first := translateForced("root-a") + second := translateForced("root-b") + require.Equal(t, first, second) + require.Contains(t, first, "s5_suffix_seeded_reverse.path") + require.Contains(t, first, "select s5.n2 as \"id(head)\", s5.n4 as \"id(terminal)\"") + require.NotContains(t, first, "ordered_edge_ids_to_path") + require.NotContains(t, first, "s2(root_id, next_id, depth, satisfied, is_cycle, path)") +} + +// TestForcedSuffixSeededReversePreservesBoundaryConstraints verifies that predicates attached at the suffix boundary survive reversal. +func TestForcedSuffixSeededReversePreservesBoundaryConstraints(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH (root:ExpansionRoot) + WHERE root.root_key = $root_key + MATCH (root)-[:Expand*0..16]->(boundary:ExpansionNode {enabled: true})-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) + RETURN id(head), id(terminal) + `) + require.NoError(t, err) + + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": "forced-fixed-suffix-root", + }, DefaultGraphID, ToolOptions{ForceExpansionSearchStrategy: optimize.ExpansionSearchSuffixSeededReverse}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "_suffix_seeded_suffix as materialized") + require.Contains(t, formatted, "n1.kind_ids operator (pg_catalog.@>)") + require.Contains(t, formatted, "n1.properties -> 'enabled'") + require.Contains(t, formatted, "to_jsonb((true)::bool)") +} + +// TestForcedFixedSuffixSearchRejectsUnsupportedStrategy verifies that tooling cannot force a strategy outside the candidate family. +func TestForcedFixedSuffixSearchRejectsUnsupportedStrategy(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) + RETURN id(head), id(terminal) + `) + require.NoError(t, err) + + _, err = TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), nil, DefaultGraphID, ToolOptions{ + ForceExpansionSearchStrategy: optimize.ExpansionSearchFactoredSuffixForward, + }) + require.ErrorContains(t, err, "unsupported forced expansion-search strategy") +} + +// TestForcedFixedSuffixSearchRejectsStructurallyIneligibleTarget verifies that forcing does not bypass structural qualification. +func TestForcedFixedSuffixSearchRejectsStructurallyIneligibleTarget(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead) + RETURN id(head) + `) + require.NoError(t, err) + + _, err = TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), nil, DefaultGraphID, ToolOptions{ + ForceExpansionSearchStrategy: optimize.ExpansionSearchSuffixSeededReverse, + }) + require.ErrorContains(t, err, "has no structurally eligible target") +} + +// TestForcedExpansionSearchRequiresExactlyOneEligibleTarget verifies that +// tooling fails closed before mutating any decision when a force is ambiguous. +func TestForcedExpansionSearchRequiresExactlyOneEligibleTarget(t *testing.T) { + plan := optimize.Plan{LoweringPlan: optimize.LoweringPlan{ + ExpansionSearchStrategy: []optimize.ExpansionSearchStrategyDecision{ + { + CandidateStrategy: optimize.ExpansionSearchSuffixSeededReverse, + SelectedStrategy: optimize.ExpansionSearchStepwiseForward, + StructurallyEligible: true, + }, + { + CandidateStrategy: optimize.ExpansionSearchSuffixSeededReverse, + SelectedStrategy: optimize.ExpansionSearchStepwiseForward, + StructurallyEligible: true, + }, + }, + }} + before := append([]optimize.ExpansionSearchStrategyDecision(nil), plan.LoweringPlan.ExpansionSearchStrategy...) + + err := applyForcedExpansionSearchStrategy(&plan, optimize.ExpansionSearchSuffixSeededReverse) + require.ErrorContains(t, err, "matched 2 structurally eligible targets; expected exactly one") + require.Equal(t, before, plan.LoweringPlan.ExpansionSearchStrategy) +} + +// TestShortestDistanceExecutorIsAutomaticallySelectedAndReportedApplied verifies automatic scalar-distance selection and matching diagnostics. +func TestShortestDistanceExecutorIsAutomaticallySelectedAndReportedApplied(t *testing.T) { + translation := optimizerSafetyTranslation(t, ` + MATCH p = shortestPath((s)-[:MemberOf*1..64]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `) + + requirePlannedOptimizationLowering(t, translation.Optimization, optimize.LoweringShortestPathExecutor) + requireOptimizationLowering(t, translation.Optimization, optimize.LoweringShortestPathExecutor) + requireNoSkippedOptimizationLowering(t, translation.Optimization, optimize.LoweringShortestPathExecutor) + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, "SP", outcome.Family) + require.Equal(t, []string{"SP-S0", "SP-S0-DIRECT", "SP-S1", "SP-S2", "SP-S3-U-D", "SP-S3-U-E+MAT-M0", "SP-S4-C-D", "SP-S4-C-WE+MAT-M0", "SP-I1-C-D", "SP-I1-U-E+MAT-M0", "SP-I1-C-WE+MAT-M0", "SP-B1-C-ALT-NODE-D", "SP-B1-C-ALT-NODE-WE+MAT-M0", "SP-B2-C-MIN-LEVEL-D", "SP-B2-C-MIN-LEVEL-WE+MAT-M0"}, outcome.PlannedCandidates) + require.Equal(t, string(optimize.ShortestPathSchedulerSingleEndedLevel), outcome.Scheduler) + require.Contains(t, outcome.EligibilityFacts, TargetEligibilityFact{ + Name: "one_static_id_equality_per_endpoint", + Eligible: true, + }) + require.Equal(t, string(optimize.ShortestPathObservationDistance), outcome.ObservationMode) + require.NotNil(t, outcome.Eligible) + require.True(t, *outcome.Eligible) + require.Equal(t, "static", outcome.SelectionMode) + require.Equal(t, "sp-static-v3", outcome.SelectorVersion) + require.Equal(t, string(optimize.ShortestPathExecutorS3Unidirectional), outcome.Selected) + require.Equal(t, string(optimize.ShortestPathExecutorS3Unidirectional), outcome.Applied) + require.Equal(t, string(optimize.ShortestPathExecutorIncumbentWorkspace), outcome.Fallback) + require.Empty(t, outcome.SkipReason) +} + +// TestGreedyProjectionMaterializesShortestPathAndEntities verifies that RETURN * hydrates the path and every visible endpoint. +func TestGreedyProjectionMaterializesShortestPathAndEntities(t *testing.T) { + translation := optimizerSafetyTranslation(t, ` + MATCH p = shortestPath((s:Group)-[:MemberOf*1..4]->(e:Group)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN * + `) + + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, string(optimize.ShortestPathObservationOnePath), outcome.ObservationMode) + + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "::pathcomposite") + require.Contains(t, formatted, "::nodecomposite") +} + +// TestGreedyProjectionMaterializesRelationships verifies that RETURN * hydrates relationship bindings. +func TestGreedyProjectionMaterializesRelationships(t *testing.T) { + formatted := optimizerSafetySQL(t, ` + MATCH (s:Group)-[r:MemberOf]->(e:Group) + RETURN * + `) + + require.Contains(t, formatted, "::nodecomposite") + require.Contains(t, formatted, "::edgecomposite") +} + +// TestGreedyWithProjectionCarriesFullShortestPath verifies that WITH * preserves a complete shortest-path value across query parts. +func TestGreedyWithProjectionCarriesFullShortestPath(t *testing.T) { + translation := optimizerSafetyTranslation(t, ` + MATCH p = shortestPath((s:Group)-[:MemberOf*1..4]->(e:Group)) + WHERE id(s) = $start_id AND id(e) = $end_id + WITH * + RETURN p + `) + + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "::pathcomposite") + require.NotContains(t, formatted, "ordered_edge_ids_to_path") + require.Contains(t, formatted, "m0_hydrated") +} + +// TestShortestExecutorV4SelectsDeepInboundCompactDistance verifies canonical distance selection and inbound physical topology diagnostics. +func TestShortestExecutorV4SelectsDeepInboundCompactDistance(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((e)<-[:MemberOf*1..8]-(s)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `) + require.NoError(t, err) + translation, err := Translate(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "shortest_path_compact") + require.NotContains(t, formatted, "sp_harness") + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, "inbound", outcome.Direction) + require.Equal(t, "end_id", outcome.PhysicalExpansion) + require.Equal(t, 1, outcome.RelationshipKindCount) + require.False(t, outcome.UntypedRelationship) + require.Equal(t, "physical_inbound_deep", outcome.TopologyClassification) + require.NotNil(t, outcome.Eligible) + require.True(t, *outcome.Eligible) + require.NotNil(t, outcome.StaticallyEligible) + require.True(t, *outcome.StaticallyEligible) + require.Equal(t, string(optimize.ShortestPathExecutorS4CanonicalDistance), outcome.Selected) + require.Equal(t, string(optimize.ShortestPathExecutorS4CanonicalDistance), outcome.Applied) + require.Empty(t, outcome.SkipReason) +} + +// TestShortestExecutorV4SelectsCompactMultiKindPathAndKeepsS3Distance verifies observation-dependent selection for multi-kind paths. +func TestShortestExecutorV4SelectsCompactMultiKindPathAndKeepsS3Distance(t *testing.T) { + for _, test := range []struct { + // observation is the return expression that consumes the shortest path. + observation string + // selected is the executor expected for that observation. + selected optimize.ShortestPathExecutor + // reason is the expected translation skip reason, if any. + reason string + }{ + { + observation: "p", + selected: optimize.ShortestPathExecutorS4CanonicalWitness, + }, + { + observation: "length(p)", + selected: optimize.ShortestPathExecutorS3Unidirectional, + }, + } { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), fmt.Sprintf(` + MATCH p = shortestPath((s)-[:MemberOf|SuffixEdgeOne*1..8]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN %s + `, test.observation)) + require.NoError(t, err) + translation, err := Translate(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, 2, outcome.RelationshipKindCount) + require.Equal(t, string(test.selected), outcome.Selected) + require.Equal(t, test.reason, outcome.SkipReason) + if test.selected == optimize.ShortestPathExecutorS4CanonicalWitness { + require.Contains(t, formatted, "generate_subscripts(s1.path, 1)") + require.NotContains(t, formatted, "ordered_edge_ids_to_path") + } + } +} + +// TestAllShortestDAGIsAutomaticallySelectedAndUsesTypedStaticExecutor verifies typed predecessor-DAG execution for bound all-shortest paths. +func TestAllShortestDAGIsAutomaticallySelectedAndUsesTypedStaticExecutor(t *testing.T) { + translation := optimizerSafetyTranslationWithParameters(t, ` + MATCH p = allShortestPaths((s)-[*1..]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `, map[string]any{"start_id": int64(1), "end_id": int64(2)}) + + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "all_shortest_paths_dag") + require.NotContains(t, formatted, "bidirectional_asp_harness") + require.NotContains(t, formatted, "traversal_pair_filter") + require.Contains(t, formatted, "array []::int2[]") + + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, "ASP", outcome.Family) + require.Equal(t, []string{"SP-S0", "ASP-A1-DAG", "ASP-I1-U-DAG+MAT-M0", "ASP-B1-DAG-ALT-NODE", "ASP-B2-DAG-MIN-LEVEL"}, outcome.PlannedCandidates) + require.Equal(t, string(optimize.ShortestPathSchedulerSingleEndedLevel), outcome.Scheduler) + require.Equal(t, string(optimize.ShortestPathObservationAllPaths), outcome.ObservationMode) + require.Equal(t, string(optimize.ShortestPathExecutorASPA1DAG), outcome.Selected) + require.Equal(t, string(optimize.ShortestPathExecutorASPA1DAG), outcome.Applied) + require.Equal(t, "asp-static-v1", outcome.SelectorVersion) + require.Empty(t, outcome.SkipReason) +} + +// TestForcedCompactBidirectionalExecutorsUseTypedKernels verifies every SP B1/B2 +// identity reaches its scheduler wrapper without changing automatic selection. +func TestForcedCompactBidirectionalExecutorsUseTypedKernels(t *testing.T) { + tests := []struct { + executor optimize.ShortestPathExecutor + result string + functionName string + }{ + {optimize.ShortestPathExecutorB1AlternatingNodeDistance, "length(p)", "shortest_path_b1_strict_alternating"}, + {optimize.ShortestPathExecutorB1AlternatingNodeWitness, "p", "shortest_path_b1_strict_alternating"}, + {optimize.ShortestPathExecutorB2SmallerCurrentLevelDistance, "length(p)", "shortest_path_b2_smaller_current_level"}, + {optimize.ShortestPathExecutorB2SmallerCurrentLevelWitness, "p", "shortest_path_b2_smaller_current_level"}, + } + for _, test := range tests { + t.Run(string(test.executor), func(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), fmt.Sprintf(` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN %s + `, test.result)) + require.NoError(t, err) + + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: test.executor}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, test.functionName) + require.Equal(t, 3, strings.Count(formatted, "100000"), formatted) + require.NotContains(t, formatted, "bidirectional_sp_harness") + + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{QueryPartIndex: 0, ClauseIndex: 0, PatternIndex: 0, StepIndex: 0}) + require.Equal(t, string(test.executor), outcome.Selected) + require.Equal(t, string(test.executor), outcome.Applied) + require.Equal(t, string(test.executor.Scheduler()), outcome.Scheduler) + require.Equal(t, "forced_tool", outcome.SelectionMode) + }) + } +} + +// TestProductionCanaryShortestExecutorUsesVersionedSelectionMetadata verifies +// the production policy path emits the same qualified kernel while remaining +// distinguishable from tool forcing. +func TestProductionCanaryShortestExecutorUsesVersionedSelectionMetadata(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)<-[:MemberOf*1..64]-(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + translation, err := TranslateWithProductionOptions(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ProductionOptions{ + ShortestPathExecutor: optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, + SelectorVersion: optimize.ShortestPathSelectorStaticV6, + ShortestPathCaps: &ProductionShortestPathCaps{ + StateLimit: 1000, PredecessorLimit: 1000, EnumerationLimit: 1000, OutputBytesLimit: 1 << 20, + }, + AuthorizedBucket: &ProductionTraversalBucket{Direction: "inbound", ObservationMode: "one_path", MinimumDepth: 1, MaximumDepth: 64, RelationshipKindCount: 1}, + }) + require.NoError(t, err) + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{QueryPartIndex: 0, ClauseIndex: 0, PatternIndex: 0, StepIndex: 0}) + require.Equal(t, "production_canary", outcome.SelectionMode) + require.Equal(t, optimize.ShortestPathSelectorStaticV6, outcome.SelectorVersion) + require.Equal(t, string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), outcome.Applied) +} + +func TestProductionCanonicalSPRequiresExactStaticV6Envelope(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)<-[:MemberOf*1..64]-(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + base := ProductionOptions{ + ShortestPathExecutor: optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, + SelectorVersion: optimize.ShortestPathSelectorStaticV6, + ShortestPathCaps: &ProductionShortestPathCaps{ + StateLimit: 1000, PredecessorLimit: 1000, EnumerationLimit: 1000, OutputBytesLimit: 1 << 20, + }, + AuthorizedBucket: &ProductionTraversalBucket{ + Direction: "inbound", ObservationMode: "one_path", MinimumDepth: 1, MaximumDepth: 64, RelationshipKindCount: 1, + }, + } + + tests := map[string]func(*ProductionOptions){ + "selector": func(options *ProductionOptions) { options.SelectorVersion = "sp-static-v5-contained" }, + "outbound": func(options *ProductionOptions) { options.AuthorizedBucket.Direction = "outbound" }, + "maximum": func(options *ProductionOptions) { options.AuthorizedBucket.MaximumDepth = 63 }, + "kinds": func(options *ProductionOptions) { options.AuthorizedBucket.RelationshipKindCount = 2 }, + "untyped": func(options *ProductionOptions) { + options.AuthorizedBucket.RelationshipKindCount = 0 + options.AuthorizedBucket.UntypedRelationship = true + }, + } + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + options := base + bucket := *base.AuthorizedBucket + options.AuthorizedBucket = &bucket + mutate(&options) + _, err := TranslateWithProductionOptions(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, options) + require.Error(t, err) + }) + } +} + +func TestProductionRejectsToolOnlyBidirectionalShortestExecutor(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + _, err = TranslateWithProductionOptions(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ProductionOptions{ + ShortestPathExecutor: optimize.ShortestPathExecutorB2SmallerCurrentLevelWitness, + SelectorVersion: "traversal-production-g7", + }) + require.ErrorContains(t, err, "not production-canary eligible") +} + +// TestForcedBidirectionalASPExecutorsUseTypedKernels verifies the tool-only +// candidates reach their two-sided predecessor-DAG wrappers while automatic +// production selection remains ASP-A1-DAG. +func TestForcedBidirectionalASPExecutorsUseTypedKernels(t *testing.T) { + tests := []struct { + executor optimize.ShortestPathExecutor + functionName string + }{ + {optimize.ShortestPathExecutorASPB1AlternatingNodeDAG, "all_shortest_paths_b1_strict_alternating"}, + {optimize.ShortestPathExecutorASPB2SmallerCurrentLevelDAG, "all_shortest_paths_b2_smaller_current_level"}, + } + for _, test := range tests { + t.Run(string(test.executor), func(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = allShortestPaths((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: test.executor}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, test.functionName) + require.NotContains(t, formatted, "bidirectional_asp_harness") + require.Equal(t, 4, strings.Count(formatted, "100000"), formatted) + require.Contains(t, formatted, "67108864") + + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{QueryPartIndex: 0, ClauseIndex: 0, PatternIndex: 0, StepIndex: 0}) + require.Equal(t, string(test.executor), outcome.Selected) + require.Equal(t, string(test.executor), outcome.Applied) + require.Equal(t, string(test.executor.Scheduler()), outcome.Scheduler) + require.Equal(t, "forced_tool", outcome.SelectionMode) + require.Equal(t, "asp-tool-v1", outcome.SelectorVersion) + require.Equal(t, string(optimize.ShortestPathExecutorASPA1DAG), outcome.Fallback) + require.Equal(t, int64(100_000), outcome.EnumerationLimit) + require.Equal(t, int64(64*1024*1024), outcome.OutputBytesLimit) + }) + } +} + +// TestForcedInlineASPExecutorUsesGuardedTypedStatement verifies the I1 +// production-shaped emitter is forceable for qualification without changing +// the automatic ASP-A1 selection. +func TestForcedInlineASPExecutorUsesGuardedTypedStatement(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = allShortestPaths((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: optimize.ShortestPathExecutorASPI1DAG}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "asp_i1_distance") + require.Contains(t, formatted, "asp_i1_direct") + require.Contains(t, formatted, "asp_i1_predecessor_bounded") + require.Contains(t, formatted, "asp_i1_paths_bounded") + require.Contains(t, formatted, "asp_i1_admission") + require.Contains(t, formatted, "asp_i1_candidate_marker") + require.Contains(t, formatted, "asp_i1_fallback_marker") + require.Contains(t, formatted, "all_shortest_paths_dag") + require.Contains(t, formatted, "record_requested_traversal_runtime_attestation_v1") + require.Contains(t, formatted, "record_requested_traversal_runtime_attestation_v1(case when asp_i1_admission.overflow") + require.Contains(t, formatted, "end, asp_i1_admission.overflow, case when asp_i1_admission.overflow") + require.Equal(t, 7, strings.Count(formatted, "offset 100000 limit 1"), formatted) + require.Equal(t, 1, strings.Count(formatted, "67108864"), formatted) + + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{QueryPartIndex: 0, ClauseIndex: 0, PatternIndex: 0, StepIndex: 0}) + require.Equal(t, string(optimize.ShortestPathExecutorASPI1DAG), outcome.Selected) + require.Equal(t, string(optimize.ShortestPathExecutorASPI1DAG), outcome.Applied) + require.Equal(t, "guarded_dual_arm", outcome.ExecutionBoundary) + require.Equal(t, string(optimize.ShortestPathExecutorASPA1DAG), outcome.Fallback) + require.Equal(t, "forced_tool", outcome.SelectionMode) + require.Equal(t, "asp-tool-v1", outcome.SelectorVersion) +} + +func TestProductionInlineASPUsesAuthorizedBucketAndImmutableCaps(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = allShortestPaths((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + options := ProductionOptions{ + ShortestPathExecutor: optimize.ShortestPathExecutorASPI1DAG, + ShortestPathCaps: &ProductionShortestPathCaps{ + StateLimit: 31, PredecessorLimit: 37, EnumerationLimit: 41, OutputBytesLimit: 43000, + }, + AuthorizedBucket: &ProductionTraversalBucket{ + Direction: "outbound", ObservationMode: "all_paths", MinimumDepth: 1, MaximumDepth: 4, + RelationshipKindCount: 1, UntypedRelationship: false, + }, + SelectorVersion: "asp-i1-canary-v1", + } + translation, err := TranslateWithProductionOptions(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, options) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + for _, limit := range []string{"31", "37", "41", "43000"} { + require.Contains(t, formatted, limit) + } + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{QueryPartIndex: 0, ClauseIndex: 0, PatternIndex: 0, StepIndex: 0}) + require.Equal(t, "production_canary", outcome.SelectionMode) + require.Equal(t, "asp-i1-canary-v1", outcome.SelectorVersion) + require.Equal(t, "asp-i1-guarded-v1", outcome.EmittedPolicy) + require.Equal(t, []string{"ASP-I1-U-DAG+MAT-M0", "ASP-A1-DAG"}, outcome.EmittedCandidates) + require.Equal(t, "guarded_dual_arm", outcome.ExecutionBoundary) + require.Zero(t, outcome.FrontierLimit) + + options.AuthorizedBucket.MaximumDepth = 8 + _, err = TranslateWithProductionOptions(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, options) + require.ErrorContains(t, err, "does not match its authorized promotion bucket") + + options.AuthorizedBucket = nil + _, err = TranslateWithProductionOptions(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, options) + require.ErrorContains(t, err, "requires an exact authorized bucket") +} + +// TestForcedBidirectionalASPExecutorsFailClosedOutsideEnvelope verifies tool +// forcing cannot broaden the singleton, directed, predicate-free, read-only, +// minimum-depth-one all-path observation contract. +func TestForcedBidirectionalASPExecutorsFailClosedOutsideEnvelope(t *testing.T) { + tests := []struct { + name string + query string + }{ + {name: "wrong observation", query: `MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p`}, + {name: "zero minimum", query: `MATCH p = allShortestPaths((s)-[:MemberOf*0..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p`}, + {name: "minimum two", query: `MATCH p = allShortestPaths((s)-[:MemberOf*2..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p`}, + {name: "maximum sixty five", query: `MATCH p = allShortestPaths((s)-[:MemberOf*1..65]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p`}, + {name: "directionless", query: `MATCH p = allShortestPaths((s)-[:MemberOf*1..4]-(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p`}, + {name: "path relationship predicate", query: `MATCH p = allShortestPaths((s)-[:MemberOf*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id AND all(r IN relationships(p) WHERE type(r) = 'MemberOf') RETURN p`}, + {name: "optional", query: `OPTIONAL MATCH p = allShortestPaths((s)-[:MemberOf*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p`}, + {name: "mutation", query: `MATCH p = allShortestPaths((s)-[:MemberOf*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id SET s.flag = true RETURN p`}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), test.query) + require.NoError(t, err) + for _, executor := range []optimize.ShortestPathExecutor{ + optimize.ShortestPathExecutorASPB1AlternatingNodeDAG, + optimize.ShortestPathExecutorASPB2SmallerCurrentLevelDAG, + } { + _, err = TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: executor}) + require.ErrorContains(t, err, "no structurally eligible all-paths target") + } + }) + } +} + +// TestForcedCompactBidirectionalExecutorsRejectUnsupportedDepth verifies the +// bounded maximum-depth envelope cannot be broadened by tool forcing. +func TestForcedCompactBidirectionalExecutorsRejectUnsupportedDepth(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..65]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `) + require.NoError(t, err) + for _, executor := range []optimize.ShortestPathExecutor{ + optimize.ShortestPathExecutorB1AlternatingNodeDistance, + optimize.ShortestPathExecutorB2SmallerCurrentLevelDistance, + } { + _, err = TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: executor}) + require.ErrorContains(t, err, "no structurally eligible distance-only target") + } +} + +// TestForcedShortestDistanceExecutorEmitsNativeScalarState verifies the scalar recursive state emitted by a forced distance executor. +func TestForcedShortestDistanceExecutorEmitsNativeScalarState(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `) + require.NoError(t, err) + + incumbent, err := Translate(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID) + require.NoError(t, err) + incumbentSQL, err := Translated(incumbent) + require.NoError(t, err) + productionOutcome := requireTraversalTargetOutcome(t, incumbent.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, string(optimize.ShortestPathExecutorS3Unidirectional), productionOutcome.Selected) + require.Equal(t, string(optimize.ShortestPathExecutorS3Unidirectional), productionOutcome.Applied) + require.Equal(t, "static", productionOutcome.SelectionMode) + require.Equal(t, "sp-static-v3", productionOutcome.SelectorVersion) + + forced, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: optimize.ShortestPathExecutorS3Unidirectional}) + require.NoError(t, err) + forcedSQL, err := Translated(forced) + require.NoError(t, err) + + require.Equal(t, incumbentSQL, forcedSQL) + require.Contains(t, forcedSQL, "with recursive") + require.Contains(t, forcedSQL, "s1(next_id, depth)") + require.NotContains(t, forcedSQL, "s1(root_id, next_id, depth)") + require.Contains(t, forcedSQL, "select singleton_endpoints.root_id, 0 from singleton_endpoints") + require.Contains(t, forcedSQL, "(select singleton_endpoints.root_id from singleton_endpoints) as n0") + require.NotContains(t, forcedSQL, "sp_harness") + require.NotContains(t, forcedSQL, "path)") + require.NotContains(t, forcedSQL, "is_cycle") + require.NotContains(t, forcedSQL, "cardinality") + require.Contains(t, forcedSQL, "order by") + require.Contains(t, forcedSQL, "depth limit 1") + require.NotContains(t, forcedSQL, "join node") + + outcome := requireTraversalTargetOutcome(t, forced.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, string(optimize.ShortestPathExecutorS3Unidirectional), outcome.Selected) + require.Equal(t, string(optimize.ShortestPathExecutorS3Unidirectional), outcome.Applied) + require.Equal(t, "forced_tool", outcome.SelectionMode) + require.Empty(t, outcome.SkipReason) + requireOptimizationLowering(t, forced.Optimization, optimize.LoweringShortestPathExecutor) + requireNoSkippedOptimizationLowering(t, forced.Optimization, optimize.LoweringShortestPathExecutor) +} + +// TestForcedShortestIncumbentEmitsExactWorkspaceHarness verifies that forcing the incumbent preserves its workspace-table harness. +func TestForcedShortestIncumbentEmitsExactWorkspaceHarness(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `) + require.NoError(t, err) + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ + ForceShortestPathExecutor: optimize.ShortestPathExecutorIncumbentWorkspace, + }) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "sp_harness") + require.NotContains(t, formatted, "s1(next_id, depth)") + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, string(optimize.ShortestPathExecutorIncumbentWorkspace), outcome.Selected) + require.Equal(t, string(optimize.ShortestPathExecutorIncumbentWorkspace), outcome.Applied) + require.Equal(t, "forced_tool", outcome.SelectionMode) +} + +// TestForcedShortestDirectPreflightGatesWorkspaceFallback verifies that direct preflight gates the incumbent workspace branch. +func TestForcedShortestDirectPreflightGatesWorkspaceFallback(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((e)<-[:MemberOf|SuffixEdgeOne*1..8]-(s)) + WHERE id(e) = $end_id AND id(s) = $start_id + RETURN p + `) + require.NoError(t, err) + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ + ForceShortestPathExecutor: optimize.ShortestPathExecutorS0Direct, + }) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "direct_shortest(root_id, next_id, depth, satisfied, is_cycle, path) as materialized") + require.Contains(t, formatted, "fallback_endpoints as (select * from singleton_endpoints where not exists") + require.Contains(t, formatted, "workspace_shortest(root_id, next_id, depth, satisfied, is_cycle, path)") + require.Contains(t, formatted, "from fallback_endpoints, bidirectional_sp_harness") + require.Contains(t, formatted, "select * from direct_shortest union all select * from workspace_shortest") + + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, string(optimize.ShortestPathExecutorS0Direct), outcome.Selected) + require.Equal(t, string(optimize.ShortestPathExecutorS0Direct), outcome.Applied) + require.Equal(t, "forced_tool", outcome.SelectionMode) +} + +// TestForcedShortestDirectPreflightRejectsZeroMinimumDepth verifies that forcing cannot bypass the direct executor's positive-depth requirement. +func TestForcedShortestDirectPreflightRejectsZeroMinimumDepth(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*0..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + + _, err = TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ + ForceShortestPathExecutor: optimize.ShortestPathExecutorS0Direct, + }) + require.ErrorContains(t, err, "no structurally eligible depth-one target") +} + +// TestForcedShortestDirectPreflightRejectsMutation verifies that statement mutation prevents direct shortest-path execution. +func TestForcedShortestDirectPreflightRejectsMutation(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + CREATE (:Group) + RETURN p + `) + require.NoError(t, err) + + _, err = TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ + ForceShortestPathExecutor: optimize.ShortestPathExecutorS0Direct, + }) + require.ErrorContains(t, err, "no structurally eligible depth-one target") +} + +// TestForcedShortestDirectPreflightPreservesPathThroughWithAlias verifies that a path witness survives aliasing across WITH. +func TestForcedShortestDirectPreflightPreservesPathThroughWithAlias(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + WITH p AS q + RETURN q + `) + require.NoError(t, err) + + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ + ForceShortestPathExecutor: optimize.ShortestPathExecutorS0Direct, + }) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "direct_shortest") + require.Contains(t, formatted, "ordered_edge_ids_to_path") + require.Contains(t, formatted, "as q") +} + +// TestForcedShortestDistanceExecutorRejectsIneligibleObservation verifies that a path consumer cannot force distance-only execution. +func TestForcedShortestDistanceExecutorRejectsIneligibleObservation(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + + _, err = TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: optimize.ShortestPathExecutorS3Unidirectional}) + require.ErrorContains(t, err, "no structurally eligible distance-only target") +} + +// TestForcedShortestPathEdgeM0ExecutorEmitsNativeEdgeTrailAndMaterializer verifies ordered edge-trail state and deferred path hydration. +func TestForcedShortestPathEdgeM0ExecutorEmitsNativeEdgeTrailAndMaterializer(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + + incumbent, err := Translate(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID) + require.NoError(t, err) + incumbentSQL, err := Translated(incumbent) + require.NoError(t, err) + productionOutcome := requireTraversalTargetOutcome(t, incumbent.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, string(optimize.ShortestPathExecutorS3EdgeM0), productionOutcome.Selected) + require.Equal(t, string(optimize.ShortestPathExecutorS3EdgeM0), productionOutcome.Applied) + require.Equal(t, "static", productionOutcome.SelectionMode) + require.Equal(t, "sp-static-v5-contained", productionOutcome.SelectorVersion) + require.NotContains(t, incumbentSQL, "shortest_path_compact") + + forced, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: optimize.ShortestPathExecutorS3EdgeM0}) + require.NoError(t, err) + forcedSQL, err := Translated(forced) + require.NoError(t, err) + + require.Equal(t, incumbentSQL, forcedSQL, "forcing the contained S3 winner must reproduce the default SQL") + require.Contains(t, forcedSQL, "with recursive") + require.Contains(t, forcedSQL, "s1(next_id, depth, path)") + require.Contains(t, forcedSQL, "generate_subscripts(s1.path, 1)") + require.Equal(t, 1, strings.Count(forcedSQL, "generate_subscripts(s1.path, 1)"), forcedSQL) + require.Contains(t, forcedSQL, "array_agg((m0_terminal.id, m0_terminal.kind_ids, m0_terminal.properties)::nodecomposite order by m0_path_index)") + require.Contains(t, forcedSQL, "m0_hydrated.hydrated_count = cardinality(s1.path)") + require.Contains(t, forcedSQL, "m0_terminal.id = m0_edge.end_id") + require.Contains(t, forcedSQL, "::pathcomposite") + require.NotContains(t, forcedSQL, "sp_harness") + require.NotContains(t, forcedSQL, "ordered_edge_ids_to_path") + + outcome := requireTraversalTargetOutcome(t, forced.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, string(optimize.ShortestPathExecutorS3EdgeM0), outcome.Selected) + require.Equal(t, string(optimize.ShortestPathExecutorS3EdgeM0), outcome.Applied) + require.Equal(t, "forced_tool", outcome.SelectionMode) + require.Empty(t, outcome.SkipReason) +} + +// TestForcedShortestPathEdgeM0ExecutorIsDirectionAware verifies that edge-trail recursion joins the correct physical endpoint for each direction. +func TestForcedShortestPathEdgeM0ExecutorIsDirectionAware(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((e)<-[:MemberOf*1..8]-(s)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + + forced, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: optimize.ShortestPathExecutorS3EdgeM0}) + require.NoError(t, err) + forcedSQL, err := Translated(forced) + require.NoError(t, err) + + require.Contains(t, forcedSQL, "join edge e0 on e0.end_id = s1.next_id") + require.Contains(t, forcedSQL, "m0_terminal.id = m0_edge.start_id") +} + +// TestForcedShortestPathExecutorsRejectMismatchedObservation verifies tool +// forcing cannot broaden distance and witness observation contracts. +func TestForcedShortestPathExecutorsRejectMismatchedObservation(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `) + require.NoError(t, err) + + _, err = TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: optimize.ShortestPathExecutorS3EdgeM0}) + require.ErrorContains(t, err, "no structurally eligible one-path target") + + for _, executor := range []optimize.ShortestPathExecutor{ + optimize.ShortestPathExecutorB1AlternatingNodeWitness, + optimize.ShortestPathExecutorB2SmallerCurrentLevelWitness, + } { + _, err = TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: executor}) + require.ErrorContains(t, err, "no structurally eligible one-path target") + } + + witnessQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + for _, executor := range []optimize.ShortestPathExecutor{ + optimize.ShortestPathExecutorB1AlternatingNodeDistance, + optimize.ShortestPathExecutorB2SmallerCurrentLevelDistance, + } { + _, err = TranslateForTool(context.Background(), witnessQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: executor}) + require.ErrorContains(t, err, "no structurally eligible distance-only target") + } +} + +// TestForcedShortestPathEdgeM0ExecutorPreservesPathThroughWithAlias verifies that a materialized witness survives aliasing across WITH. +func TestForcedShortestPathEdgeM0ExecutorPreservesPathThroughWithAlias(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + WITH p AS q + RETURN q + `) + require.NoError(t, err) + + forced, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: optimize.ShortestPathExecutorS3EdgeM0}) + require.NoError(t, err) + forcedSQL, err := Translated(forced) + require.NoError(t, err) + + require.Contains(t, forcedSQL, "::pathcomposite") + require.Contains(t, forcedSQL, "as q") + require.NotContains(t, forcedSQL, "ordered_edge_ids_to_path") + + outcome := requireTraversalTargetOutcome(t, forced.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, string(optimize.ShortestPathExecutorS3EdgeM0), outcome.Applied) +} + +// TestForcedShortestDistanceExecutorIsDirectionAwareAndParameterStable verifies physical direction and deterministic parameters for scalar search. +func TestForcedShortestDistanceExecutorIsDirectionAwareAndParameterStable(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((e)<-[:MemberOf*1..8]-(s)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `) + require.NoError(t, err) + + translateForced := func(startID, endID int64) string { + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": startID, "end_id": endID, + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: optimize.ShortestPathExecutorS3Unidirectional}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + return formatted + } + + firstSQL := translateForced(1, 2) + secondSQL := translateForced(100, 200) + require.Equal(t, firstSQL, secondSQL) + require.Contains(t, firstSQL, "select e0.start_id, s1.depth + 1") + require.NotContains(t, firstSQL, "select s1.root_id, e0.start_id, s1.depth + 1") + require.Contains(t, firstSQL, "join edge e0 on e0.end_id = s1.next_id") +} + +// TestForcedShortestDistanceExecutorSupportsZeroDepthWithoutSelfEndpointError verifies legal same-endpoint zero-length paths. +func TestForcedShortestDistanceExecutorSupportsZeroDepthWithoutSelfEndpointError(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*0..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) AS distance + `) + require.NoError(t, err) + + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(1), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: optimize.ShortestPathExecutorS3Unidirectional}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "s1.depth >= 0") + require.NotContains(t, formatted, "shortest_path_self_endpoint_error") + require.Contains(t, formatted, "(s0.ep0)::int as distance") +} + +func TestProductionRejectsUnderGuardedInlineDistanceExecutor(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)<-[:MemberOf*1..64]-(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `) + require.NoError(t, err) + _, err = TranslateWithProductionOptions(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ProductionOptions{ShortestPathExecutor: optimize.ShortestPathExecutorI1CanonicalDistance, SelectorVersion: "sp-i1-canary-v1"}) + require.ErrorContains(t, err, "not production-canary eligible") +} + +func TestProductionInlineWitnessExecutorKeepsEdgeIDsAtMaterializationBoundary(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)<-[:MemberOf*1..64]-(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + translation, err := TranslateWithProductionOptions(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ProductionOptions{ + ShortestPathExecutor: optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, + SelectorVersion: optimize.ShortestPathSelectorStaticV6, + ShortestPathCaps: &ProductionShortestPathCaps{ + StateLimit: 1000, PredecessorLimit: 1000, EnumerationLimit: 1000, OutputBytesLimit: 1 << 20, + }, + AuthorizedBucket: &ProductionTraversalBucket{Direction: "inbound", ObservationMode: "one_path", MinimumDepth: 1, MaximumDepth: 64, RelationshipKindCount: 1}, + }) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "with recursive") + require.Contains(t, formatted, "generate_subscripts(s1.path, 1)") + require.NotContains(t, formatted, "ordered_edge_ids_to_path") + require.Contains(t, formatted, "shortest_path_compact") + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, optimize.TraversalStepTarget{}) + require.Equal(t, string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), outcome.Applied) + require.Equal(t, string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), outcome.Candidate) + require.Equal(t, optimize.ShortestPathPolicyI1CanonicalGuardedV1, outcome.EmittedPolicy) + require.Equal(t, []string{ + string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + string(optimize.ShortestPathExecutorS4CanonicalWitness), + }, outcome.EmittedCandidates) + require.Equal(t, string(optimize.ShortestPathExecutorS4CanonicalWitness), outcome.Fallback) + require.Equal(t, "guarded_dual_arm", outcome.ExecutionBoundary) + require.Equal(t, "production_canary", outcome.SelectionMode) +} + +// TestForcedShortestDistanceExecutorPreservesDistanceThroughWithAlias verifies that scalar distance survives aliasing across WITH. +func TestForcedShortestDistanceExecutorPreservesDistanceThroughWithAlias(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + WITH length(p) AS distance + RETURN distance + `) + require.NoError(t, err) + + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: optimize.ShortestPathExecutorS3Unidirectional}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.NotContains(t, formatted, "cardinality") + require.NotContains(t, formatted, "ordered_edge_ids_to_path") + require.Contains(t, formatted, "::int as i0") + require.Contains(t, formatted, "s0.i0 as distance") +} + +// requireTraversalTargetOutcome returns the diagnostic outcome for one lowering and traversal target. +func requireTraversalTargetOutcome(t *testing.T, summary OptimizationSummary, lowering string, target optimize.TraversalStepTarget) TargetLoweringOutcome { + t.Helper() + + for _, outcome := range summary.TargetOutcomes { + if outcome.Lowering == lowering && outcome.TraversalTarget != nil && *outcome.TraversalTarget == target { + return outcome + } + } + + require.FailNowf(t, "missing target outcome", "lowering %s target %+v", lowering, target) + return TargetLoweringOutcome{} +} + +func TestTraversalEnvelopeAnalysisHasExplicitTargetOutcomes(t *testing.T) { + t.Parallel() + + translation := optimizerSafetyTranslation(t, ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) IN [1, 2] AND id(e) = 3 + AND all(n IN nodes(p) WHERE n.enabled = true) + RETURN p + `) + target := optimize.PatternTarget{QueryPartIndex: 0, ClauseIndex: 0, PatternIndex: 0}.TraversalStep(0) + + endpoint := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringEndpointResolution, target) + require.Equal(t, "endpoint_resolution", endpoint.TargetKind) + require.Equal(t, "endpoint_resolution", endpoint.Family) + require.Equal(t, "SP", endpoint.TraversalFamily) + require.Equal(t, string(optimize.EndpointResolutionPlanBounded), endpoint.Candidate) + require.Equal(t, string(optimize.EndpointResolutionPlanIncumbent), endpoint.Selected) + require.Equal(t, endpoint.Selected, endpoint.Applied) + require.Equal(t, "analysis_only", endpoint.SelectionMode) + require.Equal(t, optimize.EndpointResolutionFallbackPlannedOnly, endpoint.SkipReason) + require.NotNil(t, endpoint.EndpointRoot) + require.Equal(t, optimize.EndpointResolutionClassExplicitSmallSet, endpoint.EndpointRoot.Class) + require.Equal(t, 2, endpoint.EndpointRoot.StaticValueCount) + require.NotNil(t, endpoint.EndpointTerminal) + require.Equal(t, optimize.EndpointResolutionClassIDEquality, endpoint.EndpointTerminal.Class) + require.Equal(t, &optimize.EndpointResolutionCaps{ + SingletonLimit: optimize.EndpointResolutionSingletonLimit, + SingletonSentinel: optimize.EndpointResolutionSingletonSentinel, + SmallSetLimit: optimize.EndpointResolutionSmallSetLimit, + SmallSetSentinel: optimize.EndpointResolutionSmallSetSentinel, + }, endpoint.EndpointResolutionCaps) + + var predicate *TargetLoweringOutcome + for index := range translation.Optimization.TargetOutcomes { + outcome := &translation.Optimization.TargetOutcomes[index] + if outcome.Lowering == optimize.LoweringTraversalPredicateClassification && outcome.PredicateClass == optimize.TraversalPredicateClassUniversalAllNodes { + predicate = outcome + break + } + } + require.NotNil(t, predicate) + require.Equal(t, "traversal_predicate", predicate.TargetKind) + require.Equal(t, string(optimize.TraversalPredicatePlanStep), predicate.Candidate) + require.Equal(t, string(optimize.TraversalPredicatePlanIncumbent), predicate.Selected) + require.Equal(t, predicate.Selected, predicate.Applied) + require.Equal(t, optimize.TraversalPredicateFallbackPlannedOnly, predicate.SkipReason) + require.Equal(t, "analysis_only", predicate.SelectionMode) + require.NotNil(t, predicate.PredicateIndex) +} + +// requireSQLContainsInOrder requires each SQL fragment to occur after the preceding fragment. func requireSQLContainsInOrder(t *testing.T, sql string, parts ...string) { t.Helper() @@ -200,6 +1583,7 @@ func requireSQLContainsInOrder(t *testing.T, sql string, parts ...string) { } } +// TestOptimizerSafetyCountStoreFastPathUsesBaseNodeCount verifies unconstrained node counts use the graph-wide node count source. func TestOptimizerSafetyCountStoreFastPathUsesBaseNodeCount(t *testing.T) { t.Parallel() @@ -209,7 +1593,7 @@ func TestOptimizerSafetyCountStoreFastPathUsesBaseNodeCount(t *testing.T) { requirePlannedOptimizationLowering(t, translation.Optimization, optimize.LoweringCountStoreFastPath) requireOptimizationLowering(t, translation.Optimization, optimize.LoweringCountStoreFastPath) - require.Empty(t, translation.Optimization.SkippedLowerings) + requireSkippedOptimizationLowering(t, translation.Optimization, optimize.LoweringFieldRequirements, "analysis_metadata_only") require.Equal(t, "select count(*)::int8 from node n0;", strings.Join(strings.Fields(formattedQuery), " ")) } @@ -250,10 +1634,11 @@ func TestOptimizerSafetyCountStoreFastPathUsesBaseEdgeCount(t *testing.T) { require.Equal(t, "select count(*)::int8 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [10]::int2[]);", strings.Join(strings.Fields(formattedQuery), " ")) } +// TestOptimizerSafetyCountStoreFastPathUsesSparseEdgeKindCount verifies a typed edge count reads only the selected kind's sparse count. func TestOptimizerSafetyCountStoreFastPathUsesSparseEdgeKindCount(t *testing.T) { t.Parallel() - translation := optimizerSafetyTranslation(t, `MATCH ()-[r:Enroll]->() RETURN count(r)`) + translation := optimizerSafetyTranslation(t, `MATCH ()-[r:SuffixEdgeOne]->() RETURN count(r)`) formattedQuery, err := Translated(translation) require.NoError(t, err) normalizedQuery := strings.Join(strings.Fields(formattedQuery), " ") @@ -295,10 +1680,11 @@ func TestOptimizerSafetyCountStoreFastPathSupportsEdgeCountStar(t *testing.T) { require.Equal(t, "select count(*)::int8 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [10]::int2[]);", strings.Join(strings.Fields(formattedQuery), " ")) } -func TestOptimizerSafetyADCSQueryPrunesExpansionEdgeCarry(t *testing.T) { +// TestOptimizerSafetyFixedSuffixQueryPrunesExpansionEdgeCarry verifies that unobserved expansion edges are omitted from recursive state. +func TestOptimizerSafetyFixedSuffixQueryPrunesExpansionEdgeCarry(t *testing.T) { t.Parallel() - translation := optimizerSafetyTranslation(t, optimizerADCSQuery) + translation := optimizerSafetyTranslation(t, optimizerFixedSuffixQuery) formattedQuery, err := Translated(translation) require.NoError(t, err) normalizedQuery := strings.Join(strings.Fields(formattedQuery), " ") @@ -315,13 +1701,15 @@ func TestOptimizerSafetyADCSQueryPrunesExpansionEdgeCarry(t *testing.T) { require.Contains(t, normalizedQuery, "select distinct (s9.n2).id as root_id from s9") require.Contains(t, normalizedQuery, "s5.ep0 as ep0") require.NotContains(t, normalizedQuery, "s5.e0 as e0") - require.Contains(t, normalizedQuery, "from unnest(s12.ep0)") - require.Contains(t, normalizedQuery, "from unnest(array [s12.e1]::int8[])") + require.Contains(t, normalizedQuery, "ordered_edge_ids_to_path(0, s12.n0, s12.ep0 || array [s12.e1]::int8[] || array [s12.e2]::int8[] || array [s12.e3]::int8[]") + require.Equal(t, 2, strings.Count(normalizedQuery, "ordered_edge_ids_to_path("), normalizedQuery) + require.NotContains(t, normalizedQuery, "ordered_edges_to_path(") + require.NotContains(t, normalizedQuery, "from unnest(") require.NotContains(t, normalizedQuery, "array [s12.e1]::edgecomposite[]") require.Contains(t, normalizedQuery, "from s5, s7") requireSQLContainsInOrder(t, normalizedQuery, "where s7.satisfied and exists (select 1 from edge e5 join node n6", - "properties -> 'authenticationenabled'", + "properties -> 'eligible'", "join edge e6 on n6.id = e6.start_id", "e6.end_id = (s5.n2).id", "and (s5.n0).id = s7.root_id", @@ -333,6 +1721,7 @@ func TestOptimizerSafetyADCSQueryPrunesExpansionEdgeCarry(t *testing.T) { ) } +// assertOptimizerSafetyRelationshipStaysComposite requires a relationship consumer to retain composite rather than scalar-ID state. func assertOptimizerSafetyRelationshipStaysComposite(t *testing.T, cypherQuery string) { t.Helper() @@ -433,13 +1822,58 @@ RETURN p requireOptimizationLowering(t, translation.Optimization, "ExpandIntoDetection") } +func TestOptimizerSafetyFixedHopExpandIntoPreservesCarriedOuterMultiplicity(t *testing.T) { + t.Parallel() + + normalizedQuery := optimizerSafetySQL(t, ` + MATCH (a:Group), (b:User) + WITH a, b, [1, 2] AS copies + UNWIND copies AS copy + MATCH (a)-[:MemberOf|AdminTo]->(b) + RETURN copy + `) + + require.Contains(t, normalizedQuery, "from s0 join edge e0 on (s0.n0).id = e0.start_id and (s0.n1).id = e0.end_id, unnest(i0) as i1") + require.NotContains(t, normalizedQuery, "join node") +} + +func TestOptimizerSafetyFixedHopExpandIntoScopesNodeUnwindBeforePairPredicate(t *testing.T) { + t.Parallel() + + normalizedQuery := optimizerSafetySQL(t, ` + MATCH (a:Group), (b:User) + WITH collect(a) AS sources, b + UNWIND sources AS source + MATCH (source)-[:MemberOf]->(b) + RETURN source + `) + + require.Contains(t, normalizedQuery, "from s0, edge e0, unnest(i0) as i1 where") + require.Contains(t, normalizedQuery, "i1.id = e0.start_id and (s0.n1).id = e0.end_id") + require.NotContains(t, normalizedQuery, "join edge e0 on i1.id") +} + +func TestOptimizerSafetyDirectionlessExpandIntoUsesPairwiseEndpoints(t *testing.T) { + t.Parallel() + + normalizedQuery := optimizerSafetySQL(t, ` + MATCH (a:Group), (b:User) + MATCH (a)-[:MemberOf]-(b) + RETURN a, b + `) + + require.Contains(t, normalizedQuery, "(((s1.n0).id = e0.start_id and (s1.n1).id = e0.end_id) or ((s1.n1).id = e0.start_id and (s1.n0).id = e0.end_id))") + require.NotContains(t, normalizedQuery, "(s1.n0).id <> (s1.n1).id") +} + +// TestOptimizerSafetyReordersIndependentNodeAnchor verifies an independent selective node can become the traversal anchor without changing semantics. func TestOptimizerSafetyReordersIndependentNodeAnchor(t *testing.T) { t.Parallel() var ( normalizedQuery = optimizerSafetySQL(t, ` MATCH (a) - MATCH (b:EnterpriseCA {name: 'target'}) + MATCH (b:SuffixNodeOne {name: 'target'}) MATCH p = (a)-[:MemberOf]->(b) RETURN p `) @@ -454,11 +1888,12 @@ func TestOptimizerSafetyReordersIndependentNodeAnchor(t *testing.T) { require.Contains(t, normalizedQuery, "(s1.n0).id = e0.end_id") } +// TestOptimizerSafetyExpansionTerminalPushdownForFixedSuffix verifies an eligible fixed suffix is pushed into terminal expansion filtering. func TestOptimizerSafetyExpansionTerminalPushdownForFixedSuffix(t *testing.T) { t.Parallel() normalizedQuery := optimizerSafetySQL(t, ` -MATCH p = (n:Group)-[:MemberOf*1..]->(m)-[:Enroll]->(ca:EnterpriseCA) +MATCH p = (n:Group)-[:MemberOf*1..]->(m)-[:SuffixEdgeOne]->(ca:SuffixNodeOne) RETURN p `) @@ -468,11 +1903,12 @@ RETURN p require.Contains(t, normalizedQuery, "n2.kind_ids operator (pg_catalog.@>) array [5]::int2[]") } +// TestOptimizerSafetySuffixPredicatePlacementStaysInsideTerminalExists verifies suffix predicates remain scoped to the terminal existence check. func TestOptimizerSafetySuffixPredicatePlacementStaysInsideTerminalExists(t *testing.T) { t.Parallel() normalizedQuery := optimizerSafetySQL(t, ` -MATCH p = (n:Group)-[:MemberOf*1..]->(m)-[:Enroll]->(ca:EnterpriseCA) +MATCH p = (n:Group)-[:MemberOf*1..]->(m)-[:SuffixEdgeOne]->(ca:SuffixNodeOne) WHERE ca.name = 'target' RETURN p `) @@ -484,11 +1920,12 @@ RETURN p ) } +// TestOptimizerSafetyPredicatePlacementRecordsExpansionRootConstraint verifies root predicates are recorded and emitted at the expansion root. func TestOptimizerSafetyPredicatePlacementRecordsExpansionRootConstraint(t *testing.T) { t.Parallel() translation := optimizerSafetyTranslation(t, ` -MATCH p = (src:Group)-[:MemberOf*1..]->(mid)-[:Enroll]->(ca:EnterpriseCA) +MATCH p = (src:Group)-[:MemberOf*1..]->(mid)-[:SuffixEdgeOne]->(ca:SuffixNodeOne) WHERE src.name = 'source' RETURN p `) @@ -547,11 +1984,12 @@ RETURN s requireOptimizationLowering(t, translation.Optimization, "PredicatePlacement") } +// TestOptimizerSafetyContinuationRelationshipsExcludePriorPathRelationships verifies suffix traversal cannot reuse relationships from the expanded prefix. func TestOptimizerSafetyContinuationRelationshipsExcludePriorPathRelationships(t *testing.T) { t.Parallel() expandedPrefixQuery := optimizerSafetySQL(t, ` -MATCH p = (n:Group)-[:MemberOf*1..]->(m)-[:Enroll]-(ca:EnterpriseCA) +MATCH p = (n:Group)-[:MemberOf*1..]->(m)-[:SuffixEdgeOne]-(ca:SuffixNodeOne) RETURN p `) @@ -559,18 +1997,19 @@ RETURN p require.Contains(t, expandedPrefixQuery, "ep0") fixedPrefixQuery := optimizerSafetySQL(t, ` -MATCH p = (n:Group)-[:MemberOf]->(m)-[:Enroll]->(ca:EnterpriseCA) +MATCH p = (n:Group)-[:MemberOf]->(m)-[:SuffixEdgeOne]->(ca:SuffixNodeOne) RETURN p `) require.Contains(t, fixedPrefixQuery, "e1.id != s0.e0") } +// TestOptimizerSafetyDirectionBalancedExpansionDoesNotPlanStaleSuffixPushdown verifies reoriented traversal targets do not retain obsolete suffix decisions. func TestOptimizerSafetyDirectionBalancedExpansionDoesNotPlanStaleSuffixPushdown(t *testing.T) { t.Parallel() translation := optimizerSafetyTranslation(t, ` -MATCH p = (n)-[:MemberOf*1..]->(ca:EnterpriseCA)-[:TrustedForNTAuth]->(d:Domain) +MATCH p = (n)-[:MemberOf*1..]->(ca:SuffixNodeOne)-[:SuffixEdgeTwo]->(d:Domain) RETURN p `) @@ -643,11 +2082,12 @@ RETURN p require.Contains(t, normalizedQuery, "array [") } +// TestOptimizerSafetyExactTwoHopRangePreservesLaterSourceStepTargets verifies exact-range expansion does not renumber later source-step decisions. func TestOptimizerSafetyExactTwoHopRangePreservesLaterSourceStepTargets(t *testing.T) { t.Parallel() translation := optimizerSafetyTranslation(t, ` -MATCH (a)-[:MemberOf*2..2]->(b)-[:Enroll]->(c) +MATCH (a)-[:MemberOf*2..2]->(b)-[:SuffixEdgeOne]->(c) RETURN a `) formattedQuery, err := Translated(translation) @@ -656,11 +2096,12 @@ RETURN a requirePlannedOptimizationLowering(t, translation.Optimization, optimize.LoweringExactRangeExpansion) requireOptimizationLowering(t, translation.Optimization, optimize.LoweringExactRangeExpansion) - require.Contains(t, normalizedQuery, "on (s1.n2).id = e2.start_id") + require.Contains(t, normalizedQuery, "on s1.n2 = e2.start_id") require.NotContains(t, normalizedQuery, "on n2.id = e2.start_id") } -func TestOptimizerSafetyExactTwoHopRangeKeepsSyntheticIntermediateNode(t *testing.T) { +// TestOptimizerSafetyExactTwoHopRangeCarriesSyntheticIntermediateNodeID verifies that exact-range lowering retains the intermediate join identity. +func TestOptimizerSafetyExactTwoHopRangeCarriesSyntheticIntermediateNodeID(t *testing.T) { t.Parallel() normalizedQuery := strings.ToLower(optimizerSafetySQL(t, ` @@ -668,15 +2109,16 @@ MATCH (a)-[:MemberOf*2..2]->(b) RETURN a `)) - require.Contains(t, normalizedQuery, "on (s0.n1).id = e1.start_id") + require.Contains(t, normalizedQuery, "on s0.n1 = e1.start_id") require.NotContains(t, normalizedQuery, "on n1.id = e1.start_id") } +// TestOptimizerSafetyConsecutiveExactRangesUseSourceStepTargets verifies consecutive expansions retain their original source-step coordinates. func TestOptimizerSafetyConsecutiveExactRangesUseSourceStepTargets(t *testing.T) { t.Parallel() translation := optimizerSafetyTranslation(t, ` -MATCH p = (a)-[:MemberOf*2..2]->(b)-[:Enroll*1..1]->(c) +MATCH p = (a)-[:MemberOf*2..2]->(b)-[:SuffixEdgeOne*1..1]->(c) RETURN p `) formattedQuery, err := Translated(translation) @@ -691,11 +2133,12 @@ RETURN p require.Contains(t, normalizedQuery, "join edge e2") } +// TestOptimizerSafetyExactRangePrefixPreservesSuffixPushdownTargets verifies prefix expansion leaves fixed-suffix decisions keyed to source coordinates. func TestOptimizerSafetyExactRangePrefixPreservesSuffixPushdownTargets(t *testing.T) { t.Parallel() translation := optimizerSafetyTranslation(t, ` -MATCH p = (a)-[:MemberOf*2..2]->(b)-[:AdminTo*1..]->(c)-[:Enroll]->(d) +MATCH p = (a)-[:MemberOf*2..2]->(b)-[:AdminTo*1..]->(c)-[:SuffixEdgeOne]->(d) RETURN p `) @@ -1249,11 +2692,12 @@ func TestOptimizerSafetyShortestPathTerminalCarriesUnwindSources(t *testing.T) { requirePlanParameterContains(t, translation, "(n1.properties ->> 'name') = i0") } +// TestOptimizerSafetyTranslationReportsOptimizerMetadata verifies translation reports planned, applied, and targeted lowering diagnostics. func TestOptimizerSafetyTranslationReportsOptimizerMetadata(t *testing.T) { t.Parallel() regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` -MATCH p = (n:Group)-[:MemberOf*1..]->(m)-[:Enroll]->(ca:EnterpriseCA) +MATCH p = (n:Group)-[:MemberOf*1..]->(m)-[:SuffixEdgeOne]->(ca:SuffixNodeOne) WHERE ca.name = 'target' RETURN p `) @@ -1279,11 +2723,12 @@ RETURN p requireOptimizationLowering(t, translation.Optimization, "PredicatePlacement") } +// TestOptimizerSafetyExpansionTerminalPushdownForZeroDepthExpansion verifies terminal filtering preserves the zero-edge expansion alternative. func TestOptimizerSafetyExpansionTerminalPushdownForZeroDepthExpansion(t *testing.T) { t.Parallel() normalizedQuery := optimizerSafetySQL(t, ` -MATCH p = (n:Group)-[:MemberOf*0..]->(m)-[:Enroll]->(ca:EnterpriseCA) +MATCH p = (n:Group)-[:MemberOf*0..]->(m)-[:SuffixEdgeOne]->(ca:SuffixNodeOne) RETURN p `) @@ -1293,12 +2738,13 @@ RETURN p require.Contains(t, normalizedQuery, "n2.kind_ids operator (pg_catalog.@>) array [5]::int2[]") } +// TestOptimizerSafetyExpansionTerminalPushdownForBoundEndpointSuffixChain verifies a bound suffix endpoint is honored inside supplemental search. func TestOptimizerSafetyExpansionTerminalPushdownForBoundEndpointSuffixChain(t *testing.T) { t.Parallel() normalizedQuery := optimizerSafetySQL(t, ` -MATCH (ca:EnterpriseCA {name: 'target'}) -MATCH p = (n:Group)-[:MemberOf*0..]->(m)-[:Enroll]->(ct:CertTemplate)-[:PublishedTo]->(ca) +MATCH (ca:SuffixNodeOne {name: 'target'}) +MATCH p = (n:Group)-[:MemberOf*0..]->(m)-[:SuffixEdgeOne]->(ct:CertTemplate)-[:PublishedTo]->(ca) WHERE ct.authenticationenabled = true RETURN p `) @@ -1318,12 +2764,13 @@ RETURN p ) } +// TestOptimizerSafetyExpansionTerminalPushdownIncludesConstrainedBoundEndpoint verifies bound-endpoint predicates are included in terminal filtering. func TestOptimizerSafetyExpansionTerminalPushdownIncludesConstrainedBoundEndpoint(t *testing.T) { t.Parallel() translation := optimizerSafetyTranslation(t, ` MATCH (ca) -MATCH p = (n:Group)-[:MemberOf*0..]->(m)-[:Enroll]->(ct:CertTemplate)-[:PublishedTo]->(ca:EnterpriseCA) +MATCH p = (n:Group)-[:MemberOf*0..]->(m)-[:SuffixEdgeOne]->(ct:CertTemplate)-[:PublishedTo]->(ca:SuffixNodeOne) RETURN p `) formattedQuery, err := Translated(translation) @@ -1340,12 +2787,13 @@ RETURN p require.Contains(t, normalizedQuery, "(s0.n0).kind_ids operator (pg_catalog.@>)") } +// TestOptimizerSafetyExpansionTerminalPushdownForBoundDomainSuffix verifies domain-bound suffix nodes remain constrained during supplemental search. func TestOptimizerSafetyExpansionTerminalPushdownForBoundDomainSuffix(t *testing.T) { t.Parallel() normalizedQuery := optimizerSafetySQL(t, ` MATCH (d:Domain {name: 'target'}) -MATCH p = (ca:EnterpriseCA)-[:IssuedSignedBy|EnterpriseCAFor*1..]->(root:RootCA)-[:RootCAFor]->(d) +MATCH p = (ca:SuffixNodeOne)-[:IssuedSignedBy|SuffixNodeOneFor*1..]->(root:RootCA)-[:RootCAFor]->(d) RETURN p `) @@ -1356,11 +2804,12 @@ RETURN p require.Contains(t, normalizedQuery, "e1.end_id = (s0.n0).id") } +// TestOptimizerSafetyExpansionTerminalPushdownForInboundFixedSuffix verifies inbound suffix direction is preserved in terminal filtering. func TestOptimizerSafetyExpansionTerminalPushdownForInboundFixedSuffix(t *testing.T) { t.Parallel() normalizedQuery := optimizerSafetySQL(t, ` -MATCH p = (ca:EnterpriseCA)<-[:PublishedTo*1..]-(ct)<-[:Enroll]-(m:Group) +MATCH p = (ca:SuffixNodeOne)<-[:PublishedTo*1..]-(ct)<-[:SuffixEdgeOne]-(m:Group) RETURN p `) @@ -1370,11 +2819,12 @@ RETURN p require.Contains(t, normalizedQuery, "n2.kind_ids operator (pg_catalog.@>)") } +// TestOptimizerSafetyExpansionTerminalPushdownSkipsDirectionlessSuffix verifies undirected suffixes are excluded from terminal-filter pushdown. func TestOptimizerSafetyExpansionTerminalPushdownSkipsDirectionlessSuffix(t *testing.T) { t.Parallel() normalizedQuery := optimizerSafetySQL(t, ` -MATCH p = (n:Group)-[:MemberOf*1..]->(m)-[:Enroll]-(ca:EnterpriseCA) +MATCH p = (n:Group)-[:MemberOf*1..]->(m)-[:SuffixEdgeOne]-(ca:SuffixNodeOne) RETURN p `) diff --git a/cypher/models/pgsql/translate/path_functions.go b/cypher/models/pgsql/translate/path_functions.go index ad2e77e9..1aec05b0 100644 --- a/cypher/models/pgsql/translate/path_functions.go +++ b/cypher/models/pgsql/translate/path_functions.go @@ -6,6 +6,7 @@ import ( "github.com/specterops/dawgs/cypher/models/pgsql" ) +// pathCompositeEdgesExpression returns the edge-array expression represented by a path binding. func pathCompositeEdgesExpression(scope *Scope, pathBinding *BoundIdentifier) (pgsql.Expression, error) { var edgeArrayReferences []pgsql.Expression @@ -40,6 +41,7 @@ func pathCompositeEdgesExpression(scope *Scope, pathBinding *BoundIdentifier) (p return pgsql.ArrayLiteral{CastType: pgsql.EdgeCompositeArray}, nil } +// pathCompositeEdgeIDArrayExpression returns ordered edge IDs from any supported carried path representation. func pathCompositeEdgeIDArrayExpression(scope *Scope, pathBinding *BoundIdentifier) (pgsql.Expression, error) { var edgeIDArrayReferences []pgsql.Expression @@ -78,6 +80,7 @@ func pathCompositeEdgeIDArrayExpression(scope *Scope, pathBinding *BoundIdentifi }, nil } +// buildPathEdgeIDArrayFutures records deferred replacements for path edge-ID references in a query part. func (s *Translator) buildPathEdgeIDArrayFutures() error { for _, future := range s.query.CurrentPart().pathEdgeIDArrayFutures { if edgeIDArrayExpression, err := pathCompositeEdgeIDArrayExpression(s.scope, future.Data); err != nil { @@ -90,6 +93,7 @@ func (s *Translator) buildPathEdgeIDArrayFutures() error { return nil } +// resolvePathCompositeFieldReference replaces a deferred path field with the expression that materializes it. func resolvePathCompositeFieldReference(scope *Scope, reference pgsql.RowColumnReference) (pgsql.Expression, bool, error) { identifier, isIdentifier := unwrapParenthetical(reference.Identifier).(pgsql.Identifier) if !isIdentifier { @@ -129,6 +133,7 @@ func resolvePathCompositeFieldReference(scope *Scope, reference pgsql.RowColumnR } } +// resolvePathCompositeFieldReferencesInProjection resolves deferred path fields in every projection item. func resolvePathCompositeFieldReferencesInProjection(scope *Scope, projection pgsql.Projection) (pgsql.Projection, error) { rewritten := make(pgsql.Projection, len(projection)) @@ -155,6 +160,7 @@ func resolvePathCompositeFieldReferencesInProjection(scope *Scope, projection pg return rewritten, nil } +// resolvePathCompositeFieldReferencesInFromClause resolves deferred path fields in a source and its join constraints. func resolvePathCompositeFieldReferencesInFromClause(scope *Scope, fromClause pgsql.FromClause) (pgsql.FromClause, error) { if resolvedSource, err := resolvePathCompositeFieldReferences(scope, fromClause.Source); err != nil { return pgsql.FromClause{}, err @@ -183,6 +189,7 @@ func resolvePathCompositeFieldReferencesInFromClause(scope *Scope, fromClause pg return fromClause, nil } +// resolvePathCompositeFieldReferencesInFromClauses resolves deferred path fields across all query sources. func resolvePathCompositeFieldReferencesInFromClauses(scope *Scope, fromClauses []pgsql.FromClause) ([]pgsql.FromClause, error) { rewritten := make([]pgsql.FromClause, len(fromClauses)) @@ -198,6 +205,7 @@ func resolvePathCompositeFieldReferencesInFromClauses(scope *Scope, fromClauses return rewritten, nil } +// resolvePathCompositeFieldReferences walks a query part and substitutes every recorded path-field future. func resolvePathCompositeFieldReferences(scope *Scope, expression pgsql.Expression) (pgsql.Expression, error) { switch typedExpression := expression.(type) { case nil: @@ -250,6 +258,16 @@ func resolvePathCompositeFieldReferences(scope *Scope, expression pgsql.Expressi typedExpression.Parameters[idx] = resolved } } + for _, orderBy := range typedExpression.OrderBy { + if orderBy == nil { + continue + } + resolved, err := resolvePathCompositeFieldReferences(scope, orderBy.Expression) + if err != nil { + return nil, err + } + orderBy.Expression = resolved + } return typedExpression, nil diff --git a/cypher/models/pgsql/translate/pattern.go b/cypher/models/pgsql/translate/pattern.go index a77d03ce..ebb8531c 100644 --- a/cypher/models/pgsql/translate/pattern.go +++ b/cypher/models/pgsql/translate/pattern.go @@ -1,8 +1,11 @@ package translate import ( + "fmt" + "github.com/specterops/dawgs/cypher/models/cypher" "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" ) type BindingResult struct { @@ -10,6 +13,7 @@ type BindingResult struct { AlreadyBound bool } +// bindPatternExpression binds a completed traversal result to its pattern variable when one was declared. func (s *Translator) bindPatternExpression(cypherExpression cypher.Expression, dataType pgsql.DataType) (BindingResult, error) { if cypherBinding, hasCypherBinding, err := extractIdentifierFromCypherExpression(cypherExpression); err != nil { return BindingResult{}, err @@ -32,6 +36,7 @@ func (s *Translator) bindPatternExpression(cypherExpression cypher.Expression, d } } +// translatePatternPart dispatches shortest-path, variable-expansion, and fixed traversal patterns to their builders. func (s *Translator) translatePatternPart(patternPart *cypher.PatternPart) error { // We expect this to be a node select if there aren't enough pattern elements for a traversal newPatternPart := s.query.CurrentPart().currentPattern.NewPart() @@ -60,6 +65,7 @@ func (s *Translator) translatePatternPart(patternPart *cypher.PatternPart) error return nil } +// buildPatternPart finalizes a translated pattern part and exports its visible bindings. func (s *Translator) buildPatternPart(part *PatternPart) error { if part.IsTraversal { return s.buildTraversalPatternPart(part) @@ -68,6 +74,7 @@ func (s *Translator) buildPatternPart(part *PatternPart) error { } } +// buildTraversalPattern emits fixed traversal steps and applies any exact-range unrolling decisions. func (s *Translator) buildTraversalPattern(traversalStep *TraversalStep, isRootStep bool) error { if isRootStep { if traversalStepQuery, err := s.buildTraversalPatternRoot(traversalStep.Frame, traversalStep); err != nil { @@ -101,6 +108,7 @@ func (s *Translator) buildTraversalPattern(traversalStep *TraversalStep, isRootS return nil } +// buildExpansionPattern emits an ordinary variable expansion and any qualified specialized-search rewrite. func (s *Translator) buildExpansionPattern(traversalStepContext TraversalStepContext, expansion *ExpansionBuilder) error { traversalStep := traversalStepContext.CurrentStep @@ -131,6 +139,7 @@ func (s *Translator) buildExpansionPattern(traversalStepContext TraversalStepCon return nil } +// buildShortestPathsExpansionPattern emits the selected shortest-path executor and its projection frame. func (s *Translator) buildShortestPathsExpansionPattern(traversalStepContext TraversalStepContext, expansion *ExpansionBuilder, allPaths bool) error { traversalStep := traversalStepContext.CurrentStep @@ -138,7 +147,34 @@ func (s *Translator) buildShortestPathsExpansionPattern(traversalStepContext Tra expansion.SetUnwindClauses(s.query.CurrentPart().ConsumeUnwindClauses()) if allPaths { - if traversalStep.Expansion.UseBidirectionalSearch { + if compactShortestExecutor(traversalStep.Expansion.ShortestPathExecutor) { + var ( + traversalStepQuery pgsql.Query + err error + ) + switch traversalStep.Expansion.ShortestPathExecutor { + case optimize.ShortestPathExecutorASPA1DAG: + traversalStepQuery, err = expansion.BuildAllShortestPathsDAGRoot() + case optimize.ShortestPathExecutorASPI1DAG: + traversalStepQuery, err = expansion.BuildInlineAllShortestPathsDAGRoot() + case optimize.ShortestPathExecutorASPB1AlternatingNodeDAG: + traversalStepQuery, err = expansion.BuildB1AllShortestPathsDAGRoot() + case optimize.ShortestPathExecutorASPB2SmallerCurrentLevelDAG: + traversalStepQuery, err = expansion.BuildB2AllShortestPathsDAGRoot() + default: + err = fmt.Errorf("compact executor %q does not implement all-shortest-path enumeration", traversalStep.Expansion.ShortestPathExecutor) + } + if err != nil { + return err + } + s.recordShortestPathExecutor(traversalStep.Expansion.ShortestPathTarget, traversalStep.Expansion.ShortestPathExecutor) + s.query.CurrentPart().Model.AddCTE(pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: traversalStep.Frame.Binding.Identifier, + }, + Query: traversalStepQuery, + }) + } else if traversalStep.Expansion.UseBidirectionalSearch { if traversalStepQuery, err := expansion.BuildBiDirectionalAllShortestPathsRoot(); err != nil { return err } else { @@ -165,7 +201,21 @@ func (s *Translator) buildShortestPathsExpansionPattern(traversalStepContext Tra err error ) - if traversalStep.Expansion.UseBidirectionalSearch { + if traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorS3Unidirectional || traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalDistance { + traversalStepQuery, err = expansion.BuildShortestDistanceRoot() + } else if traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorS3EdgeM0 || traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalWitness { + traversalStepQuery, err = expansion.BuildShortestPathEdgeM0Root() + } else if traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness { + traversalStepQuery, err = expansion.BuildInlineCanonicalShortestPathRoot() + } else if traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorB1AlternatingNodeDistance || traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorB1AlternatingNodeWitness { + traversalStepQuery, err = expansion.BuildB1CompactShortestPathRoot() + } else if traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorB2SmallerCurrentLevelDistance || traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorB2SmallerCurrentLevelWitness { + traversalStepQuery, err = expansion.BuildB2CompactShortestPathRoot() + } else if traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorS4CanonicalDistance || traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorS4CanonicalWitness { + traversalStepQuery, err = expansion.BuildCompactShortestPathRoot() + } else if traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorS0Direct { + traversalStepQuery, err = expansion.BuildBiDirectionalShortestPathsRootWithDirectPreflight() + } else if traversalStep.Expansion.UseBidirectionalSearch { traversalStepQuery, err = expansion.BuildBiDirectionalShortestPathsRoot() } else { traversalStepQuery, err = expansion.BuildShortestPathsRoot() @@ -174,6 +224,10 @@ func (s *Translator) buildShortestPathsExpansionPattern(traversalStepContext Tra if err != nil { return err } + if traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorS3Unidirectional || traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorS3EdgeM0 || traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalDistance || traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalWitness || traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness || compactShortestExecutor(traversalStep.Expansion.ShortestPathExecutor) || traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorS0Direct || + (traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorIncumbentWorkspace && decisionIsForcedShortest(s, traversalStep.Expansion.ShortestPathTarget)) { + s.recordShortestPathExecutor(traversalStep.Expansion.ShortestPathTarget, traversalStep.Expansion.ShortestPathExecutor) + } s.query.CurrentPart().Model.AddCTE(pgsql.CommonTableExpression{ Alias: pgsql.TableAlias{ @@ -204,7 +258,13 @@ type TraversalStepContext struct { IsRootStep bool } +// buildTraversalPatternPart translates all steps in a non-expanding pattern chain. func (s *Translator) buildTraversalPatternPart(part *PatternPart) error { + firstCTE := len(s.query.CurrentPart().Model.CommonTableExpressions.Expressions) + fixedSuffixDecision, useFixedSuffixStrategy := selectedFixedSuffixDecision(part, s.expansionSearchStrategyDecisions) + guardedSuffixDecision, useGuardedSuffixStrategy := selectedGuardedFixedSuffixDecision(part, s.expansionSearchStrategyDecisions) + endpointSeededDecision, useEndpointSeededStrategy := selectedEndpointSeededDecision(part, s.expansionSearchStrategyDecisions) + for idx, traversalStep := range part.TraversalSteps { var ( isRootStep = idx == 0 @@ -219,7 +279,7 @@ func (s *Translator) buildTraversalPatternPart(part *PatternPart) error { } if traversalStep.Expansion != nil { - if expansion, err := NewExpansionBuilder(s.translation.Parameters, traversalStep); err != nil { + if expansion, err := NewExpansionBuilder(s.translation.Parameters, traversalStep, s.graphID); err != nil { return err } else if part.ShortestPath || part.AllShortestPaths { if err := s.buildShortestPathsExpansionPattern(traversalStepContext, expansion, part.AllShortestPaths); err != nil { @@ -235,5 +295,15 @@ func (s *Translator) buildTraversalPatternPart(part *PatternPart) error { s.allowLimitPushdownForStep(part, idx, traversalStep) } + if useFixedSuffixStrategy { + return s.rewriteTraversalPatternAsSuffixSeededReverse(part, fixedSuffixDecision, firstCTE) + } + if useGuardedSuffixStrategy { + return s.rewriteTraversalPatternAsGuardedSuffixOrientation(part, guardedSuffixDecision, firstCTE) + } + if useEndpointSeededStrategy { + return s.rewriteTraversalPatternAsEndpointSeededReverse(part, endpointSeededDecision, firstCTE) + } + return nil } diff --git a/cypher/models/pgsql/translate/projection.go b/cypher/models/pgsql/translate/projection.go index d83f7a4a..8983b4df 100644 --- a/cypher/models/pgsql/translate/projection.go +++ b/cypher/models/pgsql/translate/projection.go @@ -1,9 +1,11 @@ package translate import ( + "bytes" "fmt" "github.com/specterops/dawgs/cypher/models/cypher" + cypherFormat "github.com/specterops/dawgs/cypher/models/cypher/format" "github.com/specterops/dawgs/cypher/models/walk" "github.com/specterops/dawgs/cypher/models" @@ -11,11 +13,16 @@ import ( "github.com/specterops/dawgs/cypher/models/pgsql/optimize" ) +// BoundProjections pairs rendered select items with the identifiers they carry into the next frame. type BoundProjections struct { - Items pgsql.Projection + // Items contains the SQL expressions emitted by the projection. + Items pgsql.Projection + + // Bindings contains the scope bindings represented by Items. Bindings []*BoundIdentifier } +// rewriteConstraintIdentifierReferences resolves constraint bindings through the preceding projection frame. func rewriteConstraintIdentifierReferences(scope *Scope, frame *Frame, constraints []*Constraint) error { if frame.Previous == nil { return nil @@ -30,6 +37,7 @@ func rewriteConstraintIdentifierReferences(scope *Scope, frame *Frame, constrain return nil } +// buildExternalProjection renders user-visible projection expressions and applies their requested aliases. func buildExternalProjection(scope *Scope, projections []*Projection) (pgsql.Projection, error) { var sqlProjection pgsql.Projection @@ -82,6 +90,7 @@ func buildExternalProjection(scope *Scope, projections []*Projection) (pgsql.Pro return sqlProjection, nil } +// buildInternalProjection renders each distinct bound identifier required by an internal frame. func buildInternalProjection(scope *Scope, projectedBindings []*BoundIdentifier) (BoundProjections, error) { var ( boundProjections = BoundProjections{ @@ -113,6 +122,7 @@ func buildInternalProjection(scope *Scope, projectedBindings []*BoundIdentifier) return boundProjections, nil } +// buildVisibleProjections renders the bindings known to the current scope frame. func buildVisibleProjections(scope *Scope) (BoundProjections, error) { currentFrame := scope.CurrentFrame() @@ -123,7 +133,21 @@ func buildVisibleProjections(scope *Scope) (BoundProjections, error) { } } +// buildProjectionForExpansionPath projects an expansion's distance or accumulated edge-identifier path. func buildProjectionForExpansionPath(alias pgsql.Identifier, projected *BoundIdentifier, scope *Scope, referenceFrame *Frame) ([]pgsql.SelectItem, error) { + if projected.DistanceOnly { + reference := scope.CurrentFrame().Binding.Identifier + column := expansionDepth + if projected.LastProjection != nil { + reference = referenceFrame.Binding.Identifier + column = projected.Identifier + } + return []pgsql.SelectItem{&pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{reference, column}, + Alias: pgsql.AsOptionalIdentifier(alias), + }}, nil + } + if projected.LastProjection != nil { return []pgsql.SelectItem{ &pgsql.AliasedExpression{ @@ -141,6 +165,7 @@ func buildProjectionForExpansionPath(alias pgsql.Identifier, projected *BoundIde }, nil } +// concatenatePathCompositeParts joins ordered path fragments into one array expression. func concatenatePathCompositeParts(parts []pgsql.Expression) pgsql.Expression { if len(parts) == 0 { return nil @@ -154,6 +179,7 @@ func concatenatePathCompositeParts(parts []pgsql.Expression) pgsql.Expression { return joined } +// bindingFrameReference returns the qualified reference to a binding in its latest projection frame. func bindingFrameReference(scope *Scope, binding *BoundIdentifier) pgsql.CompoundIdentifier { frameIdentifier := scope.CurrentFrameBinding().Identifier if binding.LastProjection != nil { @@ -163,6 +189,7 @@ func bindingFrameReference(scope *Scope, binding *BoundIdentifier) pgsql.Compoun return pgsql.CompoundIdentifier{frameIdentifier, binding.Identifier} } +// pathBindingReference resolves a path binding against its latest available frame. func pathBindingReference(scope *Scope, binding *BoundIdentifier) pgsql.Expression { if binding.LastProjection != nil { return pgsql.CompoundIdentifier{binding.LastProjection.Binding.Identifier, binding.Identifier} @@ -175,6 +202,7 @@ func pathBindingReference(scope *Scope, binding *BoundIdentifier) pgsql.Expressi return binding.Identifier } +// pathCompositeReference returns a projected path value or constructs a composite from table columns. func pathCompositeReference(scope *Scope, binding *BoundIdentifier, columns []pgsql.Identifier) pgsql.Expression { if binding.LastProjection != nil || scope.CurrentFrameBinding() != nil { return pathBindingReference(scope, binding) @@ -191,6 +219,7 @@ func pathCompositeReference(scope *Scope, binding *BoundIdentifier, columns []pg } } +// edgeCompositeValue constructs an edge composite from a table alias or row-valued expression. func edgeCompositeValue(expression pgsql.Expression) pgsql.CompositeValue { value := pgsql.CompositeValue{ DataType: pgsql.EdgeComposite, @@ -211,6 +240,7 @@ func edgeCompositeValue(expression pgsql.Expression) pgsql.CompositeValue { return value } +// pathCompositeColumnReference addresses a column of either a projected path composite or its source table. func pathCompositeColumnReference(scope *Scope, binding *BoundIdentifier, column pgsql.Identifier) pgsql.Expression { if binding.LastProjection != nil || scope.CurrentFrameBinding() != nil { return pgsql.RowColumnReference{ @@ -222,6 +252,7 @@ func pathCompositeColumnReference(scope *Scope, binding *BoundIdentifier, column return pgsql.CompoundIdentifier{binding.Identifier, column} } +// pathEdgeIDReference resolves the identifier of an edge used as a path component. func pathEdgeIDReference(scope *Scope, binding *BoundIdentifier) pgsql.Expression { if binding.LastProjection != nil || scope.CurrentFrameBinding() != nil { return pathBindingReference(scope, binding) @@ -230,23 +261,30 @@ func pathEdgeIDReference(scope *Scope, binding *BoundIdentifier) pgsql.Expressio return pgsql.CompoundIdentifier{binding.Identifier, pgsql.ColumnID} } -func pathEdgeArrayExpression(scope *Scope, edge *BoundIdentifier) pgsql.Expression { +// edgeArrayFromPathIDs creates a graph-scoped edge-array materializer for ordered edge identifiers. +func edgeArrayFromPathIDs(scope *Scope, pathIDs pgsql.Expression) *pgsql.EdgeArrayFromPathIDs { return &pgsql.EdgeArrayFromPathIDs{ - PathIDs: pgsql.ArrayLiteral{ - Values: []pgsql.Expression{ - pathEdgeIDReference(scope, edge), - }, - CastType: pgsql.Int8Array, - }, + PathIDs: pathIDs, + GraphID: pgsql.NewLiteral(scope.GraphID(), pgsql.Int4), } } +// pathEdgeArrayExpression materializes one path-edge binding as an edge-composite array. +func pathEdgeArrayExpression(scope *Scope, edge *BoundIdentifier) pgsql.Expression { + return edgeArrayFromPathIDs(scope, pgsql.ArrayLiteral{ + Values: []pgsql.Expression{ + pathEdgeIDReference(scope, edge), + }, + CastType: pgsql.Int8Array, + }) +} + +// expansionPathEdgeArrayExpression materializes an expansion's edge-identifier path as edge composites. func expansionPathEdgeArrayExpression(scope *Scope, expansionPath *BoundIdentifier) (pgsql.Expression, error) { - return &pgsql.EdgeArrayFromPathIDs{ - PathIDs: pathBindingReference(scope, expansionPath), - }, nil + return edgeArrayFromPathIDs(scope, pathBindingReference(scope, expansionPath)), nil } +// optionalOr combines two predicates while treating a nil operand as absent. func optionalOr(leftOperand, rightOperand pgsql.Expression) pgsql.Expression { if leftOperand == nil { return rightOperand @@ -257,17 +295,19 @@ func optionalOr(leftOperand, rightOperand pgsql.Expression) pgsql.Expression { return pgsql.NewBinaryExpression(leftOperand, pgsql.OperatorOr, rightOperand) } +// expressionIsNull builds an SQL null test for an expression. func expressionIsNull(expression pgsql.Expression) pgsql.Expression { return pgsql.NewBinaryExpression(expression, pgsql.OperatorIs, pgsql.NullLiteral()) } +// pathCompositeDependencyNullGuard returns the null test appropriate for a path component binding. func pathCompositeDependencyNullGuard(scope *Scope, dependency *BoundIdentifier) pgsql.Expression { if dependency == nil { return nil } switch dependency.DataType { - case pgsql.ExpansionPath: + case pgsql.ExpansionPath, pgsql.PathComposite: return expressionIsNull(pathBindingReference(scope, dependency)) case pgsql.EdgeComposite: @@ -284,6 +324,7 @@ func pathCompositeDependencyNullGuard(scope *Scope, dependency *BoundIdentifier) } } +// nullGuardPathCompositeExpression yields SQL null instead of constructing a path when a dependency is null. func nullGuardPathCompositeExpression(expression, nullGuard pgsql.Expression) pgsql.Expression { if nullGuard == nil { return expression @@ -296,6 +337,7 @@ func nullGuardPathCompositeExpression(expression, nullGuard pgsql.Expression) pg } } +// expressionForPathComposite assembles a path value from complete paths, node composites, and ordered edge components. func expressionForPathComposite(projected *BoundIdentifier, scope *Scope) (pgsql.Expression, error) { if projected.LastProjection != nil { return pgsql.CompoundIdentifier{projected.LastProjection.Binding.Identifier, projected.Identifier}, nil @@ -306,11 +348,27 @@ func expressionForPathComposite(projected *BoundIdentifier, scope *Scope) (pgsql nodeReferences []pgsql.Expression directNodeReferences []pgsql.Expression directEdgeReferences []pgsql.Expression + allRawPathIDParts []pgsql.Expression seenExpansionPath = false seenPathEdge = false + seenDirectEdge = false + directPath pgsql.Expression nullGuard pgsql.Expression + pendingPathIDParts []pgsql.Expression ) + flushPathIDParts := func() { + if len(pendingPathIDParts) == 0 { + return + } + + edgeArrayReferences = append(edgeArrayReferences, edgeArrayFromPathIDs( + scope, + concatenatePathCompositeParts(pendingPathIDParts), + )) + pendingPathIDParts = nil + } + // Path composite components are encoded as dependencies on the bound identifier representing the // path. This is not ideal as it escapes normal translation flow as driven by the structure of the // originating cypher AST. @@ -318,15 +376,21 @@ func expressionForPathComposite(projected *BoundIdentifier, scope *Scope) (pgsql nullGuard = optionalOr(nullGuard, pathCompositeDependencyNullGuard(scope, dependency)) switch dependency.DataType { + case pgsql.PathComposite: + if directPath != nil { + return nil, fmt.Errorf("path rendering contains multiple complete path dependencies") + } + directPath = pathBindingReference(scope, dependency) + case pgsql.ExpansionPath: seenExpansionPath = true - if edgeArrayReference, err := expansionPathEdgeArrayExpression(scope, dependency); err != nil { - return nil, err - } else { - edgeArrayReferences = append(edgeArrayReferences, edgeArrayReference) - } + pathIDs := pathBindingReference(scope, dependency) + pendingPathIDParts = append(pendingPathIDParts, pathIDs) + allRawPathIDParts = append(allRawPathIDParts, pathIDs) case pgsql.EdgeComposite: + seenDirectEdge = true + flushPathIDParts() directEdgeReference := pathCompositeReference(scope, dependency, pgsql.EdgeTableColumns) directEdgeReferences = append(directEdgeReferences, directEdgeReference) @@ -337,7 +401,12 @@ func expressionForPathComposite(projected *BoundIdentifier, scope *Scope) (pgsql case pgsql.PathEdge: seenPathEdge = true - edgeArrayReferences = append(edgeArrayReferences, pathEdgeArrayExpression(scope, dependency)) + pathIDs := pgsql.ArrayLiteral{ + Values: []pgsql.Expression{pathEdgeIDReference(scope, dependency)}, + CastType: pgsql.Int8Array, + } + pendingPathIDParts = append(pendingPathIDParts, pathIDs) + allRawPathIDParts = append(allRawPathIDParts, pathIDs) case pgsql.NodeComposite, pgsql.ExpansionRootNode, pgsql.ExpansionTerminalNode: directNodeReferences = append(directNodeReferences, pathCompositeReference(scope, dependency, pgsql.NodeTableColumns)) @@ -347,6 +416,13 @@ func expressionForPathComposite(projected *BoundIdentifier, scope *Scope) (pgsql return nil, fmt.Errorf("unsupported type for path rendering: %s", dependency.DataType) } } + flushPathIDParts() + if directPath != nil { + if seenExpansionPath || seenPathEdge || seenDirectEdge { + return nil, fmt.Errorf("complete path dependency cannot be mixed with edge path components") + } + return nullGuardPathCompositeExpression(directPath, nullGuard), nil + } // Direct, non-expansion path bindings already have their node and edge composites in scope. Keep // those explicit components instead of reconstructing the path from edge IDs: this preserves path @@ -373,6 +449,34 @@ func expressionForPathComposite(projected *BoundIdentifier, scope *Scope) (pgsql return nil, fmt.Errorf("expansion path %s does not contain a root node reference", projected.Identifier) } + knownNodes := pgsql.ArrayLiteral{ + Values: directNodeReferences, + CastType: pgsql.NodeCompositeArray, + } + + // Read expansions carry edge IDs in path order. When every edge + // component is still an ID, let the graph-scoped linear materializer + // hydrate and walk the stream once. A direct edge composite indicates a + // mixed or mutation-returning path and retains the conservative generic + // materializer below. + if !seenDirectEdge { + pathIDs := concatenatePathCompositeParts(allRawPathIDParts) + if pathIDs == nil { + pathIDs = pgsql.ArrayLiteral{CastType: pgsql.Int8Array} + } + + return nullGuardPathCompositeExpression(pgsql.FunctionCall{ + Function: pgsql.FunctionOrderedEdgeIDsToPath, + Parameters: []pgsql.Expression{ + pgsql.NewLiteral(scope.GraphID(), pgsql.Int4), + directNodeReferences[0], + pathIDs, + knownNodes, + }, + CastType: pgsql.PathComposite, + }, nullGuard), nil + } + edgeArrayExpression := concatenatePathCompositeParts(edgeArrayReferences) if edgeArrayExpression == nil { edgeArrayExpression = pgsql.ArrayLiteral{CastType: pgsql.EdgeCompositeArray} @@ -381,12 +485,10 @@ func expressionForPathComposite(projected *BoundIdentifier, scope *Scope) (pgsql return nullGuardPathCompositeExpression(pgsql.FunctionCall{ Function: pgsql.FunctionOrderedEdgesToPath, Parameters: []pgsql.Expression{ + pgsql.NewLiteral(scope.GraphID(), pgsql.Int4), directNodeReferences[0], edgeArrayExpression, - pgsql.ArrayLiteral{ - Values: directNodeReferences, - CastType: pgsql.NodeCompositeArray, - }, + knownNodes, }, CastType: pgsql.PathComposite, }, nullGuard), nil @@ -394,6 +496,7 @@ func expressionForPathComposite(projected *BoundIdentifier, scope *Scope) (pgsql return nullGuardPathCompositeExpression(pgsql.FunctionCall{ Function: pgsql.FunctionNodesToPath, Parameters: []pgsql.Expression{ + pgsql.NewLiteral(scope.GraphID(), pgsql.Int4), pgsql.Variadic{ Expression: pgsql.ArrayLiteral{ Values: nodeReferences, @@ -408,7 +511,21 @@ func expressionForPathComposite(projected *BoundIdentifier, scope *Scope) (pgsql return nil, fmt.Errorf("path variable does not contain valid components") } +// buildProjectionForPathComposite projects either a path's distance or its assembled composite value. func buildProjectionForPathComposite(alias pgsql.Identifier, projected *BoundIdentifier, scope *Scope) ([]pgsql.SelectItem, error) { + if projected.DistanceOnly { + reference := scope.CurrentFrame().Binding.Identifier + column := expansionDepth + if projected.LastProjection != nil { + reference = projected.LastProjection.Binding.Identifier + column = projected.Identifier + } + return []pgsql.SelectItem{&pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{reference, column}, + Alias: pgsql.AsOptionalIdentifier(alias), + }}, nil + } + if expression, err := expressionForPathComposite(projected, scope); err != nil { return nil, err } else { @@ -421,7 +538,20 @@ func buildProjectionForPathComposite(alias pgsql.Identifier, projected *BoundIde } } +// buildProjectionForExpansionNode projects an expansion endpoint as an identifier or hydrated node composite. func buildProjectionForExpansionNode(alias pgsql.Identifier, projected *BoundIdentifier, referenceFrame *Frame) ([]pgsql.SelectItem, error) { + if projected.IDOnly { + var expression pgsql.Expression = pgsql.CompoundIdentifier{projected.Identifier, pgsql.ColumnID} + if projected.LastProjection != nil { + expression = pgsql.CompoundIdentifier{referenceFrame.Binding.Identifier, projected.Identifier} + } + + return []pgsql.SelectItem{&pgsql.AliasedExpression{ + Expression: expression, + Alias: pgsql.AsOptionalIdentifier(alias), + }}, nil + } + if projected.LastProjection != nil { return []pgsql.SelectItem{ &pgsql.AliasedExpression{ @@ -451,7 +581,20 @@ func buildProjectionForExpansionNode(alias pgsql.Identifier, projected *BoundIde }, nil } +// buildProjectionForNodeComposite projects an existing node binding as an identifier or node composite. func buildProjectionForNodeComposite(alias pgsql.Identifier, projected *BoundIdentifier, referenceFrame *Frame) ([]pgsql.SelectItem, error) { + if projected.IDOnly { + var expression pgsql.Expression = pgsql.CompoundIdentifier{projected.Identifier, pgsql.ColumnID} + if projected.LastProjection != nil { + expression = pgsql.CompoundIdentifier{referenceFrame.Binding.Identifier, projected.Identifier} + } + + return []pgsql.SelectItem{&pgsql.AliasedExpression{ + Expression: expression, + Alias: pgsql.AsOptionalIdentifier(alias), + }}, nil + } + if projected.LastProjection != nil { return []pgsql.SelectItem{ &pgsql.AliasedExpression{ @@ -478,6 +621,7 @@ func buildProjectionForNodeComposite(alias pgsql.Identifier, projected *BoundIde }, nil } +// buildProjectionForExpansionEdge materializes an expansion path's edge identifiers as edge composites. func buildProjectionForExpansionEdge(alias pgsql.Identifier, projected *BoundIdentifier, scope *Scope) ([]pgsql.SelectItem, error) { // Change the type to the edge composite now that this is projected projected.DataType = pgsql.EdgeComposite @@ -485,17 +629,16 @@ func buildProjectionForExpansionEdge(alias pgsql.Identifier, projected *BoundIde // Create a new final projection that's aliased to the visible binding's identifier return []pgsql.SelectItem{ &pgsql.AliasedExpression{ - Expression: &pgsql.EdgeArrayFromPathIDs{ - PathIDs: pgsql.CompoundIdentifier{ - scope.CurrentFrame().Binding.Identifier, - pgsql.ColumnPath, - }, - }, + Expression: edgeArrayFromPathIDs(scope, pgsql.CompoundIdentifier{ + scope.CurrentFrame().Binding.Identifier, + pgsql.ColumnPath, + }), Alias: pgsql.AsOptionalIdentifier(alias), }, }, nil } +// buildProjectionForEdgeComposite projects an edge binding from its latest frame or source columns. func buildProjectionForEdgeComposite(alias pgsql.Identifier, projected *BoundIdentifier, referenceFrame *Frame) ([]pgsql.SelectItem, error) { if projected.LastProjection != nil { return []pgsql.SelectItem{ @@ -515,6 +658,7 @@ func buildProjectionForEdgeComposite(alias pgsql.Identifier, projected *BoundIde }, nil } +// buildProjectionForPathEdge projects the identifier carried by a single-edge path component. func buildProjectionForPathEdge(alias pgsql.Identifier, projected *BoundIdentifier, referenceFrame *Frame) ([]pgsql.SelectItem, error) { var expression pgsql.Expression @@ -536,7 +680,21 @@ func buildProjectionForPathEdge(alias pgsql.Identifier, projected *BoundIdentifi }, nil } +// buildProjection dispatches a bound identifier to the projection form required by its data type. func buildProjection(alias pgsql.Identifier, projected *BoundIdentifier, scope *Scope, referenceFrame *Frame) ([]pgsql.SelectItem, error) { + if projected.DistanceOnly { + reference := scope.CurrentFrame().Binding.Identifier + column := expansionDepth + if projected.LastProjection != nil { + reference = referenceFrame.Binding.Identifier + column = projected.Identifier + } + return []pgsql.SelectItem{&pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{reference, column}, + Alias: pgsql.AsOptionalIdentifier(alias), + }}, nil + } + switch projected.DataType { case pgsql.ExpansionPath: return buildProjectionForExpansionPath(alias, projected, scope, referenceFrame) @@ -577,6 +735,7 @@ func buildProjection(alias pgsql.Identifier, projected *BoundIdentifier, scope * } } +// buildInlineProjection renders a query part's prepared expressions directly into a select statement. func (s *Translator) buildInlineProjection(part *QueryPart) (pgsql.Select, error) { sqlSelect := pgsql.Select{ Distinct: part.projections.Distinct, @@ -638,6 +797,7 @@ func (s *Translator) buildInlineProjection(part *QueryPart) (pgsql.Select, error return sqlSelect, nil } +// collectProjectionFromFrames collects the frame sources required by projected bindings and path dependencies. func (s *Translator) collectProjectionFromFrames(projections []*Projection) []pgsql.FromClause { fromClauseBuilder := NewFromClauseBuilder() @@ -670,6 +830,7 @@ func (s *Translator) collectProjectionFromFrames(projections []*Projection) []pg return fromClauseBuilder.Clauses() } +// countLimitPushdownShortestPathHarnessCalls counts eligible shortest-path harness calls throughout a query's CTE tree. func countLimitPushdownShortestPathHarnessCalls(query pgsql.Query) int { var count int @@ -691,10 +852,12 @@ func countLimitPushdownShortestPathHarnessCalls(query pgsql.Query) int { return count } +// isLimitPushdownShortestPathHarness reports whether a function accepts the shortest-path limit parameter. func isLimitPushdownShortestPathHarness(function pgsql.Identifier) bool { return function == pgsql.FunctionUnidirectionalSPHarness || function == pgsql.FunctionBidirectionalSPHarness } +// appendLimitToShortestPathHarness passes a limit to each eligible harness and bounds its containing function scan. func appendLimitToShortestPathHarness(query *pgsql.Query, limit pgsql.Expression) { if query.CommonTableExpressions != nil { for idx := range query.CommonTableExpressions.Expressions { @@ -703,6 +866,7 @@ func appendLimitToShortestPathHarness(query *pgsql.Query, limit pgsql.Expression } if selectBody, isSelect := query.Body.(pgsql.Select); isSelect { + containsHarness := false for idx := range selectBody.From { if functionCall, isFunctionCall := selectBody.From[idx].Source.(pgsql.FunctionCall); isFunctionCall && isLimitPushdownShortestPathHarness(functionCall.Function) { @@ -711,13 +875,22 @@ func appendLimitToShortestPathHarness(query *pgsql.Query, limit pgsql.Expression // outer query will discard. functionCall.Parameters = append(functionCall.Parameters, pgsql.NewTypeCast(limit, pgsql.Int8)) selectBody.From[idx].Source = functionCall + containsHarness = true } } query.Body = selectBody + if containsHarness { + // Keep the internal limit so the BFS can stop early, and also bound + // the containing FunctionScan so downstream planning sees the same + // cardinality ceiling. In particular, LIMIT 0 must prevent invoking + // the harness because the harness uses zero to mean "unlimited". + query.Limit = limit + } } } +// selectContainsAggregate reports whether a select body contains an aggregate function call. func selectContainsAggregate(selectBody pgsql.Select) bool { containsAggregate := false @@ -732,6 +905,7 @@ func selectContainsAggregate(selectBody pgsql.Select) bool { return containsAggregate } +// compoundIdentifierEqual reports whether two qualified identifiers contain the same components. func compoundIdentifierEqual(left, right pgsql.CompoundIdentifier) bool { if len(left) != len(right) { return false @@ -746,6 +920,7 @@ func compoundIdentifierEqual(left, right pgsql.CompoundIdentifier) bool { return true } +// directShortestPathHarnessFrame returns the sole CTE that directly invokes an eligible shortest-path harness. func directShortestPathHarnessFrame(query pgsql.Query) (pgsql.Identifier, bool) { if query.CommonTableExpressions == nil { return "", false @@ -774,11 +949,13 @@ func directShortestPathHarnessFrame(query pgsql.Query) (pgsql.Identifier, bool) return harnessFrame, harnessFrame != "" } +// isCompoundIdentifierOperand reports whether an expression is the requested qualified identifier. func isCompoundIdentifierOperand(expression pgsql.Expression, identifier pgsql.CompoundIdentifier) bool { compoundIdentifier, isCompoundIdentifier := unwrapParenthetical(expression).(pgsql.CompoundIdentifier) return isCompoundIdentifier && compoundIdentifierEqual(compoundIdentifier, identifier) } +// isEqualityBetweenCompoundIdentifiers recognizes equality between two qualified identifiers in either order. func isEqualityBetweenCompoundIdentifiers(expression pgsql.Expression, left, right pgsql.CompoundIdentifier) bool { binaryExpression, isBinaryExpression := unwrapParenthetical(expression).(*pgsql.BinaryExpression) if !isBinaryExpression || binaryExpression.Operator != pgsql.OperatorEquals { @@ -789,6 +966,7 @@ func isEqualityBetweenCompoundIdentifiers(expression pgsql.Expression, left, rig (isCompoundIdentifierOperand(binaryExpression.LOperand, right) && isCompoundIdentifierOperand(binaryExpression.ROperand, left)) } +// expansionEndpointJoin identifies a node-table join to the root or terminal column of a harness frame. func expansionEndpointJoin(join pgsql.Join, harnessFrame pgsql.Identifier) (pgsql.Identifier, pgsql.Identifier, bool) { tableReference, isTableReference := join.Table.(pgsql.TableReference) if !isTableReference || @@ -815,6 +993,7 @@ func expansionEndpointJoin(join pgsql.Join, harnessFrame pgsql.Identifier) (pgsq return "", "", false } +// shortestPathEndpointAliases finds distinct node aliases joined to a shortest-path harness's root and terminal columns. func shortestPathEndpointAliases(query pgsql.Query) (pgsql.Identifier, pgsql.Identifier, bool) { harnessFrame, hasHarnessFrame := directShortestPathHarnessFrame(query) if !hasHarnessFrame { @@ -844,6 +1023,7 @@ func shortestPathEndpointAliases(query pgsql.Query) (pgsql.Identifier, pgsql.Ide return rootAlias, terminalAlias, rootAlias != "" && terminalAlias != "" && rootAlias != terminalAlias } +// harnessEndpointColumn recognizes a root or terminal identifier column belonging to a harness frame. func harnessEndpointColumn(expression pgsql.Expression, harnessFrame pgsql.Identifier) (pgsql.Identifier, bool) { compoundIdentifier, isCompoundIdentifier := unwrapParenthetical(expression).(pgsql.CompoundIdentifier) if !isCompoundIdentifier || @@ -856,6 +1036,7 @@ func harnessEndpointColumn(expression pgsql.Expression, harnessFrame pgsql.Ident return compoundIdentifier[1], true } +// rowIDReferenceAlias extracts the row alias named by a composite identifier-field reference. func rowIDReferenceAlias(expression pgsql.Expression) (pgsql.Identifier, bool) { rowColumnReference, isRowColumnReference := unwrapParenthetical(expression).(pgsql.RowColumnReference) if !isRowColumnReference || rowColumnReference.Column != pgsql.ColumnID { @@ -870,6 +1051,7 @@ func rowIDReferenceAlias(expression pgsql.Expression) (pgsql.Identifier, bool) { return compoundIdentifier[1], true } +// sourceAliasMatchesEndpointColumn reports whether a source alias corresponds to a harness endpoint column. func sourceAliasMatchesEndpointColumn(sourceAlias, endpointColumn, rootAlias, terminalAlias pgsql.Identifier) bool { switch endpointColumn { case expansionRootID: @@ -881,6 +1063,7 @@ func sourceAliasMatchesEndpointColumn(sourceAlias, endpointColumn, rootAlias, te } } +// isBoundEndpointProjectionConstraint recognizes a shape-preserving equality between a harness endpoint and its node alias. func isBoundEndpointProjectionConstraint(expression pgsql.Expression, harnessFrame, rootAlias, terminalAlias pgsql.Identifier) bool { binaryExpression, isBinaryExpression := unwrapParenthetical(expression).(*pgsql.BinaryExpression) if !isBinaryExpression || binaryExpression.Operator != pgsql.OperatorEquals { @@ -896,6 +1079,7 @@ func isBoundEndpointProjectionConstraint(expression pgsql.Expression, harnessFra (rightIsEndpoint && leftIsRowIDReference && sourceAliasMatchesEndpointColumn(leftSourceAlias, rightEndpointColumn, rootAlias, terminalAlias)) } +// shortestPathSourceWhereTransparent reports whether a source CTE filters only by endpoint projection equalities. func shortestPathSourceWhereTransparent(query pgsql.Query, rootAlias, terminalAlias pgsql.Identifier) bool { harnessFrame, hasHarnessFrame := directShortestPathHarnessFrame(query) if !hasHarnessFrame { @@ -923,6 +1107,7 @@ func shortestPathSourceWhereTransparent(query pgsql.Query, rootAlias, terminalAl return true } +// endpointIDReference extracts an endpoint alias from an identifier-field reference in the source frame. func endpointIDReference(expression pgsql.Expression, sourceFrame pgsql.Identifier) (pgsql.Identifier, bool) { rowColumnReference, isRowColumnReference := unwrapParenthetical(expression).(pgsql.RowColumnReference) compoundIdentifier, isCompoundIdentifier := unwrapParenthetical(rowColumnReference.Identifier).(pgsql.CompoundIdentifier) @@ -937,11 +1122,13 @@ func endpointIDReference(expression pgsql.Expression, sourceFrame pgsql.Identifi return compoundIdentifier[1], true } +// isEndpointAliasPair reports whether two aliases are the root and terminal aliases in either order. func isEndpointAliasPair(leftAlias, rightAlias, rootAlias, terminalAlias pgsql.Identifier) bool { return (leftAlias == rootAlias && rightAlias == terminalAlias) || (leftAlias == terminalAlias && rightAlias == rootAlias) } +// isEndpointInequality recognizes a non-equality predicate between the source frame's root and terminal identifiers. func isEndpointInequality(expression pgsql.Expression, sourceFrame, rootAlias, terminalAlias pgsql.Identifier) bool { binaryExpression, isBinaryExpression := unwrapParenthetical(expression).(*pgsql.BinaryExpression) if !isBinaryExpression || @@ -955,6 +1142,7 @@ func isEndpointInequality(expression pgsql.Expression, sourceFrame, rootAlias, t return hasLeftAlias && hasRightAlias && isEndpointAliasPair(leftAlias, rightAlias, rootAlias, terminalAlias) } +// shortestPathLimitPushdownTransparentWhere permits only the endpoint anti-reflexive predicate above a transparent source CTE. func shortestPathLimitPushdownTransparentWhere(currentPart *QueryPart, sourceFrame pgsql.Identifier, where pgsql.Expression) bool { if where == nil { return true @@ -982,6 +1170,7 @@ func shortestPathLimitPushdownTransparentWhere(currentPart *QueryPart, sourceFra return true } +// limitPushdownTailSource returns the sole pass-through source CTE when the tail select preserves limit semantics. func limitPushdownTailSource(currentPart *QueryPart, tailSelect pgsql.Select) (pgsql.Identifier, bool) { // Keep this intentionally narrow: LIMIT can move into the harness only when // the tail SELECT is a simple pass-through over one shortest-path CTE. Sorts, @@ -1026,6 +1215,7 @@ func limitPushdownTailSource(currentPart *QueryPart, tailSelect pgsql.Select) (p return sourceFrame, true } +// pushDownShortestPathLimit moves an outer limit into a single eligible shortest-path harness call. func pushDownShortestPathLimit(currentPart *QueryPart, tailSelect pgsql.Select) bool { sourceFrame, canPushDown := limitPushdownTailSource(currentPart, tailSelect) if !canPushDown { @@ -1044,6 +1234,7 @@ func pushDownShortestPathLimit(currentPart *QueryPart, tailSelect pgsql.Select) return false } +// findCTE returns the named top-level common table expression, if present. func findCTE(query *pgsql.Query, cteName pgsql.Identifier) *pgsql.CommonTableExpression { if query.CommonTableExpressions == nil { return nil @@ -1060,6 +1251,7 @@ func findCTE(query *pgsql.Query, cteName pgsql.Identifier) *pgsql.CommonTableExp return nil } +// applyLimitToCTE assigns a limit to the named common table expression. func applyLimitToCTE(query *pgsql.Query, cteName pgsql.Identifier, limit pgsql.Expression) bool { if cte := findCTE(query, cteName); cte != nil { cte.Query.Limit = limit @@ -1069,6 +1261,7 @@ func applyLimitToCTE(query *pgsql.Query, cteName pgsql.Identifier, limit pgsql.E return false } +// pushDownTraversalLimit moves an outer limit to a semantically transparent traversal CTE. func pushDownTraversalLimit(currentPart *QueryPart, tailSelect pgsql.Select) bool { sourceFrame, canPushDown := limitPushdownTailSource(currentPart, tailSelect) if !canPushDown || !currentPart.CanPushDownLimitTo(sourceFrame) { @@ -1078,6 +1271,7 @@ func pushDownTraversalLimit(currentPart *QueryPart, tailSelect pgsql.Select) boo return applyLimitToCTE(currentPart.Model, sourceFrame, currentPart.Limit) } +// projectionAliasBindings maps internal binding identifiers to their visible projection aliases. func projectionAliasBindings(scope *Scope, projections []*Projection) map[pgsql.Identifier]pgsql.Identifier { aliases := map[pgsql.Identifier]pgsql.Identifier{} @@ -1094,6 +1288,7 @@ func projectionAliasBindings(scope *Scope, projections []*Projection) map[pgsql. return aliases } +// rewriteOrderByProjectionAlias replaces an internal ORDER BY identifier with its visible projection alias. func rewriteOrderByProjectionAlias(orderBy *pgsql.OrderBy, aliases map[pgsql.Identifier]pgsql.Identifier) { identifier, isIdentifier := orderBy.Expression.(pgsql.Identifier) if !isIdentifier { @@ -1105,21 +1300,32 @@ func rewriteOrderByProjectionAlias(orderBy *pgsql.OrderBy, aliases map[pgsql.Ide } } +// pathCompositeReferenceCount records how a path and each of its component arrays are reused by a projection stage. type pathCompositeReferenceCount struct { + // binding is the unmaterialized path binding being counted. binding *BoundIdentifier - full int - nodes int - edges int + + // full counts references to the complete path value. + full int + + // nodes counts references to the path's node array. + nodes int + + // edges counts references to the path's edge array. + edges int } +// componentReferences returns the combined number of node-array and edge-array references. func (s pathCompositeReferenceCount) componentReferences() int { return s.nodes + s.edges } +// totalReferences returns the number of complete-path and component-array references. func (s pathCompositeReferenceCount) totalReferences() int { return s.full + s.componentReferences() } +// pathCompositeBinding resolves an identifier to an unmaterialized path-composite binding. func pathCompositeBinding(scope *Scope, identifier pgsql.Identifier) (*BoundIdentifier, bool) { binding, bound := scope.Lookup(identifier) if !bound { @@ -1133,6 +1339,7 @@ func pathCompositeBinding(scope *Scope, identifier pgsql.Identifier) (*BoundIden return binding, true } +// ensurePathCompositeReferenceCount returns the stable counter for a binding and records first-seen order. func ensurePathCompositeReferenceCount( counts map[pgsql.Identifier]*pathCompositeReferenceCount, orderedCounts *[]*pathCompositeReferenceCount, @@ -1152,6 +1359,7 @@ func ensurePathCompositeReferenceCount( return count } +// countPathCompositeComponents counts references to node and edge arrays of unmaterialized path composites. func countPathCompositeComponents(scope *Scope, expressions ...pgsql.Expression) ([]*pathCompositeReferenceCount, error) { var ( counts = map[pgsql.Identifier]*pathCompositeReferenceCount{} @@ -1194,6 +1402,7 @@ func countPathCompositeComponents(scope *Scope, expressions ...pgsql.Expression) return orderedCounts, nil } +// countPathCompositeProjectionReferences counts complete and component references made by projection items. func countPathCompositeProjectionReferences(scope *Scope, projections []*Projection) ([]*pathCompositeReferenceCount, error) { var ( counts = map[pgsql.Identifier]*pathCompositeReferenceCount{} @@ -1231,6 +1440,7 @@ func countPathCompositeProjectionReferences(scope *Scope, projections []*Project return orderedCounts, nil } +// tailPathCompositeStageBindings selects paths whose node arrays must be staged for a tail constraint. func tailPathCompositeStageBindings(scope *Scope, expression pgsql.Expression) ([]*BoundIdentifier, error) { counts, err := countPathCompositeComponents(scope, expression) if err != nil { @@ -1247,6 +1457,7 @@ func tailPathCompositeStageBindings(scope *Scope, expression pgsql.Expression) ( return bindings, nil } +// projectionPathCompositeStageBindings selects paths reused enough to warrant one intermediate materialization. func projectionPathCompositeStageBindings(scope *Scope, projections []*Projection) ([]*BoundIdentifier, error) { counts, err := countPathCompositeProjectionReferences(scope, projections) if err != nil { @@ -1268,6 +1479,7 @@ func projectionPathCompositeStageBindings(scope *Scope, projections []*Projectio return bindings, nil } +// mergePathCompositeStageBindings combines binding lists in first-seen order without duplicates. func mergePathCompositeStageBindings(bindingSets ...[]*BoundIdentifier) []*BoundIdentifier { var ( merged = make([]*BoundIdentifier, 0) @@ -1288,6 +1500,7 @@ func mergePathCompositeStageBindings(bindingSets ...[]*BoundIdentifier) []*Bound return merged } +// stagePathCompositeBindings adds lateral sources that materialize selected paths once for downstream reuse. func (s *Translator) stagePathCompositeBindings(fromClauses []pgsql.FromClause, bindings []*BoundIdentifier) ([]pgsql.FromClause, error) { for _, binding := range bindings { stageBinding, err := s.scope.DefineNew(pgsql.Scope) @@ -1327,6 +1540,7 @@ func (s *Translator) stagePathCompositeBindings(fromClauses []pgsql.FromClause, return fromClauses, nil } +// buildTailProjection renders the final select, stages reused paths, and applies grouping, ordering, skip, and limit. func (s *Translator) buildTailProjection() error { var ( currentPart = s.query.CurrentPart() @@ -1426,6 +1640,7 @@ func (s *Translator) buildTailProjection() error { return nil } +// ensureProjectionAliasBinding defines an inferred scope binding for an expression alias not already known. func (s *Translator) ensureProjectionAliasBinding(alias pgsql.Identifier, selectItem pgsql.SelectItem) error { if _, isBound := s.scope.AliasedLookup(alias); isBound { return nil @@ -1445,6 +1660,7 @@ func (s *Translator) ensureProjectionAliasBinding(alias pgsql.Identifier, select return nil } +// ensureSortItemProjectionAliases registers visible aliases that ORDER BY items may reference. func (s *Translator) ensureSortItemProjectionAliases() error { currentPart := s.query.CurrentPart() if currentPart.projections == nil { @@ -1468,7 +1684,53 @@ func (s *Translator) ensureSortItemProjectionAliases() error { return nil } +// isGreedyProjectionItem reports whether a Cypher projection item is the wildcard expression. +func isGreedyProjectionItem(projectionItem *cypher.ProjectionItem) bool { + variable, isVariable := projectionItem.Expression.(*cypher.Variable) + return isVariable && variable.Symbol == cypher.TokenLiteralAsterisk +} + +// translateGreedyProjection replaces a wildcard placeholder with every named binding visible in the frame. +func (s *Translator) translateGreedyProjection(scope *Scope) error { + currentPart := s.query.CurrentPart() + if _, err := s.treeTranslator.PopOperand(); err != nil { + return err + } + if currentPart.projections == nil || len(currentPart.projections.Items) == 0 { + return fmt.Errorf("greedy projection has no prepared projection item") + } + + // Entering the projection item reserves one slot. Replace that placeholder + // with a projection for every named binding visible at this boundary. + currentPart.projections.Items = currentPart.projections.Items[:len(currentPart.projections.Items)-1] + projected := 0 + for _, identifier := range scope.CurrentFrame().Known().Slice() { + binding, found := scope.Lookup(identifier) + if !found { + return fmt.Errorf("unable to resolve greedy projection binding %s", identifier) + } + for _, symbol := range scope.Symbols(binding) { + currentPart.projections.Items = append(currentPart.projections.Items, &Projection{ + SelectItem: binding.Identifier, + Alias: models.OptionalValue(symbol), + }) + projected++ + } + } + + if projected == 0 { + return fmt.Errorf("greedy projection requires at least one named binding") + } + currentPart.projections.Frame = scope.CurrentFrame() + return nil +} + +// translateProjectionItem records one translated select expression and establishes its explicit or implicit alias. func (s *Translator) translateProjectionItem(scope *Scope, projectionItem *cypher.ProjectionItem) error { + if isGreedyProjectionItem(projectionItem) { + return s.translateGreedyProjection(scope) + } + if alias, hasAlias, err := extractIdentifierFromCypherExpression(projectionItem); err != nil { return err } else if nextExpression, err := s.treeTranslator.PopOperand(); err != nil { @@ -1483,6 +1745,15 @@ func (s *Translator) translateProjectionItem(scope *Scope, projectionItem *cyphe s.query.CurrentPart().projections.Frame = s.scope.CurrentFrame() } + if !hasAlias { + var buffer bytes.Buffer + if err := cypherFormat.NewCypherEmitter(false).WriteExpression(&buffer, projectionItem.Expression); err != nil { + return fmt.Errorf("format implicit projection name: %w", err) + } + alias = pgsql.Identifier(buffer.String()) + hasAlias = true + } + switch typedSelectItem := unwrapParenthetical(selectItem).(type) { case pgsql.Identifier: // If this is an identifier then assume the identifier as the projection alias since the translator @@ -1527,6 +1798,7 @@ func (s *Translator) translateProjectionItem(scope *Scope, projectionItem *cyphe return nil } +// prepareProjection initializes a query part's projection state and validates literal SKIP and LIMIT values. func (s *Translator) prepareProjection(projection *cypher.Projection) error { currentPart := s.query.CurrentPart() currentPart.PrepareProjections(projection.Distinct) diff --git a/cypher/models/pgsql/translate/relationship.go b/cypher/models/pgsql/translate/relationship.go index 51ff6381..13cf867b 100644 --- a/cypher/models/pgsql/translate/relationship.go +++ b/cypher/models/pgsql/translate/relationship.go @@ -8,6 +8,7 @@ import ( "github.com/specterops/dawgs/cypher/models/pgsql/optimize" ) +// translateRelationshipPattern validates a relationship pattern and records its binding, kinds, and range. func (s *Translator) translateRelationshipPattern(relationshipPattern *cypher.RelationshipPattern) error { var ( currentQueryPart = s.query.CurrentPart() @@ -41,6 +42,11 @@ func (s *Translator) translateRelationshipPattern(relationshipPattern *cypher.Re return fmt.Errorf("failed to translate kinds: %w", err) } else { for _, edgeBinding := range edgeBindings { + for _, step := range patternPart.TraversalSteps { + if step.Edge == edgeBinding && step.Expansion != nil { + step.Expansion.RelationshipKindIDs = append([]int16(nil), kindIDs...) + } + } if err := s.treeTranslator.AddTranslationConstraint(pgsql.NewIdentifierSet().Add(edgeBinding.Identifier), pgsql.NewBinaryExpression( pgsql.CompoundIdentifier{edgeBinding.Identifier, pgsql.ColumnKindID}, pgsql.OperatorEquals, @@ -56,6 +62,7 @@ func (s *Translator) translateRelationshipPattern(relationshipPattern *cypher.Re return nil } +// collectCreateEdgePattern records the endpoints, kind, properties, and binding needed to create one edge. func (s *Translator) collectCreateEdgePattern(relationshipPattern *cypher.RelationshipPattern, part *PatternPart, bindingResult BindingResult) error { var ( queryPart = s.query.CurrentPart() @@ -99,6 +106,7 @@ func (s *Translator) collectCreateEdgePattern(relationshipPattern *cypher.Relati return nil } +// exactRangeExpansionDecision returns the planned exact-range unrolling decision for target. func (s *Translator) exactRangeExpansionDecision(sourceTarget optimize.TraversalStepTarget, hasSourceTarget bool, relationshipPattern *cypher.RelationshipPattern) (optimize.ExactRangeExpansionDecision, bool) { if !hasSourceTarget || relationshipPattern == nil { return optimize.ExactRangeExpansionDecision{}, false @@ -112,6 +120,7 @@ func (s *Translator) exactRangeExpansionDecision(sourceTarget optimize.Traversal return decision, true } +// translateExactRangeRelationshipPatternToSteps expands a fixed-depth relationship range into synthetic single-hop traversal steps. func (s *Translator) translateExactRangeRelationshipPatternToSteps( firstEdge *BoundIdentifier, part *PatternPart, @@ -191,6 +200,7 @@ func (s *Translator) translateExactRangeRelationshipPatternToSteps( return edgeBindings, nil } +// translateRelationshipPatternToStep attaches one translated relationship pattern to the current traversal step. func (s *Translator) translateRelationshipPatternToStep(bindingResult BindingResult, part *PatternPart, relationshipPattern *cypher.RelationshipPattern) ([]*BoundIdentifier, error) { var ( expansion *Expansion diff --git a/cypher/models/pgsql/translate/renamer.go b/cypher/models/pgsql/translate/renamer.go index f516cbbc..f8e2c191 100644 --- a/cypher/models/pgsql/translate/renamer.go +++ b/cypher/models/pgsql/translate/renamer.go @@ -7,6 +7,7 @@ import ( "github.com/specterops/dawgs/cypher/models/walk" ) +// rewriteCompositeTypeFieldReference rewrites the binding portion of a composite-field reference through mappings. func rewriteCompositeTypeFieldReference(scopeIdentifier pgsql.Identifier, compositeReference pgsql.CompoundIdentifier) pgsql.RowColumnReference { return pgsql.RowColumnReference{ Identifier: pgsql.CompoundIdentifier{scopeIdentifier, compositeReference.Root()}, @@ -14,6 +15,7 @@ func rewriteCompositeTypeFieldReference(scopeIdentifier pgsql.Identifier, compos } } +// rewriteIdentifierScopeReference replaces an identifier when mappings contains a scoped rename. func rewriteIdentifierScopeReference(scope *Scope, identifier pgsql.Identifier) (pgsql.SelectItem, error) { if !pgsql.IsReservedIdentifier(identifier) { if binding, bound := scope.Lookup(identifier); bound { @@ -27,9 +29,14 @@ func rewriteIdentifierScopeReference(scope *Scope, identifier pgsql.Identifier) return identifier, nil } +// rewriteCompoundIdentifierScopeReference replaces the root binding of a compound identifier through mappings. func rewriteCompoundIdentifierScopeReference(scope *Scope, identifier pgsql.CompoundIdentifier) (pgsql.SelectItem, error) { if binding, bound := scope.Lookup(identifier[0]); bound { if binding.LastProjection != nil { + if binding.IDOnly && len(identifier) == 2 && identifier[1] == pgsql.ColumnID { + return pgsql.CompoundIdentifier{binding.LastProjection.Binding.Identifier, binding.Identifier}, nil + } + return pgsql.RowColumnReference{ Identifier: pgsql.CompoundIdentifier{binding.LastProjection.Binding.Identifier, binding.Identifier}, Column: identifier[1], @@ -41,6 +48,7 @@ func rewriteCompoundIdentifierScopeReference(scope *Scope, identifier pgsql.Comp return identifier, nil } +// rewriteExpressionScopeReference rewrites identifier-bearing expression variants through mappings. func rewriteExpressionScopeReference(scope *Scope, expression pgsql.Expression) (pgsql.Expression, bool, error) { switch typedExpression := expression.(type) { case pgsql.Identifier: @@ -62,6 +70,7 @@ type FrameBindingRewriter struct { scope *Scope } +// rewriteArraySlice rewrites identifier references in an array expression and its slice bounds. func (s *FrameBindingRewriter) rewriteArraySlice(slice *pgsql.ArraySlice) error { if slice == nil { return nil @@ -92,6 +101,7 @@ func (s *FrameBindingRewriter) rewriteArraySlice(slice *pgsql.ArraySlice) error return nil } +// rewriteArrayLiteral rewrites identifier references in every array literal element. func (s *FrameBindingRewriter) rewriteArrayLiteral(literal *pgsql.ArrayLiteral) error { if literal == nil { return nil @@ -106,6 +116,7 @@ func (s *FrameBindingRewriter) rewriteArrayLiteral(literal *pgsql.ArrayLiteral) return nil } +// rewriteExpression recursively rewrites every supported identifier-bearing SQL expression. func (s *FrameBindingRewriter) rewriteExpression(expression *pgsql.Expression) error { if expression == nil || *expression == nil { return nil @@ -148,6 +159,7 @@ func (s *FrameBindingRewriter) rewriteExpression(expression *pgsql.Expression) e return nil } +// rewriteCase rewrites identifier references in a CASE operand, branches, and fallback. func (s *FrameBindingRewriter) rewriteCase(caseExpression *pgsql.Case) error { if caseExpression == nil { return nil @@ -172,6 +184,7 @@ func (s *FrameBindingRewriter) rewriteCase(caseExpression *pgsql.Case) error { return s.rewriteExpression(&caseExpression.Else) } +// enter rewrites a node's inbound references and pushes aliases that become visible to its children. func (s *FrameBindingRewriter) enter(node pgsql.SyntaxNode) error { switch typedExpression := node.(type) { case pgsql.Case: @@ -450,7 +463,10 @@ func (s *FrameBindingRewriter) enter(node pgsql.SyntaxNode) error { } case *pgsql.EdgeArrayFromPathIDs: - return s.rewriteExpression(&typedExpression.PathIDs) + if err := s.rewriteExpression(&typedExpression.PathIDs); err != nil { + return err + } + return s.rewriteExpression(&typedExpression.GraphID) case *pgsql.AliasedExpression: switch typedInnerExpression := typedExpression.Expression.(type) { @@ -656,6 +672,7 @@ func (s *FrameBindingRewriter) Enter(node pgsql.SyntaxNode) { } } +// exit removes aliases whose scope ends after the visited node. func (s *FrameBindingRewriter) exit(node pgsql.SyntaxNode) error { switch node.(type) { } diff --git a/cypher/models/pgsql/translate/semantic_drift_test.go b/cypher/models/pgsql/translate/semantic_drift_test.go index a20b7d69..b17a3693 100644 --- a/cypher/models/pgsql/translate/semantic_drift_test.go +++ b/cypher/models/pgsql/translate/semantic_drift_test.go @@ -5,6 +5,8 @@ import ( "testing" "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/cypher/models/cypher" + "github.com/specterops/dawgs/cypher/models/walk" "github.com/specterops/dawgs/drivers/pg/pgutil" "github.com/specterops/dawgs/graph" "github.com/stretchr/testify/require" @@ -40,3 +42,21 @@ func TestTranslatorRejectsUnsupportedPropertyLookupSourcesDirectly(t *testing.T) require.Error(t, err) require.Contains(t, err.Error(), "unsupported property lookup prop on expression type int8[]") } + +// TestTranslatorRejectsEmptyPropertyLookupKeys verifies that invalid empty keys cannot reach SQL translation. +func TestTranslatorRejectsEmptyPropertyLookupKeys(t *testing.T) { + kindMapper := pgutil.NewInMemoryKindMapper() + + query, err := frontend.ParseCypher(frontend.NewContext(), `MATCH (n) RETURN n.name`) + require.NoError(t, err) + + err = walk.CypherStructural(query, walk.NewSimpleVisitor[cypher.SyntaxNode](func(node cypher.SyntaxNode, _ walk.VisitorHandler) { + if propertyLookup, typeOK := node.(*cypher.PropertyLookup); typeOK { + propertyLookup.Symbol = "" + } + })) + require.NoError(t, err) + + _, err = Translate(context.Background(), query, kindMapper, nil, DefaultGraphID) + require.ErrorIs(t, err, cypher.ErrEmptyPropertyKeyName) +} diff --git a/cypher/models/pgsql/translate/shortest_workspace_test.go b/cypher/models/pgsql/translate/shortest_workspace_test.go new file mode 100644 index 00000000..9d2bbdd0 --- /dev/null +++ b/cypher/models/pgsql/translate/shortest_workspace_test.go @@ -0,0 +1,22 @@ +package translate + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestShortestPathWorkspaceFragmentUsesDedicatedTablesAndConstraints verifies isolation and key constraints for each workspace relation. +func TestShortestPathWorkspaceFragmentUsesDedicatedTablesAndConstraints(t *testing.T) { + fragment := "insert into next_front select * from forward_front " + + "where not exists (select 1 from forward_visited) " + + "on conflict on constraint forward_visited_pkey do nothing" + + rewritten := shortestPathWorkspaceFragment(fragment) + require.Equal(t, + "insert into pg_temp.bsp_next_front select * from pg_temp.bsp_forward_front "+ + "where not exists (select 1 from pg_temp.bsp_forward_visited) "+ + "on conflict on constraint bsp_forward_visited_pkey do nothing", + rewritten, + ) +} diff --git a/cypher/models/pgsql/translate/tracking.go b/cypher/models/pgsql/translate/tracking.go index 38f9058e..2a9e0477 100644 --- a/cypher/models/pgsql/translate/tracking.go +++ b/cypher/models/pgsql/translate/tracking.go @@ -2,6 +2,7 @@ package translate import ( "fmt" + "sort" "strconv" "github.com/specterops/dawgs/cypher/models" @@ -109,13 +110,30 @@ func (s *Frame) Reveal(identifier pgsql.Identifier) { // all visible projections. This is required when disambiguating references that otherwise belong to // a frame. type Scope struct { + // nextFrameID is the sequence value assigned to the next scope frame. nextFrameID int - stack []*Frame - generator IdentifierGenerator - aliases map[pgsql.Identifier]pgsql.Identifier + // graphID identifies the graph whose concrete partitions translation targets. + graphID int32 + // stack contains active scope frames from outermost to innermost. + stack []*Frame + // generator allocates collision-free PostgreSQL identifiers by data type. + generator IdentifierGenerator + // aliases maps Cypher-visible symbols to their canonical translated identifiers. + aliases map[pgsql.Identifier]pgsql.Identifier + // definitions maps canonical translated identifiers to their binding metadata. definitions map[pgsql.Identifier]*BoundIdentifier } +// SetGraphID sets the graph used for graph-scoped table references created in this scope. +func (s *Scope) SetGraphID(graphID int32) { + s.graphID = graphID +} + +// GraphID returns the graph used for graph-scoped table references in this scope. +func (s *Scope) GraphID() int32 { + return s.graphID +} + func NewScope() *Scope { return &Scope{ nextFrameID: 0, @@ -378,18 +396,29 @@ func (s *Scope) Define(identifier pgsql.Identifier, dataType pgsql.DataType) *Bo // will eagerly bind anonymous identifiers for traversal steps and rebind existing identifiers and their // aliases to prevent naming collisions. type BoundIdentifier struct { - Identifier pgsql.Identifier - Alias models.Optional[pgsql.Identifier] - Parameter *pgsql.Parameter + // Identifier is the canonical PostgreSQL name allocated for the binding. + Identifier pgsql.Identifier + // Alias is the optional source-visible name projected for the binding. + Alias models.Optional[pgsql.Identifier] + // Parameter is the translated SQL parameter represented by this binding, when applicable. + Parameter *pgsql.Parameter + // LastProjection is the most recent frame that materialized the binding. LastProjection *Frame - Dependencies []*BoundIdentifier - DataType pgsql.DataType + // Dependencies are the bindings required to reconstruct this value. + Dependencies []*BoundIdentifier + // DataType is the PostgreSQL representation carried by the binding. + DataType pgsql.DataType + // IDOnly reports that the binding is represented by a scalar entity ID instead of a composite. + IDOnly bool + // DistanceOnly reports that the binding carries only shortest-path distance state. + DistanceOnly bool } func (s *BoundIdentifier) MaterializedBy(frame *Frame) { s.LastProjection = frame } +// Copy returns an independent binding whose dependency slice can be modified without affecting the source. func (s *BoundIdentifier) Copy() *BoundIdentifier { dependenciesCopy := make([]*BoundIdentifier, len(s.Dependencies)) copy(dependenciesCopy, s.Dependencies) @@ -401,7 +430,34 @@ func (s *BoundIdentifier) Copy() *BoundIdentifier { LastProjection: s.LastProjection, Dependencies: dependenciesCopy, DataType: s.DataType, + IDOnly: s.IDOnly, + DistanceOnly: s.DistanceOnly, + } +} + +// Symbol returns the first deterministic symbol that aliases binding. +func (s *Scope) Symbol(binding *BoundIdentifier) (pgsql.Identifier, bool) { + if symbols := s.Symbols(binding); len(symbols) > 0 { + return symbols[0], true + } + + return "", false +} + +// Symbols returns every symbol that aliases binding in lexical order. +func (s *Scope) Symbols(binding *BoundIdentifier) []pgsql.Identifier { + if binding == nil { + return nil + } + + var symbols []pgsql.Identifier + for symbol, identifier := range s.aliases { + if identifier == binding.Identifier { + symbols = append(symbols, symbol) + } } + sort.Slice(symbols, func(left, right int) bool { return symbols[left] < symbols[right] }) + return symbols } func (s *BoundIdentifier) Dematerialize() { diff --git a/cypher/models/pgsql/translate/translator.go b/cypher/models/pgsql/translate/translator.go index 8a53e5c1..24df003a 100644 --- a/cypher/models/pgsql/translate/translator.go +++ b/cypher/models/pgsql/translate/translator.go @@ -12,45 +12,85 @@ import ( "github.com/specterops/dawgs/graph" ) -// DefaultGraphID is the graph_id used by callers that do not have a specific -// graph target available (tests, tooling, and visualization passes that only -// exercise translation output). +// DefaultGraphID selects graph zero for tests and tooling that do not target a concrete graph. const DefaultGraphID int32 = 0 +// Translator walks an optimized Cypher AST and constructs the corresponding PostgreSQL AST. type Translator struct { + // Visitor supplies traversal control and error propagation for the Cypher walk. walk.Visitor[cypher.SyntaxNode] - ctx context.Context - kindMapper *contextAwareKindMapper - graphID int32 - parameters map[string]any - translation Result + // ctx carries cancellation and deadlines through translation. + ctx context.Context + // kindMapper resolves graph kind names within the translation context. + kindMapper *contextAwareKindMapper + // graphID identifies the concrete graph partitions targeted by generated SQL. + graphID int32 + // parameters is an isolated copy of the caller's Cypher parameter values. + parameters map[string]any + // translation accumulates the statement, generated parameters, and diagnostics. + translation Result + // treeTranslator lowers the current Cypher expression tree into PostgreSQL expressions. treeTranslator *ExpressionTreeTranslator - query *Query - scope *Scope - unwindTargets map[*cypher.Variable]struct{} - + // query holds the PostgreSQL query model under construction. + query *Query + // scope tracks translated bindings and their materialization frames. + scope *Scope + // unwindTargets contains UNWIND variables awaiting source translation. + unwindTargets map[*cypher.Variable]struct{} + + // collectIDMembershipAliases identifies collect projections eligible to carry scalar entity IDs. collectIDMembershipAliases map[pgsql.Identifier]struct{} - collectIDProjectionDepth int - - appliedLoweringCounts map[string]int - patternTargets map[*cypher.PatternPart]optimize.PatternTarget - patternPredicateTargets map[*cypher.PatternPredicate]optimize.PatternTarget - projectionPruningDecisions map[optimize.TraversalStepTarget]optimize.ProjectionPruningDecision - latePathDecisions map[optimize.TraversalStepTarget][]optimize.LatePathMaterializationDecision - suffixPushdownDecisions map[optimize.TraversalStepTarget][]optimize.ExpansionSuffixPushdownDecision - predicatePlacementDecisions map[optimize.TraversalStepTarget][]optimize.PredicatePlacementDecision - expandIntoDecisions map[optimize.TraversalStepTarget]optimize.ExpandIntoDecision - traversalDirectionDecisions map[optimize.TraversalStepTarget]optimize.TraversalDirectionDecision - shortestPathStrategyDecisions map[optimize.TraversalStepTarget]optimize.ShortestPathStrategyDecision - shortestPathFilterDecisions map[optimize.TraversalStepTarget][]optimize.ShortestPathFilterDecision - limitPushdownDecisions map[optimize.TraversalStepTarget][]optimize.LimitPushdownDecision - patternPredicateDecisions map[optimize.TraversalStepTarget]optimize.PatternPredicatePlacementDecision - exactRangeExpansionDecisions map[optimize.TraversalStepTarget]optimize.ExactRangeExpansionDecision + // collectIDProjectionDepth tracks nesting within an ID-only collect projection. + collectIDProjectionDepth int + + // appliedLoweringCounts counts emitted applications of each planned lowering. + appliedLoweringCounts map[string]int + // appliedShortestPathExecutors records the physical executor emitted for each optimized traversal. + appliedShortestPathExecutors map[optimize.TraversalStepTarget]optimize.ShortestPathExecutor + // appliedExpansionSearchStrategies records the physical search emitted for each optimized expansion. + appliedExpansionSearchStrategies map[optimize.TraversalStepTarget]optimize.ExpansionSearchStrategy + // emittedExpansionSearchPolicies records runtime selection policies emitted for optimized expansions. + emittedExpansionSearchPolicies map[optimize.TraversalStepTarget]optimize.ExpansionSearchPolicy + // patternTargets maps source pattern parts to their stable optimizer coordinates. + patternTargets map[*cypher.PatternPart]optimize.PatternTarget + // patternPredicateTargets maps source pattern predicates to their stable optimizer coordinates. + patternPredicateTargets map[*cypher.PatternPredicate]optimize.PatternTarget + // projectionPruningDecisions indexes planned projection omissions by traversal target. + projectionPruningDecisions map[optimize.TraversalStepTarget]optimize.ProjectionPruningDecision + // latePathDecisions indexes deferred path-materialization decisions by traversal target. + latePathDecisions map[optimize.TraversalStepTarget][]optimize.LatePathMaterializationDecision + // suffixPushdownDecisions indexes fixed-suffix pushdown decisions by traversal target. + suffixPushdownDecisions map[optimize.TraversalStepTarget][]optimize.ExpansionSuffixPushdownDecision + // predicatePlacementDecisions indexes predicate attachment decisions by traversal target. + predicatePlacementDecisions map[optimize.TraversalStepTarget][]optimize.PredicatePlacementDecision + // expandIntoDecisions indexes bound-endpoint expansion choices by traversal target. + expandIntoDecisions map[optimize.TraversalStepTarget]optimize.ExpandIntoDecision + // traversalDirectionDecisions indexes physical traversal direction choices by traversal target. + traversalDirectionDecisions map[optimize.TraversalStepTarget]optimize.TraversalDirectionDecision + // shortestPathStrategyDecisions indexes directional shortest-path search choices by traversal target. + shortestPathStrategyDecisions map[optimize.TraversalStepTarget]optimize.ShortestPathStrategyDecision + // shortestPathFilterDecisions indexes shortest-path filter decisions by traversal target. + shortestPathFilterDecisions map[optimize.TraversalStepTarget][]optimize.ShortestPathFilterDecision + // shortestPathExecutorDecisions indexes planned shortest-path executor choices by traversal target. + shortestPathExecutorDecisions map[optimize.TraversalStepTarget]optimize.ShortestPathExecutorDecision + // expansionSearchStrategyDecisions indexes planned variable-expansion strategies by traversal target. + expansionSearchStrategyDecisions map[optimize.TraversalStepTarget]optimize.ExpansionSearchStrategyDecision + // limitPushdownDecisions indexes planned traversal limits by source target. + limitPushdownDecisions map[optimize.TraversalStepTarget][]optimize.LimitPushdownDecision + // patternPredicateDecisions indexes planned existence lowering by traversal target. + patternPredicateDecisions map[optimize.TraversalStepTarget]optimize.PatternPredicatePlacementDecision + // exactRangeExpansionDecisions indexes fixed-depth unrolling choices by source target. + exactRangeExpansionDecisions map[optimize.TraversalStepTarget]optimize.ExactRangeExpansionDecision + // pathRelationshipPredicateDecisions indexes path quantifier lowering by stable quantifier target. pathRelationshipPredicateDecisions map[optimize.QuantifierTarget]optimize.PathRelationshipPredicateDecision - quantifierTargets []optimize.QuantifierTarget + // fieldRequirementDecisions indexes binding representation requirements by query part and symbol. + fieldRequirementDecisions map[int]map[string]optimize.FieldRequirementDecision + // quantifierTargets records stable coordinates for visited quantified traversals. + quantifierTargets []optimize.QuantifierTarget } +// NewTranslator initializes translation state for the supplied graph and copies the caller's parameter map. func NewTranslator(ctx context.Context, kindMapper pgsql.KindMapper, parameters map[string]any, graphID int32) *Translator { if parameters == nil { parameters = map[string]any{} @@ -66,10 +106,12 @@ func NewTranslator(ctx context.Context, kindMapper pgsql.KindMapper, parameters ctxAwareKindMapper = newContextAwareKindMapper(ctx, kindMapper, translatedParameters) ) - return &Translator{ + translator := &Translator{ Visitor: walk.NewVisitor[cypher.SyntaxNode](), translation: Result{ - Parameters: translatedParameters, + Parameters: translatedParameters, + ParameterSources: map[string]string{}, + GraphID: graphID, }, ctx: ctx, kindMapper: ctxAwareKindMapper, @@ -80,8 +122,12 @@ func NewTranslator(ctx context.Context, kindMapper pgsql.KindMapper, parameters scope: NewScope(), unwindTargets: map[*cypher.Variable]struct{}{}, } + + translator.scope.SetGraphID(graphID) + return translator } +// SetOptimizationPlan indexes lowering decisions by their stable targets for use during AST traversal. func (s *Translator) SetOptimizationPlan(plan optimize.Plan) { s.patternTargets = optimize.IndexPatternTargets(plan.Query) s.patternPredicateTargets = optimize.IndexPatternPredicateTargets(plan.Query) @@ -93,10 +139,13 @@ func (s *Translator) SetOptimizationPlan(plan optimize.Plan) { s.traversalDirectionDecisions = map[optimize.TraversalStepTarget]optimize.TraversalDirectionDecision{} s.shortestPathStrategyDecisions = map[optimize.TraversalStepTarget]optimize.ShortestPathStrategyDecision{} s.shortestPathFilterDecisions = map[optimize.TraversalStepTarget][]optimize.ShortestPathFilterDecision{} + s.shortestPathExecutorDecisions = map[optimize.TraversalStepTarget]optimize.ShortestPathExecutorDecision{} + s.expansionSearchStrategyDecisions = map[optimize.TraversalStepTarget]optimize.ExpansionSearchStrategyDecision{} s.limitPushdownDecisions = map[optimize.TraversalStepTarget][]optimize.LimitPushdownDecision{} s.patternPredicateDecisions = map[optimize.TraversalStepTarget]optimize.PatternPredicatePlacementDecision{} s.exactRangeExpansionDecisions = map[optimize.TraversalStepTarget]optimize.ExactRangeExpansionDecision{} s.pathRelationshipPredicateDecisions = map[optimize.QuantifierTarget]optimize.PathRelationshipPredicateDecision{} + s.fieldRequirementDecisions = map[int]map[string]optimize.FieldRequirementDecision{} for _, decision := range plan.LoweringPlan.ProjectionPruning { s.projectionPruningDecisions[decision.Target] = decision @@ -130,6 +179,14 @@ func (s *Translator) SetOptimizationPlan(plan optimize.Plan) { s.shortestPathFilterDecisions[decision.Target] = append(s.shortestPathFilterDecisions[decision.Target], decision) } + for _, decision := range plan.LoweringPlan.ShortestPathExecutor { + s.shortestPathExecutorDecisions[decision.Target] = decision + } + + for _, decision := range plan.LoweringPlan.ExpansionSearchStrategy { + s.expansionSearchStrategyDecisions[decision.Target] = decision + } + for _, decision := range plan.LoweringPlan.LimitPushdown { s.limitPushdownDecisions[decision.Target] = append(s.limitPushdownDecisions[decision.Target], decision) } @@ -145,8 +202,18 @@ func (s *Translator) SetOptimizationPlan(plan optimize.Plan) { for _, decision := range plan.LoweringPlan.PathRelationshipPredicate { s.pathRelationshipPredicateDecisions[decision.Target] = decision } + + for _, decision := range plan.LoweringPlan.FieldRequirements { + bySymbol := s.fieldRequirementDecisions[decision.QueryPartIndex] + if bySymbol == nil { + bySymbol = map[string]optimize.FieldRequirementDecision{} + s.fieldRequirementDecisions[decision.QueryPartIndex] = bySymbol + } + bySymbol[decision.Symbol] = decision + } } +// Enter translates a Cypher syntax node when the walker reaches it. func (s *Translator) Enter(expression cypher.SyntaxNode) { switch typedExpression := expression.(type) { case *cypher.RegularQuery, *cypher.SingleQuery, *cypher.PatternElement, @@ -227,6 +294,9 @@ func (s *Translator) Enter(expression cypher.SyntaxNode) { } else { // Lift the parameter value into the parameters map s.translation.Parameters[parameterBinding.Identifier.String()] = negotiatedValue + if typedExpression.Symbol != "" { + s.translation.ParameterSources[parameterBinding.Identifier.String()] = typedExpression.Symbol + } parameterBinding.Parameter = newParameter } @@ -238,7 +308,11 @@ func (s *Translator) Enter(expression cypher.SyntaxNode) { s.treeTranslator.PushOperand(binding.Parameter) case *cypher.Variable: - if binding, isUnwindTarget, err := s.prepareUnwindTarget(typedExpression); err != nil { + if typedExpression.Symbol == cypher.TokenLiteralAsterisk { + // Greedy projections are expanded to their named scope bindings when + // the enclosing projection item is completed. + s.treeTranslator.PushOperand(pgsql.Identifier(cypher.TokenLiteralAsterisk)) + } else if binding, isUnwindTarget, err := s.prepareUnwindTarget(typedExpression); err != nil { s.SetError(err) } else if isUnwindTarget { s.treeTranslator.PushOperand(binding.Identifier) @@ -334,6 +408,7 @@ func (s *Translator) Enter(expression cypher.SyntaxNode) { } } +// resolveParameterValue returns the caller-supplied value for a Cypher parameter or reports an unknown parameter. func (s *Translator) resolveParameterValue(parameter *cypher.Parameter) any { if value, hasValue := s.parameters[parameter.Symbol]; hasValue { return value @@ -342,6 +417,7 @@ func (s *Translator) resolveParameterValue(parameter *cypher.Parameter) any { return parameter.Value } +// coalescePropertyLookupExpression builds a coalesce call from a property lookup and translated fallback operands. func coalescePropertyLookupExpression(expression pgsql.Expression) pgsql.Expression { if propertyLookup, isPropertyLookup := expressionToPropertyLookupBinaryExpression(expression); isPropertyLookup { return pgsql.FunctionCall{ @@ -357,6 +433,7 @@ func coalescePropertyLookupExpression(expression pgsql.Expression) pgsql.Express return expression } +// rewriteNegatedStringPredicateExpression preserves Cypher null behavior when negating a string predicate. func rewriteNegatedStringPredicateExpression(expression pgsql.Expression) pgsql.Expression { switch typedExpression := expression.(type) { case *pgsql.Parenthetical: @@ -639,19 +716,151 @@ func (s *Translator) Exit(expression cypher.SyntaxNode) { } } +// Result contains the translated PostgreSQL statement, parameters, graph target, and optimization diagnostics. type Result struct { - Statement pgsql.Statement - Parameters map[string]any + // Statement is the translated PostgreSQL AST. + Statement pgsql.Statement + // Parameters contains SQL parameters generated during translation. + Parameters map[string]any + // ParameterSources maps generated SQL parameter names back to Cypher parameter names. + ParameterSources map[string]string + // Optimization summarizes planned, applied, and skipped lowering decisions. Optimization OptimizationSummary + // GraphID identifies the graph partitions targeted by the statement. + GraphID int32 } +// OptimizationSummary records which optimizer decisions were planned, applied, or skipped during translation. type OptimizationSummary struct { - Rules []optimize.RuleResult `json:"rules,omitempty"` + // Rules contains the semantic optimizer rule results in execution order. + Rules []optimize.RuleResult `json:"rules,omitempty"` + // PredicateAttachments records optimizer-selected predicate scopes. PredicateAttachments []optimize.PredicateAttachment `json:"predicate_attachments,omitempty"` - PlannedLowerings []optimize.LoweringDecision `json:"planned_lowerings,omitempty"` - Lowerings []optimize.LoweringDecision `json:"lowerings,omitempty"` - SkippedLowerings []SkippedLowering `json:"skipped_lowerings,omitempty"` - LoweringPlan *optimize.LoweringPlan `json:"lowering_plan,omitempty"` + // PlannedLowerings summarizes lowering categories selected by the optimizer. + PlannedLowerings []optimize.LoweringDecision `json:"planned_lowerings,omitempty"` + // Lowerings summarizes lowering categories actually emitted by translation. + Lowerings []optimize.LoweringDecision `json:"lowerings,omitempty"` + // SkippedLowerings explains planned lowering applications that translation did not emit. + SkippedLowerings []SkippedLowering `json:"skipped_lowerings,omitempty"` + // TargetOutcomes reports selection and application results for each lowering target. + TargetOutcomes []TargetLoweringOutcome `json:"target_outcomes,omitempty"` + // LoweringPlan exposes the optimizer decisions used to translate the statement. + LoweringPlan *optimize.LoweringPlan `json:"lowering_plan,omitempty"` +} + +// TargetLoweringOutcome reports how one planned lowering target was qualified, selected, and applied. +type TargetLoweringOutcome struct { + // Lowering names the lowering pass that produced this outcome. + Lowering string `json:"lowering"` + // TargetKind identifies the kind of syntax or binding targeted by the lowering. + TargetKind string `json:"target_kind"` + // TraversalTarget locates a traversal-step target when the lowering applies to one. + TraversalTarget *optimize.TraversalStepTarget `json:"traversal_target,omitempty"` + // QueryPartIndex locates a query-part target when the lowering applies to one. + QueryPartIndex *int `json:"query_part_index,omitempty"` + // Symbol identifies a binding target when the lowering applies to one. + Symbol string `json:"symbol,omitempty"` + // Family names the candidate-selection family that produced this outcome. + Family string `json:"family,omitempty"` + // TraversalFamily preserves the SP/ASP family for analysis-only decisions + // whose outcome family must remain distinct from an executable traversal. + TraversalFamily string `json:"traversal_family,omitempty"` + // PlannedPolicy identifies the runtime policy intended for this candidate + // family, whether or not it was emitted. + PlannedPolicy string `json:"planned_policy,omitempty"` + // EmittedPolicy identifies a runtime policy present in translated SQL. A + // single incumbent or tool-forced arm has no emitted policy identity. + EmittedPolicy string `json:"emitted_policy,omitempty"` + // PlannedCandidates lists the candidates considered in preference order. + PlannedCandidates []string `json:"planned_candidates,omitempty"` + // EmittedCandidates lists the arms present in translated SQL. Runtime + // telemetry separately records which arm executed. + EmittedCandidates []string `json:"emitted_candidates,omitempty"` + // ProbeCaps records bounded evidence inputs for an expansion policy. + ProbeCaps *optimize.ExpansionSearchProbeCaps `json:"probe_caps,omitempty"` + // Admission records the specialized-state gate and exact fallback chain. + Admission *optimize.ExpansionSearchAdmission `json:"admission,omitempty"` + // EndpointRoot and EndpointTerminal describe the bounded endpoint inputs + // considered by analysis without implying that translation emitted them. + EndpointRoot *optimize.EndpointResolutionInput `json:"endpoint_root,omitempty"` + EndpointTerminal *optimize.EndpointResolutionInput `json:"endpoint_terminal,omitempty"` + // EndpointPairClass records a correlation class when endpoint resolution + // must preserve a paired input rather than independent endpoint sets. + EndpointPairClass optimize.EndpointResolutionClass `json:"endpoint_pair_class,omitempty"` + // EndpointResolutionCaps records immutable 1/2/32/33 admission sentinels. + EndpointResolutionCaps *optimize.EndpointResolutionCaps `json:"endpoint_resolution_caps,omitempty"` + // PredicateClass and its source/index expose conservative traversal + // predicate placement analysis as a first-class target outcome. + PredicateClass optimize.TraversalPredicateClass `json:"predicate_class,omitempty"` + PredicateSource string `json:"predicate_source,omitempty"` + PredicateIndex *int `json:"predicate_index,omitempty"` + // Scheduler identifies the selected shortest-path frontier scheduling policy. + Scheduler string `json:"scheduler,omitempty"` + // ExecutionBoundary identifies whether the selected executor is inline SQL, + // a stored helper, or a guarded multi-arm statement. + ExecutionBoundary string `json:"execution_boundary,omitempty"` + // Candidate is the specialized candidate proposed by analysis. + Candidate string `json:"candidate,omitempty"` + // EligibilityFacts records named qualification checks for the candidate. + EligibilityFacts []TargetEligibilityFact `json:"eligibility_facts,omitempty"` + // ObservationMode describes how downstream clauses consume the target. + ObservationMode string `json:"observation_mode,omitempty"` + // Direction records the target's logical traversal direction. + Direction string `json:"direction,omitempty"` + // PhysicalExpansion records the stored edge endpoint used to advance traversal. + PhysicalExpansion string `json:"physical_expansion,omitempty"` + // RelationshipKindCount is the number of statically resolved relationship kinds. + RelationshipKindCount int `json:"relationship_kind_count,omitempty"` + // UntypedRelationship reports whether the pattern omitted relationship kinds. + UntypedRelationship bool `json:"untyped_relationship,omitempty"` + // TopologyClassification summarizes logical direction, physical direction, and depth. + TopologyClassification string `json:"topology_classification,omitempty"` + // Eligible reports the structural qualification result when one is available. + Eligible *bool `json:"eligible,omitempty"` + // StaticallyEligible reports the literal- and kind-based qualification result when available. + StaticallyEligible *bool `json:"statically_eligible,omitempty"` + // SelectionMode records whether selection was automatic or forced by tooling. + SelectionMode string `json:"selection_mode,omitempty"` + // SelectorVersion identifies the policy version that ranked candidates. + SelectorVersion string `json:"selector_version,omitempty"` + // Fallback names the candidate used if the preferred lowering was not applied. + Fallback string `json:"fallback,omitempty"` + // MinimumDepth is the target's inclusive lower traversal-depth bound. + MinimumDepth *int64 `json:"minimum_depth,omitempty"` + // MaximumDepth is the target's inclusive upper traversal-depth bound when finite. + MaximumDepth *int64 `json:"maximum_depth,omitempty"` + // StateLimit is the maximum intermediate-state count admitted by the candidate. + StateLimit int64 `json:"state_limit,omitempty"` + // FrontierLimit is the maximum current or queued frontier size admitted by a shortest-path candidate. + FrontierLimit int64 `json:"frontier_limit,omitempty"` + // PredecessorLimit is the maximum retained witness predecessor state admitted by a shortest-path candidate. + PredecessorLimit int64 `json:"predecessor_limit,omitempty"` + // EnumerationLimit is the maximum distinct ordered path count staged by an all-shortest-path candidate. + EnumerationLimit int64 `json:"enumeration_limit,omitempty"` + // OutputBytesLimit is the maximum staged ordered edge-array bytes admitted by an all-shortest-path candidate. + OutputBytesLimit int64 `json:"output_bytes_limit,omitempty"` + // EndpointLimit is the maximum endpoint-seed count admitted by the candidate. + EndpointLimit int64 `json:"endpoint_limit,omitempty"` + // SeedPredicateClass describes the predicate used to bound search seeds. + SeedPredicateClass string `json:"seed_predicate_class,omitempty"` + // PrefixLength is the number of fixed steps before the variable expansion. + PrefixLength int `json:"prefix_length,omitempty"` + // HasFinalLimit reports whether a final row limit influenced candidate selection. + HasFinalLimit bool `json:"has_final_limit,omitempty"` + // Selected names the candidate selected by the optimizer. + Selected string `json:"selected,omitempty"` + // Applied names the candidate actually emitted by translation. + Applied string `json:"applied,omitempty"` + // SkipReason explains why a planned candidate was not emitted. + SkipReason string `json:"skip_reason,omitempty"` +} + +// TargetEligibilityFact reports one named qualification result in a translated target outcome. +type TargetEligibilityFact struct { + // Name identifies the qualification check. + Name string `json:"name"` + // Eligible reports whether the target passed the named check. + Eligible bool `json:"eligible"` } type SkippedLowering struct { @@ -660,6 +869,7 @@ type SkippedLowering struct { Count int `json:"count,omitempty"` } +// recordLowering increments the applied count for one lowering name. func (s *Translator) recordLowering(name string) { if s.appliedLoweringCounts == nil { s.appliedLoweringCounts = map[string]int{} @@ -675,6 +885,34 @@ func (s *Translator) recordLowering(name string) { s.translation.Optimization.Lowerings = append(s.translation.Optimization.Lowerings, optimize.LoweringDecision{Name: name}) } +// recordShortestPathExecutor records the executor actually emitted for a traversal target. +func (s *Translator) recordShortestPathExecutor(target optimize.TraversalStepTarget, executor optimize.ShortestPathExecutor) { + if s.appliedShortestPathExecutors == nil { + s.appliedShortestPathExecutors = map[optimize.TraversalStepTarget]optimize.ShortestPathExecutor{} + } + s.appliedShortestPathExecutors[target] = executor + s.recordLowering(optimize.LoweringShortestPathExecutor) +} + +// recordExpansionSearchStrategy records the expansion strategy actually emitted for a traversal target. +func (s *Translator) recordExpansionSearchStrategy(target optimize.TraversalStepTarget, strategy optimize.ExpansionSearchStrategy) { + if s.appliedExpansionSearchStrategies == nil { + s.appliedExpansionSearchStrategies = map[optimize.TraversalStepTarget]optimize.ExpansionSearchStrategy{} + } + s.appliedExpansionSearchStrategies[target] = strategy + s.recordLowering(optimize.LoweringExpansionSearchStrategy) +} + +// recordExpansionSearchPolicy records a runtime expansion policy actually emitted for a traversal target. +func (s *Translator) recordExpansionSearchPolicy(target optimize.TraversalStepTarget, policy optimize.ExpansionSearchPolicy) { + if s.emittedExpansionSearchPolicies == nil { + s.emittedExpansionSearchPolicies = map[optimize.TraversalStepTarget]optimize.ExpansionSearchPolicy{} + } + s.emittedExpansionSearchPolicies[target] = policy + s.recordLowering(optimize.LoweringExpansionSearchStrategy) +} + +// appliedLoweringCountSnapshot merges optimizer-declared and translator-observed lowering counts into the snapshot used to diagnose unapplied plans. func (s *Translator) appliedLoweringCountSnapshot() map[string]int { applied := map[string]int{} @@ -689,12 +927,14 @@ func (s *Translator) appliedLoweringCountSnapshot() map[string]int { return applied } +// recordSkippedLowerings compares the plan with applied counts and emits aggregated skip diagnostics. func (s *Translator) recordSkippedLowerings() { if s.translation.Optimization.LoweringPlan == nil { return } applied := s.appliedLoweringCountSnapshot() + s.recordTargetOutcomes(*s.translation.Optimization.LoweringPlan) for _, planned := range plannedLoweringCounts(*s.translation.Optimization.LoweringPlan) { if planned.Count == 0 { @@ -714,6 +954,246 @@ func (s *Translator) recordSkippedLowerings() { } } +// recordTargetOutcomes converts per-target plan decisions and applied choices into diagnostic outcomes. +func (s *Translator) recordTargetOutcomes(plan optimize.LoweringPlan) { + if len(s.translation.Optimization.TargetOutcomes) != 0 { + return + } + for _, decision := range plan.ShortestPathExecutor { + target := decision.Target + eligible, staticallyEligible := decision.StructurallyEligible, decision.StaticallyEligible + minimumDepth, maximumDepth := decision.MinimumDepth, decision.MaximumDepth + applied := string(s.appliedShortestPathExecutors[target]) + outcome := TargetLoweringOutcome{ + Lowering: optimize.LoweringShortestPathExecutor, + TargetKind: "traversal", + TraversalTarget: &target, + Family: decision.Family, + PlannedCandidates: shortestPathCandidateNames(decision.PlannedCandidates), + Scheduler: string(decision.Scheduler), + ExecutionBoundary: decision.ExecutionBoundary, + EligibilityFacts: shortestPathEligibilityFacts(decision.Eligibility), + ObservationMode: string(decision.ObservationMode), + Direction: decision.Direction.String(), + PhysicalExpansion: string(decision.PhysicalExpansion), + RelationshipKindCount: decision.RelationshipKindCount, + UntypedRelationship: decision.UntypedRelationship, + TopologyClassification: string(decision.TopologyClassification), + Eligible: &eligible, + StaticallyEligible: &staticallyEligible, + SelectionMode: decision.SelectionMode, + SelectorVersion: decision.SelectorVersion, + Selected: string(decision.SelectedExecutor), + Applied: applied, + Fallback: string(decision.FallbackExecutor), + SkipReason: decision.FallbackReason, + MinimumDepth: &minimumDepth, + MaximumDepth: &maximumDepth, + StateLimit: decision.StateLimit, + FrontierLimit: decision.FrontierLimit, + PredecessorLimit: decision.PredecessorLimit, + EnumerationLimit: decision.EnumerationLimit, + OutputBytesLimit: decision.OutputBytesLimit, + } + if decision.SelectedExecutor == optimize.ShortestPathExecutorASPI1DAG && applied == string(optimize.ShortestPathExecutorASPI1DAG) { + outcome.Candidate = string(optimize.ShortestPathExecutorASPI1DAG) + outcome.EmittedPolicy = optimize.ShortestPathPolicyASPI1GuardedV1 + outcome.EmittedCandidates = []string{ + string(optimize.ShortestPathExecutorASPI1DAG), + string(optimize.ShortestPathExecutorASPA1DAG), + } + } + if decision.SelectedExecutor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness && applied == string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness) { + outcome.Candidate = string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness) + outcome.EmittedPolicy = optimize.ShortestPathPolicyI1CanonicalGuardedV1 + outcome.EmittedCandidates = []string{ + string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + string(optimize.ShortestPathExecutorS4CanonicalWitness), + } + } + s.translation.Optimization.TargetOutcomes = append(s.translation.Optimization.TargetOutcomes, outcome) + } + for _, decision := range plan.ExpansionSearchStrategy { + target := decision.Target + eligible, staticallyEligible := decision.StructurallyEligible, decision.StaticallyEligible + minimumDepth, maximumDepth := decision.MinimumDepth, decision.MaximumDepth + applied := string(s.appliedExpansionSearchStrategies[target]) + probeCaps, admission := decision.ProbeCaps, decision.Admission + s.translation.Optimization.TargetOutcomes = append(s.translation.Optimization.TargetOutcomes, TargetLoweringOutcome{ + Lowering: optimize.LoweringExpansionSearchStrategy, + TargetKind: "traversal", + TraversalTarget: &target, + Family: decision.Family, + PlannedPolicy: string(decision.PlannedPolicy), + EmittedPolicy: string(decision.EmittedPolicy), + PlannedCandidates: expansionSearchCandidateNames(decision.PlannedCandidates), + EmittedCandidates: expansionSearchCandidateNames(decision.EmittedCandidates), + ExecutionBoundary: decision.ExecutionBoundary, + ProbeCaps: &probeCaps, + Admission: &admission, + Candidate: string(decision.CandidateStrategy), + EligibilityFacts: expansionSearchEligibilityFacts(decision.EligibilityFacts), + ObservationMode: string(decision.ObservationMode), + Eligible: &eligible, + StaticallyEligible: &staticallyEligible, + SelectionMode: decision.SelectionMode, + SelectorVersion: decision.SelectorVersion, + Selected: string(decision.SelectedStrategy), + Applied: applied, + Fallback: string(decision.FallbackStrategy), + SkipReason: decision.FallbackReason, + MinimumDepth: &minimumDepth, + MaximumDepth: &maximumDepth, + StateLimit: decision.StateLimit, + EndpointLimit: decision.EndpointLimit, + SeedPredicateClass: decision.SeedPredicateClass, + PrefixLength: decision.PrefixLength, + HasFinalLimit: decision.HasFinalLimit, + }) + } + for _, decision := range plan.EndpointResolution { + target := decision.Target + eligible, staticallyEligible := decision.StructurallyEligible, decision.StaticallyEligible + root, terminal, caps := decision.Root, decision.Terminal, decision.Caps + s.translation.Optimization.TargetOutcomes = append(s.translation.Optimization.TargetOutcomes, TargetLoweringOutcome{ + Lowering: optimize.LoweringEndpointResolution, + TargetKind: "endpoint_resolution", + TraversalTarget: &target, + Family: "endpoint_resolution", + TraversalFamily: decision.Family, + PlannedCandidates: endpointResolutionCandidateNames(decision.PlannedCandidates), + EndpointRoot: &root, + EndpointTerminal: &terminal, + EndpointPairClass: decision.PairClass, + EndpointResolutionCaps: &caps, + Candidate: string(decision.CandidatePlan), + EligibilityFacts: endpointResolutionEligibilityFacts(decision.EligibilityFacts), + Eligible: &eligible, + StaticallyEligible: &staticallyEligible, + SelectionMode: decision.SelectionMode, + SelectorVersion: decision.SelectorVersion, + Selected: string(decision.SelectedPlan), + Applied: string(decision.SelectedPlan), + Fallback: string(decision.FallbackPlan), + SkipReason: decision.FallbackReason, + }) + } + for _, decision := range plan.TraversalPredicate { + target, predicateIndex := decision.Target, decision.PredicateIndex + eligible, staticallyEligible := decision.StructurallyEligible, decision.StaticallyEligible + s.translation.Optimization.TargetOutcomes = append(s.translation.Optimization.TargetOutcomes, TargetLoweringOutcome{ + Lowering: optimize.LoweringTraversalPredicateClassification, + TargetKind: "traversal_predicate", + TraversalTarget: &target, + Family: "traversal_predicate", + PlannedCandidates: traversalPredicateCandidateNames(decision.PlannedCandidates), + PredicateClass: decision.Class, + PredicateSource: decision.Source, + PredicateIndex: &predicateIndex, + Candidate: string(decision.CandidatePlan), + EligibilityFacts: traversalPredicateEligibilityFacts(decision.EligibilityFacts), + Eligible: &eligible, + StaticallyEligible: &staticallyEligible, + SelectionMode: decision.SelectionMode, + SelectorVersion: decision.ClassifierVersion, + Selected: string(decision.SelectedPlan), + Applied: string(decision.SelectedPlan), + Fallback: string(decision.FallbackPlan), + SkipReason: decision.FallbackReason, + }) + } + for _, decision := range plan.FieldRequirements { + queryPartIndex := decision.QueryPartIndex + s.translation.Optimization.TargetOutcomes = append(s.translation.Optimization.TargetOutcomes, TargetLoweringOutcome{ + Lowering: optimize.LoweringFieldRequirements, + TargetKind: "field_requirement", + QueryPartIndex: &queryPartIndex, + Symbol: decision.Symbol, + Selected: "analysis_only", + SkipReason: "analysis_metadata_only", + }) + } +} + +// shortestPathCandidateNames converts executor candidates to their stable diagnostic names. +func shortestPathCandidateNames(candidates []optimize.ShortestPathExecutor) []string { + names := make([]string, len(candidates)) + for idx, candidate := range candidates { + names[idx] = string(candidate) + } + return names +} + +// expansionSearchCandidateNames converts expansion candidates to their stable diagnostic names. +func expansionSearchCandidateNames(candidates []optimize.ExpansionSearchStrategy) []string { + names := make([]string, len(candidates)) + for idx, candidate := range candidates { + names[idx] = string(candidate) + } + return names +} + +// endpointResolutionCandidateNames converts analysis-only endpoint plans to +// their stable diagnostic identities. +func endpointResolutionCandidateNames(candidates []optimize.EndpointResolutionPlan) []string { + names := make([]string, len(candidates)) + for idx, candidate := range candidates { + names[idx] = string(candidate) + } + return names +} + +// traversalPredicateCandidateNames converts predicate-placement plans to +// their stable diagnostic identities. +func traversalPredicateCandidateNames(candidates []optimize.TraversalPredicatePlan) []string { + names := make([]string, len(candidates)) + for idx, candidate := range candidates { + names[idx] = string(candidate) + } + return names +} + +// shortestPathEligibilityFacts converts executor qualification facts to public diagnostic records. +func shortestPathEligibilityFacts(facts []optimize.ShortestPathEligibilityFact) []TargetEligibilityFact { + outcomes := make([]TargetEligibilityFact, len(facts)) + for idx, fact := range facts { + outcomes[idx] = TargetEligibilityFact{ + Name: fact.Name, + Eligible: fact.Eligible, + } + } + return outcomes +} + +// expansionSearchEligibilityFacts converts search-strategy qualification facts to public diagnostic records. +func expansionSearchEligibilityFacts(facts []optimize.ExpansionSearchEligibilityFact) []TargetEligibilityFact { + outcomes := make([]TargetEligibilityFact, len(facts)) + for idx, fact := range facts { + outcomes[idx] = TargetEligibilityFact{ + Name: fact.Name, + Eligible: fact.Eligible, + } + } + return outcomes +} + +func endpointResolutionEligibilityFacts(facts []optimize.EndpointResolutionEligibilityFact) []TargetEligibilityFact { + outcomes := make([]TargetEligibilityFact, len(facts)) + for idx, fact := range facts { + outcomes[idx] = TargetEligibilityFact{Name: fact.Name, Eligible: fact.Eligible} + } + return outcomes +} + +func traversalPredicateEligibilityFacts(facts []optimize.TraversalPredicateEligibilityFact) []TargetEligibilityFact { + outcomes := make([]TargetEligibilityFact, len(facts)) + for idx, fact := range facts { + outcomes[idx] = TargetEligibilityFact{Name: fact.Name, Eligible: fact.Eligible} + } + return outcomes +} + +// plannedLoweringCounts converts each lowering target collection into a named count so planned work can be reconciled with applied work. func plannedLoweringCounts(plan optimize.LoweringPlan) []SkippedLowering { return []SkippedLowering{ { @@ -748,6 +1228,10 @@ func plannedLoweringCounts(plan optimize.LoweringPlan) []SkippedLowering { Name: optimize.LoweringExpansionSuffixPushdown, Count: len(plan.ExpansionSuffixPushdown), }, + { + Name: optimize.LoweringExpansionSearchStrategy, + Count: len(plan.ExpansionSearchStrategy), + }, { Name: optimize.LoweringPredicatePlacement, Count: len(plan.PredicatePlacement) + len(plan.PatternPredicate), @@ -768,10 +1252,22 @@ func plannedLoweringCounts(plan optimize.LoweringPlan) []SkippedLowering { Name: optimize.LoweringAggregateTraversalCount, Count: len(plan.AggregateTraversalCount), }, + { + Name: optimize.LoweringFieldRequirements, + Count: len(plan.FieldRequirements), + }, + { + Name: optimize.LoweringShortestPathExecutor, + Count: len(plan.ShortestPathExecutor), + }, } } +// skippedLoweringReason explains why planned lowering work was not observed, including metadata-only analyses and lowerings superseded by a stronger fast path. func skippedLoweringReason(name string, applied map[string]int, plan optimize.LoweringPlan) string { + if name == optimize.LoweringFieldRequirements { + return "analysis_metadata_only" + } if applied[optimize.LoweringCountStoreFastPath] > 0 && name != optimize.LoweringCountStoreFastPath { return "superseded by CountStoreFastPath" } @@ -786,6 +1282,18 @@ func skippedLoweringReason(name string, applied map[string]int, plan optimize.Lo if reason := skippedTraversalDirectionReason(plan); reason != "" { return reason } + case optimize.LoweringExpansionSearchStrategy: + for _, decision := range plan.ExpansionSearchStrategy { + if decision.FallbackReason != "" { + return decision.FallbackReason + } + } + case optimize.LoweringShortestPathExecutor: + for _, decision := range plan.ShortestPathExecutor { + if decision.FallbackReason != "" { + return decision.FallbackReason + } + } default: return "planned lowering did not change the emitted SQL" } @@ -793,6 +1301,7 @@ func skippedLoweringReason(name string, applied map[string]int, plan optimize.Lo return "planned lowering did not change the emitted SQL" } +// skippedTraversalDirectionReason returns the first recorded reason a planned traversal direction was retained. func skippedTraversalDirectionReason(plan optimize.LoweringPlan) string { for _, decision := range plan.TraversalDirection { if !decision.Flip && decision.Reason != "" { @@ -803,11 +1312,131 @@ func skippedTraversalDirectionReason(plan optimize.LoweringPlan) string { return "" } +// ToolOptions controls experimental lowering selection exposed only to repository tooling. +type ToolOptions struct { + // ForceShortestPathExecutor requests a qualified shortest-path executor instead of automatic selection. + ForceShortestPathExecutor optimize.ShortestPathExecutor + // ForceExpansionSearchStrategy requests a qualified variable-expansion strategy instead of automatic selection. + ForceExpansionSearchStrategy optimize.ExpansionSearchStrategy + // ExpansionOrientationPolicy selects the immutable orientation selector + // identity used by an enabled tournament or shadow mode. The zero value + // preserves orientation-probe-v1. + ExpansionOrientationPolicy optimize.ExpansionSearchPolicy + // EnableExpansionOrientationTournament emits a guarded orientation policy + // for one qualified fixed-suffix expansion. It defaults to + // orientation-probe-v1 and is intentionally tool-only while selectors are + // being shadow-qualified. + EnableExpansionOrientationTournament bool + // EnableExpansionOrientationShadow emits the same bounded orientation + // probes and SQL-visible would_select metadata while executing only the + // exact incumbent traversal arm. + EnableExpansionOrientationShadow bool + // DisableEndpointSeededReverse is an emergency production rollback switch. + DisableEndpointSeededReverse bool +} + +// ProductionOptions contains the deliberately narrow subset of experimental +// lowerings that may be enabled by the PostgreSQL driver's versioned, +// query-allowlisted canary policy. The zero value preserves all incumbent +// production choices. +type ProductionOptions struct { + ShortestPathExecutor optimize.ShortestPathExecutor + ShortestPathCaps *ProductionShortestPathCaps + AuthorizedBucket *ProductionTraversalBucket + EnableExpansionOrientation bool + DisableEndpointSeededReverse bool + DisableInlineASPDAG bool + DisableInlineSPWitness bool + SelectorVersion string +} + +// ProductionShortestPathCaps are immutable manifest-authorized limits. They +// are copied into the lowering decision and therefore into emitted SQL. +type ProductionShortestPathCaps struct { + StateLimit int64 `json:"state_limit"` + PredecessorLimit int64 `json:"predecessor_limit"` + EnumerationLimit int64 `json:"enumeration_limit"` + OutputBytesLimit int64 `json:"output_bytes_limit"` +} + +// ProductionTraversalBucket binds an exact-query authorization to the +// structural target characteristics independently qualified by evidence. +type ProductionTraversalBucket struct { + Direction string `json:"direction"` + ObservationMode string `json:"observation_mode"` + MinimumDepth int64 `json:"minimum_depth"` + MaximumDepth int64 `json:"maximum_depth"` + RelationshipKindCount int `json:"relationship_kind_count"` + UntypedRelationship bool `json:"untyped_relationship"` +} + +// Translate optimizes and translates a Cypher query for the selected graph using production lowering choices. func Translate(ctx context.Context, cypherQuery *cypher.RegularQuery, kindMapper pgsql.KindMapper, parameters map[string]any, graphID int32) (Result, error) { + return translate(ctx, cypherQuery, kindMapper, parameters, graphID, ToolOptions{}) +} + +// TranslateWithProductionOptions applies a validated canary policy. B +// executors remain unavailable unless the driver has independently established +// the required transaction snapshot; this function only controls lowering. +func TranslateWithProductionOptions(ctx context.Context, cypherQuery *cypher.RegularQuery, kindMapper pgsql.KindMapper, parameters map[string]any, graphID int32, options ProductionOptions) (Result, error) { + if options.SelectorVersion == "" { + return Result{}, fmt.Errorf("production traversal policy requires a selector version") + } + if options.ShortestPathExecutor != "" && !productionShortestPathExecutor(options.ShortestPathExecutor) { + return Result{}, fmt.Errorf("shortest-path executor %q is not production-canary eligible", options.ShortestPathExecutor) + } + toolOptions := ToolOptions{ + ForceShortestPathExecutor: options.ShortestPathExecutor, + EnableExpansionOrientationTournament: options.EnableExpansionOrientation, + DisableEndpointSeededReverse: options.DisableEndpointSeededReverse, + } optimizedPlan, err := optimize.Optimize(cypherQuery) if err != nil { return Result{}, err } + if err := applyToolOptions(&optimizedPlan, toolOptions); err != nil { + return Result{}, err + } + applyProductionShortestPathRollback(&optimizedPlan, options) + if err := applyProductionShortestPathAuthorization(&optimizedPlan, options); err != nil { + return Result{}, err + } + for idx := range optimizedPlan.LoweringPlan.ShortestPathExecutor { + decision := &optimizedPlan.LoweringPlan.ShortestPathExecutor[idx] + if decision.SelectionMode == "forced_tool" { + decision.SelectionMode = "production_canary" + decision.SelectorVersion = options.SelectorVersion + } + } + for idx := range optimizedPlan.LoweringPlan.ExpansionSearchStrategy { + decision := &optimizedPlan.LoweringPlan.ExpansionSearchStrategy[idx] + if decision.SelectionMode == "guarded_tool" { + decision.SelectionMode = "production_canary" + decision.SelectorVersion = options.SelectorVersion + } + } + return translateOptimized(ctx, optimizedPlan, kindMapper, parameters, graphID, toolOptions) +} + +// TranslateForTool exposes qualified experimental lowerings to repository +// tooling without making them selectable through the production query API. +func TranslateForTool(ctx context.Context, cypherQuery *cypher.RegularQuery, kindMapper pgsql.KindMapper, parameters map[string]any, graphID int32, options ToolOptions) (Result, error) { + return translate(ctx, cypherQuery, kindMapper, parameters, graphID, options) +} + +// translate optimizes a Cypher query, applies optional tooling overrides, emits PostgreSQL, and records diagnostics. +func translate(ctx context.Context, cypherQuery *cypher.RegularQuery, kindMapper pgsql.KindMapper, parameters map[string]any, graphID int32, options ToolOptions) (Result, error) { + optimizedPlan, err := optimize.Optimize(cypherQuery) + if err != nil { + return Result{}, err + } + if err := applyToolOptions(&optimizedPlan, options); err != nil { + return Result{}, err + } + return translateOptimized(ctx, optimizedPlan, kindMapper, parameters, graphID, options) +} + +func translateOptimized(ctx context.Context, optimizedPlan optimize.Plan, kindMapper pgsql.KindMapper, parameters map[string]any, graphID int32, options ToolOptions) (Result, error) { translator := NewTranslator(ctx, kindMapper, parameters, graphID) if membershipAliases, err := collectIDMembershipAliases(optimizedPlan.Query); err != nil { @@ -841,11 +1470,436 @@ func Translate(ctx context.Context, cypherQuery *cypher.RegularQuery, kindMapper if err := walk.Cypher(optimizedPlan.Query, translator); err != nil { return Result{}, err } + if options.ForceExpansionSearchStrategy != "" && len(translator.appliedExpansionSearchStrategies) == 0 { + return Result{}, fmt.Errorf("forced expansion-search strategy %q was selected but not emitted", options.ForceExpansionSearchStrategy) + } + if options.EnableExpansionOrientationTournament && len(translator.emittedExpansionSearchPolicies) == 0 { + return Result{}, fmt.Errorf("expansion orientation tournament was selected but not emitted") + } + if options.EnableExpansionOrientationShadow && len(translator.emittedExpansionSearchPolicies) == 0 { + return Result{}, fmt.Errorf("expansion orientation shadow was selected but not emitted") + } + if options.ForceShortestPathExecutor != "" && len(translator.appliedShortestPathExecutors) == 0 { + return Result{}, fmt.Errorf("forced shortest-path executor %q was selected but not emitted", options.ForceShortestPathExecutor) + } translator.recordSkippedLowerings() return translator.translation, nil } +func productionShortestPathExecutor(executor optimize.ShortestPathExecutor) bool { + switch executor { + case optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, + optimize.ShortestPathExecutorASPI1DAG: + return true + default: + return false + } +} + +func applyProductionShortestPathAuthorization(plan *optimize.Plan, options ProductionOptions) error { + if options.ShortestPathExecutor == "" { + return nil + } + if options.DisableInlineASPDAG && options.ShortestPathExecutor == optimize.ShortestPathExecutorASPI1DAG { + return fmt.Errorf("inline ASP DAG is disabled by production policy") + } + for idx := range plan.LoweringPlan.ShortestPathExecutor { + decision := &plan.LoweringPlan.ShortestPathExecutor[idx] + if decision.SelectedExecutor != options.ShortestPathExecutor || decision.SelectionMode != "forced_tool" { + continue + } + if options.AuthorizedBucket != nil { + bucket := options.AuthorizedBucket + if decision.Direction.String() != bucket.Direction || + string(decision.ObservationMode) != bucket.ObservationMode || + decision.MinimumDepth != bucket.MinimumDepth || + decision.MaximumDepth != bucket.MaximumDepth || + decision.RelationshipKindCount != bucket.RelationshipKindCount || + decision.UntypedRelationship != bucket.UntypedRelationship { + return fmt.Errorf("production traversal target does not match its authorized promotion bucket") + } + } + if options.ShortestPathExecutor == optimize.ShortestPathExecutorASPI1DAG || options.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness { + if options.AuthorizedBucket == nil { + return fmt.Errorf("guarded inline shortest-path production policy requires an exact authorized bucket") + } + if options.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness { + bucket := options.AuthorizedBucket + if options.SelectorVersion != optimize.ShortestPathSelectorStaticV6 { + return fmt.Errorf("canonical SP-I1 production policy requires selector %q", optimize.ShortestPathSelectorStaticV6) + } + if bucket.Direction != "inbound" || bucket.ObservationMode != string(optimize.ShortestPathObservationOnePath) || + bucket.MinimumDepth != 1 || bucket.MaximumDepth != 64 || bucket.RelationshipKindCount != 1 || bucket.UntypedRelationship { + return fmt.Errorf("canonical SP-I1 production policy requires the qualified inbound typed single-kind one-path depth 1..64 bucket") + } + } + if options.ShortestPathCaps == nil { + return fmt.Errorf("guarded inline shortest-path production policy requires immutable caps") + } + caps := options.ShortestPathCaps + if caps.StateLimit <= 0 || caps.PredecessorLimit <= 0 || caps.EnumerationLimit <= 0 || caps.OutputBytesLimit <= 0 { + return fmt.Errorf("guarded inline shortest-path production policy requires positive immutable caps") + } + decision.StateLimit = caps.StateLimit + decision.PredecessorLimit = caps.PredecessorLimit + decision.EnumerationLimit = caps.EnumerationLimit + decision.OutputBytesLimit = caps.OutputBytesLimit + decision.ExecutionBoundary = "guarded_dual_arm" + } + return nil + } + return fmt.Errorf("production shortest-path executor %q was not selected", options.ShortestPathExecutor) +} + +// applyProductionShortestPathRollback is deliberately post-optimization: an +// emergency switch must rewrite both a policy-forced candidate and any future +// statically preferred candidate. Returning to the exact incumbent also resets +// candidate-only limits and boundary metadata so cached SQL cannot retain a +// disabled guarded arm. +func applyProductionShortestPathRollback(plan *optimize.Plan, options ProductionOptions) { + for idx := range plan.LoweringPlan.ShortestPathExecutor { + decision := &plan.LoweringPlan.ShortestPathExecutor[idx] + switch { + case options.DisableInlineASPDAG && decision.SelectedExecutor == optimize.ShortestPathExecutorASPI1DAG: + decision.SelectedExecutor = optimize.ShortestPathExecutorASPA1DAG + decision.FallbackExecutor = optimize.ShortestPathExecutorIncumbentWorkspace + case options.DisableInlineSPWitness && decision.SelectedExecutor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness: + decision.SelectedExecutor = optimize.ShortestPathExecutorS4CanonicalWitness + decision.FallbackExecutor = optimize.ShortestPathExecutorIncumbentWorkspace + default: + continue + } + decision.Scheduler = decision.SelectedExecutor.Scheduler() + decision.ExecutionBoundary = decision.SelectedExecutor.ExecutionBoundary() + decision.SelectionMode = "production_kill_switch" + decision.SelectorVersion = options.SelectorVersion + decision.FallbackReason = "disabled_by_production_policy" + decision.FrontierLimit = 0 + decision.PredecessorLimit = 0 + decision.EnumerationLimit = 0 + decision.OutputBytesLimit = 0 + } +} + +// applyToolOptions applies supported forced executor and expansion-strategy requests to an optimized plan. +func applyToolOptions(plan *optimize.Plan, options ToolOptions) error { + if options.EnableExpansionOrientationTournament && options.EnableExpansionOrientationShadow { + return fmt.Errorf("expansion orientation tournament and shadow modes are mutually exclusive") + } + if (options.EnableExpansionOrientationTournament || options.EnableExpansionOrientationShadow) && options.ForceExpansionSearchStrategy != "" { + return fmt.Errorf("expansion orientation policy and forced expansion-search strategy are mutually exclusive") + } + orientationPolicy, err := requestedExpansionOrientationPolicy(options) + if err != nil { + return err + } + if err := applyForcedShortestPathExecutor(plan, options.ForceShortestPathExecutor); err != nil { + return err + } + if options.DisableEndpointSeededReverse { + for idx := range plan.LoweringPlan.ExpansionSearchStrategy { + decision := &plan.LoweringPlan.ExpansionSearchStrategy[idx] + if decision.SelectedStrategy == optimize.ExpansionSearchEndpointSeededReverse { + decision.SelectedStrategy = optimize.ExpansionSearchStepwiseForward + decision.EmittedPolicy = "" + decision.EmittedCandidates = []optimize.ExpansionSearchStrategy{optimize.ExpansionSearchStepwiseForward} + decision.ExecutionBoundary = optimize.ExpansionSearchExecutionBoundaryInlineStatement + decision.SelectionMode = "production_kill_switch" + decision.SelectorVersion = "endpoint-seeded-disabled-v1" + decision.FallbackReason = "disabled_by_production_policy" + } + } + } + if options.EnableExpansionOrientationTournament { + return applyExpansionOrientationTournamentPolicy(plan, orientationPolicy) + } + if options.EnableExpansionOrientationShadow { + return applyExpansionOrientationShadowPolicy(plan, orientationPolicy) + } + return applyForcedExpansionSearchStrategy(plan, options.ForceExpansionSearchStrategy) +} + +func requestedExpansionOrientationPolicy(options ToolOptions) (optimize.ExpansionSearchPolicy, error) { + policy := options.ExpansionOrientationPolicy + if policy == "" { + return optimize.ExpansionSearchPolicyOrientationProbeV1, nil + } + if !options.EnableExpansionOrientationTournament && !options.EnableExpansionOrientationShadow { + return "", fmt.Errorf("expansion orientation policy %q requires tournament or shadow mode", policy) + } + if !supportedExpansionOrientationPolicy(policy) { + return "", fmt.Errorf("unsupported expansion orientation policy %q", policy) + } + return policy, nil +} + +func supportedExpansionOrientationPolicy(policy optimize.ExpansionSearchPolicy) bool { + switch policy { + case optimize.ExpansionSearchPolicyOrientationProbeV1, + optimize.ExpansionSearchPolicyOrientationProbeV2: + return true + default: + return false + } +} + +// applyForcedShortestPathExecutor selects the requested executor only when exactly one qualified shortest-path target supports it. +func applyForcedShortestPathExecutor(plan *optimize.Plan, executor optimize.ShortestPathExecutor) error { + if executor == "" { + return nil + } + if !supportedForcedShortestPathExecutor(executor) { + return fmt.Errorf("unsupported forced shortest-path executor %q", executor) + } + if executor == optimize.ShortestPathExecutorIncumbentWorkspace || executor == optimize.ShortestPathExecutorS0Direct { + forced := 0 + for idx := range plan.LoweringPlan.ShortestPathExecutor { + decision := &plan.LoweringPlan.ShortestPathExecutor[idx] + if !decision.StructurallyEligible { + continue + } + if executor == optimize.ShortestPathExecutorS0Direct && (decision.MinimumDepth != 1 || decision.MaximumDepth < 1) { + continue + } + decision.SelectedExecutor = executor + decision.Scheduler = executor.Scheduler() + decision.ExecutionBoundary = executor.ExecutionBoundary() + decision.SelectionMode = "forced_tool" + decision.SelectorVersion = "sp-tool-v1" + decision.FallbackReason = "" + forced++ + } + if forced == 0 { + if executor == optimize.ShortestPathExecutorS0Direct { + return fmt.Errorf("forced shortest-path executor %q has no structurally eligible depth-one target", executor) + } + return fmt.Errorf("forced shortest-path executor %q has no structurally eligible target", executor) + } + return nil + } + expectedObservation := optimize.ShortestPathObservationDistance + expectedDescription := "distance-only" + if executor == optimize.ShortestPathExecutorS3EdgeM0 || executor == optimize.ShortestPathExecutorS4CanonicalWitness || executor == optimize.ShortestPathExecutorI1CanonicalWitness || executor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness || executor == optimize.ShortestPathExecutorB1AlternatingNodeWitness || executor == optimize.ShortestPathExecutorB2SmallerCurrentLevelWitness { + expectedObservation = optimize.ShortestPathObservationOnePath + expectedDescription = "one-path" + } else if executor == optimize.ShortestPathExecutorASPA1DAG || executor == optimize.ShortestPathExecutorASPI1DAG || executor == optimize.ShortestPathExecutorASPB1AlternatingNodeDAG || executor == optimize.ShortestPathExecutorASPB2SmallerCurrentLevelDAG { + expectedObservation = optimize.ShortestPathObservationAllPaths + expectedDescription = "all-paths" + } + + allShortestExecutor := executor == optimize.ShortestPathExecutorASPA1DAG || + executor == optimize.ShortestPathExecutorASPI1DAG || + executor == optimize.ShortestPathExecutorASPB1AlternatingNodeDAG || + executor == optimize.ShortestPathExecutorASPB2SmallerCurrentLevelDAG + forced := 0 + for idx := range plan.LoweringPlan.ShortestPathExecutor { + decision := &plan.LoweringPlan.ShortestPathExecutor[idx] + if !decision.StructurallyEligible { + continue + } + if decision.ObservationMode != expectedObservation { + continue + } + // Two-sided predecessor-DAG discovery is proven only for one distinct, + // directed singleton endpoint pair with minimum depth exactly one. The + // shared structural facts enforce every condition except this narrower + // minimum-depth check. Tool forcing must not broaden that envelope. + if allShortestExecutor && (decision.Family != "ASP" || decision.MinimumDepth != 1 || decision.MaximumDepth < 1 || decision.MaximumDepth > 64) { + continue + } + + decision.SelectedExecutor = executor + decision.ExecutionBoundary = executor.ExecutionBoundary() + if executor == optimize.ShortestPathExecutorASPI1DAG || executor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness { + decision.ExecutionBoundary = "guarded_dual_arm" + decision.FrontierLimit = 0 + } + decision.Scheduler = executor.Scheduler() + decision.SelectionMode = "forced_tool" + decision.SelectorVersion = "sp-tool-v1" + decision.FallbackReason = "" + if allShortestExecutor { + decision.SelectorVersion = "asp-tool-v1" + if executor != optimize.ShortestPathExecutorASPA1DAG { + decision.FallbackExecutor = optimize.ShortestPathExecutorASPA1DAG + } + } + if executor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness { + decision.SelectorVersion = "sp-i1-canonical-tool-v1" + decision.FallbackExecutor = optimize.ShortestPathExecutorS4CanonicalWitness + } + forced++ + } + if forced == 0 { + return fmt.Errorf("forced shortest-path executor %q has no structurally eligible %s target", executor, expectedDescription) + } + return nil +} + +func supportedForcedShortestPathExecutor(executor optimize.ShortestPathExecutor) bool { + switch executor { + case optimize.ShortestPathExecutorIncumbentWorkspace, + optimize.ShortestPathExecutorS0Direct, + optimize.ShortestPathExecutorS3Unidirectional, + optimize.ShortestPathExecutorS3EdgeM0, + optimize.ShortestPathExecutorS4CanonicalDistance, + optimize.ShortestPathExecutorS4CanonicalWitness, + optimize.ShortestPathExecutorASPA1DAG, + optimize.ShortestPathExecutorASPI1DAG, + optimize.ShortestPathExecutorI1CanonicalDistance, + optimize.ShortestPathExecutorI1CanonicalWitness, + optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, + optimize.ShortestPathExecutorB1AlternatingNodeDistance, + optimize.ShortestPathExecutorB1AlternatingNodeWitness, + optimize.ShortestPathExecutorB2SmallerCurrentLevelDistance, + optimize.ShortestPathExecutorB2SmallerCurrentLevelWitness, + optimize.ShortestPathExecutorASPB1AlternatingNodeDAG, + optimize.ShortestPathExecutorASPB2SmallerCurrentLevelDAG: + return true + default: + return false + } +} + +// applyForcedExpansionSearchStrategy selects the requested strategy only when exactly one qualified expansion target supports it. +func applyForcedExpansionSearchStrategy(plan *optimize.Plan, strategy optimize.ExpansionSearchStrategy) error { + if strategy == "" { + return nil + } + if strategy != optimize.ExpansionSearchSuffixSeededReverse && strategy != optimize.ExpansionSearchEndpointSeededReverse { + return fmt.Errorf("unsupported forced expansion-search strategy %q", strategy) + } + + var matching []int + for idx := range plan.LoweringPlan.ExpansionSearchStrategy { + decision := plan.LoweringPlan.ExpansionSearchStrategy[idx] + if !decision.StructurallyEligible { + continue + } + if strategy == optimize.ExpansionSearchSuffixSeededReverse && decision.CandidateStrategy != optimize.ExpansionSearchSuffixSeededReverse { + continue + } + if strategy == optimize.ExpansionSearchEndpointSeededReverse && decision.CandidateStrategy != optimize.ExpansionSearchEndpointSeededReverse { + continue + } + matching = append(matching, idx) + } + if len(matching) == 0 { + return fmt.Errorf("forced expansion-search strategy %q has no structurally eligible target", strategy) + } + if len(matching) != 1 { + return fmt.Errorf("forced expansion-search strategy %q matched %d structurally eligible targets; expected exactly one", strategy, len(matching)) + } + + decision := &plan.LoweringPlan.ExpansionSearchStrategy[matching[0]] + decision.SelectedStrategy = strategy + decision.SelectionMode = "forced_tool" + decision.EmittedPolicy = "" + decision.EmittedCandidates = []optimize.ExpansionSearchStrategy{strategy} + decision.ExecutionBoundary = optimize.ExpansionSearchExecutionBoundaryInlineStatement + if strategy == optimize.ExpansionSearchSuffixSeededReverse { + decision.SelectorVersion = "suffix-seeded-reverse-tool-v1" + } else { + decision.SelectorVersion = "endpoint-seeded-reverse-tool-v1" + decision.EmittedPolicy = optimize.ExpansionSearchPolicyEndpointGuardV1 + decision.EmittedCandidates = []optimize.ExpansionSearchStrategy{ + optimize.ExpansionSearchStepwiseForward, + optimize.ExpansionSearchEndpointSeededReverse, + } + decision.ExecutionBoundary = optimize.ExpansionSearchExecutionBoundaryGuardedDualArm + } + decision.FallbackReason = "" + + return nil +} + +// applyExpansionOrientationTournament emits orientation-probe-v1 only when a +// single already-qualified fixed-suffix target exists. It preserves the +// compile-time incumbent identity because the runtime arm is not known during +// translation. +func applyExpansionOrientationTournament(plan *optimize.Plan) error { + return applyExpansionOrientationTournamentPolicy(plan, optimize.ExpansionSearchPolicyOrientationProbeV1) +} + +func applyExpansionOrientationTournamentPolicy(plan *optimize.Plan, policy optimize.ExpansionSearchPolicy) error { + if !supportedExpansionOrientationPolicy(policy) { + return fmt.Errorf("unsupported expansion orientation policy %q", policy) + } + var matching []int + for idx, decision := range plan.LoweringPlan.ExpansionSearchStrategy { + if decision.Family != "fixed_suffix_expansion" || + decision.CandidateStrategy != optimize.ExpansionSearchSuffixSeededReverse || + !decision.StructurallyEligible || !decision.StaticallyEligible { + continue + } + matching = append(matching, idx) + } + if len(matching) == 0 { + return fmt.Errorf("expansion orientation tournament has no structurally eligible fixed-suffix target") + } + if len(matching) != 1 { + return fmt.Errorf("expansion orientation tournament matched %d structurally eligible fixed-suffix targets; expected exactly one", len(matching)) + } + + decision := &plan.LoweringPlan.ExpansionSearchStrategy[matching[0]] + decision.SelectedStrategy = optimize.ExpansionSearchStepwiseForward + decision.PlannedPolicy = policy + decision.SelectionMode = "guarded_tool" + decision.SelectorVersion = string(policy) + decision.EmittedPolicy = policy + decision.EmittedCandidates = []optimize.ExpansionSearchStrategy{ + optimize.ExpansionSearchStepwiseForward, + optimize.ExpansionSearchSuffixSeededReverse, + } + decision.ExecutionBoundary = optimize.ExpansionSearchExecutionBoundaryGuardedDualArm + decision.FallbackReason = "" + + return nil +} + +// applyExpansionOrientationShadow emits orientation-probe-v1 for one +// qualified fixed-suffix target while retaining the exact incumbent as the +// only emitted traversal arm. The generated policy CTE records which arm the +// selector would have chosen without dispatching it. +func applyExpansionOrientationShadow(plan *optimize.Plan) error { + return applyExpansionOrientationShadowPolicy(plan, optimize.ExpansionSearchPolicyOrientationProbeV1) +} + +func applyExpansionOrientationShadowPolicy(plan *optimize.Plan, policy optimize.ExpansionSearchPolicy) error { + if !supportedExpansionOrientationPolicy(policy) { + return fmt.Errorf("unsupported expansion orientation policy %q", policy) + } + var matching []int + for idx, decision := range plan.LoweringPlan.ExpansionSearchStrategy { + if decision.Family != "fixed_suffix_expansion" || + decision.CandidateStrategy != optimize.ExpansionSearchSuffixSeededReverse || + !decision.StructurallyEligible || !decision.StaticallyEligible { + continue + } + matching = append(matching, idx) + } + if len(matching) == 0 { + return fmt.Errorf("expansion orientation shadow has no structurally eligible fixed-suffix target") + } + if len(matching) != 1 { + return fmt.Errorf("expansion orientation shadow matched %d structurally eligible fixed-suffix targets; expected exactly one", len(matching)) + } + + decision := &plan.LoweringPlan.ExpansionSearchStrategy[matching[0]] + decision.SelectedStrategy = optimize.ExpansionSearchStepwiseForward + decision.PlannedPolicy = policy + decision.SelectionMode = "shadow_tool" + decision.SelectorVersion = string(policy) + decision.EmittedPolicy = policy + decision.EmittedCandidates = []optimize.ExpansionSearchStrategy{optimize.ExpansionSearchStepwiseForward} + decision.ExecutionBoundary = optimize.ExpansionSearchExecutionBoundaryInlineStatement + decision.FallbackReason = "" + + return nil +} + +// decodeCypherStringLiteral decodes Cypher escape sequences by interpreting the token as a quoted Go string. func decodeCypherStringLiteral(raw string) (string, error) { if len(raw) < 2 { return "", fmt.Errorf("invalid cypher string literal: %q", raw) diff --git a/cypher/models/pgsql/translate/traversal.go b/cypher/models/pgsql/translate/traversal.go index 3a6eb873..53c82069 100644 --- a/cypher/models/pgsql/translate/traversal.go +++ b/cypher/models/pgsql/translate/traversal.go @@ -10,23 +10,24 @@ import ( "github.com/specterops/dawgs/graph" ) -func boundEndpointIDReference(frame *Frame, binding *BoundIdentifier) pgsql.RowColumnReference { +// projectedNodeIDReference returns the scalar ID expression exposed for node by frame. +func projectedNodeIDReference(frameIdentifier pgsql.Identifier, binding *BoundIdentifier) pgsql.Expression { + if binding != nil && binding.IDOnly { + return pgsql.CompoundIdentifier{frameIdentifier, binding.Identifier} + } + return pgsql.RowColumnReference{ - Identifier: pgsql.CompoundIdentifier{frame.Binding.Identifier, binding.Identifier}, + Identifier: pgsql.CompoundIdentifier{frameIdentifier, binding.Identifier}, Column: pgsql.ColumnID, } } -func boundEndpointInequality(frame *Frame, traversalStep *TraversalStep) pgsql.Expression { - return pgsql.NewParenthetical( - pgsql.NewBinaryExpression( - boundEndpointIDReference(frame, traversalStep.LeftNode), - pgsql.OperatorCypherNotEquals, - boundEndpointIDReference(frame, traversalStep.RightNode), - ), - ) +// boundEndpointIDReference returns the previous-frame scalar ID for a bound traversal endpoint. +func boundEndpointIDReference(frame *Frame, binding *BoundIdentifier) pgsql.Expression { + return projectedNodeIDReference(frame.Binding.Identifier, binding) } +// sourceTargetForTraversalStep returns optimizer coordinates for a step that originated in the source query. func sourceTargetForTraversalStep(part *PatternPart, stepIndex int) (optimize.TraversalStepTarget, bool) { if part == nil || stepIndex < 0 || stepIndex >= len(part.TraversalSteps) { return optimize.TraversalStepTarget{}, false @@ -43,6 +44,26 @@ func sourceTargetForTraversalStep(part *PatternPart, stepIndex int) (optimize.Tr return part.Target.TraversalStep(stepIndex), true } +// shortestPathExecutorDecision returns the planned physical executor for a source traversal step. +func (s *Translator) shortestPathExecutorDecision(part *PatternPart, stepIndex int) (optimize.ShortestPathExecutorDecision, bool) { + target, hasTarget := sourceTargetForTraversalStep(part, stepIndex) + if !hasTarget { + return optimize.ShortestPathExecutorDecision{}, false + } + decision, hasDecision := s.shortestPathExecutorDecisions[target] + return decision, hasDecision +} + +// decisionIsForcedShortest reports whether tooling forced a non-incumbent shortest-path executor. +func decisionIsForcedShortest(translator *Translator, target optimize.TraversalStepTarget) bool { + if translator == nil { + return false + } + decision, found := translator.shortestPathExecutorDecisions[target] + return found && decision.SelectionMode == "forced_tool" +} + +// traversalStepIsFirstForSourceTarget reports whether step is the first translated step for its source target. func traversalStepIsFirstForSourceTarget(part *PatternPart, stepIndex int) bool { target, hasTarget := sourceTargetForTraversalStep(part, stepIndex) if !hasTarget || stepIndex == 0 { @@ -53,6 +74,7 @@ func traversalStepIsFirstForSourceTarget(part *PatternPart, stepIndex int) bool return !previousHasTarget || previousTarget != target } +// traversalStepIsLastForSourceTarget reports whether step is the final translated step for its source target. func traversalStepIsLastForSourceTarget(part *PatternPart, stepIndex int) bool { target, hasTarget := sourceTargetForTraversalStep(part, stepIndex) if !hasTarget || stepIndex+1 >= len(part.TraversalSteps) { @@ -63,6 +85,7 @@ func traversalStepIsLastForSourceTarget(part *PatternPart, stepIndex int) bool { return !nextHasTarget || nextTarget != target } +// shouldUseExpandInto reports whether a planned bound-endpoint traversal applies to this source step. func (s *Translator) shouldUseExpandInto(part *PatternPart, stepIndex int, traversalStep *TraversalStep) bool { if traversalStep == nil || traversalStep.Expansion != nil || !traversalStep.LeftNodeBound || !traversalStep.RightNodeBound { return false @@ -79,6 +102,7 @@ func (s *Translator) shouldUseExpandInto(part *PatternPart, stepIndex int, trave return true } +// traversalDirectionDecision returns the planned direction choice for a source traversal step. func (s *Translator) traversalDirectionDecision(part *PatternPart, stepIndex int) (optimize.TraversalDirectionDecision, bool) { target, hasTarget := sourceTargetForTraversalStep(part, stepIndex) if !hasTarget { @@ -89,6 +113,7 @@ func (s *Translator) traversalDirectionDecision(part *PatternPart, stepIndex int return decision, hasDecision } +// applyPatternConstraintBalance swaps endpoint constraints and reverses path state when the plan flips traversal direction. func (s *Translator) applyPatternConstraintBalance(part *PatternPart, stepIndex int, constraints *PatternConstraints, traversalStep *TraversalStep) error { if decision, hasDecision := s.traversalDirectionDecision(part, stepIndex); hasDecision { if decision.Flip { @@ -117,6 +142,7 @@ func (s *Translator) applyPatternConstraintBalance(part *PatternPart, stepIndex return nil } +// shortestPathStrategyDecision returns the planned unidirectional or bidirectional strategy for a source step. func (s *Translator) shortestPathStrategyDecision(part *PatternPart, stepIndex int) (optimize.ShortestPathStrategyDecision, bool) { target, hasTarget := sourceTargetForTraversalStep(part, stepIndex) if !hasTarget { @@ -127,6 +153,7 @@ func (s *Translator) shortestPathStrategyDecision(part *PatternPart, stepIndex i return decision, hasDecision } +// useBidirectionalShortestPathStrategy reports whether a qualified plan selects bidirectional search for step. func (s *Translator) useBidirectionalShortestPathStrategy(part *PatternPart, stepIndex int, traversalStep *TraversalStep) (bool, error) { if decision, hasDecision := s.shortestPathStrategyDecision(part, stepIndex); hasDecision { if decision.Strategy != optimize.ShortestPathStrategyBidirectional { @@ -153,6 +180,7 @@ func (s *Translator) useBidirectionalShortestPathStrategy(part *PatternPart, ste return false, nil } +// shortestPathFilterDecisionsForStep returns every planned filter materialization for a source traversal step. func (s *Translator) shortestPathFilterDecisionsForStep(part *PatternPart, stepIndex int) []optimize.ShortestPathFilterDecision { target, hasTarget := sourceTargetForTraversalStep(part, stepIndex) if !hasTarget { @@ -162,6 +190,7 @@ func (s *Translator) shortestPathFilterDecisionsForStep(part *PatternPart, stepI return s.shortestPathFilterDecisions[target] } +// applyShortestPathFilterMaterialization enables terminal or endpoint-pair filters selected for the source step. func (s *Translator) applyShortestPathFilterMaterialization(part *PatternPart, stepIndex int, traversalStep *TraversalStep, expansionModel *Expansion) { for _, decision := range s.shortestPathFilterDecisionsForStep(part, stepIndex) { switch decision.Mode { @@ -180,6 +209,7 @@ func (s *Translator) applyShortestPathFilterMaterialization(part *PatternPart, s } } +// hasLimitPushdownDecision reports whether target has the requested limit-pushdown mode. func (s *Translator) hasLimitPushdownDecision(part *PatternPart, stepIndex int, mode optimize.LimitPushdownMode) bool { target, hasTarget := sourceTargetForTraversalStep(part, stepIndex) if !hasTarget { @@ -195,6 +225,7 @@ func (s *Translator) hasLimitPushdownDecision(part *PatternPart, stepIndex int, return false } +// allowLimitPushdownForStep authorizes the step's frame to consume a matching planned limit internally. func (s *Translator) allowLimitPushdownForStep(part *PatternPart, stepIndex int, traversalStep *TraversalStep) { if traversalStep == nil || traversalStep.Frame == nil { return @@ -215,14 +246,19 @@ func (s *Translator) allowLimitPushdownForStep(part *PatternPart, stepIndex int, } } +// buildBoundEndpointTraversalPattern emits a one-hop join between two endpoints already visible in the previous frame. func (s *Translator) buildBoundEndpointTraversalPattern(partFrame *Frame, traversalStep *TraversalStep) (pgsql.Query, error) { if partFrame == nil || partFrame.Previous == nil { return pgsql.Query{}, errors.New("expected previous frame for bound endpoint traversal") } var ( - previousFrame = partFrame.Previous - nextSelect = pgsql.Select{ + previousFrame = partFrame.Previous + edgeConstraint = pgsql.OptionalAnd( + traversalStep.EdgeJoinCondition, + traversalStep.RightNodeJoinCondition, + ) + nextSelect = pgsql.Select{ Projection: traversalStep.Projection, From: []pgsql.FromClause{{ Source: pgsql.TableReference{ @@ -234,25 +270,38 @@ func (s *Translator) buildBoundEndpointTraversalPattern(partFrame *Frame, traver Binding: models.OptionalValue(traversalStep.Edge.Identifier), }, JoinOperator: pgsql.JoinOperator{ - JoinType: pgsql.JoinTypeInner, - Constraint: pgsql.OptionalAnd( - traversalStep.EdgeJoinCondition, - traversalStep.RightNodeJoinCondition, - ), + JoinType: pgsql.JoinTypeInner, + Constraint: edgeConstraint, }, }}, }}, } ) + if traversalStep.Direction == graph.DirectionBoth { + edgeConstraint = buildDirectionlessPairwiseEdgeConstraintForRefs( + boundEndpointIDReference(previousFrame, traversalStep.LeftNode), + boundEndpointIDReference(previousFrame, traversalStep.RightNode), + traversalStep.Edge.Identifier, + ) + nextSelect.From[0].Joins[0].JoinOperator.Constraint = edgeConstraint + } + if referencesUnwind, err := expressionReferencesUnwindBinding(edgeConstraint, s.query.CurrentPart().unwindClauses); err != nil { + return pgsql.Query{}, err + } else if referencesUnwind { + // An UNWIND alias is appended as a comma source after this builder + // returns. PostgreSQL JOIN ... ON cannot see a later comma source, while + // WHERE can see the complete FROM list. Keep the exact pair predicate + // and edge scan together in that shared scope. + edgeJoin := nextSelect.From[0].Joins[0] + nextSelect.From[0].Joins = nil + nextSelect.From = append(nextSelect.From, pgsql.FromClause{Source: edgeJoin.Table}) + nextSelect.Where = pgsql.OptionalAnd(edgeConstraint, nextSelect.Where) + } nextSelect.Where = pgsql.OptionalAnd(traversalStep.LeftNodeConstraints, nextSelect.Where) nextSelect.Where = pgsql.OptionalAnd(traversalStep.EdgeConstraints.Expression, nextSelect.Where) nextSelect.Where = pgsql.OptionalAnd(traversalStep.RightNodeConstraints, nextSelect.Where) - if traversalStep.Direction == graph.DirectionBoth && traversalStep.LeftNode.Identifier != traversalStep.RightNode.Identifier { - nextSelect.Where = pgsql.OptionalAnd(boundEndpointInequality(previousFrame, traversalStep), nextSelect.Where) - } - return pgsql.Query{ Body: nextSelect, }, nil @@ -368,12 +417,16 @@ func (s *Translator) buildTraversalPatternRootWithOuterCorrelation(partFrame *Fr } } +// buildTraversalPatternRoot emits the first node source, constraints, and projection for a traversal pattern. func (s *Translator) buildTraversalPatternRoot(partFrame *Frame, traversalStep *TraversalStep) (pgsql.Query, error) { if traversalStep.Direction == graph.DirectionBoth { return s.buildDirectionlessTraversalPatternRoot(traversalStep) } - if traversalStep.UseExpandInto { + // Dual-bound fixed hops must always use the exact pair join. The optimizer + // decision records and measures this shape, but correctness must not depend + // on that analysis recognizing every supported binding source. + if traversalStep.UseExpandInto || (traversalStep.LeftNodeBound && traversalStep.RightNodeBound) { return s.buildBoundEndpointTraversalPattern(partFrame, traversalStep) } @@ -558,8 +611,12 @@ func (s *Translator) buildTraversalPatternRoot(partFrame *Frame, traversalStep * }, nil } +// buildTraversalPatternStep emits one relationship join, terminal node join, constraints, and projection frame. func (s *Translator) buildTraversalPatternStep(partFrame *Frame, traversalStep *TraversalStep) (pgsql.Query, error) { - if traversalStep.UseExpandInto { + // Keep the dual-bound semantic fallback independent of optimizer coverage; + // otherwise a missed decision can introduce an uncorrelated terminal-node + // join and multiply the outer bag. + if traversalStep.UseExpandInto || (traversalStep.LeftNodeBound && traversalStep.RightNodeBound) { return s.buildBoundEndpointTraversalPattern(partFrame, traversalStep) } @@ -626,6 +683,7 @@ func (s *Translator) buildTraversalPatternStep(partFrame *Frame, traversalStep * }, nil } +// translateTraversalPatternPart prepares source targets, constraints, and state for translating one pattern part. func (s *Translator) translateTraversalPatternPart(part *PatternPart, isolatedProjection bool, allowProjectionPruning bool) error { var scopeSnapshot *Scope @@ -671,6 +729,7 @@ func (s *Translator) translateTraversalPatternPart(part *PatternPart, isolatedPr return nil } +// applyExpansionSuffixPushdown attaches planned fixed-suffix predicates and records any applied predicate placement. func (s *Translator) applyExpansionSuffixPushdown(part *PatternPart) (int, error) { if part == nil || !part.HasTarget { return applyExpansionSuffixPushdown(part) @@ -691,6 +750,7 @@ func (s *Translator) applyExpansionSuffixPushdown(part *PatternPart) (int, error for _, decision := range decisions { if decision.SuffixLength <= 0 || + !decision.ApplySupplemental || decision.SuffixStartStep <= target.StepIndex || decision.SuffixEndStep < decision.SuffixStartStep || decision.SuffixEndStep-decision.SuffixStartStep+1 != decision.SuffixLength { @@ -742,10 +802,120 @@ func (s *Translator) applyExpansionSuffixPushdown(part *PatternPart) (int, error return applied, nil } +// traversalStepHasContinuation reports whether another translated step follows in the pattern part. func traversalStepHasContinuation(part *PatternPart, stepIndex int) bool { return part != nil && stepIndex+1 < len(part.TraversalSteps) } +// fieldRequirementAllowsIDOnly reports whether all external uses of symbol can consume a scalar entity ID. +func fieldRequirementAllowsIDOnly(decision optimize.FieldRequirementDecision) bool { + observesID := false + for _, use := range decision.Uses { + for _, field := range use.Fields { + if !use.Internal && field == optimize.FieldRequirementEntityID { + observesID = true + } + + if !use.Internal && field != optimize.FieldRequirementEntityID { + return false + } + + if field == optimize.FieldRequirementFullEntity || field == optimize.FieldRequirementFullPath { + return false + } + } + } + + return observesID +} + +// fieldRequirementAllowsIDOnlyContinuation reports whether later pattern use can continue from scalar ID state. +func fieldRequirementAllowsIDOnlyContinuation(decision optimize.FieldRequirementDecision) bool { + for _, use := range decision.Uses { + for _, field := range use.Fields { + if field == optimize.FieldRequirementFullEntity || field == optimize.FieldRequirementFullPath { + return false + } + + if !use.Internal && field != optimize.FieldRequirementEntityID { + return false + } + } + } + + return true +} + +// traversalStepContinuesFromBinding reports whether the next step starts from binding. +func traversalStepContinuesFromBinding(part *PatternPart, stepIndex int, binding *BoundIdentifier) bool { + if part == nil || binding == nil || stepIndex < 0 || stepIndex+1 >= len(part.TraversalSteps) { + return false + } + + currentStep := part.TraversalSteps[stepIndex] + nextStep := part.TraversalSteps[stepIndex+1] + + return currentStep != nil && nextStep != nil && + currentStep.RightNode == binding && nextStep.LeftNode == binding +} + +// applyIDOnlyNodeProjection replaces an eligible node composite projection with its scalar ID. +func (s *Translator) applyIDOnlyNodeProjection(part *PatternPart, stepIndex int, binding *BoundIdentifier) bool { + if part == nil || binding == nil || !part.HasTarget { + return false + } + + var ( + isContinuation = traversalStepContinuesFromBinding(part, stepIndex, binding) + isTerminal = !traversalStepHasContinuation(part, stepIndex) + ) + if !isContinuation && !isTerminal { + return false + } + + if part.PatternBinding != nil { + for _, pathSymbol := range s.scope.Symbols(part.PatternBinding) { + if decision, found := s.fieldRequirementDecisions[part.Target.QueryPartIndex][pathSymbol.String()]; found { + for _, field := range decision.Fields { + if field == optimize.FieldRequirementFullPath { + return false + } + } + } + } + } + + foundDecision := false + for _, symbol := range s.scope.Symbols(binding) { + if decision, found := s.fieldRequirementDecisions[part.Target.QueryPartIndex][symbol.String()]; found { + foundDecision = true + allowsIDOnly := fieldRequirementAllowsIDOnly(decision) + if isContinuation { + allowsIDOnly = fieldRequirementAllowsIDOnlyContinuation(decision) + } + + if !allowsIDOnly { + return false + } + } + } + if foundDecision { + binding.IDOnly = true + return true + } + + // Anonymous or otherwise unobserved intermediate nodes have no source-level + // field-requirement decision. Their identity is still required to join the + // next relationship, so carry that identity as a scalar between steps. + if isContinuation && !foundDecision { + binding.IDOnly = true + return true + } + + return false +} + +// relationshipIDReference returns the scalar relationship ID exposed by a composite or ID-only binding. func relationshipIDReference(scope *Scope, binding *BoundIdentifier) pgsql.Expression { if binding != nil && binding.DataType == pgsql.EdgeComposite { return pathCompositeColumnReference(scope, binding, pgsql.ColumnID) @@ -754,6 +924,7 @@ func relationshipIDReference(scope *Scope, binding *BoundIdentifier) pgsql.Expre return pathEdgeIDReference(scope, binding) } +// relationshipIDNotInPath builds the edge-uniqueness predicate for a relationship and accumulated path. func relationshipIDNotInPath(edgeID, pathIDs pgsql.Expression) pgsql.Expression { return pgsql.NewBinaryExpression( edgeID, @@ -762,6 +933,7 @@ func relationshipIDNotInPath(edgeID, pathIDs pgsql.Expression) pgsql.Expression ) } +// previousRelationshipUniquenessConstraint excludes a relationship ID already used by a prior fixed step. func previousRelationshipUniquenessConstraint(scope *Scope, part *PatternPart, stepIndex int, traversalStep *TraversalStep) pgsql.Expression { if scope == nil || part == nil || stepIndex <= 0 || traversalStep == nil || traversalStep.Edge == nil { return nil @@ -801,6 +973,7 @@ func previousRelationshipUniquenessConstraint(scope *Scope, part *PatternPart, s return constraint } +// projectionPruningDecision returns the planned omitted fields for a source traversal step. func (s *Translator) projectionPruningDecision(part *PatternPart, stepIndex int) (optimize.ProjectionPruningDecision, bool) { target, hasTarget := sourceTargetForTraversalStep(part, stepIndex) if !hasTarget { @@ -811,6 +984,7 @@ func (s *Translator) projectionPruningDecision(part *PatternPart, stepIndex int) return decision, hasDecision } +// prepareProjectionPruning applies pruning flags and records the bindings removed from a traversal projection. func (s *Translator) prepareProjectionPruning(part *PatternPart, stepIndex int, traversalStep *TraversalStep) { decision, hasDecision := s.projectionPruningDecision(part, stepIndex) if !hasDecision || traversalStep == nil { @@ -834,6 +1008,7 @@ func (s *Translator) prepareProjectionPruning(part *PatternPart, stepIndex int, } } +// latePathMaterializationDecision returns the requested deferred materialization mode for target. func (s *Translator) latePathMaterializationDecision(part *PatternPart, stepIndex int, mode optimize.LatePathMaterializationMode) (optimize.LatePathMaterializationDecision, bool) { target, hasTarget := sourceTargetForTraversalStep(part, stepIndex) if !hasTarget { @@ -849,6 +1024,7 @@ func (s *Translator) latePathMaterializationDecision(part *PatternPart, stepInde return optimize.LatePathMaterializationDecision{}, false } +// applyPathEdgeIDMaterialization replaces a path binding with ordered edge-ID state for later hydration. func (s *Translator) applyPathEdgeIDMaterialization(part *PatternPart, stepIndex int, traversalStep *TraversalStep) bool { if traversalStep == nil || traversalStep.Edge == nil || @@ -864,6 +1040,7 @@ func (s *Translator) applyPathEdgeIDMaterialization(part *PatternPart, stepIndex return true } +// unexportFrameBinding removes binding and its alias from a frame's exported identifiers. func unexportFrameBinding(frame *Frame, identifier pgsql.Identifier) bool { if frame == nil { return false @@ -874,6 +1051,7 @@ func unexportFrameBinding(frame *Frame, identifier pgsql.Identifier) bool { return exported } +// traversalStepBindingBound reports whether binding is an endpoint or relationship already bound for step. func traversalStepBindingBound(traversalStep *TraversalStep, binding *BoundIdentifier) bool { if traversalStep == nil || binding == nil { return false @@ -890,6 +1068,7 @@ func traversalStepBindingBound(traversalStep *TraversalStep, binding *BoundIdent return false } +// unexportPrunedNodeBinding removes a pruned node and its aliases unless another step still requires the binding. func unexportPrunedNodeBinding(traversalStep *TraversalStep, binding *BoundIdentifier) bool { if binding == nil || traversalStepBindingBound(traversalStep, binding) { return false @@ -898,6 +1077,7 @@ func unexportPrunedNodeBinding(traversalStep *TraversalStep, binding *BoundIdent return unexportFrameBinding(traversalStep.Frame, binding.Identifier) } +// pruneTraversalStepProjectionExports removes planned node, relationship, and path exports from a fixed step. func pruneTraversalStepProjectionExports(part *PatternPart, stepIndex int, traversalStep *TraversalStep) bool { var applied bool @@ -910,6 +1090,7 @@ func pruneTraversalStepProjectionExports(part *PatternPart, stepIndex int, trave return applied } +// pruneExpansionStepProjectionExports removes planned node, relationship, and path exports from an expansion step. func pruneExpansionStepProjectionExports(part *PatternPart, stepIndex int, traversalStep *TraversalStep) bool { if traversalStep == nil || traversalStep.Expansion == nil { return false @@ -927,6 +1108,7 @@ func pruneExpansionStepProjectionExports(part *PatternPart, stepIndex int, trave return applied } +// translateTraversalPatternPartWithoutExpansion emits each fixed step, applying pruning and scalar-ID continuation where qualified. func (s *Translator) translateTraversalPatternPartWithoutExpansion(part *PatternPart, stepIndex int, traversalStep *TraversalStep, allowProjectionPruning bool) error { isFirstTraversalStep := stepIndex == 0 @@ -1023,6 +1205,12 @@ func (s *Translator) translateTraversalPatternPartWithoutExpansion(part *Pattern } } + leftNodeIDOnly := s.applyIDOnlyNodeProjection(part, stepIndex, traversalStep.LeftNode) + rightNodeIDOnly := s.applyIDOnlyNodeProjection(part, stepIndex, traversalStep.RightNode) + if leftNodeIDOnly || rightNodeIDOnly { + s.recordLowering(optimize.LoweringFieldRequirements) + } + if boundProjections, err := buildVisibleProjections(s.scope); err != nil { return err } else { diff --git a/cypher/models/pgsql/translate/traversal_directionless.go b/cypher/models/pgsql/translate/traversal_directionless.go index 7f51c0ba..c7660c86 100644 --- a/cypher/models/pgsql/translate/traversal_directionless.go +++ b/cypher/models/pgsql/translate/traversal_directionless.go @@ -174,11 +174,6 @@ func (s *Translator) buildPairwiseDirectionlessTraversalPatternRoot(traversalSte nextSelect.Where = pgsql.OptionalAnd(leftJoinExternal, nextSelect.Where) nextSelect.Where = pgsql.OptionalAnd(rightJoinExternal, nextSelect.Where) - // Only apply endpoint inequality when the bound nodes are different, to allow for self-referential relationships - if traversalStep.LeftNode.Identifier != traversalStep.RightNode.Identifier { - nextSelect.Where = pgsql.OptionalAnd(boundEndpointInequality(traversalStep.Frame.Previous, traversalStep), nextSelect.Where) - } - return pgsql.Query{Body: nextSelect}, nil } @@ -244,15 +239,11 @@ func (s *Translator) buildUnboundDirectionlessTraversalPatternRoot(traversalStep nextSelect.Where = pgsql.OptionalAnd(leftJoinExternal, nextSelect.Where) nextSelect.Where = pgsql.OptionalAnd(traversalStep.EdgeConstraints.Expression, nextSelect.Where) nextSelect.Where = pgsql.OptionalAnd(rightJoinExternal, nextSelect.Where) - - // AND (n0.id <> n1.id) - ensures edges are properly constrained to the specified nodes nextSelect.Where = pgsql.OptionalAnd( - pgsql.NewParenthetical( - pgsql.NewBinaryExpression( - pgsql.CompoundIdentifier{traversalStep.LeftNode.Identifier, pgsql.ColumnID}, - pgsql.OperatorCypherNotEquals, - pgsql.CompoundIdentifier{traversalStep.RightNode.Identifier, pgsql.ColumnID}, - ), + buildDirectionlessPairwiseEdgeConstraintForRefs( + pgsql.CompoundIdentifier{traversalStep.LeftNode.Identifier, pgsql.ColumnID}, + pgsql.CompoundIdentifier{traversalStep.RightNode.Identifier, pgsql.ColumnID}, + traversalStep.Edge.Identifier, ), nextSelect.Where, ) @@ -316,18 +307,11 @@ func (s *Translator) buildSingleBoundDirectionlessTraversalRoot(traversalStep *T }) nextSelect.Where = plan.whereConstraint - - // selected node is not joined here, so the guard must reference the bound node through the previous frame nextSelect.Where = pgsql.OptionalAnd( - pgsql.NewParenthetical( - pgsql.NewBinaryExpression( - pgsql.RowColumnReference{ - Identifier: pgsql.CompoundIdentifier{previousFrame.Binding.Identifier, plan.boundNode.Identifier}, - Column: pgsql.ColumnID, - }, - pgsql.OperatorCypherNotEquals, - pgsql.CompoundIdentifier{plan.unboundNodeIdentifier, pgsql.ColumnID}, - ), + buildDirectionlessPairwiseEdgeConstraintForRefs( + boundEndpointIDReference(previousFrame, plan.boundNode), + pgsql.CompoundIdentifier{plan.unboundNodeIdentifier, pgsql.ColumnID}, + traversalStep.Edge.Identifier, ), nextSelect.Where, ) @@ -449,15 +433,11 @@ func (s *Translator) buildSingleBoundDirectionlessTraversalRootWithOuterCorrelat nextSelect.Where = pgsql.OptionalAnd(plan.boundNodeConstraints, nextSelect.Where) nextSelect.Where = pgsql.OptionalAnd(plan.boundNodeJoinCondition, nextSelect.Where) nextSelect.Where = pgsql.OptionalAnd(plan.whereConstraint, nextSelect.Where) - - // selected node is not joined here, so the guard must reference the bound node through the previous frame nextSelect.Where = pgsql.OptionalAnd( - pgsql.NewParenthetical( - pgsql.NewBinaryExpression( - boundEndpointIDReference(previousFrame, plan.boundNode), - pgsql.OperatorCypherNotEquals, - pgsql.CompoundIdentifier{plan.unboundNodeIdentifier, pgsql.ColumnID}, - ), + buildDirectionlessPairwiseEdgeConstraintForRefs( + boundEndpointIDReference(previousFrame, plan.boundNode), + pgsql.CompoundIdentifier{plan.unboundNodeIdentifier, pgsql.ColumnID}, + traversalStep.Edge.Identifier, ), nextSelect.Where, ) @@ -504,10 +484,5 @@ func (s *Translator) buildPairwiseDirectionlessTraversalPatternRootWithOuterCorr nextSelect.Where = pgsql.OptionalAnd(leftJoinExternal, nextSelect.Where) nextSelect.Where = pgsql.OptionalAnd(rightJoinExternal, nextSelect.Where) - // Only apply endpoint inequality when the bound nodes are different, to allow for self-referential relationships - if traversalStep.LeftNode.Identifier != traversalStep.RightNode.Identifier { - nextSelect.Where = pgsql.OptionalAnd(boundEndpointInequality(traversalStep.Frame.Previous, traversalStep), nextSelect.Where) - } - return pgsql.Query{Body: nextSelect}, nil } diff --git a/cypher/models/pgsql/translate/traversal_test.go b/cypher/models/pgsql/translate/traversal_test.go new file mode 100644 index 00000000..5cff5407 --- /dev/null +++ b/cypher/models/pgsql/translate/traversal_test.go @@ -0,0 +1,54 @@ +package translate + +import ( + "testing" + + "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/specterops/dawgs/graph" + "github.com/stretchr/testify/require" +) + +// TestDualBoundTraversalUsesExactPairJoinWithoutOptimizerMarker verifies fixed +// traversal correctness does not depend on ExpandInto analysis being exhaustive. +func TestDualBoundTraversalUsesExactPairJoinWithoutOptimizerMarker(t *testing.T) { + previousFrame := &Frame{Binding: &BoundIdentifier{Identifier: "s0"}} + currentFrame := &Frame{ + Previous: previousFrame, + Binding: &BoundIdentifier{Identifier: "s1"}, + } + left := &BoundIdentifier{Identifier: "n0"} + right := &BoundIdentifier{Identifier: "n1"} + edge := &BoundIdentifier{Identifier: "e0"} + step := &TraversalStep{ + Frame: currentFrame, + Direction: graph.DirectionOutbound, + LeftNode: left, + LeftNodeBound: true, + Edge: edge, + EdgeConstraints: &Constraint{}, + EdgeJoinCondition: pgsql.NewBinaryExpression( + boundEndpointIDReference(previousFrame, left), + pgsql.OperatorEquals, + pgsql.CompoundIdentifier{edge.Identifier, pgsql.ColumnStartID}, + ), + RightNode: right, + RightNodeBound: true, + RightNodeJoinCondition: pgsql.NewBinaryExpression( + boundEndpointIDReference(previousFrame, right), + pgsql.OperatorEquals, + pgsql.CompoundIdentifier{edge.Identifier, pgsql.ColumnEndID}, + ), + } + + translator := &Translator{query: &Query{Parts: []*QueryPart{{}}}} + query, err := translator.buildTraversalPatternRoot(currentFrame, step) + require.NoError(t, err) + + selectBody, ok := query.Body.(pgsql.Select) + require.True(t, ok) + require.Len(t, selectBody.From, 1) + require.Len(t, selectBody.From[0].Joins, 1, "dual-bound fallback must not add an uncorrelated terminal-node join") + edgeTable, ok := selectBody.From[0].Joins[0].Table.(pgsql.TableReference) + require.True(t, ok) + require.Equal(t, pgsql.CompoundIdentifier{pgsql.TableEdge}, edgeTable.Name) +} diff --git a/cypher/models/pgsql/translate/with.go b/cypher/models/pgsql/translate/with.go index 38860366..624afddb 100644 --- a/cypher/models/pgsql/translate/with.go +++ b/cypher/models/pgsql/translate/with.go @@ -6,6 +6,7 @@ import ( "github.com/specterops/dawgs/cypher/models/pgsql" ) +// translateWith closes the current query part, projects WITH items, and opens the scope consumed by the next part. func (s *Translator) translateWith() error { currentPart := s.query.CurrentPart() @@ -14,6 +15,7 @@ func (s *Translator) translateWith() error { } else { var ( projectedItems = pgsql.NewIdentifierSet() + materialized []*BoundIdentifier // aggregatedItems contains a set of symbols of projected aggregate functions. aggregatedItems = pgsql.NewSymbolTable() @@ -124,8 +126,11 @@ func (s *Translator) translateWith() error { currentPart.projections.Items[idx].Alias = pgsql.AsOptionalIdentifier(projectedBinding.Identifier) } - // Assign the frame to the binding's last projection backref - projectedBinding.MaterializedBy(currentPart.Frame) + // Delay the back-reference update until every select item has + // been built. Path projections may depend on node bindings that + // appear earlier in a greedy WITH projection, and those + // dependencies must still reference the input frame here. + materialized = append(materialized, projectedBinding) // Reveal and export the identifier in the current multipart query part's frame currentPart.Frame.Reveal(projectedBinding.Identifier) @@ -143,8 +148,7 @@ func (s *Translator) translateWith() error { // Track this projected item for scope pruning projectedItems.Add(binding.Identifier) - // Assign the frame to the binding's last projection backref - binding.LastProjection = currentPart.Frame + materialized = append(materialized, binding) // Reveal and export the identifier in the current multipart query part's frame currentPart.Frame.Reveal(binding.Identifier) @@ -156,6 +160,9 @@ func (s *Translator) translateWith() error { } } } + for _, binding := range materialized { + binding.MaterializedBy(currentPart.Frame) + } if !aggregatedItems.IsEmpty() { currentPart.projections.GroupBy = append(currentPart.projections.GroupBy, groupByItems...) diff --git a/cypher/models/walk/walk_pgsql.go b/cypher/models/walk/walk_pgsql.go index f3ac3945..6ab08c84 100644 --- a/cypher/models/walk/walk_pgsql.go +++ b/cypher/models/walk/walk_pgsql.go @@ -6,10 +6,12 @@ import ( "github.com/specterops/dawgs/cypher/models/pgsql" ) +// pgsqlSyntaxNodeSliceTypeConvert widens a concrete PostgreSQL syntax-node slice for the generic walker without changing element order. func pgsqlSyntaxNodeSliceTypeConvert[F any, FS []F](fs FS) ([]pgsql.SyntaxNode, error) { return ConvertSliceType[pgsql.SyntaxNode](fs) } +// newSQLCaseWalkCursor creates a cursor that visits a CASE operand, conditions, results, and fallback in SQL order. func newSQLCaseWalkCursor(node pgsql.SyntaxNode, caseExpr pgsql.Case) (*Cursor[pgsql.SyntaxNode], error) { if len(caseExpr.Conditions) != len(caseExpr.Then) { return nil, fmt.Errorf("case expression has %d conditions and %d then expressions", len(caseExpr.Conditions), len(caseExpr.Then)) @@ -34,6 +36,7 @@ func newSQLCaseWalkCursor(node pgsql.SyntaxNode, caseExpr pgsql.Case) (*Cursor[p return nextCursor, nil } +// newSQLWalkCursor creates a structural cursor for the concrete PostgreSQL AST node type. func newSQLWalkCursor(node pgsql.SyntaxNode) (*Cursor[pgsql.SyntaxNode], error) { if isNilNode(node) { return nil, fmt.Errorf("unable to negotiate sql type %T into a translation cursor", node) @@ -210,15 +213,22 @@ func newSQLWalkCursor(node pgsql.SyntaxNode) (*Cursor[pgsql.SyntaxNode], error) }, nil case *pgsql.EdgeArrayFromPathIDs: + branches := []pgsql.SyntaxNode{typedNode.PathIDs} + if typedNode.GraphID != nil { + branches = append(branches, typedNode.GraphID) + } return &Cursor[pgsql.SyntaxNode]{ Node: node, - Branches: []pgsql.SyntaxNode{typedNode.PathIDs}, + Branches: branches, }, nil case pgsql.FunctionCall: if branches, err := pgsqlSyntaxNodeSliceTypeConvert(typedNode.Parameters); err != nil { return nil, err } else { + for _, orderBy := range typedNode.OrderBy { + branches = append(branches, orderBy) + } return &Cursor[pgsql.SyntaxNode]{ Node: node, Branches: branches, @@ -229,6 +239,9 @@ func newSQLWalkCursor(node pgsql.SyntaxNode) (*Cursor[pgsql.SyntaxNode], error) if branches, err := pgsqlSyntaxNodeSliceTypeConvert(typedNode.Parameters); err != nil { return nil, err } else { + for _, orderBy := range typedNode.OrderBy { + branches = append(branches, orderBy) + } return &Cursor[pgsql.SyntaxNode]{ Node: node, Branches: branches, diff --git a/cypher/test/cases/mutation_tests.json b/cypher/test/cases/mutation_tests.json index dc73b031..3b8338c1 100644 --- a/cypher/test/cases/mutation_tests.json +++ b/cypher/test/cases/mutation_tests.json @@ -44,7 +44,7 @@ "name": "JD's Create User Example", "type": "string_match", "details": { - "query": "merge (x:Base {objectid: '\u003cobjId\u003e'}) set x:User, x.name = 'BOB@TEST.LAB' set x += {arr: ['abc', 'def', 'ghi']} return x", + "query": "merge (x:Base {objectid: ''}) set x:User, x.name = 'BOB@TEST.LAB' set x += {arr: ['abc', 'def', 'ghi']} return x", "fitness": 6 } }, @@ -52,7 +52,7 @@ "name": "JD's Create Edges Example", "type": "string_match", "details": { - "query": "match (x) match (y) merge (x)-[:Edge]-\u003e(y)", + "query": "match (x) match (y) merge (x)-[:Edge]->(y)", "fitness": 1 } }, @@ -100,7 +100,7 @@ "name": "Create relationship", "type": "string_match", "details": { - "query": "create p = (:Label {p: '1234'})-[:Link {r: 1234}]-\u003e(b {p: '4321'}) return p", + "query": "create p = (:Label {p: '1234'})-[:Link {r: 1234}]->(b {p: '4321'}) return p", "fitness": 12 } }, @@ -108,7 +108,7 @@ "name": "Create relationship with decimal properties parameter", "type": "string_match", "details": { - "query": "create p = (:Label {p: '1234'})-[:Link $1]-\u003e(b {p: '4321'}) return p", + "query": "create p = (:Label {p: '1234'})-[:Link $1]->(b {p: '4321'}) return p", "fitness": 9 } }, @@ -116,7 +116,7 @@ "name": "Create relationship with named properties parameter", "type": "string_match", "details": { - "query": "create p = (:Label {p: '1234'})-[:Link $named]-\u003e(b {p: '4321'}) return p", + "query": "create p = (:Label {p: '1234'})-[:Link $named]->(b {p: '4321'}) return p", "fitness": 9 } }, @@ -124,7 +124,7 @@ "name": "Create relationship with matching", "type": "string_match", "details": { - "query": "match (a), (b) where a.name = 'a' and b.linked = id(a) create p = (a)-[:Linked]-\u003e(b) return p", + "query": "match (a), (b) where a.name = 'a' and b.linked = id(a) create p = (a)-[:Linked]->(b) return p", "fitness": 12 } }, @@ -248,6 +248,86 @@ "query": "match (a:Thing1), (b:Thing2) detach delete a, b return b", "fitness": 4 } + }, + { + "name": "LOGIC-04 filtered relationship delete preserves mutation binding", + "type": "string_match", + "details": { + "query": "match (s:RegressionKind05)-[r:RegressionKind06]->(e:RegressionKind07) where e.objectid = $object_id and r.shoulddelete = $should_delete delete r", + "fitness": 16 + } + }, + { + "name": "LOGIC-04 filtered detach node delete preserves mutation binding", + "type": "string_match", + "details": { + "query": "match (n:RegressionKind08) where n.objectid = $object_id detach delete n", + "fitness": 9 + } + }, + { + "name": "REC-01 inbound reconciliation delete with thirty relationship kinds", + "type": "string_match", + "details": { + "query": "match ()-[r:RegressionKind01|RegressionKind02|RegressionKind03|RegressionKind04|RegressionKind05|RegressionKind06|RegressionKind07|RegressionKind08|RegressionKind09|RegressionKind10|RegressionKind11|RegressionKind12|RegressionKind13|RegressionKind14|RegressionKind15|RegressionKind16|RegressionKind17|RegressionKind18|RegressionKind19|RegressionKind20|RegressionKind21|RegressionKind22|RegressionKind23|RegressionKind24|RegressionKind25|RegressionKind26|RegressionKind27|RegressionKind28|RegressionKind29|RegressionKind30]->(e:RegressionKind31) where e.objectid = $object_id delete r", + "fitness": 8 + } + }, + { + "name": "REC-02 outbound reconciliation delete with thirty relationship kinds", + "type": "string_match", + "details": { + "query": "match (s:RegressionKind31)-[r:RegressionKind01|RegressionKind02|RegressionKind03|RegressionKind04|RegressionKind05|RegressionKind06|RegressionKind07|RegressionKind08|RegressionKind09|RegressionKind10|RegressionKind11|RegressionKind12|RegressionKind13|RegressionKind14|RegressionKind15|RegressionKind16|RegressionKind17|RegressionKind18|RegressionKind19|RegressionKind20|RegressionKind21|RegressionKind22|RegressionKind23|RegressionKind24|RegressionKind25|RegressionKind26|RegressionKind27|RegressionKind28|RegressionKind29|RegressionKind30]->() where s.objectid = $object_id delete r", + "fitness": 8 + } + }, + { + "name": "REC-03 inbound primary group relationship delete", + "type": "string_match", + "details": { + "query": "match ()-[r:RegressionKind32]->(e:RegressionKind31) where e.objectid = $object_id and r.isprimarygroup = $flag delete r", + "fitness": 14 + } + }, + { + "name": "REC-03 outbound primary group relationship delete", + "type": "string_match", + "details": { + "query": "match (s:RegressionKind31)-[r:RegressionKind32]->() where s.objectid = $object_id and r.isprimarygroup = $flag delete r", + "fitness": 14 + } + }, + { + "name": "REC-04 endpoint object ID list relationship delete", + "type": "string_match", + "details": { + "query": "match ()-[r:RegressionKind32]->(e:RegressionKind31) where e.objectid in $object_ids delete r", + "fitness": 8 + } + }, + { + "name": "REC-06 delegated enrollment relationship delete by endpoint IDs", + "type": "string_match", + "details": { + "query": "match ()-[r:RegressionKind37]->(e:RegressionKind35) where id(e) in $template_ids delete r", + "fitness": 4 + } + }, + { + "name": "REC-07 HostsCAService relationship delete", + "type": "string_match", + "details": { + "query": "match ()-[r:RegressionKind39]->(e:RegressionKind38) where e.objectid = $object_id delete r", + "fitness": 10 + } + }, + { + "name": "REC-08 AD entity detach delete by object ID list", + "type": "string_match", + "details": { + "query": "match (n:RegressionKind31) where n.objectid in $object_ids detach delete n", + "fitness": 7 + } } ] } diff --git a/cypher/test/cases/positive_tests.json b/cypher/test/cases/positive_tests.json index cedfccdc..b27bcd6a 100644 --- a/cypher/test/cases/positive_tests.json +++ b/cypher/test/cases/positive_tests.json @@ -36,7 +36,7 @@ "name": "Support filter and quantifier expressions", "type": "string_match", "details": { - "query": "match (g:GPO) optional match (g)-[r1:GPLink {enforced: false}]-\u003e(container1) with g, container1 optional match (g)-[r2:GPLink {enforced: true}]-\u003e(container2) with g, container1, container2 optional match p1 = (g)-[r1:GPLink]-\u003e(container1)-[r2:Contains*1..]-\u003e(n1:Computer) where none(x in nodes(p1) where x.blocksinheritance = true and labels(x) = 'OU') with g, p1, container2, n1 optional match p2 = (g)-[r1:GPLink]-\u003e(container2)-[r2:Contains*1..]-\u003e(n2:Computer) return p1, p2", + "query": "match (g:GPO) optional match (g)-[r1:GPLink {enforced: false}]->(container1) with g, container1 optional match (g)-[r2:GPLink {enforced: true}]->(container2) with g, container1, container2 optional match p1 = (g)-[r1:GPLink]->(container1)-[r2:Contains*1..]->(n1:Computer) where none(x in nodes(p1) where x.blocksinheritance = true and labels(x) = 'OU') with g, p1, container2, n1 optional match p2 = (g)-[r1:GPLink]->(container2)-[r2:Contains*1..]->(n2:Computer) return p1, p2", "fitness": -6 } }, @@ -90,34 +90,34 @@ } }, { - "name": "Filter nodes using WHERE clause with \u003c operator", + "name": "Filter nodes using WHERE clause with < operator", "type": "string_match", "details": { - "query": "match (p:Person) where p.age \u003c 50 return p", + "query": "match (p:Person) where p.age < 50 return p", "fitness": 3 } }, { - "name": "Filter nodes using WHERE clause with \u003e operator", + "name": "Filter nodes using WHERE clause with > operator", "type": "string_match", "details": { - "query": "match (p:Person) where p.age \u003e 50 return p", + "query": "match (p:Person) where p.age > 50 return p", "fitness": 3 } }, { - "name": "Filter nodes using WHERE clause with \u003c= operator", + "name": "Filter nodes using WHERE clause with <= operator", "type": "string_match", "details": { - "query": "match (p:Person) where p.age \u003c= 50 return p", + "query": "match (p:Person) where p.age <= 50 return p", "fitness": 3 } }, { - "name": "Filter nodes using WHERE clause with \u003e= operator", + "name": "Filter nodes using WHERE clause with >= operator", "type": "string_match", "details": { - "query": "match (p:Person) where p.age \u003e= 50 return p", + "query": "match (p:Person) where p.age >= 50 return p", "fitness": 3 } }, @@ -125,7 +125,7 @@ "name": "Filter nodes using WHERE clause with not equal to", "type": "string_match", "details": { - "query": "match (p:Person) where p.name \u003c\u003e 'Tom Hanks' return p", + "query": "match (p:Person) where p.name <> 'Tom Hanks' return p", "fitness": 5 } }, @@ -149,7 +149,7 @@ "name": "Traverse relationship by specifying edge type, filter query using where clause", "type": "string_match", "details": { - "query": "match (p:Person)-[:ACTED_IN]-\u003e(m:Movie) where p.name = 'Tom Hanks' return m", + "query": "match (p:Person)-[:ACTED_IN]->(m:Movie) where p.name = 'Tom Hanks' return m", "fitness": 12 } }, @@ -157,7 +157,7 @@ "name": "Traverse relationship by specifying edge type, filter query using property matcher", "type": "string_match", "details": { - "query": "match (p:Person {name: 'Tom Hanks'})-[:ACTED_IN]-\u003e(m:Movie) return m", + "query": "match (p:Person {name: 'Tom Hanks'})-[:ACTED_IN]->(m:Movie) return m", "fitness": 9 } }, @@ -165,7 +165,7 @@ "name": "Traverse relationship by specifying multiple edge types", "type": "string_match", "details": { - "query": "match (p:Person)-[:ACTED_IN|DIRECTED]-\u003e(m:Movie) return m", + "query": "match (p:Person)-[:ACTED_IN|DIRECTED]->(m:Movie) return m", "fitness": 4 } }, @@ -173,7 +173,7 @@ "name": "Specify left to right relationship", "type": "string_match", "details": { - "query": "match (p:Person)-[]-\u003e(m:Movie) return m", + "query": "match (p:Person)-[]->(m:Movie) return m", "fitness": 3 } }, @@ -181,7 +181,7 @@ "name": "Specify right to left relationship", "type": "string_match", "details": { - "query": "match (p:Person)\u003c-[]-(m:Movie) return m", + "query": "match (p:Person)<-[]-(m:Movie) return m", "fitness": 3 } }, @@ -197,7 +197,7 @@ "name": "Filter query by specifying node labels in the where clause", "type": "string_match", "details": { - "query": "match (p)-[:ACTED_IN]-\u003e(m) where p:Person and m:Movie and m.title = 'The Matrix' return p.name", + "query": "match (p)-[:ACTED_IN]->(m) where p:Person and m:Movie and m.title = 'The Matrix' return p.name", "fitness": 9 } }, @@ -205,7 +205,7 @@ "name": "Filter using ranges in where clause", "type": "string_match", "details": { - "query": "match (p:Person)-[:ACTED_IN]-\u003e(m:Movie) where 2000 \u003c m.released \u003c 2003 and 100 \u003e m.last \u003c 200 return p.name", + "query": "match (p:Person)-[:ACTED_IN]->(m:Movie) where 2000 < m.released < 2003 and 100 > m.last < 200 return p.name", "fitness": 10 } }, @@ -285,7 +285,7 @@ "name": "Filter by list inclusion: list comes from the edge property named `r.roles`", "type": "string_match", "details": { - "query": "match (p:Person)-[r:ACTED_IN]-\u003e(m:Movie) where 'Neo' in r.roles return p.name", + "query": "match (p:Person)-[r:ACTED_IN]->(m:Movie) where 'Neo' in r.roles return p.name", "fitness": 6 } }, @@ -301,7 +301,7 @@ "name": "Query for the properties of an edge using keys()", "type": "string_match", "details": { - "query": "match ()-[e:EDGE_OF_INTEREST]-\u003e() return keys(e)", + "query": "match ()-[e:EDGE_OF_INTEREST]->() return keys(e)", "fitness": 1 } }, @@ -373,7 +373,7 @@ "name": "Eliminate duplicate rows returned", "type": "string_match", "details": { - "query": "match (p:Person)-[]-\u003e(m:Movie) return distinct p.name, m.title", + "query": "match (p:Person)-[]->(m:Movie) return distinct p.name, m.title", "fitness": 4 } }, @@ -413,7 +413,7 @@ "name": "Aggregation using collect() to return a list", "type": "string_match", "details": { - "query": "match (p:Person)-[:ACTED_IN]-\u003e(m:Movie) return p.name, collect(m.title)", + "query": "match (p:Person)-[:ACTED_IN]->(m:Movie) return p.name, collect(m.title)", "fitness": 5 } }, @@ -421,7 +421,7 @@ "name": "Eliminate duplication in lists", "type": "string_match", "details": { - "query": "match (p:Person)-[:ACTED_IN]-\u003e(m:Movie) where m.year = 1920 return collect(distinct (m.title))", + "query": "match (p:Person)-[:ACTED_IN]->(m:Movie) where m.year = 1920 return collect(distinct (m.title))", "fitness": 8 } }, @@ -429,7 +429,7 @@ "name": "Collecting nodes", "type": "string_match", "details": { - "query": "match (p:Person)-[:ACTED_IN]-\u003e(m:Movie) where p.name = 'tom cruise' return collect(m) as tomCruiseMovies", + "query": "match (p:Person)-[:ACTED_IN]->(m:Movie) where p.name = 'tom cruise' return collect(m) as tomCruiseMovies", "fitness": 12 } }, @@ -485,7 +485,7 @@ "name": "Conjunction", "type": "string_match", "details": { - "query": "match (n) where n.indexed \u003e= 1 and n.other_1 = 2 return n", + "query": "match (n) where n.indexed >= 1 and n.other_1 = 2 return n", "fitness": 5 } }, @@ -493,7 +493,7 @@ "name": "Multiple conjunctions", "type": "string_match", "details": { - "query": "match (n) where n.indexed \u003e= 1 and n.other_1 = 2 and n.other_2 = 3 return n", + "query": "match (n) where n.indexed >= 1 and n.other_1 = 2 and n.other_2 = 3 return n", "fitness": 8 } }, @@ -501,7 +501,7 @@ "name": "Conjunction with disjunction", "type": "string_match", "details": { - "query": "match (n) where n.indexed \u003e= 1 and (n.other_1 = 2 or n.other_2 = 3) return n", + "query": "match (n) where n.indexed >= 1 and (n.other_1 = 2 or n.other_2 = 3) return n", "fitness": 7 } }, @@ -509,7 +509,7 @@ "name": "Disjunction", "type": "string_match", "details": { - "query": "match (n) where (n.indexed \u003e= 1 or n.other_1 = 2) return n", + "query": "match (n) where (n.indexed >= 1 or n.other_1 = 2) return n", "fitness": 3 } }, @@ -517,7 +517,7 @@ "name": "Multiple disjunctions", "type": "string_match", "details": { - "query": "match (n) where (n.indexed \u003e= 1 or n.other_1 = 2 or n.other_2 = 3) return n", + "query": "match (n) where (n.indexed >= 1 or n.other_1 = 2 or n.other_2 = 3) return n", "fitness": 6 } }, @@ -557,7 +557,7 @@ "name": "Match patterns with range literal", "type": "string_match", "details": { - "query": "match (n)-[:NestedEdge*]-\u003e() where id(n) = 1 return n", + "query": "match (n)-[:NestedEdge*]->() where id(n) = 1 return n", "fitness": 1 } }, @@ -565,7 +565,7 @@ "name": "Match patterns with range literal with at least one edge", "type": "string_match", "details": { - "query": "match (n)-[:NestedEdge*1..]-\u003e() where id(n) = 1 return n", + "query": "match (n)-[:NestedEdge*1..]->() where id(n) = 1 return n", "fitness": 5 } }, @@ -573,7 +573,7 @@ "name": "Match patterns with range literal with 1 to 2 edges", "type": "string_match", "details": { - "query": "match (n)-[:NestedEdge*1..2]-\u003e() where id(n) = 1 return n", + "query": "match (n)-[:NestedEdge*1..2]->() where id(n) = 1 return n", "fitness": 3 } }, @@ -581,7 +581,7 @@ "name": "Match patterns with where and return clauses", "type": "string_match", "details": { - "query": "match (n {property: true})\u003c-[r {property: n.name}]-(s)-[v]-\u003e() where n.indexed = false return n, r.other", + "query": "match (n {property: true})<-[r {property: n.name}]-(s)-[v]->() where n.indexed = false return n, r.other", "fitness": 2 } }, @@ -613,7 +613,7 @@ "name": "Find All Domain Admins", "type": "string_match", "details": { - "query": "match p = (n:Group)\u003c-[:MemberOf*1..]-(m) where n.objectid =~ '(?i)S-1-5-.*-512' return p", + "query": "match p = (n:Group)<-[:MemberOf*1..]-(m) where n.objectid =~ '(?i)S-1-5-.*-512' return p", "fitness": 10 } }, @@ -621,7 +621,7 @@ "name": "Map Domain Trusts", "type": "string_match", "details": { - "query": "match p = (n:Domain)-[]-\u003e(m:Domain) return p", + "query": "match p = (n:Domain)-[]->(m:Domain) return p", "fitness": 3 } }, @@ -629,7 +629,7 @@ "name": "Find principals with DCSync rights", "type": "string_match", "details": { - "query": "match p = ()-[:DCSync|AllExtendedRights|GenericAll]-\u003e(:Domain {name: 'DOMAIN.PAIN'}) return p", + "query": "match p = ()-[:DCSync|AllExtendedRights|GenericAll]->(:Domain {name: 'DOMAIN.PAIN'}) return p", "fitness": 6 } }, @@ -637,7 +637,7 @@ "name": "Principals with Foreign Domain Group Membership", "type": "string_match", "details": { - "query": "match p = (n:Base)-[:MemberOf]-\u003e(m:Group) where n.domain = 'DOMAIN.PAIN' and m.domain \u003c\u003e n.domain return p", + "query": "match p = (n:Base)-[:MemberOf]->(m:Group) where n.domain = 'DOMAIN.PAIN' and m.domain <> n.domain return p", "fitness": 8 } }, @@ -645,7 +645,7 @@ "name": "Find Computers where Domain Users are Local Admin", "type": "string_match", "details": { - "query": "match p = (m:Group {name: 'DOMAIN USERS@DOMAIN.PAIN'})-[:AdminTo]-\u003e(n:Computer) return p", + "query": "match p = (m:Group {name: 'DOMAIN USERS@DOMAIN.PAIN'})-[:AdminTo]->(n:Computer) return p", "fitness": 9 } }, @@ -653,7 +653,7 @@ "name": "Find Computers where Domain Users can read LAPS passwords", "type": "string_match", "details": { - "query": "match p = (Group {name: 'DOMAIN USERS@DOMAIN.PAIN'})-[:MemberOf*0..]-\u003e(g:Group)-[:AllExtendedRights|ReadLAPSPassword]-\u003e(n:Computer) return p", + "query": "match p = (Group {name: 'DOMAIN USERS@DOMAIN.PAIN'})-[:MemberOf*0..]->(g:Group)-[:AllExtendedRights|ReadLAPSPassword]->(n:Computer) return p", "fitness": 4 } }, @@ -661,7 +661,7 @@ "name": "Find All Paths from Domain Users to High Value Targets", "type": "string_match", "details": { - "query": "match p = shortestPath((g:Group {name: 'DOMAIN USERS@DOMAIN.PAIN'})-[*1..]-\u003e(n {highvalue: true})) where g \u003c\u003e n return p", + "query": "match p = shortestPath((g:Group {name: 'DOMAIN USERS@DOMAIN.PAIN'})-[*1..]->(n {highvalue: true})) where g <> n return p", "fitness": 13 } }, @@ -669,7 +669,7 @@ "name": "Find all shortest paths to workstations where Domain Users can RDP", "type": "string_match", "details": { - "query": "match p = allShortestPaths((g:Group {name: 'DOMAIN USERS@DOMAIN.PAIN'})-[:CanRDP]-\u003e(c:Computer)) where not (c.operatingsystem contains 'Server') return p", + "query": "match p = allShortestPaths((g:Group {name: 'DOMAIN USERS@DOMAIN.PAIN'})-[:CanRDP]->(c:Computer)) where not (c.operatingsystem contains 'Server') return p", "fitness": 14 } }, @@ -677,7 +677,7 @@ "name": "Find Workstations where Domain Users can RDP", "type": "string_match", "details": { - "query": "match p = (g:Group {name: 'DOMAIN USERS@DOMAIN.PAIN'})-[:CanRDP]-\u003e(c:Computer) where not (c.operatingsystem contains 'Server') return p", + "query": "match p = (g:Group {name: 'DOMAIN USERS@DOMAIN.PAIN'})-[:CanRDP]->(c:Computer) where not (c.operatingsystem contains 'Server') return p", "fitness": 10 } }, @@ -685,7 +685,7 @@ "name": "Find Servers where Domain Users can RDP", "type": "string_match", "details": { - "query": "match p = (g:Group {name: 'DOMAIN USERS@DOMAIN.PAIN'})-[:CanRDP]-\u003e(c:Computer) where c.operatingsystem contains 'Server' return p", + "query": "match p = (g:Group {name: 'DOMAIN USERS@DOMAIN.PAIN'})-[:CanRDP]->(c:Computer) where c.operatingsystem contains 'Server' return p", "fitness": 11 } }, @@ -693,7 +693,7 @@ "name": "Find Dangerous Privileges for Domain Users Groups", "type": "string_match", "details": { - "query": "match p = (m:Group)-[:Owns|GenericAll|GenericWrite|WriteOwner|WriteDacl|MemberOf|ForceChangePassword|AllExtendedRights|AddMember|HasSession|CanApplyGPO|AllowedToDelegate|CoerceToTGT|SameForestTrust|AllowedToAct|AdminTo|CanPSRemote|CanRDP|ExecuteDCOM|HasSIDHistory|AddSelf|DCSync|ReadLAPSPassword|ReadGMSAPassword|DumpSMSAPassword|SQLAdmin|AddAllowedToAct|WriteSPN|AddKeyCredentialLink|SyncLAPSPassword|WriteAccountRestrictions|GoldenCert|ADCSESC1|ADCSESC3|ADCSESC4|ADCSESC5|ADCSESC6a|ADCSESC6b|ADCSESC7|ADCSESC9a|ADCSESC9b|ADCSESC10a|ADCSESC10b|ADCSESC13|DCFor|SyncedToEntraUser]-\u003e(n:Base) where m.objectid ends with '-513' return p", + "query": "match p = (m:Group)-[:Owns|GenericAll|GenericWrite|WriteOwner|WriteDacl|MemberOf|ForceChangePassword|AllExtendedRights|AddMember|HasSession|CanApplyGPO|AllowedToDelegate|CoerceToTGT|SameForestTrust|AllowedToAct|AdminTo|CanPSRemote|CanRDP|ExecuteDCOM|HasSIDHistory|AddSelf|DCSync|ReadLAPSPassword|ReadGMSAPassword|DumpSMSAPassword|SQLAdmin|AddAllowedToAct|WriteSPN|AddKeyCredentialLink|SyncLAPSPassword|WriteAccountRestrictions|GoldenCert|ADCSESC1|ADCSESC3|ADCSESC4|ADCSESC5|ADCSESC6a|ADCSESC6b|ADCSESC7|ADCSESC9a|ADCSESC9b|ADCSESC10a|ADCSESC10b|ADCSESC13|DCFor|SyncedToEntraUser]->(n:Base) where m.objectid ends with '-513' return p", "fitness": 9 } }, @@ -701,7 +701,7 @@ "name": "Find Domain Admins Logons to non-Domain Controllers", "type": "string_match", "details": { - "query": "match (dc)-[r:MemberOf*0..]-\u003e(g:Group) where g.objectid ends with '-516' with collect(dc) as exclude match p = (c:Computer)-[n:HasSession]-\u003e(u:User)-[r2:MemberOf*1..]-\u003e(g:Group) where g.objectid ends with '-512' and not (c in exclude) return p", + "query": "match (dc)-[r:MemberOf*0..]->(g:Group) where g.objectid ends with '-516' with collect(dc) as exclude match p = (c:Computer)-[n:HasSession]->(u:User)-[r2:MemberOf*1..]->(g:Group) where g.objectid ends with '-512' and not (c in exclude) return p", "fitness": 17 } }, @@ -789,7 +789,7 @@ "name": "Find Kerberoastable Users with most privileges", "type": "string_match", "details": { - "query": "match (u:User {hasspn: true}) optional match (u)-[:AdminTo]-\u003e(c1:Computer) optional match (u)-[:MemberOf*1..]-\u003e(:Group)-[:AdminTo]-\u003e(c2:Computer) with u, collect(c1) + collect(c2) as tempVar unwind tempVar as comps return u.name, count(distinct (comps)) order by count(distinct (comps)) desc", + "query": "match (u:User {hasspn: true}) optional match (u)-[:AdminTo]->(c1:Computer) optional match (u)-[:MemberOf*1..]->(:Group)-[:AdminTo]->(c2:Computer) with u, collect(c1) + collect(c2) as tempVar unwind tempVar as comps return u.name, count(distinct (comps)) order by count(distinct (comps)) desc", "fitness": 2 } }, @@ -797,7 +797,7 @@ "name": "Find Kerberoastable Members of High Value Groups", "type": "string_match", "details": { - "query": "match p = shortestPath((n:User)-[:MemberOf]-\u003e(g:Group)) where g.highvalue = true and n.hasspn = true return p", + "query": "match p = shortestPath((n:User)-[:MemberOf]->(g:Group)) where g.highvalue = true and n.hasspn = true return p", "fitness": 17 } }, @@ -805,7 +805,7 @@ "name": "Shortest Paths to Unconstrained Delegation Systems", "type": "string_match", "details": { - "query": "match p = shortestPath((n)-[:HasSession|AdminTo|Contains|AZLogicAppContributor*1..]-\u003e(m:Computer {unconstraineddelegation: true})) where not (n = m) return p", + "query": "match p = shortestPath((n)-[:HasSession|AdminTo|Contains|AZLogicAppContributor*1..]->(m:Computer {unconstraineddelegation: true})) where not (n = m) return p", "fitness": 13 } }, @@ -813,7 +813,7 @@ "name": "Shortest Paths from Kerberoastable Users", "type": "string_match", "details": { - "query": "match p = shortestPath((n)-[:HasSession|AdminTo|Contains|AZLogicAppContributor*1..]-\u003e(m:Computer {unconstraineddelegation: true})) where not (n = m) return p", + "query": "match p = shortestPath((n)-[:HasSession|AdminTo|Contains|AZLogicAppContributor*1..]->(m:Computer {unconstraineddelegation: true})) where not (n = m) return p", "fitness": 13 } }, @@ -821,7 +821,7 @@ "name": "Shortest Paths to Domain Admins from Kerberoastable Users", "type": "string_match", "details": { - "query": "match p = shortestPath((n:User {hasspn: true})-[:HasSession|AdminTo|Contains|AZLogicAppContributor*1..]-\u003e(m:Group {name: 'DOMAIN ADMINS@DOMAIN.PAIN'})) return p", + "query": "match p = shortestPath((n:User {hasspn: true})-[:HasSession|AdminTo|Contains|AZLogicAppContributor*1..]->(m:Group {name: 'DOMAIN ADMINS@DOMAIN.PAIN'})) return p", "fitness": 17 } }, @@ -829,7 +829,7 @@ "name": "Shortest Paths from Owned Principals", "type": "string_match", "details": { - "query": "match p = shortestPath((n:User {hasspn: true})-[:HasSession|AdminTo|Contains|AZLogicAppContributor*1..]-\u003e(m:Group {name: 'DOMAIN ADMINS@DOMAIN.PAIN'})) return p", + "query": "match p = shortestPath((n:User {hasspn: true})-[:HasSession|AdminTo|Contains|AZLogicAppContributor*1..]->(m:Group {name: 'DOMAIN ADMINS@DOMAIN.PAIN'})) return p", "fitness": 17 } }, @@ -837,7 +837,7 @@ "name": "Shortest Paths to High Value Targets", "type": "string_match", "details": { - "query": "match p = shortestPath((n)-[*1..]-\u003e(m {highvalue: true})) where m.domain = 'DOMAIN.PAIN' and m \u003c\u003e n return p", + "query": "match p = shortestPath((n)-[*1..]->(m {highvalue: true})) where m.domain = 'DOMAIN.PAIN' and m <> n return p", "fitness": 11 } }, @@ -853,7 +853,7 @@ "name": "Shortest Paths from Domain Users to High Value Targets", "type": "string_match", "details": { - "query": "match p = shortestPath((g:Group {name: 'DOMAIN USERS@DOMAIN.PAIN'})-[*1..]-\u003e(n {highvalue: true})) where g.objectid ends with '-513' and g \u003c\u003e n return p", + "query": "match p = shortestPath((g:Group {name: 'DOMAIN USERS@DOMAIN.PAIN'})-[*1..]->(n {highvalue: true})) where g.objectid ends with '-513' and g <> n return p", "fitness": 20 } }, @@ -861,7 +861,7 @@ "name": "Find Shortest Paths to Domain Admins", "type": "string_match", "details": { - "query": "match p = shortestPath((n)-[:HasSession|AdminTo|Contains|AZLogicAppContributor*1..]-\u003e(m:Group {name: 'DOMAIN ADMINS@DOMAIN.PAIN'})) where not (n = m) return p", + "query": "match p = shortestPath((n)-[:HasSession|AdminTo|Contains|AZLogicAppContributor*1..]->(m:Group {name: 'DOMAIN ADMINS@DOMAIN.PAIN'})) where not (n = m) return p", "fitness": 14 } }, @@ -869,7 +869,7 @@ "name": "Find Shortest Paths to Domain Admins with Traversal Limit", "type": "string_match", "details": { - "query": "match p = shortestPath((n)-[:HasSession|AdminTo|Contains|AZLogicAppContributor*5..1]-\u003e(m:Group {name: 'DOMAIN ADMINS@DOMAIN.PAIN'})) where not (n = m) return p", + "query": "match p = shortestPath((n)-[:HasSession|AdminTo|Contains|AZLogicAppContributor*5..1]->(m:Group {name: 'DOMAIN ADMINS@DOMAIN.PAIN'})) where not (n = m) return p", "fitness": 17 } }, diff --git a/cypher/test/test.go b/cypher/test/test.go index b4c340cf..b94ac08d 100644 --- a/cypher/test/test.go +++ b/cypher/test/test.go @@ -21,13 +21,18 @@ import ( "github.com/stretchr/testify/require" ) +// testCaseFiles embeds the parser and analyzer fixture cases consumed by Runner. +// //go:embed cases var testCaseFiles embed.FS type Type = string const ( - TypeStringMatch Type = "string_match" + // TypeStringMatch identifies a case that compares formatted query text. + TypeStringMatch Type = "string_match" + + // TypeNegativeCase identifies a case that expects parsing or analysis errors. TypeNegativeCase Type = "negative_case" ) @@ -208,6 +213,7 @@ func LoadFixture(t *testing.T, filename string) Cases { return fixture } +// testRunner loads one embedded fixture and dispatches it to the runner selected by its case type. func testRunner[T Runner](testCase Case) func(t *testing.T) { return func(t *testing.T) { // Run the test case if it isn't ignored @@ -221,6 +227,7 @@ func testRunner[T Runner](testCase Case) func(t *testing.T) { } } +// testCase parses one named JSON fixture from fs into the concrete case type requested by its metadata. func testCase(test Case) func(t *testing.T) { switch test.Type { case TypeStringMatch: @@ -236,6 +243,7 @@ func testCase(test Case) func(t *testing.T) { } } +// updatedCasesDir returns the caller-provided fixture update directory or an isolated temporary directory. func updatedCasesDir() (string, error) { if workingDir, err := os.Getwd(); err != nil { return "", err @@ -250,6 +258,7 @@ func updatedCasesDir() (string, error) { } } +// UpdatePositiveTestCasesFitness rewrites positive fixtures with their current PostgreSQL translations. func UpdatePositiveTestCasesFitness() error { if updatedCasesPath, err := updatedCasesDir(); err != nil { return err @@ -291,10 +300,13 @@ func UpdatePositiveTestCasesFitness() error { } else { details.ExpectedFitness = &complexity.RelativeFitness - if updatedDetails, err := json.Marshal(details); err != nil { + var updatedDetails bytes.Buffer + encoder := json.NewEncoder(&updatedDetails) + encoder.SetEscapeHTML(false) + if err := encoder.Encode(details); err != nil { return fmt.Errorf("error marshalling test case details: %v", err) } else { - nextCase.Details = updatedDetails + nextCase.Details = bytes.TrimSpace(updatedDetails.Bytes()) } } @@ -309,7 +321,10 @@ func UpdatePositiveTestCasesFitness() error { } else { defer output.Close() - if err := json.NewEncoder(output).Encode(updatedCases); err != nil { + encoder := json.NewEncoder(output) + encoder.SetEscapeHTML(false) + encoder.SetIndent("", " ") + if err := encoder.Encode(updatedCases); err != nil { return err } } diff --git a/databaseguard/guard.go b/databaseguard/guard.go new file mode 100644 index 00000000..e1a8c679 --- /dev/null +++ b/databaseguard/guard.go @@ -0,0 +1,151 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +// Package databaseguard prevents destructive database workflows from running +// against a target that was not explicitly named as disposable by the +// operator. +package databaseguard + +import ( + "fmt" + "net" + "net/url" + "os" + "slices" + "strconv" + "strings" + + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + // AllowDestructiveEnv names the environment variable that must equal "1" before destructive database work is permitted. + AllowDestructiveEnv = "DAWGS_INTEGRATION_ALLOW_DESTRUCTIVE" + + // DisposableTargetsEnv names the environment variable containing the exact, credential-free targets approved for destructive work. + DisposableTargetsEnv = "DAWGS_INTEGRATION_DISPOSABLE_TARGETS" + + // allowDestructiveValue is the acknowledgement value required by Validate. + allowDestructiveValue = "1" +) + +// Target returns a credential-free, stable database endpoint identity suitable +// for explicit operator confirmation. The identity is derived from the +// effective driver configuration so endpoint-changing connection parameters +// cannot authorize a different target than the driver will use. +func Target(connection string) (string, error) { + parsed, err := url.Parse(connection) + if err != nil { + return "", fmt.Errorf("invalid database connection string") + } + + switch strings.ToLower(parsed.Scheme) { + case "postgres", "postgresql": + return postgresTarget(connection) + case "neo4j", "neo4j+s", "neo4j+ssc": + return neo4jTarget(parsed) + case "": + return "", fmt.Errorf("connection string must include a scheme and host") + default: + return "", fmt.Errorf("unsupported database connection scheme") + } +} + +// postgresTarget returns the canonical PostgreSQL endpoint and database that the parsed pgx configuration will use. +func postgresTarget(connection string) (string, error) { + config, err := pgxpool.ParseConfig(connection) + if err != nil { + return "", fmt.Errorf("invalid PostgreSQL connection string") + } + + host := strings.ToLower(strings.TrimSpace(config.ConnConfig.Host)) + port := config.ConnConfig.Port + if host == "" || port == 0 { + return "", fmt.Errorf("PostgreSQL connection string must resolve to one host and port") + } + + for _, fallback := range config.ConnConfig.Fallbacks { + if !strings.EqualFold(strings.TrimSpace(fallback.Host), host) || fallback.Port != port { + return "", fmt.Errorf("destructive PostgreSQL connections must resolve to one endpoint") + } + } + + database := config.ConnConfig.Database + if database == "" { + database = "" + } + + return "postgresql://" + net.JoinHostPort(host, strconv.FormatUint(uint64(port), 10)) + "/" + url.PathEscape(database), nil +} + +// neo4jTarget returns a credential-free Neo4j target with an explicit port and escaped database name. +func neo4jTarget(parsed *url.URL) (string, error) { + host := strings.ToLower(strings.TrimSpace(parsed.Hostname())) + if host == "" { + return "", fmt.Errorf("Neo4j connection string must include a host") + } + + port := uint64(7687) + if parsedPort := parsed.Port(); parsedPort != "" { + parsedValue, err := strconv.ParseUint(parsedPort, 10, 16) + if err != nil || parsedValue == 0 { + return "", fmt.Errorf("invalid Neo4j connection port") + } + port = parsedValue + } + + database := strings.Trim(parsed.EscapedPath(), "/") + if database == "" { + database = "" + } else if decoded, err := url.PathUnescape(database); err != nil || strings.Contains(decoded, "/") { + return "", fmt.Errorf("invalid Neo4j database name") + } else { + database = url.PathEscape(decoded) + } + + return strings.ToLower(parsed.Scheme) + "://" + net.JoinHostPort(host, strconv.FormatUint(port, 10)) + "/" + database, nil +} + +// Validate requires both an explicit destructive-operation acknowledgement and +// an exact target allowlist match. Errors expose only the sanitized target. +func Validate(connection, acknowledgement, disposableTargets string) error { + target, err := Target(connection) + if err != nil { + return err + } + if acknowledgement != allowDestructiveValue { + return fmt.Errorf("destructive database access to %s is disabled: set %s=%s and include the target in %s", target, AllowDestructiveEnv, allowDestructiveValue, DisposableTargetsEnv) + } + + targets := splitTargets(disposableTargets) + if !slices.Contains(targets, target) { + return fmt.Errorf("destructive database target %s is not confirmed in %s", target, DisposableTargetsEnv) + } + + return nil +} + +// ValidateEnvironment validates a destructive target using the process-wide +// acknowledgement and exact-target allowlist. Destructive entry points should +// call this immediately before opening or mutating a database rather than rely +// on a command wrapper to have performed the check. +func ValidateEnvironment(connection string) error { + return Validate( + connection, + os.Getenv(AllowDestructiveEnv), + os.Getenv(DisposableTargetsEnv), + ) +} + +// splitTargets parses a comma-separated target allowlist, trimming whitespace and discarding empty entries. +func splitTargets(value string) []string { + var targets []string + for _, target := range strings.Split(value, ",") { + if target = strings.TrimSpace(target); target != "" { + targets = append(targets, target) + } + } + return targets +} diff --git a/databaseguard/guard_test.go b/databaseguard/guard_test.go new file mode 100644 index 00000000..04e5936c --- /dev/null +++ b/databaseguard/guard_test.go @@ -0,0 +1,90 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package databaseguard + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestTargetRedactsCredentialsAndQuery verifies canonical targets never expose connection credentials or query parameters. +func TestTargetRedactsCredentialsAndQuery(t *testing.T) { + target, err := Target("postgresql://user:secret@LOCALHOST:65432/dawgs?sslmode=disable&password=other") + require.NoError(t, err) + require.Equal(t, "postgresql://localhost:65432/dawgs", target) + require.NotContains(t, target, "user") + require.NotContains(t, target, "secret") + require.NotContains(t, target, "password") +} + +// TestTargetNamesDefaultDatabase verifies a missing Neo4j database is represented by an explicit placeholder. +func TestTargetNamesDefaultDatabase(t *testing.T) { + target, err := Target("neo4j://localhost:7687") + require.NoError(t, err) + require.Equal(t, "neo4j://localhost:7687/", target) +} + +// TestTargetUsesEffectivePostgreSQLEndpoint verifies pgx query parameters override authority components during target canonicalization. +func TestTargetUsesEffectivePostgreSQLEndpoint(t *testing.T) { + target, err := Target("postgresql://user:secret@localhost:65432/disposable?host=PROD&port=5433&dbname=live") + require.NoError(t, err) + require.Equal(t, "postgresql://prod:5433/live", target) +} + +// TestTargetCanonicalizesPostgreSQLSchemeAndDefaultPort verifies PostgreSQL aliases and implicit ports produce one stable identity. +func TestTargetCanonicalizesPostgreSQLSchemeAndDefaultPort(t *testing.T) { + target, err := Target("postgres://user:secret@LOCALHOST/dawgs?sslmode=disable") + require.NoError(t, err) + require.Equal(t, "postgresql://localhost:5432/dawgs", target) +} + +// TestTargetCanonicalizesNeo4jDefaultPortAndEscapedDatabase verifies IPv6 hosts, default ports, and escaped database names remain stable. +func TestTargetCanonicalizesNeo4jDefaultPortAndEscapedDatabase(t *testing.T) { + target, err := Target("neo4j+s://user:secret@[2001:DB8::1]/Case%20Sensitive") + require.NoError(t, err) + require.Equal(t, "neo4j+s://[2001:db8::1]:7687/Case%20Sensitive", target) +} + +// TestValidateRequiresAcknowledgementAndExactTarget verifies both safety gates are mandatory and target matching is exact. +func TestValidateRequiresAcknowledgementAndExactTarget(t *testing.T) { + connection := "postgresql://user:secret@localhost:65432/dawgs" + target := "postgresql://localhost:65432/dawgs" + + require.ErrorContains(t, Validate(connection, "", target), AllowDestructiveEnv) + require.ErrorContains(t, Validate(connection, "1", "postgresql://localhost:65432/other"), DisposableTargetsEnv) + require.NoError(t, Validate(connection, "1", "neo4j://localhost:7687/, "+target)) + require.ErrorContains(t, Validate("postgresql://localhost:65432/CaseSensitive", "1", "postgresql://localhost:65432/casesensitive"), DisposableTargetsEnv) +} + +// TestValidateEnvironment verifies process environment values authorize the corresponding canonical target. +func TestValidateEnvironment(t *testing.T) { + t.Setenv(AllowDestructiveEnv, "1") + t.Setenv(DisposableTargetsEnv, "postgresql://localhost:5432/dawgs") + require.NoError(t, ValidateEnvironment("postgres://user:secret@localhost/dawgs")) +} + +// TestTargetRejectsIncompleteConnection verifies target derivation rejects connection strings without a scheme and host. +func TestTargetRejectsIncompleteConnection(t *testing.T) { + _, err := Target("localhost/dawgs") + require.Error(t, err) +} + +// TestTargetErrorsDoNotExposeCredentials verifies malformed connection errors do not echo sensitive input. +func TestTargetErrorsDoNotExposeCredentials(t *testing.T) { + connection := "postgresql://user:super-secret@localhost/%zz" + _, err := Target(connection) + require.Error(t, err) + require.NotContains(t, err.Error(), "user") + require.NotContains(t, err.Error(), "super-secret") + require.NotContains(t, err.Error(), connection) +} + +// TestTargetRejectsMultiplePostgreSQLEndpoints verifies destructive authorization cannot cover a multi-host PostgreSQL failover configuration. +func TestTargetRejectsMultiplePostgreSQLEndpoints(t *testing.T) { + _, err := Target("postgresql://user:secret@localhost/dawgs?host=one,two") + require.ErrorContains(t, err, "one endpoint") +} diff --git a/docs/cysql_traversal_priorities.md b/docs/cysql_traversal_priorities.md new file mode 100644 index 00000000..8ca42230 --- /dev/null +++ b/docs/cysql_traversal_priorities.md @@ -0,0 +1,1009 @@ +# CySQL traversal performance priorities + +Date: 2026-08-12 + +Status: implementation complete; promotion evidence pending + +The code and current activation disposition are recorded in +[`experiments/traversal_priority_implementation_status_v1.md`](experiments/traversal_priority_implementation_status_v1.md). +New production promotion remains evidence-gated as specified below. + +This plan turns the fresh CySQL/PostgreSQL versus Cypher/Neo4j benchmark and +source review into an implementation and qualification program. It focuses on +ordinary variable-length traversal orientation, bound `shortestPath` (SP), +`allShortestPaths` (ASP), and fixed one-hop `ExpandInto` behavior. + +The principal decision is to build one exact, observable traversal-selection +framework rather than add another isolated lowering. The first production +targets are a topology-aware forward/reverse orientation tournament and compact +bidirectional SP candidates. Bidirectional ASP follows after the shared search +kernel and telemetry are qualified. Fixed one-hop `ExpandInto` is a narrow, +measure-first opportunity. A persistent topology synopsis is deferred until +runtime probes prove that its maintenance and cache complexity are warranted. + +## Executive priority order + +Engineering effort should proceed in this order: + +| Priority | Work | Reason | +| --- | --- | --- | +| P0 | Shared telemetry, matched plan deltas, and frozen qualification corpus | Current PostgreSQL function scans hide traversal work, while Neo4j 4.4 SP/ASP profiles do not count internal relationship traversal. Selector work is not explainable or safely promotable without independent counters. | +| P1 | General ordinary-expansion orientation tournament | The measured fixed-suffix crossover is the largest ordinary-traversal opportunity: reverse is dramatically better on sparse terminal topology and materially worse under high reverse fan-in. | +| P2 | Compact SP architecture and scheduler tournament | Current S4 witness and deep/inbound execution is the main SP loss, while exact inline references show that the gap is not inherent to PostgreSQL storage. | +| P3 | Compact bidirectional ASP predecessor DAG | Recursive ASP is materially slower than Neo4j and currently lacks independent predecessor/output gates. It should reuse the proven SP search and telemetry foundation. | +| P4 | Bounded endpoint resolution and step-local predicate support | The current singleton-ID envelope excludes unique property seeks, small endpoint sets, and safe universal predicates that Neo4j can prepare before traversal. | +| P5 | Fixed one-hop `ExpandInto` endpoint choice and pair reuse | Neo4j's lower-degree scan and pair cache are useful hypotheses, but PostgreSQL may already choose an efficient plan for the current bound-pair join, including a parameterized index lookup or `Memoize`; this must be measured before adding probes. | +| P6 | Optional versioned topology synopsis | Persistent estimates may reduce probe cost, but they are advisory, mutation-sensitive, and absent from the current translation-cache identity. Runtime evidence comes first. | + +This is the engineering-priority order, not necessarily the automatic-promotion +order. A semantically narrow fixed-hop candidate may graduate before a recursive +candidate if it independently passes every gate. Orientation and SP reference +work can proceed in parallel after P0. ASP depends on the common bidirectional +state model and its counters. + +## Outcomes and success measures + +The program should deliver: + +1. Exact runtime selection between forward and reverse ordinary expansions for + qualified shapes, with a same-statement forward incumbent on uncertainty or + overflow. +2. Exact SP comparison among the current single-ended compact executor, + Neo4j-4.4-style strict per-node alternation, and current-Neo4j-style + smaller-current-level expansion. +3. Exact ASP comparison among the current single-ended predecessor DAG and two + bidirectional predecessor-DAG schedulers, with independent discovery, + predecessor, and output-enumeration gates. +4. Executor-reported work metrics that explain a choice in terms of seeds, + directional degree, frontier growth, edge scans, reconvergence, + predecessor multiplicity, meeting width, fallback, and hydration. +5. Matched PostgreSQL/Neo4j plan-delta reports that identify starting side, + physical direction, predicate placement, estimate error, and traversal + setup without treating unlike backend operator counters as equivalent. +6. Versioned selectors, reference identities, negative-result records, and a + reversible rollout path. + +Promotion is not defined as "beat Neo4j everywhere." Neo4j is an exact-result +and descriptive latency oracle. A CySQL candidate is promoted only when it is +exact, beats or contains its PostgreSQL incumbent on predeclared topology +buckets, and stays within resource and operational limits. + +For tied singleton SP, "exact" means the same minimum distance and one valid +minimum relationship-unique witness, not the same arbitrary witness as Neo4j +or another CySQL executor. ASP and bag-valued ordinary traversals require their +complete logical result multisets. + +## Scope and explicit non-goals + +The initial scope is read-only, directed, bounded traversal with a single +variable region or one statically proven endpoint pair. It includes the current +endpoint-seeded and three-hop fixed-suffix envelopes, singleton bound SP/ASP, +and fixed one-hop `ExpandInto`. + +The first program does not: + +- implement a general IDP query-graph solver or reorder arbitrary Cypher + components; +- infer correctness from planner estimates or make mutable statistics a + translation-time dependency; +- change trail, bag, tie, optional-match, mutation, or predicate semantics; +- make legacy full-trail bidirectional harnesses production candidates; +- revive the retired suffix keyset-continuation design; +- force one SP/ASP scheduler across every topology or observation mode; +- use Neo4j latency or opaque 4.4 `ShortestPath` DB hits as a CySQL release + threshold. + +## Baseline evidence to freeze + +The 2026-08-12 discovery capture used PostgreSQL 17.10 and Neo4j 4.4.44. It +contained two backend-order-balanced rounds, ten warmups, and thirty measured +samples per round. These results motivate the work, but they are not a release +gate and must be recaptured as milestone M0. + +| Shape | Discovery result | Planning implication | +| --- | --- | --- | +| Bounded outbound SP distance | CySQL S3 was about 6-30x faster | Preserve S3 as a real tournament arm; do not replace it globally. | +| SP witness and deep physical-inbound search | Neo4j was about 5-16x faster | Tournament both execution boundary and bidirectional scheduler. | +| Recursive ASP at depths 3 and 16 | Neo4j was about 5.7-13.1x faster | Shallow two-hop fixtures are insufficient; exercise the predecessor workspace. | +| Sparse fixed suffix | Neo4j was about 51x faster than production CySQL | General orientation selection has high expected value. | +| Forced CySQL suffix reverse on that sparse case | About 460x faster than forward endpoint-ID output | The reverse implementation is viable when topology is favorable. | +| High reverse fan-in | CySQL forward was about 3.7-4.4x faster than Neo4j; forced reverse was about 3.4x slower than forward | Static "always reverse" is unsafe as a performance policy. | +| Exact inline PostgreSQL references | About 2.5-45.7x faster than corresponding compact production functions on selected cases | Function/workspace overhead and algorithm must be separated in the tournament. | + +The source capture, raw benchmark records, and local review currently live +under `.coverage/fresh-plan-delta-20260812`. M0 must create a checksummed capture +bundle and commit only compact, credential-free decision records; raw +environment-specific artifacts remain ignored. + +## Neo4j lessons to use deliberately + +The primary source target is the measured Neo4j 4.4.44 tag at commit +[`17d7609`](https://github.com/neo4j/neo4j/tree/17d7609361109bd9b08ea149a5ed5966f1115324). +Current upstream behavior is pinned separately to the reviewed 2026.06 commit +[`eccd584`](https://github.com/neo4j/neo4j/tree/eccd584a64d468af3daeab421478fe78567c518f). +Current behavior must not be projected backward onto the measured server. + +The source review establishes these design inputs: + +- Ordinary relationship planning creates candidates from both endpoints and + lets bounded IDP retain the cheapest orientation. The suffix-first benchmark + plan is a general enumeration result, not a special suffix rule. See + [`SingleComponentPlanner`](https://github.com/neo4j/neo4j/blob/17d7609361109bd9b08ea149a5ed5966f1115324/community/cypher/cypher-planner/src/main/scala/org/neo4j/cypher/internal/compiler/planner/logical/idp/SingleComponentPlanner.scala#L215-L244). +- Neo4j 4.4 statistics contain global node, label, relationship-step, and index + selectivity values, but no endpoint-local degree, frontier survival, + reconvergence, meeting-cut width, or predecessor/output multiplicity. See + [`GraphStatistics`](https://github.com/neo4j/neo4j/blob/17d7609361109bd9b08ea149a5ed5966f1115324/community/cypher/planner-spi/src/main/scala/org/neo4j/cypher/internal/planner/spi/GraphStatistics.scala#L27-L66). +- Generic `VarLengthExpand(All/Into)` is a single-ended stack-based DFS in its + planned orientation. `Into` checks the bound target when emitting; it does + not become target-directed or bidirectional. See + [`VarLengthExpandPipe`](https://github.com/neo4j/neo4j/blob/17d7609361109bd9b08ea149a5ed5966f1115324/community/cypher/interpreted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/interpreted/pipes/VarLengthExpandPipe.scala#L50-L135). +- Fixed one-hop `ExpandInto` is different: Neo4j can scan the lower-degree + endpoint and cache a node-pair result. See + [`CachingExpandInto`](https://github.com/neo4j/neo4j/blob/17d7609361109bd9b08ea149a5ed5966f1115324/community/cypher/runtime-util/src/main/java/org/neo4j/internal/kernel/api/helpers/CachingExpandInto.java#L139-L207). +- Bound SP/ASP is attached only after both endpoints are available. Neo4j + 4.4's specialized bidirectional BFS alternates one newly discovered node per + side and retains same-depth predecessor relationships. See + [`ShortestPath`](https://github.com/neo4j/neo4j/blob/17d7609361109bd9b08ea149a5ed5966f1115324/community/graph-algo/src/main/java/org/neo4j/graphalgo/impl/path/ShortestPath.java#L207-L343). +- Current Neo4j expands a complete level from the side with the smaller current + level, a materially different scheduler. See + [`BiDirectionalBFSImpl`](https://github.com/neo4j/neo4j/blob/eccd584a64d468af3daeab421478fe78567c518f/community/cypher/runtime-util/src/main/java/org/neo4j/internal/kernel/api/helpers/traversal/BiDirectionalBFSImpl.java#L167-L195). +- Neo4j 4.4's Cypher profiler does not expose internal SP relationship reads. + Raw `ShortestPath` DB-hit counts must be marked opaque, not compared to + PostgreSQL recursive rows or edge probes. + +The plan adopts orientation enumeration, endpoint binding, bidirectional BFS, +frontier-aware scheduling, and two-sided predecessor reconstruction as +candidate ideas. It does not adopt Neo4j's global-average cost blindness, +opaque SP/ASP telemetry, or generic DFS behavior as CySQL requirements. + +## Architecture and decision boundaries + +The target decision flow is: + +```text +Cypher shape analysis + | + v +exact candidate envelope + observation classification + | + +---------------- compile-time diagnostics ----------------+ + | | + v v +same-statement capped probes or executor frontier state plan-delta record + | + v +versioned runtime policy + | + +---------+-----------+------------------+ + | | | | + v v v v +forward/reverse SP arm ASP arm fixed-hop arm + | | | | + +---------+-----------+------------------+ + | + v + exact gated output or incumbent fallback + | + v + late hydration + runtime telemetry +``` + +Compile-time facts and runtime facts must remain distinct: + +- The optimizer records the correctness envelope, candidates, observation + mode, selector version, caps, and fallback policy. +- The emitted SQL or executor records probes performed, scheduler decisions, + runtime arm, work, overflow, and fallback actually executed. +- GraphBench must not claim that a compile-time candidate ran merely because it + was planned or emitted. +- Tool forcing may choose among structurally eligible candidates; it may never + broaden their correctness envelope. + +The current translation cache is keyed by normalized query text, graph ID, and +parameter-name/type shape. Mutable parameter values or graph statistics must +therefore be consulted inside the generated statement. If a future selector +embeds a synopsis value at translation time, a statistics generation and +invalidation contract must first be added to the cache key. + +Mutable rollout policy is subject to the same rule. Feature-gate state, +selector version, and caps are not in the current cache key. A policy that can +change during a driver's lifetime must be supplied at execution time, add an +explicit cache generation, or invalidate affected translations. Otherwise a +rollback can leave cached tournament SQL active. Immutable caps may be SQL +literals; planner-created SQL parameters without `ParameterSources` currently +make a translation non-cacheable and need explicit rebinding/cache support if +that behavior is not desired. + +## Non-negotiable semantic contract + +Every candidate, probe, and fallback must preserve: + +- graph partition and resolved relationship-kind filtering; +- logical direction and the correct physical adjacency index; +- inclusive minimum and maximum depth, including qualified zero-length paths; +- relationship-trail uniqueness while permitting repeated nodes where Cypher + permits them; +- ordered relationship and node IDs in logical source-to-target order; +- prefix/suffix relationship non-reuse across stitched path regions; +- duplicate root rows, endpoint rows, suffix rows, and output bag + multiplicity; +- SP's one arbitrary valid minimum trail and ASP's complete set of + relationship-distinct minimum trails; +- predicate null behavior, locality, determinism, and evaluation count; +- optional-match and mutation visibility rules; +- one top-level SQL statement for probes, candidate, and fallback, plus an + explicit snapshot contract. SQL-only CTE arms share a statement snapshot; + `VOLATILE` PL/pgSQL internal statements under `READ COMMITTED` must not be + assumed to do so. Function-backed candidates require a deliberate mechanism + such as repeatable-read execution, or an independently proven equivalent, + before claiming snapshot-stable fallback; +- no candidate row exposure until every fallback-triggering gate has passed; +- prompt cancellation, rollback recovery, and clean reuse of a pooled session. + +The singleton SP tie policy remains the contract in +[`shortest_path_tie_policy.md`](shortest_path_tie_policy.md). Physical edge ID +or insertion order is not public. ASP may not use the singleton tie policy to +discard equal-depth predecessors. + +The PostgreSQL schema currently has a unique +`(start_id, end_id, kind_id, graph_id)` relationship constraint. Same-kind +parallel physical relationships cannot be represented in the current backend. +Cross-kind parallel relationships must be covered now; same-kind parallel-edge +parity remains an explicit storage boundary, not a silently skipped test. + +## Workstream 0: observability and matched plan deltas + +This is the prerequisite for every selector change. + +### 0.1 PostgreSQL executor telemetry + +Add a versioned `TraversalExecutionTelemetry` schema to GraphBench records and +PostgreSQL full-comparator records. Preserve `PostgresPlanMetrics` for measured +plan facts, but do not infer hidden PL/pgSQL work from a `Function Scan` loop. + +Use two telemetry levels: + +- A lightweight summary: requested/planned/emitted/runtime/applied identity, + selector and scheduler version, caps, runtime branch, overflow, and fallback. +- A tool-only diagnostic replay on the same connection: per-level and + per-stage executor counters. It runs outside the timed sample block so + detailed instrumentation does not contaminate latency evidence. + +Replay counters describe that untimed invocation, not a particular timed +sample. Store them in a separate diagnostic boundary and do not combine their +resource values with the production timing record. + +Missing required telemetry is a qualification failure, not a zero value. Every +derived field carries provenance naming the function, CTE, or executor metric +that produced it. + +Record at minimum: + +| Family | Required runtime counters | +| --- | --- | +| Ordinary DFS/recursive CTE | roots, edge candidates, admitted states, relationship-repeat rejects, recursive rows, peak state, emitted trails, hydration rows | +| Orientation policy | forward/reverse seeds, duplicate seeds, suffix rows, distinct boundaries, typed directional degree samples, shallow survival, probe rows/time/buffers, scores, selected side, sentinel overflow, branch loops | +| SP | scheduler actions, per-side depth/frontier, candidate edges, distinct new nodes, seen/frontier/queue peaks, meeting candidates, frozen distance, witness rows, fallback | +| ASP | SP counters plus same-depth predecessor additions, predecessor peak, meeting nodes, cut depth, saturating path-count estimate, enumerated candidates, duplicate rejects, output paths/edge cells/bytes | +| Hydration | path count, node/edge lookups, loops, rows, time, and bytes separately from discovery | + +Candidate workspace metrics must be invocation-keyed and session-local so +concurrent pooled sessions cannot collide. Cancellation and SQL errors +propagate; they are not converted into performance fallbacks. + +### 0.2 Neo4j read profiling + +Extend GraphBench to run a read-only `PROFILE` pass after the timed block while +retaining `EXPLAIN` for writes. Persist: + +- planner and runtime version; +- ordered operator tree and child order; +- estimated and actual rows, loops, DB hits, page-cache hits/misses, and + operator time where the server exposes them; +- leaf variables, access predicates, expansion direction, and starting side; +- an explicit `internal_traversal_work=opaque` marker for 4.4 SP/ASP. + +Normalize the current doubled `@neo4j` operator suffix and verify endpoint-child +fidelity. Neo4j profile data remains descriptive and must not become a CySQL +release gate. + +### 0.3 Paired PlanCorpus record + +Add a versioned PostgreSQL/Neo4j plan-delta record keyed by dataset, case, +workload hash, source revision, and backend plan fingerprints. It should +compare semantic stages rather than raw operator names: + +- starting and terminal access; +- logical and physical traversal direction; +- predicate placement and endpoint binding; +- ordinary expand versus SP/ASP operator family; +- estimated seeds, traversal multiplier/frontier, output, and Q-error; +- PostgreSQL planned/emitted/runtime/fallback identities; +- whether Neo4j reordered the pattern and whether the chosen side did less + observed work. + +Rank opposite-side choices, largest estimate disagreements, predicate moves, +fallback/cap cases, and hydration deltas. Incomplete pairs must be explicit; +they must not disappear through intersection-only reporting. PlanCorpus remains +the plan inventory and GraphBench remains the runtime authority. + +## Workstream 1: ordinary traversal orientation tournament + +The strategy should be general in framework and deliberately narrow at first +activation. + +### 1.1 Candidate model + +Introduce runtime policy identity `orientation-probe-v1`. Keep executed arm +identities separate: + +- `EXPANSION-STEPWISE-FORWARD` is the permanent exact incumbent. +- `EXPANSION-SUFFIX-SEEDED-REVERSE` is the exact fixed-suffix reverse arm. +- `EXPANSION-ENDPOINT-SEEDED-REVERSE` remains the exact terminal-seeded arm. +- factored-forward and backward-viability arms remain references until they + independently qualify. + +Do not overload compile-time `SelectedStrategy` to imply a runtime choice. Add +emitted-policy, probe-cap, admission, and candidate fields to the typed +`ExpansionSearchStrategyDecision` and translation outcome. Record the actual +arm, probe results, overflow, and fallback only in execution/GraphBench +telemetry; translation cannot know them, and a translation-cache hit does not +reconstruct a fresh runtime outcome. + +Initial eligibility remains conservative: + +- one read-only, non-optional ordinary pattern region; +- one directed, bounded variable expansion with maximum depth at most 64; +- a bound/safely materializable seed region on each considered side; +- no relationship variable or relationship/path-dependent predicate; +- no cross-region correlation or limit-pushdown conflict; +- endpoint-ID, ordered-ID, or full-path observation with proven projection + alignment. + +The first suffix activation must reproduce the current envelope exactly: a +bound root; one outbound, single-kind variable expansion; exactly three +outbound, single-kind fixed suffix hops; exactly one right-node kind on every +suffix hop; and the existing dependency, observation, and no-function-call +restrictions. Endpoint-seeded migration likewise preserves its current +identity-function exception and all other restrictions. "Deterministic" is not +enough to broaden expression eligibility because repeated probing can change +evaluation count and exception behavior. Other predicates or contiguous fixed +regions wait for the predicate-class workstream and their own decision record. + +### 1.2 Same-statement probe and branch design + +Emit one statement containing: + +1. A capped forward-root materialization. +2. A capped reverse seed materialization: terminal endpoints or exact suffix + rows plus distinct boundary nodes. +3. Capped typed directional-degree probes using the existing covering + `(start_id, kind_id)` and `(end_id, kind_id)` indexes. +4. An optional, statically enabled one-level survival probe with an explicit + row/edge cap; its cost envelope is qualified offline. +5. A versioned score and hysteresis decision CTE. +6. A reverse-state admission relation capped at `state_limit + 1`. +7. Strictly disjoint reverse and forward-incumbent branches. + +Every cap uses a `cap + 1` sentinel. Probe relations must actually contain an +explicit bound. The existing unused `buildFixedSuffixProbeCTE` helper is not +currently limited despite its comment; bounding or replacing it is a +prerequisite, not evidence that suffix probing is already safe. + +Capped relations are evidence, not automatically exact query inputs. Keep an +uncapped exact source for the incumbent. A candidate may consume a capped root, +endpoint, or suffix relation only after its sentinel proves that the relation +is complete; overflow must not feed truncated rows to either arm. If a complete +probe relation is reused to avoid duplicate work, tests must prove that it +retains the exact duplicate and suffix-bag multiplicity required by that arm. + +Record: + +- distinct and duplicate roots; +- reverse seed rows and distinct seed nodes; +- suffix row multiplicity and distinct boundary count; +- first-hop typed adjacency rows, maximum sampled degree, and a high percentile + when the seed set is small; +- one-level admitted-next-node ratio; +- reverse states consumed before admission; +- total probe latency and buffers. + +Latency and buffers are post-execution telemetry used to qualify the policy; +plain CTE SQL cannot observe them in time to choose a branch within that same +statement. + +The initial policy is dominance-based, not a fragile learned formula: + +- choose reverse only when required probes are complete below their caps and + its versioned score beats forward by a qualified hysteresis margin; +- choose forward on overflow, missing evidence, ties, or ambiguous + correlation; +- if reverse-state admission crosses its sentinel, discard all candidate state + and run the exact forward incumbent before returning a row. + +Thresholds are derived from predeclared GraphBench training buckets and frozen +before the holdout is opened. Parameter values and topology stay runtime inputs, +so cached SQL remains safe. + +### 1.3 Implementation sequence + +1. Refactor fixed-prefix and fixed-suffix analysis in + `cypher/models/pgsql/optimize/lowering_plan.go` into a common contiguous + orientation-candidate analyzer while retaining specific fallback reasons. +2. Extend typed decisions in `cypher/models/pgsql/optimize/lowering.go` and + outcomes in `cypher/models/pgsql/translate/translator.go`. +3. Add `cypher/models/pgsql/translate/expansion_orientation.go` and extract + reusable seed, reverse recursion, projection alignment, overflow, and + incumbent-gating helpers from `expansion_endpoint_seeded.go` and + `expansion_suffix_seeded.go`. +4. Emit the incumbent first, then wrap it with probes and disjoint gates in + `pattern.go`. Distinguish tournament emission from runtime arm execution. +5. Migrate endpoint-seeded reverse to the common framework without changing + its current 32-endpoint/4096-state behavior. +6. Add guarded suffix reverse; keep the existing force seams as independent + A/B controls. +7. Run shadow selection before changing production. The shadow can compute + `would_select` while executing the incumbent; regret comes from separate + matched GraphBench runs that execute the exact forced arms. + +The retired keyset-continuation experiment is not a candidate. Its confirmed +negative result remains authoritative unless a materially different design is +given a new identity and hypothesis. + +## Workstream 2: compact SP scheduler tournament + +SP must tournament algorithm, scheduler, and execution boundary. Current +production winners remain controls: + +- `SP-S3-U-D` for qualified outbound distance and shallow physical-inbound + distance; +- `SP-S4-C-D` for qualified deep physical-inbound distance; +- `SP-S4-C-WE+MAT-M0` for qualified one-path witnesses; +- `SP-S0` as the exact broad-envelope incumbent. + +The specialized SP envelope requires an explicit bounded maximum depth at most +64. The current ASP envelope differs: an omitted maximum is admitted as depth +15, while minimum depth must be one for `ASP-A1-DAG`. Preserve those distinctions +in candidate eligibility, comparator choice, and serialized decisions. + +Reserve stable candidate identities before capture: + +| Candidate | Scheduler ID | Observation | Reference arm | +| --- | --- | --- | --- | +| `SP-B1-C-ALT-NODE-D` | `strict_alternating_node` | distance | `sp_b1_strict_alternating_distance` | +| `SP-B1-C-ALT-NODE-WE+MAT-M0` | `strict_alternating_node` | one witness | `sp_b1_strict_alternating_witness_m0` | +| `SP-B2-C-MIN-LEVEL-D` | `smaller_current_level` | distance | `sp_b2_smaller_frontier_distance` | +| `SP-B2-C-MIN-LEVEL-WE+MAT-M0` | `smaller_current_level` | one witness | `sp_b2_smaller_frontier_witness_m0` | + +Add a typed scheduler field to `ShortestPathExecutorDecision`; scheduler +behavior must not be inferred from a display name. Freeze +`single_ended_level` for S3/S4/A1 as well as the two candidate scheduler values +before the first artifact. + +### 2.1 Shared compact kernel + +Prototype a typed, graph-scoped bound-pair kernel with distinct forward and +backward structures: + +- node/depth frontier and next-front state; +- minimum-depth seen state per side; +- one deterministic predecessor/successor per accepted node for SP witness; +- per-node FIFO queue state for strict alternation; +- invocation telemetry and independently versioned limits. + +Keep relationship and node IDs only until one late hydration boundary. Preserve +logical source-to-target relationship order even when physical search begins at +the target. Outbound logical search uses `start_id -> end_id` forward and +`end_id -> start_id` backward; inbound search reverses those accesses. + +The legacy `bidirectional_sp_harness` already contains smaller-frontier control +logic, but it retains full path arrays, executes generated SQL text, and uses +generic pathspace tables. Reuse its control-flow lessons only. Do not promote or +rename it as a compact candidate. + +Strict alternation must dequeue one accepted node from each side in turn; +alternating whole SQL levels is a different scheduler. Smaller-frontier must +expand a complete level and use a deterministic tie break. Both schedulers need +a documented lower-bound termination proof: do not stop merely at the first +intersection, and complete enough depth on both sides to prove that no shorter +path remains. + +Retain exact zero-, one-, and two-hop arms before workspace allocation. Their +latency is a setup control, not evidence that distinguishes recursive +schedulers. + +### 2.2 Architecture boundary tournament + +The discovery references show that inline recursive SQL can be much faster than +the current session-workspace functions. Therefore: + +- retain exact inline S3/S4/ASP full comparators; +- implement compact bidirectional references with explicit internal counters; +- compare a typed function/workspace boundary to the smallest viable inline or + SQL-visible boundary where the scheduler permits it; +- attribute search, workspace reset, predecessor reconstruction, and hydration + separately. + +Do not select a scheduler based on a comparison that also changes hydration or +public observation. Each pair must share the same output boundary. + +### 2.3 Gates and fallback + +SP admission gates are separate counters: + +- total distinct seen nodes across both sides; +- current/next frontier or queue rows; +- retained witness-predecessor rows; +- optionally bounded meeting candidates. + +No recursive result is emitted until all gates pass. Overflow invokes +the production incumbent for the candidate's bucket in the same top-level +statement: S3 for S3 distance buckets, and S4 for deep-inbound distance or +witness buckets. Alternatively, restrict the first B1/B2 production activation +to S4 buckets. Candidate workspace names must be distinct from the current +`spd_*` workspace so nested fallback cannot corrupt state. Record the complete +fallback chain when S4 invokes its relationship-trail fallback, and establish +the function snapshot contract described above before calling the chain +snapshot-stable. + +After confirmation, a new `sp-static-v5` may select candidates only for the +topology and observation buckets that pass. A global scheduler winner is not +required: S3 or S4 may remain best for shallow or selective shapes. + +Before shadow or production use, define a versioned mapping from facts available +to the real query—query shape, observation, physical direction, depth, bounded +endpoint/degree probes, or executor frontier state—to each selectable topology +bucket. Fixture metadata and post-run telemetry label evaluation strata; they +cannot drive production selection. If a bucket cannot be recognized from +runtime inputs, it remains a diagnostic classification. + +## Workstream 3: bidirectional ASP predecessor DAG + +ASP begins only after the shared bidirectional search kernel, termination proof, +and SP telemetry pass qualification. + +Reserve: + +| Candidate | Scheduler ID | Reference arm | +| --- | --- | --- | +| `ASP-B1-DAG-ALT-NODE` | `strict_alternating_node` | `asp_b1_bidirectional_dag_strict_m0` | +| `ASP-B2-DAG-MIN-LEVEL` | `smaller_current_level` | `asp_b2_bidirectional_dag_smaller_frontier_m0` | + +The current `ASP-A1-DAG` remains the single-ended exact production control. +The legacy `bidirectional_asp_harness` carries complete trails and is not the +new candidate. + +### 3.1 State and reconstruction + +Each side retains: + +- minimum reached depth per node; +- every relationship-distinct predecessor or successor that reaches that node + at the same minimum depth; +- frontier state and scheduler order independently from predecessor state. + +When minimum distance `L` is proven, select one deterministic completed meeting +cut `k`. Enumerate source predecessor paths to nodes at depth `k`, target +successor paths from the same nodes at depth `L-k`, and stitch ordered edge ID +arrays. Using one cut ensures that a complete path is not emitted once per +overlap level. For the initial singleton pair, uniquely stage ordered +`edge_ids` and assert relationship uniqueness before public output. Endpoint +broadening must key uniqueness by input-pair identity plus `edge_ids`, then +reapply duplicate input-pair multiplicity; otherwise repeated endpoint rows +would be collapsed. + +Within the initial distinct-endpoint, minimum-depth-one envelope, an unweighted +minimum path cannot repeat a node because removing the intervening cycle would +make it shorter. This justifies minimum-node-depth discovery for this envelope +only. It does not justify directionless traversal, positive-minimum self cycles, +whole-path predicates, or broader trail semantics. + +### 3.2 Independent resource gates + +ASP has three different explosion modes and therefore three limits: + +1. Discovery: distinct seen/frontier nodes. +2. Predecessors: same-minimum-depth relationship-distinct predecessor rows. +3. Enumeration: distinct ordered edge arrays and materialized bytes. + +Before enumeration, calculate a saturating path-count bound over the predecessor +DAG. Stage output under `limit + 1` sentinels. Any overflow clears candidate +state and invokes `all_shortest_paths_dag` before exposing a row. This fallback +uses the same top-level statement, but still requires the deliberate function +snapshot contract before it can be described as one-snapshot execution. + +These are candidate-admission guards, not public result limits. ASP may never +silently truncate a required path set. If the exact incumbent itself cannot +complete within an external statement/resource policy, propagate that error; +do not relabel truncation as fallback success. + +After independent confirmation, `asp-static-v2` may select a qualified +bidirectional arm. If enumeration dominates total latency or no candidate +contains predecessor/output risk, retain A1 and record the new arm as a frozen +negative result. + +## Workstream 4: endpoints and predicate classes + +The first SP/ASP candidates retain the current one-literal-ID-per-endpoint +envelope. Broaden only after their core algorithms are stable. + +### 4.1 Bounded endpoint resolution + +Materialize endpoint resolution once with explicit 1/2/32/33 sentinels and exact +fallback. Qualify independently: + +- ID equality; +- unique indexed property equality; +- nonunique property equality that returns a small bounded set; +- explicitly supplied small endpoint sets; +- endpoint pairs whose correlation must be preserved rather than treated as a + Cartesian product. + +Record input rows, distinct endpoint IDs, duplicate multiplicity, pair count, +resolution plan/index, and overflow. Endpoint cardinality is runtime evidence; +predicate syntax alone is not selectivity proof. + +Keep the compact bidirectional ASP kernel singleton-only until a wrapper assigns +stable input-pair identities, deduplicates paths within each pair, and reapplies +duplicate pair-row multiplicity. Endpoint broadening must not make global +`edge_ids` uniqueness collapse the Cypher result bag. + +### 4.2 Predicate classification + +Add an explicit classifier for: + +- step-local node predicates; +- step-local relationship predicates; +- universal `ALL`/`NONE` predicates over path nodes or relationships that can + be evaluated on each expansion step; +- whole-path predicates requiring a complete materialized candidate. + +Only step-local or proven universal predicates may enter the compact expander. +Whole-path predicates retain an exact fallback-capable exhaustive plan. Each +predicate class needs mutation and translation fixtures because placement can +change evaluation and output semantics. + +## Workstream 5: fixed one-hop `ExpandInto` + +This work applies only when both endpoints of a fixed, one-hop relationship are +bound. It must not be generalized to variable-length `Into`. + +Start with a three-way plan study: + +1. Current bound-endpoint edge join, recording the plan PostgreSQL actually + chooses (for example, parameterized index lookup, hash join, or another + shape). +2. Typed lower-degree endpoint probe followed by adjacency scan and opposite + endpoint check. +3. The bound-pair join plus PostgreSQL `Memoize` or an explicit + statement-local distinct-pair cache for repeated input pairs. + +Measure wildcard and multi-kind cases separately. An actual parameterized pair +index plan may make lower-degree probing redundant for singleton typed pairs, +while pair reuse may matter only with duplicate outer rows. Add policy metadata +to the currently marker-only `ExpandIntoDecision` only if a candidate +demonstrates a real crossover. + +Pair caching stores or reproduces all matching relationship rows, not only a +connectivity boolean. It must preserve relationship IDs/properties, one-per-kind +multiplicity, wildcard/multi-kind and directionless behavior, self-loops, and +duplicate outer-row multiplicity even when it deduplicates lookup work. Qualify +cache hit/miss, missing endpoints, cross-kind parallel relationships, +cancellation, and generic/custom plans. + +## Workstream 6: statistics and probe roadmap + +Runtime capped probes are the first authority because they use the current +parameters and graph contents in the executing statement. Function-backed +search and fallback remain subject to the explicit snapshot contract above. +The useful evidence is: + +| Evidence | Primary use | +| --- | --- | +| Root/terminal endpoint rows and distinct IDs | Bound pair count and seed cost | +| Typed directional degree at each endpoint | First-step orientation and frontier risk | +| Suffix rows, distinct boundaries, and path multiplicity | Reverse seed and reconstruction cost | +| One-level survival and distinct-next ratio | Predicate selectivity and reconvergence hint | +| Per-level frontier and candidate edges | Adaptive SP/ASP scheduler choice | +| Seen-to-frontier and candidate-to-new-node ratios | Cycle/reconvergence cost | +| Same-depth predecessor additions | ASP predecessor memory risk | +| Meeting-node count and cut width | Bidirectional reconstruction cost | +| Saturating returned-path count and edge cells | ASP output/hydration risk | + +An optional synopsis is a later optimization, never a correctness proof. A +versioned synopsis may contain: + +- node counts by graph and kind; +- relationship counts by graph, direction, kind, and endpoint kind; +- distinct start/end counts and most-common endpoints; +- directional degree quantiles and heavy hitters; +- observed frontier survival/reconvergence buckets by depth; +- predecessor and output multiplicity buckets for qualified generated shapes. + +Node multi-kind membership makes endpoint-kind estimates overlapping rather +than additive. Sampling, refresh cadence, mutation overhead, stale-data +behavior, and graph drop/reload handling require an explicit design record. The +runtime guard remains authoritative. Prefer reading a synopsis at execution +time; embedding it in translated SQL requires a synopsis epoch in +`cypherTranslationCacheKey` and mutation-safe invalidation. + +## Qualification corpus + +Preserve the scale corpus's `normal`, `envelope`, and `stress` tiers. Gate normal +and envelope; use stress for exact fallback and failure-mode diagnosis. Expand +the existing deterministic generators before adding a new generator family. + +| Area | Required axes | +| --- | --- | +| Orientation | root and terminal seeds `0/1/2/32/33/128/512/513`; independent forward/reverse typed degree `0/1/4/32/128/1000/16000`; productive fraction `0/sparse/half/all`; mirrored fan-out/fan-in; hidden spike at first/middle/final depth | +| Common traversal | depth `0/1/2/4/8/16/32/64`; outbound/inbound/directionless; one/multiple kinds; fixed prefix/suffix `0/1/3`; disconnected decoys; cycles; self-loops; convergence; payload | +| SP | direct and two-hop controls; highly asymmetric endpoints; alternating-frontier crossovers; shallow target plus huge continuation; disconnected exhaustion; intermediate skew; one/equal witnesses; distance and path observations | +| ASP | depths `3/8/16`; diamond width and path count `1/2/16/128+`; same node count with different predecessor density; multiple meeting nodes; merge-then-split DAG; modest state with explosive output; large predecessor state with modest output | +| `ExpandInto` | asymmetric degrees; typed/wildcard/multi-kind; missing endpoints; self-loop; repeated pair hit/miss; duplicate outer rows | +| Endpoints/predicates | ID, unique property, nonunique property, small sets; local node/edge universal and whole-path predicates | +| Limits | every probe/state/predecessor/output cap at `N-1/N/N+1`, including current `32/33` and `4096/4097` boundaries | +| Output | scalar/count, endpoint IDs, ordered witness, full path/hydration, `LIMIT` absent/one/small | + +Freeze a topology holdout before selector thresholds are tuned. Include textually +permuted multi-`MATCH` and multi-pattern forms to compare Neo4j reorder +invariance with CySQL clause ordering. Record unsupported same-kind parallel +edges as a storage boundary while covering cross-kind multiplicity. + +## Tests required for every behavior change + +### Unit, translation, and mutation coverage + +- Optimizer table tests for candidate lists, exact eligibility facts, physical + direction, policy/scheduler versions, caps, and stable fallback reasons. +- SQL-shape tests for materialized probes, explicit `LIMIT cap+1`, disjoint + branch dependencies, ID-only state, edge-index orientation, and late + hydration. +- Fail-closed forcing tests for wrong observation, predicates, mutation, + correlation, optional match, directionless traversal, multiple calls, and + unsupported depth. +- Reverse path-order, relationship-overlap, suffix bag multiplicity, duplicate + roots, parameter rebinding, and generic/custom-plan tests. +- Source translation-case updates plus generated artifacts and mutation tests + for parsing, lowering, rendering, and predicate placement changes. + +### Semantic integration + +- Shared backend-equivalent cases validate logical stable observations; no + driver-specific expected values or skips belong in the shared corpus. +- PostgreSQL-scoped tests validate candidate branch loops, exact fallback, + workspace state, edge indexes, caps, buffers, and function invocation. +- Cover missing/null/equal endpoints, zero depth, maximum-depth miss, both + directions, cycles, repeated nodes without repeated relationships, suffix + multiplicity, empty/disconnected sides, and every accepted/rejected predicate + class. +- For singleton SP ties, compare distance and validate that each returned trail + is minimum and relationship-unique; use unique-witness cases when an exact + ordered-ID reference comparator is required. +- For ASP, compare the full stable path multiset and predecessor/output cap + boundaries, not only row count. + +### Operational integration + +- Pool sizes `1/2/8` and concurrency `1/8/16`. +- Prompt cancellation followed by successful rollback and reuse of the same + PostgreSQL backend PID. +- A concurrent-writer semantic test proving the selected snapshot mechanism or + rejecting function-backed fallback under the default isolation behavior. +- Low `work_mem`, forced generic plan, forced custom plan, and normal `auto` + plan modes. +- No cross-invocation workspace or telemetry contamination. +- Schema-up/schema-down symmetry and upgrade coverage for every new helper or + temporary workspace. + +## Performance and resource gates + +Use the existing balanced GraphBench protocols: + +- Discovery: at least five independently reloaded rounds, five warmups, and ten + samples per arm. +- Confirmation: 10-20 independently reloaded rounds, at least 20 warmups and + 50 samples per arm, seeded 97.5% intervals, and balanced arm order. +- Before accepting three-arm SP or ASP evidence, add and freeze a balanced + three-arm Latin/Williams schedule. The current non-five-arm forward/reverse + ordering leaves the middle arm in the middle and is not carryover-balanced. +- A/A calibration: derive per-host p50/p95 absolute and ratio resolution before + applying materiality. +- Complete declarations only: filtered or adaptive artifacts are diagnostic and + cannot pass a release gate. + +Initial promotion thresholds are policy inputs and must be versioned: + +- target p50 candidate/incumbent ratio upper bound at most `0.95`, or absolute + saving lower bound at least `100us`; +- no p95 regression beyond the greater of host A/A noise and 5%, using the + greater of A/A absolute noise and `100us` for very fast cases; +- no confirmed normal/envelope regression outside that same noise band; +- selector regret versus the fastest exact arm: ratio upper bound at most + `1.10` or within the A/A absolute floor; +- probe overhead versus the forced selected arm: at most 10% or `100us`; +- production/reference closure: retain the existing `1.10` ratio/A/A floor; +- Neo4j latency and PROFILE remain descriptive. + +Extend the resource gate to enforce numeric envelopes, not only spill classes: + +- probe rows at or below `cap + 1`; +- frontier, queue, seen, predecessor, output, and bytes at declared ceilings; +- no executor temp-file read/write or WAL for non-mutating candidates; +- local workspace only for explicitly workspace-qualified architectures; +- measured per-session and pool memory ceilings; +- no unexpected fallback in admitted normal/envelope buckets; +- exactly attributed fallback in stress buckets. + +The identities must form a valid chain: the translation-applied policy matches +the planned candidate set, the runtime arm belongs to that emitted policy, and +any runtime fallback matches the declared incumbent chain. Probes execute at +most once, unselected arms show zero work, and fallback executes once before any +output. Any missing or contradictory attribution fails the gate. + +## Milestones and exit criteria + +| Milestone | Deliverables | Exit criterion | +| --- | --- | --- | +| M0: freeze baseline | Clean-source capture bundle; PostgreSQL/Neo4j environment fingerprints; current plans; A/A calibration; stable candidate IDs; topology holdout split | Checksummed artifacts reproduce exact observations and the discovery findings without credentials. | +| M1: observability | `TraversalExecutionTelemetry`; PostgreSQL diagnostic counters; Neo4j read `PROFILE`; paired PlanCorpus deltas; numeric resource schema | No result change; measured telemetry overhead is within A/A noise or disabled outside diagnostic replay; missing counters fail qualification. | +| M2: orientation framework | Common candidate analyzer; bounded seed/degree probes; endpoint-reverse migration; guarded suffix reverse; forced and shadow modes | Exact parity across semantic/cap cases; disjoint branches; selector-regret and probe-overhead reports exist. Production still uses the incumbent except the already qualified endpoint family. | +| M3: SP references | Strict-alternating and smaller-level compact reference arms; typed scheduler metadata; balanced three-arm schedule; formal termination invariant; inline/function boundary comparison | Exact distance/witness results, bounded state, cancellation/reuse, and discovery report across asymmetric topology buckets. | +| M4: SP production qualification | Incumbent-specific same-statement fallback; snapshot contract; complete confirmation/holdout/resource/reference-closure reports; `sp-static-v5` policy | Only runtime-recognizable, passing topology/observation buckets select a new arm; all other shapes preserve S3/S4/S0 with precise reasons. | +| M5: ASP references and qualification | Two-sided predecessor state; canonical meeting cut; three independent gates; full multiset comparator; ASP stress corpus | Exact ASP output, no truncation, bounded candidate state, confirmation and holdout pass; otherwise freeze a negative result and retain A1. | +| M6: envelope broadening | Bounded property/small-set endpoints; step-local/universal predicates; fixed one-hop `ExpandInto` study and any qualified policy | Each class has its own eligibility, exact fallback, corpus, and decision record. No broadening by tool forcing. | +| M7: optional synopsis | Synopsis ADR, schema/refresh/cache design, shadow comparison against runtime probes | Implement only if it materially reduces probe/selector regret and its mutation/cache cost passes independent gates. | + +M2 and M3 may proceed in parallel after M1. M5 begins after the shared SP +kernel and telemetry stabilize. M6's `ExpandInto` plan study may run earlier, +but automatic behavior still requires its own evidence. + +## Repository implementation map + +| Concern | Primary files | +| --- | --- | +| Typed decisions and selectors | `cypher/models/pgsql/optimize/lowering.go`, `lowering_plan.go`, `optimizer_test.go` | +| Ordinary orientation emission | `cypher/models/pgsql/translate/expansion_orientation.go` (new), `expansion_endpoint_seeded.go`, `expansion_suffix_seeded.go`, `pattern.go`, `traversal.go`, `translator.go` | +| SP/ASP builders and dispatch | `cypher/models/pgsql/translate/expansion.go`, `pattern.go`, `optimizer_safety_test.go`, `cypher/models/pgsql/functions.go` | +| Compact workspaces/functions | `drivers/pg/query/sql/schema_up.sql`, `schema_down.sql`, `drivers/pg/query/sql_workspace_test.go`, schema-upgrade integration tests | +| Translation cache contract | `drivers/pg/translation_cache.go` and tests; change if mutable rollout policy is translated rather than supplied at execution, or if a synopsis is embedded | +| GraphBench telemetry/references | `cmd/graphbench/results.go`, `postgres_plan.go`, `neo4j.go`, `references.go`, `datasets.go`, `main.go` and tests | +| Gates and reports | `cmd/graphbench/resource_gate.go`, `perf_gate.go`, reference-pair/closure reports, backend-delta report | +| Matched plan deltas | `cmd/plancorpus/types.go`, `report.go`, capture/report tests | +| Deterministic topology generators | `testutil/perf_shortest_v2.go`, `perf_endpoint_seeded.go`, `perf_fixtures.go` | +| Scale declarations | `benchmark/testdata/scale/cases/generated_shortest_paths_v2.json`, `generated_endpoint_seeded_expansion_v1.json`, `generated_fixed_suffix_expansion.json` | +| Semantic fixtures | `integration/testdata/cases`, `integration/testdata/templates`, PostgreSQL-scoped plan-invariant tests | +| Documentation and evidence | this plan, `recursive_descent_cost_controls.md`, `postgresql_translation.md`, GraphBench/scale READMEs, and versioned `docs/experiments` records | + +Changes should be sliced so telemetry, candidate implementation, selector +activation, and envelope broadening are separately reviewable. Do not combine a +new algorithm, new semantic support, and automatic selection in one change. + +## Rollout and rollback + +Every candidate follows the same stages: + +1. Telemetry only; no selection change. +2. Exact benchmark reference arm with a frozen implementation ID. +3. Tool-forced production emitter, failing closed outside its envelope. +4. Shadow selection that records `would_select` while executing the incumbent; + matched diagnostic arms calculate regret. +5. Explicit opt-in with same-statement exact fallback and an established + snapshot contract for function-backed arms. +6. Narrow automatic selection for named, passing topology buckets. +7. One-bucket-at-a-time expansion after new holdout confirmation. + +Keep the incumbent selector and previous function/schema identity available for +at least one release after automatic activation. A feature gate must be able to +return all traffic to the incumbent without a data migration, and changing it +must invalidate cached translated SQL or be an execution-time policy input. + +Immediately disable automatic selection on: + +- any correctness or ASP multiplicity mismatch; +- planned/emitted/runtime attribution disagreement; +- cap breach, partial candidate output, unexpected spill, or read-query WAL; +- cancellation poisoning or workspace/telemetry cross-talk; +- unstable SQL/plan fingerprint outside a declared change; +- abnormal fallback frequency in a qualified bucket; +- a confirmed p95 regression outside the A/A/materiality envelope. + +Do not retune a failed identity post hoc. Preserve the failed arm and compact +evidence in `docs/experiments`, assign a new ID to a materially changed design, +and reopen discovery with a new hypothesis. + +## Risk register + +| Risk | Mitigation | +| --- | --- | +| Probe overhead erases the orientation win | Cap every probe, materialize once, measure probe-only cost, use hysteresis, and keep forward on ambiguous small gains. | +| Reverse admission plus fallback doubles expensive work | Gate before output, measure fallback regret explicitly, lower admission caps, and qualify overflow buckets separately. | +| Mutable topology or rollout policy invalidates cached SQL | Keep topology values inside same-statement probes; make mutable policy an execution input or cache generation; require a synopsis epoch before embedding statistics. | +| Bidirectional search stops at a nonminimal first meeting | Require a documented lower-bound termination proof and adversarial asymmetric/reconvergent tests. | +| ASP predecessor or output explosion is hidden by node-state counts | Enforce separate discovery, predecessor, path-count, output-row, and byte gates. | +| Session workspaces consume excessive pool memory or collide | Use invocation/session isolation, explicit per-session/pool ceilings, concurrency tests, and prompt cleanup on error/cancel. | +| Detailed telemetry changes the measured algorithm | Keep detailed counters in untimed diagnostic replay; separately measure lightweight summary overhead. | +| Fixed `ExpandInto` copies a Neo4j optimization that PostgreSQL does not need | Compare direct pair index lookup, lower-degree scan, and `Memoize`/pair cache before implementation. | +| Predicate pushdown changes evaluation semantics | Classify locality/universality, retain exact fallback, and require mutation plus cross-backend semantic fixtures. | +| Aggregate benchmark wins hide topology regressions | Gate by predeclared buckets, worst-case containment, and a frozen holdout rather than aggregate median alone. | +| Neo4j version differences corrupt interpretation | Pin source commits and server version in every artifact; keep 4.4 strict alternation and current smaller-level scheduling as separate arms. | + +## Validation and evidence workflow + +After code changes, run formatting and unit validation: + +```bash +make format +make test +make lint +``` + +Run backend-specific full validation separately, using only disposable targets +and the repository's destructive-integration guards: + +```bash +DAWGS_INTEGRATION_ALLOW_DESTRUCTIVE=1 \ +DAWGS_INTEGRATION_DISPOSABLE_TARGETS="$PG_DISPOSABLE_TARGET" \ + CONNECTION_STRING="$PG_CONNECTION_STRING" make test_all + +DAWGS_INTEGRATION_ALLOW_DESTRUCTIVE=1 \ +DAWGS_INTEGRATION_DISPOSABLE_TARGETS="$NEO4J_DISPOSABLE_TARGET" \ + CONNECTION_STRING="$NEO4J_CONNECTION_STRING" make test_all +``` + +Then run both-backend PlanCorpus and the staged GraphBench workflow: + +1. plan corpus and matched delta capture; +2. discovery plus exact reference comparisons; +3. A/A calibration; +4. 10-20-round confirmation; +5. numeric resource gate; +6. production/reference closure; +7. concurrency, cancellation, and session-reuse cases; +8. topology holdout; +9. descriptive backend delta; +10. complete performance gate and capture bundle checksum. + +Never place connection strings, endpoint IDs from sensitive graphs, query +parameters, or credentials in durable artifacts. Existing-graph confirmation +uses the current redacted anchor-manifest workflow and cannot substitute for the +deterministic correctness corpus. + +For every accepted or rejected candidate, add +`docs/experiments/_vN.md` containing: + +- immutable implementation and selector IDs; +- source and artifact SHA-256 values; +- backend versions and relevant settings; +- corpus declaration and holdout identity; +- rounds, warmups, samples, order balancing, and confidence policy; +- correctness, performance, resource, fallback, concurrency, and cancellation + results; +- the promotion/rejection decision and unchanged incumbent behavior. + +Raw captures remain under `.coverage`; compact canonical reports may be +committed when they contain no secrets or unstable physical identifiers. + +## Definition of done + +This priority plan is complete when: + +- traversal decisions and runtime execution are separately observable and + matched across plan records; +- Neo4j read plans include actual evidence with SP/ASP opacity represented + honestly; +- ordinary orientation, SP, and ASP each have exact incumbent and candidate + arms with stable identities; +- every candidate has bounded probes/state, disjoint output/fallback behavior, + and precise machine-readable fallback reasons; +- semantic, cap-boundary, operational, resource, performance, and holdout gates + run reproducibly; +- production selectors enable only independently passing topology/observation + buckets and remain quickly reversible; +- nonwinning candidates are retired with durable negative evidence rather than + left as ambiguous code paths; +- documentation describes current production behavior separately from future + candidates and their qualification status. + +Success may legitimately conclude that S3/S4/A1 or direct PostgreSQL pair +lookup remains best for some or all buckets. The required outcome is a measured, +exact, explainable selector program—not a predetermined Neo4j-shaped executor. diff --git a/docs/development.md b/docs/development.md index b39bcfe7..15155238 100644 --- a/docs/development.md +++ b/docs/development.md @@ -21,6 +21,8 @@ Run the integration suite when a backend is available: ```bash export CONNECTION_STRING="postgresql://dawgs:weneedbetterpasswords@localhost:65432/dawgs" +export DAWGS_INTEGRATION_ALLOW_DESTRUCTIVE=1 +export DAWGS_INTEGRATION_DISPOSABLE_TARGETS="postgresql://localhost:65432/dawgs" make test_integration ``` @@ -34,6 +36,15 @@ export CONNECTION_STRING="postgresql://dawgs:weneedbetterpasswords@localhost:654 export CONNECTION_STRING="neo4j://neo4j:weneedbetterpasswords@localhost:7687" ``` +`DAWGS_INTEGRATION_DISPOSABLE_TARGETS` is a comma-separated list of exact, +credential-free targets in `:///` form. Use +`/` when the connection URL selects the driver's default database. +`make test_all`, `make test_pg`, `make test_neo4j`, and fixture-loading +GraphBench runs refuse targets absent from the list. GraphBench +`-existing-graph` mode remains exempt because it rejects writes and validates +before/after cardinalities. Its PostgreSQL sessions remain read-write so +temporary traversal workspaces retain production behavior. + Use backend-specific targets when needed: ```bash @@ -59,7 +70,17 @@ Run: make format ``` -The target uses `goimports`; install it locally if it is missing from your environment. +The target uses `goimports`; install it locally if it is missing from your +environment. Sandboxed or nonstandard installations can supply its explicit +path without changing `PATH`: + +```bash +make format GOIMPORTS_CMD=/absolute/path/to/goimports +``` + +`make lint` runs the standard Go vet analyzers across the repository. The unreachable-code analyzer is rerun only for +handwritten packages because ANTLR emits intentional terminal branches in `cypher/parser`; generated parser code still +receives every other vet analyzer. ## Quality And Metrics @@ -108,7 +129,8 @@ The defaults can be adjusted with `CYCLO_TOP`, `CYCLO_OVER`, `CRAP_TOP`, `CRAP_O `make plan_corpus` captures plan diagnostics for the shared Cypher integration corpus. It accepts either `CONNECTION_STRING` for one backend or `PG_CONNECTION_STRING` and `NEO4J_CONNECTION_STRING` for both backends, then -writes JSONL captures and markdown/JSON summaries under `.coverage/`. +writes JSONL captures and markdown/JSON summaries under `.coverage/`. Fixture loading requires the same destructive +acknowledgement and exact credential-free allowlist entries as integration testing. Run it when changing PostgreSQL Cypher planning, lowering, or SQL emission. The summaries rank expensive PostgreSQL plans and report recursive CTEs, `SubPlan`, `Function Scan on unnest`, planned/applied optimizer lowerings, and @@ -120,13 +142,42 @@ See [Plan Corpus Capture](../cmd/plancorpus/README.md) for flags and review guid `go run ./cmd/graphbench` captures runtime diagnostics for the scale corpus under `benchmark/testdata/scale`. -Current modes are: +Implemented modes are: - `postgres_sql` -- `local_traversal` - `neo4j` +`local_traversal` emits non-gating `not_implemented` diagnostics only; it is not an implemented executor. + AGE is reference-design input only and is not a direct comparison mode. The command can emit JSONL records plus Markdown and JSON summaries, and can compare current timings against a previous JSONL baseline. +The PostgreSQL scale-plan correctness gate shares the scale runner. It checks the +required stable query-form IDs, declared read/write cardinalities, rollback-safe +mutation post-state, `EXPLAIN ANALYZE` capture, and stable plan invariants. It +runs under `make test_all` for PostgreSQL or can be selected directly: + +```bash +DAWGS_INTEGRATION_ALLOW_DESTRUCTIVE=1 \ +DAWGS_INTEGRATION_DISPOSABLE_TARGETS="postgresql://localhost:65432/dawgs" \ + CONNECTION_STRING="$PG_CONNECTION_STRING" \ + go test -tags manual_integration ./cmd/graphbench \ + -run 'Test(PostgreSQLScalePlanInvariants|ScaleCorpusRequiredRepresentativesDeclareCardinality)' \ + -count=1 +``` + +Store graphbench and plan-corpus captures under `.coverage/`; they are +environment-specific review artifacts, not committed correctness goldens. + See [Graph Benchmark Capture](../cmd/graphbench/README.md) for command examples. + +## BloodHound Source-Parity Audits + +When the reviewed BHE or BHCE snapshots change, repeat the call-site inventory, +active-entry-point trace, normalized query-form mapping, and commit recording in +[BloodHound Regression Source Parity](regression_source_parity.md). + +Dormant `FUTURE-*` forms stay manifest-only until a reviewed caller is enabled. +The unit suites reject dormant IDs from both shared plan inputs and scale cases; +activating a form requires updating those gates together with its required +semantic, plan, and scale coverage. diff --git a/docs/experiments/asp_i1_inline_v1.md b/docs/experiments/asp_i1_inline_v1.md new file mode 100644 index 00000000..057e3aed --- /dev/null +++ b/docs/experiments/asp_i1_inline_v1.md @@ -0,0 +1,75 @@ +# Inline all-shortest-path predecessor DAG v1 + +Date: 2026-08-12 + +Status: implemented as a default-off production canary; automatic selection +withheld pending clean qualification evidence + +`ASP-I1-U-DAG+MAT-M0` is the typed, inline PostgreSQL comparator for qualified +`allShortestPaths` queries. It is intentionally distinct from the stored +helper implementation `ASP-A1-DAG` so benchmark arms and production receipts +identify the executable code path rather than only the algorithm family. + +## Correctness and resource boundary + +The emitter accepts one read-only, non-optional, directed endpoint pair with +static singleton endpoint IDs, minimum depth one, and a bounded maximum depth +from 1 through 64. It discovers minimum node distances, retains every +relationship-distinct predecessor at that minimum layer, and enumerates the +predecessor DAG into ordered relationship-ID arrays. Existing outer +translation performs path hydration. + +The production emitter resolves exact one- and two-hop targets first. These +bounded preflight rows participate in the enumeration cap+1 gate, and recursive +distance discovery runs only when no early target exists. + +Every recursive producer is consumed through a materialized cap+1 relation. +Separate immutable limits cover discovered states, predecessor rows, all +intermediate enumeration states, and serialized output bytes. The guarded +decision is complete before either public-output arm opens. A cap overflow +selects exact `ASP-A1-DAG` in the same statement and stable snapshot; candidate +rows cannot mix with fallback rows. + +Materialized candidate and fallback markers provide singular plan evidence. +The runtime attestation receipt schema v2 records an ordered event chain. A +non-nested I1 execution records one of: + +- `inline_predecessor_dag` with runtime identity `ASP-I1-U-DAG+MAT-M0`; +- `inline_no_path` with runtime identity `ASP-I1-U-DAG+MAT-M0`; +- `exact_a1_fallback` with runtime identity `ASP-A1-DAG`. + +GraphBench replays distance, predecessor, enumeration, output, marker, and +branch-row counters. Qualification fails when attribution is absent, +contradictory, over cap, or shows rows from the inactive output arm. + +## Production policy + +The driver can select I1 only under Repeatable Read or Serializable isolation. +The verified schema-v2 promotion manifest must name the candidate and exact A1 fallback, +use `guarded_dual_arm`, declare all four positive caps, and authorize the exact +normalized-query SHA plus direction, all-path observation, depth, +relationship-kind count, and typed/untyped bucket. Every evidence report must +repeat that complete authorization identity. Query allowlisting and the +policy generation partition the translation cache. Read Committed, unmatched +queries, and the zero policy retain the incumbent. `DisableInlineASPDAG` +provides an evidence-free immediate rollback switch. + +Tool forcing remains available for controlled comparison but does not broaden +the structural envelope. B1/B2 shortest and ASP experiments remain tool-only; +the production allowlist is centralized on the implemented inline families. + +## Qualification sequence + +1. Capture balanced A/A and A1-versus-I1 runs from a clean source tree. +2. Require exact full path-multiset parity on training, frozen holdout, and + diagnostic cases, including inbound, disconnected, parallel-kind, + early-target, diamond, cycle, and self-loop topologies. +3. Pass confirmation materiality/p95, selector-regret, resource, + reference-closure, cancellation, concurrency, and session-isolation gates. +4. Generate a checksummed manifest for only the independently passing query + and topology buckets, then canary at stable isolation. +5. Expand allowlisted buckets only with new clean evidence. Keep A1 automatic + and retain the kill switch until post-canary production telemetry closes. + +No result from a dirty diagnostic tree is promotion evidence, and this +implementation does not change the automatic `asp-static-v1` selector. diff --git a/docs/experiments/fixed_suffix_cardinality_metadata_audit.md b/docs/experiments/fixed_suffix_cardinality_metadata_audit.md new file mode 100644 index 00000000..d19adb62 --- /dev/null +++ b/docs/experiments/fixed_suffix_cardinality_metadata_audit.md @@ -0,0 +1,40 @@ +# Fixed-suffix cardinality metadata audit + +Status: **no hard pre-translation bound is currently available**. + +This audit asks whether production translation can directly select +`EXPANSION-SUFFIX-SEEDED-REVERSE` only when it can prove both physical suffix +rows and reverse states are at most 512, without executing the retired runtime +probe/fallback design. + +## Existing inputs + +- The public translator receives the Cypher AST, kind mapper, parameters, and + graph ID. It has no database connection or graph-cardinality provider. +- Graph schema metadata describes names, kinds, indexes, and constraints. It + does not contain degree, suffix-row, path, or reverse-state bounds. +- The PostgreSQL `graph` catalog contains only graph ID and name. Partition + models contain table names, indexes, and constraints. +- `OptimizeStorage` reads approximate live/dead tuple counts for vacuum + decisions. These counts are database-storage statistics, not per-root or + per-kind hard bounds. +- PostgreSQL planner statistics and `pg_class.reltuples` are estimates. They + are neither correctness-grade upper bounds nor available to the optimizer + before SQL emission. +- Translation caching is keyed by query text, graph ID, and parameter types. + A selector dependent on parameter values or mutable graph cardinality would + require new invalidation and cache-identity rules. + +## Finding + +Suffix rows and reverse states depend on the selected root, relationship kinds, +query depth, physical trail multiplicity, and current graph contents. Global +node/edge counts or planner estimates cannot prove either 512 ceiling. No +existing schema constraint establishes these limits, and no maintained +per-graph or per-root synopsis supplies conservative upper bounds. + +Therefore the S511/S512 wins do not currently support automatic production +dispatch. Production must continue selecting `EXPANSION-STEPWISE-FORWARD` for +this family. A future attempt would require a new proof-bearing metadata/API +contract plus mutation-safe maintenance, cache invalidation, and independent +qualification; that work is outside this completed audit. diff --git a/docs/experiments/guarded_suffix_keyset_continuation_v1.md b/docs/experiments/guarded_suffix_keyset_continuation_v1.md new file mode 100644 index 00000000..f32a9cef --- /dev/null +++ b/docs/experiments/guarded_suffix_keyset_continuation_v1.md @@ -0,0 +1,41 @@ +# Guarded suffix keyset continuation v1 + +Status: **rejected and retired negative result**. The historical implementation +identity `t16_s512_r512_e1_e2_e3_boundary_keyset_v1` is frozen in these +artifacts. Its GraphBench reference arm and experiment-specific telemetry have +been removed, and it was never part of production translation. +This confirmation is the canonical upstream record; later local reruns are not +part of the submitted evidence set. + +The confirmation run used an isolated PostgreSQL 18.4 database with +`plan_cache_mode=auto`, 10 matched reload rounds, 20 warmups per round, and 50 +measurements per arm per round (500 samples per arm). Intervals are paired +97.5% confidence intervals. The source artifact SHA-256 is +`e6aa00733de4861b9684d8f1276e922ff1e8059671e57400703a8266ca88ee25`. + +| Case | Baseline p50 | Candidate p50 | Median ratio (97.5% CI) | Candidate shared hits | Interpretation | +| --- | ---: | ---: | ---: | ---: | --- | +| S511 | 11.254 ms | 4.758 ms | 0.416 [0.407, 0.439] | 6,866 | Existing bounded reverse branch wins | +| S512 | 11.165 ms | 4.748 ms | 0.428 [0.408, 0.453] | 6,879 | Existing bounded reverse branch wins | +| S513 | 11.247 ms | 20.065 ms | 1.791 [1.752, 1.875] | 54,020 | Continuation is 79% slower | +| S600 | 11.566 ms | 68.950 ms | 5.898 [5.649, 6.462] | 55,523 | Non-empty continuation is 490% slower | + +S511 and S512 do not validate keyset continuation: they select the previously +known bounded reverse branch. S513 and S600 are the cases that exercise the new +continuation path, and both regress decisively. The reconstruction after the +prefix/remainder probes accounts for roughly 45,093 shared-buffer hits in both +overflow cases, so tuning the keyset predicate alone is not a credible next +step. + +The experiment's unpublished resource gate v5 passed all 40 candidate records: +there was no temporary or +local workspace, WAL, sentinel-budget violation, or inactive-branch execution. +Correctness and structured-plan checks also passed under `auto`, +`force_custom_plan`, and `force_generic_plan`. This makes the rejection a +performance decision rather than a correctness or spill failure. + +The compact machine-readable evidence is preserved in +`guarded_suffix_keyset_continuation_v1_pair.json` and +`guarded_suffix_keyset_continuation_v1_resources.json`. The JSON retains the +historical case names for artifact comparability; the active corpus uses +generic `GFSE-BOUNDARY-*` names for these fixed-suffix expansion holdouts. diff --git a/docs/experiments/guarded_suffix_keyset_continuation_v1_pair.json b/docs/experiments/guarded_suffix_keyset_continuation_v1_pair.json new file mode 100644 index 00000000..9da571fa --- /dev/null +++ b/docs/experiments/guarded_suffix_keyset_continuation_v1_pair.json @@ -0,0 +1,58 @@ +{ + "version": 1, + "decision": "rejected", + "implementation_id": "t16_s512_r512_e1_e2_e3_boundary_keyset_v1", + "source_artifact_sha256": "e6aa00733de4861b9684d8f1276e922ff1e8059671e57400703a8266ca88ee25", + "environment": "isolated local PostgreSQL 18.4, plan_cache_mode=auto", + "protocol": { + "reload_rounds": 10, + "warmups_per_round": 20, + "samples_per_arm_per_round": 50, + "samples_per_arm": 500, + "confidence_level": 0.975 + }, + "baseline": "complete_reference", + "candidate": "guarded_suffix_keyset_continuation", + "cases": [ + { + "name": "GFSE-GUARDED-S511-admitted-suffix-limit-minus-one", + "baseline_p50_ns": 11254109, + "candidate_p50_ns": 4758027, + "baseline_p95_ns": 12290083, + "candidate_p95_ns": 5301188, + "median_ratio": {"estimate": 0.41555464313812607, "lower": 0.40705827827777386, "upper": 0.4385313114263852}, + "p95_ratio": {"estimate": 0.43133866549151867, "lower": 0.42443171773646776, "upper": 0.44452360755289955}, + "median_change_ns": {"estimate": -6587660, "lower": -6747944, "upper": -6252829} + }, + { + "name": "GFSE-GUARDED-S512-admitted-suffix-limit-exact", + "baseline_p50_ns": 11164527, + "candidate_p50_ns": 4747871, + "baseline_p95_ns": 12142325, + "candidate_p95_ns": 5565890, + "median_ratio": {"estimate": 0.427560127676012, "lower": 0.40752095743455247, "upper": 0.4528344352178298}, + "p95_ratio": {"estimate": 0.4583874999227907, "lower": 0.4396809723294706, "upper": 0.4702413504336159}, + "median_change_ns": {"estimate": -6335235, "lower": -6732953, "upper": -6004886} + }, + { + "name": "GFSE-GUARDED-S513-admitted-suffix-limit-plus-one", + "baseline_p50_ns": 11247438, + "candidate_p50_ns": 20064748, + "baseline_p95_ns": 12188292, + "candidate_p95_ns": 21819920, + "median_ratio": {"estimate": 1.7906082254074516, "lower": 1.7518498795013928, "upper": 1.8748309114021493}, + "p95_ratio": {"estimate": 1.7902360724537942, "lower": 1.7578466359917437, "upper": 1.827209494526714}, + "median_change_ns": {"estimate": 8847280, "lower": 8396170, "upper": 9581455} + }, + { + "name": "GFSE-KEYSET-S600-productive-nonempty-remainder", + "baseline_p50_ns": 11565607, + "candidate_p50_ns": 68949562, + "baseline_p95_ns": 12595535, + "candidate_p95_ns": 75773743, + "median_ratio": {"estimate": 5.898025791399814, "lower": 5.649302617882263, "upper": 6.462465544247631}, + "p95_ratio": {"estimate": 6.015920959292321, "lower": 5.936216979766291, "upper": 6.159786543310964}, + "median_change_ns": {"estimate": 56902893, "lower": 56055344, "upper": 60770683} + } + ] +} diff --git a/docs/experiments/guarded_suffix_keyset_continuation_v1_resources.json b/docs/experiments/guarded_suffix_keyset_continuation_v1_resources.json new file mode 100644 index 00000000..38e80112 --- /dev/null +++ b/docs/experiments/guarded_suffix_keyset_continuation_v1_resources.json @@ -0,0 +1,24 @@ +{ + "version": 5, + "decision": "rejected", + "implementation_id": "t16_s512_r512_e1_e2_e3_boundary_keyset_v1", + "source_artifact_sha256": "e6aa00733de4861b9684d8f1276e922ff1e8059671e57400703a8266ca88ee25", + "passed": true, + "evaluated_records": 120, + "candidate_records": 40, + "candidate_failures": 0, + "candidate_cases": 4, + "candidate_shared_hit_blocks": { + "GFSE-GUARDED-S511-admitted-suffix-limit-minus-one": 6866, + "GFSE-GUARDED-S512-admitted-suffix-limit-exact": 6879, + "GFSE-GUARDED-S513-admitted-suffix-limit-plus-one": 54020, + "GFSE-KEYSET-S600-productive-nonempty-remainder": 55523 + }, + "observed": { + "temporary_blocks": 0, + "local_blocks": 0, + "wal_records": 0, + "sentinel_budget_violations": 0, + "inactive_branch_executions": 0 + } +} diff --git a/docs/experiments/traversal_priority_implementation_status_v1.md b/docs/experiments/traversal_priority_implementation_status_v1.md new file mode 100644 index 00000000..35415a6e --- /dev/null +++ b/docs/experiments/traversal_priority_implementation_status_v1.md @@ -0,0 +1,77 @@ +# Traversal priority implementation status v1 + +Date: 2026-08-12 + +Status: canonical-I1 qualified; production promotion withheld pending rollout closure + +This record separates repository implementation from empirical promotion for +[`cysql_traversal_priorities.md`](../cysql_traversal_priorities.md). The +candidate algorithms, exact fallbacks, diagnostic surfaces, qualification +corpora, and fail-closed gates are repository code. This change does not claim +new latency results and does not fabricate a clean M0 capture from a modified +working tree. Consequently, no new automatic suffix, SP, ASP, endpoint, +predicate, or `ExpandInto` selector is enabled. + +## Immutable identities + +| Concern | Implemented identity | +| --- | --- | +| Ordinary orientation policy | `orientation-probe-v1` | +| Ordinary incumbent | `EXPANSION-STEPWISE-FORWARD` | +| Fixed-suffix candidate | `EXPANSION-SUFFIX-SEEDED-REVERSE` | +| Existing endpoint candidate | `EXPANSION-ENDPOINT-SEEDED-REVERSE` | +| SP strict node alternation | `SP-B1-C-ALT-NODE-D`, `SP-B1-C-ALT-NODE-WE+MAT-M0` | +| SP smaller current level | `SP-B2-C-MIN-LEVEL-D`, `SP-B2-C-MIN-LEVEL-WE+MAT-M0` | +| ASP strict node alternation | `ASP-B1-DAG-ALT-NODE` | +| ASP smaller current level | `ASP-B2-DAG-MIN-LEVEL` | +| SP/ASP production controls | `SP-S3-U-D`, `SP-S3-U-E+MAT-M0`, `SP-S4-C-D`, `SP-S4-C-WE+MAT-M0`, `ASP-A1-DAG`, `SP-S0` | +| Inline production canaries | `SP-I1-C-WE+MAT-M0`, `ASP-I1-U-DAG+MAT-M0` | +| Inline tool-only executors | `SP-I1-C-D`, `SP-I1-U-E+MAT-M0` | +| Bounded endpoint analysis | `endpoint-resolution-v1` | +| Traversal predicate analysis | `traversal-predicate-v1` | +| Fixed one-hop study | `expand-into-study-v1` | + +## Milestone disposition + +| Milestone | Repository implementation | Promotion disposition | +| --- | --- | --- | +| M0 | Capture bundle v3 binds source state, patch and untracked payloads, dependency files, executable, the complete sorted corpus declaration and identity, evidence checksums, and sanitized environment metadata. Its independent verifier reconstructs and validates the bundled source and corpus fingerprints. Host-bound A/A now requires two explicitly executed, order-balanced arms; frozen training/holdout declarations are enforced. | A fresh clean-source capture is still required. A dirty diagnostic bundle cannot qualify promotion. | +| M1 | Traversal telemetry v1 separates summary identity from untimed diagnostic replay and carries per-field provenance/completeness. PostgreSQL diagnostics fail closed for hidden function work. Neo4j reads use `PROFILE`, preserve ordered children and actual metrics, and explicitly mark opaque SP/ASP internals. Plan-delta v2 uses union pairing and semantic stages. Resource gate v3 enforces attribution, caps, measured memory, spill/WAL policy, fallback, hydration, and inactive-arm work. | Missing, hidden, contradictory, or unattributable counters fail qualification; they are never treated as zero. | +| M2 | The common typed orientation decision records planned/emitted policies, candidates, caps, admission, and fallback separately. Guarded and shadow fixed-suffix statements use bounded root/suffix/directional-degree probes, cap+1 sentinels, strict 3/4 hysteresis, bounded reverse state, and exact forward fallback. Expensive candidate and incumbent output chains are independently marker-gated. | Guarded/shadow execution is tool-only. Production fixed-suffix translation remains the exact forward incumbent. The already-shipped endpoint family retains its established 32/33 endpoint and 4096/4097 state guards. | +| M3 | Compact B1/B2 SP functions retain ID-only two-sided frontier/seen/predecessor state, exact 0/1/2-hop controls, typed schedulers, lower-bound termination, deterministic minimum witnesses, late hydration, invocation-local diagnostics, and exact S4 fallback on cap overflow. GraphBench exposes four full-comparator reference arms on a carryover-balanced three-arm schedule. `SP-I1-C-WE+MAT-M0` now has a guarded canonical-predecessor emitter with four cap+1 gates, inline M0 hydration, S4 fallback, complete nested receipts, an exact-bucket stable-snapshot driver canary, and an evidence-free rollback switch. S4/A1 share workspace v2, while `sp-static-v5-contained` restores S3 for qualified shallow single-kind witnesses. | B1/B2 and the under-guarded `SP-I1-C-D`/legacy witness executors are forceable/reference candidates only. Canonical predecessor SP is the sole inline SP production canary. `sp-static-v6` limits it to the confirmed inbound typed single-kind `1..64` bucket; broader activation remains unauthorized. | +| M4 | Confirmation, generic three/five-arm Williams tournaments, performance, selector-regret, resource, and reference-closure reports are machine-readable and evidence-gated. Promotion requires explicit materiality targets, a stable training/holdout winner, median materiality, p95 containment, and per-timed-invocation non-fallback attribution. Function-backed and guarded candidates now write a singular session-local branch receipt around every pool-size-one timed invocation; same-case diagnostic replay remains separate. The driver has default-off, generation-keyed, normalized-query-SHA allowlisted canaries and immediate rollback. It consumes the exact manifest bytes and verifies their digest, candidate, selector, execution boundary, caps, buckets, training/holdout split, query cohort, and required evidence digests. Endpoint-seeded reverse has an evidence-free emergency disable switch. | The clean `6d56a609` canonical-I1 confirmation passed all four training and three holdout cases with zero fallback and resource-gate v5 passing all 70 case-round records. Automatic production remains unchanged pending exact production-statement, reference-closure, and operational evidence. | +| M5 | B1/B2 ASP functions retain all same-minimum-depth predecessors on each side, select one deterministic completed meeting cut, saturate pre-enumeration counts, stage unique ordered edge arrays, and enforce separate discovery, predecessor, enumeration, and output-byte sentinels before exact A1 fallback. Full-multiset references and stress/cap cases are included. `ASP-I1-U-DAG+MAT-M0` has a typed inline emitter, exact bounded one/two-hop preflights, four cap+1 guards, exact A1 same-statement fallback, event-chain runtime receipts, inactive-arm evidence, exact-query manifest buckets, a kill switch, and live driver-policy/isolation/cache/rollback coverage. | B1/B2 ASP remain forceable/reference candidates. `ASP-A1-DAG` remains the automatic production choice. I1 is a default-off, stable-snapshot, exact-query canary; broader activation still requires clean evidence. | +| M6 | Optimizer diagnostics conservatively classify bounded endpoint sources and traversal predicate locality without changing execution. A property name alone is never considered a uniqueness proof; parameterized and literal small sets use the 32/33 contract. Fixed one-hop translation has an optimizer-independent exact dual-bound fallback, recognizes carried and node-valued `UNWIND` endpoints, and preserves directionless self-loops in unbound, single-bound, and dual-bound forms. The corpus and three exact PostgreSQL study arms cover pair join, lower-degree scan, pair reuse, both logical directions, wildcard/multi-kind edges, missing pairs, duplicates, and self-loops. Confirmation now requires material improvement, p95 containment, and one stable winner across separate training and holdout partitions. | Endpoint/predicate broadening remains analysis-only until the SP/ASP candidates it would feed qualify. The `ExpandInto` report is a study and cannot activate a policy. | +| M7 | The versioned topology-synopsis ADR records schema, mutation, refresh, staleness, cache-key, graph-lifecycle, and rollout requirements. | Deferred. Runtime probes remain authoritative; no synopsis schema or cache dependency is introduced. | + +## Qualification invariants + +Release-eligible evidence must satisfy all of the following: + +- complete declared corpus coverage, with diagnostic selections unable to pass; +- checksummed host-matched A/A evidence and balanced rounds at 97.5% confidence; +- a target p50 improvement clearing 5% or 100 microseconds and contained p95; +- independent, nonempty training and frozen-holdout passes for every concrete + prioritized candidate family; +- exact stable observations and SP witness validity or complete ASP/ordinary + result multisets as appropriate; +- complete required search and hydration telemetry with measured, attributable + resource use; +- at-most-once probes, zero work in unselected arms, and an exact, single, + declared fallback before output; +- cancellation, rollback, session reuse, pool isolation, and schema-down + symmetry. + +Stress cases are correctness/resource diagnostics. Their timing cannot tune or +promote a selector, and a stress fallback is accepted only where the case and +candidate declare that exact fallback. + +## Evidence still required for promotion + +Promotion is a later evidence-producing change. It must start from a clean +source checkout and publish credential-free checksums for the baseline and +candidate binaries, corpus declaration, source revision, database versions, +host A/A report, matched plans, discovery, confirmation, frozen holdout, +resource, reference-closure, cancellation/concurrency, and bundle-verification +reports. A passing report then enables only the named runtime-recognizable +topology and observation buckets; all other shapes retain their incumbents. diff --git a/docs/experiments/traversal_topology_synopsis_adr_v1.md b/docs/experiments/traversal_topology_synopsis_adr_v1.md new file mode 100644 index 00000000..9a4c23f0 --- /dev/null +++ b/docs/experiments/traversal_topology_synopsis_adr_v1.md @@ -0,0 +1,106 @@ +# Traversal topology synopsis ADR v1 + +Status: **deferred; no synopsis is read by production translation or execution**. + +Decision ID: `traversal-topology-synopsis-v1`. This record defines the design +and qualification boundary requested by M7 of +[`cysql_traversal_priorities.md`](../cysql_traversal_priorities.md). It does not +authorize a schema migration or selector change. Same-statement capped probes +and executor frontier state remain authoritative until a synopsis demonstrates +lower selector regret or lower probe overhead on the frozen holdout and also +passes the mutation, cache, and resource gates below. + +## Decision + +Do not add persistent topology tables yet. First capture the M1 diagnostic +counters and complete the M2--M6 candidate studies. Those artifacts provide the +runtime labels needed to test whether a synopsis predicts anything useful. A +synopsis implementation may proceed only as a separately versioned experiment; +it may influence a candidate score, but it may never prove correctness, bypass +an admission sentinel, or suppress exact fallback. + +If the experiment proceeds, prefer reading the current synopsis at execution +time. Embedding a synopsis value in translated SQL is forbidden until its epoch +is part of `cypherTranslationCacheKey` and an epoch change either invalidates or +misses every affected cached translation. Mutable rollout policy is likewise an +execution input or an explicit cache generation, never unkeyed translator +state. + +## Proposed storage contract + +The candidate schema is graph-scoped and generation-scoped. All rows for a new +generation become visible atomically by advancing one graph metadata row after +the generation is complete. + +| Relation | Key | Candidate values | +| --- | --- | --- | +| `traversal_synopsis_generation` | `(graph_id)` | `epoch`, schema/estimator version, source mutation epoch, build start/end, sampled/full mode, status | +| `traversal_synopsis_node_count` | `(graph_id, epoch, kind_id)` | exact or sampled count and error bound | +| `traversal_synopsis_edge_count` | `(graph_id, epoch, direction, kind_id, endpoint_kind_id)` | count, distinct starts/ends, error bound | +| `traversal_synopsis_degree` | `(graph_id, epoch, direction, kind_id, bucket)` | quantiles, heavy-hitter threshold, sample size | +| `traversal_synopsis_frontier` | `(graph_id, epoch, shape_bucket, depth_bucket)` | survival and reconvergence distributions, sample size | +| `traversal_synopsis_risk` | `(graph_id, epoch, shape_bucket)` | predecessor/output multiplicity buckets and saturation rate | + +Multi-kind node membership is represented by separate overlapping strata; the +reader must not sum them as disjoint populations. Every estimate carries sample +size, method, error bound, build timestamp, source mutation epoch, and estimator +version. Missing, stale, building, failed, or incompatible generations produce +`synopsis_unavailable` and leave the runtime-probe policy unchanged. + +## Refresh and mutation contract + +- Graph creation starts with no usable generation. Graph drop removes or makes + unreachable all generations for that graph. +- Bulk load or fixture replacement builds a fresh generation after the load and + publishes it atomically. Readers never mix epochs. +- Incremental node/edge mutations advance a graph mutation epoch. A published + synopsis whose source epoch differs is stale and advisory-only; the initial + experiment treats it as unavailable rather than estimating staleness. +- Refresh work runs outside query latency measurements, has bounded memory and + temporary storage, and records its own WAL, CPU, elapsed time, and table size. +- Failed or cancelled refresh leaves the previous generation intact but stale. + Cleanup is idempotent and cannot delete the currently published generation. +- The first implementation must include schema-up/schema-down symmetry, + concurrent reader/refresh tests, graph reload/drop tests, and an upgrade test. + +## Shadow comparison + +Shadow mode records a synopsis prediction beside the same-statement runtime +probe decision while executing the incumbent. It must not alter emitted arms. +Each record binds the workload, fixture and holdout identity, source revision, +graph mutation epoch, synopsis epoch/version, runtime policy version, probe +caps, predicted arm/score, observed probe values, actual selected exact arm, +fallback, and measured probe overhead. + +Evaluate normal and envelope tiers on the frozen holdout. Stress remains a +fallback/staleness diagnostic. Report at least: + +- prediction coverage and stale/unavailable frequency; +- selector regret against every exact arm; +- disagreement with capped runtime probes and executor frontier decisions; +- probe latency and buffer work saved after charging synopsis lookup cost; +- refresh latency, WAL, persistent bytes, and mutation write amplification; +- cache hit/miss and invalidation behavior across epoch changes; +- correctness, fallback, cancellation, pool-reuse, and concurrent-writer results. + +## Admission gate + +The synopsis experiment is rejected or remains deferred unless all of these are +shown with checksummed discovery and confirmation artifacts under the standard +97.5% protocol: + +1. The synopsis materially lowers selector regret or probe overhead on both the + declared corpus and frozen holdout after lookup cost. +2. No normal/envelope bucket regresses beyond the host A/A timing floor, and + resource limits pass without unexpected WAL or spill in read execution. +3. Stale, absent, incompatible, or partially refreshed data always selects the + unchanged runtime-probe/incumbent chain with a precise reason. +4. Mutation and refresh overhead passes an independently declared budget; it is + not hidden inside query measurements. +5. Translation-cache tests prove that no SQL can retain an unkeyed embedded + epoch or rollout policy. + +Until those gates pass, `traversal-topology-synopsis-v1` has no database schema, +no cache-key effect, no production feature gate, and no automatic selector +bucket. This is the reversible outcome required by the priority plan: runtime +evidence remains the authority, and lack of a synopsis is normal operation. diff --git a/docs/postgresql_translation.md b/docs/postgresql_translation.md index c05d2c94..f56f2cdb 100644 --- a/docs/postgresql_translation.md +++ b/docs/postgresql_translation.md @@ -28,7 +28,74 @@ Current PostgreSQL optimization coverage includes: `size(relationships(p))`, `startNode`, `endNode`, and `type`. - Recursive traversal optimizations for endpoint kind/property predicates, relationship type predicates, bound-node filters, traversal direction selection, and limit pushdown where ordering and distinct semantics permit it. +- Static shortest-path executor selection for one read-only, uncorrelated, directed traversal with one ID equality per + endpoint and no observed relationship/path predicate. Distance observations use scalar `SP-S3-U-D` state, with deep + physical-inbound searches sent to `SP-S4-C-D`. Bounded directed single-kind one-path witnesses use + `SP-S3-U-E+MAT-M0`; deep inbound and multi-kind or untyped witnesses use + `SP-S4-C-WE+MAT-M0`. Both S4 executors canonicalize + expansion, keep recursive state ID-only, enforce a bounded state + ceiling, and fall back to an exact relationship-trail query in the same statement and snapshot before returning a + row. Singleton ties return one valid minimal trail; physical edge-ID order is not public. See + `docs/shortest_path_tie_policy.md`. +- Default-off compact bidirectional SP candidates preserve that singleton + endpoint and observation envelope. `SP-B1-C-ALT-NODE-D` and + `SP-B1-C-ALT-NODE-WE+MAT-M0` alternate one accepted node per side; + `SP-B2-C-MIN-LEVEL-D` and `SP-B2-C-MIN-LEVEL-WE+MAT-M0` expand the smaller + complete current level. Both use ID-only invocation-local state, a + lower-bound stop condition, late witness hydration, independent + seen/frontier/predecessor caps, and exact S4 fallback before output. They are + reference and explicit-tool arms; the production driver rejects them. + `SP-I1-C-D` is likewise tool-only until it has the same cap, exact-fallback, + receipt, and kill-switch contract as guarded witness and ASP I1. Eligible + canaries require SHA-256-allowlisted queries under repeatable-read or + serializable isolation and a schema-v2 promotion manifest whose reports + repeat its complete authorization identity. The ordinary production path + remains unchanged. +- Static `allShortestPaths` selection through `asp-static-v1` for a single directed, read-only endpoint pair with + minimum depth one. `ASP-A1-DAG` has exact one- and two-hop arms, discovers minimum node-depth layers, retains every + relationship-distinct predecessor at those layers, and enumerates the predecessor DAG. Open maximum ranges use the + documented depth cap of 15. Unsupported or ambiguous forms retain exact `SP-S0` with a machine-readable reason. +- Default-off `ASP-B1-DAG-ALT-NODE` and `ASP-B2-DAG-MIN-LEVEL` reuse compact + two-sided search while retaining every same-minimum-depth predecessor. They + enumerate at one canonical completed meeting cut and apply separate + discovery, predecessor, saturating path-count, staged-output, and byte gates. + Overflow clears candidate state and invokes exact `ASP-A1-DAG` before output. + Production remains on A1 until independent training, frozen-holdout, + resource, and reference-closure reports pass; the allowlisted canary seam + uses the same explicit stable-snapshot requirement as SP. - Expansion suffix pushdown and `ExpandInto` detection for fixed suffixes and shared-endpoint fanout patterns. +- Typed compound expansion-search planning for directed bounded expansions followed by fixed suffixes. The decision + records its fixed-suffix expansion family, planned candidates, exact eligibility facts, observation mode, suffix + bounds, + selected/fallback strategy, selector version/mode, and stable fallback code separately from the legacy + boolean suffix prefilter. Correlated suffix bindings and predicates spanning the expansion/suffix boundary have + distinct conservative fallback codes. Candidate factored-forward and backward-viability SQL remains + reference-only. `EXPANSION-SUFFIX-SEEDED-REVERSE` has a repository-native, + qualification-only emitter. Explicit tool options select it and fail closed + unless translation records the matching target as applied. Production deliberately + retains the `EXPANSION-STEPWISE-FORWARD` translator and reports + `tournament_unqualified` for otherwise eligible three-hop forms because no + hard suffix-density or reverse-state bound is available before translation. +- The default-off `orientation-probe-v1` guarded and shadow statements measure + bounded duplicate-preserving roots, suffix rows/distinct boundaries, and + typed first-hop work from both sides. Every relation has a cap+1 sentinel; + reverse must beat forward by the versioned strict 3/4 hysteresis rule. + Guarded execution also caps reverse state and marker-gates candidate and + incumbent output chains independently. Probe and state overflow select the + exact forward fallback and produce a truthful runtime receipt. Shadow + execution always runs the incumbent, records only `would_select`, and emits + its marker-first receipt even for an empty result. Plan telemetry attributes + work from exact CTE materialization subplans so repeated consumer scans cannot + inflate probe or branch loops. A versioned query-allowlisted + driver canary can emit the guarded form only when it also binds a verified + promotion-manifest SHA-256, while the zero policy and every non-allowlisted + query remain forward. +- Guarded endpoint-seeded expansion selection covers a separate + `fixed_prefix_terminal_expansion` family: exactly one directed fixed prefix followed by one terminal, directed, + single-kind variable expansion with minimum depth one and a local selective terminal predicate. Production emits + `EXPANSION-ENDPOINT-SEEDED-REVERSE` with at most 32 terminal seeds and 4096 reverse states. Sentinel rows select an + exact stepwise-forward fallback inside the same statement and snapshot before candidate rows are exposed. Both arms + preserve ordered relationship IDs and enforce relationship uniqueness across the fixed prefix and expansion. - Strict string property equality lowering through `jsonb_typeof(properties -> key) = 'string'` plus `properties ->> key = value`, preserving JSON scalar semantics while allowing existing text expression indexes on selective fields such as `objectid` and `name`. @@ -37,6 +104,22 @@ Current PostgreSQL optimization coverage includes: correlations are sufficient. - Membership-only `collect(entity)` ID-array lowering with `id = any(...)` membership predicates. - Shortest-path strategy and terminal-filter planning for selective endpoint predicates and kind-only terminal filters. +- Analysis-only endpoint resolution metadata classifies ID equality, bounded + nonunique property equality, literal or parameterized small sets, and + correlated pairs with explicit 1/2/32/33 contracts. Property syntax is not a + uniqueness proof. Analysis-only traversal predicate metadata distinguishes + step-local and universal node/relationship forms from whole-path and + unsupported forms. Neither diagnostic broadens execution until the compact + candidates and that semantic class independently qualify. +- The fixed one-hop, bound-pair `ExpandInto` study exposes exact direct-pair, + lower-degree adjacency, and statement-local pair-reuse reference arms. It + covers outbound, inbound, directionless, wildcard/multi-kind, duplicate, + missing, and self-loop behavior but does not select a production policy. + Fixed-hop correctness does not depend on the study marker: dual-bound steps + always retain an exact pair-join fallback, including endpoints carried across + `WITH` or introduced by node-valued `UNWIND`. Directionless fixed hops use + paired endpoint orientations so self-loops are emitted once for unbound, + single-bound, and dual-bound forms. - Exact anonymous directed fixed-range expansion lowering for non-shortest-path `*1..1` and `*2..2` patterns. These shapes use fixed traversal steps instead of recursive CTEs, preserve path projection semantics, and enforce relationship uniqueness across emitted fixed steps. The explicit SQL-size cap is depth 2; broader exact ranges @@ -45,6 +128,103 @@ Current PostgreSQL optimization coverage includes: path edge IDs, avoiding full `edgecomposite[]` materialization when the final projection does not require it. - Dependency-safe clause reordering inside non-optional read regions, using existing selectivity heuristics while preserving stable tie order and pinning clauses with unresolved external dependencies. +- Field-sensitive continuation lowering carries node IDs as scalar columns between eligible fixed or recursive + traversal steps. Property, full-entity, path, cross-pattern, and mutation consumers retain composite bindings; + ID-only expansion endpoints still join the graph-scoped node partition so orphan filtering and multiplicity remain + unchanged. + +## Repeated-query compilation + +Each PostgreSQL driver keeps bounded least-recently-used caches of 256 successfully parsed Cypher ASTs and 256 safe SQL +translations. Parse-cache keys are +the trimmed query text; invalid input is not retained, and queries larger than 64 KiB bypass the cache. Concurrent misses +for the same text are coalesced. Cached ASTs remain immutable: the optimizer copies an AST before applying rules, so +parallel executions cannot mutate shared parser output. + +The cache deliberately retains complete trimmed query text, including literals, until LRU eviction or driver close. +That lifetime is bounded to 256 entries per driver; closing the driver clears all retained keys and AST references and +prevents in-flight misses from repopulating the cache. Queries whose source text exceeds 64 KiB bypass retention. Cache +diagnostics expose aggregate hit, miss, bypass, eviction, coalesced-miss, entry, and pending counts only—never query +text, literals, parameters, or credentials. + +The translation cache is keyed by trimmed query text, graph ID, parameter names, the PostgreSQL data type negotiated +for each parameter, and the exact effective traversal-policy identity. Values are rebound on every hit. This deliberately separates empty untyped lists from typed lists +and separates different graph partitions. A translation containing generated/static fragment parameters is not cached, +because those values cannot be reconstructed safely from caller parameters. Concurrent cacheable misses are coalesced; +waiters rebuild uncacheable translations rather than inheriting the first caller's values. Driver close clears both +caches. `ParseCacheStats` and `TranslationCacheStats` expose aggregate, query-text-free counters. + +`pg.TraversalPolicy` is default-off and admits one candidate family per +nonzero generation. It requires a nonempty allowlist built with +`pg.TraversalPolicyQuerySHA256` and the exact verified promotion-manifest bytes. +The driver checks the manifest digest and binds its candidate, selector, +execution boundary, immutable caps, training/holdout buckets, exact query +cohort, and required evidence digests before accepting the policy. Generation +and policy contents partition the translation cache. Setting the zero policy +makes older candidate entries immediately unreachable. B1/B2 candidates are +not production-canary eligible. `DisableEndpointSeededReverse` is an emergency +rollback control and intentionally requires no promotion artifact. Policy +forcing never broadens a lowering's structural correctness envelope. + +The same policy boundary now admits `ASP-I1-U-DAG+MAT-M0` as a default-off, +exact-query canary under Repeatable Read or Serializable isolation. Its +manifest must authorize the query SHA and exact direction/observation/depth/ +relationship-kind bucket, declare positive immutable state, predecessor, +enumeration, and output-byte caps, name `ASP-A1-DAG` as fallback, and use the +`guarded_dual_arm` boundary. Exact one- and two-hop targets bypass recursive +discovery. The inline statement materializes cap+1 preflight, distance, +predecessor, and enumeration relations before opening either output arm. A +version-2 runtime receipt retains the complete ordered event chain and +identifies `inline_predecessor_dag`, `inline_no_path`, or `exact_a1_fallback`; +the unselected arm emits no rows. Read Committed and +queries outside the exact allowlist retain A1. `DisableInlineASPDAG` is the +evidence-free emergency rollback control. + +Fixed-suffix expansion orientation uses the same fail-closed manifest boundary. +An `orientation-probe-v1` production manifest must name +`EXPANSION-STEPWISE-FORWARD` as fallback, use `guarded_dual_arm`, and bind the +immutable `root_row_limit=512`, `reverse_seed_row_limit=512`, +`directional_degree_row_limit=16384`, and `state_limit=4096` caps. The guarded +statement exposes that boundary in traversal telemetry; shadow and forced +single-arm statements report `inline_statement` and cannot stand in for +production-boundary evidence. + +Runtime receipt workspaces must exist on the exact PostgreSQL session before +an explicit read-only transaction begins. GraphBench satisfies this by pinning +and preparing one session. Driver callers that intentionally arm receipts from +inside a graph transaction can pass +`pg.OptionInitializeTraversalRuntimeAttestation()`; the driver then prepares +the acquired session immediately before `BEGIN READ ONLY`. + +The driver automatically prepares the production S4 and A1 session-local +workspaces before every explicit Repeatable Read or Serializable read-only +graph transaction. The underlying PostgreSQL transaction uses `READ WRITE` +access because workspace reset mutates session-local temporary tables; graph +data remains non-mutating. This keeps incumbent execution and a guarded +candidate's exact fallback valid on a fresh pooled connection. + +`SP-I1-C-WE+MAT-M0` uses the same guarded production boundary for singleton +one-path observations, with `SP-S4-C-WE+MAT-M0` as its declared fallback. The +manifest must authorize an exact `one_path` bucket and the same four positive +caps. It is admitted only at Repeatable Read or Serializable isolation; +`DisableInlineSPWitness` immediately restores the statically selected S3/S4 +incumbent and changes the cache identity without requiring evidence. + +The shortest-path functions use session-local `ON COMMIT PRESERVE ROWS` +workspace-v2 tables with invocation versions. Calls reset seen, candidate, and +predecessor state once, then derive each frontier from depth-tagged seen rows; +they do not create, drop, swap, or truncate frontier tables at every level. The functions set a +local `recursive_worktable_factor`, declare explicit `COST`/`ROWS` estimates, and carry graph/node/edge IDs until one +outer hydration boundary. Temporary-workspace buffers are expected for S4/ASP; executor temp-file spill and WAL remain +resource-gate failures. + +Raw PostgreSQL graph-composite values are driver implementation details. Use the result value mapper or +`graph.ScanNextResult` for nodes, relationships, paths, and their arrays instead of depending on pgx's historical +`map[string]any` composite representation. + +`Result.Keys()` returns metadata cached once for the result set; callers must treat that slice and its strings as +immutable for the result lifetime. `Result.Values()` remains row-scoped raw driver data. Public graph values produced +through the mapper are owned independently of later row advancement and pooled connection reuse. ## Indexing Notes diff --git a/docs/recursive_descent_cost_controls.md b/docs/recursive_descent_cost_controls.md new file mode 100644 index 00000000..c90c6c41 --- /dev/null +++ b/docs/recursive_descent_cost_controls.md @@ -0,0 +1,82 @@ +# Recursive-descent cost controls + +Date: 2026-08-09 + +This implementation addresses the six recursive-descent findings from the PostgreSQL/Neo4j delta review. It changes +the PostgreSQL execution architecture; it does not claim that the cross-backend latency gap is closed until the same +corpus is recaptured against both supplied backends. + +The next-phase orientation, SP/ASP, topology-evidence, and qualification work is +sequenced in the [CySQL traversal performance priorities](cysql_traversal_priorities.md), +with current candidate and promotion status recorded in +[the implementation status](experiments/traversal_priority_implementation_status_v1.md). + +| Finding | Implemented control | +|---|---| +| 1. `allShortestPaths` retained too much trail state | `ASP-A1-DAG` performs minimum-layer discovery, stores all relationship-distinct predecessors only for minimum layers, then enumerates the predecessor DAG. | +| 2. Small depths paid recursive setup cost | Both production functions have exact one-hop and two-hop SQL arms before workspace allocation. | +| 3. Breadth-first levels churned temporary catalog objects | Session-local workspace v2 is created once per connection and reset once per invocation. A1 and S4 derive each frontier from depth-tagged seen state and share one candidate relation instead of swapping or repeatedly truncating frontier tables. | +| 4. Singleton shortest paths needed a bounded compact search | `SP-S4-C-D` and `SP-S4-C-WE+MAT-M0` use canonical ID-only BFS state, a 100,000-state default ceiling, and exact same-statement fallback. | +| 5. Recursive rows hydrated entities too early | New executors carry node/relationship IDs and perform one ordered path hydration after search. | +| 6. Repeated compilation and unstable recursive estimates added overhead | Functions declare `COST`/`ROWS` and set `recursive_worktable_factor`; the driver has a bounded, coalescing, parameter-shape-aware translation cache. | + +Terminal-selective ordinary expansions also have a guarded reverse lowering. The optimizer only selects it for one +fixed directed prefix hop followed by a terminal directed expansion (`*1..64`) with one relationship kind and a local +terminal ID/property search. The statement probes 33 endpoints and 4097 reverse states: up to 32/4096 uses the reverse +candidate, while either sentinel activates the exact forward incumbent in the same snapshot. Candidate output is +gated until both probes finish, so overflow and cancellation cannot leak partial results. + +## Selection boundaries + +`asp-static-v1` selects `ASP-A1-DAG` only for one read-only, non-optional, directed `allShortestPaths` traversal with one +static ID equality per endpoint, minimum depth one, no path/relationship predicate, and no observed relationship value. +An open maximum uses depth 15. Minimum-depth-zero, self-endpoint, directionless, correlated, mutation, and predicate +shapes retain the incumbent exact executor. + +`sp-static-v5-contained` retains `SP-S3-U-D` for qualified distance work, with +`SP-S4-C-D` for deep physical-inbound distance searches. Already-qualified, +bounded, directed, single-kind one-path witnesses use `SP-S3-U-E+MAT-M0`; +deep inbound and multi-kind or untyped witnesses retain +`SP-S4-C-WE+MAT-M0`. This containment avoids paying the S4 workspace boundary +where the relationship-trail executor is the better incumbent. S4 checks a +cap+1 state ceiling before emitting any row and records its exact +`SP-S3-U-E+MAT-M0` fallback in the same statement and snapshot. + +`SP-I1-C-WE+MAT-M0` is a separate default-off canonical-predecessor canary. +Selector `sp-static-v6` restricts it to the qualified inbound, typed, +single-kind singleton one-path envelope with `min=1` and `max=64`; different +directions, kind shapes, or depth bounds fail closed. Its guarded inline statement uses +four cap+1 gates, hydrates only after admission, and falls back through S4. A +state overflow can therefore produce the auditable event chain +`SP-I1-C-WE+MAT-M0 -> SP-S4-C-WE+MAT-M0 -> SP-S3-U-E+MAT-M0` without exposing +rows from an abandoned arm. Stable isolation, an exact manifest bucket, and +positive immutable caps are mandatory; `DisableInlineSPWitness` is the +evidence-free rollback switch. + +`ASP-I1-U-DAG+MAT-M0` is also available through the production policy as a +default-off exact-query canary. It is limited to a singleton directed endpoint +pair, `allShortestPaths`, minimum depth one, and an explicit maximum no greater +than 64. Exact one- and two-hop targets are resolved before recursive +discovery. The typed recursive statement bounds distance discovery, +same-minimum-depth predecessor retention, all intermediate enumeration states, +and output bytes with immutable cap+1 sentinels. It exposes candidate and +fallback markers only after every guard is known; any overflow selects exact +`ASP-A1-DAG` before public output. Its canary requires a stable transaction +snapshot and a manifest whose topology bucket matches the optimized target. +Runtime receipts use schema v2 and retain the complete ordered branch-event +chain rather than overwriting nested fallback evidence. The automatic +`asp-static-v1` choice remains A1 until clean confirmation, +holdout, resource, and reference-closure evidence authorizes broader rollout. + +`EXPANSION-SUFFIX-SEEDED-REVERSE` remains tool-only. Existing evidence showed a +fixed-suffix expansion topology crossover that query shape alone does not safely +bound, so this work does not activate the strategy in production. The rejected +bounded-fallback and continuation experiments are retained only as historical +decision records under `docs/experiments`. + +## Qualification contract + +GraphBench recognizes `ASP-A1-DAG`, `ASP-I1-U-DAG+MAT-M0`, `SP-S4-C-D`, and `SP-S4-C-WE+MAT-M0` as applied architectures. Their resource gate +allows the declared local workspace but rejects executor temporary-file reads/writes and WAL for non-mutating queries. +Use the generated depth/fanout corpus, exact path observations, planner modes, concurrency, cancellation/session reuse, +and matched PostgreSQL/Neo4j delta report before treating the implementation as performance-qualified. diff --git a/docs/regression_source_parity.md b/docs/regression_source_parity.md new file mode 100644 index 00000000..13eedbad --- /dev/null +++ b/docs/regression_source_parity.md @@ -0,0 +1,134 @@ +# BloodHound Regression Source Parity + +This workflow keeps the stable query-form manifest synchronized with reviewed +BloodHound Enterprise (BHE) and BloodHound Community Edition (BHCE) source +snapshots. It records query shapes only; DAWGS must not import application +business logic or reproduce complete BloodHound traversal behavior. + +## Dormant tier + +`FUTURE-01` is the outbound tenant reconciliation form: + +```cypher +MATCH (s:AZEntity)-[r:K]->() +WHERE s.tenantid IN $tenant_ids +DELETE r +``` + +At BHE commit `c9f61530f45b`, its callers in +`lib/go/daemons/datapipe/ingest.go` are inside the block labeled "Disabled for +now". The compiled `ReconcileOutboundKindsForTenants` helper does not by itself +make the form production-active. + +Keep `FUTURE-01` in the dormant section of +`regression_coverage_manifest.md`. Do not add it to +`integration/testdata/cases`, `integration/testdata/templates`, or +`benchmark/testdata/scale/cases` while the caller remains disabled. Unit gates +in `cmd/plancorpus` and `cmd/graphbench` reject every `FUTURE-*` ID from those +active corpora. + +When a reviewed source snapshot enables the caller: + +1. Record the enabling entry point and source commit before changing the tier. +2. Move the manifest row from dormant to active and update the corpus gates in + the same change. +3. Add the exact outbound builder composition and the `PG`, `IT`, `PC`, and + `SC` layers required by `regression_coverage_manifest.md`. +4. Cover empty, single-item, 1,000-item, boundary, and stress tenant lists; + include direction, kind, tenant, endpoint, and missing/null decoys. +5. Use exact mutation post-state and rollback/reset isolation. Reuse the + `REC-04` matrix, but do not reuse its inbound query as proof of outbound + orientation. +6. Capture the PostgreSQL plan/runtime baseline with the same source metadata. + +## Audit procedure + +Set source roots to reviewed, immutable checkouts. These sources are audit +inputs and are not copied into DAWGS: + +```bash +export BHE_ROOT=/path/to/bhe +export BHCE_ROOT=/path/to/bhce +git -C "$BHE_ROOT" rev-parse HEAD +git -C "$BHCE_ROOT" rev-parse HEAD +git rev-parse HEAD +``` + +Start with a broad call-site inventory. This intentionally includes helpers and +commented code; activity is classified during the trace step: + +```bash +rg -n --glob '*.go' \ + '\b(Filterf?|Query|First|Count|Fetch[A-Za-z0-9_]*|Create[A-Za-z0-9_]*|Delete[A-Za-z0-9_]*|Update[A-Za-z0-9_]*|BatchOperation)\b' \ + "$BHE_ROOT" "$BHCE_ROOT" +``` + +For each candidate: + +1. Trace the helper to an active reconciliation, post-processing, or changelog + entry point. Label helper-only, test-only, and commented-out forms. +2. Normalize active forms by anchor, pattern, direction, relationship kinds, + predicates, projection, cardinality, mutation target, and execution path. +3. Map the tuple to an existing stable ID or add a new manifest row and source + link. A new operator, grouping, direction, anchor, projection, or mutation + target requires a distinct ID. +4. Treat stepwise traversal evidence as standalone `HOP-*` shapes only. Never + add a test that sequences the application traversal. +5. Recheck projection independently from predicates, and recheck relationship + kind-list and ID-list cardinalities after schema-set changes. +6. Apply the coverage contract from `regression_coverage_manifest.md`, then run + both backend suites and refresh PostgreSQL plan/scale captures when + applicable. + +## Ongoing parity checklist + +For each reviewed BHE/BHCE update: + +- [ ] Search active reconciliation and post-processing entry points for new + `Filter`, `Filterf`, `Query`, `First`, `Count`, `Fetch*`, `Create*`, + `Delete*`, `Update*`, and `BatchOperation` calls. +- [ ] Trace helpers to an active entry point and label helper-only, test-only, + commented-out, or dormant forms accurately. +- [ ] Normalize every active call with the tuple in + `regression_coverage_manifest.md`. +- [ ] Map the tuple to an existing stable ID or add a new ID and source link. +- [ ] If stepwise traversal criteria change, update only the corresponding + standalone `HOP-*` cases; do not sequence the downstream traversal. +- [ ] Recheck projections independently from predicates. +- [ ] Recheck relationship-kind and ID-list cardinalities when schema sets + change. +- [ ] Record the BHE, BHCE, and DAWGS commits used for the audit. + +## Audit record template + +Append one record per reviewed source update: + +```markdown +### YYYY-MM-DD source parity audit + +- BHE commit: `` +- BHCE commit: `` +- DAWGS commit/worktree: `` +- Active entry points reviewed: `` +- Existing IDs confirmed: `` +- IDs added or changed: `` +- Dormant/helper-only forms: `` +- Projection/cardinality changes: `

` +- Validation and captures: `` +``` + +## Seed audit record + +### 2026-08-04 source parity audit + +- BHE commit: `c9f61530f45b` +- BHCE commit: `74dd3daa58a8` +- DAWGS baseline: `v0.6.0-13-g6638cc2`; implementation worktree based on + `8c5fba7` with the dormant-form parity-gate changes +- Active IDs: the `LOGIC-*`, `REC-*`, `TRUST-*`, `PRUNE-*`, `HOP-*`, + `SCAN-*`, `LOOKUP-*`, and `WRITE-*` rows in + `regression_coverage_manifest.md` +- Dormant forms: `FUTURE-01`; both reviewed callers remain in the disabled + Azure reconciliation block +- Validation: PostgreSQL and Neo4j `make test_all`; PostgreSQL scale-plan and + scale captures under `.coverage/` diff --git a/docs/shortest_path_tie_policy.md b/docs/shortest_path_tie_policy.md new file mode 100644 index 00000000..29b60cae --- /dev/null +++ b/docs/shortest_path_tie_policy.md @@ -0,0 +1,23 @@ +# Singleton shortest-path tie policy + +Date: 2026-08-07 + +`shortestPath` promises one valid relationship-unique trail of minimum length. +It does not promise which equally short trail is selected, and PostgreSQL +physical relationship IDs or insertion order are not part of the public +contract. Callers that require every relationship-distinct minimum trail must +use `allShortestPaths`. + +An executor may use a deterministic internal tie breaker for repeatability, +but changing that internal choice is not a semantic change when the returned +trail remains valid and minimal. PostgreSQL/Neo4j compatibility fixtures +therefore compare logical node identities, relationship kinds, and stable +`logical_key` properties. They do not require both backends to select the same +physical relationship ID for singleton output. + +This policy permits a future singleton witness executor to retain one +predecessor per accepted node/depth state. It does not permit deduplication for +`allShortestPaths`, relationship/path predicates, relationship variables, or +other forms whose validity or output multiplicity depends on the complete +trail. Those forms retain their exact incumbent unless independently +qualified. diff --git a/drivers/pg/batch.go b/drivers/pg/batch.go index d7978cc6..89c992f8 100644 --- a/drivers/pg/batch.go +++ b/drivers/pg/batch.go @@ -6,7 +6,6 @@ import ( "fmt" "log/slog" "strconv" - "strings" "github.com/jackc/pgtype" "github.com/jackc/pgx/v5" @@ -18,6 +17,8 @@ import ( ) const ( + // LargeNodeUpdateThreshold is the node count above which batch updates use + // the large-update execution path. LargeNodeUpdateThreshold = 1_000_000 ) @@ -41,21 +42,46 @@ func (s *Int2ArrayEncoder) Encode(values []int16) string { return s.buffer.String() } +// batch buffers graph mutations and applies them through one PostgreSQL transaction in insertion order. type batch struct { - ctx context.Context - innerTransaction *transaction - schemaManager *SchemaManager - nodeDeletionBuffer []graph.ID + // ctx scopes database operations performed while flushing buffered mutations. + ctx context.Context + + // innerTransaction owns the PostgreSQL transaction through which every buffered mutation is applied. + innerTransaction *transaction + + // schemaManager resolves graph metadata and maps graph kinds to their database identifiers. + schemaManager *SchemaManager + + // nodeDeletionBuffer retains node identifiers awaiting a bulk delete. + nodeDeletionBuffer []graph.ID + + // relationshipDeletionBuffer retains relationship identifiers awaiting a bulk delete. relationshipDeletionBuffer []graph.ID - nodeCreateBuffer []*graph.Node - nodeUpdateBuffer []*graph.Node - nodeUpdateByBuffer []graph.NodeUpdate - relationshipCreateBuffer []*graph.Relationship + + // nodeCreateBuffer retains nodes awaiting a bulk insert. + nodeCreateBuffer []*graph.Node + + // nodeUpdateBuffer retains complete node replacements awaiting a bulk update. + nodeUpdateBuffer []*graph.Node + + // nodeUpdateByBuffer retains identity-property node upserts awaiting validation and execution. + nodeUpdateByBuffer []graph.NodeUpdate + + // relationshipCreateBuffer retains relationships awaiting conflict coalescing and insertion. + relationshipCreateBuffer []*graph.Relationship + + // relationshipUpdateByBuffer retains identity-based relationship upserts awaiting validation and execution. relationshipUpdateByBuffer []graph.RelationshipUpdate - batchWriteSize int - kindIDEncoder Int2ArrayEncoder + + // batchWriteSize is the buffer length that triggers an automatic flush. + batchWriteSize int + + // kindIDEncoder reuses one buffer when serializing PostgreSQL int2 arrays for node writes. + kindIDEncoder Int2ArrayEncoder } +// newBatch opens the transaction used by a mutation batch and applies its configured flush threshold. func newBatch(ctx context.Context, conn *pgxpool.Conn, schemaManager *SchemaManager, cfg *Config) (*batch, error) { if tx, err := newTransactionWrapper(ctx, conn, schemaManager, cfg, false); err != nil { return nil, err @@ -288,6 +314,7 @@ func (s *batch) UpdateNodes(nodes []*graph.Node) error { return nil } +// flushNodeDeleteBuffer deletes the buffered node IDs and clears the buffer after a successful execution. func (s *batch) flushNodeDeleteBuffer() error { if _, err := s.innerTransaction.conn.Exec(s.ctx, deleteNodeWithIDStatement, s.nodeDeletionBuffer); err != nil { return err @@ -297,6 +324,7 @@ func (s *batch) flushNodeDeleteBuffer() error { return nil } +// flushRelationshipDeleteBuffer deletes the buffered relationship IDs and clears the buffer after a successful execution. func (s *batch) flushRelationshipDeleteBuffer() error { if _, err := s.innerTransaction.conn.Exec(s.ctx, deleteEdgeWithIDStatement, s.relationshipDeletionBuffer); err != nil { return err @@ -306,6 +334,7 @@ func (s *batch) flushRelationshipDeleteBuffer() error { return nil } +// flushNodeCreateBuffer rejects mixed ID allocation modes and dispatches the buffered nodes to the matching insert path. func (s *batch) flushNodeCreateBuffer() error { var ( withoutIDs = false @@ -331,6 +360,7 @@ func (s *batch) flushNodeCreateBuffer() error { return s.flushNodeCreateBufferWithIDs() } +// flushNodeCreateBufferWithIDs inserts buffered nodes whose IDs were assigned by the caller. func (s *batch) flushNodeCreateBufferWithIDs() error { var ( numCreates = len(s.nodeCreateBuffer) @@ -368,6 +398,7 @@ func (s *batch) flushNodeCreateBufferWithIDs() error { return nil } +// flushNodeCreateBufferWithoutIDs inserts buffered nodes using database-generated IDs. func (s *batch) flushNodeCreateBufferWithoutIDs() error { var ( numCreates = len(s.nodeCreateBuffer) @@ -402,6 +433,7 @@ func (s *batch) flushNodeCreateBufferWithoutIDs() error { return nil } +// flushNodeUpsertBatch validates and executes one identity-based node upsert batch for the target graph. func (s *batch) flushNodeUpsertBatch(updates *sql.NodeUpdateBatch) error { parameters := NewNodeUpsertParameters(len(updates.Updates)) @@ -438,6 +470,7 @@ func (s *batch) flushNodeUpsertBatch(updates *sql.NodeUpdateBatch) error { return nil } +// tryFlushNodeUpdateByBuffer validates, writes, and clears the buffered identity-based node updates. func (s *batch) tryFlushNodeUpdateByBuffer() error { if updates, err := sql.ValidateNodeUpdateByBatch(s.nodeUpdateByBuffer); err != nil { return err @@ -449,6 +482,7 @@ func (s *batch) tryFlushNodeUpdateByBuffer() error { return nil } +// flushNodeUpdateBatch writes complete node replacements for the supplied nodes. func (s *batch) flushNodeUpdateBatch(nodes []*graph.Node) error { parameters := NewNodeUpdateParameters(len(nodes)) @@ -471,6 +505,7 @@ func (s *batch) flushNodeUpdateBatch(nodes []*graph.Node) error { } } +// tryFlushNodeUpdateBuffer writes and clears the buffered complete node updates. func (s *batch) tryFlushNodeUpdateBuffer() error { if err := s.flushNodeUpdateBatch(s.nodeUpdateBuffer); err != nil { return err @@ -648,6 +683,7 @@ func (s *RelationshipUpdateByParameters) AppendAll(ctx context.Context, updates return nil } +// flushRelationshipUpdateByBuffer upserts prerequisite nodes and then applies identity-based relationship updates. func (s *batch) flushRelationshipUpdateByBuffer(updates *sql.RelationshipUpdateBatch) error { if err := s.flushNodeUpsertBatch(updates.NodeUpdates); err != nil { return err @@ -672,6 +708,7 @@ func (s *batch) flushRelationshipUpdateByBuffer(updates *sql.RelationshipUpdateB return nil } +// tryFlushRelationshipUpdateByBuffer validates, writes, and clears the buffered identity-based relationship updates. func (s *batch) tryFlushRelationshipUpdateByBuffer() error { if updateBatch, err := sql.ValidateRelationshipUpdateByBatch(s.relationshipUpdateByBuffer); err != nil { return err @@ -683,13 +720,22 @@ func (s *batch) tryFlushRelationshipUpdateByBuffer() error { return nil } +// relationshipCreateBatch stores column-oriented values for one relationship insert statement. type relationshipCreateBatch struct { - startIDs []uint64 - endIDs []uint64 - edgeKindIDs []int16 + // startIDs contains each relationship's start-node identifier in insert-row order. + startIDs []uint64 + + // endIDs contains each relationship's end-node identifier in insert-row order. + endIDs []uint64 + + // edgeKindIDs contains each relationship's database kind identifier in insert-row order. + edgeKindIDs []int16 + + // edgePropertyBags contains each relationship's JSONB properties in insert-row order. edgePropertyBags []pgtype.JSONB } +// newRelationshipCreateBatch allocates relationship insert columns with capacity for size rows. func newRelationshipCreateBatch(size int) *relationshipCreateBatch { return &relationshipCreateBatch{ startIDs: make([]uint64, 0, size), @@ -717,18 +763,35 @@ func (s *relationshipCreateBatch) EncodeProperties(edgePropertiesBatch []*graph. return nil } +// relationshipCreateBatchBuilder coalesces duplicate relationship keys while retaining their merged properties. type relationshipCreateBatchBuilder struct { - keyToEdgeID map[string]uint64 + // keyToPropertiesIndex locates the property bag associated with each unique relationship key. + keyToPropertiesIndex map[relationshipCreateKey]int + + // relationshipUpdateBatch accumulates the column values emitted for unique relationship keys. relationshipUpdateBatch *relationshipCreateBatch - edgePropertiesIndex map[uint64]int - edgePropertiesBatch []*graph.Properties + + // edgePropertiesBatch retains mergeable properties parallel to relationshipUpdateBatch rows. + edgePropertiesBatch []*graph.Properties +} + +// relationshipCreateKey identifies a relationship by endpoints and kind for conflict coalescing. +type relationshipCreateKey struct { + // startID identifies the relationship's starting node. + startID graph.ID + + // endID identifies the relationship's ending node. + endID graph.ID + + // kind identifies the relationship kind independently of its property bag. + kind string } +// newRelationshipCreateBatchBuilder allocates a conflict index and column buffers for size relationship inputs. func newRelationshipCreateBatchBuilder(size int) *relationshipCreateBatchBuilder { return &relationshipCreateBatchBuilder{ - keyToEdgeID: map[string]uint64{}, + keyToPropertiesIndex: map[relationshipCreateKey]int{}, relationshipUpdateBatch: newRelationshipCreateBatch(size), - edgePropertiesIndex: map[uint64]int{}, } } @@ -736,21 +799,19 @@ func (s *relationshipCreateBatchBuilder) Build() (*relationshipCreateBatch, erro return s.relationshipUpdateBatch, s.relationshipUpdateBatch.EncodeProperties(s.edgePropertiesBatch) } +// Add coalesces edge into the relationship batch, merging properties when its endpoints and kind repeat. func (s *relationshipCreateBatchBuilder) Add(ctx context.Context, kindMapper KindMapper, edge *graph.Relationship) error { - keyBuilder := strings.Builder{} - - keyBuilder.WriteString(edge.StartID.String()) - keyBuilder.WriteString(edge.EndID.String()) - keyBuilder.WriteString(edge.Kind.String()) - - key := keyBuilder.String() + key := relationshipCreateKey{ + startID: edge.StartID, + endID: edge.EndID, + kind: edge.Kind.String(), + } - if existingPropertiesIdx, hasExisting := s.keyToEdgeID[key]; hasExisting { + if existingPropertiesIdx, hasExisting := s.keyToPropertiesIndex[key]; hasExisting { s.edgePropertiesBatch[existingPropertiesIdx].Merge(edge.Properties) } else { var ( startID = edge.StartID.Uint64() - edgeID = edge.ID.Uint64() endID = edge.EndID.Uint64() edgeProperties = edge.Properties.Clone() ) @@ -761,15 +822,14 @@ func (s *relationshipCreateBatchBuilder) Add(ctx context.Context, kindMapper Kin s.relationshipUpdateBatch.Add(startID, endID, edgeKindID) } - s.keyToEdgeID[key] = edgeID - + s.keyToPropertiesIndex[key] = len(s.edgePropertiesBatch) s.edgePropertiesBatch = append(s.edgePropertiesBatch, edgeProperties) - s.edgePropertiesIndex[edgeID] = len(s.edgePropertiesBatch) - 1 } return nil } +// flushRelationshipCreateBuffer coalesces duplicate keys, inserts the resulting relationships, and clears the input buffer. func (s *batch) flushRelationshipCreateBuffer() error { batchBuilder := newRelationshipCreateBatchBuilder(len(s.relationshipCreateBuffer)) @@ -784,7 +844,7 @@ func (s *batch) flushRelationshipCreateBuffer() error { } else if graphTarget, err := s.innerTransaction.getTargetGraph(); err != nil { return err } else if _, err := s.innerTransaction.conn.Exec(s.ctx, createEdgeBatchStatement, graphTarget.ID, createBatch.startIDs, createBatch.endIDs, createBatch.edgeKindIDs, createBatch.edgePropertyBags); err != nil { - slog.Info(fmt.Sprintf("Num merged property bags: %d - Num edge keys: %d - StartID batch size: %d", len(batchBuilder.edgePropertiesIndex), len(batchBuilder.keyToEdgeID), len(batchBuilder.relationshipUpdateBatch.startIDs))) + slog.Info(fmt.Sprintf("Num property bags: %d - Num edge keys: %d - StartID batch size: %d", len(batchBuilder.edgePropertiesBatch), len(batchBuilder.keyToPropertiesIndex), len(batchBuilder.relationshipUpdateBatch.startIDs))) return err } @@ -792,6 +852,7 @@ func (s *batch) flushRelationshipCreateBuffer() error { return nil } +// tryFlush writes any mutation buffer whose length exceeds batchWriteSize. func (s *batch) tryFlush(batchWriteSize int) error { if len(s.nodeUpdateByBuffer) > batchWriteSize { if err := s.tryFlushNodeUpdateByBuffer(); err != nil { diff --git a/drivers/pg/batch_test.go b/drivers/pg/batch_test.go new file mode 100644 index 00000000..8d01de5f --- /dev/null +++ b/drivers/pg/batch_test.go @@ -0,0 +1,90 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package pg + +import ( + "context" + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/stretchr/testify/require" +) + +// staticKindMapper returns deterministic kind mappings for relationship batch tests. +type staticKindMapper struct { +} + +// MapKindID returns the fixed kind associated with every synthetic ID. +func (s staticKindMapper) MapKindID(context.Context, int16) (graph.Kind, error) { + return graph.StringKind("WriteCreateRelationship"), nil +} + +// MapKindIDs returns the fixed kind set used by the batch fixture. +func (s staticKindMapper) MapKindIDs(context.Context, []int16) (graph.Kinds, error) { + return graph.Kinds{graph.StringKind("WriteCreateRelationship")}, nil +} + +// MapKind returns the fixed database ID associated with every synthetic kind. +func (s staticKindMapper) MapKind(context.Context, graph.Kind) (int16, error) { + return 1, nil +} + +// MapKinds returns the fixed database ID set used by the batch fixture. +func (s staticKindMapper) MapKinds(context.Context, graph.Kinds) ([]int16, error) { + return []int16{1}, nil +} + +// AssertKinds accepts every supplied kind and returns the fixture's fixed database ID. +func (s staticKindMapper) AssertKinds(context.Context, graph.Kinds) ([]int16, error) { + return []int16{1}, nil +} + +// TestRelationshipCreateBatchBuilderMergesPropertiesByConflictKey verifies distinct endpoint tuples cannot collide and duplicate tuples merge properties. +func TestRelationshipCreateBatchBuilderMergesPropertiesByConflictKey(t *testing.T) { + var ( + ctx = context.Background() + kind = graph.StringKind("WriteCreateRelationship") + builder = newRelationshipCreateBatchBuilder(4) + ) + + updates := []*graph.Relationship{ + // These two endpoint pairs had the same concatenated key ("123...") + // before the batch builder used a structured conflict key. + graph.NewRelationship(0, 1, 23, graph.NewProperties().SetAll(map[string]any{"custom": "a-first", "a": true}), kind), + graph.NewRelationship(0, 1, 23, graph.NewProperties().SetAll(map[string]any{"custom": "a-last", "a-last": true}), kind), + graph.NewRelationship(0, 12, 3, graph.NewProperties().SetAll(map[string]any{"custom": "b-first", "b": true}), kind), + graph.NewRelationship(0, 12, 3, graph.NewProperties().SetAll(map[string]any{"custom": "b-last", "b-last": true}), kind), + } + for _, update := range updates { + require.NoError(t, builder.Add(ctx, staticKindMapper{}, update)) + } + + require.Len(t, builder.edgePropertiesBatch, 2) + require.Equal(t, "a-last", builder.edgePropertiesBatch[0].Get("custom").Any()) + require.Equal(t, true, builder.edgePropertiesBatch[0].Get("a").Any()) + require.Equal(t, true, builder.edgePropertiesBatch[0].Get("a-last").Any()) + require.False(t, builder.edgePropertiesBatch[0].Exists("b-last")) + require.Equal(t, "b-last", builder.edgePropertiesBatch[1].Get("custom").Any()) + require.Equal(t, true, builder.edgePropertiesBatch[1].Get("b").Any()) + require.Equal(t, true, builder.edgePropertiesBatch[1].Get("b-last").Any()) + require.False(t, builder.edgePropertiesBatch[1].Exists("a-last")) + + batch, err := builder.Build() + require.NoError(t, err) + require.Len(t, batch.startIDs, 2) + require.Len(t, batch.edgePropertyBags, 2) +} diff --git a/drivers/pg/composite_codec.go b/drivers/pg/composite_codec.go new file mode 100644 index 00000000..237d45da --- /dev/null +++ b/drivers/pg/composite_codec.go @@ -0,0 +1,194 @@ +package pg + +import ( + sqldriver "database/sql/driver" + "fmt" + + "github.com/jackc/pgx/v5/pgtype" + "github.com/specterops/dawgs/cypher/models/pgsql" +) + +// ownedComposite is the set of PostgreSQL composites that have a stable, +// driver-owned Go representation. Keeping this set closed makes it difficult +// to accidentally register an unrelated composite with a decoder whose field +// order does not match its PostgreSQL definition. +type ownedComposite interface { + // The closed type set limits optimized decoding to composites whose PostgreSQL field order is owned by this driver. + nodeComposite | edgeComposite | pathComposite +} + +// ownedCompositeCodec retains pgx's encoding and explicit Scan behavior while +// replacing CompositeCodec.DecodeValue's map[string]any result. Rows.Values +// uses DecodeValue, so decoding directly into the concrete representation +// avoids a map and one interface value per field. The field scanners allocate +// their slices and JSON maps, which also makes the returned value independent +// of pgx's reusable wire buffer. +type ownedCompositeCodec[T ownedComposite] struct { + // compositeCodec retains pgx's standard encoding and scan-plan implementation. + compositeCodec *pgtype.CompositeCodec +} + +// ownedCompositeArrayCodec decodes the common, non-null-element case directly +// into []T. PostgreSQL arrays may contain NULL composite elements, so a typed +// scan failure falls back to pgx's []any representation instead of discarding +// that information. +type ownedCompositeArrayCodec[T ownedComposite] struct { + // arrayCodec retains pgx's array metadata and fallback decoding behavior. + arrayCodec *pgtype.ArrayCodec +} + +// FormatSupported reports whether the wrapped composite codec accepts format. +func (s *ownedCompositeCodec[T]) FormatSupported(format int16) bool { + return s.compositeCodec.FormatSupported(format) +} + +// PreferredFormat returns the wire format preferred by the wrapped composite codec. +func (s *ownedCompositeCodec[T]) PreferredFormat() int16 { + return s.compositeCodec.PreferredFormat() +} + +// PlanEncode delegates composite encoding to pgx's registered composite codec. +func (s *ownedCompositeCodec[T]) PlanEncode(m *pgtype.Map, oid uint32, format int16, value any) pgtype.EncodePlan { + return s.compositeCodec.PlanEncode(m, oid, format, value) +} + +// PlanScan preserves pgx's explicit-target composite scanning behavior. +func (s *ownedCompositeCodec[T]) PlanScan(m *pgtype.Map, oid uint32, format int16, target any) pgtype.ScanPlan { + return s.compositeCodec.PlanScan(m, oid, format, target) +} + +// DecodeDatabaseSQLValue delegates database/sql decoding to pgx's composite codec. +func (s *ownedCompositeCodec[T]) DecodeDatabaseSQLValue( + m *pgtype.Map, + oid uint32, + format int16, + src []byte, +) (sqldriver.Value, error) { + return s.compositeCodec.DecodeDatabaseSQLValue(m, oid, format, src) +} + +// DecodeValue decodes non-null composites into their owned Go representation and falls back for nullable fields. +func (s *ownedCompositeCodec[T]) DecodeValue(m *pgtype.Map, oid uint32, format int16, src []byte) (any, error) { + if src == nil { + return nil, nil + } + + var value T + target, typeOK := any(&value).(pgtype.CompositeIndexScanner) + if !typeOK { + return nil, fmt.Errorf("owned composite target %T does not implement pgtype.CompositeIndexScanner", &value) + } + + plan := s.compositeCodec.PlanScan(m, oid, format, target) + if plan == nil { + return nil, fmt.Errorf("unable to scan PostgreSQL composite OID %d in format %d into %T", oid, format, &value) + } + + if err := plan.Scan(src, target); err != nil { + // PostgreSQL permits NULL fields inside a non-NULL composite, while the + // hot-path representation deliberately uses non-nullable scalar fields. + // Preserve the old map representation for those uncommon values rather + // than turning a valid row into a decode error. + return s.compositeCodec.DecodeValue(m, oid, format, src) + } + + return value, nil +} + +// FormatSupported reports whether the wrapped array codec accepts format. +func (s *ownedCompositeArrayCodec[T]) FormatSupported(format int16) bool { + return s.arrayCodec.FormatSupported(format) +} + +// PreferredFormat returns the wire format preferred by the wrapped array codec. +func (s *ownedCompositeArrayCodec[T]) PreferredFormat() int16 { + return s.arrayCodec.PreferredFormat() +} + +// PlanEncode delegates composite-array encoding to pgx's registered array codec. +func (s *ownedCompositeArrayCodec[T]) PlanEncode( + m *pgtype.Map, + oid uint32, + format int16, + value any, +) pgtype.EncodePlan { + return s.arrayCodec.PlanEncode(m, oid, format, value) +} + +// PlanScan preserves pgx's explicit-target composite-array scanning behavior. +func (s *ownedCompositeArrayCodec[T]) PlanScan( + m *pgtype.Map, + oid uint32, + format int16, + target any, +) pgtype.ScanPlan { + return s.arrayCodec.PlanScan(m, oid, format, target) +} + +// DecodeDatabaseSQLValue delegates database/sql decoding to pgx's array codec. +func (s *ownedCompositeArrayCodec[T]) DecodeDatabaseSQLValue( + m *pgtype.Map, + oid uint32, + format int16, + src []byte, +) (sqldriver.Value, error) { + return s.arrayCodec.DecodeDatabaseSQLValue(m, oid, format, src) +} + +// DecodeValue decodes arrays without null elements into []T and otherwise preserves pgx's nullable representation. +func (s *ownedCompositeArrayCodec[T]) DecodeValue(m *pgtype.Map, oid uint32, format int16, src []byte) (any, error) { + if src == nil { + return nil, nil + } + + var values []T + if plan := m.PlanScan(oid, format, &values); plan != nil { + if err := plan.Scan(src, &values); err == nil { + return values, nil + } + } + + // A []T cannot represent a NULL composite array element. Preserve pgx's + // nullable []any behavior for that less common case. + return s.arrayCodec.DecodeValue(m, oid, format, src) +} + +// installOwnedCompositeCodec replaces a supported pgx codec with the matching driver-owned scalar or array decoder. +func installOwnedCompositeCodec(dataType pgsql.DataType, definition *pgtype.Type) error { + switch dataType { + case pgsql.NodeCompositeArray: + arrayCodec, typeOK := definition.Codec.(*pgtype.ArrayCodec) + if !typeOK { + return fmt.Errorf("expected PostgreSQL type %s to use *pgtype.ArrayCodec but received %T", dataType, definition.Codec) + } + + definition.Codec = &ownedCompositeArrayCodec[nodeComposite]{arrayCodec: arrayCodec} + return nil + case pgsql.EdgeCompositeArray: + arrayCodec, typeOK := definition.Codec.(*pgtype.ArrayCodec) + if !typeOK { + return fmt.Errorf("expected PostgreSQL type %s to use *pgtype.ArrayCodec but received %T", dataType, definition.Codec) + } + + definition.Codec = &ownedCompositeArrayCodec[edgeComposite]{arrayCodec: arrayCodec} + return nil + } + + compositeCodec, typeOK := definition.Codec.(*pgtype.CompositeCodec) + if !typeOK { + return fmt.Errorf("expected PostgreSQL type %s to use *pgtype.CompositeCodec but received %T", dataType, definition.Codec) + } + + switch dataType { + case pgsql.NodeComposite: + definition.Codec = &ownedCompositeCodec[nodeComposite]{compositeCodec: compositeCodec} + case pgsql.EdgeComposite: + definition.Codec = &ownedCompositeCodec[edgeComposite]{compositeCodec: compositeCodec} + case pgsql.PathComposite: + definition.Codec = &ownedCompositeCodec[pathComposite]{compositeCodec: compositeCodec} + default: + return fmt.Errorf("PostgreSQL type %s does not have an owned composite decoder", dataType) + } + + return nil +} diff --git a/drivers/pg/composite_codec_integration_test.go b/drivers/pg/composite_codec_integration_test.go new file mode 100644 index 00000000..852e3336 --- /dev/null +++ b/drivers/pg/composite_codec_integration_test.go @@ -0,0 +1,273 @@ +package pg + +import ( + "context" + "os" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/stretchr/testify/require" +) + +// postgresIntegrationConnectionString returns CONNECTION_STRING only for a PostgreSQL target and skips the driver-scoped test otherwise. +func postgresIntegrationConnectionString(t *testing.T) string { + t.Helper() + + connectionString := os.Getenv("CONNECTION_STRING") + if connectionString == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + + normalizedConnectionString := strings.ToLower(connectionString) + if !strings.HasPrefix(normalizedConnectionString, "postgres://") && + !strings.HasPrefix(normalizedConnectionString, "postgresql://") { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + return connectionString +} + +// connectCompositeCodecIntegration opens a timeout-bounded PostgreSQL connection and registers cleanup for composite-codec integration tests. +func connectCompositeCodecIntegration(t *testing.T) (context.Context, *pgx.Conn) { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + t.Cleanup(cancel) + + config, err := pgx.ParseConfig(postgresIntegrationConnectionString(t)) + require.NoError(t, err) + + conn, err := pgx.ConnectConfig(ctx, config) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, conn.Close(context.Background())) + }) + + // Keep this driver-scoped test independent of application data and schema + // state. PostgreSQL drops types in pg_temp with the connection. + _, err = conn.Exec(ctx, ` +set search_path = pg_temp, public; +create type pg_temp.nodeComposite as ( + id bigint, + kind_ids smallint[], + properties jsonb +); +create type pg_temp.edgeComposite as ( + id bigint, + start_id bigint, + end_id bigint, + kind_id smallint, + properties jsonb +); +create type pg_temp.pathComposite as ( + nodes nodeComposite[], + edges edgeComposite[] +);`) + require.NoError(t, err) + + require.NoError(t, AfterPooledConnectionEstablished(ctx, conn)) + + return ctx, conn +} + +// TestPostgresOwnedCompositeCodecRegistration verifies pooled connections register optimized codecs for every owned composite type. +func TestPostgresOwnedCompositeCodecRegistration(t *testing.T) { + _, conn := connectCompositeCodecIntegration(t) + typeMap := conn.TypeMap() + + nodeType, typeOK := typeMap.TypeForName(pgsql.NodeComposite.String()) + require.True(t, typeOK) + require.IsType(t, &ownedCompositeCodec[nodeComposite]{}, nodeType.Codec) + + nodeArrayType, typeOK := typeMap.TypeForName(pgsql.NodeCompositeArray.String()) + require.True(t, typeOK) + nodeArrayCodec, typeOK := nodeArrayType.Codec.(*ownedCompositeArrayCodec[nodeComposite]) + require.True(t, typeOK) + require.Same(t, nodeType, nodeArrayCodec.arrayCodec.ElementType) + + edgeType, typeOK := typeMap.TypeForName(pgsql.EdgeComposite.String()) + require.True(t, typeOK) + require.IsType(t, &ownedCompositeCodec[edgeComposite]{}, edgeType.Codec) + + pathType, typeOK := typeMap.TypeForName(pgsql.PathComposite.String()) + require.True(t, typeOK) + require.IsType(t, &ownedCompositeCodec[pathComposite]{}, pathType.Codec) +} + +// TestPostgresOwnedCompositeCodecRowsValues verifies Rows.Values returns driver-owned node and edge composites. +func TestPostgresOwnedCompositeCodecRowsValues(t *testing.T) { + ctx, conn := connectCompositeCodecIntegration(t) + + for _, testCase := range []struct { + // name identifies the wire-format subtest. + name string + + // format selects the pgx result format used by the query. + format int16 + }{ + { + name: "binary", + format: pgtype.BinaryFormatCode, + }, + { + name: "text", + format: pgtype.TextFormatCode, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + rows, err := conn.Query(ctx, ` +select (series.id, array[1::smallint, 2::smallint], jsonb_build_object('id', series.id))::nodeComposite +from generate_series(101::bigint, 102::bigint) series(id) +order by series.id`, pgx.QueryResultFormats{testCase.format}) + require.NoError(t, err) + defer rows.Close() + + require.True(t, rows.Next()) + firstValues, err := rows.Values() + require.NoError(t, err) + require.Len(t, firstValues, 1) + first, typeOK := firstValues[0].(nodeComposite) + require.True(t, typeOK) + require.Equal(t, int64(101), first.ID) + require.Equal(t, []int16{1, 2}, first.KindIDs) + + require.True(t, rows.Next()) + secondValues, err := rows.Values() + require.NoError(t, err) + second, typeOK := secondValues[0].(nodeComposite) + require.True(t, typeOK) + require.Equal(t, int64(102), second.ID) + + // Reading the next row must not overwrite data retained from the + // first Rows.Values call. + require.Equal(t, int64(101), first.ID) + require.Equal(t, []int16{1, 2}, first.KindIDs) + require.Equal(t, float64(101), first.Properties["id"]) + + require.False(t, rows.Next()) + require.NoError(t, rows.Err()) + }) + } +} + +// TestPostgresOwnedCompositeCodecArraysAndPaths verifies composite arrays and paths decode into their typed graph representations. +func TestPostgresOwnedCompositeCodecArraysAndPaths(t *testing.T) { + ctx, conn := connectCompositeCodecIntegration(t) + + for _, testCase := range []struct { + // name identifies the wire-format subtest. + name string + + // format selects the pgx result format used by the query. + format int16 + }{ + { + name: "binary", + format: pgtype.BinaryFormatCode, + }, + { + name: "text", + format: pgtype.TextFormatCode, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + rows, err := conn.Query(ctx, ` +select + array[ + (101, array[1::smallint], '{"name":"first"}'::jsonb)::nodeComposite, + null::nodeComposite, + (102, array[2::smallint], '{"name":"second"}'::jsonb)::nodeComposite + ]::nodeComposite[], + ( + array[ + (101, array[1::smallint], '{"name":"first"}'::jsonb)::nodeComposite, + (102, array[2::smallint], '{"name":"second"}'::jsonb)::nodeComposite + ]::nodeComposite[], + array[ + (201, 101, 102, 3::smallint, '{"name":"edge"}'::jsonb)::edgeComposite + ]::edgeComposite[] + )::pathComposite`, pgx.QueryResultFormats{testCase.format}) + require.NoError(t, err) + defer rows.Close() + + require.True(t, rows.Next()) + values, err := rows.Values() + require.NoError(t, err) + require.Len(t, values, 2) + + nodes, typeOK := values[0].([]any) + require.True(t, typeOK) + require.Len(t, nodes, 3) + require.IsType(t, nodeComposite{}, nodes[0]) + require.Nil(t, nodes[1]) + require.IsType(t, nodeComposite{}, nodes[2]) + + path, typeOK := values[1].(pathComposite) + require.True(t, typeOK) + require.Len(t, path.Nodes, 2) + require.Len(t, path.Edges, 1) + require.Equal(t, int64(101), path.Nodes[0].ID) + require.Equal(t, int64(102), path.Nodes[1].ID) + require.Equal(t, int64(201), path.Edges[0].ID) + + require.False(t, rows.Next()) + require.NoError(t, rows.Err()) + }) + } +} + +// TestPostgresOwnedCompositeCodecNullInternalFieldFallback verifies nullable composite fields retain pgx's lossless fallback representation. +func TestPostgresOwnedCompositeCodecNullInternalFieldFallback(t *testing.T) { + ctx, conn := connectCompositeCodecIntegration(t) + + for _, testCase := range []struct { + // name identifies the wire-format subtest. + name string + + // format selects the pgx result format used by the query. + format int16 + }{ + { + name: "binary", + format: pgtype.BinaryFormatCode, + }, + { + name: "text", + format: pgtype.TextFormatCode, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + rows, err := conn.Query(ctx, ` +select + (null::bigint, array[1::smallint], '{"name":"nullable"}'::jsonb)::nodeComposite, + array[(null::bigint, array[1::smallint], '{"name":"nullable"}'::jsonb)::nodeComposite]::nodeComposite[]`, + pgx.QueryResultFormats{testCase.format}) + require.NoError(t, err) + defer rows.Close() + + require.True(t, rows.Next()) + values, err := rows.Values() + require.NoError(t, err) + require.Len(t, values, 2) + + node, typeOK := values[0].(map[string]any) + require.True(t, typeOK) + require.Nil(t, node["id"]) + require.Equal(t, []any{int16(1)}, node["kind_ids"]) + + nodes, typeOK := values[1].([]any) + require.True(t, typeOK) + require.Len(t, nodes, 1) + node, typeOK = nodes[0].(map[string]any) + require.True(t, typeOK) + require.Nil(t, node["id"]) + + require.False(t, rows.Next()) + require.NoError(t, rows.Err()) + }) + } +} diff --git a/drivers/pg/composite_codec_test.go b/drivers/pg/composite_codec_test.go new file mode 100644 index 00000000..6e246299 --- /dev/null +++ b/drivers/pg/composite_codec_test.go @@ -0,0 +1,531 @@ +package pg + +import ( + "reflect" + "testing" + + "github.com/jackc/pgx/v5/pgtype" + "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/stretchr/testify/require" +) + +const ( + // testNodeCompositeOID is the synthetic scalar node OID registered by codec unit tests. + testNodeCompositeOID uint32 = 91_001 + + // testNodeCompositeArrayOID is the synthetic node-array OID registered by codec unit tests. + testNodeCompositeArrayOID uint32 = 91_002 + + // testEdgeCompositeOID is the synthetic scalar edge OID registered by codec unit tests. + testEdgeCompositeOID uint32 = 91_003 + + // testEdgeCompositeArrayOID is the synthetic edge-array OID registered by codec unit tests. + testEdgeCompositeArrayOID uint32 = 91_004 + + // testPathCompositeOID is the synthetic path OID registered by codec unit tests. + testPathCompositeOID uint32 = 91_005 +) + +// compositeCodecTestTypes provides a controllable test double for PostgreSQL composite decoding; graph values round-trip through pgx without losing identity or properties. +type compositeCodecTestTypes struct { + // node is the registered scalar node type. + node *pgtype.Type + + // nodeArray is the registered node-array type. + nodeArray *pgtype.Type + + // edge is the registered scalar edge type. + edge *pgtype.Type + + // edgeArray is the registered edge-array type. + edgeArray *pgtype.Type + + // path is the registered scalar path type. + path *pgtype.Type +} + +// requirePGType returns the type registered for oid and fails the test when the registration is absent. +func requirePGType(t testing.TB, typeMap *pgtype.Map, oid uint32) *pgtype.Type { + t.Helper() + + dataType, typeOK := typeMap.TypeForOID(oid) + require.True(t, typeOK, "expected PostgreSQL type OID %d", oid) + + return dataType +} + +// newCompositeCodecTestMap registers synthetic node, edge, and path definitions, optionally installing owned codecs. +func newCompositeCodecTestMap(t testing.TB, owned bool) (*pgtype.Map, compositeCodecTestTypes) { + t.Helper() + + typeMap := pgtype.NewMap() + types := compositeCodecTestTypes{} + types.node = &pgtype.Type{ + Name: pgsql.NodeComposite.String(), + OID: testNodeCompositeOID, + Codec: &pgtype.CompositeCodec{ + Fields: []pgtype.CompositeCodecField{ + { + Name: "id", + Type: requirePGType(t, typeMap, pgtype.Int8OID), + }, + { + Name: "kind_ids", + Type: requirePGType(t, typeMap, pgtype.Int2ArrayOID), + }, + { + Name: "properties", + Type: requirePGType(t, typeMap, pgtype.JSONBOID), + }, + }, + }, + } + if owned { + require.NoError(t, installOwnedCompositeCodec(pgsql.NodeComposite, types.node)) + } + typeMap.RegisterType(types.node) + + types.nodeArray = &pgtype.Type{ + Name: pgsql.NodeCompositeArray.String(), + OID: testNodeCompositeArrayOID, + Codec: &pgtype.ArrayCodec{ + ElementType: types.node, + }, + } + if owned { + require.NoError(t, installOwnedCompositeCodec(pgsql.NodeCompositeArray, types.nodeArray)) + } + typeMap.RegisterType(types.nodeArray) + + types.edge = &pgtype.Type{ + Name: pgsql.EdgeComposite.String(), + OID: testEdgeCompositeOID, + Codec: &pgtype.CompositeCodec{ + Fields: []pgtype.CompositeCodecField{ + { + Name: "id", + Type: requirePGType(t, typeMap, pgtype.Int8OID), + }, + { + Name: "start_id", + Type: requirePGType(t, typeMap, pgtype.Int8OID), + }, + { + Name: "end_id", + Type: requirePGType(t, typeMap, pgtype.Int8OID), + }, + { + Name: "kind_id", + Type: requirePGType(t, typeMap, pgtype.Int2OID), + }, + { + Name: "properties", + Type: requirePGType(t, typeMap, pgtype.JSONBOID), + }, + }, + }, + } + if owned { + require.NoError(t, installOwnedCompositeCodec(pgsql.EdgeComposite, types.edge)) + } + typeMap.RegisterType(types.edge) + + types.edgeArray = &pgtype.Type{ + Name: pgsql.EdgeCompositeArray.String(), + OID: testEdgeCompositeArrayOID, + Codec: &pgtype.ArrayCodec{ + ElementType: types.edge, + }, + } + if owned { + require.NoError(t, installOwnedCompositeCodec(pgsql.EdgeCompositeArray, types.edgeArray)) + } + typeMap.RegisterType(types.edgeArray) + + types.path = &pgtype.Type{ + Name: pgsql.PathComposite.String(), + OID: testPathCompositeOID, + Codec: &pgtype.CompositeCodec{ + Fields: []pgtype.CompositeCodecField{ + { + Name: "nodes", + Type: types.nodeArray, + }, + { + Name: "edges", + Type: types.edgeArray, + }, + }, + }, + } + if owned { + require.NoError(t, installOwnedCompositeCodec(pgsql.PathComposite, types.path)) + } + typeMap.RegisterType(types.path) + + return typeMap, types +} + +// testNodeComposite returns a representative node value with the requested ID. +func testNodeComposite(id int64) nodeComposite { + return nodeComposite{ + ID: id, + KindIDs: []int16{1, 2}, + Properties: map[string]any{"id": float64(id), "name": "node"}, + } +} + +// testEdgeComposite returns a representative edge value with the requested identity and endpoints. +func testEdgeComposite(id, startID, endID int64) edgeComposite { + return edgeComposite{ + ID: id, + StartID: startID, + EndID: endID, + KindID: 3, + Properties: map[string]any{"id": float64(id), "name": "edge"}, + } +} + +// TestOwnedCompositeCodecDecodeValue verifies scalar node, edge, and path values decode into owned concrete types. +func TestOwnedCompositeCodecDecodeValue(t *testing.T) { + typeMap, types := newCompositeCodecTestMap(t, true) + expectedNode := testNodeComposite(101) + expectedEdge := testEdgeComposite(201, 101, 102) + expectedPath := pathComposite{ + Nodes: []nodeComposite{expectedNode, testNodeComposite(102)}, + Edges: []edgeComposite{expectedEdge}, + } + + for _, testCase := range []struct { + // name identifies the composite type and wire-format subtest. + name string + + // format selects the pgx encoding format. + format int16 + + // dataType supplies the composite codec under test. + dataType *pgtype.Type + + // value is the concrete composite expected after decoding. + value any + }{ + { + name: "node/binary", + format: pgtype.BinaryFormatCode, + dataType: types.node, + value: expectedNode, + }, + { + name: "node/text", + format: pgtype.TextFormatCode, + dataType: types.node, + value: expectedNode, + }, + { + name: "edge/binary", + format: pgtype.BinaryFormatCode, + dataType: types.edge, + value: expectedEdge, + }, + { + name: "edge/text", + format: pgtype.TextFormatCode, + dataType: types.edge, + value: expectedEdge, + }, + { + name: "path/binary", + format: pgtype.BinaryFormatCode, + dataType: types.path, + value: expectedPath, + }, + { + name: "path/text", + format: pgtype.TextFormatCode, + dataType: types.path, + value: expectedPath, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + src, err := typeMap.Encode(testCase.dataType.OID, testCase.format, testCase.value, nil) + require.NoError(t, err) + + decoded, err := testCase.dataType.Codec.DecodeValue(typeMap, testCase.dataType.OID, testCase.format, src) + require.NoError(t, err) + require.IsType(t, testCase.value, decoded) + require.Equal(t, testCase.value, decoded) + + // pgx may reuse its receive buffer after Rows.Values returns. None of + // the concrete composite's slices, strings, or maps may alias it. + clear(src) + require.Equal(t, testCase.value, decoded) + }) + } +} + +// TestOwnedCompositeCodecPreservesExplicitScanAndNull verifies explicit scan targets and null composites keep pgx semantics. +func TestOwnedCompositeCodecPreservesExplicitScanAndNull(t *testing.T) { + typeMap, types := newCompositeCodecTestMap(t, true) + expected := testNodeComposite(101) + + for _, format := range []int16{pgtype.BinaryFormatCode, pgtype.TextFormatCode} { + src, err := typeMap.Encode(types.node.OID, format, expected, nil) + require.NoError(t, err) + + var decoded nodeComposite + require.NoError(t, typeMap.Scan(types.node.OID, format, src, &decoded)) + require.Equal(t, expected, decoded) + + nullValue, err := types.node.Codec.DecodeValue(typeMap, types.node.OID, format, nil) + require.NoError(t, err) + require.Nil(t, nullValue) + } +} + +// TestOwnedCompositeCodecFallsBackForNullInternalFields verifies nullable internal fields retain pgx's lossless map representation. +func TestOwnedCompositeCodecFallsBackForNullInternalFields(t *testing.T) { + typeMap, types := newCompositeCodecTestMap(t, true) + value := pgtype.CompositeFields{nil, []int16{1, 2}, map[string]any{"name": "nullable"}} + + for _, format := range []int16{pgtype.BinaryFormatCode, pgtype.TextFormatCode} { + src, err := typeMap.Encode(types.node.OID, format, value, nil) + require.NoError(t, err) + + decoded, err := types.node.Codec.DecodeValue(typeMap, types.node.OID, format, src) + require.NoError(t, err) + require.Equal(t, map[string]any{ + "id": nil, + "kind_ids": []any{int16(1), int16(2)}, + "properties": map[string]any{"name": "nullable"}, + }, decoded) + + arraySource := []pgtype.CompositeFields{value} + src, err = typeMap.Encode(types.nodeArray.OID, format, arraySource, nil) + require.NoError(t, err) + + decoded, err = types.nodeArray.Codec.DecodeValue(typeMap, types.nodeArray.OID, format, src) + require.NoError(t, err) + require.Equal(t, []any{map[string]any{ + "id": nil, + "kind_ids": []any{int16(1), int16(2)}, + "properties": map[string]any{"name": "nullable"}, + }}, decoded) + } +} + +// TestOwnedCompositeCodecSupportsArrays verifies non-null composite arrays decode directly into typed slices. +func TestOwnedCompositeCodecSupportsArrays(t *testing.T) { + typeMap, types := newCompositeCodecTestMap(t, true) + first := testNodeComposite(101) + second := testNodeComposite(102) + expectedNodes := []nodeComposite{first, second} + expectedEdges := []edgeComposite{ + testEdgeComposite(201, 101, 102), + testEdgeComposite(202, 102, 103), + } + + for _, format := range []int16{pgtype.BinaryFormatCode, pgtype.TextFormatCode} { + src, err := typeMap.Encode(types.nodeArray.OID, format, expectedNodes, nil) + require.NoError(t, err) + + decoded, err := types.nodeArray.Codec.DecodeValue(typeMap, types.nodeArray.OID, format, src) + require.NoError(t, err) + require.Equal(t, expectedNodes, decoded) + + var typedValues []nodeComposite + require.NoError(t, typeMap.Scan(types.nodeArray.OID, format, src, &typedValues)) + require.Equal(t, expectedNodes, typedValues) + + src, err = typeMap.Encode(types.edgeArray.OID, format, expectedEdges, nil) + require.NoError(t, err) + + decoded, err = types.edgeArray.Codec.DecodeValue(typeMap, types.edgeArray.OID, format, src) + require.NoError(t, err) + require.Equal(t, expectedEdges, decoded) + + var typedEdges []edgeComposite + require.NoError(t, typeMap.Scan(types.edgeArray.OID, format, src, &typedEdges)) + require.Equal(t, expectedEdges, typedEdges) + } +} + +// TestOwnedCompositeCodecArrayPreservesNullElements verifies arrays containing null composites retain a nullable representation. +func TestOwnedCompositeCodecArrayPreservesNullElements(t *testing.T) { + typeMap, types := newCompositeCodecTestMap(t, true) + first := testNodeComposite(101) + values := []*nodeComposite{&first, nil} + + for _, format := range []int16{pgtype.BinaryFormatCode, pgtype.TextFormatCode} { + src, err := typeMap.Encode(types.nodeArray.OID, format, values, nil) + require.NoError(t, err) + + decoded, err := types.nodeArray.Codec.DecodeValue(typeMap, types.nodeArray.OID, format, src) + require.NoError(t, err) + require.Equal(t, []any{first, nil}, decoded) + } +} + +// TestInstallOwnedCompositeCodec verifies supported definitions are wrapped and incompatible definitions are rejected. +func TestInstallOwnedCompositeCodec(t *testing.T) { + for _, testCase := range []struct { + // dataType identifies the supported composite definition to install. + dataType pgsql.DataType + + // value selects the concrete owned codec type expected for dataType. + value any + }{ + { + dataType: pgsql.NodeComposite, + value: nodeComposite{}, + }, + { + dataType: pgsql.EdgeComposite, + value: edgeComposite{}, + }, + { + dataType: pgsql.PathComposite, + value: pathComposite{}, + }, + } { + t.Run(testCase.dataType.String(), func(t *testing.T) { + definition := &pgtype.Type{ + Name: testCase.dataType.String(), + OID: testNodeCompositeOID, + Codec: &pgtype.CompositeCodec{}, + } + + require.NoError(t, installOwnedCompositeCodec(testCase.dataType, definition)) + require.NotEqual(t, reflect.TypeOf(&pgtype.CompositeCodec{}), reflect.TypeOf(definition.Codec)) + }) + } + + arrayDefinition := &pgtype.Type{Codec: &pgtype.ArrayCodec{}} + require.NoError(t, installOwnedCompositeCodec(pgsql.NodeCompositeArray, arrayDefinition)) + require.IsType(t, &ownedCompositeArrayCodec[nodeComposite]{}, arrayDefinition.Codec) + + invalidDefinition := &pgtype.Type{Codec: pgtype.TextCodec{}} + require.ErrorContains(t, installOwnedCompositeCodec(pgsql.NodeComposite, invalidDefinition), "*pgtype.CompositeCodec") +} + +// compositeCodecBenchmarkSink retains decoded values so benchmark work cannot be optimized away. +var compositeCodecBenchmarkSink any + +// benchmarkCompositeDecodeValue repeatedly decodes one encoded value through the selected codec implementation. +func benchmarkCompositeDecodeValue( + b *testing.B, + owned bool, + dataType func(compositeCodecTestTypes) *pgtype.Type, + value any, +) { + b.Helper() + + typeMap, types := newCompositeCodecTestMap(b, owned) + selectedType := dataType(types) + src, err := typeMap.Encode(selectedType.OID, pgtype.BinaryFormatCode, value, nil) + require.NoError(b, err) + + b.ReportAllocs() + b.ResetTimer() + for range b.N { + decoded, err := selectedType.Codec.DecodeValue(typeMap, selectedType.OID, pgtype.BinaryFormatCode, src) + if err != nil { + b.Fatal(err) + } + compositeCodecBenchmarkSink = decoded + } +} + +// BenchmarkNodeCompositeDecodeValue compares scalar node decoding through stock and owned codecs. +func BenchmarkNodeCompositeDecodeValue(b *testing.B) { + value := testNodeComposite(101) + for _, testCase := range []struct { + // name identifies whether the benchmark uses stock or owned decoding. + name string + + // owned enables the owned composite codec when true. + owned bool + }{ + { + name: "map", + owned: false, + }, + { + name: "owned", + owned: true, + }, + } { + b.Run(testCase.name, func(b *testing.B) { + benchmarkCompositeDecodeValue(b, testCase.owned, func(types compositeCodecTestTypes) *pgtype.Type { + return types.node + }, value) + }) + } +} + +// BenchmarkNodeCompositeArrayDecodeValue compares node-array decoding through stock and owned codecs. +func BenchmarkNodeCompositeArrayDecodeValue(b *testing.B) { + values := make([]nodeComposite, 128) + for idx := range values { + values[idx] = testNodeComposite(int64(idx + 1)) + } + + for _, testCase := range []struct { + // name identifies whether the benchmark uses stock or owned decoding. + name string + + // owned enables the owned composite codec when true. + owned bool + }{ + { + name: "map", + owned: false, + }, + { + name: "owned", + owned: true, + }, + } { + b.Run(testCase.name, func(b *testing.B) { + benchmarkCompositeDecodeValue(b, testCase.owned, func(types compositeCodecTestTypes) *pgtype.Type { + return types.nodeArray + }, values) + }) + } +} + +// BenchmarkPathCompositeDecodeValue compares path decoding through stock and owned codecs. +func BenchmarkPathCompositeDecodeValue(b *testing.B) { + value := pathComposite{ + Nodes: make([]nodeComposite, 32), + Edges: make([]edgeComposite, 31), + } + for idx := range value.Nodes { + value.Nodes[idx] = testNodeComposite(int64(idx + 1)) + } + for idx := range value.Edges { + value.Edges[idx] = testEdgeComposite(int64(idx+1), int64(idx+1), int64(idx+2)) + } + + for _, testCase := range []struct { + // name identifies whether the benchmark uses stock or owned decoding. + name string + + // owned enables the owned composite codec when true. + owned bool + }{ + { + name: "map", + owned: false, + }, + { + name: "owned", + owned: true, + }, + } { + b.Run(testCase.name, func(b *testing.B) { + benchmarkCompositeDecodeValue(b, testCase.owned, func(types compositeCodecTestTypes) *pgtype.Type { + return types.path + }, value) + }) + } +} diff --git a/drivers/pg/driver.go b/drivers/pg/driver.go index 36833741..1d258ed8 100644 --- a/drivers/pg/driver.go +++ b/drivers/pg/driver.go @@ -3,6 +3,7 @@ package pg import ( "context" "fmt" + "strings" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" @@ -11,11 +12,15 @@ import ( ) var ( - batchWriteSize = defaultBatchWriteSize + // batchWriteSize is the process-wide flush threshold used by new batch operations. + batchWriteSize = defaultBatchWriteSize + + // readOnlyTxOptions configures transactions that must not mutate PostgreSQL state. readOnlyTxOptions = pgx.TxOptions{ AccessMode: pgx.ReadOnly, } + // readWriteTxOptions configures transactions that may mutate PostgreSQL state. readWriteTxOptions = pgx.TxOptions{ AccessMode: pgx.ReadWrite, } @@ -26,6 +31,8 @@ type Config struct { QueryExecMode pgx.QueryExecMode QueryResultFormats pgx.QueryResultFormats BatchWriteSize int + + initializeTraversalRuntimeAttestation bool } func OptionSetQueryExecMode(queryExecMode pgx.QueryExecMode) graph.TransactionOption { @@ -36,6 +43,37 @@ func OptionSetQueryExecMode(queryExecMode pgx.QueryExecMode) graph.TransactionOp } } +// OptionSetTransactionIsolation requests an explicit PostgreSQL transaction at +// the supplied isolation level. B traversal candidates are selected only for +// REPEATABLE READ or SERIALIZABLE transactions. The driver prepares the +// production shortest-path and all-shortest-path temporary workspaces on the +// acquired session before beginning either stable-snapshot transaction and +// uses PostgreSQL READ WRITE access so those session-local tables can reset. +func OptionSetTransactionIsolation(isolation pgx.TxIsoLevel) graph.TransactionOption { + return func(config *graph.TransactionConfig) { + if pgCfg, typeOK := config.DriverConfig.(*Config); typeOK { + pgCfg.Options.IsoLevel = isolation + if stableSnapshotIsolation(isolation) { + pgCfg.Options.AccessMode = pgx.ReadWrite + } + } + } +} + +// OptionInitializeTraversalRuntimeAttestation prepares the acquired PostgreSQL +// session before an explicit read-only transaction begins. Callers that arm +// traversal runtime receipts inside a graph transaction need this option +// because PostgreSQL forbids creating the temporary workspace after BEGIN READ +// ONLY. GraphBench normally pins and prepares its session before the timed +// transaction instead. +func OptionInitializeTraversalRuntimeAttestation() graph.TransactionOption { + return func(config *graph.TransactionConfig) { + if pgCfg, typeOK := config.DriverConfig.(*Config); typeOK { + pgCfg.initializeTraversalRuntimeAttestation = true + } + } +} + type Driver struct { pool *pgxpool.Pool *SchemaManager @@ -94,11 +132,35 @@ func (s *Driver) BatchOperation(ctx context.Context, batchDelegate graph.BatchDe } } +// Close stops the driver's query caches before releasing its PostgreSQL pool. func (s *Driver) Close(ctx context.Context) error { + if s.SchemaManager != nil { + s.SchemaManager.parseCache.Close() + s.SchemaManager.translationCache.Close() + } s.pool.Close() return nil } +// TranslationCacheStats returns query-text-free counters for this driver's +// bounded Cypher-to-SQL translation cache. +func (s *Driver) TranslationCacheStats() TranslationCacheStats { + if s == nil || s.SchemaManager == nil { + return TranslationCacheStats{} + } + return s.SchemaManager.translationCache.Stats() +} + +// ParseCacheStats returns query-text-free counters for this driver's bounded Cypher parse cache. +func (s *Driver) ParseCacheStats() ParseCacheStats { + if s == nil || s.SchemaManager == nil { + return ParseCacheStats{} + } + return s.SchemaManager.parseCache.Stats() +} + +// renderConfig applies transaction options to PostgreSQL defaults and rejects +// a driver configuration of the wrong concrete type. func renderConfig(batchWriteSize int, pgxOptions pgx.TxOptions, userOptions []graph.TransactionOption) (*Config, error) { graphCfg := graph.TransactionConfig{ DriverConfig: &Config{ @@ -210,3 +272,124 @@ func (s *Driver) WipeGraph(ctx context.Context, retain graph.TransactionDelegate return nil }) } + +// resolveKindIDs maps kinds to their integer IDs, refreshing the schema cache once on a miss. It returns the resolved +// IDs alongside any kinds that remain undefined after the refresh, so callers can decide whether an unresolved kind is +// a tolerable no-op (include predicates) or must fail closed (exclude predicates). +func (s *Driver) resolveKindIDs(ctx context.Context, kinds graph.Kinds) ([]int16, graph.Kinds, error) { + if len(kinds) == 0 { + return nil, nil, nil + } + + s.lock.RLock() + if kindIDs, missingKinds := s.mapKinds(kinds); len(missingKinds) == 0 { + s.lock.RUnlock() + return kindIDs, nil, nil + } + s.lock.RUnlock() + + s.lock.Lock() + defer s.lock.Unlock() + + if err := s.Fetch(ctx); err != nil { + return nil, nil, err + } + + kindIDs, missingKinds := s.mapKinds(kinds) + return kindIDs, missingKinds, nil +} + +// DeleteNodesByKinds performs a server-side, set-based delete of nodes using the kind_ids GIN index instead of +// streaming node IDs through the application. A node is deleted when its kind_ids overlap includeAny (or, when +// includeAny is empty, for every node) and do not overlap excludeAny. Deleting nodes fires the delete_node_edges +// trigger, cascading the attached edge deletes. +// +// includeAny is mapped to kind IDs tolerantly: include kinds that are not defined in the database map to no IDs and +// therefore match no nodes, so a request that targets only undefined kinds is a safe no-op rather than an accidental +// full delete. excludeAny is mapped fail-closed: if any exclude kind is undefined the delete is refused, because +// silently dropping an exclusion would widen the delete and could remove protected nodes (e.g. an unresolved +// MigrationData would turn a guarded wipe into an unguarded delete from node). +func (s *Driver) DeleteNodesByKinds(ctx context.Context, includeAny graph.Kinds, excludeAny graph.Kinds) error { + includeIDs, _, err := s.resolveKindIDs(ctx, includeAny) + if err != nil { + return err + } + + excludeIDs, excludeMissing, err := s.resolveKindIDs(ctx, excludeAny) + if err != nil { + return err + } + if len(excludeMissing) > 0 { + return fmt.Errorf("cannot exclude undefined kinds from node delete: %v", excludeMissing) + } + + statement, arguments := buildNodeDeleteStatement(len(includeAny) > 0, includeIDs, excludeIDs) + + return s.execDelete(ctx, "node", statement, arguments...) +} + +// buildNodeDeleteStatement renders the node delete statement and its positional arguments for the given resolved kind +// IDs. The include predicate is emitted whenever an include filter was requested (includeRequested), even if includeIDs +// is empty, so that targeting only undefined kinds matches no nodes. The exclude predicate is emitted only when +// excludeIDs is non-empty, so an unresolved exclusion can never widen the delete into an unguarded wipe. +func buildNodeDeleteStatement(includeRequested bool, includeIDs []int16, excludeIDs []int16) (string, []any) { + var ( + predicates []string + arguments []any + ) + + if includeRequested { + arguments = append(arguments, includeIDs) + predicates = append(predicates, fmt.Sprintf("kind_ids operator (pg_catalog.&&) $%d::int2[]", len(arguments))) + } + + if len(excludeIDs) > 0 { + arguments = append(arguments, excludeIDs) + predicates = append(predicates, fmt.Sprintf("not (kind_ids operator (pg_catalog.&&) $%d::int2[])", len(arguments))) + } + + statement := "delete from node" + if len(predicates) > 0 { + statement += " where " + strings.Join(predicates, " and ") + } + + return statement, arguments +} + +// DeleteRelationshipsByKinds performs a server-side, set-based delete of relationships whose kind_id matches any of +// the given kinds, using the edge_kind_id_id_start_id_end_id_index covering index instead of streaming relationship +// IDs through the application. +// +// kinds are mapped to kind IDs tolerantly: kinds that are not defined in the database map to no IDs. An empty kinds +// argument, or one that maps entirely to undefined kinds, deletes nothing rather than every relationship. +func (s *Driver) DeleteRelationshipsByKinds(ctx context.Context, kinds graph.Kinds) error { + if len(kinds) == 0 { + return nil + } + + kindIDs, _, err := s.resolveKindIDs(ctx, kinds) + if err != nil { + return err + } + + const statement = "delete from edge where kind_id = any($1::int2[])" + + return s.execDelete(ctx, "relationship", statement, kindIDs) +} + +// execDelete acquires a pooled connection and runs a delete statement, wrapping acquisition and execution errors. label +// names the delete for the acquire error message; statement and arguments are passed through unchanged so each caller +// preserves its own SQL, positional arguments, and statement error wrapping. +func (s *Driver) execDelete(ctx context.Context, label, statement string, arguments ...any) error { + conn, err := s.pool.Acquire(ctx) + if err != nil { + return fmt.Errorf("acquire connection for %s delete: %w", label, err) + } + defer conn.Release() + + if _, err := conn.Exec(ctx, statement, arguments...); err != nil { + return fmt.Errorf("%s: %w", statement, err) + } + + return nil +} diff --git a/drivers/pg/driver_test.go b/drivers/pg/driver_test.go new file mode 100644 index 00000000..8c860b94 --- /dev/null +++ b/drivers/pg/driver_test.go @@ -0,0 +1,129 @@ +package pg + +import ( + "context" + "testing" + + "github.com/jackc/pgx/v5" + "github.com/specterops/dawgs/graph" + "github.com/stretchr/testify/require" +) + +// TestBuildNodeDeleteStatement covers the statement/argument construction for DeleteNodesByKinds, including the guard +// that prevents an unresolved exclusion from widening the delete into an unguarded wipe. +func TestBuildNodeDeleteStatement(t *testing.T) { + var ( + includeIDs = []int16{1, 2} + excludeIDs = []int16{9} + ) + + t.Run("no filters deletes all nodes", func(t *testing.T) { + statement, arguments := buildNodeDeleteStatement(false, nil, nil) + require.Equal(t, "delete from node", statement) + require.Empty(t, arguments) + }) + + t.Run("include only", func(t *testing.T) { + statement, arguments := buildNodeDeleteStatement(true, includeIDs, nil) + require.Equal(t, "delete from node where kind_ids operator (pg_catalog.&&) $1::int2[]", statement) + require.Equal(t, []any{includeIDs}, arguments) + }) + + t.Run("exclude only", func(t *testing.T) { + statement, arguments := buildNodeDeleteStatement(false, nil, excludeIDs) + require.Equal(t, "delete from node where not (kind_ids operator (pg_catalog.&&) $1::int2[])", statement) + require.Equal(t, []any{excludeIDs}, arguments) + }) + + t.Run("include and exclude are positionally numbered", func(t *testing.T) { + statement, arguments := buildNodeDeleteStatement(true, includeIDs, excludeIDs) + require.Equal(t, "delete from node where kind_ids operator (pg_catalog.&&) $1::int2[] and not (kind_ids operator (pg_catalog.&&) $2::int2[])", statement) + require.Equal(t, []any{includeIDs, excludeIDs}, arguments) + }) + + t.Run("empty excludeIDs cannot widen the delete", func(t *testing.T) { + // A requested-but-unresolved exclusion must never emit a not(... && '{}') clause that matches every row. + statement, arguments := buildNodeDeleteStatement(false, nil, []int16{}) + require.Equal(t, "delete from node", statement) + require.Empty(t, arguments) + + statement, arguments = buildNodeDeleteStatement(true, includeIDs, []int16{}) + require.Equal(t, "delete from node where kind_ids operator (pg_catalog.&&) $1::int2[]", statement) + require.Equal(t, []any{includeIDs}, arguments) + }) + + t.Run("include requested with empty IDs is a tolerant no-op predicate", func(t *testing.T) { + statement, arguments := buildNodeDeleteStatement(true, []int16{}, nil) + require.Equal(t, "delete from node where kind_ids operator (pg_catalog.&&) $1::int2[]", statement) + require.Equal(t, []any{[]int16{}}, arguments) + }) +} + +// TestResolveKindIDsDefinedFastPath exercises the cache-hit path of resolveKindIDs, which resolves defined kinds +// without touching the database. The cache-miss/refresh and fail-closed exclude paths require a live pool and are +// covered by the integration suite. +func TestResolveKindIDsDefinedFastPath(t *testing.T) { + ctx := context.Background() + + driver := &Driver{ + SchemaManager: NewSchemaManager(nil, 0), + } + + var ( + userKind = graph.StringKind("User") + groupKind = graph.StringKind("Group") + ) + driver.kindsByID[userKind] = 1 + driver.kindsByID[groupKind] = 2 + + t.Run("defined kinds resolve with no missing", func(t *testing.T) { + ids, missing, err := driver.resolveKindIDs(ctx, graph.Kinds{userKind, groupKind}) + require.NoError(t, err) + require.Empty(t, missing) + require.ElementsMatch(t, []int16{1, 2}, ids) + }) + + t.Run("empty kinds short-circuit", func(t *testing.T) { + ids, missing, err := driver.resolveKindIDs(ctx, nil) + require.NoError(t, err) + require.Nil(t, ids) + require.Nil(t, missing) + }) +} + +// TestDeleteRelationshipsByKindsEmptyIsNoop verifies that an empty kinds request returns before acquiring a +// connection, so it is a safe no-op rather than deleting every relationship. +func TestDeleteRelationshipsByKindsEmptyIsNoop(t *testing.T) { + ctx := context.Background() + + driver := &Driver{SchemaManager: NewSchemaManager(nil, 0)} + + require.NoError(t, driver.DeleteRelationshipsByKinds(ctx, nil)) + require.NoError(t, driver.DeleteRelationshipsByKinds(ctx, graph.Kinds{})) +} + +func TestOptionInitializeTraversalRuntimeAttestation(t *testing.T) { + cfg, err := renderConfig(defaultBatchWriteSize, readOnlyTxOptions, []graph.TransactionOption{ + OptionInitializeTraversalRuntimeAttestation(), + }) + require.NoError(t, err) + require.True(t, cfg.initializeTraversalRuntimeAttestation) +} + +func TestStableSnapshotIsolation(t *testing.T) { + require.False(t, stableSnapshotIsolation("")) + require.False(t, stableSnapshotIsolation(pgx.ReadCommitted)) + require.True(t, stableSnapshotIsolation(pgx.RepeatableRead)) + require.True(t, stableSnapshotIsolation(pgx.Serializable)) +} + +func TestOptionSetStableSnapshotIsolationAllowsTemporaryWorkspaceWrites(t *testing.T) { + for _, isolation := range []pgx.TxIsoLevel{pgx.RepeatableRead, pgx.Serializable} { + cfg, err := renderConfig(defaultBatchWriteSize, readOnlyTxOptions, []graph.TransactionOption{ + OptionSetTransactionIsolation(isolation), + }) + require.NoError(t, err) + require.Equal(t, isolation, cfg.Options.IsoLevel) + require.Equal(t, pgx.ReadWrite, cfg.Options.AccessMode) + } +} diff --git a/drivers/pg/manager.go b/drivers/pg/manager.go index 4ce56419..f69f67f7 100644 --- a/drivers/pg/manager.go +++ b/drivers/pg/manager.go @@ -32,20 +32,51 @@ func KindMapperFromGraphDatabase(graphDB graph.Database) (KindMapper, error) { } } +// SchemaManager coordinates graph and kind metadata with the query caches that depend on that schema state. type SchemaManager struct { - defaultGraph model.Graph - pool *pgxpool.Pool - hasDefaultGraph bool - graphs map[string]model.Graph - kindsByID map[graph.Kind]int16 - kindIDsByKind map[int16]graph.Kind - lock *sync.RWMutex + // defaultGraph caches the first graph selected as the schema default. + defaultGraph model.Graph + + // pool supplies PostgreSQL connections for schema operations. + pool *pgxpool.Pool + + // parseCache retains immutable Cypher ASTs keyed by normalized query text. + parseCache *cypherParseCache + + // translationCache retains parameter-rebindable SQL translations by graph and parameter shape. + translationCache *cypherTranslationCache + + // hasDefaultGraph distinguishes a cached default graph from the zero-value graph model. + hasDefaultGraph bool + + // graphs indexes asserted database graph models by schema name. + graphs map[string]model.Graph + + // kindsByID maps graph kind names to their PostgreSQL int2 identifiers. + kindsByID map[graph.Kind]int16 + + // kindIDsByKind maps PostgreSQL int2 identifiers back to graph kind names. + kindIDsByKind map[int16]graph.Kind + + // lock protects cached graph and kind metadata from concurrent access. + lock *sync.RWMutex + + // graphQueryMemoryLimit caps memory available to a graph query transaction. graphQueryMemoryLimit size.Size + + // traversalPolicyLock protects the versioned default-off production canary policy. + traversalPolicyLock sync.RWMutex + + // traversalPolicy is copied on reads so callers cannot mutate live selection state. + traversalPolicy TraversalPolicy } +// NewSchemaManager creates an empty metadata manager with bounded parse and translation caches for pool. func NewSchemaManager(pool *pgxpool.Pool, graphQueryMemoryLimit size.Size) *SchemaManager { return &SchemaManager{ pool: pool, + parseCache: newCypherParseCache(defaultCypherParseCacheEntries), + translationCache: newCypherTranslationCache(defaultCypherTranslationCacheEntries), hasDefaultGraph: false, graphs: map[string]model.Graph{}, kindsByID: map[graph.Kind]int16{}, @@ -77,6 +108,7 @@ func (s *SchemaManager) WriteTransaction(ctx context.Context, txDelegate graph.T } } +// fetch replaces both in-memory kind indexes with the kinds visible through tx. func (s *SchemaManager) fetch(tx graph.Transaction) error { if kinds, err := query.On(tx).SelectKinds(); err != nil { return err @@ -97,12 +129,15 @@ func (s *SchemaManager) GetKindIDsByKind() map[int16]graph.Kind { return s.kindIDsByKind } +// Fetch refreshes both in-memory kind indexes from a read transaction against the current schema. func (s *SchemaManager) Fetch(ctx context.Context) error { - return s.WriteTransaction(ctx, func(tx graph.Transaction) error { + return s.ReadTransaction(ctx, func(tx graph.Transaction) error { return s.fetch(tx) }, OptionSetQueryExecMode(pgx.QueryExecModeSimpleProtocol)) } +// defineKinds inserts any missing kinds and records their database IDs in both +// in-memory indexes. func (s *SchemaManager) defineKinds(tx graph.Transaction, kinds graph.Kinds) error { for _, kind := range kinds { if kindID, err := query.On(tx).InsertOrGetKind(kind); err != nil { @@ -116,6 +151,7 @@ func (s *SchemaManager) defineKinds(tx graph.Transaction, kinds graph.Kinds) err return nil } +// mapKinds partitions semantic kinds into cached database IDs and unresolved kinds without refreshing the cache. func (s *SchemaManager) mapKinds(kinds graph.Kinds) ([]int16, graph.Kinds) { var ( missingKinds = make(graph.Kinds, 0, len(kinds)) @@ -185,17 +221,48 @@ func (s *SchemaManager) ReadTransaction(ctx context.Context, txDelegate graph.Tr return err } else { defer conn.Release() + if stableSnapshotIsolation(cfg.Options.IsoLevel) { + if err := initializeStableSnapshotTraversalWorkspaces(ctx, conn); err != nil { + return err + } + } + if cfg.initializeTraversalRuntimeAttestation { + if _, err := conn.Exec(ctx, "select public.ensure_traversal_runtime_attestation_workspace_v1()"); err != nil { + return fmt.Errorf("initialize traversal runtime attestation workspace: %w", err) + } + } + allocateTransaction := cfg.Options.IsoLevel != "" + wrapper, err := newTransactionWrapper(ctx, conn, s, cfg, allocateTransaction) + if err != nil { + return err + } + defer wrapper.Close() + if err := txDelegate(wrapper); err != nil { + return err + } + if allocateTransaction { + return wrapper.Commit() + } + return nil + } +} - return txDelegate(&transaction{ - schemaManager: s, - queryExecMode: cfg.QueryExecMode, - ctx: ctx, - conn: conn, - targetSchemaSet: false, - }) +func stableSnapshotIsolation(isolation pgx.TxIsoLevel) bool { + return isolation == pgx.RepeatableRead || isolation == pgx.Serializable +} + +func initializeStableSnapshotTraversalWorkspaces(ctx context.Context, conn *pgxpool.Conn) error { + const initializeSQL = `select + public.ensure_shortest_dag_workspace(), + public.ensure_bidirectional_shortest_path_workspace(), + public.ensure_bidirectional_all_shortest_path_workspace()` + if _, err := conn.Exec(ctx, initializeSQL); err != nil { + return fmt.Errorf("initialize stable-snapshot traversal workspaces: %w", err) } + return nil } +// mapKindIDs partitions database kind IDs into cached semantic kinds and unresolved IDs without refreshing the cache. func (s *SchemaManager) mapKindIDs(kindIDs []int16) (graph.Kinds, []int16) { var ( missingIDs = make([]int16, 0, len(kindIDs)) @@ -244,6 +311,7 @@ func (s *SchemaManager) MapKindIDs(ctx context.Context, kindIDs []int16) (graph. } } +// assertKinds defines any missing kinds while holding the write lock and returns IDs from the refreshed in-memory mapping. func (s *SchemaManager) assertKinds(ctx context.Context, kinds graph.Kinds) ([]int16, error) { // Acquire a write-lock and release on-exit s.lock.Lock() @@ -279,6 +347,7 @@ func (s *SchemaManager) AssertKinds(ctx context.Context, kinds graph.Kinds) ([]i return s.assertKinds(ctx, kinds) } +// setDefaultGraph caches the first successfully resolved default graph and ignores later attempts to replace it. func (s *SchemaManager) setDefaultGraph(defaultGraph model.Graph, schema graph.Graph) { s.lock.Lock() defer s.lock.Unlock() @@ -325,6 +394,7 @@ func (s *SchemaManager) DefaultGraph() (model.Graph, bool) { return s.defaultGraph, s.hasDefaultGraph } +// assertGraph creates or validates schema's graph definition in tx and records the resulting database model. func (s *SchemaManager) assertGraph(tx graph.Transaction, schema graph.Graph) (model.Graph, error) { var assertedGraph model.Graph @@ -376,6 +446,7 @@ func (s *SchemaManager) AssertGraph(tx graph.Transaction, schema graph.Graph) (m return s.assertGraph(tx, schema) } +// assertSchema creates schema storage and defines every node and relationship kind required by its graphs. func (s *SchemaManager) assertSchema(tx graph.Transaction, schema graph.Schema) error { if err := query.On(tx).CreateSchema(); err != nil { return err diff --git a/drivers/pg/mapper.go b/drivers/pg/mapper.go index 0195f2f4..c7fbc4ad 100644 --- a/drivers/pg/mapper.go +++ b/drivers/pg/mapper.go @@ -7,10 +7,14 @@ import ( ) const ( + // minKindID is the smallest integer representable by PostgreSQL's int2 kind column. minKindID = -1 << 15 + + // maxKindID is the largest integer representable by PostgreSQL's int2 kind column. maxKindID = 1<<15 - 1 ) +// mapKindIDs resolves database kind IDs and reports false when the mapper rejects any ID. func mapKindIDs(ctx context.Context, kindMapper KindMapper, kindIDs []int16) (graph.Kinds, bool) { if len(kindIDs) == 0 { return graph.Kinds{}, true @@ -23,6 +27,7 @@ func mapKindIDs(ctx context.Context, kindMapper KindMapper, kindIDs []int16) (gr return nil, false } +// asKindID converts supported integer representations to int16 without truncation. func asKindID(value any) (int16, bool) { switch typedValue := value.(type) { case int: @@ -78,6 +83,7 @@ func asKindID(value any) (int16, bool) { } } +// mapAnyKinds maps a homogeneous list of kind names or numeric IDs and rejects mixed or unsupported values. func mapAnyKinds(ctx context.Context, kindMapper KindMapper, values []any) (graph.Kinds, bool) { if len(values) == 0 { return graph.Kinds{}, true @@ -113,6 +119,7 @@ func mapAnyKinds(ctx context.Context, kindMapper KindMapper, values []any) (grap return mapKindIDs(ctx, kindMapper, kindIDs) } +// mapKinds accepts the slice representations emitted by pgx for graph kind arrays. func mapKinds(ctx context.Context, kindMapper KindMapper, untypedValue any) (graph.Kinds, bool) { switch typedValue := untypedValue.(type) { case []any: @@ -128,6 +135,7 @@ func mapKinds(ctx context.Context, kindMapper KindMapper, untypedValue any) (gra return nil, false } +// mapNodeCompositeArray converts a raw PostgreSQL composite array into graph nodes with resolved kinds. func mapNodeCompositeArray(ctx context.Context, kindMapper KindMapper, value any) ([]*graph.Node, bool) { nodeComposites, err := nodeCompositesFromRaw(value) if err != nil { @@ -147,6 +155,7 @@ func mapNodeCompositeArray(ctx context.Context, kindMapper KindMapper, value any return nodes, true } +// mapEdgeCompositeArray converts a raw PostgreSQL composite array into graph relationships with resolved kinds. func mapEdgeCompositeArray(ctx context.Context, kindMapper KindMapper, value any) ([]*graph.Relationship, bool) { edgeComposites, err := edgeCompositesFromRaw(value) if err != nil { @@ -166,17 +175,14 @@ func mapEdgeCompositeArray(ctx context.Context, kindMapper KindMapper, value any return relationships, true } +// newMapFunc returns the result mapper that recognizes graph composites, arrays, paths, and kind slices. func newMapFunc(ctx context.Context, kindMapper KindMapper) graph.MapFunc { return func(value, target any) bool { switch typedTarget := target.(type) { case *graph.Relationship: - if compositeMap, typeOK := value.(map[string]any); typeOK { - edge := edgeComposite{} - - if edge.TryMap(compositeMap) { - if err := edge.ToRelationship(ctx, kindMapper, typedTarget); err == nil { - return true - } + if edge, typeOK := edgeCompositeFromRaw(value); typeOK { + if err := edge.ToRelationship(ctx, kindMapper, typedTarget); err == nil { + return true } } @@ -200,13 +206,9 @@ func newMapFunc(ctx context.Context, kindMapper KindMapper) graph.MapFunc { } case *graph.Node: - if compositeMap, typeOK := value.(map[string]any); typeOK { - node := nodeComposite{} - - if node.TryMap(compositeMap) { - if err := node.ToNode(ctx, kindMapper, typedTarget); err == nil { - return true - } + if node, typeOK := nodeCompositeFromRaw(value); typeOK { + if err := node.ToNode(ctx, kindMapper, typedTarget); err == nil { + return true } } @@ -230,13 +232,9 @@ func newMapFunc(ctx context.Context, kindMapper KindMapper) graph.MapFunc { } case *graph.Path: - if compositeMap, typeOK := value.(map[string]any); typeOK { - path := pathComposite{} - - if path.TryMap(compositeMap) { - if err := path.ToPath(ctx, kindMapper, typedTarget); err == nil { - return true - } + if path, typeOK := pathCompositeFromRaw(value); typeOK { + if err := path.ToPath(ctx, kindMapper, typedTarget); err == nil { + return true } } diff --git a/drivers/pg/mapper_test.go b/drivers/pg/mapper_test.go index 3145327c..76356eda 100644 --- a/drivers/pg/mapper_test.go +++ b/drivers/pg/mapper_test.go @@ -84,6 +84,7 @@ func TestValueMapperMapsStringArraysByTargetType(t *testing.T) { require.Equal(t, []string{"Alice", "Bob"}, stringTarget) } +// TestValueMapperMapsCompositeArrays verifies typed node and relationship arrays preserve order and graph metadata. func TestValueMapperMapsCompositeArrays(t *testing.T) { ctx := context.Background() mapper := pgutil.NewInMemoryKindMapper() @@ -114,6 +115,28 @@ func TestValueMapperMapsCompositeArrays(t *testing.T) { require.Equal(t, "Alice", nodes[0].Properties.Get("name").Any()) }) + t.Run("typed node array preserves order", func(t *testing.T) { + rawNodes := []any{ + nodeComposite{ + ID: 1, + KindIDs: []int16{userKindID}, + Properties: map[string]any{"name": "Alice"}, + }, + nodeComposite{ + ID: 2, + KindIDs: []int16{userKindID}, + Properties: map[string]any{"name": "Bob"}, + }, + } + + var nodes []*graph.Node + require.True(t, valueMapper.Map(rawNodes, &nodes)) + require.Len(t, nodes, 2) + require.Equal(t, graph.ID(1), nodes[0].ID) + require.Equal(t, graph.ID(2), nodes[1].ID) + require.Equal(t, "Alice", nodes[0].Properties.Get("name").Any()) + }) + t.Run("relationship array preserves order", func(t *testing.T) { rawRelationships := []any{ map[string]any{ @@ -139,6 +162,73 @@ func TestValueMapperMapsCompositeArrays(t *testing.T) { require.Equal(t, graph.ID(11), relationships[1].ID) require.Equal(t, graph.StringKind("MemberOf"), relationships[0].Kind) }) + + t.Run("typed relationship array preserves order", func(t *testing.T) { + rawRelationships := []edgeComposite{ + { + ID: 10, + StartID: 1, + EndID: 2, + KindID: memberOfKindID, + Properties: map[string]any{"ordinal": int64(1)}, + }, + { + ID: 11, + StartID: 2, + EndID: 3, + KindID: memberOfKindID, + Properties: map[string]any{"ordinal": int64(2)}, + }, + } + + var relationships []graph.Relationship + require.True(t, valueMapper.Map(rawRelationships, &relationships)) + require.Len(t, relationships, 2) + require.Equal(t, graph.ID(10), relationships[0].ID) + require.Equal(t, graph.ID(11), relationships[1].ID) + }) +} + +// TestValueMapperMapsTypedComposites verifies owned node, edge, and path composites map to graph-native values. +func TestValueMapperMapsTypedComposites(t *testing.T) { + ctx := context.Background() + mapper := pgutil.NewInMemoryKindMapper() + userKindID := mapper.Put(graph.StringKind("User")) + memberOfKindID := mapper.Put(graph.StringKind("MemberOf")) + valueMapper := NewValueMapper(ctx, mapper) + + rawNode := nodeComposite{ + ID: 1, + KindIDs: []int16{userKindID}, + Properties: map[string]any{"name": "Alice"}, + } + rawEdge := edgeComposite{ + ID: 10, + StartID: 1, + EndID: 2, + KindID: memberOfKindID, + Properties: map[string]any{"ordinal": int64(1)}, + } + + var node graph.Node + require.True(t, valueMapper.Map(rawNode, &node)) + require.Equal(t, graph.ID(1), node.ID) + require.Equal(t, graph.StringKind("User"), node.Kinds[0]) + + var relationship graph.Relationship + require.True(t, valueMapper.Map(&rawEdge, &relationship)) + require.Equal(t, graph.ID(10), relationship.ID) + require.Equal(t, graph.StringKind("MemberOf"), relationship.Kind) + + var path graph.Path + require.True(t, valueMapper.Map(pathComposite{ + Nodes: []nodeComposite{rawNode}, + Edges: []edgeComposite{rawEdge}, + }, &path)) + require.Len(t, path.Nodes, 1) + require.Len(t, path.Edges, 1) + require.Equal(t, graph.ID(1), path.Nodes[0].ID) + require.Equal(t, graph.ID(10), path.Edges[0].ID) } func TestAsKindID(t *testing.T) { diff --git a/drivers/pg/optimize.go b/drivers/pg/optimize.go index 8da0cfb8..4b1efe55 100644 --- a/drivers/pg/optimize.go +++ b/drivers/pg/optimize.go @@ -10,60 +10,13 @@ import ( "github.com/jackc/pgx/v5/pgconn" ) -// deadTupleThreshold is the minimum fraction of dead tuples a partitioned -// parent must accumulate across its partitions before OptimizeStorage will -// vacuum it. -const deadTupleThreshold = 0.1 - -// Sum n_dead_tup and n_live_tup across every leaf partition of the parent; -const optimizeStorageStatsQuery = ` - SELECT - COALESCE(SUM(stat.n_dead_tup), 0), - COALESCE(SUM(stat.n_live_tup), 0) - FROM pg_partition_tree($1::regclass) tree - LEFT JOIN pg_stat_user_tables stat ON stat.relid = tree.relid - WHERE tree.isleaf -` - type optimizeStorageConn interface { Exec(ctx context.Context, sql string, arguments ...any) (pgconn.CommandTag, error) - QueryRow(ctx context.Context, sql string, arguments ...any) pgx.Row } +// optimizeStorage vacuums and analyzes the partitioned node and edge parents using a simple-protocol statement. func optimizeStorage(ctx context.Context, conn optimizeStorageConn) error { - var targets []string - for _, table := range []string{"node", "edge"} { - var dead, live int64 - if err := conn.QueryRow(ctx, optimizeStorageStatsQuery, table).Scan(&dead, &live); err != nil { - return fmt.Errorf("query dead tuple stats for %s: %w", table, err) - } - - total := dead + live - var deadTupleRatio float64 - if total > 0 { - deadTupleRatio = float64(dead) / float64(total) - } - - slog.InfoContext(ctx, "Queried PostgreSQL table storage statistics", - slog.String("table", table), - slog.Int64("dead_tuples", dead), - slog.Int64("live_tuples", live), - slog.Int64("total_tuples", total), - slog.Float64("dead_tuple_ratio", deadTupleRatio), - slog.Float64("dead_tuple_threshold", deadTupleThreshold), - ) - - if total == 0 { - continue - } - if deadTupleRatio >= deadTupleThreshold { - targets = append(targets, table) - } - } - - if len(targets) == 0 { - return nil - } + targets := []string{"node", "edge"} // Targeting the partitioned parents cascades to every partition. stmt := "VACUUM (ANALYZE) " + strings.Join(targets, ", ") diff --git a/drivers/pg/optimize_test.go b/drivers/pg/optimize_test.go index 8fd0f6c5..c5a7677c 100644 --- a/drivers/pg/optimize_test.go +++ b/drivers/pg/optimize_test.go @@ -2,7 +2,6 @@ package pg import ( "context" - "errors" "testing" "github.com/jackc/pgx/v5" @@ -10,68 +9,17 @@ import ( "github.com/stretchr/testify/require" ) +// TestOptimizeStorage verifies optimization vacuums both graph storage parents in one statement. func TestOptimizeStorage(t *testing.T) { - t.Run("skips vacuum when dead tuple ratios are below threshold", func(t *testing.T) { + t.Run("always vacuums node and edge", func(t *testing.T) { ctx := context.Background() conn := newOptimizeStorageMockConn(t) - expectOptimizeStorageStats(conn, "node", 9, 91) - expectOptimizeStorageStats(conn, "edge", 0, 0) - - require.NoError(t, optimizeStorage(ctx, conn)) - require.NoError(t, conn.ExpectationsWereMet()) - }) - - t.Run("vacuums node only", func(t *testing.T) { - ctx := context.Background() - conn := newOptimizeStorageMockConn(t) - - expectOptimizeStorageStats(conn, "node", 10, 90) - expectOptimizeStorageStats(conn, "edge", 9, 91) - expectOptimizeStorageVacuum(conn, "VACUUM (ANALYZE) node") - - require.NoError(t, optimizeStorage(ctx, conn)) - require.NoError(t, conn.ExpectationsWereMet()) - }) - - t.Run("vacuums edge only", func(t *testing.T) { - ctx := context.Background() - conn := newOptimizeStorageMockConn(t) - - expectOptimizeStorageStats(conn, "node", 9, 91) - expectOptimizeStorageStats(conn, "edge", 10, 90) - expectOptimizeStorageVacuum(conn, "VACUUM (ANALYZE) edge") - - require.NoError(t, optimizeStorage(ctx, conn)) - require.NoError(t, conn.ExpectationsWereMet()) - }) - - t.Run("vacuums node and edge", func(t *testing.T) { - ctx := context.Background() - conn := newOptimizeStorageMockConn(t) - - expectOptimizeStorageStats(conn, "node", 10, 90) - expectOptimizeStorageStats(conn, "edge", 10, 90) expectOptimizeStorageVacuum(conn, "VACUUM (ANALYZE) node, edge") require.NoError(t, optimizeStorage(ctx, conn)) require.NoError(t, conn.ExpectationsWereMet()) }) - - t.Run("returns query error", func(t *testing.T) { - ctx := context.Background() - conn := newOptimizeStorageMockConn(t) - expectedErr := errors.New("stats unavailable") - - conn.ExpectQuery(optimizeStorageStatsQuery). - WithArgs("node"). - WillReturnError(expectedErr) - - err := optimizeStorage(ctx, conn) - require.ErrorIs(t, err, expectedErr) - require.ErrorContains(t, err, "query dead tuple stats for node") - require.NoError(t, conn.ExpectationsWereMet()) - }) } func newOptimizeStorageMockConn(t *testing.T) pgxmock.PgxConnIface { @@ -83,12 +31,6 @@ func newOptimizeStorageMockConn(t *testing.T) pgxmock.PgxConnIface { return conn } -func expectOptimizeStorageStats(conn pgxmock.PgxConnIface, table string, dead, live int64) { - conn.ExpectQuery(optimizeStorageStatsQuery). - WithArgs(table). - WillReturnRows(pgxmock.NewRows([]string{"dead", "live"}).AddRow(dead, live)) -} - func expectOptimizeStorageVacuum(conn pgxmock.PgxConnIface, stmt string) { conn.ExpectExec(stmt). WithArgs(pgx.QueryExecModeSimpleProtocol). diff --git a/drivers/pg/pg.go b/drivers/pg/pg.go index 88a5d17e..e255ada8 100644 --- a/drivers/pg/pg.go +++ b/drivers/pg/pg.go @@ -14,20 +14,27 @@ import ( ) const ( + // DriverName is the connection-string scheme registered by the PostgreSQL + // driver. DriverName = "pg" // defaultBatchWriteSize is currently set to 2k. This is meant to strike a balance between the cost of thousands // of round-trips against the cost of locking tables for too long. - defaultBatchWriteSize = 2_000 + defaultBatchWriteSize = 2_000 + + // poolInitConnectionTimeout limits how long pool setup waits for the first connection to initialize. poolInitConnectionTimeout = time.Second * 10 ) +// AfterPooledConnectionEstablished loads and registers the driver's owned graph composite types on a new pooled connection. func AfterPooledConnectionEstablished(ctx context.Context, conn *pgx.Conn) error { for _, dataType := range pgsql.CompositeTypes { if definition, err := conn.LoadType(ctx, dataType.String()); err != nil { if !StateObjectDoesNotExist.ErrorMatches(err) { return fmt.Errorf("failed to match composite type %s to database: %w", dataType, err) } + } else if err := installOwnedCompositeCodec(dataType, definition); err != nil { + return fmt.Errorf("failed to configure composite type %s: %w", dataType, err) } else { conn.TypeMap().RegisterType(definition) } diff --git a/drivers/pg/query/schema_upgrade_integration_test.go b/drivers/pg/query/schema_upgrade_integration_test.go new file mode 100644 index 00000000..e20abde7 --- /dev/null +++ b/drivers/pg/query/schema_upgrade_integration_test.go @@ -0,0 +1,377 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +//go:build manual_integration && integration + +package query + +import ( + "context" + "encoding/json" + "os" + "testing" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/specterops/dawgs/databaseguard" + "github.com/stretchr/testify/require" +) + +// TestSchemaUpgradeRemovesLegacyPathMaterializerOverloads verifies an upgrade drops obsolete unscoped path functions while retaining graph-scoped signatures. +func TestSchemaUpgradeRemovesLegacyPathMaterializerOverloads(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + target, err := databaseguard.Target(connection) + require.NoError(t, err) + if len(target) < len("postgresql://") || target[:len("postgresql://")] != "postgresql://" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + require.NoError(t, databaseguard.ValidateEnvironment(connection)) + + ctx := context.Background() + pool, err := pgxpool.New(ctx, connection) + require.NoError(t, err) + t.Cleanup(pool.Close) + + _, err = pool.Exec(ctx, sqlSchemaUp) + require.NoError(t, err) + + _, err = pool.Exec(ctx, ` + drop function public.nodes_to_path(int4, int8[]); + drop function public.edges_to_path(int4, int8[]); + drop function public.ordered_edges_to_path(int4, nodeComposite, edgeComposite[], nodeComposite[]); + create function public.nodes_to_path(nodes variadic int8[]) returns pathComposite language sql immutable strict as $$ + select row(array[]::nodeComposite[], array[]::edgeComposite[])::pathComposite + $$; + create function public.edges_to_path(path variadic int8[]) returns pathComposite language sql immutable strict as $$ + select row(array[]::nodeComposite[], array[]::edgeComposite[])::pathComposite + $$; + create function public.ordered_edges_to_path(root nodeComposite, edges edgeComposite[], known_nodes nodeComposite[]) returns pathComposite language sql immutable strict as $$ + select row(array[root]::nodeComposite[], edges)::pathComposite + $$; + `) + require.NoError(t, err) + + _, err = pool.Exec(ctx, sqlSchemaUp) + require.NoError(t, err) + + var legacyNodes, legacyEdges, legacyOrdered, scopedNodes, scopedEdges, scopedOrdered bool + err = pool.QueryRow(ctx, `select + to_regprocedure('public.nodes_to_path(bigint[])') is not null, + to_regprocedure('public.edges_to_path(bigint[])') is not null, + to_regprocedure('public.ordered_edges_to_path(nodecomposite,edgecomposite[],nodecomposite[])') is not null, + to_regprocedure('public.nodes_to_path(integer,bigint[])') is not null, + to_regprocedure('public.edges_to_path(integer,bigint[])') is not null, + to_regprocedure('public.ordered_edges_to_path(integer,nodecomposite,edgecomposite[],nodecomposite[])') is not null + `).Scan(&legacyNodes, &legacyEdges, &legacyOrdered, &scopedNodes, &scopedEdges, &scopedOrdered) + require.NoError(t, err) + require.False(t, legacyNodes) + require.False(t, legacyEdges) + require.False(t, legacyOrdered) + require.True(t, scopedNodes) + require.True(t, scopedEdges) + require.True(t, scopedOrdered) +} + +// TestBidirectionalAllShortestPathCapBoundaries proves that every candidate +// admission gate is exact at N, fails closed at N-1, and preserves the full +// ASP-A1 multiset on fallback. The fixture reconverges through two middle +// nodes so equal-depth, relationship-distinct predecessor rows are required +// to produce all six shortest paths. +func TestBidirectionalAllShortestPathCapBoundaries(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + target, err := databaseguard.Target(connection) + require.NoError(t, err) + if len(target) < len("postgresql://") || target[:len("postgresql://")] != "postgresql://" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + require.NoError(t, databaseguard.ValidateEnvironment(connection)) + + ctx := context.Background() + pool, err := pgxpool.New(ctx, connection) + require.NoError(t, err) + t.Cleanup(pool.Close) + connectionHandle, err := pool.Acquire(ctx) + require.NoError(t, err) + defer connectionHandle.Release() + + _, err = connectionHandle.Exec(ctx, sqlSchemaUp) + require.NoError(t, err) + _, err = connectionHandle.Exec(ctx, ` + create temporary table edge + ( + id int8 not null, + graph_id int4 not null, + start_id int8 not null, + end_id int8 not null, + kind_id int2 not null, + properties jsonb not null + ) on commit preserve rows; + insert into edge(id, graph_id, start_id, end_id, kind_id, properties) values + (101, 1, 1, 2, 1, '{}'), (102, 1, 1, 3, 1, '{}'), (103, 1, 1, 4, 1, '{}'), + (104, 1, 2, 5, 1, '{}'), (105, 1, 2, 6, 1, '{}'), + (106, 1, 3, 5, 1, '{}'), (107, 1, 3, 6, 1, '{}'), + (108, 1, 4, 5, 1, '{}'), (109, 1, 4, 6, 1, '{}'), + (110, 1, 5, 9, 1, '{}'), (111, 1, 6, 9, 1, '{}'); + `) + require.NoError(t, err) + + tx, err := connectionHandle.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadWrite}) + require.NoError(t, err) + defer func() { _ = tx.Rollback(ctx) }() + + readPaths := func(query string, args ...any) []string { + rows, queryErr := tx.Query(ctx, query, args...) + require.NoError(t, queryErr) + defer rows.Close() + paths := []string{} + for rows.Next() { + var path string + require.NoError(t, rows.Scan(&path)) + paths = append(paths, path) + } + require.NoError(t, rows.Err()) + return paths + } + + exact := readPaths(` + select path::text + from public.all_shortest_paths_dag(1, 1, 9, 1, 8, array[]::int2[], false) + order by path`) + require.Len(t, exact, 6) + + type limits struct { + state int64 + frontier int64 + predecessor int64 + enumeration int64 + outputBytes int64 + } + type diagnostic struct { + RuntimeBranch string `json:"runtime_branch"` + Overflowed bool `json:"overflowed"` + FallbackExecuted bool `json:"fallback_executed"` + Counters struct { + SeenPeak int64 `json:"seen_peak"` + FrontierPeak int64 `json:"frontier_peak"` + PredecessorPeak int64 `json:"predecessor_peak"` + OutputPaths int64 `json:"output_paths"` + OutputBytes int64 `json:"output_bytes"` + } `json:"counters"` + } + const candidateQuery = ` + select path::text + from public.all_shortest_paths_b1_strict_alternating( + 1, 1, 9, 1, 8, array[]::int2[], false, $1, $2, $3, $4, $5) + order by path` + runCandidate := func(invocationID string, caps limits) ([]string, diagnostic) { + _, execErr := tx.Exec(ctx, "select public.begin_bidirectional_all_shortest_path_diagnostic_v1($1)", invocationID) + require.NoError(t, execErr) + paths := readPaths(candidateQuery, caps.state, caps.frontier, caps.predecessor, caps.enumeration, caps.outputBytes) + var raw string + require.NoError(t, tx.QueryRow(ctx, + "select public.read_bidirectional_all_shortest_path_diagnostic_v1($1)::text", invocationID).Scan(&raw)) + var report diagnostic + require.NoError(t, json.Unmarshal([]byte(raw), &report)) + _, execErr = tx.Exec(ctx, "select public.clear_bidirectional_all_shortest_path_diagnostic_v1($1)", invocationID) + require.NoError(t, execErr) + return paths, report + } + + large := limits{state: 1_000_000, frontier: 1_000_000, predecessor: 1_000_000, enumeration: 1_000_000, outputBytes: 1 << 30} + for _, scheduler := range []struct { + name string + query string + }{ + {name: "B1 strict alternating", query: candidateQuery}, + {name: "B2 smaller level", query: ` + select path::text + from public.all_shortest_paths_b2_smaller_current_level( + 1, 1, 9, 1, 8, array[]::int2[], false, $1, $2, $3, $4, $5) + order by path`}, + } { + t.Run(scheduler.name+" retains the exact multiset", func(t *testing.T) { + paths := readPaths(scheduler.query, large.state, large.frontier, large.predecessor, large.enumeration, large.outputBytes) + require.Equal(t, exact, paths) + }) + } + + baselinePaths, baseline := runCandidate("asp-cap-baseline", large) + require.Equal(t, exact, baselinePaths) + require.Equal(t, "bidirectional_search", baseline.RuntimeBranch) + require.False(t, baseline.Overflowed) + require.False(t, baseline.FallbackExecuted) + require.Positive(t, baseline.Counters.SeenPeak) + require.Positive(t, baseline.Counters.FrontierPeak) + require.Positive(t, baseline.Counters.PredecessorPeak) + require.Equal(t, int64(len(exact)), baseline.Counters.OutputPaths) + require.Positive(t, baseline.Counters.OutputBytes) + + boundaries := []struct { + name string + get func(limits) int64 + set func(*limits, int64) + }{ + {name: "state", get: func(_ limits) int64 { return baseline.Counters.SeenPeak }, set: func(value *limits, limit int64) { value.state = limit }}, + {name: "frontier", get: func(_ limits) int64 { return baseline.Counters.FrontierPeak }, set: func(value *limits, limit int64) { value.frontier = limit }}, + {name: "predecessor", get: func(_ limits) int64 { return baseline.Counters.PredecessorPeak }, set: func(value *limits, limit int64) { value.predecessor = limit }}, + {name: "enumeration", get: func(_ limits) int64 { return baseline.Counters.OutputPaths }, set: func(value *limits, limit int64) { value.enumeration = limit }}, + {name: "output bytes", get: func(_ limits) int64 { return baseline.Counters.OutputBytes }, set: func(value *limits, limit int64) { value.outputBytes = limit }}, + } + for _, boundary := range boundaries { + boundary := boundary + n := boundary.get(large) + for _, delta := range []int64{-1, 0, 1} { + delta := delta + name := boundary.name + map[int64]string{-1: " N-1", 0: " N", 1: " N+1"}[delta] + t.Run(name, func(t *testing.T) { + caps := large + boundary.set(&caps, n+delta) + paths, report := runCandidate("asp-cap-"+boundary.name+map[int64]string{-1: "-minus", 0: "-exact", 1: "-plus"}[delta], caps) + require.Equal(t, exact, paths, "candidate and fallback must preserve the complete ordered multiset") + if delta < 0 { + require.Equal(t, "exact_a1_fallback", report.RuntimeBranch) + require.True(t, report.Overflowed) + require.True(t, report.FallbackExecuted) + } else { + require.Equal(t, "bidirectional_search", report.RuntimeBranch) + require.False(t, report.Overflowed) + require.False(t, report.FallbackExecuted) + } + }) + } + } + + require.NoError(t, tx.Rollback(ctx)) +} + +// TestBidirectionalShortestPathLowerBoundAndWitnesses exercises a graph where +// strict alternation encounters a length-five meeting before the unique +// length-four route. Returning the shorter route proves the queue-head +// lower-bound check continued beyond the first intersection. The tie and +// inbound assertions separately validate the one-witness contract: minimum +// depth, relationship uniqueness, and logical source-to-target edge order. +func TestBidirectionalShortestPathLowerBoundAndWitnesses(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + target, err := databaseguard.Target(connection) + require.NoError(t, err) + if len(target) < len("postgresql://") || target[:len("postgresql://")] != "postgresql://" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + require.NoError(t, databaseguard.ValidateEnvironment(connection)) + + ctx := context.Background() + pool, err := pgxpool.New(ctx, connection) + require.NoError(t, err) + t.Cleanup(pool.Close) + connectionHandle, err := pool.Acquire(ctx) + require.NoError(t, err) + defer connectionHandle.Release() + _, err = connectionHandle.Exec(ctx, sqlSchemaUp) + require.NoError(t, err) + _, err = connectionHandle.Exec(ctx, ` + create temporary table edge + ( + id int8 not null, + graph_id int4 not null, + start_id int8 not null, + end_id int8 not null, + kind_id int2 not null, + properties jsonb not null + ) on commit preserve rows; + -- Graph 2: the low-ID length-five branch meets first under B1. Two + -- target-side dead ends delay acceptance of the unique length-four arm. + insert into edge(id, graph_id, start_id, end_id, kind_id, properties) values + (201, 2, 1000, 1001, 1, '{}'), (203, 2, 1001, 1002, 1, '{}'), + (205, 2, 1002, 1003, 1, '{}'), (207, 2, 1003, 1004, 1, '{}'), + (209, 2, 1004, 1999, 1, '{}'), + (202, 2, 1000, 1100, 1, '{}'), (204, 2, 1100, 1101, 1, '{}'), + (206, 2, 1101, 1102, 1, '{}'), (999, 2, 1102, 1999, 1, '{}'), + (210, 2, 1200, 1999, 1, '{}'), (211, 2, 1201, 1999, 1, '{}'), + -- Graph 3: two equally short, relationship-disjoint witnesses. + (301, 3, 2000, 2001, 1, '{}'), (302, 3, 2001, 2002, 1, '{}'), + (303, 3, 2002, 2999, 1, '{}'), + (304, 3, 2000, 2101, 1, '{}'), (305, 3, 2101, 2102, 1, '{}'), + (306, 3, 2102, 2999, 1, '{}'); + `) + require.NoError(t, err) + + tx, err := connectionHandle.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadWrite}) + require.NoError(t, err) + defer func() { _ = tx.Rollback(ctx) }() + + type result struct { + depth int32 + path []int64 + } + run := func(function string, graphID, sourceID, targetID int64, inbound bool) result { + query := `select depth, path from public.` + function + `( + $1::int4, $2::int8, $3::int8, 1, 8, array[]::int2[], $4, 100000, 100000, 100000)` + var value result + require.NoError(t, tx.QueryRow(ctx, query, graphID, sourceID, targetID, inbound).Scan(&value.depth, &value.path)) + return value + } + + _, err = tx.Exec(ctx, "select public.begin_bidirectional_shortest_path_diagnostic_v1('sp-adversarial-b1')") + require.NoError(t, err) + b1 := run("shortest_path_b1_strict_alternating", 2, 1000, 1999, false) + require.Equal(t, int32(4), b1.depth) + require.Equal(t, []int64{202, 204, 206, 999}, b1.path) + var raw string + require.NoError(t, tx.QueryRow(ctx, + "select public.read_bidirectional_shortest_path_diagnostic_v1('sp-adversarial-b1')::text").Scan(&raw)) + var report struct { + RuntimeBranch string `json:"runtime_branch"` + Counters struct { + MeetingCandidates int64 `json:"meeting_candidates"` + FrozenDistance int32 `json:"frozen_distance"` + WitnessRows int64 `json:"witness_rows"` + } `json:"counters"` + } + require.NoError(t, json.Unmarshal([]byte(raw), &report)) + require.Equal(t, "bidirectional_search", report.RuntimeBranch) + require.GreaterOrEqual(t, report.Counters.MeetingCandidates, int64(2), "the longer and shorter intersections must both be observed") + require.Equal(t, int32(4), report.Counters.FrozenDistance) + require.Equal(t, int64(1), report.Counters.WitnessRows) + _, err = tx.Exec(ctx, "select public.clear_bidirectional_shortest_path_diagnostic_v1('sp-adversarial-b1')") + require.NoError(t, err) + + for _, scheduler := range []string{ + "shortest_path_b1_strict_alternating", + "shortest_path_b2_smaller_current_level", + } { + t.Run(scheduler+" unique and inbound witnesses", func(t *testing.T) { + outbound := run(scheduler, 2, 1000, 1999, false) + require.Equal(t, int32(4), outbound.depth) + require.Equal(t, []int64{202, 204, 206, 999}, outbound.path) + require.Len(t, outbound.path, int(outbound.depth)) + + inbound := run(scheduler, 2, 1999, 1000, true) + require.Equal(t, int32(4), inbound.depth) + require.Equal(t, []int64{999, 206, 204, 202}, inbound.path) + require.Len(t, inbound.path, int(inbound.depth)) + + tie := run(scheduler, 3, 2000, 2999, false) + require.Equal(t, int32(3), tie.depth) + require.Len(t, tie.path, int(tie.depth)) + require.Contains(t, [][]int64{{301, 302, 303}, {304, 305, 306}}, tie.path) + relationships := map[int64]struct{}{} + for _, edgeID := range tie.path { + relationships[edgeID] = struct{}{} + } + require.Len(t, relationships, len(tie.path), "a shortest witness may not repeat a relationship") + }) + } + + require.NoError(t, tx.Rollback(ctx)) +} diff --git a/drivers/pg/query/sql/schema_down.sql b/drivers/pg/query/sql/schema_down.sql index 6e2c0de0..44cfaec3 100644 --- a/drivers/pg/query/sql/schema_down.sql +++ b/drivers/pg/query/sql/schema_down.sql @@ -28,11 +28,52 @@ drop function if exists index_utilization; drop function if exists _format_asp_where_clause; drop function if exists _format_asp_query; drop function if exists asp_harness; -drop function if exists create_traversal_filter_tables; +drop function if exists create_traversal_filter_tables(); drop function if exists create_traversal_filter_tables(text, text, text); drop function if exists create_traversal_filter_tables(text, text); drop function if exists create_traversal_filter_tables(int8[], int8[]); drop function if exists shortest_path_self_endpoint_error(int8, int8); +drop function if exists all_shortest_paths_b1_strict_alternating(int4, int8, int8, int4, int4, int2[], bool, int8, int8, int8, int8, int8); +drop function if exists all_shortest_paths_b2_smaller_current_level(int4, int8, int8, int4, int4, int2[], bool, int8, int8, int8, int8, int8); +drop function if exists all_shortest_paths_bidirectional_compact_v1(int4, int8, int8, int4, int4, int2[], bool, int8, int8, int8, int8, int8, text); +drop function if exists _finish_bidirectional_all_shortest_path_diagnostic_call_v1(text, int8, text, int8, int8, int8, int8, int8, int8, int8, int8, int4, int8, int8, int8, int4, int8, bool, int8, int8, int8, int8, int8, bool, bool); +drop function if exists _record_bidirectional_all_shortest_path_diagnostic_level_v1(text, int8, int8, text, text, int4, int8, int8, int8, int8, int8, int8, int8); +drop function if exists _start_bidirectional_all_shortest_path_diagnostic_call_v1(text, text, int8, int8, int8, int8, int8, int8, int8); +drop function if exists clear_bidirectional_all_shortest_path_diagnostic_v1(text); +drop function if exists read_bidirectional_all_shortest_path_diagnostic_v1(text); +drop function if exists begin_bidirectional_all_shortest_path_diagnostic_v1(text); +drop function if exists ensure_bidirectional_all_shortest_path_telemetry_workspace(); +drop function if exists clear_bidirectional_all_shortest_path_workspace(); +drop function if exists reset_bidirectional_all_shortest_path_workspace(); +drop function if exists ensure_bidirectional_all_shortest_path_workspace(); +drop function if exists shortest_path_b1_strict_alternating(int4, int8, int8, int4, int4, int2[], bool, int8, int8, int8); +drop function if exists shortest_path_b2_smaller_current_level(int4, int8, int8, int4, int4, int2[], bool, int8, int8, int8); +drop function if exists shortest_path_bidirectional_compact_v1(int4, int8, int8, int4, int4, int2[], bool, int8, int8, int8, text); +drop function if exists _finish_bidirectional_shortest_path_diagnostic_call_v1(text, int8, text, int8, int8, int8, int8, int8, int8, int8, int8, int4, int8, bool, bool); +drop function if exists _record_bidirectional_shortest_path_diagnostic_level_v1(text, int8, int8, text, text, int4, int8, int8, int8, int8, int8, int8, int8); +drop function if exists _start_bidirectional_shortest_path_diagnostic_call_v1(text, text, int8, int8, int8, int8, int8); +drop function if exists clear_bidirectional_shortest_path_diagnostic_v1(text); +drop function if exists read_bidirectional_shortest_path_diagnostic_v1(text); +drop function if exists begin_bidirectional_shortest_path_diagnostic_v1(text); +drop function if exists ensure_bidirectional_shortest_path_telemetry_workspace(); +drop function if exists reset_bidirectional_shortest_path_workspace(); +drop function if exists ensure_bidirectional_shortest_path_workspace(); +drop function if exists clear_traversal_runtime_attestation_v1(text); +drop function if exists read_traversal_runtime_attestation_v1(text); +drop function if exists record_requested_traversal_runtime_attestation_v1(text, bool, text); +drop function if exists record_traversal_runtime_attestation_v1(text, text, bool); +drop function if exists begin_traversal_runtime_attestation_v1(text, text); +drop function if exists ensure_traversal_runtime_attestation_workspace_v1(); +drop function if exists shortest_path_compact(int4, int8, int8, int4, int4, int2[], bool, int8); +drop function if exists all_shortest_paths_dag(int4, int8, int8, int4, int4, int2[], bool); +drop function if exists reset_shortest_dag_workspace(); +drop function if exists ensure_shortest_dag_workspace(); +drop function if exists bsp_workspace_fragment(text); +drop function if exists load_bsp_filter_tables(text, text, text); +drop function if exists reset_bsp_workspace(bool); +drop function if exists ensure_bsp_generic_workspace(); +drop function if exists ensure_bsp_core_workspace(); +drop function if exists graphbench_s1_distance_bfs(int4, int8, int8, int4, int4, int2[], bool, int4); drop function if exists unidirectional_sp_harness(text, text, int4); drop function if exists unidirectional_sp_harness(text, text, int4, int8); drop function if exists unidirectional_sp_harness(text, text, int4, text, text); @@ -53,13 +94,20 @@ drop function if exists bidirectional_asp_harness(text, text, text, text, int4, drop function if exists bidirectional_sp_harness(text, text, text, text, int4); drop function if exists bidirectional_sp_harness(text, text, text, text, int4, int8); drop function if exists bidirectional_sp_harness(text, text, text, text, int4, text, text, text, int8); +drop function if exists bidirectional_sp_harness(text, text, text, text, int4, text, text, text, bool, int8); +drop function if exists bidirectional_sp_harness(text, text, text, text, int4, text, text, text, bool); drop function if exists bidirectional_sp_harness(text, text, text, text, int4, text, text, text); drop function if exists bidirectional_sp_harness(text, text, text, text, int4, text, text, int8); +drop function if exists bidirectional_sp_harness(text, text, text, text, int4, text, text, bool, int8); +drop function if exists bidirectional_sp_harness(text, text, text, text, int4, text, text, bool); drop function if exists bidirectional_sp_harness(text, text, text, text, int4, text, text); drop function if exists bidirectional_sp_harness(text, text, text, text, int4, int8[], int8); drop function if exists bidirectional_sp_harness(text, text, text, text, int4, int8[]); drop function if exists bidirectional_sp_harness(text, text, text, text, int4, int8[], int8[], int8); +drop function if exists bidirectional_sp_harness(text, text, text, text, int4, int8[], int8[], bool, int8); +drop function if exists bidirectional_sp_harness(text, text, text, text, int4, int8[], int8[], bool); drop function if exists bidirectional_sp_harness(text, text, text, text, int4, int8[], int8[]); +drop function if exists _bidirectional_sp_harness(text, text, text, text, int4, text, text, text, int8[], int8[], int8, bool, bool); drop function if exists _bidirectional_sp_harness(text, text, text, text, int4, text, text, text, int8[], int8[], int8, bool); drop function if exists _bidirectional_sp_harness(text, text, text, text, int4, text, text, text, int8[], int8[], bool); drop function if exists _bidirectional_sp_harness(text, text, text, text, int4, text, text, int8[], int8[], bool); @@ -73,8 +121,13 @@ drop function if exists _format_traversal_query; drop function if exists _format_traversal_initial_query; drop function if exists expand_traversal_step; drop function if exists traverse; +drop function if exists ordered_edges_to_path(int4, nodeComposite, edgeComposite[], nodeComposite[]); drop function if exists ordered_edges_to_path(nodeComposite, edgeComposite[], nodeComposite[]); -drop function if exists edges_to_path; +drop function if exists ordered_edge_ids_to_path(int4, nodeComposite, int8[], nodeComposite[]); +drop function if exists nodes_to_path(int4, int8[]); +drop function if exists nodes_to_path(int8[]); +drop function if exists edges_to_path(int4, int8[]); +drop function if exists edges_to_path(int8[]); drop function if exists traverse_paths; -- Drop all tables in order of dependency. diff --git a/drivers/pg/query/sql/schema_up.sql b/drivers/pg/query/sql/schema_up.sql index f112002d..a567fd4a 100644 --- a/drivers/pg/query/sql/schema_up.sql +++ b/drivers/pg/query/sql/schema_up.sql @@ -638,31 +638,41 @@ $$ parallel safe strict; -create or replace function public.nodes_to_path(nodes variadic int8[]) returns pathComposite as +-- CREATE OR REPLACE does not replace a function when its argument signature +-- changes. Remove the pre-graph-scope overloads explicitly so upgrades cannot +-- retain helpers that hydrate entities from a different graph partition. +drop function if exists public.nodes_to_path(int8[]); +drop function if exists public.edges_to_path(int8[]); +drop function if exists public.ordered_edges_to_path(nodeComposite, edgeComposite[], nodeComposite[]); + +create or replace function public.nodes_to_path(target_graph_id int4, nodes variadic int8[]) returns pathComposite as $$ select row (array_agg(distinct (n.id, n.kind_ids, n.properties)::nodeComposite)::nodeComposite[], array []::edgeComposite[])::pathComposite from node n -where n.id = any (nodes); +where n.graph_id = target_graph_id + and n.id = any (nodes); $$ language sql immutable parallel safe strict; -create or replace function public.edges_to_path(path variadic int8[]) returns pathComposite as +create or replace function public.edges_to_path(target_graph_id int4, path variadic int8[]) returns pathComposite as $$ select row ( (select array_agg(distinct (n.id, n.kind_ids, n.properties)::nodeComposite) from node n - where n.id in ( - select start_id from edge where id = any(path) + where n.graph_id = target_graph_id + and n.id in ( + select start_id from edge where graph_id = target_graph_id and id = any(path) union - select end_id from edge where id = any(path) + select end_id from edge where graph_id = target_graph_id and id = any(path) )), (select array_agg(distinct (r.id, r.start_id, r.end_id, r.kind_id, r.properties)::edgeComposite) from edge r - where r.id = any(path)) + where r.graph_id = target_graph_id + and r.id = any(path)) )::pathComposite; $$ language sql @@ -670,7 +680,7 @@ $$ parallel safe strict; -create or replace function public.ordered_edges_to_path(root nodeComposite, edges edgeComposite[], known_nodes nodeComposite[]) returns pathComposite as +create or replace function public.ordered_edges_to_path(target_graph_id int4, root nodeComposite, edges edgeComposite[], known_nodes nodeComposite[]) returns pathComposite as $$ with recursive edge_bounds(edge_count) as ( @@ -745,7 +755,7 @@ select row ( where candidate.id = ordered_node.id limit 1 ) known_node on true - left join node n on n.id = ordered_node.id and known_node.node is null + left join node n on n.id = ordered_node.id and n.graph_id = target_graph_id and known_node.node is null ), ( select coalesce( @@ -764,6 +774,87 @@ $$ parallel safe strict; +-- ordered_edge_ids_to_path is the read-expansion materializer. Expansion +-- lowering already knows the edge order, so this helper walks that order once +-- instead of repeatedly searching the remaining edge array. Every persistent +-- lookup is constrained by target_graph_id because entity IDs are only unique +-- within a graph partition. +create or replace function public.ordered_edge_ids_to_path(target_graph_id int4, root nodeComposite, edge_ids int8[], known_nodes nodeComposite[]) returns pathComposite as +$$ +with recursive +edge_count(value) as +( + select coalesce(cardinality(edge_ids), 0) +), +hydrated_edges as materialized +( + select path_edge.ordinality::int4 as ordinality, + (e.id, e.start_id, e.end_id, e.kind_id, e.properties)::edgeComposite as edge + from unnest(edge_ids) with ordinality as path_edge(id, ordinality) + join edge e + on e.id = path_edge.id + and e.graph_id = target_graph_id +), +path_walk(idx, current_node_id, node_ids) as +( + select 0::int4, (root).id, array [(root).id]::int8[] + union all + select path_walk.idx + 1, + case + when path_walk.current_node_id = (next_edge.edge).start_id then (next_edge.edge).end_id + else (next_edge.edge).start_id + end, + path_walk.node_ids || case + when path_walk.current_node_id = (next_edge.edge).start_id then (next_edge.edge).end_id + else (next_edge.edge).start_id + end + from path_walk + join hydrated_edges next_edge + on next_edge.ordinality = path_walk.idx + 1 + and path_walk.current_node_id in ((next_edge.edge).start_id, (next_edge.edge).end_id) +), +final_walk as +( + select path_walk.node_ids + from path_walk + cross join edge_count + where path_walk.idx = edge_count.value +) +select row ( + ( + select coalesce( + array_agg(coalesce(known_node.node, (n.id, n.kind_ids, n.properties)::nodeComposite) order by ordered_node.ordinality)::nodeComposite[], + array []::nodeComposite[] + ) + from final_walk + cross join lateral unnest(final_walk.node_ids) with ordinality as ordered_node(id, ordinality) + left join lateral + ( + select (candidate.id, candidate.kind_ids, candidate.properties)::nodeComposite as node + from unnest(known_nodes) as candidate(id, kind_ids, properties) + where candidate.id = ordered_node.id + limit 1 + ) known_node on true + left join node n + on n.id = ordered_node.id + and n.graph_id = target_graph_id + and known_node.node is null + ), + ( + select coalesce( + array_agg(hydrated_edges.edge order by hydrated_edges.ordinality)::edgeComposite[], + array []::edgeComposite[] + ) + from hydrated_edges + ) +)::pathComposite +from final_walk; +$$ + language sql + stable + parallel safe + strict; + create or replace function public.create_unidirectional_pathspace_tables() returns void as $$ @@ -771,7 +862,7 @@ begin -- The path column is not used as a primary key. Deduplication is handled by DISTINCT ON clauses in the -- harness functions. Removing the PK on the variable-length int8[] array eliminates O(n)-key B-tree -- maintenance that grows with traversal depth. - create temporary table forward_front + create temporary table if not exists forward_front ( root_id int8 not null, next_id int8 not null, @@ -779,9 +870,9 @@ begin satisfied bool, is_cycle bool not null, path int8[] not null - ) on commit drop; + ) on commit preserve rows; - create temporary table next_front + create temporary table if not exists next_front ( root_id int8 not null, next_id int8 not null, @@ -789,15 +880,17 @@ begin satisfied bool, is_cycle bool not null, path int8[] not null - ) on commit drop; + ) on commit preserve rows; - create index forward_front_next_id_index on forward_front using btree (next_id); - create index forward_front_satisfied_index on forward_front using btree (root_id, next_id, depth) where satisfied; - create index forward_front_is_cycle_index on forward_front using btree (root_id, next_id) where is_cycle; + create index if not exists forward_front_next_id_index on forward_front using btree (next_id); + create index if not exists forward_front_satisfied_index on forward_front using btree (root_id, next_id, depth) where satisfied; + create index if not exists forward_front_is_cycle_index on forward_front using btree (root_id, next_id) where is_cycle; - create index next_front_next_id_index on next_front using btree (next_id); - create index next_front_satisfied_index on next_front using btree (root_id, next_id, depth) where satisfied; - create index next_front_is_cycle_index on next_front using btree (root_id, next_id) where is_cycle; + create index if not exists next_front_next_id_index on next_front using btree (next_id); + create index if not exists next_front_satisfied_index on next_front using btree (root_id, next_id, depth) where satisfied; + create index if not exists next_front_is_cycle_index on next_front using btree (root_id, next_id) where is_cycle; + + truncate table forward_front, next_front; end; $$ language plpgsql @@ -809,14 +902,14 @@ create or replace function public.create_unidirectional_shortest_path_tables() returns void as $$ begin - create temporary table visited + create temporary table if not exists visited ( root_id int8 not null, id int8 not null, primary key (root_id, id) - ) on commit drop; + ) on commit preserve rows; - create temporary table paths + create temporary table if not exists paths ( root_id int8 not null, next_id int8 not null, @@ -824,19 +917,21 @@ begin satisfied bool, is_cycle bool not null, path int8[] not null - ) on commit drop; + ) on commit preserve rows; - create temporary table resolved_roots + create temporary table if not exists resolved_roots ( root_id int8 not null, primary key (root_id) - ) on commit drop; + ) on commit preserve rows; + + truncate table visited, paths, resolved_roots; perform create_unidirectional_pathspace_tables(); - create index forward_front_root_id_next_id_index on forward_front using btree (root_id, next_id); - create index next_front_root_id_next_id_index on next_front using btree (root_id, next_id); - create index paths_root_id_next_id_index on paths using btree (root_id, next_id); + create index if not exists forward_front_root_id_next_id_index on forward_front using btree (root_id, next_id); + create index if not exists next_front_root_id_next_id_index on next_front using btree (root_id, next_id); + create index if not exists paths_root_id_next_id_index on paths using btree (root_id, next_id); end; $$ language plpgsql @@ -844,9 +939,8 @@ $$ strict; -- create_traversal_filter_tables materializes the root, terminal and pair filter sets into temporary tables that the --- harness functions join against. The tables use `on commit drop`, so a single transaction can only host one harness --- invocation that depends on these tables; concurrent or sequential expansions in the same transaction will conflict --- on the temporary table names. +-- harness functions join against. Definitions persist for the physical +-- session; each invocation truncates its row state before loading a new filter. create or replace function public.create_traversal_filter_tables() returns void as $$ @@ -855,120 +949,3690 @@ begin ( id int8 not null, primary key (id) - ) on commit drop; + ) on commit preserve rows; create temporary table if not exists traversal_terminal_filter ( id int8 not null, primary key (id) - ) on commit drop; + ) on commit preserve rows; create temporary table if not exists traversal_pair_filter ( root_id int8 not null, terminal_id int8 not null, primary key (root_id, terminal_id) - ) on commit drop; + ) on commit preserve rows; + + create index if not exists traversal_pair_filter_terminal_id_root_id_index on traversal_pair_filter using btree (terminal_id, root_id); + + truncate table traversal_root_filter; + truncate table traversal_terminal_filter; + truncate table traversal_pair_filter; + + return; +end; +$$ + language plpgsql + volatile; + +create or replace function public.create_traversal_filter_tables(root_ids int8[], terminal_ids int8[]) + returns void as +$$ +begin + perform create_traversal_filter_tables(); + + insert into traversal_root_filter + select distinct root_id + from unnest(root_ids) as root_ids(root_id) + where root_id is not null + on conflict (id) do nothing; + + insert into traversal_terminal_filter + select distinct terminal_id + from unnest(terminal_ids) as terminal_ids(terminal_id) + where terminal_id is not null + on conflict (id) do nothing; + + analyze traversal_root_filter; + analyze traversal_terminal_filter; + + return; +end; +$$ + language plpgsql + volatile + strict; + +create or replace function public.create_traversal_filter_tables(root_filter text, terminal_filter text, pair_filter text) + returns void as +$$ +begin + perform create_traversal_filter_tables(); + + if length(pair_filter) > 0 then + execute pair_filter; + end if; + + if length(root_filter) > 0 then + execute root_filter; + elsif length(pair_filter) > 0 then + insert into traversal_root_filter + select distinct root_id + from traversal_pair_filter + on conflict (id) do nothing; + end if; + + if length(terminal_filter) > 0 then + execute terminal_filter; + elsif length(pair_filter) > 0 then + insert into traversal_terminal_filter + select distinct terminal_id + from traversal_pair_filter + on conflict (id) do nothing; + end if; + + analyze traversal_root_filter; + analyze traversal_terminal_filter; + analyze traversal_pair_filter; + + return; +end; +$$ + language plpgsql + volatile + strict; + +create or replace function public.create_traversal_filter_tables(root_filter text, terminal_filter text) + returns void as +$$ +select public.create_traversal_filter_tables(root_filter, terminal_filter, ''::text); +$$ + language sql + volatile + strict; + +create or replace function public.shortest_path_self_endpoint_error(root_id int8, terminal_id int8) + returns bool as +$$ +begin + raise exception using + errcode = '22023', + message = format('shortest path endpoints must not resolve to the same node: root_id=%s terminal_id=%s', + root_id, + terminal_id); + + return false; +end; +$$ + language plpgsql + volatile + strict; + +-- Compact bound-pair shortest-path searches share a session-local workspace. +-- The tables survive transaction boundaries so their catalog objects and +-- indexes are paid for once per physical connection. Every public executor +-- resets row state before use; an aborted call is therefore harmless to the +-- next invocation on the same pooled connection. +create or replace function public.ensure_shortest_dag_workspace() + returns void as +$$ +declare + expected_version constant int4 := 2; + present_version int4; +begin + if to_regclass('pg_temp.spd_workspace_version') is not null then + select version into present_version from pg_temp.spd_workspace_version limit 1; + end if; + + if to_regclass('pg_temp.spd_workspace_version') is not null + and present_version is distinct from expected_version then + drop table if exists pg_temp.spd_predecessor; + drop table if exists pg_temp.spd_candidate; + drop table if exists pg_temp.spd_seen; + drop table if exists pg_temp.spd_next; + drop table if exists pg_temp.spd_front; + drop table if exists pg_temp.spd_workspace_version; + end if; + + if to_regclass('pg_temp.spd_workspace_version') is null then + create temporary table spd_workspace_version + ( + version int4 not null primary key + ) on commit preserve rows; + + create temporary table spd_seen + ( + node_id int8 not null primary key, + depth int4 not null + ) on commit preserve rows; + + create temporary table spd_candidate + ( + node_id int8 not null, + depth int4 not null, + predecessor_id int8 not null, + edge_id int8 not null, + primary key (depth, node_id, predecessor_id, edge_id) + ) on commit preserve rows; + create index spd_candidate_node_id_depth_index + on spd_candidate using btree (node_id, depth); + + create temporary table spd_predecessor + ( + node_id int8 not null, + depth int4 not null, + predecessor_id int8 not null, + edge_id int8 not null, + primary key (node_id, depth, predecessor_id, edge_id) + ) on commit preserve rows; + create index spd_predecessor_predecessor_id_depth_index + on spd_predecessor using btree (predecessor_id, depth); + + insert into spd_workspace_version(version) values (expected_version); + end if; +end; +$$ + language plpgsql + volatile; + +create or replace function public.reset_shortest_dag_workspace() + returns void as +$$ +begin + perform public.ensure_shortest_dag_workspace(); + truncate table pg_temp.spd_seen, pg_temp.spd_candidate, pg_temp.spd_predecessor; +end; +$$ + language plpgsql + volatile; + +-- all_shortest_paths_dag separates minimum-depth discovery from path +-- enumeration. It retains every relationship-distinct predecessor edge at a +-- node's minimum depth, then enumerates only the resulting predecessor DAG. +-- The min_depth=1/distinct-endpoint contract is enforced by the production +-- selector; the guards below keep direct SQL callers honest as well. +create or replace function public.all_shortest_paths_dag(target_graph_id int4, source_id int8, target_id int8, + min_depth int4, max_depth int4, + edge_kind_ids int2[], inbound bool) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +#variable_conflict use_column +declare + search_depth int4; + target_depth int4; + emitted_count int8; +begin + if source_id is null or target_id is null or max_depth < 1 then + return; + end if; + if min_depth <> 1 then + raise exception using errcode = '22023', message = 'all_shortest_paths_dag requires min_depth = 1'; + end if; + if source_id = target_id then + perform public.shortest_path_self_endpoint_error(source_id, target_id); + end if; + + -- Exact depth-one fast arm. Every qualifying parallel edge is observable. + if not inbound then + return query + select source_id, target_id, 1::int4, true, false, array[e.id]::int8[] + from edge e + where e.graph_id = target_graph_id + and e.start_id = source_id and e.end_id = target_id + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + order by e.id; + else + return query + select source_id, target_id, 1::int4, true, false, array[e.id]::int8[] + from edge e + where e.graph_id = target_graph_id + and e.end_id = source_id and e.start_id = target_id + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + order by e.id; + end if; + get diagnostics emitted_count = row_count; + if emitted_count > 0 then + return; + end if; + + -- Exact depth-two fast arm. Relationship uniqueness is explicit so self + -- loops and reciprocal patterns cannot reuse one physical relationship. + if max_depth >= 2 then + if not inbound then + return query + select source_id, target_id, 2::int4, true, false, array[e1.id, e2.id]::int8[] + from edge e1 + join edge e2 on e2.graph_id = target_graph_id and e2.start_id = e1.end_id + where e1.graph_id = target_graph_id + and e1.start_id = source_id and e2.end_id = target_id + and e1.id <> e2.id + and (cardinality(edge_kind_ids) = 0 or e1.kind_id = any(edge_kind_ids)) + and (cardinality(edge_kind_ids) = 0 or e2.kind_id = any(edge_kind_ids)) + order by e1.id, e2.id; + else + return query + select source_id, target_id, 2::int4, true, false, array[e1.id, e2.id]::int8[] + from edge e1 + join edge e2 on e2.graph_id = target_graph_id and e2.end_id = e1.start_id + where e1.graph_id = target_graph_id + and e1.end_id = source_id and e2.start_id = target_id + and e1.id <> e2.id + and (cardinality(edge_kind_ids) = 0 or e1.kind_id = any(edge_kind_ids)) + and (cardinality(edge_kind_ids) = 0 or e2.kind_id = any(edge_kind_ids)) + order by e1.id, e2.id; + end if; + get diagnostics emitted_count = row_count; + if emitted_count > 0 then + return; + end if; + end if; + + if max_depth <= 2 then + return; + end if; + + perform public.reset_shortest_dag_workspace(); + insert into pg_temp.spd_seen(node_id, depth) values (source_id, 0); + + for search_depth in 1..max_depth loop + if not inbound then + insert into pg_temp.spd_candidate(node_id, depth, predecessor_id, edge_id) + select e.end_id, search_depth, f.node_id, e.id + from pg_temp.spd_seen f + join edge e on e.graph_id = target_graph_id and e.start_id = f.node_id + where f.depth = search_depth - 1 + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.spd_seen s where s.node_id = e.end_id) + on conflict do nothing; + else + insert into pg_temp.spd_candidate(node_id, depth, predecessor_id, edge_id) + select e.start_id, search_depth, f.node_id, e.id + from pg_temp.spd_seen f + join edge e on e.graph_id = target_graph_id and e.end_id = f.node_id + where f.depth = search_depth - 1 + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.spd_seen s where s.node_id = e.start_id) + on conflict do nothing; + end if; + + if not exists (select 1 from pg_temp.spd_candidate where depth = search_depth) then + exit; + end if; + + insert into pg_temp.spd_predecessor(node_id, depth, predecessor_id, edge_id) + select node_id, search_depth, predecessor_id, edge_id + from pg_temp.spd_candidate + where depth = search_depth + on conflict do nothing; + + insert into pg_temp.spd_seen(node_id, depth) + select distinct node_id, search_depth from pg_temp.spd_candidate + where depth = search_depth + on conflict do nothing; + + if exists (select 1 from pg_temp.spd_candidate where depth = search_depth and node_id = target_id) then + target_depth = search_depth; + exit; + end if; + end loop; + + if target_depth is null then + return; + end if; + + return query + with recursive shortest_paths(node_id, path_depth, edge_ids) as ( + select target_id, target_depth, array []::int8[] + union all + select predecessor.predecessor_id, + shortest_paths.path_depth - 1, + array[predecessor.edge_id]::int8[] || shortest_paths.edge_ids + from shortest_paths + join pg_temp.spd_predecessor predecessor + on predecessor.node_id = shortest_paths.node_id + and predecessor.depth = shortest_paths.path_depth + ) + select source_id, target_id, target_depth, true, false, shortest_paths.edge_ids + from shortest_paths + where shortest_paths.node_id = source_id and shortest_paths.path_depth = 0 + order by shortest_paths.edge_ids; +end; +$$ + language plpgsql + volatile + strict + cost 100 + set recursive_worktable_factor = 1 + rows 100; + +-- shortest_path_compact keeps one deterministic predecessor per minimum-depth +-- node. If its bounded state budget is exceeded it restarts an exact +-- relationship-trail recursive search before returning any row, preserving the +-- transaction snapshot and the incumbent relationship-simple semantics. +create or replace function public.shortest_path_compact(target_graph_id int4, source_id int8, target_id int8, + min_depth int4, max_depth int4, + edge_kind_ids int2[], inbound bool, + state_limit int8) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +#variable_conflict use_column +declare + search_depth int4; + target_depth int4; + emitted_count int8; + retained_state int8; + overflowed bool := false; +begin + if source_id is null or target_id is null or max_depth < min_depth then + return; + end if; + if min_depth <> 0 and min_depth <> 1 then + raise exception using errcode = '22023', message = 'shortest_path_compact requires min_depth = 0 or 1'; + end if; + if source_id = target_id then + if min_depth = 0 then + return query select source_id, target_id, 0::int4, true, false, array []::int8[]; + return; + end if; + perform public.shortest_path_self_endpoint_error(source_id, target_id); + end if; + + if min_depth <= 1 and max_depth >= 1 then + if not inbound then + return query + select source_id, target_id, 1::int4, true, false, array[e.id]::int8[] + from edge e + where e.graph_id = target_graph_id + and e.start_id = source_id and e.end_id = target_id + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + order by e.id limit 1; + else + return query + select source_id, target_id, 1::int4, true, false, array[e.id]::int8[] + from edge e + where e.graph_id = target_graph_id + and e.end_id = source_id and e.start_id = target_id + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + order by e.id limit 1; + end if; + get diagnostics emitted_count = row_count; + if emitted_count > 0 then + perform public.record_requested_traversal_runtime_attestation_v1('one_hop_preflight', false, 'SP-S4-C-WE+MAT-M0'); + return; + end if; + end if; + + if min_depth <= 2 and max_depth >= 2 then + if not inbound then + return query + select source_id, target_id, 2::int4, true, false, array[e1.id, e2.id]::int8[] + from edge e1 + join edge e2 on e2.graph_id = target_graph_id and e2.start_id = e1.end_id + where e1.graph_id = target_graph_id + and e1.start_id = source_id and e2.end_id = target_id and e1.id <> e2.id + and (cardinality(edge_kind_ids) = 0 or e1.kind_id = any(edge_kind_ids)) + and (cardinality(edge_kind_ids) = 0 or e2.kind_id = any(edge_kind_ids)) + order by e1.id, e2.id limit 1; + else + return query + select source_id, target_id, 2::int4, true, false, array[e1.id, e2.id]::int8[] + from edge e1 + join edge e2 on e2.graph_id = target_graph_id and e2.end_id = e1.start_id + where e1.graph_id = target_graph_id + and e1.end_id = source_id and e2.start_id = target_id and e1.id <> e2.id + and (cardinality(edge_kind_ids) = 0 or e1.kind_id = any(edge_kind_ids)) + and (cardinality(edge_kind_ids) = 0 or e2.kind_id = any(edge_kind_ids)) + order by e1.id, e2.id limit 1; + end if; + get diagnostics emitted_count = row_count; + if emitted_count > 0 then + perform public.record_requested_traversal_runtime_attestation_v1('two_hop_preflight', false, 'SP-S4-C-WE+MAT-M0'); + return; + end if; + end if; + + if max_depth <= 2 then + perform public.record_requested_traversal_runtime_attestation_v1('preflight_no_path', false, 'SP-S4-C-WE+MAT-M0'); + return; + end if; + + perform public.reset_shortest_dag_workspace(); + insert into pg_temp.spd_seen(node_id, depth) values (source_id, 0); + + for search_depth in 1..max_depth loop + if not inbound then + insert into pg_temp.spd_candidate(node_id, depth, predecessor_id, edge_id) + select distinct on (e.end_id) e.end_id, search_depth, f.node_id, e.id + from pg_temp.spd_seen f + join edge e on e.graph_id = target_graph_id and e.start_id = f.node_id + where f.depth = search_depth - 1 + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.spd_seen s where s.node_id = e.end_id) + order by e.end_id, e.id, f.node_id + limit case when state_limit > 0 then greatest(state_limit - (select count(*) from pg_temp.spd_seen) + 1, 0) else 9223372036854775807 end + on conflict do nothing; + else + insert into pg_temp.spd_candidate(node_id, depth, predecessor_id, edge_id) + select distinct on (e.start_id) e.start_id, search_depth, f.node_id, e.id + from pg_temp.spd_seen f + join edge e on e.graph_id = target_graph_id and e.end_id = f.node_id + where f.depth = search_depth - 1 + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.spd_seen s where s.node_id = e.start_id) + order by e.start_id, e.id, f.node_id + limit case when state_limit > 0 then greatest(state_limit - (select count(*) from pg_temp.spd_seen) + 1, 0) else 9223372036854775807 end + on conflict do nothing; + end if; + + if not exists (select 1 from pg_temp.spd_candidate where depth = search_depth) then + exit; + end if; + + if state_limit > 0 then + select (select count(*) from pg_temp.spd_seen) + + (select count(distinct node_id) from pg_temp.spd_candidate where depth = search_depth) + into retained_state; + if retained_state > state_limit then + overflowed = true; + exit; + end if; + end if; + + insert into pg_temp.spd_predecessor(node_id, depth, predecessor_id, edge_id) + select distinct on (node_id) node_id, search_depth, predecessor_id, edge_id + from pg_temp.spd_candidate + where depth = search_depth + order by node_id, edge_id, predecessor_id + on conflict do nothing; + + insert into pg_temp.spd_seen(node_id, depth) + select distinct node_id, search_depth from pg_temp.spd_candidate + where depth = search_depth + on conflict do nothing; + + if search_depth >= min_depth and exists ( + select 1 from pg_temp.spd_candidate where depth = search_depth and node_id = target_id + ) then + target_depth = search_depth; + exit; + end if; + end loop; + + if overflowed then + perform public.record_requested_traversal_runtime_attestation_v1('exact_relationship_trail_fallback', true, 'SP-S3-U-E+MAT-M0'); + if not inbound then + return query + with recursive trails(node_id, trail_depth, edge_ids) as ( + select source_id, 0::int4, array []::int8[] + union all + select e.end_id, trails.trail_depth + 1, trails.edge_ids || e.id + from trails + join edge e on e.graph_id = target_graph_id and e.start_id = trails.node_id + where trails.trail_depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not e.id = any(trails.edge_ids) + ) + select source_id, target_id, trails.trail_depth, true, false, trails.edge_ids + from trails + where trails.node_id = target_id and trails.trail_depth >= min_depth + order by trails.trail_depth, trails.edge_ids + limit 1; + else + return query + with recursive trails(node_id, trail_depth, edge_ids) as ( + select source_id, 0::int4, array []::int8[] + union all + select e.start_id, trails.trail_depth + 1, trails.edge_ids || e.id + from trails + join edge e on e.graph_id = target_graph_id and e.end_id = trails.node_id + where trails.trail_depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not e.id = any(trails.edge_ids) + ) + select source_id, target_id, trails.trail_depth, true, false, trails.edge_ids + from trails + where trails.node_id = target_id and trails.trail_depth >= min_depth + order by trails.trail_depth, trails.edge_ids + limit 1; + end if; + return; + end if; + + if target_depth is null then + perform public.record_requested_traversal_runtime_attestation_v1('compact_no_path', false, 'SP-S4-C-WE+MAT-M0'); + return; + end if; + + perform public.record_requested_traversal_runtime_attestation_v1('compact_workspace_witness', false, 'SP-S4-C-WE+MAT-M0'); + + return query + with recursive witness(node_id, path_depth, edge_ids) as ( + select target_id, target_depth, array []::int8[] + union all + select predecessor.predecessor_id, + witness.path_depth - 1, + array[predecessor.edge_id]::int8[] || witness.edge_ids + from witness + join pg_temp.spd_predecessor predecessor + on predecessor.node_id = witness.node_id + and predecessor.depth = witness.path_depth + ) + select source_id, target_id, target_depth, true, false, witness.edge_ids + from witness + where witness.node_id = source_id and witness.path_depth = 0 + order by witness.edge_ids + limit 1; +end; +$$ + language plpgsql + volatile + strict + cost 100 + set recursive_worktable_factor = 1 + rows 1; + +-- Compact bidirectional shortest-path candidates use a workspace that is +-- deliberately disjoint from spd_*. An overflow can therefore invoke the +-- production S4 executor in the same top-level statement without corrupting +-- either search. The version row makes pooled-session reuse fail closed when +-- the typed workspace shape changes. +create or replace function public.ensure_bidirectional_shortest_path_workspace() + returns void as +$$ +declare + expected_version constant int4 := 1; + present_version int4; +begin + if to_regclass('pg_temp.spb_workspace_version') is not null then + select version into present_version from pg_temp.spb_workspace_version limit 1; + end if; + + if to_regclass('pg_temp.spb_workspace_version') is not null + and present_version is distinct from expected_version then + drop table if exists pg_temp.spb_predecessor; + drop table if exists pg_temp.spb_candidate; + drop table if exists pg_temp.spb_active; + drop table if exists pg_temp.spb_seen; + drop table if exists pg_temp.spb_front; + drop table if exists pg_temp.spb_workspace_version; + end if; + + if to_regclass('pg_temp.spb_workspace_version') is null then + create temporary table spb_workspace_version + ( + version int4 not null primary key + ) on commit preserve rows; + + -- side is f for logical source search and b for reverse search from the + -- logical target. queue_order is a stable FIFO order for B1; B2 groups the + -- same ID-only rows by depth into complete levels. + create temporary table spb_front + ( + side char(1) not null, + node_id int8 not null, + depth int4 not null, + queue_order int8 not null, + primary key (side, node_id), + unique (side, queue_order), + check (side in ('f', 'b')) + ) on commit preserve rows; + create index spb_front_side_depth_index on spb_front using btree (side, depth, queue_order); + + create temporary table spb_seen + ( + side char(1) not null, + node_id int8 not null, + depth int4 not null, + primary key (side, node_id), + check (side in ('f', 'b')) + ) on commit preserve rows; + create index spb_seen_node_side_index on spb_seen using btree (node_id, side, depth); + + create temporary table spb_active + ( + side char(1) not null, + node_id int8 not null, + depth int4 not null, + primary key (side, node_id), + check (side in ('f', 'b')) + ) on commit preserve rows; + + create temporary table spb_candidate + ( + side char(1) not null, + node_id int8 not null, + depth int4 not null, + adjacent_id int8 not null, + edge_id int8 not null, + primary key (side, node_id), + check (side in ('f', 'b')) + ) on commit preserve rows; + + -- For f rows adjacent_id is the predecessor toward source. For b rows it + -- is the successor toward target. One stable edge is retained per node. + create temporary table spb_predecessor + ( + side char(1) not null, + node_id int8 not null, + depth int4 not null, + adjacent_id int8 not null, + edge_id int8 not null, + primary key (side, node_id), + check (side in ('f', 'b')) + ) on commit preserve rows; + create index spb_predecessor_adjacent_side_index on spb_predecessor using btree (adjacent_id, side, depth); + + insert into spb_workspace_version(version) values (expected_version); + end if; +end; +$$ + language plpgsql + volatile; + +create or replace function public.reset_bidirectional_shortest_path_workspace() + returns void as +$$ +begin + perform public.ensure_bidirectional_shortest_path_workspace(); + truncate table pg_temp.spb_front, pg_temp.spb_seen, pg_temp.spb_active, + pg_temp.spb_candidate, pg_temp.spb_predecessor; +end; +$$ + language plpgsql + volatile; + +-- Runtime receipts bind a GraphBench latency sample to the branch executed by +-- that exact statement. The receipt is armed and read outside the timed block +-- on the same session. Instrumentation is inert unless an invocation is armed. +create or replace function public.ensure_traversal_runtime_attestation_workspace_v1() + returns void as +$$ +begin + if to_regclass('pg_temp.traversal_runtime_attestation_v1') is null then + create temporary table traversal_runtime_attestation_v1 + ( + invocation_id text not null primary key, + requested_identity text not null, + runtime_identity text, + runtime_branch text, + fallback_executed bool, + record_count int4 not null default 0, + events jsonb not null default '[]'::jsonb, + check (btrim(invocation_id) <> ''), + check (btrim(requested_identity) <> '') + ) on commit preserve rows; + end if; + -- Avoid issuing even a no-op ALTER in ordinary read-only transactions. + -- The conditional branch is retained for pooled sessions whose temporary + -- v1 receipt table predates the event-chain column. + if not exists ( + select 1 + from pg_attribute + where attrelid = 'pg_temp.traversal_runtime_attestation_v1'::regclass + and attname = 'events' + and not attisdropped + ) then + alter table pg_temp.traversal_runtime_attestation_v1 + add column events jsonb not null default '[]'::jsonb; + end if; +end; +$$ + language plpgsql + volatile; + +create or replace function public.begin_traversal_runtime_attestation_v1( + target_invocation_id text, + target_requested_identity text) + returns void as +$$ +begin + if target_invocation_id is null or btrim(target_invocation_id) = '' or length(target_invocation_id) > 256 then + raise exception using errcode = '22023', message = 'traversal runtime invocation ID must contain 1 to 256 characters'; + end if; + if target_requested_identity is null or btrim(target_requested_identity) = '' or length(target_requested_identity) > 256 then + raise exception using errcode = '22023', message = 'traversal runtime requested identity must contain 1 to 256 characters'; + end if; + perform public.ensure_traversal_runtime_attestation_workspace_v1(); + delete from pg_temp.traversal_runtime_attestation_v1 where invocation_id = target_invocation_id; + insert into pg_temp.traversal_runtime_attestation_v1(invocation_id, requested_identity) + values (target_invocation_id, target_requested_identity); + -- Session scope deliberately survives the arming autocommit. The matching + -- clear call executes immediately after the timed statement. + perform set_config('dawgs.traversal_runtime_invocation_id', target_invocation_id, false); +end; +$$ + language plpgsql + volatile + strict; + +create or replace function public.record_traversal_runtime_attestation_v1( + target_runtime_identity text, + target_runtime_branch text, + target_fallback_executed bool) + returns bool as +$$ +declare + target_invocation_id text := nullif(current_setting('dawgs.traversal_runtime_invocation_id', true), ''); +begin + if target_invocation_id is null then + return true; + end if; + update pg_temp.traversal_runtime_attestation_v1 receipt + set runtime_identity = target_runtime_identity, + runtime_branch = target_runtime_branch, + fallback_executed = coalesce(receipt.fallback_executed, false) or target_fallback_executed, + record_count = receipt.record_count + 1, + events = receipt.events || jsonb_build_array(jsonb_build_object( + 'ordinal', receipt.record_count + 1, + 'runtime_identity', target_runtime_identity, + 'runtime_branch', target_runtime_branch, + 'fallback_executed', target_fallback_executed + )) + where receipt.invocation_id = target_invocation_id; + if not found then + raise exception using errcode = '55000', message = 'traversal runtime receipt is missing'; + end if; + return true; +end; +$$ + language plpgsql + volatile + strict; + +create or replace function public.record_requested_traversal_runtime_attestation_v1( + target_runtime_branch text, + target_fallback_executed bool, + target_fallback_identity text) + returns bool as +$$ +declare + target_invocation_id text := nullif(current_setting('dawgs.traversal_runtime_invocation_id', true), ''); + target_requested_identity text; +begin + if target_invocation_id is null then + return true; + end if; + select requested_identity into target_requested_identity + from pg_temp.traversal_runtime_attestation_v1 + where invocation_id = target_invocation_id; + if target_requested_identity is null then + raise exception using errcode = '55000', message = 'armed traversal runtime receipt is missing'; + end if; + if target_fallback_executed and target_fallback_identity = 'SP-S4' then + target_fallback_identity = case when target_requested_identity like '%-D' + then 'SP-S4-C-D' else 'SP-S4-C-WE+MAT-M0' end; + end if; + return public.record_traversal_runtime_attestation_v1( + case when target_fallback_executed then target_fallback_identity else target_requested_identity end, + target_runtime_branch, + target_fallback_executed + ); +end; +$$ + language plpgsql + volatile + strict; + +create or replace function public.read_traversal_runtime_attestation_v1(target_invocation_id text) + returns jsonb as +$$ +begin + return ( + select jsonb_build_object( + 'schema_version', 2, + 'invocation_id', invocation_id, + 'requested_identity', requested_identity, + 'runtime_identity', runtime_identity, + 'runtime_branch', runtime_branch, + 'fallback_executed', fallback_executed, + 'record_count', record_count, + 'events', events + ) + from pg_temp.traversal_runtime_attestation_v1 + where invocation_id = target_invocation_id + ); +end; +$$ + language plpgsql + stable + strict; + +create or replace function public.clear_traversal_runtime_attestation_v1(target_invocation_id text) + returns void as +$$ +begin + if to_regclass('pg_temp.traversal_runtime_attestation_v1') is not null then + delete from pg_temp.traversal_runtime_attestation_v1 where invocation_id = target_invocation_id; + end if; + if nullif(current_setting('dawgs.traversal_runtime_invocation_id', true), '') = target_invocation_id then + perform set_config('dawgs.traversal_runtime_invocation_id', '', false); + end if; +end; +$$ + language plpgsql + volatile + strict; + +-- Detailed bidirectional SP counters live in a second, independently +-- versioned temporary workspace. GraphBench enables this workspace only for +-- an untimed replay. The transaction-local invocation setting means pooled +-- sessions cannot accidentally attribute a later statement to an earlier +-- replay, while the explicit invocation key keeps every row attributable. +create or replace function public.ensure_bidirectional_shortest_path_telemetry_workspace() + returns void as +$$ +declare + expected_version constant int4 := 1; + present_version int4; +begin + if to_regclass('pg_temp.spb_telemetry_workspace_version') is not null then + select version into present_version + from pg_temp.spb_telemetry_workspace_version + limit 1; + end if; + + if to_regclass('pg_temp.spb_telemetry_workspace_version') is not null + and present_version is distinct from expected_version then + drop table if exists pg_temp.spb_telemetry_level; + drop table if exists pg_temp.spb_telemetry_call; + drop table if exists pg_temp.spb_telemetry_invocation; + drop table if exists pg_temp.spb_telemetry_workspace_version; + end if; + + if to_regclass('pg_temp.spb_telemetry_workspace_version') is null then + create temporary table spb_telemetry_workspace_version + ( + version int4 not null primary key + ) on commit preserve rows; + + create temporary table spb_telemetry_invocation + ( + invocation_id text not null primary key, + schema_version int4 not null, + scheduler text, + state_limit int8, + frontier_limit int8, + predecessor_limit int8, + next_search_id int8 not null default 0, + check (btrim(invocation_id) <> '') + ) on commit preserve rows; + + create temporary table spb_telemetry_call + ( + invocation_id text not null, + search_id int8 not null, + source_id int8 not null, + target_id int8 not null, + runtime_branch text not null default 'started', + scheduler_actions int8 not null default 0, + candidate_edges int8 not null default 0, + distinct_new_nodes int8 not null default 0, + seen_peak int8 not null default 0, + frontier_peak int8 not null default 0, + queue_peak int8 not null default 0, + predecessor_peak int8 not null default 0, + meeting_candidates int8 not null default 0, + frozen_distance int4, + witness_rows int8 not null default 0, + overflowed bool not null default false, + fallback_executed bool not null default false, + primary key (invocation_id, search_id) + ) on commit preserve rows; + + create temporary table spb_telemetry_level + ( + invocation_id text not null, + search_id int8 not null, + action_index int8 not null, + side text not null, + action text not null, + depth int4 not null, + frontier_rows int8 not null, + candidate_edges int8 not null, + distinct_new_nodes int8 not null, + seen_rows int8 not null, + queue_rows int8 not null, + predecessor_rows int8 not null, + meeting_candidates int8 not null, + primary key (invocation_id, search_id, action_index) + ) on commit preserve rows; + + insert into spb_telemetry_workspace_version(version) values (expected_version); + end if; +end; +$$ + language plpgsql + volatile; + +-- begin_bidirectional_shortest_path_diagnostic_v1 must be called inside the +-- same explicit transaction and on the same PostgreSQL connection as the +-- diagnostic replay. It clears only its own invocation key and enables +-- instrumentation through a transaction-local setting. +create or replace function public.begin_bidirectional_shortest_path_diagnostic_v1(invocation_id text) + returns void as +$$ +begin + if invocation_id is null or btrim(invocation_id) = '' or length(invocation_id) > 256 then + raise exception using errcode = '22023', message = 'bidirectional shortest-path diagnostic invocation ID must contain 1 to 256 characters'; + end if; + + perform public.ensure_bidirectional_shortest_path_telemetry_workspace(); + delete from pg_temp.spb_telemetry_level where spb_telemetry_level.invocation_id = begin_bidirectional_shortest_path_diagnostic_v1.invocation_id; + delete from pg_temp.spb_telemetry_call where spb_telemetry_call.invocation_id = begin_bidirectional_shortest_path_diagnostic_v1.invocation_id; + delete from pg_temp.spb_telemetry_invocation where spb_telemetry_invocation.invocation_id = begin_bidirectional_shortest_path_diagnostic_v1.invocation_id; + insert into pg_temp.spb_telemetry_invocation(invocation_id, schema_version) + values (invocation_id, 1); + perform set_config('dawgs.spb_diagnostic_invocation_id', invocation_id, true); +end; +$$ + language plpgsql + volatile; + +-- The reader returns one self-describing document. Aggregate counters support +-- the common single-bound-pair replay, while calls preserve exact per-pair +-- attribution if a translated statement invokes the kernel more than once. +create or replace function public.read_bidirectional_shortest_path_diagnostic_v1(target_invocation_id text) + returns jsonb as +$$ +declare + result jsonb; +begin + select jsonb_build_object( + 'schema_version', invocation.schema_version, + 'invocation_id', invocation.invocation_id, + 'scheduler', invocation.scheduler, + 'state_limit', invocation.state_limit, + 'frontier_limit', invocation.frontier_limit, + 'predecessor_limit', invocation.predecessor_limit, + 'search_calls', coalesce(call_totals.search_calls, 0), + 'runtime_branch', coalesce(call_totals.runtime_branch, 'missing'), + 'overflowed', coalesce(call_totals.overflowed, false), + 'fallback_executed', coalesce(call_totals.fallback_executed, false), + 'counters', jsonb_build_object( + 'scheduler_actions', coalesce(call_totals.scheduler_actions, 0), + 'candidate_edges', coalesce(call_totals.candidate_edges, 0), + 'distinct_new_nodes', coalesce(call_totals.distinct_new_nodes, 0), + 'seen_peak', coalesce(call_totals.seen_peak, 0), + 'frontier_peak', coalesce(call_totals.frontier_peak, 0), + 'queue_peak', coalesce(call_totals.queue_peak, 0), + 'predecessor_peak', coalesce(call_totals.predecessor_peak, 0), + 'meeting_candidates', coalesce(call_totals.meeting_candidates, 0), + -- -1 is the explicit no-frozen-meeting sentinel. Exact values are + -- retained per call below when a statement evaluates many pairs. + 'frozen_distance', coalesce(call_totals.frozen_distance, -1), + 'witness_rows', coalesce(call_totals.witness_rows, 0), + 'levels', coalesce(levels.rows, '[]'::jsonb) + ), + 'calls', coalesce(calls.rows, '[]'::jsonb) + ) +into result +from pg_temp.spb_telemetry_invocation invocation +left join lateral ( + select count(*)::int8 as search_calls, + case when count(distinct call.runtime_branch) = 1 + then min(call.runtime_branch) else 'mixed' end as runtime_branch, + bool_or(call.overflowed) as overflowed, + bool_or(call.fallback_executed) as fallback_executed, + sum(call.scheduler_actions)::int8 as scheduler_actions, + sum(call.candidate_edges)::int8 as candidate_edges, + sum(call.distinct_new_nodes)::int8 as distinct_new_nodes, + max(call.seen_peak)::int8 as seen_peak, + max(call.frontier_peak)::int8 as frontier_peak, + max(call.queue_peak)::int8 as queue_peak, + max(call.predecessor_peak)::int8 as predecessor_peak, + sum(call.meeting_candidates)::int8 as meeting_candidates, + min(call.frozen_distance)::int4 as frozen_distance, + sum(call.witness_rows)::int8 as witness_rows + from pg_temp.spb_telemetry_call call + where call.invocation_id = invocation.invocation_id +) call_totals on true +left join lateral ( + select jsonb_agg(jsonb_build_object( + 'search_id', level.search_id, + 'action_index', level.action_index, + 'side', level.side, + 'action', level.action, + 'depth', level.depth, + 'frontier_rows', level.frontier_rows, + 'candidate_edges', level.candidate_edges, + 'distinct_new_nodes', level.distinct_new_nodes, + 'seen_rows', level.seen_rows, + 'queue_rows', level.queue_rows, + 'predecessor_rows', level.predecessor_rows, + 'meeting_candidates', level.meeting_candidates + ) order by level.search_id, level.action_index) as rows + from pg_temp.spb_telemetry_level level + where level.invocation_id = invocation.invocation_id +) levels on true +left join lateral ( + select jsonb_agg(to_jsonb(call) - 'invocation_id' order by call.search_id) as rows + from pg_temp.spb_telemetry_call call + where call.invocation_id = invocation.invocation_id +) calls on true + where invocation.invocation_id = target_invocation_id; + return result; +end; +$$ + language plpgsql + stable + strict; + +create or replace function public.clear_bidirectional_shortest_path_diagnostic_v1(target_invocation_id text) + returns void as +$$ +begin + if to_regclass('pg_temp.spb_telemetry_invocation') is not null then + delete from pg_temp.spb_telemetry_level where invocation_id = target_invocation_id; + delete from pg_temp.spb_telemetry_call where invocation_id = target_invocation_id; + delete from pg_temp.spb_telemetry_invocation where invocation_id = target_invocation_id; + end if; + if nullif(current_setting('dawgs.spb_diagnostic_invocation_id', true), '') = target_invocation_id then + perform set_config('dawgs.spb_diagnostic_invocation_id', '', true); + end if; +end; +$$ + language plpgsql + volatile + strict; + +create or replace function public._start_bidirectional_shortest_path_diagnostic_call_v1( + target_invocation_id text, + target_scheduler text, + target_state_limit int8, + target_frontier_limit int8, + target_predecessor_limit int8, + target_source_id int8, + target_target_id int8) + returns int8 as +$$ +declare + target_search_id int8; +begin + if target_invocation_id is null then + return null; + end if; + if to_regclass('pg_temp.spb_telemetry_invocation') is null then + raise exception using errcode = '55000', message = 'bidirectional shortest-path diagnostic replay was not initialized on this session'; + end if; + + update pg_temp.spb_telemetry_invocation invocation + set scheduler = coalesce(invocation.scheduler, target_scheduler), + state_limit = coalesce(invocation.state_limit, target_state_limit), + frontier_limit = coalesce(invocation.frontier_limit, target_frontier_limit), + predecessor_limit = coalesce(invocation.predecessor_limit, target_predecessor_limit), + next_search_id = invocation.next_search_id + 1 + where invocation.invocation_id = target_invocation_id + and (invocation.scheduler is null or invocation.scheduler = target_scheduler) + and (invocation.state_limit is null or invocation.state_limit = target_state_limit) + and (invocation.frontier_limit is null or invocation.frontier_limit = target_frontier_limit) + and (invocation.predecessor_limit is null or invocation.predecessor_limit = target_predecessor_limit) + returning invocation.next_search_id into target_search_id; + + if target_search_id is null then + raise exception using + errcode = '55000', + message = 'bidirectional shortest-path diagnostic invocation is missing or mixes scheduler/cap identities'; + end if; + + insert into pg_temp.spb_telemetry_call(invocation_id, search_id, source_id, target_id) + values (target_invocation_id, target_search_id, target_source_id, target_target_id); + return target_search_id; +end; +$$ + language plpgsql + volatile; + +create or replace function public._record_bidirectional_shortest_path_diagnostic_level_v1( + target_invocation_id text, + target_search_id int8, + target_action_index int8, + target_side text, + target_action text, + target_depth int4, + target_frontier_rows int8, + target_candidate_edges int8, + target_distinct_new_nodes int8, + target_seen_rows int8, + target_queue_rows int8, + target_predecessor_rows int8, + target_meeting_candidates int8) + returns void as +$$ +begin + insert into pg_temp.spb_telemetry_level( + invocation_id, search_id, action_index, side, action, depth, + frontier_rows, candidate_edges, distinct_new_nodes, seen_rows, + queue_rows, predecessor_rows, meeting_candidates) + values ( + target_invocation_id, target_search_id, target_action_index, target_side, + target_action, target_depth, target_frontier_rows, target_candidate_edges, + target_distinct_new_nodes, target_seen_rows, target_queue_rows, + target_predecessor_rows, target_meeting_candidates); +end; +$$ + language plpgsql + volatile + strict; + +create or replace function public._finish_bidirectional_shortest_path_diagnostic_call_v1( + target_invocation_id text, + target_search_id int8, + target_runtime_branch text, + target_scheduler_actions int8, + target_candidate_edges int8, + target_distinct_new_nodes int8, + target_seen_peak int8, + target_frontier_peak int8, + target_queue_peak int8, + target_predecessor_peak int8, + target_meeting_candidates int8, + target_frozen_distance int4, + target_witness_rows int8, + target_overflowed bool, + target_fallback_executed bool) + returns void as +$$ +begin + update pg_temp.spb_telemetry_call call + set runtime_branch = target_runtime_branch, + scheduler_actions = target_scheduler_actions, + candidate_edges = target_candidate_edges, + distinct_new_nodes = target_distinct_new_nodes, + seen_peak = target_seen_peak, + frontier_peak = target_frontier_peak, + queue_peak = target_queue_peak, + predecessor_peak = target_predecessor_peak, + meeting_candidates = target_meeting_candidates, + frozen_distance = target_frozen_distance, + witness_rows = target_witness_rows, + overflowed = target_overflowed, + fallback_executed = target_fallback_executed + where call.invocation_id = target_invocation_id + and call.search_id = target_search_id; + + if not found then + raise exception using errcode = '55000', message = 'bidirectional shortest-path diagnostic call is missing'; + end if; +end; +$$ + language plpgsql + volatile; + +-- shortest_path_bidirectional_compact_v1 is the common typed kernel for the +-- B1 and B2 tournament arms. Queue-head depths are lower bounds on every +-- undiscovered source/target distance. Once their sum is at least the best +-- completed meeting distance, no unexpanded pair can produce a shorter path. +-- B1 applies this proof after deterministic one-node alternation; B2 applies it +-- only between complete-level expansions. Merely finding an intersection is +-- never a termination condition. +-- +-- Admission is fail-closed. Candidate state is materialized with LIMIT cap+1 +-- before any seen/front/predecessor mutation. If total seen rows, queued +-- frontier rows, or retained predecessors exceed their independent bound, the +-- function invokes exact S4 before returning any candidate row. VOLATILE +-- PL/pgSQL statements do not provide one transaction snapshot at READ +-- COMMITTED, so the kernel rejects that isolation level. At REPEATABLE READ or +-- SERIALIZABLE, candidate search and nested S4 fallback observe the same +-- transaction snapshot; spb_/spd_ state remains disjoint. +create or replace function public.shortest_path_bidirectional_compact_v1( + target_graph_id int4, + source_id int8, + target_id int8, + min_depth int4, + max_depth int4, + edge_kind_ids int2[], + inbound bool, + state_limit int8, + frontier_limit int8, + predecessor_limit int8, + scheduler text) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +#variable_conflict use_column +declare + chosen_side char(1); + strict_side char(1) := 'f'; + forward_depth int4; + backward_depth int4; + forward_width int8; + backward_width int8; + forward_tail int8 := 0; + backward_tail int8 := 0; + seen_rows int8; + active_rows int8; + frontier_rows int8; + predecessor_rows int8; + candidate_rows int8; + admission_limit int8; + candidate_meeting int8; + candidate_distance int4; + best_meeting int8; + best_distance int4; + emitted_count int8; + overflowed bool := false; + telemetry_invocation_id text := nullif(current_setting('dawgs.spb_diagnostic_invocation_id', true), ''); + telemetry_search_id int8; + telemetry_action_index int8 := 0; + telemetry_action_depth int4 := 0; + telemetry_action_candidate_edges int8 := 0; + telemetry_action_meetings int8 := 0; + telemetry_scheduler_actions int8 := 0; + telemetry_candidate_edges int8 := 0; + telemetry_distinct_new_nodes int8 := 0; + telemetry_seen_peak int8 := 0; + telemetry_frontier_peak int8 := 0; + telemetry_queue_peak int8 := 0; + telemetry_predecessor_peak int8 := 0; + telemetry_meeting_candidates int8 := 0; +begin + if source_id is null or target_id is null or max_depth < min_depth then + return; + end if; + if scheduler <> 'strict_alternating_node' and scheduler <> 'smaller_current_level' then + raise exception using errcode = '22023', message = 'unknown compact bidirectional shortest-path scheduler'; + end if; + if min_depth <> 0 and min_depth <> 1 then + raise exception using errcode = '22023', message = 'compact bidirectional shortest path requires min_depth = 0 or 1'; + end if; + if max_depth > 64 then + raise exception using errcode = '22023', message = 'compact bidirectional shortest path requires max_depth <= 64'; + end if; + if state_limit <= 0 or frontier_limit <= 0 or predecessor_limit <= 0 then + raise exception using errcode = '22023', message = 'compact bidirectional shortest path requires positive state, frontier, and predecessor limits'; + end if; + if current_setting('transaction_isolation') <> 'repeatable read' + and current_setting('transaction_isolation') <> 'serializable' then + raise exception using + errcode = '25001', + message = 'compact bidirectional shortest path requires REPEATABLE READ or SERIALIZABLE transaction isolation'; + end if; + + telemetry_search_id = public._start_bidirectional_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, scheduler, state_limit, frontier_limit, + predecessor_limit, source_id, target_id); + + -- Exact zero-hop preflight precedes workspace allocation. + if source_id = target_id then + if min_depth = 0 then + return query select source_id, target_id, 0::int4, true, false, array []::int8[]; + get diagnostics emitted_count = row_count; + if telemetry_search_id is not null then + telemetry_action_index = telemetry_action_index + 1; + perform public._record_bidirectional_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + 'none', 'preflight_zero_hop', 0, 0, 0, 0, 0, 0, 0, emitted_count); + perform public._finish_bidirectional_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'zero_hop_preflight', + 0, 0, 0, 0, 0, 0, 0, emitted_count, 0, emitted_count, false, false); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('zero_hop_preflight', false, 'SP-S4'); + return; + end if; + perform public.shortest_path_self_endpoint_error(source_id, target_id); + end if; + + -- Exact one-hop preflight chooses the same deterministic edge ordering as S4. + if min_depth <= 1 and max_depth >= 1 then + if not inbound then + return query + select source_id, target_id, 1::int4, true, false, array[e.id]::int8[] + from edge e + where e.graph_id = target_graph_id + and e.start_id = source_id and e.end_id = target_id + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + order by e.id limit 1; + else + return query + select source_id, target_id, 1::int4, true, false, array[e.id]::int8[] + from edge e + where e.graph_id = target_graph_id + and e.end_id = source_id and e.start_id = target_id + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + order by e.id limit 1; + end if; + get diagnostics emitted_count = row_count; + if emitted_count > 0 then + if telemetry_search_id is not null then + if not inbound then + select count(*) into telemetry_action_candidate_edges + from edge e + where e.graph_id = target_graph_id + and e.start_id = source_id and e.end_id = target_id + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)); + else + select count(*) into telemetry_action_candidate_edges + from edge e + where e.graph_id = target_graph_id + and e.end_id = source_id and e.start_id = target_id + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)); + end if; + telemetry_candidate_edges = telemetry_candidate_edges + telemetry_action_candidate_edges; + telemetry_meeting_candidates = telemetry_meeting_candidates + telemetry_action_candidate_edges; + telemetry_action_index = telemetry_action_index + 1; + perform public._record_bidirectional_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + 'none', 'preflight_one_hop', 1, 0, telemetry_action_candidate_edges, + 0, 0, 0, 0, telemetry_action_candidate_edges); + perform public._finish_bidirectional_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'one_hop_preflight', + 0, telemetry_candidate_edges, 0, 0, 0, 0, 0, + telemetry_meeting_candidates, 1, emitted_count, false, false); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('one_hop_preflight', false, 'SP-S4'); + return; + end if; + end if; + + -- Exact two-hop preflight retains relationship uniqueness and public order. + if min_depth <= 2 and max_depth >= 2 then + if not inbound then + return query + select source_id, target_id, 2::int4, true, false, array[e1.id, e2.id]::int8[] + from edge e1 + join edge e2 on e2.graph_id = target_graph_id and e2.start_id = e1.end_id + where e1.graph_id = target_graph_id + and e1.start_id = source_id and e2.end_id = target_id and e1.id <> e2.id + and (cardinality(edge_kind_ids) = 0 or e1.kind_id = any(edge_kind_ids)) + and (cardinality(edge_kind_ids) = 0 or e2.kind_id = any(edge_kind_ids)) + order by e1.id, e2.id limit 1; + else + return query + select source_id, target_id, 2::int4, true, false, array[e1.id, e2.id]::int8[] + from edge e1 + join edge e2 on e2.graph_id = target_graph_id and e2.end_id = e1.start_id + where e1.graph_id = target_graph_id + and e1.end_id = source_id and e2.start_id = target_id and e1.id <> e2.id + and (cardinality(edge_kind_ids) = 0 or e1.kind_id = any(edge_kind_ids)) + and (cardinality(edge_kind_ids) = 0 or e2.kind_id = any(edge_kind_ids)) + order by e1.id, e2.id limit 1; + end if; + get diagnostics emitted_count = row_count; + if emitted_count > 0 then + if telemetry_search_id is not null then + if not inbound then + select count(*) * 2 into telemetry_action_candidate_edges + from edge e1 + join edge e2 on e2.graph_id = target_graph_id and e2.start_id = e1.end_id + where e1.graph_id = target_graph_id + and e1.start_id = source_id and e2.end_id = target_id and e1.id <> e2.id + and (cardinality(edge_kind_ids) = 0 or e1.kind_id = any(edge_kind_ids)) + and (cardinality(edge_kind_ids) = 0 or e2.kind_id = any(edge_kind_ids)); + else + select count(*) * 2 into telemetry_action_candidate_edges + from edge e1 + join edge e2 on e2.graph_id = target_graph_id and e2.end_id = e1.start_id + where e1.graph_id = target_graph_id + and e1.end_id = source_id and e2.start_id = target_id and e1.id <> e2.id + and (cardinality(edge_kind_ids) = 0 or e1.kind_id = any(edge_kind_ids)) + and (cardinality(edge_kind_ids) = 0 or e2.kind_id = any(edge_kind_ids)); + end if; + telemetry_candidate_edges = telemetry_candidate_edges + telemetry_action_candidate_edges; + telemetry_action_meetings = telemetry_action_candidate_edges / 2; + telemetry_meeting_candidates = telemetry_meeting_candidates + telemetry_action_meetings; + telemetry_action_index = telemetry_action_index + 1; + perform public._record_bidirectional_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + 'none', 'preflight_two_hop', 2, 0, telemetry_action_candidate_edges, + 0, 0, 0, 0, telemetry_action_meetings); + perform public._finish_bidirectional_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'two_hop_preflight', + 0, telemetry_candidate_edges, 0, 0, 0, 0, 0, + telemetry_meeting_candidates, 2, emitted_count, false, false); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('two_hop_preflight', false, 'SP-S4'); + return; + end if; + end if; + if max_depth <= 2 then + if telemetry_search_id is not null then + telemetry_action_index = telemetry_action_index + 1; + perform public._record_bidirectional_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + 'none', 'preflight_no_path', max_depth, 0, 0, 0, 0, 0, 0, 0); + perform public._finish_bidirectional_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'preflight_no_path', + 0, 0, 0, 0, 0, 0, 0, 0, null, 0, false, false); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('preflight_no_path', false, 'SP-S4'); + return; + end if; + + -- Both roots count toward seen and frontier admission. Overflow falls back + -- before allocating or exposing candidate state. + if state_limit < 2 or frontier_limit < 2 then + overflowed = true; + if telemetry_search_id is not null then + telemetry_action_index = telemetry_action_index + 1; + perform public._record_bidirectional_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + 'none', 'root_admission', 0, 2, 0, 0, 2, 2, 0, 0); + telemetry_frontier_peak = 2; + telemetry_queue_peak = 2; + end if; + else + perform public.reset_bidirectional_shortest_path_workspace(); + insert into pg_temp.spb_front(side, node_id, depth, queue_order) + values ('f', source_id, 0, 0), ('b', target_id, 0, 0); + insert into pg_temp.spb_seen(side, node_id, depth) + values ('f', source_id, 0), ('b', target_id, 0); + telemetry_seen_peak = 2; + telemetry_frontier_peak = 2; + telemetry_queue_peak = 2; + end if; + + while not overflowed loop + select min(depth), count(*) filter (where depth = (select min(depth) from pg_temp.spb_front where side = 'f')) + into forward_depth, forward_width + from pg_temp.spb_front where side = 'f'; + select min(depth), count(*) filter (where depth = (select min(depth) from pg_temp.spb_front where side = 'b')) + into backward_depth, backward_width + from pg_temp.spb_front where side = 'b'; + + if forward_depth is null or backward_depth is null then + exit; + end if; + + -- Dijkstra/BFS lower bound over the two next accepted queue depths. + if best_distance is not null and forward_depth + backward_depth >= best_distance then + exit; + end if; + + truncate table pg_temp.spb_active, pg_temp.spb_candidate; + if scheduler = 'strict_alternating_node' then + chosen_side = strict_side; + if (chosen_side = 'f' and forward_width = 0) or (chosen_side = 'b' and backward_width = 0) then + chosen_side = case chosen_side when 'f' then 'b' else 'f' end; + end if; + strict_side = case chosen_side when 'f' then 'b' else 'f' end; + + insert into pg_temp.spb_active(side, node_id, depth) + select side, node_id, depth + from pg_temp.spb_front + where side = chosen_side + order by queue_order + limit 1; + else + -- B2 expands the complete smaller current level. Equality always chooses + -- the forward side, freezing the tie break across artifacts. + chosen_side = case when forward_width <= backward_width then 'f' else 'b' end; + insert into pg_temp.spb_active(side, node_id, depth) + select side, node_id, depth + from pg_temp.spb_front + where side = chosen_side + and depth = case chosen_side when 'f' then forward_depth else backward_depth end + order by queue_order; + end if; + + delete from pg_temp.spb_front front + using pg_temp.spb_active active + where front.side = active.side and front.node_id = active.node_id; + + telemetry_scheduler_actions = telemetry_scheduler_actions + 1; + telemetry_action_candidate_edges = 0; + telemetry_action_meetings = 0; + select min(depth) into telemetry_action_depth from pg_temp.spb_active; + + if not exists (select 1 from pg_temp.spb_active where depth < max_depth) then + if telemetry_search_id is not null then + select count(*) into seen_rows from pg_temp.spb_seen; + select count(*) into active_rows from pg_temp.spb_active; + select count(*) into frontier_rows from pg_temp.spb_front; + select count(*) into predecessor_rows from pg_temp.spb_predecessor; + telemetry_action_index = telemetry_action_index + 1; + telemetry_seen_peak = greatest(telemetry_seen_peak, seen_rows); + telemetry_frontier_peak = greatest(telemetry_frontier_peak, active_rows + frontier_rows); + telemetry_queue_peak = greatest(telemetry_queue_peak, frontier_rows); + telemetry_predecessor_peak = greatest(telemetry_predecessor_peak, predecessor_rows); + perform public._record_bidirectional_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + chosen_side::text, + case scheduler when 'strict_alternating_node' then 'dequeue_node' else 'expand_level' end, + telemetry_action_depth, active_rows + frontier_rows, 0, 0, + seen_rows, frontier_rows, predecessor_rows, 0); + end if; + continue; + end if; + + select count(*) into seen_rows from pg_temp.spb_seen; + select count(*) into active_rows from pg_temp.spb_active; + select count(*) into frontier_rows from pg_temp.spb_front; + select count(*) into predecessor_rows from pg_temp.spb_predecessor; + admission_limit = least(state_limit - seen_rows, + frontier_limit - active_rows - frontier_rows, + predecessor_limit - predecessor_rows); + if admission_limit < 0 then + overflowed = true; + if telemetry_search_id is not null then + telemetry_action_index = telemetry_action_index + 1; + telemetry_seen_peak = greatest(telemetry_seen_peak, seen_rows); + telemetry_frontier_peak = greatest(telemetry_frontier_peak, active_rows + frontier_rows); + telemetry_queue_peak = greatest(telemetry_queue_peak, frontier_rows); + telemetry_predecessor_peak = greatest(telemetry_predecessor_peak, predecessor_rows); + perform public._record_bidirectional_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + chosen_side::text, + case scheduler when 'strict_alternating_node' then 'dequeue_node' else 'expand_level' end, + telemetry_action_depth, active_rows + frontier_rows, 0, 0, + seen_rows, frontier_rows, predecessor_rows, 0); + end if; + exit; + end if; + + -- Candidate selection is graph scoped, ID only, and bounded at cap+1. + -- DISTINCT ON freezes one predecessor/successor before workspace mutation. + if chosen_side = 'f' and not inbound then + if telemetry_search_id is not null then + select count(*) into telemetry_action_candidate_edges + from pg_temp.spb_active active + join edge e on e.graph_id = target_graph_id and e.start_id = active.node_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.spb_seen seen where seen.side = 'f' and seen.node_id = e.end_id); + end if; + insert into pg_temp.spb_candidate(side, node_id, depth, adjacent_id, edge_id) + select 'f', candidate.node_id, candidate.depth, candidate.adjacent_id, candidate.edge_id + from ( + select distinct on (e.end_id) e.end_id as node_id, active.depth + 1 as depth, + active.node_id as adjacent_id, e.id as edge_id + from pg_temp.spb_active active + join edge e on e.graph_id = target_graph_id and e.start_id = active.node_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.spb_seen seen where seen.side = 'f' and seen.node_id = e.end_id) + order by e.end_id, e.id, active.node_id + limit admission_limit + 1 + ) candidate; + elsif chosen_side = 'f' and inbound then + if telemetry_search_id is not null then + select count(*) into telemetry_action_candidate_edges + from pg_temp.spb_active active + join edge e on e.graph_id = target_graph_id and e.end_id = active.node_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.spb_seen seen where seen.side = 'f' and seen.node_id = e.start_id); + end if; + insert into pg_temp.spb_candidate(side, node_id, depth, adjacent_id, edge_id) + select 'f', candidate.node_id, candidate.depth, candidate.adjacent_id, candidate.edge_id + from ( + select distinct on (e.start_id) e.start_id as node_id, active.depth + 1 as depth, + active.node_id as adjacent_id, e.id as edge_id + from pg_temp.spb_active active + join edge e on e.graph_id = target_graph_id and e.end_id = active.node_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.spb_seen seen where seen.side = 'f' and seen.node_id = e.start_id) + order by e.start_id, e.id, active.node_id + limit admission_limit + 1 + ) candidate; + elsif chosen_side = 'b' and not inbound then + if telemetry_search_id is not null then + select count(*) into telemetry_action_candidate_edges + from pg_temp.spb_active active + join edge e on e.graph_id = target_graph_id and e.end_id = active.node_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.spb_seen seen where seen.side = 'b' and seen.node_id = e.start_id); + end if; + insert into pg_temp.spb_candidate(side, node_id, depth, adjacent_id, edge_id) + select 'b', candidate.node_id, candidate.depth, candidate.adjacent_id, candidate.edge_id + from ( + select distinct on (e.start_id) e.start_id as node_id, active.depth + 1 as depth, + active.node_id as adjacent_id, e.id as edge_id + from pg_temp.spb_active active + join edge e on e.graph_id = target_graph_id and e.end_id = active.node_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.spb_seen seen where seen.side = 'b' and seen.node_id = e.start_id) + order by e.start_id, e.id, active.node_id + limit admission_limit + 1 + ) candidate; + else + if telemetry_search_id is not null then + select count(*) into telemetry_action_candidate_edges + from pg_temp.spb_active active + join edge e on e.graph_id = target_graph_id and e.start_id = active.node_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.spb_seen seen where seen.side = 'b' and seen.node_id = e.end_id); + end if; + insert into pg_temp.spb_candidate(side, node_id, depth, adjacent_id, edge_id) + select 'b', candidate.node_id, candidate.depth, candidate.adjacent_id, candidate.edge_id + from ( + select distinct on (e.end_id) e.end_id as node_id, active.depth + 1 as depth, + active.node_id as adjacent_id, e.id as edge_id + from pg_temp.spb_active active + join edge e on e.graph_id = target_graph_id and e.start_id = active.node_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.spb_seen seen where seen.side = 'b' and seen.node_id = e.end_id) + order by e.end_id, e.id, active.node_id + limit admission_limit + 1 + ) candidate; + end if; + + select count(*) into candidate_rows from pg_temp.spb_candidate; + if telemetry_search_id is not null then + select count(*) into telemetry_action_meetings + from pg_temp.spb_candidate candidate + join pg_temp.spb_seen opposite + on opposite.node_id = candidate.node_id and opposite.side <> candidate.side + where candidate.depth + opposite.depth between min_depth and max_depth; + telemetry_candidate_edges = telemetry_candidate_edges + telemetry_action_candidate_edges; + telemetry_distinct_new_nodes = telemetry_distinct_new_nodes + candidate_rows; + telemetry_meeting_candidates = telemetry_meeting_candidates + telemetry_action_meetings; + telemetry_seen_peak = greatest(telemetry_seen_peak, seen_rows + candidate_rows); + telemetry_frontier_peak = greatest(telemetry_frontier_peak, active_rows + frontier_rows + candidate_rows); + telemetry_queue_peak = greatest(telemetry_queue_peak, frontier_rows + candidate_rows); + telemetry_predecessor_peak = greatest(telemetry_predecessor_peak, predecessor_rows + candidate_rows); + telemetry_action_index = telemetry_action_index + 1; + perform public._record_bidirectional_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + chosen_side::text, + case scheduler when 'strict_alternating_node' then 'dequeue_node' else 'expand_level' end, + telemetry_action_depth, active_rows + frontier_rows + candidate_rows, + telemetry_action_candidate_edges, candidate_rows, seen_rows + candidate_rows, + frontier_rows + candidate_rows, predecessor_rows + candidate_rows, + telemetry_action_meetings); + end if; + if seen_rows + candidate_rows > state_limit + or active_rows + frontier_rows + candidate_rows > frontier_limit + or predecessor_rows + candidate_rows > predecessor_limit then + overflowed = true; + exit; + end if; + + insert into pg_temp.spb_predecessor(side, node_id, depth, adjacent_id, edge_id) + select side, node_id, depth, adjacent_id, edge_id + from pg_temp.spb_candidate + order by side, node_id; + insert into pg_temp.spb_seen(side, node_id, depth) + select side, node_id, depth from pg_temp.spb_candidate order by side, node_id; + + if chosen_side = 'f' then + insert into pg_temp.spb_front(side, node_id, depth, queue_order) + select side, node_id, depth, + forward_tail + row_number() over (order by edge_id, node_id, adjacent_id) + from pg_temp.spb_candidate; + forward_tail = forward_tail + candidate_rows; + else + insert into pg_temp.spb_front(side, node_id, depth, queue_order) + select side, node_id, depth, + backward_tail + row_number() over (order by edge_id, node_id, adjacent_id) + from pg_temp.spb_candidate; + backward_tail = backward_tail + candidate_rows; + end if; + + candidate_meeting = null; + candidate_distance = null; + select candidate.node_id, candidate.depth + opposite.depth + into candidate_meeting, candidate_distance + from pg_temp.spb_candidate candidate + join pg_temp.spb_seen opposite + on opposite.node_id = candidate.node_id and opposite.side <> candidate.side + where candidate.depth + opposite.depth between min_depth and max_depth + order by candidate.depth + opposite.depth, candidate.node_id + limit 1; + if candidate_distance is not null + and (best_distance is null + or candidate_distance < best_distance + or (candidate_distance = best_distance and candidate_meeting < best_meeting)) then + best_distance = candidate_distance; + best_meeting = candidate_meeting; + end if; + end loop; + + if overflowed then + return query + select fallback.root_id, fallback.next_id, fallback.depth, + fallback.satisfied, fallback.is_cycle, fallback.path + from public.shortest_path_compact(target_graph_id, source_id, target_id, + min_depth, max_depth, edge_kind_ids, + inbound, state_limit) fallback; + get diagnostics emitted_count = row_count; + if telemetry_search_id is not null then + perform public._finish_bidirectional_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'exact_s4_fallback', + telemetry_scheduler_actions, telemetry_candidate_edges, + telemetry_distinct_new_nodes, telemetry_seen_peak, + telemetry_frontier_peak, telemetry_queue_peak, + telemetry_predecessor_peak, telemetry_meeting_candidates, + best_distance, emitted_count, true, true); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('exact_s4_fallback', true, 'SP-S4'); + return; + end if; + if best_distance is null then + if telemetry_search_id is not null then + perform public._finish_bidirectional_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'search_no_path', + telemetry_scheduler_actions, telemetry_candidate_edges, + telemetry_distinct_new_nodes, telemetry_seen_peak, + telemetry_frontier_peak, telemetry_queue_peak, + telemetry_predecessor_peak, telemetry_meeting_candidates, + null, 0, false, false); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('search_no_path', false, 'SP-S4'); + return; + end if; + + -- Path arrays exist only at the late output boundary. Forward predecessor + -- edges are prepended back to source; backward successor edges are appended + -- toward target, preserving logical source-to-target order for both physical + -- edge orientations. + return query + with recursive + forward_witness(node_id, edge_ids) as ( + select best_meeting, array []::int8[] + union all + select predecessor.adjacent_id, + array[predecessor.edge_id]::int8[] || forward_witness.edge_ids + from forward_witness + join pg_temp.spb_predecessor predecessor + on predecessor.side = 'f' and predecessor.node_id = forward_witness.node_id + ), + backward_witness(node_id, edge_ids) as ( + select best_meeting, array []::int8[] + union all + select successor.adjacent_id, + backward_witness.edge_ids || successor.edge_id + from backward_witness + join pg_temp.spb_predecessor successor + on successor.side = 'b' and successor.node_id = backward_witness.node_id + ) + select source_id, target_id, best_distance, true, false, + forward_witness.edge_ids || backward_witness.edge_ids + from forward_witness + join backward_witness on forward_witness.node_id = source_id + and backward_witness.node_id = target_id + limit 1; + get diagnostics emitted_count = row_count; + if telemetry_search_id is not null then + perform public._finish_bidirectional_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'bidirectional_search', + telemetry_scheduler_actions, telemetry_candidate_edges, + telemetry_distinct_new_nodes, telemetry_seen_peak, + telemetry_frontier_peak, telemetry_queue_peak, + telemetry_predecessor_peak, telemetry_meeting_candidates, + best_distance, emitted_count, false, false); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('bidirectional_search', false, 'SP-S4'); +end; +$$ + language plpgsql + volatile + strict + cost 100 + set recursive_worktable_factor = 1 + rows 1; + +-- B1 freezes Neo4j-4.4-style strict one-node alternation behind a typed +-- wrapper so scheduler identity is not inferred from generated SQL text. +create or replace function public.shortest_path_b1_strict_alternating( + target_graph_id int4, + source_id int8, + target_id int8, + min_depth int4, + max_depth int4, + edge_kind_ids int2[], + inbound bool, + state_limit int8, + frontier_limit int8, + predecessor_limit int8) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +select * +from public.shortest_path_bidirectional_compact_v1( + target_graph_id, source_id, target_id, min_depth, max_depth, + edge_kind_ids, inbound, state_limit, frontier_limit, predecessor_limit, + 'strict_alternating_node'); +$$ + language sql + volatile + strict + cost 100 + rows 1; + +-- B2 expands a complete current level from the smaller side, with a stable +-- forward-side tie break. +create or replace function public.shortest_path_b2_smaller_current_level( + target_graph_id int4, + source_id int8, + target_id int8, + min_depth int4, + max_depth int4, + edge_kind_ids int2[], + inbound bool, + state_limit int8, + frontier_limit int8, + predecessor_limit int8) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +select * +from public.shortest_path_bidirectional_compact_v1( + target_graph_id, source_id, target_id, min_depth, max_depth, + edge_kind_ids, inbound, state_limit, frontier_limit, predecessor_limit, + 'smaller_current_level'); +$$ + language sql + volatile + strict + cost 100 + rows 1; + +-- Compact bidirectional all-shortest-path candidates use a workspace that is +-- disjoint from both the production ASP-A1 spd_* state and singleton SP spb_* +-- state. Discovery, relationship-distinct predecessor retention, path-count +-- calculation, and staged output therefore have separately measurable shapes. +create or replace function public.ensure_bidirectional_all_shortest_path_workspace() + returns void as +$$ +declare + expected_version constant int4 := 1; + present_version int4; +begin + if to_regclass('pg_temp.asb_workspace_version') is not null then + select version into present_version from pg_temp.asb_workspace_version limit 1; + end if; + + if to_regclass('pg_temp.asb_workspace_version') is not null + and present_version is distinct from expected_version then + drop table if exists pg_temp.asb_output; + drop table if exists pg_temp.asb_path_count; + drop table if exists pg_temp.asb_predecessor; + drop table if exists pg_temp.asb_candidate_predecessor; + drop table if exists pg_temp.asb_candidate_node; + drop table if exists pg_temp.asb_active; + drop table if exists pg_temp.asb_seen; + drop table if exists pg_temp.asb_front; + drop table if exists pg_temp.asb_workspace_version; + end if; + + if to_regclass('pg_temp.asb_workspace_version') is null then + create temporary table asb_workspace_version + ( + version int4 not null primary key + ) on commit preserve rows; + + create temporary table asb_front + ( + side char(1) not null, + node_id int8 not null, + depth int4 not null, + queue_order int8 not null, + primary key (side, node_id), + unique (side, queue_order), + check (side in ('f', 'b')) + ) on commit preserve rows; + create index asb_front_side_depth_order_index + on asb_front using btree (side, depth, queue_order); + + create temporary table asb_seen + ( + side char(1) not null, + node_id int8 not null, + depth int4 not null, + primary key (side, node_id), + check (side in ('f', 'b')) + ) on commit preserve rows; + create index asb_seen_node_side_depth_index + on asb_seen using btree (node_id, side, depth); + + create temporary table asb_active + ( + side char(1) not null, + node_id int8 not null, + depth int4 not null, + primary key (side, node_id), + check (side in ('f', 'b')) + ) on commit preserve rows; + + create temporary table asb_candidate_node + ( + side char(1) not null, + node_id int8 not null, + depth int4 not null, + primary key (side, node_id), + check (side in ('f', 'b')) + ) on commit preserve rows; + + create temporary table asb_candidate_predecessor + ( + side char(1) not null, + node_id int8 not null, + depth int4 not null, + adjacent_id int8 not null, + edge_id int8 not null, + primary key (side, node_id, depth, adjacent_id, edge_id), + check (side in ('f', 'b')) + ) on commit preserve rows; + + -- Forward adjacent_id points toward the logical source. Backward + -- adjacent_id points toward the logical target. Equal-depth rows are not + -- collapsed: every relationship-distinct shortest predecessor/successor + -- is retained. + create temporary table asb_predecessor + ( + side char(1) not null, + node_id int8 not null, + depth int4 not null, + adjacent_id int8 not null, + edge_id int8 not null, + primary key (side, node_id, depth, adjacent_id, edge_id), + check (side in ('f', 'b')) + ) on commit preserve rows; + create index asb_predecessor_node_side_depth_index + on asb_predecessor using btree (node_id, side, depth); + create index asb_predecessor_adjacent_side_depth_index + on asb_predecessor using btree (adjacent_id, side, depth); + + create temporary table asb_path_count + ( + side char(1) not null, + node_id int8 not null, + depth int4 not null, + path_count int8 not null, + primary key (side, node_id), + check (side in ('f', 'b')), + check (path_count >= 0) + ) on commit preserve rows; + + create temporary table asb_output + ( + edge_ids int8[] not null primary key, + output_bytes int8 not null, + check (output_bytes >= 0) + ) on commit preserve rows; + + insert into asb_workspace_version(version) values (expected_version); + end if; +end; +$$ + language plpgsql + volatile; + +create or replace function public.reset_bidirectional_all_shortest_path_workspace() + returns void as +$$ +begin + perform public.ensure_bidirectional_all_shortest_path_workspace(); + truncate table pg_temp.asb_front, pg_temp.asb_seen, pg_temp.asb_active, + pg_temp.asb_candidate_node, pg_temp.asb_candidate_predecessor, + pg_temp.asb_predecessor, pg_temp.asb_path_count, + pg_temp.asb_output; +end; +$$ + language plpgsql + volatile; + +-- clear_bidirectional_all_shortest_path_workspace does not allocate state. +-- Overflow paths call it before ASP-A1 so no candidate rows survive into the +-- exact fallback boundary. +create or replace function public.clear_bidirectional_all_shortest_path_workspace() + returns void as +$$ +begin + if to_regclass('pg_temp.asb_workspace_version') is not null then + execute 'truncate table pg_temp.asb_front, pg_temp.asb_seen, pg_temp.asb_active, ' + 'pg_temp.asb_candidate_node, pg_temp.asb_candidate_predecessor, ' + 'pg_temp.asb_predecessor, pg_temp.asb_path_count, pg_temp.asb_output'; + end if; +end; +$$ + language plpgsql + volatile; + +-- Tool-only ASP diagnostic counters use a second versioned, session-local +-- workspace. The transaction-local invocation setting prevents pooled-session +-- reuse from attributing a later call to an earlier replay, while explicit +-- keys make multi-call statements and cleanup independently auditable. +create or replace function public.ensure_bidirectional_all_shortest_path_telemetry_workspace() + returns void as +$$ +declare + expected_version constant int4 := 1; + present_version int4; +begin + if to_regclass('pg_temp.asb_telemetry_workspace_version') is not null then + select version into present_version + from pg_temp.asb_telemetry_workspace_version limit 1; + end if; + if to_regclass('pg_temp.asb_telemetry_workspace_version') is not null + and present_version is distinct from expected_version then + drop table if exists pg_temp.asb_telemetry_level; + drop table if exists pg_temp.asb_telemetry_call; + drop table if exists pg_temp.asb_telemetry_invocation; + drop table if exists pg_temp.asb_telemetry_workspace_version; + end if; + if to_regclass('pg_temp.asb_telemetry_workspace_version') is null then + create temporary table asb_telemetry_workspace_version + ( + version int4 not null primary key + ) on commit preserve rows; + create temporary table asb_telemetry_invocation + ( + invocation_id text not null primary key, + schema_version int4 not null, + scheduler text, + state_limit int8, + frontier_limit int8, + predecessor_limit int8, + enumeration_limit int8, + output_bytes_limit int8, + next_search_id int8 not null default 0, + check (btrim(invocation_id) <> '') + ) on commit preserve rows; + create temporary table asb_telemetry_call + ( + invocation_id text not null, + search_id int8 not null, + source_id int8 not null, + target_id int8 not null, + runtime_branch text not null default 'started', + scheduler_actions int8 not null default 0, + candidate_edges int8 not null default 0, + distinct_new_nodes int8 not null default 0, + seen_peak int8 not null default 0, + frontier_peak int8 not null default 0, + queue_peak int8 not null default 0, + predecessor_peak int8 not null default 0, + meeting_candidates int8 not null default 0, + frozen_distance int4, + witness_rows int8 not null default 0, + same_depth_predecessor_additions int8 not null default 0, + meeting_nodes int8 not null default 0, + cut_depth int4, + path_count_estimate int8 not null default 0, + path_count_saturated bool not null default false, + enumerated_candidates int8 not null default 0, + duplicate_rejects int8 not null default 0, + output_paths int8 not null default 0, + output_edge_cells int8 not null default 0, + output_bytes int8 not null default 0, + overflowed bool not null default false, + fallback_executed bool not null default false, + primary key (invocation_id, search_id) + ) on commit preserve rows; + create temporary table asb_telemetry_level + ( + invocation_id text not null, + search_id int8 not null, + action_index int8 not null, + side text not null, + action text not null, + depth int4 not null, + frontier_rows int8 not null, + candidate_edges int8 not null, + distinct_new_nodes int8 not null, + seen_rows int8 not null, + queue_rows int8 not null, + predecessor_rows int8 not null, + meeting_candidates int8 not null, + primary key (invocation_id, search_id, action_index) + ) on commit preserve rows; + insert into asb_telemetry_workspace_version(version) values (expected_version); + end if; +end; +$$ + language plpgsql + volatile; + +create or replace function public.begin_bidirectional_all_shortest_path_diagnostic_v1(invocation_id text) + returns void as +$$ +begin + if invocation_id is null or btrim(invocation_id) = '' or length(invocation_id) > 256 then + raise exception using errcode = '22023', message = 'bidirectional all-shortest-path diagnostic invocation ID must contain 1 to 256 characters'; + end if; + perform public.ensure_bidirectional_all_shortest_path_telemetry_workspace(); + delete from pg_temp.asb_telemetry_level where asb_telemetry_level.invocation_id = begin_bidirectional_all_shortest_path_diagnostic_v1.invocation_id; + delete from pg_temp.asb_telemetry_call where asb_telemetry_call.invocation_id = begin_bidirectional_all_shortest_path_diagnostic_v1.invocation_id; + delete from pg_temp.asb_telemetry_invocation where asb_telemetry_invocation.invocation_id = begin_bidirectional_all_shortest_path_diagnostic_v1.invocation_id; + insert into pg_temp.asb_telemetry_invocation(invocation_id, schema_version) + values (invocation_id, 1); + perform set_config('dawgs.asb_diagnostic_invocation_id', invocation_id, true); +end; +$$ + language plpgsql + volatile; + +create or replace function public.read_bidirectional_all_shortest_path_diagnostic_v1(target_invocation_id text) + returns jsonb as +$$ +declare + result jsonb; +begin + select jsonb_build_object( + 'schema_version', invocation.schema_version, + 'invocation_id', invocation.invocation_id, + 'scheduler', invocation.scheduler, + 'state_limit', invocation.state_limit, + 'frontier_limit', invocation.frontier_limit, + 'predecessor_limit', invocation.predecessor_limit, + 'enumeration_limit', invocation.enumeration_limit, + 'output_bytes_limit', invocation.output_bytes_limit, + 'search_calls', coalesce(call_totals.search_calls, 0), + 'runtime_branch', coalesce(call_totals.runtime_branch, 'missing'), + 'overflowed', coalesce(call_totals.overflowed, false), + 'fallback_executed', coalesce(call_totals.fallback_executed, false), + 'counters', jsonb_build_object( + 'scheduler_actions', coalesce(call_totals.scheduler_actions, 0), + 'candidate_edges', coalesce(call_totals.candidate_edges, 0), + 'distinct_new_nodes', coalesce(call_totals.distinct_new_nodes, 0), + 'seen_peak', coalesce(call_totals.seen_peak, 0), + 'frontier_peak', coalesce(call_totals.frontier_peak, 0), + 'queue_peak', coalesce(call_totals.queue_peak, 0), + 'predecessor_peak', coalesce(call_totals.predecessor_peak, 0), + 'meeting_candidates', coalesce(call_totals.meeting_candidates, 0), + 'frozen_distance', coalesce(call_totals.frozen_distance, -1), + 'witness_rows', coalesce(call_totals.witness_rows, 0), + 'same_depth_predecessor_additions', coalesce(call_totals.same_depth_predecessor_additions, 0), + 'meeting_nodes', coalesce(call_totals.meeting_nodes, 0), + 'cut_depth', coalesce(call_totals.cut_depth, -1), + 'path_count_estimate', coalesce(call_totals.path_count_estimate, 0), + 'path_count_saturated', coalesce(call_totals.path_count_saturated, false), + 'enumerated_candidates', coalesce(call_totals.enumerated_candidates, 0), + 'duplicate_rejects', coalesce(call_totals.duplicate_rejects, 0), + 'output_paths', coalesce(call_totals.output_paths, 0), + 'output_edge_cells', coalesce(call_totals.output_edge_cells, 0), + 'output_bytes', coalesce(call_totals.output_bytes, 0), + 'levels', coalesce(levels.rows, '[]'::jsonb) + ), + 'calls', coalesce(calls.rows, '[]'::jsonb) + ) into result + from pg_temp.asb_telemetry_invocation invocation + left join lateral ( + select count(*)::int8 as search_calls, + case when count(distinct call.runtime_branch) = 1 + then min(call.runtime_branch) else 'mixed' end as runtime_branch, + bool_or(call.overflowed) as overflowed, + bool_or(call.fallback_executed) as fallback_executed, + sum(call.scheduler_actions)::int8 as scheduler_actions, + sum(call.candidate_edges)::int8 as candidate_edges, + sum(call.distinct_new_nodes)::int8 as distinct_new_nodes, + max(call.seen_peak)::int8 as seen_peak, + max(call.frontier_peak)::int8 as frontier_peak, + max(call.queue_peak)::int8 as queue_peak, + max(call.predecessor_peak)::int8 as predecessor_peak, + sum(call.meeting_candidates)::int8 as meeting_candidates, + min(call.frozen_distance)::int4 as frozen_distance, + sum(call.witness_rows)::int8 as witness_rows, + sum(call.same_depth_predecessor_additions)::int8 as same_depth_predecessor_additions, + sum(call.meeting_nodes)::int8 as meeting_nodes, + min(call.cut_depth)::int4 as cut_depth, + sum(call.path_count_estimate)::int8 as path_count_estimate, + bool_or(call.path_count_saturated) as path_count_saturated, + sum(call.enumerated_candidates)::int8 as enumerated_candidates, + sum(call.duplicate_rejects)::int8 as duplicate_rejects, + sum(call.output_paths)::int8 as output_paths, + sum(call.output_edge_cells)::int8 as output_edge_cells, + sum(call.output_bytes)::int8 as output_bytes + from pg_temp.asb_telemetry_call call + where call.invocation_id = invocation.invocation_id + ) call_totals on true + left join lateral ( + select jsonb_agg(jsonb_build_object( + 'search_id', level.search_id, + 'action_index', level.action_index, + 'side', level.side, + 'action', level.action, + 'depth', level.depth, + 'frontier_rows', level.frontier_rows, + 'candidate_edges', level.candidate_edges, + 'distinct_new_nodes', level.distinct_new_nodes, + 'seen_rows', level.seen_rows, + 'queue_rows', level.queue_rows, + 'predecessor_rows', level.predecessor_rows, + 'meeting_candidates', level.meeting_candidates + ) order by level.search_id, level.action_index) as rows + from pg_temp.asb_telemetry_level level + where level.invocation_id = invocation.invocation_id + ) levels on true + left join lateral ( + select jsonb_agg(to_jsonb(call) - 'invocation_id' order by call.search_id) as rows + from pg_temp.asb_telemetry_call call + where call.invocation_id = invocation.invocation_id + ) calls on true + where invocation.invocation_id = target_invocation_id; + return result; +end; +$$ + language plpgsql + stable + strict; + +create or replace function public.clear_bidirectional_all_shortest_path_diagnostic_v1(target_invocation_id text) + returns void as +$$ +begin + if to_regclass('pg_temp.asb_telemetry_invocation') is not null then + delete from pg_temp.asb_telemetry_level where invocation_id = target_invocation_id; + delete from pg_temp.asb_telemetry_call where invocation_id = target_invocation_id; + delete from pg_temp.asb_telemetry_invocation where invocation_id = target_invocation_id; + end if; + if nullif(current_setting('dawgs.asb_diagnostic_invocation_id', true), '') = target_invocation_id then + perform set_config('dawgs.asb_diagnostic_invocation_id', '', true); + end if; +end; +$$ + language plpgsql + volatile + strict; + +create or replace function public._start_bidirectional_all_shortest_path_diagnostic_call_v1( + target_invocation_id text, + target_scheduler text, + target_state_limit int8, + target_frontier_limit int8, + target_predecessor_limit int8, + target_enumeration_limit int8, + target_output_bytes_limit int8, + target_source_id int8, + target_target_id int8) + returns int8 as +$$ +declare + target_search_id int8; +begin + if target_invocation_id is null then + return null; + end if; + if to_regclass('pg_temp.asb_telemetry_invocation') is null then + raise exception using errcode = '55000', message = 'bidirectional all-shortest-path diagnostic replay was not initialized on this session'; + end if; + update pg_temp.asb_telemetry_invocation invocation + set scheduler = coalesce(invocation.scheduler, target_scheduler), + state_limit = coalesce(invocation.state_limit, target_state_limit), + frontier_limit = coalesce(invocation.frontier_limit, target_frontier_limit), + predecessor_limit = coalesce(invocation.predecessor_limit, target_predecessor_limit), + enumeration_limit = coalesce(invocation.enumeration_limit, target_enumeration_limit), + output_bytes_limit = coalesce(invocation.output_bytes_limit, target_output_bytes_limit), + next_search_id = invocation.next_search_id + 1 + where invocation.invocation_id = target_invocation_id + and (invocation.scheduler is null or invocation.scheduler = target_scheduler) + and (invocation.state_limit is null or invocation.state_limit = target_state_limit) + and (invocation.frontier_limit is null or invocation.frontier_limit = target_frontier_limit) + and (invocation.predecessor_limit is null or invocation.predecessor_limit = target_predecessor_limit) + and (invocation.enumeration_limit is null or invocation.enumeration_limit = target_enumeration_limit) + and (invocation.output_bytes_limit is null or invocation.output_bytes_limit = target_output_bytes_limit) + returning invocation.next_search_id into target_search_id; + if target_search_id is null then + raise exception using errcode = '55000', message = 'bidirectional all-shortest-path diagnostic invocation is missing or mixes scheduler/cap identities'; + end if; + insert into pg_temp.asb_telemetry_call(invocation_id, search_id, source_id, target_id) + values (target_invocation_id, target_search_id, target_source_id, target_target_id); + return target_search_id; +end; +$$ + language plpgsql + volatile; + +create or replace function public._record_bidirectional_all_shortest_path_diagnostic_level_v1( + target_invocation_id text, + target_search_id int8, + target_action_index int8, + target_side text, + target_action text, + target_depth int4, + target_frontier_rows int8, + target_candidate_edges int8, + target_distinct_new_nodes int8, + target_seen_rows int8, + target_queue_rows int8, + target_predecessor_rows int8, + target_meeting_candidates int8) + returns void as +$$ +begin + insert into pg_temp.asb_telemetry_level( + invocation_id, search_id, action_index, side, action, depth, + frontier_rows, candidate_edges, distinct_new_nodes, seen_rows, + queue_rows, predecessor_rows, meeting_candidates) + values ( + target_invocation_id, target_search_id, target_action_index, target_side, + target_action, target_depth, target_frontier_rows, target_candidate_edges, + target_distinct_new_nodes, target_seen_rows, target_queue_rows, + target_predecessor_rows, target_meeting_candidates); +end; +$$ + language plpgsql + volatile + strict; + +create or replace function public._finish_bidirectional_all_shortest_path_diagnostic_call_v1( + target_invocation_id text, + target_search_id int8, + target_runtime_branch text, + target_scheduler_actions int8, + target_candidate_edges int8, + target_distinct_new_nodes int8, + target_seen_peak int8, + target_frontier_peak int8, + target_queue_peak int8, + target_predecessor_peak int8, + target_meeting_candidates int8, + target_frozen_distance int4, + target_witness_rows int8, + target_same_depth_predecessor_additions int8, + target_meeting_nodes int8, + target_cut_depth int4, + target_path_count_estimate int8, + target_path_count_saturated bool, + target_enumerated_candidates int8, + target_duplicate_rejects int8, + target_output_paths int8, + target_output_edge_cells int8, + target_output_bytes int8, + target_overflowed bool, + target_fallback_executed bool) + returns void as +$$ +begin + update pg_temp.asb_telemetry_call call + set runtime_branch = target_runtime_branch, + scheduler_actions = target_scheduler_actions, + candidate_edges = target_candidate_edges, + distinct_new_nodes = target_distinct_new_nodes, + seen_peak = target_seen_peak, + frontier_peak = target_frontier_peak, + queue_peak = target_queue_peak, + predecessor_peak = target_predecessor_peak, + meeting_candidates = target_meeting_candidates, + frozen_distance = target_frozen_distance, + witness_rows = target_witness_rows, + same_depth_predecessor_additions = target_same_depth_predecessor_additions, + meeting_nodes = target_meeting_nodes, + cut_depth = target_cut_depth, + path_count_estimate = target_path_count_estimate, + path_count_saturated = target_path_count_saturated, + enumerated_candidates = target_enumerated_candidates, + duplicate_rejects = target_duplicate_rejects, + output_paths = target_output_paths, + output_edge_cells = target_output_edge_cells, + output_bytes = target_output_bytes, + overflowed = target_overflowed, + fallback_executed = target_fallback_executed + where call.invocation_id = target_invocation_id and call.search_id = target_search_id; + if not found then + raise exception using errcode = '55000', message = 'bidirectional all-shortest-path diagnostic call is missing'; + end if; +end; +$$ + language plpgsql + volatile; + +-- all_shortest_paths_bidirectional_compact_v1 is restricted to one validated, +-- distinct endpoint pair, minimum depth one, directed traversal, and maximum +-- depth 64. Within that envelope a minimum path cannot repeat a node, so two +-- minimum-node-depth predecessor DAGs preserve relationship-simple Cypher +-- semantics. +-- +-- Queue-head depth is a lower bound on every not-yet-completed path from that +-- side. A minimum distance L is proven only when one side is exhausted or the +-- two queue-head depths sum to at least L. The kernel then completes one +-- canonical cut k=floor(L/2): all forward predecessor rows into depth k and +-- all backward successor rows into depth L-k must be complete. Every shortest +-- path crosses exactly one node at this cut and is therefore stitched once, +-- even when the two searches overlap at several depths. +-- +-- Discovery nodes/frontier, relationship-distinct predecessors, enumerated +-- arrays, and materialized array bytes have independent cap+1 admissions. +-- Path counts are evaluated over the completed DAG with saturating arithmetic +-- before enumeration. No candidate row is returned until every gate passes. +-- Overflow clears asb_* and invokes exact ASP-A1 in the same top-level +-- statement. REPEATABLE READ or SERIALIZABLE is mandatory because VOLATILE +-- PL/pgSQL statements at READ COMMITTED do not share one statement snapshot. +create or replace function public.all_shortest_paths_bidirectional_compact_v1( + target_graph_id int4, + source_id int8, + target_id int8, + min_depth int4, + max_depth int4, + edge_kind_ids int2[], + inbound bool, + state_limit int8, + frontier_limit int8, + predecessor_limit int8, + enumeration_limit int8, + output_bytes_limit int8, + scheduler text) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +#variable_conflict use_column +declare + chosen_side char(1); + strict_side char(1) := 'f'; + forward_depth int4; + backward_depth int4; + forward_ready_depth int4; + backward_ready_depth int4; + forward_width int8; + backward_width int8; + forward_tail int8 := 0; + backward_tail int8 := 0; + seen_rows int8; + active_rows int8; + frontier_rows int8; + predecessor_rows int8; + candidate_node_rows int8; + candidate_predecessor_rows int8; + discovery_admission_limit int8; + predecessor_admission_limit int8; + candidate_meeting int8; + candidate_distance int4; + best_distance int4; + cut_depth int4; + count_depth int4; + meeting_nodes int8; + path_array_bytes int8; + path_count_limit int8; + path_count_sentinel int8; + path_count_estimate int8; + output_rows int8; + output_bytes int8; + emitted_count int8 := 0; + overflowed bool := false; + telemetry_invocation_id text := nullif(current_setting('dawgs.asb_diagnostic_invocation_id', true), ''); + telemetry_search_id int8; + telemetry_action_index int8 := 0; + telemetry_action_depth int4 := 0; + telemetry_action_candidate_edges int8 := 0; + telemetry_action_meetings int8 := 0; + telemetry_scheduler_actions int8 := 0; + telemetry_candidate_edges int8 := 0; + telemetry_distinct_new_nodes int8 := 0; + telemetry_seen_peak int8 := 0; + telemetry_frontier_peak int8 := 0; + telemetry_queue_peak int8 := 0; + telemetry_predecessor_peak int8 := 0; + telemetry_meeting_candidates int8 := 0; + telemetry_same_depth_predecessors int8 := 0; + telemetry_path_count_saturated bool := false; + telemetry_enumerated_candidates int8 := 0; + telemetry_duplicate_rejects int8 := 0; +begin + if source_id is null or target_id is null or max_depth < 1 then + return; + end if; + if scheduler <> 'strict_alternating_node' and scheduler <> 'smaller_current_level' then + raise exception using errcode = '22023', message = 'unknown compact bidirectional all-shortest-path scheduler'; + end if; + if min_depth <> 1 then + raise exception using errcode = '22023', message = 'compact bidirectional all-shortest paths requires min_depth = 1'; + end if; + if max_depth > 64 then + raise exception using errcode = '22023', message = 'compact bidirectional all-shortest paths requires max_depth <= 64'; + end if; + if state_limit <= 0 or frontier_limit <= 0 or predecessor_limit <= 0 + or enumeration_limit <= 0 or output_bytes_limit <= 0 + or enumeration_limit = 9223372036854775807 + or output_bytes_limit = 9223372036854775807 then + raise exception using errcode = '22023', message = 'compact bidirectional all-shortest paths requires positive bounded limits below int8 maximum'; + end if; + if current_setting('transaction_isolation') <> 'repeatable read' + and current_setting('transaction_isolation') <> 'serializable' then + raise exception using + errcode = '25001', + message = 'compact bidirectional all-shortest paths requires REPEATABLE READ or SERIALIZABLE transaction isolation'; + end if; + if source_id = target_id then + perform public.shortest_path_self_endpoint_error(source_id, target_id); + end if; + + telemetry_search_id = public._start_bidirectional_all_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, scheduler, state_limit, frontier_limit, + predecessor_limit, enumeration_limit, output_bytes_limit, + source_id, target_id); + -- This non-allocating clear prevents successful shallow preflights, no-path + -- returns, and exact fallback from inheriting an earlier invocation's state. + perform public.clear_bidirectional_all_shortest_path_workspace(); + + -- Exact depth-one preflight remains outside the candidate workspace. It + -- returns every relationship-distinct edge only when enumeration and bytes + -- gates admit the complete multiset. + path_array_bytes = pg_column_size(array_fill(0::int8, array[1])); + if path_array_bytes <= 126 then + path_array_bytes = path_array_bytes - 3; + end if; + path_count_limit = least(enumeration_limit, output_bytes_limit / path_array_bytes); + select count(*) into output_rows + from ( + select 1 + from edge e + where e.graph_id = target_graph_id + and ((not inbound and e.start_id = source_id and e.end_id = target_id) + or (inbound and e.end_id = source_id and e.start_id = target_id)) + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + limit path_count_limit + 1 + ) shallow; + if output_rows > 0 then + if telemetry_search_id is not null then + telemetry_action_index = telemetry_action_index + 1; + perform public._record_bidirectional_all_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + 'none', 'preflight_one_hop', 1, 0, output_rows, 0, 0, 0, 0, + output_rows); + end if; + if output_rows > path_count_limit then + return query + select fallback.root_id, fallback.next_id, fallback.depth, + fallback.satisfied, fallback.is_cycle, fallback.path + from public.all_shortest_paths_dag(target_graph_id, source_id, target_id, + min_depth, max_depth, edge_kind_ids, + inbound) fallback; + get diagnostics emitted_count = row_count; + if telemetry_search_id is not null then + perform public._finish_bidirectional_all_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'exact_a1_fallback', + 0, output_rows, 0, 0, 0, 0, 0, output_rows, 1, emitted_count, + 0, 1, 0, output_rows, true, output_rows, 0, + emitted_count, emitted_count, emitted_count * path_array_bytes, + true, true); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('exact_a1_fallback', true, 'ASP-A1-DAG'); + return; + end if; + if not inbound then + return query + select source_id, target_id, 1::int4, true, false, array[e.id]::int8[] + from edge e + where e.graph_id = target_graph_id + and e.start_id = source_id and e.end_id = target_id + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + order by e.id; + else + return query + select source_id, target_id, 1::int4, true, false, array[e.id]::int8[] + from edge e + where e.graph_id = target_graph_id + and e.end_id = source_id and e.start_id = target_id + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + order by e.id; + end if; + get diagnostics emitted_count = row_count; + if telemetry_search_id is not null then + perform public._finish_bidirectional_all_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'preflight_one_hop', + 0, output_rows, 0, 0, 0, 0, 0, output_rows, 1, emitted_count, + 0, 1, 0, output_rows, false, output_rows, 0, + emitted_count, emitted_count, emitted_count * path_array_bytes, + false, false); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('preflight_one_hop', false, 'ASP-A1-DAG'); + return; + end if; + + -- Exact depth-two preflight similarly stages only a cap+1 scalar count. The + -- full relationship pair multiset is emitted only after both output gates. + if max_depth >= 2 then + path_array_bytes = pg_column_size(array_fill(0::int8, array[2])); + if path_array_bytes <= 126 then + path_array_bytes = path_array_bytes - 3; + end if; + path_count_limit = least(enumeration_limit, output_bytes_limit / path_array_bytes); + if not inbound then + select count(*) into output_rows + from ( + select 1 + from edge e1 + join edge e2 on e2.graph_id = target_graph_id and e2.start_id = e1.end_id + where e1.graph_id = target_graph_id + and e1.start_id = source_id and e2.end_id = target_id and e1.id <> e2.id + and (cardinality(edge_kind_ids) = 0 or e1.kind_id = any(edge_kind_ids)) + and (cardinality(edge_kind_ids) = 0 or e2.kind_id = any(edge_kind_ids)) + limit path_count_limit + 1 + ) shallow; + else + select count(*) into output_rows + from ( + select 1 + from edge e1 + join edge e2 on e2.graph_id = target_graph_id and e2.end_id = e1.start_id + where e1.graph_id = target_graph_id + and e1.end_id = source_id and e2.start_id = target_id and e1.id <> e2.id + and (cardinality(edge_kind_ids) = 0 or e1.kind_id = any(edge_kind_ids)) + and (cardinality(edge_kind_ids) = 0 or e2.kind_id = any(edge_kind_ids)) + limit path_count_limit + 1 + ) shallow; + end if; + if output_rows > 0 then + if telemetry_search_id is not null then + telemetry_action_index = telemetry_action_index + 1; + perform public._record_bidirectional_all_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + 'none', 'preflight_two_hop', 2, 0, output_rows * 2, 0, 0, 0, 0, + output_rows); + end if; + if output_rows > path_count_limit then + return query + select fallback.root_id, fallback.next_id, fallback.depth, + fallback.satisfied, fallback.is_cycle, fallback.path + from public.all_shortest_paths_dag(target_graph_id, source_id, target_id, + min_depth, max_depth, edge_kind_ids, + inbound) fallback; + get diagnostics emitted_count = row_count; + if telemetry_search_id is not null then + perform public._finish_bidirectional_all_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'exact_a1_fallback', + 0, output_rows * 2, 0, 0, 0, 0, 0, output_rows, 2, emitted_count, + 0, 1, 1, output_rows, true, output_rows, 0, + emitted_count, emitted_count * 2, emitted_count * path_array_bytes, + true, true); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('exact_a1_fallback', true, 'ASP-A1-DAG'); + return; + end if; + if not inbound then + return query + select source_id, target_id, 2::int4, true, false, array[e1.id, e2.id]::int8[] + from edge e1 + join edge e2 on e2.graph_id = target_graph_id and e2.start_id = e1.end_id + where e1.graph_id = target_graph_id + and e1.start_id = source_id and e2.end_id = target_id and e1.id <> e2.id + and (cardinality(edge_kind_ids) = 0 or e1.kind_id = any(edge_kind_ids)) + and (cardinality(edge_kind_ids) = 0 or e2.kind_id = any(edge_kind_ids)) + order by e1.id, e2.id; + else + return query + select source_id, target_id, 2::int4, true, false, array[e1.id, e2.id]::int8[] + from edge e1 + join edge e2 on e2.graph_id = target_graph_id and e2.end_id = e1.start_id + where e1.graph_id = target_graph_id + and e1.end_id = source_id and e2.start_id = target_id and e1.id <> e2.id + and (cardinality(edge_kind_ids) = 0 or e1.kind_id = any(edge_kind_ids)) + and (cardinality(edge_kind_ids) = 0 or e2.kind_id = any(edge_kind_ids)) + order by e1.id, e2.id; + end if; + get diagnostics emitted_count = row_count; + if telemetry_search_id is not null then + perform public._finish_bidirectional_all_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'preflight_two_hop', + 0, output_rows * 2, 0, 0, 0, 0, 0, output_rows, 2, emitted_count, + 0, 1, 1, output_rows, false, output_rows, 0, + emitted_count, emitted_count * 2, emitted_count * path_array_bytes, + false, false); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('preflight_two_hop', false, 'ASP-A1-DAG'); + return; + end if; + end if; + if max_depth <= 2 then + if telemetry_search_id is not null then + telemetry_action_index = telemetry_action_index + 1; + perform public._record_bidirectional_all_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + 'none', 'preflight_no_path', max_depth, 0, 0, 0, 0, 0, 0, 0); + perform public._finish_bidirectional_all_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'preflight_no_path', + 0, 0, 0, 0, 0, 0, 0, 0, null, 0, + 0, 0, null, 0, false, 0, 0, 0, 0, 0, false, false); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('preflight_no_path', false, 'ASP-A1-DAG'); + return; + end if; + + -- The two roots are discovery/frontier state, but not predecessor state. + if state_limit < 2 or frontier_limit < 2 then + overflowed = true; + telemetry_frontier_peak = 2; + telemetry_queue_peak = 2; + if telemetry_search_id is not null then + telemetry_action_index = telemetry_action_index + 1; + perform public._record_bidirectional_all_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + 'none', 'root_admission', 0, 2, 0, 0, 2, 2, 0, 0); + end if; + else + perform public.reset_bidirectional_all_shortest_path_workspace(); + insert into pg_temp.asb_front(side, node_id, depth, queue_order) + values ('f', source_id, 0, 0), ('b', target_id, 0, 0); + insert into pg_temp.asb_seen(side, node_id, depth) + values ('f', source_id, 0), ('b', target_id, 0); + telemetry_seen_peak = 2; + telemetry_frontier_peak = 2; + telemetry_queue_peak = 2; + end if; + + while not overflowed loop + select min(depth) into forward_depth from pg_temp.asb_front where side = 'f'; + select min(depth) into backward_depth from pg_temp.asb_front where side = 'b'; + select count(*) into forward_width from pg_temp.asb_front where side = 'f' and depth = forward_depth; + select count(*) into backward_width from pg_temp.asb_front where side = 'b' and depth = backward_depth; + select coalesce(forward_depth, max(depth), 0) into forward_ready_depth + from pg_temp.asb_seen where side = 'f'; + select coalesce(backward_depth, max(depth), 0) into backward_ready_depth + from pg_temp.asb_seen where side = 'b'; + + if best_distance is null and (forward_depth is null or backward_depth is null) then + exit; + end if; + + if best_distance is not null + and (forward_depth is null or backward_depth is null + or forward_depth + backward_depth >= best_distance) then + cut_depth = best_distance / 2; + if forward_ready_depth >= cut_depth + and backward_ready_depth >= best_distance - cut_depth then + exit; + elsif forward_ready_depth < cut_depth then + chosen_side = 'f'; + else + chosen_side = 'b'; + end if; + elsif scheduler = 'strict_alternating_node' then + chosen_side = strict_side; + if (chosen_side = 'f' and forward_depth is null) + or (chosen_side = 'b' and backward_depth is null) then + chosen_side = case chosen_side when 'f' then 'b' else 'f' end; + end if; + strict_side = case chosen_side when 'f' then 'b' else 'f' end; + else + if forward_depth is null then + chosen_side = 'b'; + elsif backward_depth is null then + chosen_side = 'f'; + else + -- Stable equality tie break: forward. + chosen_side = case when forward_width <= backward_width then 'f' else 'b' end; + end if; + end if; + + truncate table pg_temp.asb_active, pg_temp.asb_candidate_node, + pg_temp.asb_candidate_predecessor; + if scheduler = 'strict_alternating_node' + and not (best_distance is not null + and (forward_depth is null or backward_depth is null + or forward_depth + backward_depth >= best_distance)) then + insert into pg_temp.asb_active(side, node_id, depth) + select side, node_id, depth + from pg_temp.asb_front + where side = chosen_side + order by queue_order + limit 1; + elsif scheduler = 'strict_alternating_node' then + -- Cut completion retains node granularity while allowing the incomplete + -- side to advance consecutively after minimum distance is proven. + insert into pg_temp.asb_active(side, node_id, depth) + select side, node_id, depth + from pg_temp.asb_front + where side = chosen_side + order by queue_order + limit 1; + else + insert into pg_temp.asb_active(side, node_id, depth) + select side, node_id, depth + from pg_temp.asb_front + where side = chosen_side + and depth = case chosen_side when 'f' then forward_depth else backward_depth end + order by queue_order; + end if; + + delete from pg_temp.asb_front front + using pg_temp.asb_active active + where front.side = active.side and front.node_id = active.node_id; + + telemetry_scheduler_actions = telemetry_scheduler_actions + 1; + telemetry_action_candidate_edges = 0; + telemetry_action_meetings = 0; + select min(depth) into telemetry_action_depth from pg_temp.asb_active; + if telemetry_search_id is not null then + if (chosen_side = 'f' and not inbound) or (chosen_side = 'b' and inbound) then + select count(*) into telemetry_action_candidate_edges + from pg_temp.asb_active active + join edge e on e.graph_id = target_graph_id and e.start_id = active.node_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)); + else + select count(*) into telemetry_action_candidate_edges + from pg_temp.asb_active active + join edge e on e.graph_id = target_graph_id and e.end_id = active.node_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)); + end if; + end if; + + if not exists (select 1 from pg_temp.asb_active where depth < max_depth) then + if telemetry_search_id is not null then + select count(*) into seen_rows from pg_temp.asb_seen; + select count(*) into active_rows from pg_temp.asb_active; + select count(*) into frontier_rows from pg_temp.asb_front; + select count(*) into predecessor_rows from pg_temp.asb_predecessor; + telemetry_action_index = telemetry_action_index + 1; + telemetry_seen_peak = greatest(telemetry_seen_peak, seen_rows); + telemetry_frontier_peak = greatest(telemetry_frontier_peak, active_rows + frontier_rows); + telemetry_queue_peak = greatest(telemetry_queue_peak, frontier_rows); + telemetry_predecessor_peak = greatest(telemetry_predecessor_peak, predecessor_rows); + perform public._record_bidirectional_all_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + chosen_side::text, + case scheduler when 'strict_alternating_node' then 'dequeue_node' else 'expand_level' end, + telemetry_action_depth, active_rows + frontier_rows, 0, 0, + seen_rows, frontier_rows, predecessor_rows, 0); + end if; + continue; + end if; + + select count(*) into seen_rows from pg_temp.asb_seen; + select count(*) into active_rows from pg_temp.asb_active; + select count(*) into frontier_rows from pg_temp.asb_front; + select count(*) into predecessor_rows from pg_temp.asb_predecessor; + discovery_admission_limit = least(state_limit - seen_rows, + frontier_limit - active_rows - frontier_rows); + if discovery_admission_limit < 0 then + overflowed = true; + if telemetry_search_id is not null then + telemetry_action_index = telemetry_action_index + 1; + telemetry_seen_peak = greatest(telemetry_seen_peak, seen_rows); + telemetry_frontier_peak = greatest(telemetry_frontier_peak, active_rows + frontier_rows); + telemetry_queue_peak = greatest(telemetry_queue_peak, frontier_rows); + telemetry_predecessor_peak = greatest(telemetry_predecessor_peak, predecessor_rows); + perform public._record_bidirectional_all_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + chosen_side::text, + case scheduler when 'strict_alternating_node' then 'dequeue_node' else 'expand_level' end, + telemetry_action_depth, active_rows + frontier_rows, 0, 0, + seen_rows, frontier_rows, predecessor_rows, 0); + end if; + exit; + end if; + + -- First admit distinct unseen nodes with a discovery cap+1 sentinel. + if chosen_side = 'f' and not inbound then + insert into pg_temp.asb_candidate_node(side, node_id, depth) + select 'f', candidate.node_id, candidate.depth + from ( + select distinct e.end_id as node_id, active.depth + 1 as depth + from pg_temp.asb_active active + join edge e on e.graph_id = target_graph_id and e.start_id = active.node_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.asb_seen seen where seen.side = 'f' and seen.node_id = e.end_id) + order by e.end_id + limit discovery_admission_limit + 1 + ) candidate; + elsif chosen_side = 'f' and inbound then + insert into pg_temp.asb_candidate_node(side, node_id, depth) + select 'f', candidate.node_id, candidate.depth + from ( + select distinct e.start_id as node_id, active.depth + 1 as depth + from pg_temp.asb_active active + join edge e on e.graph_id = target_graph_id and e.end_id = active.node_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.asb_seen seen where seen.side = 'f' and seen.node_id = e.start_id) + order by e.start_id + limit discovery_admission_limit + 1 + ) candidate; + elsif chosen_side = 'b' and not inbound then + insert into pg_temp.asb_candidate_node(side, node_id, depth) + select 'b', candidate.node_id, candidate.depth + from ( + select distinct e.start_id as node_id, active.depth + 1 as depth + from pg_temp.asb_active active + join edge e on e.graph_id = target_graph_id and e.end_id = active.node_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.asb_seen seen where seen.side = 'b' and seen.node_id = e.start_id) + order by e.start_id + limit discovery_admission_limit + 1 + ) candidate; + else + insert into pg_temp.asb_candidate_node(side, node_id, depth) + select 'b', candidate.node_id, candidate.depth + from ( + select distinct e.end_id as node_id, active.depth + 1 as depth + from pg_temp.asb_active active + join edge e on e.graph_id = target_graph_id and e.start_id = active.node_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.asb_seen seen where seen.side = 'b' and seen.node_id = e.end_id) + order by e.end_id + limit discovery_admission_limit + 1 + ) candidate; + end if; + + select count(*) into candidate_node_rows from pg_temp.asb_candidate_node; + if seen_rows + candidate_node_rows > state_limit + or active_rows + frontier_rows + candidate_node_rows > frontier_limit then + overflowed = true; + if telemetry_search_id is not null then + select count(*) into telemetry_action_meetings + from pg_temp.asb_candidate_node candidate + join pg_temp.asb_seen opposite + on opposite.node_id = candidate.node_id and opposite.side <> candidate.side + where candidate.depth + opposite.depth between min_depth and max_depth; + telemetry_candidate_edges = telemetry_candidate_edges + telemetry_action_candidate_edges; + telemetry_distinct_new_nodes = telemetry_distinct_new_nodes + candidate_node_rows; + telemetry_meeting_candidates = telemetry_meeting_candidates + telemetry_action_meetings; + telemetry_seen_peak = greatest(telemetry_seen_peak, seen_rows + candidate_node_rows); + telemetry_frontier_peak = greatest(telemetry_frontier_peak, active_rows + frontier_rows + candidate_node_rows); + telemetry_queue_peak = greatest(telemetry_queue_peak, frontier_rows + candidate_node_rows); + telemetry_action_index = telemetry_action_index + 1; + perform public._record_bidirectional_all_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + chosen_side::text, + case scheduler when 'strict_alternating_node' then 'dequeue_node' else 'expand_level' end, + telemetry_action_depth, active_rows + frontier_rows + candidate_node_rows, + telemetry_action_candidate_edges, candidate_node_rows, + seen_rows + candidate_node_rows, frontier_rows + candidate_node_rows, + predecessor_rows, telemetry_action_meetings); + end if; + exit; + end if; + + predecessor_admission_limit = predecessor_limit - predecessor_rows; + if predecessor_admission_limit < 0 then + overflowed = true; + exit; + end if; + + -- Then retain every relationship-distinct edge into a newly discovered or + -- already-seen node at the same minimum depth. This second admission is + -- independent of distinct-node discovery. + if chosen_side = 'f' and not inbound then + insert into pg_temp.asb_candidate_predecessor(side, node_id, depth, adjacent_id, edge_id) + select 'f', candidate.node_id, candidate.depth, candidate.adjacent_id, candidate.edge_id + from ( + select e.end_id as node_id, active.depth + 1 as depth, + active.node_id as adjacent_id, e.id as edge_id + from pg_temp.asb_active active + join edge e on e.graph_id = target_graph_id and e.start_id = active.node_id + left join pg_temp.asb_seen seen on seen.side = 'f' and seen.node_id = e.end_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and (seen.node_id is null or seen.depth = active.depth + 1) + and (seen.node_id is not null or exists ( + select 1 from pg_temp.asb_candidate_node admitted + where admitted.side = 'f' and admitted.node_id = e.end_id)) + order by e.end_id, e.id, active.node_id + limit predecessor_admission_limit + 1 + ) candidate; + elsif chosen_side = 'f' and inbound then + insert into pg_temp.asb_candidate_predecessor(side, node_id, depth, adjacent_id, edge_id) + select 'f', candidate.node_id, candidate.depth, candidate.adjacent_id, candidate.edge_id + from ( + select e.start_id as node_id, active.depth + 1 as depth, + active.node_id as adjacent_id, e.id as edge_id + from pg_temp.asb_active active + join edge e on e.graph_id = target_graph_id and e.end_id = active.node_id + left join pg_temp.asb_seen seen on seen.side = 'f' and seen.node_id = e.start_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and (seen.node_id is null or seen.depth = active.depth + 1) + and (seen.node_id is not null or exists ( + select 1 from pg_temp.asb_candidate_node admitted + where admitted.side = 'f' and admitted.node_id = e.start_id)) + order by e.start_id, e.id, active.node_id + limit predecessor_admission_limit + 1 + ) candidate; + elsif chosen_side = 'b' and not inbound then + insert into pg_temp.asb_candidate_predecessor(side, node_id, depth, adjacent_id, edge_id) + select 'b', candidate.node_id, candidate.depth, candidate.adjacent_id, candidate.edge_id + from ( + select e.start_id as node_id, active.depth + 1 as depth, + active.node_id as adjacent_id, e.id as edge_id + from pg_temp.asb_active active + join edge e on e.graph_id = target_graph_id and e.end_id = active.node_id + left join pg_temp.asb_seen seen on seen.side = 'b' and seen.node_id = e.start_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and (seen.node_id is null or seen.depth = active.depth + 1) + and (seen.node_id is not null or exists ( + select 1 from pg_temp.asb_candidate_node admitted + where admitted.side = 'b' and admitted.node_id = e.start_id)) + order by e.start_id, e.id, active.node_id + limit predecessor_admission_limit + 1 + ) candidate; + else + insert into pg_temp.asb_candidate_predecessor(side, node_id, depth, adjacent_id, edge_id) + select 'b', candidate.node_id, candidate.depth, candidate.adjacent_id, candidate.edge_id + from ( + select e.end_id as node_id, active.depth + 1 as depth, + active.node_id as adjacent_id, e.id as edge_id + from pg_temp.asb_active active + join edge e on e.graph_id = target_graph_id and e.start_id = active.node_id + left join pg_temp.asb_seen seen on seen.side = 'b' and seen.node_id = e.end_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and (seen.node_id is null or seen.depth = active.depth + 1) + and (seen.node_id is not null or exists ( + select 1 from pg_temp.asb_candidate_node admitted + where admitted.side = 'b' and admitted.node_id = e.end_id)) + order by e.end_id, e.id, active.node_id + limit predecessor_admission_limit + 1 + ) candidate; + end if; + + select count(*) into candidate_predecessor_rows + from pg_temp.asb_candidate_predecessor; + if telemetry_search_id is not null then + select count(*) into telemetry_action_meetings + from pg_temp.asb_candidate_node candidate + join pg_temp.asb_seen opposite + on opposite.node_id = candidate.node_id and opposite.side <> candidate.side + where candidate.depth + opposite.depth between min_depth and max_depth; + telemetry_candidate_edges = telemetry_candidate_edges + telemetry_action_candidate_edges; + telemetry_distinct_new_nodes = telemetry_distinct_new_nodes + candidate_node_rows; + telemetry_meeting_candidates = telemetry_meeting_candidates + telemetry_action_meetings; + telemetry_same_depth_predecessors = telemetry_same_depth_predecessors + + greatest(candidate_predecessor_rows - candidate_node_rows, 0); + telemetry_seen_peak = greatest(telemetry_seen_peak, seen_rows + candidate_node_rows); + telemetry_frontier_peak = greatest(telemetry_frontier_peak, active_rows + frontier_rows + candidate_node_rows); + telemetry_queue_peak = greatest(telemetry_queue_peak, frontier_rows + candidate_node_rows); + telemetry_predecessor_peak = greatest(telemetry_predecessor_peak, predecessor_rows + candidate_predecessor_rows); + telemetry_action_index = telemetry_action_index + 1; + perform public._record_bidirectional_all_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + chosen_side::text, + case scheduler when 'strict_alternating_node' then 'dequeue_node' else 'expand_level' end, + telemetry_action_depth, active_rows + frontier_rows + candidate_node_rows, + telemetry_action_candidate_edges, candidate_node_rows, + seen_rows + candidate_node_rows, frontier_rows + candidate_node_rows, + predecessor_rows + candidate_predecessor_rows, + telemetry_action_meetings); + end if; + if predecessor_rows + candidate_predecessor_rows > predecessor_limit then + overflowed = true; + exit; + end if; + + insert into pg_temp.asb_predecessor(side, node_id, depth, adjacent_id, edge_id) + select side, node_id, depth, adjacent_id, edge_id + from pg_temp.asb_candidate_predecessor + order by side, node_id, edge_id, adjacent_id + on conflict do nothing; + insert into pg_temp.asb_seen(side, node_id, depth) + select side, node_id, depth + from pg_temp.asb_candidate_node + order by side, node_id + on conflict do nothing; + + if chosen_side = 'f' then + insert into pg_temp.asb_front(side, node_id, depth, queue_order) + select side, node_id, depth, + forward_tail + row_number() over (order by node_id) + from pg_temp.asb_candidate_node; + forward_tail = forward_tail + candidate_node_rows; + else + insert into pg_temp.asb_front(side, node_id, depth, queue_order) + select side, node_id, depth, + backward_tail + row_number() over (order by node_id) + from pg_temp.asb_candidate_node; + backward_tail = backward_tail + candidate_node_rows; + end if; + + candidate_meeting = null; + candidate_distance = null; + select candidate.node_id, candidate.depth + opposite.depth + into candidate_meeting, candidate_distance + from pg_temp.asb_candidate_node candidate + join pg_temp.asb_seen opposite + on opposite.node_id = candidate.node_id and opposite.side <> candidate.side + where candidate.depth + opposite.depth between min_depth and max_depth + order by candidate.depth + opposite.depth, candidate.node_id + limit 1; + if candidate_distance is not null + and (best_distance is null or candidate_distance < best_distance) then + best_distance = candidate_distance; + end if; + end loop; + + if overflowed then + perform public.clear_bidirectional_all_shortest_path_workspace(); + return query + select fallback.root_id, fallback.next_id, fallback.depth, + fallback.satisfied, fallback.is_cycle, fallback.path + from public.all_shortest_paths_dag(target_graph_id, source_id, target_id, + min_depth, max_depth, edge_kind_ids, + inbound) fallback; + get diagnostics emitted_count = row_count; + if telemetry_search_id is not null then + perform public._finish_bidirectional_all_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'exact_a1_fallback', + telemetry_scheduler_actions, telemetry_candidate_edges, + telemetry_distinct_new_nodes, telemetry_seen_peak, + telemetry_frontier_peak, telemetry_queue_peak, + telemetry_predecessor_peak, telemetry_meeting_candidates, + best_distance, emitted_count, telemetry_same_depth_predecessors, + coalesce(meeting_nodes, 0), cut_depth, coalesce(path_count_estimate, 0), + telemetry_path_count_saturated, telemetry_enumerated_candidates, + telemetry_duplicate_rejects, emitted_count, + emitted_count * coalesce(best_distance, 0), coalesce(output_bytes, 0), + true, true); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('exact_a1_fallback', true, 'ASP-A1-DAG'); + return; + end if; + if best_distance is null then + perform public.clear_bidirectional_all_shortest_path_workspace(); + if telemetry_search_id is not null then + perform public._finish_bidirectional_all_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'search_no_path', + telemetry_scheduler_actions, telemetry_candidate_edges, + telemetry_distinct_new_nodes, telemetry_seen_peak, + telemetry_frontier_peak, telemetry_queue_peak, + telemetry_predecessor_peak, telemetry_meeting_candidates, + null, 0, telemetry_same_depth_predecessors, 0, null, 0, false, + 0, 0, 0, 0, 0, false, false); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('search_no_path', false, 'ASP-A1-DAG'); + return; + end if; + + cut_depth = best_distance / 2; + select count(*) into meeting_nodes + from pg_temp.asb_seen forward_seen + join pg_temp.asb_seen backward_seen + on backward_seen.node_id = forward_seen.node_id and backward_seen.side = 'b' + where forward_seen.side = 'f' and forward_seen.depth = cut_depth + and backward_seen.depth = best_distance - cut_depth; + if meeting_nodes = 0 then + overflowed = true; + end if; + + -- Saturating dynamic programming over each half-DAG bounds enumeration and + -- bytes before any edge array is materialized. + if not overflowed then + path_array_bytes = pg_column_size(array_fill(0::int8, array[best_distance])); + if path_array_bytes <= 126 then + path_array_bytes = path_array_bytes - 3; + end if; + path_count_limit = least(enumeration_limit, output_bytes_limit / path_array_bytes); + path_count_sentinel = path_count_limit + 1; + truncate table pg_temp.asb_path_count, pg_temp.asb_output; + insert into pg_temp.asb_path_count(side, node_id, depth, path_count) + values ('f', source_id, 0, 1), ('b', target_id, 0, 1); + + for count_depth in 1..cut_depth loop + insert into pg_temp.asb_path_count(side, node_id, depth, path_count) + select 'f', predecessor.node_id, count_depth, + least(path_count_sentinel::numeric, + sum(adjacent.path_count::numeric))::int8 + from pg_temp.asb_predecessor predecessor + join pg_temp.asb_path_count adjacent + on adjacent.side = 'f' and adjacent.node_id = predecessor.adjacent_id + and adjacent.depth = count_depth - 1 + where predecessor.side = 'f' and predecessor.depth = count_depth + group by predecessor.node_id; + end loop; + for count_depth in 1..(best_distance - cut_depth) loop + insert into pg_temp.asb_path_count(side, node_id, depth, path_count) + select 'b', predecessor.node_id, count_depth, + least(path_count_sentinel::numeric, + sum(adjacent.path_count::numeric))::int8 + from pg_temp.asb_predecessor predecessor + join pg_temp.asb_path_count adjacent + on adjacent.side = 'b' and adjacent.node_id = predecessor.adjacent_id + and adjacent.depth = count_depth - 1 + where predecessor.side = 'b' and predecessor.depth = count_depth + group by predecessor.node_id; + end loop; + + select least(path_count_sentinel::numeric, + coalesce(sum(least(path_count_sentinel::numeric, + forward_count.path_count::numeric + * backward_count.path_count::numeric)), 0))::int8 + into path_count_estimate + from pg_temp.asb_path_count forward_count + join pg_temp.asb_path_count backward_count + on backward_count.side = 'b' and backward_count.node_id = forward_count.node_id + and backward_count.depth = best_distance - cut_depth + where forward_count.side = 'f' and forward_count.depth = cut_depth; + telemetry_path_count_saturated = path_count_estimate >= path_count_sentinel; + if path_count_estimate > path_count_limit or path_count_estimate = 0 then + overflowed = true; + end if; + end if; + + if not overflowed then + insert into pg_temp.asb_output(edge_ids, output_bytes) + with recursive + meeting(node_id) as materialized ( + select forward_seen.node_id + from pg_temp.asb_seen forward_seen + join pg_temp.asb_seen backward_seen + on backward_seen.node_id = forward_seen.node_id and backward_seen.side = 'b' + where forward_seen.side = 'f' and forward_seen.depth = cut_depth + and backward_seen.depth = best_distance - cut_depth + ), + forward_paths(meeting_id, node_id, path_depth, edge_ids) as ( + select meeting.node_id, meeting.node_id, cut_depth, array []::int8[] + from meeting + union all + select forward_paths.meeting_id, predecessor.adjacent_id, + forward_paths.path_depth - 1, + array[predecessor.edge_id]::int8[] || forward_paths.edge_ids + from forward_paths + join pg_temp.asb_predecessor predecessor + on predecessor.side = 'f' and predecessor.node_id = forward_paths.node_id + and predecessor.depth = forward_paths.path_depth + ), + backward_paths(meeting_id, node_id, path_depth, edge_ids) as ( + select meeting.node_id, meeting.node_id, best_distance - cut_depth, + array []::int8[] + from meeting + union all + select backward_paths.meeting_id, successor.adjacent_id, + backward_paths.path_depth - 1, + backward_paths.edge_ids || successor.edge_id + from backward_paths + join pg_temp.asb_predecessor successor + on successor.side = 'b' and successor.node_id = backward_paths.node_id + and successor.depth = backward_paths.path_depth + ), + stitched(edge_ids) as ( + select forward_paths.edge_ids || backward_paths.edge_ids + from forward_paths + join backward_paths using (meeting_id) + where forward_paths.node_id = source_id and forward_paths.path_depth = 0 + and backward_paths.node_id = target_id and backward_paths.path_depth = 0 + ) + select staged.edge_ids, pg_column_size(staged.edge_ids)::int8 + from ( + select distinct stitched.edge_ids + from stitched + where cardinality(stitched.edge_ids) = best_distance + and cardinality(stitched.edge_ids) = ( + select count(distinct path_edge.edge_id) + from unnest(stitched.edge_ids) path_edge(edge_id)) + order by stitched.edge_ids + limit enumeration_limit + 1 + ) staged; + + select count(*), coalesce(sum(asb_output.output_bytes), 0) + into output_rows, output_bytes + from pg_temp.asb_output; + telemetry_enumerated_candidates = output_rows; + telemetry_duplicate_rejects = greatest(coalesce(path_count_estimate, 0) - output_rows, 0); + if output_rows > enumeration_limit or output_bytes > output_bytes_limit + or output_rows <> path_count_estimate then + overflowed = true; + end if; + end if; + + if overflowed then + perform public.clear_bidirectional_all_shortest_path_workspace(); + return query + select fallback.root_id, fallback.next_id, fallback.depth, + fallback.satisfied, fallback.is_cycle, fallback.path + from public.all_shortest_paths_dag(target_graph_id, source_id, target_id, + min_depth, max_depth, edge_kind_ids, + inbound) fallback; + get diagnostics emitted_count = row_count; + if telemetry_search_id is not null then + perform public._finish_bidirectional_all_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'exact_a1_fallback', + telemetry_scheduler_actions, telemetry_candidate_edges, + telemetry_distinct_new_nodes, telemetry_seen_peak, + telemetry_frontier_peak, telemetry_queue_peak, + telemetry_predecessor_peak, telemetry_meeting_candidates, + best_distance, emitted_count, telemetry_same_depth_predecessors, + coalesce(meeting_nodes, 0), cut_depth, coalesce(path_count_estimate, 0), + telemetry_path_count_saturated, telemetry_enumerated_candidates, + telemetry_duplicate_rejects, emitted_count, + emitted_count * coalesce(best_distance, 0), coalesce(output_bytes, 0), + true, true); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('exact_a1_fallback', true, 'ASP-A1-DAG'); + return; + end if; + + return query + select source_id, target_id, best_distance, true, false, output.edge_ids + from pg_temp.asb_output output + order by output.edge_ids; + get diagnostics emitted_count = row_count; + perform public.clear_bidirectional_all_shortest_path_workspace(); + if telemetry_search_id is not null then + perform public._finish_bidirectional_all_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'bidirectional_search', + telemetry_scheduler_actions, telemetry_candidate_edges, + telemetry_distinct_new_nodes, telemetry_seen_peak, + telemetry_frontier_peak, telemetry_queue_peak, + telemetry_predecessor_peak, telemetry_meeting_candidates, + best_distance, emitted_count, telemetry_same_depth_predecessors, + meeting_nodes, cut_depth, path_count_estimate, + telemetry_path_count_saturated, telemetry_enumerated_candidates, + telemetry_duplicate_rejects, emitted_count, + emitted_count * best_distance, output_bytes, false, false); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('bidirectional_search', false, 'ASP-A1-DAG'); +end; +$$ + language plpgsql + volatile + strict + cost 100 + set recursive_worktable_factor = 1 + rows 100; + +create or replace function public.all_shortest_paths_b1_strict_alternating( + target_graph_id int4, + source_id int8, + target_id int8, + min_depth int4, + max_depth int4, + edge_kind_ids int2[], + inbound bool, + state_limit int8, + frontier_limit int8, + predecessor_limit int8, + enumeration_limit int8, + output_bytes_limit int8) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +select * +from public.all_shortest_paths_bidirectional_compact_v1( + target_graph_id, source_id, target_id, min_depth, max_depth, + edge_kind_ids, inbound, state_limit, frontier_limit, predecessor_limit, + enumeration_limit, output_bytes_limit, 'strict_alternating_node'); +$$ + language sql + volatile + strict + cost 100 + rows 100; + +create or replace function public.all_shortest_paths_b2_smaller_current_level( + target_graph_id int4, + source_id int8, + target_id int8, + min_depth int4, + max_depth int4, + edge_kind_ids int2[], + inbound bool, + state_limit int8, + frontier_limit int8, + predecessor_limit int8, + enumeration_limit int8, + output_bytes_limit int8) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +select * +from public.all_shortest_paths_bidirectional_compact_v1( + target_graph_id, source_id, target_id, min_depth, max_depth, + edge_kind_ids, inbound, state_limit, frontier_limit, predecessor_limit, + enumeration_limit, output_bytes_limit, 'smaller_current_level'); +$$ + language sql + volatile + strict + cost 100 + rows 100; + +create or replace function public.bsp_workspace_fragment(fragment text) + returns text as +$$ +select replace( + replace( + replace( + case + when position('pg_temp.bsp_' in fragment) > 0 then fragment + else replace( + replace( + replace( + replace( + replace( + replace( + replace(fragment, + 'on conflict on constraint forward_visited_pkey', 'on conflict on constraint bsp_forward_visited_pkey'), + 'on conflict on constraint backward_visited_pkey', 'on conflict on constraint bsp_backward_visited_pkey'), + 'forward_visited', 'pg_temp.bsp_forward_visited'), + 'backward_visited', 'pg_temp.bsp_backward_visited'), + 'forward_front', 'pg_temp.bsp_forward_front'), + 'backward_front', 'pg_temp.bsp_backward_front'), + 'next_front', 'pg_temp.bsp_next_front') + end, + 'traversal_root_filter', 'pg_temp.bsp_root_filter'), + 'traversal_terminal_filter', 'pg_temp.bsp_terminal_filter'), + 'traversal_pair_filter', 'pg_temp.bsp_pair_filter'); +$$ + language sql + immutable + parallel safe + strict; + +-- The bidirectional shortest-path workspace is session-local and survives +-- transaction boundaries. Warm calls retain the table and index OIDs and only +-- clear row state. The version marker lets upgrades rebuild the known object +-- set without touching unrelated temporary objects in the session. +create or replace function public.ensure_bsp_core_workspace() + returns void as +$$ +declare + expected_version constant int4 := 1; + present_version int4; +begin + if to_regclass('pg_temp.bsp_workspace_version') is not null then + select version into present_version from pg_temp.bsp_workspace_version limit 1; + end if; + + if present_version is not null and present_version is distinct from expected_version then + drop table if exists pg_temp.bsp_resolved_pairs; + drop table if exists pg_temp.bsp_unresolved_pairs; + drop table if exists pg_temp.bsp_pair_filter; + drop table if exists pg_temp.bsp_terminal_filter; + drop table if exists pg_temp.bsp_root_filter; + drop table if exists pg_temp.bsp_backward_visited; + drop table if exists pg_temp.bsp_forward_visited; + drop table if exists pg_temp.bsp_backward_front; + drop table if exists pg_temp.bsp_next_front; + drop table if exists pg_temp.bsp_forward_front; + drop table if exists pg_temp.bsp_workspace_version; + end if; + + if to_regclass('pg_temp.bsp_workspace_version') is null then + create temporary table bsp_workspace_version + ( + version int4 not null primary key + ) on commit preserve rows; + + create temporary table bsp_forward_front + ( + root_id int8 not null, next_id int8 not null, depth int4 not null, + satisfied bool, is_cycle bool not null, path int8[] not null + ) on commit preserve rows; + create index bsp_forward_front_next_id_index on bsp_forward_front using btree (next_id); + create index bsp_forward_front_root_id_next_id_index on bsp_forward_front using btree (root_id, next_id); + + create temporary table bsp_backward_front + ( + root_id int8 not null, next_id int8 not null, depth int4 not null, + satisfied bool, is_cycle bool not null, path int8[] not null + ) on commit preserve rows; + create index bsp_backward_front_next_id_index on bsp_backward_front using btree (next_id); + create index bsp_backward_front_root_id_next_id_index on bsp_backward_front using btree (root_id, next_id); + + create temporary table bsp_next_front + ( + root_id int8 not null, next_id int8 not null, depth int4 not null, + satisfied bool, is_cycle bool not null, path int8[] not null + ) on commit preserve rows; + create index bsp_next_front_next_id_index on bsp_next_front using btree (next_id); + create index bsp_next_front_root_id_next_id_index on bsp_next_front using btree (root_id, next_id); - create index if not exists traversal_pair_filter_terminal_id_root_id_index on traversal_pair_filter using btree (terminal_id, root_id); + create temporary table bsp_forward_visited + ( + root_id int8 not null, + id int8 not null, + constraint bsp_forward_visited_pkey primary key (root_id, id) + ) on commit preserve rows; - truncate table traversal_root_filter; - truncate table traversal_terminal_filter; - truncate table traversal_pair_filter; + create temporary table bsp_backward_visited + ( + root_id int8 not null, + id int8 not null, + constraint bsp_backward_visited_pkey primary key (root_id, id) + ) on commit preserve rows; - return; + insert into bsp_workspace_version(version) values (expected_version); + end if; end; $$ language plpgsql volatile; -create or replace function public.create_traversal_filter_tables(root_ids int8[], terminal_ids int8[]) +create or replace function public.ensure_bsp_generic_workspace() returns void as $$ begin - perform create_traversal_filter_tables(); + perform public.ensure_bsp_core_workspace(); - insert into traversal_root_filter - select distinct root_id - from unnest(root_ids) as root_ids(root_id) - where root_id is not null - on conflict (id) do nothing; + if to_regclass('pg_temp.bsp_root_filter') is null then + create temporary table bsp_root_filter + ( + id int8 not null primary key + ) on commit preserve rows; + create temporary table bsp_terminal_filter + ( + id int8 not null primary key + ) on commit preserve rows; + create temporary table bsp_pair_filter + ( + root_id int8 not null, + terminal_id int8 not null, + primary key (root_id, terminal_id) + ) on commit preserve rows; + create index bsp_pair_filter_terminal_id_root_id_index on bsp_pair_filter using btree (terminal_id, root_id); - insert into traversal_terminal_filter - select distinct terminal_id - from unnest(terminal_ids) as terminal_ids(terminal_id) - where terminal_id is not null - on conflict (id) do nothing; + create temporary table bsp_unresolved_pairs + ( + root_id int8 not null, + terminal_id int8 not null, + constraint bsp_unresolved_pairs_pkey primary key (root_id, terminal_id) + ) on commit preserve rows; + create index bsp_unresolved_pairs_terminal_id_root_id_index on bsp_unresolved_pairs using btree (terminal_id, root_id); - analyze traversal_root_filter; - analyze traversal_terminal_filter; + create temporary table bsp_resolved_pairs + ( + root_id int8 not null, next_id int8 not null, depth int4 not null, + satisfied bool, is_cycle bool not null, path int8[] not null, + constraint bsp_resolved_pairs_pkey primary key (root_id, next_id) + ) on commit preserve rows; + end if; +end; +$$ + language plpgsql + volatile; - return; +create or replace function public.reset_bsp_workspace(include_generic bool) + returns void as +$$ +begin + if include_generic then + perform public.ensure_bsp_generic_workspace(); + truncate table pg_temp.bsp_forward_front, pg_temp.bsp_backward_front, pg_temp.bsp_next_front, + pg_temp.bsp_forward_visited, pg_temp.bsp_backward_visited, + pg_temp.bsp_root_filter, pg_temp.bsp_terminal_filter, pg_temp.bsp_pair_filter, + pg_temp.bsp_unresolved_pairs, pg_temp.bsp_resolved_pairs; + else + perform public.ensure_bsp_core_workspace(); + truncate table pg_temp.bsp_forward_front, pg_temp.bsp_backward_front, pg_temp.bsp_next_front, + pg_temp.bsp_forward_visited, pg_temp.bsp_backward_visited; + end if; end; $$ language plpgsql volatile strict; -create or replace function public.create_traversal_filter_tables(root_filter text, terminal_filter text, pair_filter text) +create or replace function public.load_bsp_filter_tables(root_filter text, terminal_filter text, pair_filter text) returns void as $$ begin - perform create_traversal_filter_tables(); - if length(pair_filter) > 0 then - execute pair_filter; + execute replace(pair_filter, 'traversal_pair_filter', 'pg_temp.bsp_pair_filter'); end if; - if length(root_filter) > 0 then - execute root_filter; + execute replace(root_filter, 'traversal_root_filter', 'pg_temp.bsp_root_filter'); elsif length(pair_filter) > 0 then - insert into traversal_root_filter - select distinct root_id - from traversal_pair_filter + insert into pg_temp.bsp_root_filter + select distinct root_id from pg_temp.bsp_pair_filter on conflict (id) do nothing; end if; - if length(terminal_filter) > 0 then - execute terminal_filter; + execute replace(terminal_filter, 'traversal_terminal_filter', 'pg_temp.bsp_terminal_filter'); elsif length(pair_filter) > 0 then - insert into traversal_terminal_filter - select distinct terminal_id - from traversal_pair_filter + insert into pg_temp.bsp_terminal_filter + select distinct terminal_id from pg_temp.bsp_pair_filter on conflict (id) do nothing; end if; - analyze traversal_root_filter; - analyze traversal_terminal_filter; - analyze traversal_pair_filter; - - return; -end; -$$ - language plpgsql - volatile - strict; - -create or replace function public.create_traversal_filter_tables(root_filter text, terminal_filter text) - returns void as -$$ -select public.create_traversal_filter_tables(root_filter, terminal_filter, ''::text); -$$ - language sql - volatile - strict; - -create or replace function public.shortest_path_self_endpoint_error(root_id int8, terminal_id int8) - returns bool as -$$ -begin - raise exception using - errcode = '22023', - message = format('shortest path endpoints must not resolve to the same node: root_id=%s terminal_id=%s', - root_id, - terminal_id); - - return false; + analyze pg_temp.bsp_root_filter; + analyze pg_temp.bsp_terminal_filter; + analyze pg_temp.bsp_pair_filter; end; $$ language plpgsql @@ -981,7 +4645,7 @@ $$ begin perform create_unidirectional_pathspace_tables(); - create temporary table backward_front + create temporary table if not exists backward_front ( root_id int8 not null, next_id int8 not null, @@ -989,11 +4653,13 @@ begin satisfied bool, is_cycle bool not null, path int8[] not null - ) on commit drop; + ) on commit preserve rows; - create index backward_front_next_id_index on backward_front using btree (next_id); - create index backward_front_satisfied_index on backward_front using btree (root_id, next_id, depth) where satisfied; - create index backward_front_is_cycle_index on backward_front using btree (root_id, next_id) where is_cycle; + create index if not exists backward_front_next_id_index on backward_front using btree (next_id); + create index if not exists backward_front_satisfied_index on backward_front using btree (root_id, next_id, depth) where satisfied; + create index if not exists backward_front_is_cycle_index on backward_front using btree (root_id, next_id) where is_cycle; + + truncate table backward_front; end; $$ language plpgsql @@ -1004,9 +4670,9 @@ create or replace function public.create_bidirectional_pair_pathspace_indexes() returns void as $$ begin - create index forward_front_root_id_next_id_index on forward_front using btree (root_id, next_id); - create index backward_front_root_id_next_id_index on backward_front using btree (root_id, next_id); - create index next_front_root_id_next_id_index on next_front using btree (root_id, next_id); + create index if not exists forward_front_root_id_next_id_index on forward_front using btree (root_id, next_id); + create index if not exists backward_front_root_id_next_id_index on backward_front using btree (root_id, next_id); + create index if not exists next_front_root_id_next_id_index on next_front using btree (root_id, next_id); end; $$ language plpgsql @@ -1017,19 +4683,21 @@ create or replace function public.create_bidirectional_shortest_path_tables() returns void as $$ begin - create temporary table forward_visited + create temporary table if not exists forward_visited ( root_id int8 not null, id int8 not null, primary key (root_id, id) - ) on commit drop; + ) on commit preserve rows; - create temporary table backward_visited + create temporary table if not exists backward_visited ( root_id int8 not null, id int8 not null, primary key (root_id, id) - ) on commit drop; + ) on commit preserve rows; + + truncate table forward_visited, backward_visited; perform create_bidirectional_pathspace_tables(); perform create_bidirectional_pair_pathspace_indexes(); @@ -1043,13 +4711,9 @@ create or replace function public.swap_forward_front() returns void as $$ begin - alter table forward_front - rename to forward_front_old; - alter table next_front - rename to forward_front; - alter table forward_front_old - rename to next_front; + truncate table forward_front; + insert into forward_front select * from next_front; truncate table next_front; delete from forward_front r where r.is_cycle; @@ -1067,13 +4731,9 @@ create or replace function public.swap_backward_front() returns void as $$ begin - alter table backward_front - rename to backward_front_old; - alter table next_front - rename to backward_front; - alter table backward_front_old - rename to next_front; + truncate table backward_front; + insert into backward_front select * from next_front; truncate table next_front; delete from backward_front r where r.is_cycle; @@ -1718,24 +5378,24 @@ begin perform create_bidirectional_pair_pathspace_indexes(); end if; - create temporary table unresolved_pairs + create temporary table if not exists unresolved_pairs ( root_id int8 not null, terminal_id int8 not null, primary key (root_id, terminal_id) - ) on commit drop; + ) on commit preserve rows; - create index unresolved_pairs_terminal_id_root_id_index on unresolved_pairs using btree (terminal_id, root_id); + create index if not exists unresolved_pairs_terminal_id_root_id_index on unresolved_pairs using btree (terminal_id, root_id); - create temporary table resolved_pair_depths + create temporary table if not exists resolved_pair_depths ( root_id int8 not null, terminal_id int8 not null, depth int4 not null, primary key (root_id, terminal_id) - ) on commit drop; + ) on commit preserve rows; - create temporary table resolved_paths + create temporary table if not exists resolved_paths ( root_id int8 not null, next_id int8 not null, @@ -1743,7 +5403,9 @@ begin satisfied bool, is_cycle bool not null, path int8[] not null - ) on commit drop; + ) on commit preserve rows; + + truncate table unresolved_pairs, resolved_pair_depths, resolved_paths; if use_pair_filter then insert into unresolved_pairs (root_id, terminal_id) @@ -2034,6 +5696,7 @@ $$ drop function if exists public._bidirectional_sp_harness(text, text, text, text, int4, text, text, int8[], int8[], bool); drop function if exists public._bidirectional_sp_harness(text, text, text, text, int4, text, text, text, int8[], int8[], bool); drop function if exists public._bidirectional_sp_harness(text, text, text, text, int4, text, text, text, int8[], int8[], int8, bool); +drop function if exists public._bidirectional_sp_harness(text, text, text, text, int4, text, text, text, int8[], int8[], int8, bool, bool); -- _bidirectional_sp_harness implements the shortest-path bidirectional BFS in two control paths selected by -- `use_array_parameters`: @@ -2051,6 +5714,7 @@ create or replace function public._bidirectional_sp_harness(forward_primer text, root_filter text, terminal_filter text, pair_filter text, root_ids int8[], terminal_ids int8[], path_limit int8, + allow_zero_depth bool, use_array_parameters bool) returns table ( @@ -2074,42 +5738,56 @@ declare use_pair_filter bool := not use_array_parameters and length(pair_filter) > 0; matched_count int8 := 0; resolved_pairs_count int8 := 0; + unresolved_pairs_remaining bool := true; begin raise debug 'bidirectional_sp_harness start'; - perform create_bidirectional_shortest_path_tables(); + -- Validate the lean array mode before allocating its session workspace. + -- NULL endpoints represent an empty endpoint relation. Equal singleton IDs + -- retain the existing shortest-path error contract. if use_array_parameters then - perform create_traversal_filter_tables(root_ids, terminal_ids); - else - perform create_traversal_filter_tables(root_filter, terminal_filter, pair_filter); + if cardinality(root_ids) = 0 or cardinality(terminal_ids) = 0 or + root_ids[1] is null or terminal_ids[1] is null then + return; + end if; + if cardinality(root_ids) = 1 and cardinality(terminal_ids) = 1 and root_ids[1] = terminal_ids[1] then + if allow_zero_depth then + return query select root_ids[1], terminal_ids[1], 0::int4, true, false, array []::int8[]; + return; + else + perform public.shortest_path_self_endpoint_error(root_ids[1], terminal_ids[1]); + end if; + end if; end if; - create temporary table unresolved_pairs - ( - root_id int8 not null, - terminal_id int8 not null, - primary key (root_id, terminal_id) - ) on commit drop; - create index unresolved_pairs_terminal_id_root_id_index on unresolved_pairs using btree (terminal_id, root_id); + -- Array-parameter calls (including the proven singleton lowering) need only + -- the frontier/visited core. Text-filter calls lazily add pair/filter state. + perform public.reset_bsp_workspace(not use_array_parameters); - create temporary table resolved_pairs - ( - root_id int8 not null, - next_id int8 not null, - depth int4 not null, - satisfied bool, - is_cycle bool not null, - path int8[] not null, - primary key (root_id, next_id) - ) on commit drop; + if not use_array_parameters then + perform public.load_bsp_filter_tables(root_filter, terminal_filter, pair_filter); + end if; if use_pair_filter then - insert into unresolved_pairs (root_id, terminal_id) + insert into pg_temp.bsp_unresolved_pairs (root_id, terminal_id) select distinct root_id, terminal_id - from traversal_pair_filter - on conflict on constraint unresolved_pairs_pkey do nothing; + from pg_temp.bsp_pair_filter + on conflict on constraint bsp_unresolved_pairs_pkey do nothing; + + if allow_zero_depth then + insert into pg_temp.bsp_resolved_pairs (root_id, next_id, depth, satisfied, is_cycle, path) + select root_id, terminal_id, 0::int4, true, false, array []::int8[] + from pg_temp.bsp_unresolved_pairs + where root_id = terminal_id + on conflict on constraint bsp_resolved_pairs_pkey do nothing; + get diagnostics resolved_pairs_count = row_count; + + delete from pg_temp.bsp_unresolved_pairs where root_id = terminal_id; + end if; + + select exists(select 1 from pg_temp.bsp_unresolved_pairs) into unresolved_pairs_remaining; end if; -- Pair-filter mode keeps expanding until each requested pair is resolved or @@ -2117,29 +5795,29 @@ begin -- current BFS depth produces results. while forward_front_depth + backward_front_depth < max_depth and (path_limit <= 0 or resolved_pairs_count < path_limit) and - (not use_pair_filter or exists(select 1 from unresolved_pairs)) and + unresolved_pairs_remaining and (forward_front_depth = 0 or forward_front_count > 0) and (backward_front_depth = 0 or backward_front_count > 0) loop if forward_front_depth = 0 or (backward_front_depth > 0 and forward_front_count <= backward_front_count) then if forward_front_depth = 0 then if use_array_parameters then - execute forward_primer using root_ids, terminal_ids; + execute public.bsp_workspace_fragment(forward_primer) using root_ids, terminal_ids; else - execute forward_primer; + execute public.bsp_workspace_fragment(forward_primer); end if; get diagnostics next_front_count = row_count; - insert into forward_visited (root_id, id) + insert into pg_temp.bsp_forward_visited (root_id, id) select distinct f.root_id, f.root_id - from next_front f - on conflict on constraint forward_visited_pkey do nothing; + from pg_temp.bsp_next_front f + on conflict on constraint bsp_forward_visited_pkey do nothing; else if use_array_parameters then - execute forward_recursive using root_ids, terminal_ids; + execute public.bsp_workspace_fragment(forward_recursive) using root_ids, terminal_ids; else - execute forward_recursive; + execute public.bsp_workspace_fragment(forward_recursive); end if; get diagnostics next_front_count = row_count; @@ -2147,65 +5825,66 @@ begin forward_front_depth = forward_front_depth + 1; - delete from next_front f where f.is_cycle; + delete from pg_temp.bsp_next_front f where f.is_cycle; get diagnostics deleted_count = row_count; next_front_count = next_front_count - deleted_count; - delete from next_front f where f.satisfied is null; + delete from pg_temp.bsp_next_front f where f.satisfied is null; get diagnostics deleted_count = row_count; next_front_count = next_front_count - deleted_count; - delete from next_front f using forward_visited v where f.root_id = v.root_id and f.next_id = v.id; + delete from pg_temp.bsp_next_front f using pg_temp.bsp_forward_visited v where f.root_id = v.root_id and f.next_id = v.id; get diagnostics deleted_count = row_count; next_front_count = next_front_count - deleted_count; raise debug 'Forward shortest expansion as step % - Available Root Paths %', forward_front_depth + backward_front_depth, next_front_count; - truncate table forward_front; + truncate table pg_temp.bsp_forward_front; - insert into forward_front + insert into pg_temp.bsp_forward_front select distinct on (f.root_id, f.next_id) f.root_id, f.next_id, f.depth, f.satisfied, f.is_cycle, f.path - from next_front f + from pg_temp.bsp_next_front f order by f.root_id, f.next_id, f.depth; get diagnostics forward_front_count = row_count; - truncate table next_front; + truncate table pg_temp.bsp_next_front; - insert into forward_visited (root_id, id) + insert into pg_temp.bsp_forward_visited (root_id, id) select f.root_id, f.next_id - from forward_front f - on conflict on constraint forward_visited_pkey do nothing; + from pg_temp.bsp_forward_front f + on conflict on constraint bsp_forward_visited_pkey do nothing; - if exists(select 1 from forward_front r where r.satisfied) then + if exists(select 1 from pg_temp.bsp_forward_front r where r.satisfied) then if use_pair_filter then -- A direct forward hit resolves only the requested pairs it satisfies. -- Frontiers for completed roots/terminals are pruned below. - insert into resolved_pairs (root_id, next_id, depth, satisfied, is_cycle, path) + insert into pg_temp.bsp_resolved_pairs (root_id, next_id, depth, satisfied, is_cycle, path) select distinct on (r.root_id, r.next_id) r.root_id, r.next_id, r.depth, r.satisfied, r.is_cycle, r.path - from forward_front r - join unresolved_pairs p on p.root_id = r.root_id and p.terminal_id = r.next_id + from pg_temp.bsp_forward_front r + join pg_temp.bsp_unresolved_pairs p on p.root_id = r.root_id and p.terminal_id = r.next_id where r.satisfied order by r.root_id, r.next_id, r.depth - on conflict on constraint resolved_pairs_pkey do nothing; + on conflict on constraint bsp_resolved_pairs_pkey do nothing; get diagnostics matched_count = row_count; resolved_pairs_count = resolved_pairs_count + matched_count; delete - from unresolved_pairs p - using resolved_pairs r + from pg_temp.bsp_unresolved_pairs p + using pg_temp.bsp_resolved_pairs r where p.root_id = r.root_id and p.terminal_id = r.next_id; + select exists(select 1 from pg_temp.bsp_unresolved_pairs) into unresolved_pairs_remaining; - delete from forward_front f where not exists(select 1 from unresolved_pairs p where p.root_id = f.root_id); + delete from pg_temp.bsp_forward_front f where not exists(select 1 from pg_temp.bsp_unresolved_pairs p where p.root_id = f.root_id); get diagnostics deleted_count = row_count; forward_front_count = forward_front_count - deleted_count; - delete from backward_front b where not exists(select 1 from unresolved_pairs p where p.terminal_id = b.root_id); + delete from pg_temp.bsp_backward_front b where not exists(select 1 from pg_temp.bsp_unresolved_pairs p where p.terminal_id = b.root_id); get diagnostics deleted_count = row_count; backward_front_count = backward_front_count - deleted_count; else @@ -2217,7 +5896,7 @@ begin r.satisfied, r.is_cycle, r.path - from forward_front r + from pg_temp.bsp_forward_front r where r.satisfied order by r.root_id, r.next_id, r.depth limit case when path_limit > 0 then path_limit else null end; @@ -2227,22 +5906,22 @@ begin else if backward_front_depth = 0 then if use_array_parameters then - execute backward_primer using root_ids, terminal_ids; + execute public.bsp_workspace_fragment(backward_primer) using root_ids, terminal_ids; else - execute backward_primer; + execute public.bsp_workspace_fragment(backward_primer); end if; get diagnostics next_front_count = row_count; - insert into backward_visited (root_id, id) + insert into pg_temp.bsp_backward_visited (root_id, id) select distinct f.root_id, f.root_id - from next_front f - on conflict on constraint backward_visited_pkey do nothing; + from pg_temp.bsp_next_front f + on conflict on constraint bsp_backward_visited_pkey do nothing; else if use_array_parameters then - execute backward_recursive using root_ids, terminal_ids; + execute public.bsp_workspace_fragment(backward_recursive) using root_ids, terminal_ids; else - execute backward_recursive; + execute public.bsp_workspace_fragment(backward_recursive); end if; get diagnostics next_front_count = row_count; @@ -2250,65 +5929,66 @@ begin backward_front_depth = backward_front_depth + 1; - delete from next_front f where f.is_cycle; + delete from pg_temp.bsp_next_front f where f.is_cycle; get diagnostics deleted_count = row_count; next_front_count = next_front_count - deleted_count; - delete from next_front f where f.satisfied is null; + delete from pg_temp.bsp_next_front f where f.satisfied is null; get diagnostics deleted_count = row_count; next_front_count = next_front_count - deleted_count; - delete from next_front f using backward_visited v where f.root_id = v.root_id and f.next_id = v.id; + delete from pg_temp.bsp_next_front f using pg_temp.bsp_backward_visited v where f.root_id = v.root_id and f.next_id = v.id; get diagnostics deleted_count = row_count; next_front_count = next_front_count - deleted_count; raise debug 'Backward shortest expansion as step % - Available Terminal Paths %', forward_front_depth + backward_front_depth, next_front_count; - truncate table backward_front; + truncate table pg_temp.bsp_backward_front; - insert into backward_front + insert into pg_temp.bsp_backward_front select distinct on (f.root_id, f.next_id) f.root_id, f.next_id, f.depth, f.satisfied, f.is_cycle, f.path - from next_front f + from pg_temp.bsp_next_front f order by f.root_id, f.next_id, f.depth; get diagnostics backward_front_count = row_count; - truncate table next_front; + truncate table pg_temp.bsp_next_front; - insert into backward_visited (root_id, id) + insert into pg_temp.bsp_backward_visited (root_id, id) select f.root_id, f.next_id - from backward_front f - on conflict on constraint backward_visited_pkey do nothing; + from pg_temp.bsp_backward_front f + on conflict on constraint bsp_backward_visited_pkey do nothing; - if exists(select 1 from backward_front r where r.satisfied) then + if exists(select 1 from pg_temp.bsp_backward_front r where r.satisfied) then if use_pair_filter then -- Symmetric direct hit from the terminal side; swap root/terminal -- columns back into the function's result shape. - insert into resolved_pairs (root_id, next_id, depth, satisfied, is_cycle, path) + insert into pg_temp.bsp_resolved_pairs (root_id, next_id, depth, satisfied, is_cycle, path) select distinct on (r.next_id, r.root_id) r.next_id, r.root_id, r.depth, r.satisfied, r.is_cycle, r.path - from backward_front r - join unresolved_pairs p on p.root_id = r.next_id and p.terminal_id = r.root_id + from pg_temp.bsp_backward_front r + join pg_temp.bsp_unresolved_pairs p on p.root_id = r.next_id and p.terminal_id = r.root_id where r.satisfied order by r.next_id, r.root_id, r.depth - on conflict on constraint resolved_pairs_pkey do nothing; + on conflict on constraint bsp_resolved_pairs_pkey do nothing; get diagnostics matched_count = row_count; resolved_pairs_count = resolved_pairs_count + matched_count; delete - from unresolved_pairs p - using resolved_pairs r + from pg_temp.bsp_unresolved_pairs p + using pg_temp.bsp_resolved_pairs r where p.root_id = r.root_id and p.terminal_id = r.next_id; + select exists(select 1 from pg_temp.bsp_unresolved_pairs) into unresolved_pairs_remaining; - delete from backward_front f where not exists(select 1 from unresolved_pairs p where p.terminal_id = f.root_id); + delete from pg_temp.bsp_backward_front f where not exists(select 1 from pg_temp.bsp_unresolved_pairs p where p.terminal_id = f.root_id); get diagnostics deleted_count = row_count; backward_front_count = backward_front_count - deleted_count; - delete from forward_front f where not exists(select 1 from unresolved_pairs p where p.root_id = f.root_id); + delete from pg_temp.bsp_forward_front f where not exists(select 1 from pg_temp.bsp_unresolved_pairs p where p.root_id = f.root_id); get diagnostics deleted_count = row_count; forward_front_count = forward_front_count - deleted_count; else @@ -2318,7 +5998,7 @@ begin r.satisfied, r.is_cycle, r.path - from backward_front r + from pg_temp.bsp_backward_front r where r.satisfied order by r.next_id, r.root_id, r.depth limit case when path_limit > 0 then path_limit else null end; @@ -2330,39 +6010,40 @@ begin if use_pair_filter then -- For unresolved pairs that meet in the middle, keep one shortest -- stitched path per pair and leave already-resolved pairs untouched. - insert into resolved_pairs (root_id, next_id, depth, satisfied, is_cycle, path) + insert into pg_temp.bsp_resolved_pairs (root_id, next_id, depth, satisfied, is_cycle, path) select p.root_id, p.terminal_id, midpoint.depth, true, false, midpoint.path - from unresolved_pairs p + from pg_temp.bsp_unresolved_pairs p join lateral ( select f.depth + b.depth as depth, f.path || b.path as path - from forward_front f - join backward_front b on b.root_id = p.terminal_id and b.next_id = f.next_id + from pg_temp.bsp_forward_front f + join pg_temp.bsp_backward_front b on b.root_id = p.terminal_id and b.next_id = f.next_id where f.root_id = p.root_id order by f.depth + b.depth limit 1 ) midpoint on true - on conflict on constraint resolved_pairs_pkey do nothing; + on conflict on constraint bsp_resolved_pairs_pkey do nothing; get diagnostics matched_count = row_count; resolved_pairs_count = resolved_pairs_count + matched_count; if matched_count > 0 then delete - from unresolved_pairs p - using resolved_pairs r + from pg_temp.bsp_unresolved_pairs p + using pg_temp.bsp_resolved_pairs r where p.root_id = r.root_id and p.terminal_id = r.next_id; + select exists(select 1 from pg_temp.bsp_unresolved_pairs) into unresolved_pairs_remaining; - delete from forward_front f where not exists(select 1 from unresolved_pairs p where p.root_id = f.root_id); + delete from pg_temp.bsp_forward_front f where not exists(select 1 from pg_temp.bsp_unresolved_pairs p where p.root_id = f.root_id); get diagnostics deleted_count = row_count; forward_front_count = forward_front_count - deleted_count; - delete from backward_front b where not exists(select 1 from unresolved_pairs p where p.terminal_id = b.root_id); + delete from pg_temp.bsp_backward_front b where not exists(select 1 from pg_temp.bsp_unresolved_pairs p where p.terminal_id = b.root_id); get diagnostics deleted_count = row_count; backward_front_count = backward_front_count - deleted_count; end if; @@ -2373,8 +6054,8 @@ begin true, false, f.path || b.path - from forward_front f - join backward_front b on f.next_id = b.next_id + from pg_temp.bsp_forward_front f + join pg_temp.bsp_backward_front b on f.next_id = b.next_id order by f.root_id, b.root_id, f.depth + b.depth limit case when path_limit > 0 then path_limit else null end; get diagnostics matched_count = row_count; @@ -2390,12 +6071,12 @@ begin -- for unresolved pairs after the first frontier-level success. if path_limit > 0 then return query select * - from resolved_pairs + from pg_temp.bsp_resolved_pairs order by root_id, next_id, depth limit path_limit; else return query select * - from resolved_pairs + from pg_temp.bsp_resolved_pairs order by root_id, next_id, depth; end if; end if; @@ -2422,7 +6103,51 @@ create or replace function public.bidirectional_sp_harness(forward_primer text, as $$ select * -from public._bidirectional_sp_harness(forward_primer, forward_recursive, backward_primer, backward_recursive, max_depth, ''::text, ''::text, ''::text, root_ids, terminal_ids, path_limit, true); +from public._bidirectional_sp_harness(forward_primer, forward_recursive, backward_primer, backward_recursive, max_depth, ''::text, ''::text, ''::text, root_ids, terminal_ids, path_limit, false, true); +$$ + language sql volatile + strict; + +create or replace function public.bidirectional_sp_harness(forward_primer text, forward_recursive text, + backward_primer text, + backward_recursive text, max_depth int4, + root_ids int8[], terminal_ids int8[], + allow_zero_depth bool, path_limit int8) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +select * +from public._bidirectional_sp_harness(forward_primer, forward_recursive, backward_primer, backward_recursive, max_depth, ''::text, ''::text, ''::text, root_ids, terminal_ids, path_limit, allow_zero_depth, true); +$$ + language sql volatile + strict; + +create or replace function public.bidirectional_sp_harness(forward_primer text, forward_recursive text, + backward_primer text, + backward_recursive text, max_depth int4, + root_ids int8[], terminal_ids int8[], + allow_zero_depth bool) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +select * +from public.bidirectional_sp_harness(forward_primer, forward_recursive, backward_primer, backward_recursive, max_depth, root_ids, terminal_ids, allow_zero_depth, 0::int8); $$ language sql volatile strict; @@ -2464,7 +6189,51 @@ create or replace function public.bidirectional_sp_harness(forward_primer text, as $$ select * -from public._bidirectional_sp_harness(forward_primer, forward_recursive, backward_primer, backward_recursive, max_depth, root_filter, terminal_filter, ''::text, array []::int8[], array []::int8[], path_limit, false); +from public._bidirectional_sp_harness(forward_primer, forward_recursive, backward_primer, backward_recursive, max_depth, root_filter, terminal_filter, ''::text, array []::int8[], array []::int8[], path_limit, false, false); +$$ + language sql volatile + strict; + +create or replace function public.bidirectional_sp_harness(forward_primer text, forward_recursive text, + backward_primer text, + backward_recursive text, max_depth int4, + root_filter text, terminal_filter text, + allow_zero_depth bool, path_limit int8) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +select * +from public._bidirectional_sp_harness(forward_primer, forward_recursive, backward_primer, backward_recursive, max_depth, root_filter, terminal_filter, ''::text, array []::int8[], array []::int8[], path_limit, allow_zero_depth, false); +$$ + language sql volatile + strict; + +create or replace function public.bidirectional_sp_harness(forward_primer text, forward_recursive text, + backward_primer text, + backward_recursive text, max_depth int4, + root_filter text, terminal_filter text, + allow_zero_depth bool) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +select * +from public.bidirectional_sp_harness(forward_primer, forward_recursive, backward_primer, backward_recursive, max_depth, root_filter, terminal_filter, allow_zero_depth, 0::int8); $$ language sql volatile strict; @@ -2507,7 +6276,51 @@ create or replace function public.bidirectional_sp_harness(forward_primer text, as $$ select * -from public._bidirectional_sp_harness(forward_primer, forward_recursive, backward_primer, backward_recursive, max_depth, root_filter, terminal_filter, pair_filter, array []::int8[], array []::int8[], path_limit, false); +from public._bidirectional_sp_harness(forward_primer, forward_recursive, backward_primer, backward_recursive, max_depth, root_filter, terminal_filter, pair_filter, array []::int8[], array []::int8[], path_limit, false, false); +$$ + language sql volatile + strict; + +create or replace function public.bidirectional_sp_harness(forward_primer text, forward_recursive text, + backward_primer text, + backward_recursive text, max_depth int4, + root_filter text, terminal_filter text, pair_filter text, + allow_zero_depth bool, path_limit int8) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +select * +from public._bidirectional_sp_harness(forward_primer, forward_recursive, backward_primer, backward_recursive, max_depth, root_filter, terminal_filter, pair_filter, array []::int8[], array []::int8[], path_limit, allow_zero_depth, false); +$$ + language sql volatile + strict; + +create or replace function public.bidirectional_sp_harness(forward_primer text, forward_recursive text, + backward_primer text, + backward_recursive text, max_depth int4, + root_filter text, terminal_filter text, pair_filter text, + allow_zero_depth bool) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +select * +from public.bidirectional_sp_harness(forward_primer, forward_recursive, backward_primer, backward_recursive, max_depth, root_filter, terminal_filter, pair_filter, allow_zero_depth, 0::int8); $$ language sql volatile strict; @@ -2615,3 +6428,92 @@ from public.bidirectional_sp_harness(forward_primer, forward_recursive, backward $$ language sql volatile strict; + +-- graphbench_s1_distance_bfs is the typed, array-resident SP-S1 distance +-- prototype. It is additive and benchmark-only: production translation does +-- not call it. The caller must transparently restart a correct fallback when +-- overflow is true. +create or replace function public.graphbench_s1_distance_bfs(target_graph_id int4, start_id int8, terminal_id int8, + min_depth int4, max_depth int4, edge_kind_ids int2[], + inbound bool, state_limit int4) + returns table + ( + depth int4, + matched bool, + overflow bool, + examined_edges int8, + retained_nodes int4 + ) +as +$$ +#variable_conflict use_variable +declare + current_depth int4 := 0; + frontier int8[] := array[start_id]::int8[]; + next_frontier int8[]; + visited int8[] := array[start_id]::int8[]; + edge_count int8; +begin + depth := null; + matched := false; + overflow := false; + examined_edges := 0; + retained_nodes := 1; + + if state_limit < 1 then + overflow := true; + return next; + return; + end if; + + if start_id = terminal_id and min_depth = 0 then + depth := 0; + matched := true; + return next; + return; + end if; + + while current_depth < max_depth and cardinality(frontier) > 0 loop + select + coalesce(array_agg(distinct candidate.next_id order by candidate.next_id) + filter (where not candidate.next_id = any(visited)), array[]::int8[]), + count(*) + into next_frontier, edge_count + from ( + select case when inbound then edge.start_id else edge.end_id end as next_id + from unnest(frontier) as active(node_id) + join edge on edge.graph_id = target_graph_id + and ((not inbound and edge.start_id = active.node_id) + or (inbound and edge.end_id = active.node_id)) + where cardinality(edge_kind_ids) = 0 or edge.kind_id = any(edge_kind_ids) + ) candidate; + + examined_edges := examined_edges + edge_count; + current_depth := current_depth + 1; + + if terminal_id = any(next_frontier) and current_depth >= min_depth then + depth := current_depth; + matched := true; + retained_nodes := cardinality(visited) + cardinality(next_frontier); + return next; + return; + end if; + + if cardinality(visited) + cardinality(next_frontier) > state_limit then + overflow := true; + retained_nodes := cardinality(visited); + return next; + return; + end if; + + visited := visited || next_frontier; + frontier := next_frontier; + retained_nodes := cardinality(visited); + end loop; + + return next; +end; +$$ + language plpgsql + volatile + strict; diff --git a/drivers/pg/query/sql_workspace_test.go b/drivers/pg/query/sql_workspace_test.go new file mode 100644 index 00000000..3dae6c70 --- /dev/null +++ b/drivers/pg/query/sql_workspace_test.go @@ -0,0 +1,463 @@ +package query + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestBidirectionalShortestPathWorkspaceIsReusable verifies shortest-path SQL creates reusable session-scoped workspace tables. +func TestBidirectionalShortestPathWorkspaceIsReusable(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public._bidirectional_sp_harness") + require.NotEqual(t, -1, start) + end := strings.Index(sqlSchemaUp[start:], "create or replace function public.bidirectional_sp_harness") + require.NotEqual(t, -1, end) + harness := sqlSchemaUp[start : start+end] + + require.Contains(t, sqlSchemaUp, "create or replace function public.ensure_bsp_core_workspace()") + require.Contains(t, sqlSchemaUp, "if present_version is not null and present_version is distinct from expected_version then") + require.Contains(t, sqlSchemaUp, "on commit preserve rows") + require.Contains(t, harness, "perform public.reset_bsp_workspace(not use_array_parameters)") + require.Contains(t, harness, "pg_temp.bsp_forward_front") + require.Contains(t, harness, "pg_temp.bsp_backward_front") + require.Contains(t, harness, "pg_temp.bsp_next_front") + require.NotContains(t, harness, "create temporary table") + require.NotContains(t, harness, "create index") + require.Contains(t, harness, "truncate table pg_temp.bsp_forward_front") + require.Contains(t, harness, "truncate table pg_temp.bsp_backward_front") + require.Contains(t, harness, "truncate table pg_temp.bsp_next_front") +} + +// TestBidirectionalShortestPathWarmWorkspaceUsesTruncate verifies repeated shortest-path execution clears existing workspace instead of recreating it. +func TestBidirectionalShortestPathWarmWorkspaceUsesTruncate(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.reset_bsp_workspace") + require.NotEqual(t, -1, start) + end := strings.Index(sqlSchemaUp[start:], "create or replace function public.load_bsp_filter_tables") + require.NotEqual(t, -1, end) + reset := sqlSchemaUp[start : start+end] + + require.Contains(t, reset, "truncate table pg_temp.bsp_forward_front") + require.Contains(t, reset, "pg_temp.bsp_resolved_pairs") + require.NotContains(t, reset, "delete from pg_temp.bsp_") + require.NotContains(t, sqlSchemaUp, "current_setting('transaction_read_only')") +} + +// TestBidirectionalShortestPathArrayModeSkipsGenericWorkspace verifies array-backed execution does not initialize table-backed workspace. +func TestBidirectionalShortestPathArrayModeSkipsGenericWorkspace(t *testing.T) { + require.Contains(t, sqlSchemaUp, "if not use_array_parameters then\nperform public.load_bsp_filter_tables") + require.Contains(t, sqlSchemaUp, "perform public.reset_bsp_workspace(not use_array_parameters)") +} + +// TestBidirectionalShortestPathFragmentsRewriteLegacyFilterTables verifies generated fragments target the current workspace filter tables. +func TestBidirectionalShortestPathFragmentsRewriteLegacyFilterTables(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.bsp_workspace_fragment") + require.NotEqual(t, -1, start) + end := strings.Index(sqlSchemaUp[start:], "create or replace function public.reset_bsp_workspace") + require.NotEqual(t, -1, end) + rewriter := sqlSchemaUp[start : start+end] + + require.Contains(t, rewriter, "'traversal_root_filter', 'pg_temp.bsp_root_filter'") + require.Contains(t, rewriter, "'traversal_terminal_filter', 'pg_temp.bsp_terminal_filter'") + require.Contains(t, rewriter, "'traversal_pair_filter', 'pg_temp.bsp_pair_filter'") +} + +// TestLinearPathMaterializerScopesPersistentLookups verifies persistent node and edge lookups include the selected graph ID. +func TestLinearPathMaterializerScopesPersistentLookups(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.ordered_edge_ids_to_path") + require.NotEqual(t, -1, start) + end := strings.Index(sqlSchemaUp[start:], "create or replace function public.create_unidirectional_pathspace_tables") + require.NotEqual(t, -1, end) + materializer := sqlSchemaUp[start : start+end] + + require.Contains(t, materializer, "e.graph_id = target_graph_id") + require.Contains(t, materializer, "n.graph_id = target_graph_id") + require.Contains(t, materializer, "next_edge.ordinality = path_walk.idx + 1") + require.NotContains(t, materializer, "order by case when") +} + +// TestLegacyPathMaterializersRequireTargetGraph verifies legacy materializer signatures cannot bypass graph scoping. +func TestLegacyPathMaterializersRequireTargetGraph(t *testing.T) { + require.Contains(t, sqlSchemaUp, "drop function if exists public.nodes_to_path(int8[])") + require.Contains(t, sqlSchemaUp, "drop function if exists public.edges_to_path(int8[])") + require.Contains(t, sqlSchemaUp, "drop function if exists public.ordered_edges_to_path(nodeComposite, edgeComposite[], nodeComposite[])") + require.Contains(t, sqlSchemaUp, "nodes_to_path(target_graph_id int4") + require.Contains(t, sqlSchemaUp, "edges_to_path(target_graph_id int4") + require.Contains(t, sqlSchemaUp, "ordered_edges_to_path(target_graph_id int4") + require.Contains(t, sqlSchemaUp, "n.graph_id = target_graph_id") + require.Contains(t, sqlSchemaUp, "r.graph_id = target_graph_id") + require.Contains(t, sqlSchemaDown, "drop function if exists nodes_to_path(int4, int8[])") + require.Contains(t, sqlSchemaDown, "drop function if exists nodes_to_path(int8[])") + require.Contains(t, sqlSchemaDown, "drop function if exists edges_to_path(int4, int8[])") + require.Contains(t, sqlSchemaDown, "drop function if exists edges_to_path(int8[])") + require.NotContains(t, sqlSchemaDown, "drop function if exists nodes_to_path;") + require.NotContains(t, sqlSchemaDown, "drop function if exists edges_to_path;") +} + +// TestGraphBenchS1DistancePrototypeIsBoundedAndGraphScoped verifies the benchmark prototype constrains depth and graph identity. +func TestGraphBenchS1DistancePrototypeIsBoundedAndGraphScoped(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.graphbench_s1_distance_bfs") + require.NotEqual(t, -1, start) + prototype := sqlSchemaUp[start:] + + require.Contains(t, prototype, "edge.graph_id = target_graph_id") + require.Contains(t, prototype, "cardinality(visited) + cardinality(next_frontier) > state_limit") + require.Contains(t, prototype, "overflow := true") + require.NotContains(t, prototype, "create temporary table") + require.NotContains(t, prototype, "insert into") + require.Contains(t, sqlSchemaDown, "drop function if exists graphbench_s1_distance_bfs") +} + +// TestCompactShortestExecutorsUseReusableTypedWorkspace verifies compact executors use typed, reusable workspace structures. +func TestCompactShortestExecutorsUseReusableTypedWorkspace(t *testing.T) { + require.Contains(t, sqlSchemaUp, "create or replace function public.ensure_shortest_dag_workspace()") + require.Contains(t, sqlSchemaUp, "create or replace function public.reset_shortest_dag_workspace()") + require.Contains(t, sqlSchemaUp, "on commit preserve rows") + require.Contains(t, sqlSchemaUp, "create or replace function public.all_shortest_paths_dag(") + require.Contains(t, sqlSchemaUp, "create or replace function public.shortest_path_compact(") + require.Contains(t, sqlSchemaUp, "rows 100") + require.Contains(t, sqlSchemaUp, "rows 1") + require.Contains(t, sqlSchemaDown, "drop function if exists all_shortest_paths_dag") + require.Contains(t, sqlSchemaDown, "drop function if exists shortest_path_compact") +} + +// TestAllShortestDAGHasExactSmallDepthArmsAndLateEnumeration verifies shallow-depth specializations precede deferred path enumeration. +func TestAllShortestDAGHasExactSmallDepthArmsAndLateEnumeration(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.all_shortest_paths_dag") + require.NotEqual(t, -1, start) + end := strings.Index(sqlSchemaUp[start:], "create or replace function public.shortest_path_compact") + require.NotEqual(t, -1, end) + executor := sqlSchemaUp[start : start+end] + + require.Contains(t, executor, "array[e.id]::int8[]") + require.Contains(t, executor, "array[e1.id, e2.id]::int8[]") + require.Contains(t, executor, "e1.id <> e2.id") + require.Contains(t, executor, "perform public.reset_shortest_dag_workspace()") + require.Contains(t, executor, "insert into pg_temp.spd_predecessor") + require.Contains(t, executor, "with recursive shortest_paths") + require.Contains(t, executor, "if exists (select 1 from pg_temp.spd_candidate where depth = search_depth and node_id = target_id) then") + require.NotContains(t, executor, "execute ") +} + +// TestCompactSingletonOverflowFallsBackBeforeReturning verifies compact overflow takes the safe fallback before emitting a result. +func TestCompactSingletonOverflowFallsBackBeforeReturning(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.shortest_path_compact") + require.NotEqual(t, -1, start) + end := strings.Index(sqlSchemaUp[start:], "create or replace function public.ensure_bidirectional_shortest_path_workspace") + require.NotEqual(t, -1, end) + executor := sqlSchemaUp[start : start+end] + + require.Contains(t, executor, "retained_state > state_limit") + require.Contains(t, executor, "if overflowed then") + require.Contains(t, executor, "with recursive trails") + require.Contains(t, executor, "not e.id = any(trails.edge_ids)") + require.NotContains(t, executor, "execute ") +} + +// TestCompactBidirectionalWorkspaceIsVersionedAndDisjoint verifies candidate +// state can coexist with the S4 fallback workspace on a pooled session. +func TestCompactBidirectionalWorkspaceIsVersionedAndDisjoint(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.ensure_bidirectional_shortest_path_workspace") + require.NotEqual(t, -1, start) + end := strings.Index(sqlSchemaUp[start:], "create or replace function public.shortest_path_bidirectional_compact_v1") + require.NotEqual(t, -1, end) + workspace := sqlSchemaUp[start : start+end] + + require.Contains(t, workspace, "expected_version constant int4 := 1") + require.Contains(t, workspace, "pg_temp.spb_workspace_version") + require.Contains(t, workspace, "create temporary table spb_front") + require.Contains(t, workspace, "create temporary table spb_seen") + require.Contains(t, workspace, "create temporary table spb_active") + require.Contains(t, workspace, "create temporary table spb_candidate") + require.Contains(t, workspace, "create temporary table spb_predecessor") + require.Contains(t, workspace, "queue_order int8 not null") + require.Contains(t, workspace, "truncate table pg_temp.spb_front") + require.NotContains(t, workspace, "spd_front") + require.NotContains(t, workspace, "path int8[]") +} + +// TestTraversalRuntimeAttestationIsSessionLocalAndSymmetric verifies the +// timed-invocation receipt cannot persist data or survive schema teardown. +func TestTraversalRuntimeAttestationIsSessionLocalAndSymmetric(t *testing.T) { + require.Contains(t, sqlSchemaUp, "create temporary table traversal_runtime_attestation_v1") + require.Contains(t, sqlSchemaUp, "on commit preserve rows") + require.Contains(t, sqlSchemaUp, "current_setting('dawgs.traversal_runtime_invocation_id', true)") + require.Contains(t, sqlSchemaUp, "record_count = receipt.record_count + 1") + require.Contains(t, sqlSchemaUp, "events = receipt.events || jsonb_build_array") + require.Contains(t, sqlSchemaUp, "'schema_version', 2") + require.Contains(t, sqlSchemaUp, "if not exists (\nselect 1\nfrom pg_attribute") + require.Contains(t, sqlSchemaUp, "create or replace function public.read_traversal_runtime_attestation_v1") + require.Contains(t, sqlSchemaUp, "create or replace function public.clear_traversal_runtime_attestation_v1") + for _, function := range []string{ + "clear_traversal_runtime_attestation_v1(text)", + "read_traversal_runtime_attestation_v1(text)", + "record_requested_traversal_runtime_attestation_v1(text, bool, text)", + "record_traversal_runtime_attestation_v1(text, text, bool)", + "begin_traversal_runtime_attestation_v1(text, text)", + "ensure_traversal_runtime_attestation_workspace_v1()", + } { + require.Contains(t, sqlSchemaDown, "drop function if exists "+function) + } +} + +// TestCompactBidirectionalKernelHasExactPreflightBoundsAndFallback verifies all +// candidate gates run before output and overflow delegates to exact S4 state. +func TestCompactBidirectionalKernelHasExactPreflightBoundsAndFallback(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.shortest_path_bidirectional_compact_v1") + require.NotEqual(t, -1, start) + end := strings.Index(sqlSchemaUp[start:], "create or replace function public.shortest_path_b1_strict_alternating") + require.NotEqual(t, -1, end) + kernel := sqlSchemaUp[start : start+end] + + zeroHop := strings.Index(kernel, "if source_id = target_id then") + oneHop := strings.Index(kernel, "if min_depth <= 1 and max_depth >= 1 then") + twoHop := strings.Index(kernel, "if min_depth <= 2 and max_depth >= 2 then") + workspaceReset := strings.Index(kernel, "reset_bidirectional_shortest_path_workspace") + require.Greater(t, zeroHop, -1) + require.Greater(t, oneHop, zeroHop) + require.Greater(t, twoHop, oneHop) + require.Greater(t, workspaceReset, twoHop) + + require.Contains(t, kernel, "forward_depth + backward_depth >= best_distance") + require.Contains(t, kernel, "current_setting('transaction_isolation') <> 'repeatable read'") + require.Contains(t, kernel, "current_setting('transaction_isolation') <> 'serializable'") + require.Contains(t, kernel, "limit admission_limit + 1") + require.Contains(t, kernel, "seen_rows + candidate_rows > state_limit") + require.Contains(t, kernel, "active_rows + frontier_rows + candidate_rows > frontier_limit") + require.Contains(t, kernel, "predecessor_rows + candidate_rows > predecessor_limit") + require.Contains(t, kernel, "from public.shortest_path_compact(") + require.Contains(t, kernel, "with recursive\nforward_witness") + require.Less(t, strings.Index(kernel, "from public.shortest_path_compact("), strings.Index(kernel, "with recursive\nforward_witness")) + require.NotContains(t, kernel, "nodeComposite") + require.NotContains(t, kernel, "edgeComposite") +} + +// TestCompactBidirectionalWrappersFreezeSchedulersAndDownMigration verifies the +// two scheduler identities have typed wrappers and symmetric teardown. +func TestCompactBidirectionalWrappersFreezeSchedulersAndDownMigration(t *testing.T) { + require.Contains(t, sqlSchemaUp, "create or replace function public.shortest_path_b1_strict_alternating(") + require.Contains(t, sqlSchemaUp, "'strict_alternating_node'") + require.Contains(t, sqlSchemaUp, "create or replace function public.shortest_path_b2_smaller_current_level(") + require.Contains(t, sqlSchemaUp, "'smaller_current_level'") + require.Contains(t, sqlSchemaDown, "drop function if exists shortest_path_b1_strict_alternating(int4, int8, int8, int4, int4, int2[], bool, int8, int8, int8)") + require.Contains(t, sqlSchemaDown, "drop function if exists shortest_path_b2_smaller_current_level(int4, int8, int8, int4, int4, int2[], bool, int8, int8, int8)") + require.Contains(t, sqlSchemaDown, "drop function if exists shortest_path_bidirectional_compact_v1(int4, int8, int8, int4, int4, int2[], bool, int8, int8, int8, text)") + require.Contains(t, sqlSchemaDown, "drop function if exists reset_bidirectional_shortest_path_workspace()") + require.Contains(t, sqlSchemaDown, "drop function if exists ensure_bidirectional_shortest_path_workspace()") +} + +// TestCompactBidirectionalDiagnosticTelemetryIsInvocationScoped verifies the +// untimed replay API records explicit internal counters in a distinct, +// session-local workspace and has symmetric teardown. +func TestCompactBidirectionalDiagnosticTelemetryIsInvocationScoped(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.ensure_bidirectional_shortest_path_telemetry_workspace") + require.NotEqual(t, -1, start) + end := strings.Index(sqlSchemaUp[start:], "create or replace function public.shortest_path_bidirectional_compact_v1") + require.NotEqual(t, -1, end) + telemetry := sqlSchemaUp[start : start+end] + + require.Contains(t, telemetry, "expected_version constant int4 := 1") + require.Contains(t, telemetry, "create temporary table spb_telemetry_invocation") + require.Contains(t, telemetry, "create temporary table spb_telemetry_call") + require.Contains(t, telemetry, "create temporary table spb_telemetry_level") + require.Contains(t, telemetry, "on commit preserve rows") + require.Contains(t, telemetry, "set_config('dawgs.spb_diagnostic_invocation_id', invocation_id, true)") + require.Contains(t, telemetry, "where invocation.invocation_id = target_invocation_id") + require.Contains(t, telemetry, "'scheduler_actions'") + require.Contains(t, telemetry, "'candidate_edges'") + require.Contains(t, telemetry, "'seen_peak'") + require.Contains(t, telemetry, "'frontier_peak'") + require.Contains(t, telemetry, "'queue_peak'") + require.Contains(t, telemetry, "'predecessor_peak'") + require.Contains(t, telemetry, "'meeting_candidates'") + require.Contains(t, telemetry, "'fallback_executed'") + require.NotContains(t, telemetry, "create unlogged table") + require.NotContains(t, telemetry, "create table public.spb_telemetry") + + kernelStart := strings.Index(sqlSchemaUp, "create or replace function public.shortest_path_bidirectional_compact_v1") + require.NotEqual(t, -1, kernelStart) + wrapperStart := strings.Index(sqlSchemaUp[kernelStart:], "create or replace function public.shortest_path_b1_strict_alternating") + require.NotEqual(t, -1, wrapperStart) + kernel := sqlSchemaUp[kernelStart : kernelStart+wrapperStart] + require.Contains(t, kernel, "_start_bidirectional_shortest_path_diagnostic_call_v1") + require.Contains(t, kernel, "_record_bidirectional_shortest_path_diagnostic_level_v1") + require.Contains(t, kernel, "_finish_bidirectional_shortest_path_diagnostic_call_v1") + require.Contains(t, kernel, "if telemetry_search_id is not null then") + require.Contains(t, kernel, "select count(*) into telemetry_action_candidate_edges") + require.Contains(t, kernel, "'exact_s4_fallback'") + require.Contains(t, kernel, "'preflight_zero_hop'") + require.Contains(t, kernel, "'preflight_one_hop'") + require.Contains(t, kernel, "'preflight_two_hop'") + + for _, function := range []string{ + "_finish_bidirectional_shortest_path_diagnostic_call_v1", + "_record_bidirectional_shortest_path_diagnostic_level_v1", + "_start_bidirectional_shortest_path_diagnostic_call_v1", + "clear_bidirectional_shortest_path_diagnostic_v1", + "read_bidirectional_shortest_path_diagnostic_v1", + "begin_bidirectional_shortest_path_diagnostic_v1", + "ensure_bidirectional_shortest_path_telemetry_workspace", + } { + require.Contains(t, sqlSchemaDown, "drop function if exists "+function) + } +} + +// TestBidirectionalAllShortestWorkspaceSeparatesDiscoveryPredecessorAndOutput +// verifies reusable candidate state is ID-only until the staged output boundary +// and remains disjoint from the exact ASP-A1 fallback workspace. +func TestBidirectionalAllShortestWorkspaceSeparatesDiscoveryPredecessorAndOutput(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.ensure_bidirectional_all_shortest_path_workspace") + require.NotEqual(t, -1, start) + end := strings.Index(sqlSchemaUp[start:], "create or replace function public.all_shortest_paths_bidirectional_compact_v1") + require.NotEqual(t, -1, end) + workspace := sqlSchemaUp[start : start+end] + + require.Contains(t, workspace, "expected_version constant int4 := 1") + for _, table := range []string{ + "asb_front", "asb_seen", "asb_active", "asb_candidate_node", + "asb_candidate_predecessor", "asb_predecessor", "asb_path_count", "asb_output", + } { + require.Contains(t, workspace, "temporary table "+table) + require.Contains(t, workspace, "pg_temp."+table) + } + require.Contains(t, workspace, "primary key (side, node_id, depth, adjacent_id, edge_id)") + require.Contains(t, workspace, "edge_ids int8[] not null primary key") + require.Contains(t, workspace, "on commit preserve rows") + require.NotContains(t, workspace, "spd_") + require.NotContains(t, workspace, "spb_") + // Discovery/frontier tables carry scalar IDs only; arrays are confined to + // asb_output after path-count admission. + discoveryEnd := strings.Index(workspace, "create temporary table asb_output") + require.Greater(t, discoveryEnd, -1) + require.NotContains(t, workspace[:discoveryEnd], "int8[]") +} + +// TestBidirectionalAllShortestKernelProvesOneCutAndGatesBeforeOutput verifies +// scheduler termination, complete equal-depth predecessor retention, and all +// independent cap+1/fallback boundaries are explicit in the SQL kernel. +func TestBidirectionalAllShortestKernelProvesOneCutAndGatesBeforeOutput(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.all_shortest_paths_bidirectional_compact_v1") + require.NotEqual(t, -1, start) + end := strings.Index(sqlSchemaUp[start:], "create or replace function public.all_shortest_paths_b1_strict_alternating") + require.NotEqual(t, -1, end) + kernel := sqlSchemaUp[start : start+end] + + require.Contains(t, kernel, "if min_depth <> 1 then") + require.Contains(t, kernel, "if max_depth > 64 then") + require.Contains(t, kernel, "current_setting('transaction_isolation') <> 'repeatable read'") + require.Contains(t, kernel, "current_setting('transaction_isolation') <> 'serializable'") + require.Contains(t, kernel, "forward_depth + backward_depth >= best_distance") + require.Contains(t, kernel, "cut_depth = best_distance / 2") + require.Contains(t, kernel, "forward_ready_depth >= cut_depth") + require.Contains(t, kernel, "backward_ready_depth >= best_distance - cut_depth") + require.Contains(t, kernel, "scheduler = 'strict_alternating_node'") + require.Contains(t, kernel, "scheduler <> 'smaller_current_level'") + require.Contains(t, kernel, "seen.depth = active.depth + 1") + require.Contains(t, kernel, "limit discovery_admission_limit + 1") + require.Contains(t, kernel, "limit predecessor_admission_limit + 1") + require.Contains(t, kernel, "path_count_sentinel = path_count_limit + 1") + require.Contains(t, kernel, "least(path_count_sentinel::numeric") + require.Contains(t, kernel, "limit enumeration_limit + 1") + require.Contains(t, kernel, "output_bytes > output_bytes_limit") + require.Contains(t, kernel, "select distinct stitched.edge_ids") + require.Contains(t, kernel, "count(distinct path_edge.edge_id)") + require.Contains(t, kernel, "join backward_paths using (meeting_id)") + + firstFallback := strings.Index(kernel, "perform public.clear_bidirectional_all_shortest_path_workspace();") + firstPublicOutput := strings.LastIndex(kernel, "from pg_temp.asb_output output") + require.Greater(t, firstFallback, -1) + require.Greater(t, firstPublicOutput, firstFallback) + require.Contains(t, kernel, "from public.all_shortest_paths_dag(") + require.NotContains(t, kernel, "nodeComposite") + require.NotContains(t, kernel, "edgeComposite") +} + +// TestBidirectionalAllShortestWrappersAndDownMigrationAreSymmetric verifies +// both frozen scheduler identities and every new helper have exact teardown. +func TestBidirectionalAllShortestWrappersAndDownMigrationAreSymmetric(t *testing.T) { + require.Contains(t, sqlSchemaUp, "create or replace function public.all_shortest_paths_b1_strict_alternating(") + require.Contains(t, sqlSchemaUp, "create or replace function public.all_shortest_paths_b2_smaller_current_level(") + require.Contains(t, sqlSchemaUp, "enumeration_limit, output_bytes_limit, 'strict_alternating_node'") + require.Contains(t, sqlSchemaUp, "enumeration_limit, output_bytes_limit, 'smaller_current_level'") + for _, signature := range []string{ + "all_shortest_paths_b1_strict_alternating(int4, int8, int8, int4, int4, int2[], bool, int8, int8, int8, int8, int8)", + "all_shortest_paths_b2_smaller_current_level(int4, int8, int8, int4, int4, int2[], bool, int8, int8, int8, int8, int8)", + "all_shortest_paths_bidirectional_compact_v1(int4, int8, int8, int4, int4, int2[], bool, int8, int8, int8, int8, int8, text)", + "clear_bidirectional_all_shortest_path_workspace()", + "reset_bidirectional_all_shortest_path_workspace()", + "ensure_bidirectional_all_shortest_path_workspace()", + } { + require.Contains(t, sqlSchemaDown, "drop function if exists "+signature) + } +} + +// TestBidirectionalAllShortestDiagnosticTelemetryIsInvocationScoped verifies +// the ASP replay API carries every required search, predecessor, cut, count, +// and output counter in session-local keyed state with symmetric teardown. +func TestBidirectionalAllShortestDiagnosticTelemetryIsInvocationScoped(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.ensure_bidirectional_all_shortest_path_telemetry_workspace") + require.NotEqual(t, -1, start) + end := strings.Index(sqlSchemaUp[start:], "create or replace function public.all_shortest_paths_bidirectional_compact_v1") + require.NotEqual(t, -1, end) + telemetry := sqlSchemaUp[start : start+end] + + require.Contains(t, telemetry, "expected_version constant int4 := 1") + for _, table := range []string{"asb_telemetry_invocation", "asb_telemetry_call", "asb_telemetry_level"} { + require.Contains(t, telemetry, "create temporary table "+table) + } + require.Contains(t, telemetry, "on commit preserve rows") + require.Contains(t, telemetry, "set_config('dawgs.asb_diagnostic_invocation_id', invocation_id, true)") + require.Contains(t, telemetry, "where invocation.invocation_id = target_invocation_id") + for _, counter := range []string{ + "scheduler_actions", "candidate_edges", "distinct_new_nodes", "seen_peak", + "frontier_peak", "queue_peak", "predecessor_peak", "meeting_candidates", + "frozen_distance", "witness_rows", "same_depth_predecessor_additions", + "meeting_nodes", "cut_depth", "path_count_estimate", "path_count_saturated", + "enumerated_candidates", "duplicate_rejects", "output_paths", + "output_edge_cells", "output_bytes", + } { + require.Contains(t, telemetry, "'"+counter+"'") + } + require.NotContains(t, telemetry, "create table public.asb_telemetry") + + kernelStart := strings.Index(sqlSchemaUp, "create or replace function public.all_shortest_paths_bidirectional_compact_v1") + wrapperStart := strings.Index(sqlSchemaUp[kernelStart:], "create or replace function public.all_shortest_paths_b1_strict_alternating") + require.NotEqual(t, kernelStart, -1) + require.NotEqual(t, wrapperStart, -1) + kernel := sqlSchemaUp[kernelStart : kernelStart+wrapperStart] + require.Contains(t, kernel, "_start_bidirectional_all_shortest_path_diagnostic_call_v1") + require.Contains(t, kernel, "_record_bidirectional_all_shortest_path_diagnostic_level_v1") + require.Contains(t, kernel, "_finish_bidirectional_all_shortest_path_diagnostic_call_v1") + require.Contains(t, kernel, "'exact_a1_fallback'") + require.Contains(t, kernel, "'preflight_one_hop'") + require.Contains(t, kernel, "'preflight_two_hop'") + require.Contains(t, kernel, "perform public.clear_bidirectional_all_shortest_path_workspace();") + + for _, function := range []string{ + "_finish_bidirectional_all_shortest_path_diagnostic_call_v1", + "_record_bidirectional_all_shortest_path_diagnostic_level_v1", + "_start_bidirectional_all_shortest_path_diagnostic_call_v1", + "clear_bidirectional_all_shortest_path_diagnostic_v1", + "read_bidirectional_all_shortest_path_diagnostic_v1", + "begin_bidirectional_all_shortest_path_diagnostic_v1", + "ensure_bidirectional_all_shortest_path_telemetry_workspace", + } { + require.Contains(t, sqlSchemaDown, "drop function if exists "+function) + } +} + +// TestLegacyASPFallbackReusesWorkspaceWithoutCatalogSwaps verifies legacy all-shortest fallback reuses workspace without replacing catalog objects. +func TestLegacyASPFallbackReusesWorkspaceWithoutCatalogSwaps(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.create_unidirectional_pathspace_tables") + require.NotEqual(t, -1, start) + legacyWorkspace := sqlSchemaUp[start:] + + require.Contains(t, legacyWorkspace, "create temporary table if not exists forward_front") + require.Contains(t, legacyWorkspace, "create temporary table if not exists backward_front") + require.Contains(t, legacyWorkspace, "on commit preserve rows") + require.Contains(t, legacyWorkspace, "truncate table forward_front, next_front") + require.Contains(t, legacyWorkspace, "insert into forward_front select * from next_front") + require.Contains(t, legacyWorkspace, "insert into backward_front select * from next_front") + require.NotContains(t, legacyWorkspace, "alter table forward_front") + require.NotContains(t, legacyWorkspace, "alter table backward_front") +} diff --git a/drivers/pg/query_cache.go b/drivers/pg/query_cache.go new file mode 100644 index 00000000..7b93e701 --- /dev/null +++ b/drivers/pg/query_cache.go @@ -0,0 +1,212 @@ +package pg + +import ( + "container/list" + "strings" + "sync" + + "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/cypher/models/cypher" +) + +const ( + // defaultCypherParseCacheEntries is the maximum number of parsed ASTs + // retained when no cache capacity is configured. + defaultCypherParseCacheEntries = 256 + + // maxCachedCypherQueryBytes excludes oversized query strings from the parse + // cache while still allowing them to be parsed. + maxCachedCypherQueryBytes = 64 * 1024 +) + +// cypherParseCacheEntry pairs an immutable parsed AST with the normalized query text used as its LRU key. +type cypherParseCacheEntry struct { + // query is the normalized, cloned cache key. + query string + + // parsed is the immutable parser result shared by cache hits. + parsed *cypher.RegularQuery +} + +// cypherParseCall publishes one in-flight parse result to callers waiting on the same query. +type cypherParseCall struct { + // done closes after parsed and err have been published. + done chan struct{} + + // parsed is the AST produced by the coalesced parse. + parsed *cypher.RegularQuery + + // err is the parser failure, if any, shared with waiters. + err error +} + +// cypherParseCache retains immutable parser output. Translation is safe to run +// concurrently against a cached query because the optimizer copies the Cypher +// AST before applying rules or lowering it. +type cypherParseCache struct { + // lock protects cache entries, pending calls, closure state, and counters. + lock sync.Mutex + + // capacity is the maximum number of completed parses retained in entries. + capacity int + + // entries indexes completed parses by normalized query text. + entries map[string]*list.Element + + // lru orders completed entries from most to least recently used. + lru *list.List + + // pending coalesces concurrent misses for the same normalized query. + pending map[string]*cypherParseCall + + // closed prevents completed or future parses from being retained. + closed bool + + // stats accumulates cache activity for this cache instance. + stats ParseCacheStats +} + +// ParseCacheStats contains aggregate, query-text-free diagnostics. It is a +// snapshot; counters are scoped to one driver instance and reset only when the +// driver is reconstructed. +type ParseCacheStats struct { + // Hits counts lookups served from completed cache entries. + Hits uint64 `json:"hits"` + + // Misses counts queries parsed by the caller that established a pending entry. + Misses uint64 `json:"misses"` + + // Bypasses counts queries parsed without retention because caching was unavailable or disallowed. + Bypasses uint64 `json:"bypasses"` + + // Evictions counts least-recently-used entries removed at capacity. + Evictions uint64 `json:"evictions"` + + // CoalescedMisses counts callers that waited for an existing parse of the same query. + CoalescedMisses uint64 `json:"coalesced_misses"` + + // Entries is the number of completed parses retained when the snapshot was taken. + Entries int `json:"entries"` + + // Pending is the number of in-flight parses when the snapshot was taken. + Pending int `json:"pending"` +} + +// newCypherParseCache initializes an empty LRU parse cache with the requested capacity. +func newCypherParseCache(capacity int) *cypherParseCache { + return &cypherParseCache{ + capacity: capacity, + entries: make(map[string]*list.Element, capacity), + lru: list.New(), + pending: map[string]*cypherParseCall{}, + } +} + +// Parse returns an immutable Cypher AST and reports whether it came from a completed or coalesced cache hit. +func (s *cypherParseCache) Parse(input string) (*cypher.RegularQuery, bool, error) { + query := strings.TrimSpace(input) + // Bound the caller-owned input rather than only the trimmed view. A short + // query padded with a very large amount of whitespace must not retain that + // backing allocation through an LRU key. + if s == nil { + parsed, err := frontend.ParseCypher(frontend.NewContext(), query) + if err != nil { + return nil, false, err + } + + return parsed, false, nil + } + + s.lock.Lock() + if s.closed || s.capacity <= 0 || len(input) > maxCachedCypherQueryBytes { + s.stats.Bypasses++ + s.lock.Unlock() + parsed, err := frontend.ParseCypher(frontend.NewContext(), query) + if err != nil { + return nil, false, err + } + + return parsed, false, nil + } + if element, found := s.entries[query]; found { + s.stats.Hits++ + s.lru.MoveToFront(element) + parsed := element.Value.(cypherParseCacheEntry).parsed + s.lock.Unlock() + return parsed, true, nil + } + if call, found := s.pending[query]; found { + s.stats.CoalescedMisses++ + s.lock.Unlock() + <-call.done + if call.err != nil { + return nil, false, call.err + } + + return call.parsed, true, nil + } + + // Lookups do not retain the caller's string. Clone only a true miss before + // using it as a pending/cache key so the zero-allocation hit path remains + // intact. + query = strings.Clone(query) + s.stats.Misses++ + call := &cypherParseCall{done: make(chan struct{})} + s.pending[query] = call + s.lock.Unlock() + + parsed, err := frontend.ParseCypher(frontend.NewContext(), query) + + s.lock.Lock() + call.parsed = parsed + call.err = err + if err == nil && !s.closed { + element := s.lru.PushFront(cypherParseCacheEntry{ + query: query, + parsed: parsed, + }) + s.entries[query] = element + if s.lru.Len() > s.capacity { + evicted := s.lru.Back() + s.lru.Remove(evicted) + delete(s.entries, evicted.Value.(cypherParseCacheEntry).query) + s.stats.Evictions++ + } + } + delete(s.pending, query) + close(call.done) + s.lock.Unlock() + + if err != nil { + return nil, false, err + } + + return parsed, false, nil +} + +// Stats returns a consistent snapshot of counters and current cache occupancy. +func (s *cypherParseCache) Stats() ParseCacheStats { + if s == nil { + return ParseCacheStats{} + } + s.lock.Lock() + defer s.lock.Unlock() + stats := s.stats + stats.Entries = len(s.entries) + stats.Pending = len(s.pending) + return stats +} + +// Close prevents future retention and releases every cached query/AST +// reference. In-flight parses wake their waiters normally but do not repopulate +// the cache after closure. +func (s *cypherParseCache) Close() { + if s == nil { + return + } + s.lock.Lock() + s.closed = true + s.entries = nil + s.lru.Init() + s.lock.Unlock() +} diff --git a/drivers/pg/query_cache_test.go b/drivers/pg/query_cache_test.go new file mode 100644 index 00000000..595815a6 --- /dev/null +++ b/drivers/pg/query_cache_test.go @@ -0,0 +1,174 @@ +package pg + +import ( + "strings" + "sync" + "testing" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/stretchr/testify/require" +) + +// TestCypherParseCacheReusesTrimmedQuery verifies whitespace-equivalent queries share one immutable AST entry. +func TestCypherParseCacheReusesTrimmedQuery(t *testing.T) { + cache := newCypherParseCache(2) + + first, hit, err := cache.Parse(" MATCH (n) RETURN n ") + require.NoError(t, err) + require.False(t, hit) + + second, hit, err := cache.Parse("MATCH (n) RETURN n") + require.NoError(t, err) + require.True(t, hit) + require.Same(t, first, second) +} + +// TestCypherParseCacheEvictsLeastRecentlyUsedQuery verifies capacity pressure removes the coldest completed parse. +func TestCypherParseCacheEvictsLeastRecentlyUsedQuery(t *testing.T) { + cache := newCypherParseCache(2) + + _, _, err := cache.Parse("MATCH (n) RETURN n") + require.NoError(t, err) + second, _, err := cache.Parse("MATCH (n) RETURN id(n)") + require.NoError(t, err) + _, hit, err := cache.Parse("MATCH (n) RETURN n") + require.NoError(t, err) + require.True(t, hit) + _, _, err = cache.Parse("MATCH (n) RETURN count(n)") + require.NoError(t, err) + + reparsed, hit, err := cache.Parse("MATCH (n) RETURN id(n)") + require.NoError(t, err) + require.False(t, hit) + require.NotSame(t, second, reparsed) +} + +// TestCypherParseCacheDoesNotRetainErrorsOrOversizedQueries verifies failed and over-limit parses always bypass retention. +func TestCypherParseCacheDoesNotRetainErrorsOrOversizedQueries(t *testing.T) { + cache := newCypherParseCache(2) + + parsed, hit, err := cache.Parse("MATCH (") + require.Error(t, err) + require.Nil(t, parsed) + require.False(t, hit) + parsed, hit, err = cache.Parse("MATCH (") + require.Error(t, err) + require.Nil(t, parsed) + require.False(t, hit) + require.Empty(t, cache.entries) + + oversized := "MATCH (n) RETURN n // " + strings.Repeat("x", maxCachedCypherQueryBytes) + _, hit, err = cache.Parse(oversized) + require.NoError(t, err) + require.False(t, hit) + require.Empty(t, cache.entries) + + padded := strings.Repeat(" ", maxCachedCypherQueryBytes) + "MATCH (n) RETURN n" + _, hit, err = cache.Parse(padded) + require.NoError(t, err) + require.False(t, hit) + require.Empty(t, cache.entries) + require.Equal(t, uint64(2), cache.Stats().Bypasses) +} + +// TestCypherParseCacheCoalescesConcurrentMissesAndSupportsConcurrentOptimization verifies one parse can safely serve simultaneous callers. +func TestCypherParseCacheCoalescesConcurrentMissesAndSupportsConcurrentOptimization(t *testing.T) { + cache := newCypherParseCache(2) + const workers = 32 + + queries := make([]any, workers) + errors := make([]error, workers) + var waitGroup sync.WaitGroup + waitGroup.Add(workers) + for idx := 0; idx < workers; idx++ { + go func(index int) { + defer waitGroup.Done() + query, _, err := cache.Parse("MATCH (n) WHERE id(n) = $id RETURN n") + if err == nil { + _, err = optimize.Optimize(query) + } + errors[index] = err + queries[index] = query + }(idx) + } + waitGroup.Wait() + + for _, err := range errors { + require.NoError(t, err) + } + for idx := 1; idx < len(queries); idx++ { + require.Same(t, queries[0], queries[idx]) + } + require.Len(t, cache.entries, 1) + require.Equal(t, uint64(workers-1), cache.Stats().Hits+cache.Stats().CoalescedMisses) +} + +// TestCypherParseCacheSupportsConcurrentDifferentKeys verifies independent queries can populate the cache concurrently. +func TestCypherParseCacheSupportsConcurrentDifferentKeys(t *testing.T) { + cache := newCypherParseCache(64) + const workers = 32 + var waitGroup sync.WaitGroup + errors := make([]error, workers) + waitGroup.Add(workers) + for idx := 0; idx < workers; idx++ { + go func(index int) { + defer waitGroup.Done() + _, _, errors[index] = cache.Parse("MATCH (n) RETURN n // key " + strings.Repeat("x", index)) + }(idx) + } + waitGroup.Wait() + for _, err := range errors { + require.NoError(t, err) + } + require.Equal(t, uint64(workers), cache.Stats().Misses) + require.Equal(t, workers, cache.Stats().Entries) +} + +// TestCypherParseCacheStatsAndCloseReleaseEntries verifies snapshots reflect activity and Close releases retained ASTs. +func TestCypherParseCacheStatsAndCloseReleaseEntries(t *testing.T) { + cache := newCypherParseCache(1) + _, _, err := cache.Parse("MATCH (n) RETURN n") + require.NoError(t, err) + _, hit, err := cache.Parse("MATCH (n) RETURN n") + require.NoError(t, err) + require.True(t, hit) + _, _, err = cache.Parse("MATCH (n) RETURN id(n)") + require.NoError(t, err) + require.Equal(t, ParseCacheStats{ + Hits: 1, + Misses: 2, + Evictions: 1, + Entries: 1, + }, cache.Stats()) + + cache.Close() + require.Zero(t, cache.Stats().Entries) + require.Nil(t, cache.entries) + _, hit, err = cache.Parse("MATCH (n) RETURN id(n)") + require.NoError(t, err) + require.False(t, hit) + require.Equal(t, uint64(1), cache.Stats().Bypasses) +} + +// BenchmarkCypherParseCache measures repeated lookup of a normalized cached query. +func BenchmarkCypherParseCache(b *testing.B) { + const query = "MATCH (n) WHERE id(n) = $id RETURN n" + b.Run("uncached", func(b *testing.B) { + for idx := 0; idx < b.N; idx++ { + cache := newCypherParseCache(0) + _, _, err := cache.Parse(query) + require.NoError(b, err) + } + }) + b.Run("cached", func(b *testing.B) { + cache := newCypherParseCache(1) + _, _, err := cache.Parse(query) + require.NoError(b, err) + b.ResetTimer() + for idx := 0; idx < b.N; idx++ { + _, hit, err := cache.Parse(query) + require.NoError(b, err) + require.True(b, hit) + } + }) +} diff --git a/drivers/pg/result.go b/drivers/pg/result.go index 1927dfa0..226bbd2c 100644 --- a/drivers/pg/result.go +++ b/drivers/pg/result.go @@ -11,11 +11,21 @@ import ( "github.com/specterops/dawgs/graph" ) +// queryResult adapts pgx rows to graph.Result, caching column names and decoding JSON values for each current row. type queryResult struct { - ctx context.Context - rows pgx.Rows - values []any - keys []string + // ctx supplies cancellation and request scope when decoded graph values require kind mapping. + ctx context.Context + + // rows is the pgx result set being adapted. + rows pgx.Rows + + // values contains the decoded values for the current row. + values []any + + // keys caches immutable column names shared by every row in the result set. + keys []string + + // kindMapper resolves database kind identifiers while scanning graph values. kindMapper KindMapper } @@ -27,18 +37,17 @@ func (s *queryResult) Keys() []string { return s.keys } +// Next advances to the next row, caching its column names and decoding JSON values before exposing it. func (s *queryResult) Next() bool { if s.rows.Next() { - s.keys = []string{} - for _, desc := range s.rows.FieldDescriptions() { - s.keys = append(s.keys, desc.Name) - } + fields := s.rows.FieldDescriptions() + s.cacheKeys(fields) // This error check exists just as a guard for a successful return of this function. The expectation is that // the pgx type will have error information attached to it which is reflected by the Error receiver function // of this type if values, err := s.rows.Values(); err == nil { - s.values = decodeJSONValues(values, s.rows.FieldDescriptions()) + s.values = decodeJSONValues(values, fields) return true } } @@ -46,6 +55,21 @@ func (s *queryResult) Next() bool { return false } +// cacheKeys records immutable column names once for the lifetime of the result set. +func (s *queryResult) cacheKeys(fields []pgconn.FieldDescription) { + if s.keys != nil { + return + } + + // A pgx Rows value represents one result set, whose field descriptions do + // not change between rows. Retain the names once instead of rebuilding the + // same slice for every row. + s.keys = make([]string, len(fields)) + for idx, field := range fields { + s.keys[idx] = field.Name + } +} + func (s *queryResult) Mapper() graph.ValueMapper { return NewValueMapper(s.ctx, s.kindMapper) } @@ -62,22 +86,26 @@ func (s *queryResult) Close() { s.rows.Close() } +// decodeJSONValues replaces raw JSON and JSONB fields in the caller-owned row slice with decoded Go values. func decodeJSONValues(values []any, fields []pgconn.FieldDescription) []any { - decodedValues := make([]any, len(values)) - copy(decodedValues, values) - + // pgx Rows.Values returns a decoded value slice for the current row. The old + // implementation made a shallow copy before replacing JSON scalars, but its + // nested values were still shared. Updating this otherwise-unexposed slice + // in place therefore preserves ownership while avoiding one allocation and + // copy per row. for idx, field := range fields { switch field.DataTypeOID { case pgtype.JSONOID, pgtype.JSONBOID: if decoded, ok := decodeJSONValue(values[idx]); ok { - decodedValues[idx] = decoded + values[idx] = decoded } } } - return decodedValues + return values } +// decodeJSONValue decodes byte JSON and structured string JSON while preserving already-decoded scalar strings. func decodeJSONValue(value any) (any, bool) { switch typedValue := value.(type) { case []byte: diff --git a/drivers/pg/result_test.go b/drivers/pg/result_test.go index a637976b..35bd498a 100644 --- a/drivers/pg/result_test.go +++ b/drivers/pg/result_test.go @@ -1,13 +1,23 @@ package pg import ( + "context" "testing" "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgtype" + "github.com/pashagolub/pgxmock/v5" "github.com/stretchr/testify/require" ) +var ( + // benchmarkDecodedJSONValues retains decoded rows so benchmark work cannot be optimized away. + benchmarkDecodedJSONValues []any + + // benchmarkResultKeys retains cached column names so benchmark work cannot be optimized away. + benchmarkResultKeys []string +) + func TestDecodeJSONValue(t *testing.T) { t.Run("number", func(t *testing.T) { value, ok := decodeJSONValue([]byte("42")) @@ -46,6 +56,7 @@ func TestDecodeJSONValue(t *testing.T) { }) } +// TestDecodeJSONValuesPreservesDecodedStringScalars verifies JSON-typed strings already decoded by pgx are not reinterpreted as JSON tokens. func TestDecodeJSONValuesPreservesDecodedStringScalars(t *testing.T) { var ( values = []any{ @@ -60,7 +71,164 @@ func TestDecodeJSONValuesPreservesDecodedStringScalars(t *testing.T) { {DataTypeOID: pgtype.JSONBOID}, {DataTypeOID: pgtype.JSONBOID}, } + expected = append([]any(nil), values...) + ) + + decoded := decodeJSONValues(values, fields) + require.Equal(t, expected, decoded) + require.Same(t, &values[0], &decoded[0]) +} + +// TestDecodeJSONValuesReusesInputSlice verifies JSON replacement occurs in the pgx-owned row slice without an extra copy. +func TestDecodeJSONValuesReusesInputSlice(t *testing.T) { + var ( + values = []any{ + []byte(`{"name":"alpha"}`), + int64(42), + } + fields = []pgconn.FieldDescription{ + {DataTypeOID: pgtype.JSONBOID}, + {DataTypeOID: pgtype.Int8OID}, + } + ) + + decoded := decodeJSONValues(values, fields) + + require.Same(t, &values[0], &decoded[0]) + require.Equal(t, map[string]any{"name": "alpha"}, decoded[0]) + require.Equal(t, int64(42), decoded[1]) +} + +// TestDecodeJSONValuesDoesNotAllocateForDecodedFields verifies already-decoded fields follow the zero-allocation path. +func TestDecodeJSONValuesDoesNotAllocateForDecodedFields(t *testing.T) { + var ( + values = []any{ + map[string]any{"name": "alpha"}, + int64(42), + } + fields = []pgconn.FieldDescription{ + {DataTypeOID: pgtype.JSONBOID}, + {DataTypeOID: pgtype.Int8OID}, + } ) - require.Equal(t, values, decodeJSONValues(values, fields)) + require.Zero(t, testing.AllocsPerRun(100, func() { + decodeJSONValues(values, fields) + })) +} + +// TestQueryResultCachesKeysAcrossRows verifies column-name storage is reused while row values remain independently owned. +func TestQueryResultCachesKeysAcrossRows(t *testing.T) { + mock, err := pgxmock.NewConn() + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, mock.Close(context.Background())) + require.NoError(t, mock.ExpectationsWereMet()) + }) + + mock.ExpectQuery("select values").WillReturnRows( + pgxmock.NewRows([]string{"name", "count"}). + AddRow("alpha", int64(1)). + AddRow("beta", int64(2)), + ) + mock.ExpectClose() + + rows, err := mock.Query(context.Background(), "select values") + require.NoError(t, err) + + result := &queryResult{ + rows: rows, + } + require.True(t, result.Next()) + require.Equal(t, []string{"name", "count"}, result.Keys()) + firstKey := &result.Keys()[0] + firstValues := result.Values() + require.Equal(t, []any{"alpha", int64(1)}, firstValues) + + require.True(t, result.Next()) + require.Same(t, firstKey, &result.Keys()[0]) + require.Equal(t, []any{"beta", int64(2)}, result.Values()) + // Rows.Values owns each returned row slice. Advancing the cursor must not + // mutate values retained by a caller or mapper from the previous row. + require.Equal(t, []any{"alpha", int64(1)}, firstValues) + require.False(t, result.Next()) + require.NoError(t, result.Error()) +} + +// TestQueryResultCacheKeysDoesNotAllocateAfterInitialization verifies repeated key access performs no allocation. +func TestQueryResultCacheKeysDoesNotAllocateAfterInitialization(t *testing.T) { + var ( + result = &queryResult{} + fields = []pgconn.FieldDescription{ + {Name: "name"}, + {Name: "count"}, + } + ) + result.cacheKeys(fields) + + require.Zero(t, testing.AllocsPerRun(100, func() { + result.cacheKeys(fields) + })) +} + +// BenchmarkDecodeJSONValuesDecodedFields compares in-place decoding with the previous shallow-copy approach. +func BenchmarkDecodeJSONValuesDecodedFields(b *testing.B) { + var ( + values = []any{ + map[string]any{"name": "alpha"}, + int64(42), + } + fields = []pgconn.FieldDescription{ + {DataTypeOID: pgtype.JSONBOID}, + {DataTypeOID: pgtype.Int8OID}, + } + ) + + b.Run("in_place", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + benchmarkDecodedJSONValues = decodeJSONValues(values, fields) + } + }) + + b.Run("shallow_copy_reference", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + copiedValues := make([]any, len(values)) + copy(copiedValues, values) + benchmarkDecodedJSONValues = decodeJSONValues(copiedValues, fields) + } + }) +} + +// BenchmarkQueryResultCacheKeys compares cached column names with rebuilding them for every row. +func BenchmarkQueryResultCacheKeys(b *testing.B) { + fields := []pgconn.FieldDescription{ + {Name: "name"}, + {Name: "count"}, + } + + b.Run("cached", func(b *testing.B) { + result := &queryResult{} + result.cacheKeys(fields) + b.ReportAllocs() + b.ResetTimer() + + for b.Loop() { + result.cacheKeys(fields) + benchmarkResultKeys = result.keys + } + }) + + b.Run("rebuild_reference", func(b *testing.B) { + result := &queryResult{} + b.ReportAllocs() + for b.Loop() { + result.keys = make([]string, len(fields)) + for idx, field := range fields { + result.keys[idx] = field.Name + } + benchmarkResultKeys = result.keys + } + }) } diff --git a/drivers/pg/transaction.go b/drivers/pg/transaction.go index 7bf4bbd7..10594dee 100644 --- a/drivers/pg/transaction.go +++ b/drivers/pg/transaction.go @@ -10,20 +10,27 @@ import ( "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgxpool" - "github.com/specterops/dawgs/cypher/frontend" "github.com/specterops/dawgs/drivers/pg/model" "github.com/specterops/dawgs/graph" "github.com/specterops/dawgs/query" "github.com/specterops/dawgs/util/size" ) +// driver is the common execution surface implemented by pooled connections and explicit pgx transactions. type driver interface { + // Exec executes a statement and returns its PostgreSQL command tag. Exec(ctx context.Context, sql string, arguments ...any) (commandTag pgconn.CommandTag, err error) + + // Query executes a statement and returns its streaming row set. Query(ctx context.Context, sql string, arguments ...any) (pgx.Rows, error) + + // QueryRow executes a statement whose first row is consumed through pgx.Row. QueryRow(ctx context.Context, sql string, arguments ...any) pgx.Row } +// inspectingDriver records SQL and arguments before delegating execution to a connection or transaction. type inspectingDriver struct { + // upstreamDriver receives each operation after its SQL and arguments have been inspected. upstreamDriver driver } @@ -42,17 +49,37 @@ func (s inspectingDriver) QueryRow(ctx context.Context, sql string, arguments .. return s.upstreamDriver.QueryRow(ctx, sql, arguments...) } +// transaction binds query execution, schema resolution, and an optional pgx transaction to one graph operation context. type transaction struct { - schemaManager *SchemaManager - queryExecMode pgx.QueryExecMode + // schemaManager resolves target graphs, kind identifiers, and cached Cypher translations. + schemaManager *SchemaManager + + // queryExecMode selects the pgx execution protocol supplied with each query. + queryExecMode pgx.QueryExecMode + + // queryResultsFormat selects the pgx wire format requested for returned columns. queryResultsFormat pgx.QueryResultFormats - ctx context.Context - conn *pgxpool.Conn - tx pgx.Tx - targetSchema graph.Graph - targetSchemaSet bool + + // ctx scopes all work performed by the graph transaction. + ctx context.Context + + // conn is the acquired pooled connection underlying this transaction wrapper. + conn *pgxpool.Conn + + // tx is the optional explicit PostgreSQL transaction used for transactional operations. + tx pgx.Tx + + // isolation records the explicit snapshot contract, if any, used to admit B candidates. + isolation pgx.TxIsoLevel + + // targetSchema identifies the graph selected explicitly for subsequent operations. + targetSchema graph.Graph + + // targetSchemaSet distinguishes an explicit target from the zero-value graph schema. + targetSchemaSet bool } +// newTransactionWrapper configures a graph transaction and optionally begins an explicit PostgreSQL transaction. func newTransactionWrapper(ctx context.Context, conn *pgxpool.Conn, schemaManager *SchemaManager, cfg *Config, allocateTransaction bool) (*transaction, error) { wrapper := &transaction{ schemaManager: schemaManager, @@ -60,6 +87,7 @@ func newTransactionWrapper(ctx context.Context, conn *pgxpool.Conn, schemaManage queryResultsFormat: cfg.QueryResultFormats, ctx: ctx, conn: conn, + isolation: cfg.Options.IsoLevel, targetSchemaSet: false, } @@ -74,6 +102,7 @@ func newTransactionWrapper(ctx context.Context, conn *pgxpool.Conn, schemaManage return wrapper, nil } +// driver returns an inspected executor backed by the active transaction or, when absent, the pooled connection. func (s *transaction) driver() driver { if s.tx != nil { return inspectingDriver{ @@ -104,6 +133,8 @@ func (s *transaction) Close() { } } +// getTargetGraph resolves the explicitly selected graph or falls back to the +// driver's default graph. func (s *transaction) getTargetGraph() (model.Graph, error) { if !s.targetSchemaSet { // Look for a default graph target @@ -117,6 +148,7 @@ func (s *transaction) getTargetGraph() (model.Graph, error) { return s.schemaManager.AssertGraph(s, s.targetSchema) } +// targetGraphID resolves the database ID of the transaction's explicit or default graph target. func (s *transaction) targetGraphID() (int32, error) { if graphTarget, err := s.getTargetGraph(); err != nil { return 0, err @@ -264,6 +296,8 @@ func (s *transaction) Relationships() graph.RelationshipQuery { } } +// query executes SQL with the transaction's configured execution mode and +// result format, adding named parameters when present. func (s *transaction) query(query string, parameters map[string]any) (pgx.Rows, error) { queryArgs := []any{s.queryExecMode, s.queryResultsFormat} @@ -274,18 +308,35 @@ func (s *transaction) query(query string, parameters map[string]any) (pgx.Rows, return s.driver().Query(s.ctx, query, queryArgs...) } +// Query parses and translates Cypher through the schema caches, returning translation failures as graph results. func (s *transaction) Query(query string, parameters map[string]any) graph.Result { - if parsedQuery, err := frontend.ParseCypher(frontend.NewContext(), query); err != nil { - return graph.NewErrorResult(err) - } else if graphTarget, err := s.getTargetGraph(); err != nil { + parsedQuery, _, err := s.schemaManager.parseCache.Parse(query) + if err != nil { return graph.NewErrorResult(err) - } else if translated, err := translate.Translate(s.ctx, parsedQuery, s.schemaManager, parameters, graphTarget.ID); err != nil { + } + graphTarget, err := s.getTargetGraph() + if err != nil { return graph.NewErrorResult(err) - } else if sqlQuery, err := translate.Translated(translated); err != nil { + } + policy, policyIdentity := s.schemaManager.effectiveTraversalPolicy(query, s.isolation) + sqlQuery, translatedParameters, err := s.schemaManager.translationCache.TranslateWithPolicy(query, graphTarget.ID, parameters, policyIdentity, func() (translate.Result, string, error) { + var translated translate.Result + var translateErr error + if policy.enabled() { + translated, translateErr = translate.TranslateWithProductionOptions(s.ctx, parsedQuery, s.schemaManager, parameters, graphTarget.ID, policy.productionOptions(query)) + } else { + translated, translateErr = translate.Translate(s.ctx, parsedQuery, s.schemaManager, parameters, graphTarget.ID) + } + if translateErr != nil { + return translate.Result{}, "", translateErr + } + formatted, formatErr := translate.Translated(translated) + return translated, formatted, formatErr + }) + if err != nil { return graph.NewErrorResult(err) - } else { - return s.Raw(sqlQuery, translated.Parameters) } + return s.Raw(sqlQuery, translatedParameters) } func (s *transaction) Raw(query string, parameters map[string]any) graph.Result { diff --git a/drivers/pg/translation_cache.go b/drivers/pg/translation_cache.go new file mode 100644 index 00000000..a1c2d97e --- /dev/null +++ b/drivers/pg/translation_cache.go @@ -0,0 +1,322 @@ +package pg + +import ( + "container/list" + "fmt" + "sort" + "strconv" + "strings" + "sync" + + model "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" +) + +// defaultCypherTranslationCacheEntries is the maximum number of translated SQL +// entries retained when no cache capacity is configured. +const defaultCypherTranslationCacheEntries = 256 + +// cypherTranslationCacheKey identifies SQL that can be reused for one query, graph, and parameter type shape. +type cypherTranslationCacheKey struct { + // query is normalized Cypher text cloned on a cache miss. + query string + + // graphID scopes generated SQL to the selected graph. + graphID int32 + + // parameterType captures sorted parameter names and negotiated PostgreSQL types. + parameterType string + + // policyIdentity partitions SQL emitted by versioned production traversal policies. + policyIdentity string +} + +// cypherTranslationCacheValue stores generated SQL and the source mapping needed to bind fresh parameter values. +type cypherTranslationCacheValue struct { + // key is the immutable identity used by the LRU index. + key cypherTranslationCacheKey + + // sql is the rendered PostgreSQL statement reused by cache hits. + sql string + + // parameterSources maps generated SQL parameters back to caller-supplied Cypher parameter names. + parameterSources map[string]string +} + +// bind negotiates current caller values for every generated parameter recorded by the cached translation. +func (s cypherTranslationCacheValue) bind(parameters map[string]any) (map[string]any, error) { + bound := make(map[string]any, len(s.parameterSources)) + for identifier, source := range s.parameterSources { + value, found := parameters[source] + if !found { + return nil, fmt.Errorf("cached translation requires missing parameter source %q", source) + } + negotiated, err := model.NegotiateValue(value) + if err != nil { + return nil, fmt.Errorf("negotiate cached parameter %s: %w", source, err) + } + bound[identifier] = negotiated + } + return bound, nil +} + +// cypherTranslationCall publishes one in-flight build result to callers waiting on the same cache key. +type cypherTranslationCall struct { + // done closes after value, err, and cacheable have been published. + done chan struct{} + + // value is the translation produced by the build owner. + value cypherTranslationCacheValue + + // err is the build failure shared with waiting callers. + err error + + // cacheable reports whether waiters may safely rebind and reuse value. + cacheable bool +} + +// cypherTranslationCache is a bounded LRU of reusable SQL translations with single-flight miss coalescing. +type cypherTranslationCache struct { + // lock protects completed entries, pending calls, closure state, and counters. + lock sync.Mutex + + // capacity is the maximum number of completed translations retained. + capacity int + + // entries indexes completed translations by their reusable input shape. + entries map[cypherTranslationCacheKey]*list.Element + + // lru orders completed translations from most to least recently used. + lru *list.List + + // pending coalesces concurrent builds for the same translation key. + pending map[cypherTranslationCacheKey]*cypherTranslationCall + + // closed prevents completed or future builds from being retained. + closed bool + + // stats accumulates cache activity for this instance. + stats TranslationCacheStats +} + +// TranslationCacheStats is a query-text-free snapshot of translation cache activity and occupancy. +type TranslationCacheStats struct { + // Hits counts translations served from completed cache entries. + Hits uint64 `json:"hits"` + + // Misses counts builds owned by callers that established pending entries. + Misses uint64 `json:"misses"` + + // Bypasses counts builds that could not be retained or safely shared. + Bypasses uint64 `json:"bypasses"` + + // Evictions counts least-recently-used translations removed at capacity. + Evictions uint64 `json:"evictions"` + + // CoalescedMisses counts callers that waited for an existing build of the same key. + CoalescedMisses uint64 `json:"coalesced_misses"` + + // Entries is the number of completed translations retained when the snapshot was taken. + Entries int `json:"entries"` + + // Pending is the number of in-flight translation builds when the snapshot was taken. + Pending int `json:"pending"` +} + +// newCypherTranslationCache initializes an empty LRU translation cache with the requested capacity. +func newCypherTranslationCache(capacity int) *cypherTranslationCache { + return &cypherTranslationCache{ + capacity: capacity, + entries: make(map[cypherTranslationCacheKey]*list.Element, capacity), + lru: list.New(), + pending: map[cypherTranslationCacheKey]*cypherTranslationCall{}, + } +} + +// translationParameterTypeKey encodes sorted parameter names and negotiated data types into an unambiguous cache-key component. +func translationParameterTypeKey(parameters map[string]any) string { + keys := make([]string, 0, len(parameters)) + for key := range parameters { + keys = append(keys, key) + } + sort.Strings(keys) + var key strings.Builder + for _, name := range keys { + value := parameters[name] + var typeName string + if value == nil { + typeName = "null" + } else if dataType, err := model.ValueToDataType(value); err == nil { + typeName = dataType.String() + } else { + // Translation will report the same unsupported value error. Retaining + // its Go type here prevents unrelated invalid shapes from coalescing. + typeName = fmt.Sprintf("invalid:%T", value) + } + + key.WriteString(strconv.Itoa(len(name))) + key.WriteByte(':') + key.WriteString(name) + key.WriteString(strconv.Itoa(len(typeName))) + key.WriteByte(':') + key.WriteString(typeName) + } + return key.String() +} + +// cacheableTranslation reports whether every translated parameter can be rebound from a current caller parameter. +func cacheableTranslation(result translate.Result, parameters map[string]any) bool { + if len(result.Parameters) != len(result.ParameterSources) { + return false + } + for identifier := range result.Parameters { + source, found := result.ParameterSources[identifier] + if !found || source == "" { + return false + } + if _, found := parameters[source]; !found { + return false + } + } + return true +} + +// cloneSources copies parameter-source metadata so cached values do not alias translator-owned maps. +func cloneSources(values map[string]string) map[string]string { + cloned := make(map[string]string, len(values)) + for key, value := range values { + cloned[key] = value + } + return cloned +} + +// Translate returns reusable SQL with values rebound from parameters, building or coalescing a translation on a miss. +func (s *cypherTranslationCache) Translate(query string, graphID int32, parameters map[string]any, build func() (translate.Result, string, error)) (string, map[string]any, error) { + return s.TranslateWithPolicy(query, graphID, parameters, "production-incumbent-v1", build) +} + +// TranslateWithPolicy returns reusable SQL partitioned by the exact effective +// production policy, making gate disablement immediately cache safe. +func (s *cypherTranslationCache) TranslateWithPolicy(query string, graphID int32, parameters map[string]any, policyIdentity string, build func() (translate.Result, string, error)) (string, map[string]any, error) { + trimmed := strings.TrimSpace(query) + if s == nil || s.capacity <= 0 || len(query) > maxCachedCypherQueryBytes { + if result, sql, err := build(); err != nil { + return "", nil, err + } else { + return sql, result.Parameters, nil + } + } + key := cypherTranslationCacheKey{ + query: trimmed, + graphID: graphID, + parameterType: translationParameterTypeKey(parameters), + policyIdentity: policyIdentity, + } + + s.lock.Lock() + if s.closed { + s.stats.Bypasses++ + s.lock.Unlock() + if result, sql, err := build(); err != nil { + return "", nil, err + } else { + return sql, result.Parameters, nil + } + } + if element, found := s.entries[key]; found { + s.stats.Hits++ + s.lru.MoveToFront(element) + value := element.Value.(cypherTranslationCacheValue) + s.lock.Unlock() + if bound, err := value.bind(parameters); err != nil { + return "", nil, err + } else { + return value.sql, bound, nil + } + } + if call, found := s.pending[key]; found { + s.stats.CoalescedMisses++ + s.lock.Unlock() + <-call.done + if call.err != nil { + return "", nil, call.err + } + if !call.cacheable { + if result, sql, err := build(); err != nil { + return "", nil, err + } else { + return sql, result.Parameters, nil + } + } + if bound, err := call.value.bind(parameters); err != nil { + return "", nil, err + } else { + return call.value.sql, bound, nil + } + } + + key.query = strings.Clone(key.query) + s.stats.Misses++ + call := &cypherTranslationCall{ + done: make(chan struct{}), + } + s.pending[key] = call + s.lock.Unlock() + + result, sql, err := build() + value := cypherTranslationCacheValue{ + key: key, + sql: sql, + parameterSources: cloneSources(result.ParameterSources), + } + cacheable := err == nil && cacheableTranslation(result, parameters) + + s.lock.Lock() + call.value, call.err, call.cacheable = value, err, cacheable + if cacheable && !s.closed { + element := s.lru.PushFront(value) + s.entries[key] = element + if s.lru.Len() > s.capacity { + evicted := s.lru.Back() + s.lru.Remove(evicted) + delete(s.entries, evicted.Value.(cypherTranslationCacheValue).key) + s.stats.Evictions++ + } + } else if err == nil { + s.stats.Bypasses++ + } + delete(s.pending, key) + close(call.done) + s.lock.Unlock() + + if err != nil { + return "", nil, err + } + + return sql, result.Parameters, nil +} + +// Stats returns a consistent snapshot of counters and current cache occupancy. +func (s *cypherTranslationCache) Stats() TranslationCacheStats { + if s == nil { + return TranslationCacheStats{} + } + s.lock.Lock() + defer s.lock.Unlock() + stats := s.stats + stats.Entries = len(s.entries) + stats.Pending = len(s.pending) + return stats +} + +// Close releases retained translations and prevents future builds from repopulating the cache. +func (s *cypherTranslationCache) Close() { + if s == nil { + return + } + s.lock.Lock() + s.closed = true + s.entries = nil + s.lru.Init() + s.lock.Unlock() +} diff --git a/drivers/pg/translation_cache_test.go b/drivers/pg/translation_cache_test.go new file mode 100644 index 00000000..53e76f99 --- /dev/null +++ b/drivers/pg/translation_cache_test.go @@ -0,0 +1,291 @@ +package pg + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + + "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/specterops/dawgs/drivers/pg/pgutil" + "github.com/stretchr/testify/require" +) + +// TestCypherTranslationCacheReturnsZeroValuesOnBuildError verifies failed builds do not leak partial SQL or parameter maps. +func TestCypherTranslationCacheReturnsZeroValuesOnBuildError(t *testing.T) { + cache := newCypherTranslationCache(2) + expectedErr := errors.New("translation failed") + + sql, parameters, err := cache.Translate("RETURN 1", 1, nil, func() (translate.Result, string, error) { + return translate.Result{ + Parameters: map[string]any{"partial": true}, + }, "partial sql", expectedErr + }) + + require.ErrorIs(t, err, expectedErr) + require.Empty(t, sql) + require.Nil(t, parameters) + require.Zero(t, cache.Stats().Entries) +} + +// TestCypherTranslationCacheRebindsTranslatedListParameters verifies a cached list translation uses values from the current caller. +func TestCypherTranslationCacheRebindsTranslatedListParameters(t *testing.T) { + cache := newCypherTranslationCache(2) + const cypherQuery = `MATCH (n) WHERE n.objectid IN $object_ids RETURN n` + mapper := pgutil.NewInMemoryKindMapper() + builds := 0 + + translateWith := func(parameters map[string]any) (string, map[string]any, error) { + parsed, err := frontend.ParseCypher(frontend.NewContext(), cypherQuery) + require.NoError(t, err) + return cache.Translate(cypherQuery, translate.DefaultGraphID, parameters, func() (translate.Result, string, error) { + builds++ + result, err := translate.Translate(context.Background(), parsed, mapper, parameters, translate.DefaultGraphID) + if err != nil { + return translate.Result{}, "", err + } + sql, err := translate.Translated(result) + if err != nil { + return translate.Result{}, "", err + } + + return result, sql, nil + }) + } + + _, first, err := translateWith(map[string]any{"object_ids": []any{}}) + require.NoError(t, err) + _, second, err := translateWith(map[string]any{"object_ids": []any{"selected"}}) + require.NoError(t, err) + _, third, err := translateWith(map[string]any{"object_ids": []any{"other"}}) + require.NoError(t, err) + require.Equal(t, 2, builds) + require.NotEqual(t, first, second) + require.NotEqual(t, second, third) +} + +// TestCypherTranslationCacheRebindsNamedParameters verifies generated SQL parameter names map back to fresh named values. +func TestCypherTranslationCacheRebindsNamedParameters(t *testing.T) { + cache := newCypherTranslationCache(2) + var builds int + build := func(value int64) func() (translate.Result, string, error) { + return func() (translate.Result, string, error) { + builds++ + return translate.Result{ + Parameters: map[string]any{"i0": value}, + ParameterSources: map[string]string{"i0": "id"}, + }, "select @i0", nil + } + } + + sql, parameters, err := cache.Translate(" MATCH (n) WHERE id(n) = $id RETURN n ", 1, map[string]any{"id": int64(1)}, build(1)) + require.NoError(t, err) + require.Equal(t, "select @i0", sql) + require.Equal(t, int64(1), parameters["i0"]) + + sql, parameters, err = cache.Translate("MATCH (n) WHERE id(n) = $id RETURN n", 1, map[string]any{"id": int64(2)}, build(999)) + require.NoError(t, err) + require.Equal(t, "select @i0", sql) + require.Equal(t, int64(2), parameters["i0"]) + require.Equal(t, 1, builds) + require.Equal(t, TranslationCacheStats{ + Hits: 1, + Misses: 1, + Entries: 1, + }, cache.Stats()) +} + +// TestCypherTranslationCacheSeparatesGraphAndParameterTypes verifies graph identity and negotiated types partition cache entries. +func TestCypherTranslationCacheSeparatesGraphAndParameterTypes(t *testing.T) { + cache := newCypherTranslationCache(4) + var builds int + build := func() (translate.Result, string, error) { + builds++ + return translate.Result{ + Parameters: map[string]any{}, + ParameterSources: map[string]string{}, + }, "select 1", nil + } + + _, _, err := cache.Translate("RETURN $value", 1, map[string]any{"value": int64(1)}, build) + require.NoError(t, err) + _, _, err = cache.Translate("RETURN $value", 2, map[string]any{"value": int64(1)}, build) + require.NoError(t, err) + _, _, err = cache.Translate("RETURN $value", 1, map[string]any{"value": "1"}, build) + require.NoError(t, err) + require.Equal(t, 3, builds) +} + +// TestCypherTranslationCacheSeparatesProductionPolicies verifies disabling a +// canary cannot reuse SQL compiled under an earlier selector generation. +func TestCypherTranslationCacheSeparatesProductionPolicies(t *testing.T) { + cache := newCypherTranslationCache(4) + builds := 0 + build := func(sql string) func() (translate.Result, string, error) { + return func() (translate.Result, string, error) { + builds++ + return translate.Result{Parameters: map[string]any{}, ParameterSources: map[string]string{}}, sql, nil + } + } + + first, _, err := cache.TranslateWithPolicy("RETURN 1", 1, nil, "candidate-g1", build("candidate")) + require.NoError(t, err) + incumbent, _, err := cache.TranslateWithPolicy("RETURN 1", 1, nil, "production-incumbent-v1", build("incumbent")) + require.NoError(t, err) + again, _, err := cache.TranslateWithPolicy("RETURN 1", 1, nil, "candidate-g1", build("wrong")) + require.NoError(t, err) + + require.Equal(t, "candidate", first) + require.Equal(t, "incumbent", incumbent) + require.Equal(t, "candidate", again) + require.Equal(t, 2, builds) +} + +// TestTranslationParameterTypeKeyIsDelimiterSafe verifies length-prefixed name and type components cannot collide. +func TestTranslationParameterTypeKeyIsDelimiterSafe(t *testing.T) { + first := translationParameterTypeKey(map[string]any{ + "a": int64(1), + "b": "value", + }) + second := translationParameterTypeKey(map[string]any{ + "a=int8;b": "value", + }) + require.NotEqual(t, first, second) +} + +// TestCypherTranslationCacheRejectsMissingParameterSources verifies incomplete source metadata bypasses retention. +func TestCypherTranslationCacheRejectsMissingParameterSources(t *testing.T) { + cache := newCypherTranslationCache(2) + var builds int + build := func() (translate.Result, string, error) { + builds++ + return translate.Result{ + Parameters: map[string]any{"i0": int64(1)}, + ParameterSources: map[string]string{"i0": "required"}, + }, "select @i0", nil + } + + for range 2 { + _, _, err := cache.Translate("RETURN $required", 1, map[string]any{"other": int64(1)}, build) + require.NoError(t, err) + } + + require.Equal(t, 2, builds) + require.Zero(t, cache.Stats().Entries) + require.Equal(t, uint64(2), cache.Stats().Bypasses) +} + +// TestCachedTranslationBindingFailsClosedOnMissingSource verifies a cache hit errors rather than binding an absent caller value. +func TestCachedTranslationBindingFailsClosedOnMissingSource(t *testing.T) { + value := cypherTranslationCacheValue{ + parameterSources: map[string]string{"i0": "required"}, + } + _, err := value.bind(map[string]any{"other": int64(1)}) + require.ErrorContains(t, err, "missing parameter source") +} + +// TestCypherTranslationCacheBypassesGeneratedParameters verifies translations with non-source parameters are rebuilt for each caller. +func TestCypherTranslationCacheBypassesGeneratedParameters(t *testing.T) { + cache := newCypherTranslationCache(2) + var builds int + build := func() (translate.Result, string, error) { + builds++ + return translate.Result{ + Parameters: map[string]any{"pi0": "insert into traversal_pair_filter ..."}, + }, "select @pi0", nil + } + + for range 2 { + _, _, err := cache.Translate("MATCH p = shortestPath((a)-[*]->(b)) RETURN p", 1, nil, build) + require.NoError(t, err) + } + require.Equal(t, 2, builds) + require.Equal(t, uint64(2), cache.Stats().Bypasses) + require.Zero(t, cache.Stats().Entries) +} + +// TestCypherTranslationCacheCoalescesConcurrentMisses verifies equivalent concurrent requests share one cacheable build. +func TestCypherTranslationCacheCoalescesConcurrentMisses(t *testing.T) { + cache := newCypherTranslationCache(2) + const workers = 16 + var builds atomic.Int64 + start := make(chan struct{}) + release := make(chan struct{}) + build := func() (translate.Result, string, error) { + if builds.Add(1) == 1 { + close(start) + } + <-release + return translate.Result{ + Parameters: map[string]any{}, + ParameterSources: map[string]string{}, + }, "select 1", nil + } + + var group sync.WaitGroup + group.Add(workers) + errs := make([]error, workers) + for idx := 0; idx < workers; idx++ { + go func(index int) { + defer group.Done() + _, _, errs[index] = cache.Translate("MATCH (n) RETURN n", 1, nil, build) + }(idx) + } + <-start + close(release) + group.Wait() + + for _, err := range errs { + require.NoError(t, err) + } + require.Equal(t, int64(1), builds.Load()) + require.Equal(t, uint64(workers-1), cache.Stats().Hits+cache.Stats().CoalescedMisses) +} + +// TestCypherTranslationCacheDoesNotShareUncacheableParametersWithWaiters verifies waiters rebuild results whose values cannot be rebound safely. +func TestCypherTranslationCacheDoesNotShareUncacheableParametersWithWaiters(t *testing.T) { + cache := newCypherTranslationCache(2) + start := make(chan struct{}) + release := make(chan struct{}) + var builds atomic.Int64 + build := func(value string, wait bool) func() (translate.Result, string, error) { + return func() (translate.Result, string, error) { + builds.Add(1) + if wait { + close(start) + <-release + } + return translate.Result{ + Parameters: map[string]any{"pi0": value}, + }, "select @pi0", nil + } + } + + var ( + first, second map[string]any + firstErr, secondErr error + ) + + done := make(chan struct{}) + go func() { + _, first, firstErr = cache.Translate("RETURN 1", 1, nil, build("first", true)) + close(done) + }() + <-start + secondDone := make(chan struct{}) + go func() { + _, second, secondErr = cache.Translate("RETURN 1", 1, nil, build("second", false)) + close(secondDone) + }() + close(release) + <-done + <-secondDone + + require.NoError(t, firstErr) + require.NoError(t, secondErr) + require.Equal(t, "first", first["pi0"]) + require.Equal(t, "second", second["pi0"]) + require.Equal(t, int64(2), builds.Load()) +} diff --git a/drivers/pg/traversal_policy.go b/drivers/pg/traversal_policy.go new file mode 100644 index 00000000..840947c5 --- /dev/null +++ b/drivers/pg/traversal_policy.go @@ -0,0 +1,405 @@ +package pg + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "slices" + "sort" + "strings" + + "github.com/jackc/pgx/v5" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" +) + +// TraversalPolicy is a default-off, query-allowlisted production canary. A +// generation is mandatory whenever a candidate is enabled and is included in +// the translation cache identity. +type TraversalPolicy struct { + Generation uint64 `json:"generation"` + PromotionManifestSHA256 string `json:"promotion_manifest_sha256"` + // PromotionManifestJSON is the exact verified authorization document. It + // is intentionally excluded from policy serialization; its digest and + // content-derived fields form the cache identity. + PromotionManifestJSON json.RawMessage `json:"-"` + QuerySHA256Allowlist []string `json:"query_sha256_allowlist"` + ShortestPathExecutor optimize.ShortestPathExecutor `json:"shortest_path_executor,omitempty"` + EnableExpansionOrientation bool `json:"enable_expansion_orientation,omitempty"` + DisableEndpointSeededReverse bool `json:"disable_endpoint_seeded_reverse,omitempty"` + DisableInlineASPDAG bool `json:"disable_inline_asp_dag,omitempty"` + DisableInlineSPWitness bool `json:"disable_inline_sp_witness,omitempty"` + compiledManifest traversalPromotionManifest + compiledBuckets map[string]traversalPromotionBucket + compiledIdentity string +} + +func (s TraversalPolicy) enabled() bool { + return s.ShortestPathExecutor != "" || s.EnableExpansionOrientation || s.DisableEndpointSeededReverse || s.DisableInlineASPDAG || s.DisableInlineSPWitness +} + +func (s TraversalPolicy) productionOptions(query string) translate.ProductionOptions { + manifest := s.compiledManifest + if manifest.SelectorVersion == "" && len(s.PromotionManifestJSON) > 0 { + manifest, _ = decodeTraversalPromotionManifest(s.PromotionManifestJSON) + } + selectorVersion := manifest.SelectorVersion + if selectorVersion == "" { + selectorVersion = fmt.Sprintf("traversal-kill-switch-g%d", s.Generation) + if s.DisableEndpointSeededReverse && !s.DisableInlineASPDAG { + selectorVersion = fmt.Sprintf("endpoint-seeded-kill-switch-g%d", s.Generation) + } else if s.DisableInlineASPDAG && !s.DisableEndpointSeededReverse { + selectorVersion = fmt.Sprintf("inline-asp-kill-switch-g%d", s.Generation) + } + } + options := translate.ProductionOptions{ + ShortestPathExecutor: s.ShortestPathExecutor, EnableExpansionOrientation: s.EnableExpansionOrientation, + DisableEndpointSeededReverse: s.DisableEndpointSeededReverse, + DisableInlineASPDAG: s.DisableInlineASPDAG, + DisableInlineSPWitness: s.DisableInlineSPWitness, + SelectorVersion: selectorVersion, + } + if s.ShortestPathExecutor == optimize.ShortestPathExecutorASPI1DAG || s.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness { + options.ShortestPathCaps = &translate.ProductionShortestPathCaps{ + StateLimit: manifest.Caps["state_limit"], + PredecessorLimit: manifest.Caps["predecessor_limit"], + EnumerationLimit: manifest.Caps["enumeration_limit"], + OutputBytesLimit: manifest.Caps["output_bytes_limit"], + } + queryDigest := TraversalPolicyQuerySHA256(query) + if bucket, found := s.compiledBuckets[queryDigest]; found { + options.AuthorizedBucket = &translate.ProductionTraversalBucket{ + Direction: bucket.Direction, + ObservationMode: bucket.ObservationMode, + MinimumDepth: bucket.MinimumDepth, + MaximumDepth: bucket.MaximumDepth, + RelationshipKindCount: bucket.RelationshipKindCount, + UntypedRelationship: bucket.UntypedRelationship, + } + } else { + for _, bucket := range manifest.Buckets { + if !slices.Contains(bucket.QuerySHA256, queryDigest) { + continue + } + options.AuthorizedBucket = &translate.ProductionTraversalBucket{ + Direction: bucket.Direction, ObservationMode: bucket.ObservationMode, + MinimumDepth: bucket.MinimumDepth, MaximumDepth: bucket.MaximumDepth, + RelationshipKindCount: bucket.RelationshipKindCount, UntypedRelationship: bucket.UntypedRelationship, + } + break + } + } + } + return options +} + +type traversalPromotionBucket struct { + QuerySHA256 []string `json:"query_sha256"` + QualificationSplit []string `json:"qualification_split"` + Direction string `json:"direction,omitempty"` + ObservationMode string `json:"observation_mode,omitempty"` + MinimumDepth int64 `json:"minimum_depth,omitempty"` + MaximumDepth int64 `json:"maximum_depth,omitempty"` + RelationshipKindCount int `json:"relationship_kind_count,omitempty"` + UntypedRelationship bool `json:"untyped_relationship,omitempty"` +} + +type traversalPromotionEvidence struct { + SHA256 string `json:"sha256"` +} + +type traversalPromotionManifest struct { + Version int `json:"version"` + Candidate string `json:"candidate"` + SelectorVersion string `json:"selector_version"` + ExecutionBoundary string `json:"execution_boundary"` + FallbackExecutor string `json:"fallback_executor,omitempty"` + SourceCommit string `json:"source_commit"` + SourceSHA256 string `json:"source_sha256"` + BinarySHA256 string `json:"binary_sha256"` + CorpusSHA256 string `json:"corpus_sha256"` + Caps map[string]int64 `json:"caps"` + Buckets []traversalPromotionBucket `json:"buckets"` + Evidence map[string]traversalPromotionEvidence `json:"evidence"` +} + +func decodeTraversalPromotionManifest(raw []byte) (traversalPromotionManifest, error) { + var manifest traversalPromotionManifest + if len(raw) == 0 { + return manifest, fmt.Errorf("enabled traversal policy requires the verified promotion manifest JSON") + } + if err := json.Unmarshal(raw, &manifest); err != nil { + return manifest, fmt.Errorf("decode promotion manifest: %w", err) + } + return manifest, nil +} + +func (s TraversalPolicy) validate() error { + if !s.enabled() { + return nil + } + if s.Generation == 0 { + return fmt.Errorf("enabled traversal policy requires a nonzero generation") + } + if s.ShortestPathExecutor == "" && !s.EnableExpansionOrientation && (s.DisableEndpointSeededReverse || s.DisableInlineASPDAG || s.DisableInlineSPWitness) { + return nil + } + if !lowerHexSHA256(s.PromotionManifestSHA256) { + return fmt.Errorf("enabled traversal policy requires a lowercase promotion manifest SHA-256 digest") + } + manifest, err := decodeTraversalPromotionManifest(s.PromotionManifestJSON) + if err != nil { + return err + } + digest := sha256.Sum256(s.PromotionManifestJSON) + if hex.EncodeToString(digest[:]) != s.PromotionManifestSHA256 { + return fmt.Errorf("promotion manifest content does not match its SHA-256 digest") + } + if manifest.Version != 2 || strings.TrimSpace(manifest.SelectorVersion) == "" { + return fmt.Errorf("promotion manifest requires version 2 and a selector version") + } + if strings.TrimSpace(manifest.SourceCommit) == "" || !lowerHexSHA256(manifest.SourceSHA256) || !lowerHexSHA256(manifest.BinarySHA256) || !lowerHexSHA256(manifest.CorpusSHA256) { + return fmt.Errorf("promotion manifest requires source commit and lowercase source, binary, and corpus SHA-256 digests") + } + expectedCandidate := string(s.ShortestPathExecutor) + if s.EnableExpansionOrientation { + expectedCandidate = "orientation-probe-v1" + } + if manifest.Candidate != expectedCandidate { + return fmt.Errorf("promotion manifest candidate %q does not authorize %q", manifest.Candidate, expectedCandidate) + } + expectedBoundary := "inline_statement" + if s.EnableExpansionOrientation { + expectedBoundary = "guarded_dual_arm" + } else if s.ShortestPathExecutor == optimize.ShortestPathExecutorASPI1DAG || s.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness { + expectedBoundary = "guarded_dual_arm" + } + if manifest.ExecutionBoundary != expectedBoundary { + return fmt.Errorf("promotion manifest execution boundary %q does not authorize %q", manifest.ExecutionBoundary, expectedBoundary) + } + if len(manifest.Caps) == 0 || len(manifest.Buckets) == 0 { + return fmt.Errorf("promotion manifest requires immutable caps and authorized buckets") + } + if s.EnableExpansionOrientation { + expectedCaps := map[string]int64{ + "root_row_limit": optimize.ExpansionSearchOrientationRootRowLimit, + "reverse_seed_row_limit": optimize.ExpansionSearchOrientationReverseSeedRowLimit, + "directional_degree_row_limit": optimize.ExpansionSearchOrientationDirectionalDegreeRowLimit, + "state_limit": optimize.ExpansionSearchOrientationStateLimit, + } + if len(manifest.Caps) != len(expectedCaps) { + return fmt.Errorf("orientation-probe-v1 promotion manifest requires exactly root-row, reverse-seed-row, directional-degree-row, and state caps") + } + for name, expected := range expectedCaps { + if actual, found := manifest.Caps[name]; !found || actual != expected { + return fmt.Errorf("orientation-probe-v1 promotion manifest requires %s=%d", name, expected) + } + } + if manifest.FallbackExecutor != string(optimize.ExpansionSearchStepwiseForward) { + return fmt.Errorf("orientation-probe-v1 promotion manifest requires fallback %q", optimize.ExpansionSearchStepwiseForward) + } + } + if s.ShortestPathExecutor == optimize.ShortestPathExecutorASPI1DAG { + expectedCaps := map[string]struct{}{ + "state_limit": {}, "predecessor_limit": {}, "enumeration_limit": {}, "output_bytes_limit": {}, + } + if len(manifest.Caps) != len(expectedCaps) { + return fmt.Errorf("ASP-I1 promotion manifest requires exactly state, predecessor, enumeration, and output-byte caps") + } + for name := range expectedCaps { + if manifest.Caps[name] <= 0 { + return fmt.Errorf("ASP-I1 promotion manifest requires positive %s", name) + } + } + if manifest.FallbackExecutor != string(optimize.ShortestPathExecutorASPA1DAG) { + return fmt.Errorf("ASP-I1 promotion manifest requires fallback %q", optimize.ShortestPathExecutorASPA1DAG) + } + for _, bucket := range manifest.Buckets { + if (bucket.Direction != "outbound" && bucket.Direction != "inbound") || bucket.ObservationMode != "all_paths" || bucket.MinimumDepth != 1 || bucket.MaximumDepth < 1 || bucket.MaximumDepth > 64 || bucket.RelationshipKindCount < 0 { + return fmt.Errorf("ASP-I1 promotion bucket does not match the supported directed all-paths depth envelope") + } + if bucket.UntypedRelationship != (bucket.RelationshipKindCount == 0) { + return fmt.Errorf("ASP-I1 promotion bucket relationship kind metadata is inconsistent") + } + } + } + if s.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness { + expectedCaps := map[string]struct{}{ + "state_limit": {}, "predecessor_limit": {}, "enumeration_limit": {}, "output_bytes_limit": {}, + } + if len(manifest.Caps) != len(expectedCaps) { + return fmt.Errorf("SP-I1 canonical promotion manifest requires exactly state, predecessor, enumeration, and output-byte caps") + } + for name := range expectedCaps { + if manifest.Caps[name] <= 0 { + return fmt.Errorf("SP-I1 canonical promotion manifest requires positive %s", name) + } + } + if manifest.FallbackExecutor != string(optimize.ShortestPathExecutorS4CanonicalWitness) { + return fmt.Errorf("SP-I1 canonical promotion manifest requires fallback %q", optimize.ShortestPathExecutorS4CanonicalWitness) + } + if manifest.SelectorVersion != optimize.ShortestPathSelectorStaticV6 { + return fmt.Errorf("SP-I1 canonical promotion manifest requires selector %q", optimize.ShortestPathSelectorStaticV6) + } + for _, bucket := range manifest.Buckets { + if bucket.Direction != "inbound" || bucket.ObservationMode != string(optimize.ShortestPathObservationOnePath) || + bucket.MinimumDepth != 1 || bucket.MaximumDepth != 64 || bucket.RelationshipKindCount != 1 || bucket.UntypedRelationship { + return fmt.Errorf("SP-I1 canonical promotion bucket must match the qualified inbound typed single-kind one-path depth 1..64 envelope") + } + } + } + manifestQueries := make([]string, 0) + for _, bucket := range manifest.Buckets { + if !slices.Contains(bucket.QualificationSplit, "training") || !slices.Contains(bucket.QualificationSplit, "holdout") { + return fmt.Errorf("each promotion bucket requires training and holdout qualification") + } + manifestQueries = append(manifestQueries, bucket.QuerySHA256...) + } + sort.Strings(manifestQueries) + manifestQueries = slices.Compact(manifestQueries) + policyQueries := append([]string(nil), s.QuerySHA256Allowlist...) + sort.Strings(policyQueries) + policyQueries = slices.Compact(policyQueries) + if !slices.Equal(manifestQueries, policyQueries) { + return fmt.Errorf("query allowlist must exactly match the promotion manifest buckets") + } + for _, role := range []string{"aa", "confirmation", "performance", "resource", "reference_closure", "operational"} { + if evidence, found := manifest.Evidence[role]; !found || !lowerHexSHA256(evidence.SHA256) { + return fmt.Errorf("promotion manifest requires digest-bound %s evidence", role) + } + } + if len(s.QuerySHA256Allowlist) == 0 { + return fmt.Errorf("enabled traversal policy requires a nonempty query SHA-256 allowlist") + } + if s.ShortestPathExecutor != "" && s.EnableExpansionOrientation { + return fmt.Errorf("one traversal policy generation may enable only one candidate family") + } + if s.ShortestPathExecutor != "" && !productionCanaryExecutor(s.ShortestPathExecutor) { + return fmt.Errorf("shortest-path executor %q is not production-canary eligible", s.ShortestPathExecutor) + } + for _, value := range s.QuerySHA256Allowlist { + if !lowerHexSHA256(value) { + return fmt.Errorf("query allowlist entry %q is not a SHA-256 digest", value) + } + } + return nil +} + +func lowerHexSHA256(value string) bool { + if value != strings.ToLower(value) { + return false + } + decoded, err := hex.DecodeString(value) + return err == nil && len(decoded) == sha256.Size +} + +func productionCanaryExecutor(executor optimize.ShortestPathExecutor) bool { + switch executor { + case optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, + optimize.ShortestPathExecutorASPI1DAG: + return true + default: + return false + } +} + +// TraversalPolicyQuerySHA256 returns the stable digest used by policy +// allowlists. Only surrounding whitespace is normalized. Collapsing interior +// whitespace is unsafe because whitespace inside string literals and escaped +// identifiers is semantically significant. +func TraversalPolicyQuerySHA256(query string) string { + normalized := strings.TrimSpace(query) + digest := sha256.Sum256([]byte(normalized)) + return hex.EncodeToString(digest[:]) +} + +// SetTraversalPolicy atomically replaces production canary selection. The +// zero value disables all candidates; old cached SQL becomes unreachable +// because the effective policy identity changes immediately. +func (s *Driver) SetTraversalPolicy(policy TraversalPolicy) error { + if s == nil || s.SchemaManager == nil { + return fmt.Errorf("PostgreSQL driver is not initialized") + } + if err := policy.validate(); err != nil { + return err + } + policy.QuerySHA256Allowlist = append([]string(nil), policy.QuerySHA256Allowlist...) + policy.PromotionManifestJSON = append(json.RawMessage(nil), policy.PromotionManifestJSON...) + sort.Strings(policy.QuerySHA256Allowlist) + policy.QuerySHA256Allowlist = slices.Compact(policy.QuerySHA256Allowlist) + policy.compiledBuckets = map[string]traversalPromotionBucket{} + if len(policy.PromotionManifestJSON) > 0 { + manifest, err := decodeTraversalPromotionManifest(policy.PromotionManifestJSON) + if err != nil { + return err + } + policy.compiledManifest = manifest + for _, bucket := range manifest.Buckets { + for _, queryDigest := range bucket.QuerySHA256 { + if _, duplicate := policy.compiledBuckets[queryDigest]; duplicate { + return fmt.Errorf("promotion manifest query %q is authorized by more than one bucket", queryDigest) + } + policy.compiledBuckets[queryDigest] = bucket + } + } + } + raw, _ := json.Marshal(policy) + digest := sha256.Sum256(raw) + policy.compiledIdentity = "production-policy-" + hex.EncodeToString(digest[:]) + s.traversalPolicyLock.Lock() + s.traversalPolicy = policy + s.traversalPolicyLock.Unlock() + return nil +} + +// TraversalPolicy returns an immutable snapshot of the active policy. +func (s *Driver) TraversalPolicy() TraversalPolicy { + if s == nil || s.SchemaManager == nil { + return TraversalPolicy{} + } + s.traversalPolicyLock.RLock() + defer s.traversalPolicyLock.RUnlock() + policy := s.traversalPolicy + policy.QuerySHA256Allowlist = append([]string(nil), policy.QuerySHA256Allowlist...) + policy.PromotionManifestJSON = append(json.RawMessage(nil), policy.PromotionManifestJSON...) + return policy +} + +func (s *SchemaManager) effectiveTraversalPolicy(query string, isolation pgx.TxIsoLevel) (TraversalPolicy, string) { + s.traversalPolicyLock.RLock() + policy := s.traversalPolicy + s.traversalPolicyLock.RUnlock() + if policy.DisableInlineASPDAG && policy.ShortestPathExecutor == optimize.ShortestPathExecutorASPI1DAG { + policy.ShortestPathExecutor = "" + } + if policy.DisableInlineSPWitness && policy.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness { + policy.ShortestPathExecutor = "" + } + + _, queryAuthorized := policy.compiledBuckets[TraversalPolicyQuerySHA256(query)] + effective := policy.enabled() && (policy.DisableEndpointSeededReverse || policy.DisableInlineASPDAG || policy.DisableInlineSPWitness || queryAuthorized) + if shortestPathExecutorRequiresStableSnapshot(policy.ShortestPathExecutor) && isolation != pgx.RepeatableRead && isolation != pgx.Serializable { + effective = false + } + if !effective { + return TraversalPolicy{}, "production-incumbent-v1" + } + return policy, policy.compiledIdentity +} + +func shortestPathExecutorRequiresStableSnapshot(executor optimize.ShortestPathExecutor) bool { + switch executor { + case optimize.ShortestPathExecutorB1AlternatingNodeDistance, + optimize.ShortestPathExecutorB1AlternatingNodeWitness, + optimize.ShortestPathExecutorB2SmallerCurrentLevelDistance, + optimize.ShortestPathExecutorB2SmallerCurrentLevelWitness, + optimize.ShortestPathExecutorASPB1AlternatingNodeDAG, + optimize.ShortestPathExecutorASPB2SmallerCurrentLevelDAG, + optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, + optimize.ShortestPathExecutorASPI1DAG: + return true + default: + return false + } +} diff --git a/drivers/pg/traversal_policy_test.go b/drivers/pg/traversal_policy_test.go new file mode 100644 index 00000000..84b86a5a --- /dev/null +++ b/drivers/pg/traversal_policy_test.go @@ -0,0 +1,317 @@ +package pg + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "testing" + + "github.com/jackc/pgx/v5" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/stretchr/testify/require" +) + +func testTraversalPolicy(query string, executor optimize.ShortestPathExecutor, orientation bool) TraversalPolicy { + candidate := string(executor) + if orientation { + candidate = "orientation-probe-v1" + } + queryDigest := TraversalPolicyQuerySHA256(query) + evidence := map[string]map[string]string{} + for _, role := range []string{"aa", "confirmation", "performance", "resource", "reference_closure", "operational"} { + evidence[role] = map[string]string{"sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"} + } + boundary := map[bool]string{true: "guarded_dual_arm", false: "inline_statement"}[orientation] + selectorVersion := "test-selector-v1" + caps := map[string]int64{"state_limit": 1000} + bucket := map[string]any{"query_sha256": []string{queryDigest}, "qualification_split": []string{"training", "holdout"}} + fallback := "" + if orientation { + caps = map[string]int64{ + "root_row_limit": optimize.ExpansionSearchOrientationRootRowLimit, + "reverse_seed_row_limit": optimize.ExpansionSearchOrientationReverseSeedRowLimit, + "directional_degree_row_limit": optimize.ExpansionSearchOrientationDirectionalDegreeRowLimit, + "state_limit": optimize.ExpansionSearchOrientationStateLimit, + } + fallback = string(optimize.ExpansionSearchStepwiseForward) + } + if executor == optimize.ShortestPathExecutorASPI1DAG { + boundary = "guarded_dual_arm" + caps = map[string]int64{ + "state_limit": 1000, "predecessor_limit": 900, "enumeration_limit": 800, "output_bytes_limit": 70000, + } + fallback = string(optimize.ShortestPathExecutorASPA1DAG) + bucket["direction"] = "outbound" + bucket["observation_mode"] = "all_paths" + bucket["minimum_depth"] = 1 + bucket["maximum_depth"] = 4 + bucket["relationship_kind_count"] = 1 + bucket["untyped_relationship"] = false + } + if executor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness { + selectorVersion = optimize.ShortestPathSelectorStaticV6 + boundary = "guarded_dual_arm" + caps = map[string]int64{ + "state_limit": 1000, "predecessor_limit": 900, "enumeration_limit": 800, "output_bytes_limit": 70000, + } + fallback = string(optimize.ShortestPathExecutorS4CanonicalWitness) + bucket["direction"] = "inbound" + bucket["observation_mode"] = "one_path" + bucket["minimum_depth"] = 1 + bucket["maximum_depth"] = 64 + bucket["relationship_kind_count"] = 1 + bucket["untyped_relationship"] = false + } + raw, err := json.Marshal(map[string]any{ + "version": 2, "candidate": candidate, "selector_version": selectorVersion, + "source_commit": "deadbeef", "source_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "binary_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "corpus_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "execution_boundary": boundary, + "fallback_executor": fallback, + "caps": caps, + "buckets": []map[string]any{bucket}, + "evidence": evidence, + }) + if err != nil { + panic(err) + } + digest := sha256.Sum256(raw) + return TraversalPolicy{ + Generation: 1, PromotionManifestSHA256: hex.EncodeToString(digest[:]), PromotionManifestJSON: raw, + QuerySHA256Allowlist: []string{queryDigest}, ShortestPathExecutor: executor, EnableExpansionOrientation: orientation, + } +} + +func rewriteTestTraversalPolicyManifest(t *testing.T, policy TraversalPolicy, mutate func(*traversalPromotionManifest)) TraversalPolicy { + t.Helper() + + var manifest traversalPromotionManifest + require.NoError(t, json.Unmarshal(policy.PromotionManifestJSON, &manifest)) + mutate(&manifest) + + raw, err := json.Marshal(manifest) + require.NoError(t, err) + digest := sha256.Sum256(raw) + policy.PromotionManifestJSON = raw + policy.PromotionManifestSHA256 = hex.EncodeToString(digest[:]) + return policy +} + +func TestTraversalPolicyAuthorizesGuardedInlineASPOnlyWithStableSnapshotAndExactCaps(t *testing.T) { + driver := &Driver{SchemaManager: NewSchemaManager(nil, 0)} + query := "MATCH p = allShortestPaths((s)-[:MemberOf*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p" + policy := testTraversalPolicy(query, optimize.ShortestPathExecutorASPI1DAG, false) + require.NoError(t, driver.SetTraversalPolicy(policy)) + + effective, _ := driver.SchemaManager.effectiveTraversalPolicy(query, pgx.ReadCommitted) + require.False(t, effective.enabled()) + effective, _ = driver.SchemaManager.effectiveTraversalPolicy(query, pgx.RepeatableRead) + require.Equal(t, optimize.ShortestPathExecutorASPI1DAG, effective.ShortestPathExecutor) + options := effective.productionOptions(query) + require.Equal(t, int64(1000), options.ShortestPathCaps.StateLimit) + require.Equal(t, int64(900), options.ShortestPathCaps.PredecessorLimit) + require.Equal(t, int64(800), options.ShortestPathCaps.EnumerationLimit) + require.Equal(t, int64(70000), options.ShortestPathCaps.OutputBytesLimit) + require.Equal(t, "outbound", options.AuthorizedBucket.Direction) +} + +func TestTraversalPolicyInlineASPKillSwitchRequiresNoEvidence(t *testing.T) { + driver := &Driver{SchemaManager: NewSchemaManager(nil, 0)} + require.NoError(t, driver.SetTraversalPolicy(TraversalPolicy{Generation: 9, DisableInlineASPDAG: true})) + effective, identity := driver.SchemaManager.effectiveTraversalPolicy("MATCH (n) RETURN n", pgx.ReadCommitted) + require.True(t, effective.DisableInlineASPDAG) + require.Empty(t, effective.ShortestPathExecutor) + require.Contains(t, identity, "production-policy-") + require.Equal(t, "inline-asp-kill-switch-g9", effective.productionOptions("MATCH (n) RETURN n").SelectorVersion) +} + +func TestTraversalPolicyIsAllowlistedSnapshotSafeAndImmediatelyReversible(t *testing.T) { + driver := &Driver{SchemaManager: NewSchemaManager(nil, 0)} + query := "MATCH p = shortestPath((s)<-[:MemberOf*1..64]-(e)) RETURN p" + policy := testTraversalPolicy(query, optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, false) + require.NoError(t, driver.SetTraversalPolicy(policy)) + + effective, _ := driver.SchemaManager.effectiveTraversalPolicy(query, pgx.ReadCommitted) + require.False(t, effective.enabled()) + effective, candidateKey := driver.SchemaManager.effectiveTraversalPolicy(query, pgx.RepeatableRead) + require.True(t, effective.enabled()) + require.Contains(t, candidateKey, "production-policy-") + + effective, _ = driver.SchemaManager.effectiveTraversalPolicy("RETURN 1", pgx.RepeatableRead) + require.False(t, effective.enabled(), "queries outside the allowlist remain on incumbents") + + require.NoError(t, driver.SetTraversalPolicy(TraversalPolicy{})) + effective, rollbackKey := driver.SchemaManager.effectiveTraversalPolicy(query, pgx.RepeatableRead) + require.False(t, effective.enabled()) + require.Equal(t, "production-incumbent-v1", rollbackKey) + require.NotEqual(t, candidateKey, rollbackKey) +} + +func TestTraversalPolicyFailsClosed(t *testing.T) { + driver := &Driver{SchemaManager: NewSchemaManager(nil, 0)} + require.Error(t, driver.SetTraversalPolicy(TraversalPolicy{Generation: 1, EnableExpansionOrientation: true})) + require.Error(t, driver.SetTraversalPolicy(TraversalPolicy{ + Generation: 1, PromotionManifestSHA256: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", QuerySHA256Allowlist: []string{"not-a-digest"}, EnableExpansionOrientation: true, + })) + require.Error(t, driver.SetTraversalPolicy(TraversalPolicy{ + Generation: 1, PromotionManifestSHA256: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", QuerySHA256Allowlist: []string{TraversalPolicyQuerySHA256("RETURN 1")}, + ShortestPathExecutor: optimize.ShortestPathExecutorS3Unidirectional, + })) + require.Error(t, driver.SetTraversalPolicy(TraversalPolicy{ + Generation: 1, QuerySHA256Allowlist: []string{TraversalPolicyQuerySHA256("RETURN 1")}, EnableExpansionOrientation: true, + }), "an enabled production policy must be traceable to verified evidence") + require.ErrorContains(t, driver.SetTraversalPolicy(testTraversalPolicy( + "MATCH p = shortestPath((s)-[*1..4]->(e)) RETURN length(p)", + optimize.ShortestPathExecutorI1CanonicalDistance, + false, + )), "not production-canary eligible") +} + +func TestTraversalPolicyCanonicalSPRequiresExactStaticV6Envelope(t *testing.T) { + query := "MATCH p = shortestPath((s)<-[:MemberOf*1..64]-(e)) RETURN p" + valid := testTraversalPolicy(query, optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, false) + require.NoError(t, (&Driver{SchemaManager: NewSchemaManager(nil, 0)}).SetTraversalPolicy(valid)) + + tests := map[string]struct { + mutate func(*traversalPromotionManifest) + errorContains string + }{ + "selector": { + mutate: func(manifest *traversalPromotionManifest) { manifest.SelectorVersion = "sp-static-v5-contained" }, + errorContains: `requires selector "sp-static-v6"`, + }, + "outbound": { + mutate: func(manifest *traversalPromotionManifest) { manifest.Buckets[0].Direction = "outbound" }, + errorContains: "qualified inbound typed single-kind one-path depth 1..64 envelope", + }, + "shallower maximum": { + mutate: func(manifest *traversalPromotionManifest) { manifest.Buckets[0].MaximumDepth = 63 }, + errorContains: "qualified inbound typed single-kind one-path depth 1..64 envelope", + }, + "multiple kinds": { + mutate: func(manifest *traversalPromotionManifest) { manifest.Buckets[0].RelationshipKindCount = 2 }, + errorContains: "qualified inbound typed single-kind one-path depth 1..64 envelope", + }, + "untyped": { + mutate: func(manifest *traversalPromotionManifest) { + manifest.Buckets[0].RelationshipKindCount = 0 + manifest.Buckets[0].UntypedRelationship = true + }, + errorContains: "qualified inbound typed single-kind one-path depth 1..64 envelope", + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + policy := rewriteTestTraversalPolicyManifest(t, valid, test.mutate) + driver := &Driver{SchemaManager: NewSchemaManager(nil, 0)} + require.ErrorContains(t, driver.SetTraversalPolicy(policy), test.errorContains) + }) + } +} + +func TestTraversalPolicyQuerySHA256PreservesSemanticWhitespace(t *testing.T) { + require.Equal(t, + TraversalPolicyQuerySHA256(" MATCH (n) RETURN n "), + TraversalPolicyQuerySHA256("MATCH (n) RETURN n"), + ) + require.NotEqual(t, + TraversalPolicyQuerySHA256(`RETURN "a b"`), + TraversalPolicyQuerySHA256(`RETURN "a b"`), + ) + require.NotEqual(t, + TraversalPolicyQuerySHA256("MATCH (`a b`) RETURN `a b`"), + TraversalPolicyQuerySHA256("MATCH (`a b`) RETURN `a b`"), + ) +} + +func TestTraversalPolicyAllowsGuardedOrientationWithoutSnapshotUpgrade(t *testing.T) { + driver := &Driver{SchemaManager: NewSchemaManager(nil, 0)} + query := "MATCH (r)-[:Expand*0..16]->()-[:Suffix]->(e) RETURN id(e)" + policy := testTraversalPolicy(query, "", true) + policy.Generation = 2 + require.NoError(t, driver.SetTraversalPolicy(policy)) + effective, identity := driver.SchemaManager.effectiveTraversalPolicy(query, pgx.ReadCommitted) + require.True(t, effective.EnableExpansionOrientation) + require.Contains(t, identity, "production-policy-") +} + +func TestTraversalPolicyGuardedOrientationRequiresExactManifestContract(t *testing.T) { + query := "MATCH (r)-[:Expand*0..16]->()-[:Suffix]->(e) RETURN id(e)" + valid := testTraversalPolicy(query, "", true) + require.NoError(t, (&Driver{SchemaManager: NewSchemaManager(nil, 0)}).SetTraversalPolicy(valid)) + + tests := map[string]struct { + mutate func(*traversalPromotionManifest) + errorContains string + }{ + "candidate": { + mutate: func(manifest *traversalPromotionManifest) { manifest.Candidate = "orientation-probe-v2" }, + errorContains: `candidate "orientation-probe-v2" does not authorize "orientation-probe-v1"`, + }, + "execution boundary": { + mutate: func(manifest *traversalPromotionManifest) { manifest.ExecutionBoundary = "inline_statement" }, + errorContains: `execution boundary "inline_statement" does not authorize "guarded_dual_arm"`, + }, + "missing cap": { + mutate: func(manifest *traversalPromotionManifest) { + delete(manifest.Caps, "root_row_limit") + }, + errorContains: "requires exactly root-row, reverse-seed-row, directional-degree-row, and state caps", + }, + "extra cap": { + mutate: func(manifest *traversalPromotionManifest) { + manifest.Caps["survival_row_limit"] = 1 + }, + errorContains: "requires exactly root-row, reverse-seed-row, directional-degree-row, and state caps", + }, + "root cap": { + mutate: func(manifest *traversalPromotionManifest) { + manifest.Caps["root_row_limit"] = optimize.ExpansionSearchOrientationRootRowLimit + 1 + }, + errorContains: "requires root_row_limit=512", + }, + "reverse seed cap": { + mutate: func(manifest *traversalPromotionManifest) { + manifest.Caps["reverse_seed_row_limit"] = optimize.ExpansionSearchOrientationReverseSeedRowLimit + 1 + }, + errorContains: "requires reverse_seed_row_limit=512", + }, + "directional degree cap": { + mutate: func(manifest *traversalPromotionManifest) { + manifest.Caps["directional_degree_row_limit"] = optimize.ExpansionSearchOrientationDirectionalDegreeRowLimit + 1 + }, + errorContains: "requires directional_degree_row_limit=16384", + }, + "state cap": { + mutate: func(manifest *traversalPromotionManifest) { + manifest.Caps["state_limit"] = optimize.ExpansionSearchOrientationStateLimit + 1 + }, + errorContains: "requires state_limit=4096", + }, + "fallback": { + mutate: func(manifest *traversalPromotionManifest) { + manifest.FallbackExecutor = string(optimize.ExpansionSearchSuffixSeededReverse) + }, + errorContains: `requires fallback "EXPANSION-STEPWISE-FORWARD"`, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + policy := rewriteTestTraversalPolicyManifest(t, valid, test.mutate) + driver := &Driver{SchemaManager: NewSchemaManager(nil, 0)} + require.ErrorContains(t, driver.SetTraversalPolicy(policy), test.errorContains) + }) + } +} + +func TestTraversalPolicyEndpointSeededKillSwitchRequiresNoPromotionEvidence(t *testing.T) { + driver := &Driver{SchemaManager: NewSchemaManager(nil, 0)} + require.NoError(t, driver.SetTraversalPolicy(TraversalPolicy{Generation: 7, DisableEndpointSeededReverse: true})) + effective, identity := driver.SchemaManager.effectiveTraversalPolicy("MATCH (n) RETURN n", pgx.ReadCommitted) + require.True(t, effective.DisableEndpointSeededReverse) + require.Contains(t, identity, "production-policy-") + require.Equal(t, "endpoint-seeded-kill-switch-g7", effective.productionOptions("MATCH (n) RETURN n").SelectorVersion) +} diff --git a/drivers/pg/types.go b/drivers/pg/types.go index 211049a9..d1f3ceb2 100644 --- a/drivers/pg/types.go +++ b/drivers/pg/types.go @@ -7,14 +7,48 @@ import ( "github.com/specterops/dawgs/graph" ) +// edgeComposite is the ordered Go representation of PostgreSQL's edge composite type. type edgeComposite struct { - ID int64 - StartID int64 - EndID int64 - KindID int16 + // ID is the database identifier of the decoded relationship. + ID int64 + + // StartID is the database identifier of the relationship's start node. + StartID int64 + + // EndID is the database identifier of the relationship's end node. + EndID int64 + + // KindID is the PostgreSQL int2 identifier of the relationship kind. + KindID int16 + + // Properties contains the relationship's decoded JSON property values. Properties map[string]any } +// ScanNull rejects a null edge because the owned scalar representation has no null state. +func (s *edgeComposite) ScanNull() error { + return fmt.Errorf("cannot scan NULL into %T", s) +} + +// ScanIndex returns the destination for a PostgreSQL edge field in schema order. +func (s *edgeComposite) ScanIndex(index int) any { + switch index { + case 0: + return &s.ID + case 1: + return &s.StartID + case 2: + return &s.EndID + case 3: + return &s.KindID + case 4: + return &s.Properties + default: + return fmt.Errorf("%T only has 5 fields: index %d is out of bounds", s, index) + } +} + +// castSlice copies either a typed slice or a pgx []any representation into []T. func castSlice[T any](raw any) ([]T, error) { switch rawSlice := raw.(type) { case []T: @@ -38,6 +72,7 @@ func castSlice[T any](raw any) ([]T, error) { } } +// castMapValueAsSliceOf retrieves key from a fallback composite map and converts its value to []T. func castMapValueAsSliceOf[T any](compositeMap map[string]any, key string) ([]T, error) { if src, hasKey := compositeMap[key]; !hasKey { return nil, fmt.Errorf("composite map does not contain expected key %s", key) @@ -46,6 +81,7 @@ func castMapValueAsSliceOf[T any](compositeMap map[string]any, key string) ([]T, } } +// castAndAssignMapValue assigns a fallback composite-map field to dst, allowing lossless widening of integer values. func castAndAssignMapValue[T any](compositeMap map[string]any, key string, dst *T) error { if src, hasKey := compositeMap[key]; !hasKey { return fmt.Errorf("composite map does not contain expected key %s", key) @@ -124,52 +160,65 @@ func castAndAssignMapValue[T any](compositeMap map[string]any, key string, dst * return nil } +// nodeCompositesFromRaw converts typed or pgx fallback arrays into owned node composites. func nodeCompositesFromRaw(raw any) ([]nodeComposite, error) { - rawNodes, typeOK := raw.([]any) - if !typeOK { - return nil, fmt.Errorf("expected raw node composite array type []any but received %T", raw) - } - - nodes := make([]nodeComposite, 0, len(rawNodes)) - for _, rawNode := range rawNodes { - compositeMap, typeOK := rawNode.(map[string]any) - if !typeOK { - return nil, fmt.Errorf("unexpected type for raw node: %T", rawNode) - } - - var node nodeComposite - if err := node.FromMap(compositeMap); err != nil { - return nil, err + switch rawNodes := raw.(type) { + case []nodeComposite: + return rawNodes, nil + case []any: + nodes := make([]nodeComposite, len(rawNodes)) + for idx, rawNode := range rawNodes { + if node, typeOK := nodeCompositeFromRaw(rawNode); !typeOK { + return nil, fmt.Errorf("unexpected type for raw node at index %d: %T", idx, rawNode) + } else { + nodes[idx] = node + } } - nodes = append(nodes, node) + return nodes, nil + default: + return nil, fmt.Errorf("expected raw node composite array type []nodeComposite or []any but received %T", raw) } - - return nodes, nil } +// edgeCompositesFromRaw converts typed or pgx fallback arrays into owned edge composites. func edgeCompositesFromRaw(raw any) ([]edgeComposite, error) { - rawEdges, typeOK := raw.([]any) - if !typeOK { - return nil, fmt.Errorf("expected raw edge composite array type []any but received %T", raw) + switch rawEdges := raw.(type) { + case []edgeComposite: + return rawEdges, nil + case []any: + edges := make([]edgeComposite, len(rawEdges)) + for idx, rawEdge := range rawEdges { + if edge, typeOK := edgeCompositeFromRaw(rawEdge); !typeOK { + return nil, fmt.Errorf("unexpected type for raw edge at index %d: %T", idx, rawEdge) + } else { + edges[idx] = edge + } + } + + return edges, nil + default: + return nil, fmt.Errorf("expected raw edge composite array type []edgeComposite or []any but received %T", raw) } +} - edges := make([]edgeComposite, 0, len(rawEdges)) - for _, rawEdge := range rawEdges { - compositeMap, typeOK := rawEdge.(map[string]any) - if !typeOK { - return nil, fmt.Errorf("unexpected type for raw edge: %T", rawEdge) +// edgeCompositeFromRaw accepts an owned edge value, pointer, or pgx fallback map. +func edgeCompositeFromRaw(raw any) (edgeComposite, bool) { + switch typedRaw := raw.(type) { + case edgeComposite: + return typedRaw, true + case *edgeComposite: + if typedRaw != nil { + return *typedRaw, true } - + case map[string]any: var edge edgeComposite - if err := edge.FromMap(compositeMap); err != nil { - return nil, err + if edge.TryMap(typedRaw) { + return edge, true } - - edges = append(edges, edge) } - return edges, nil + return edgeComposite{}, false } func (s *edgeComposite) TryMap(compositeMap map[string]any) bool { @@ -215,12 +264,56 @@ func (s *edgeComposite) ToRelationship(ctx context.Context, kindMapper KindMappe return nil } +// nodeComposite is the ordered Go representation of PostgreSQL's node composite type. type nodeComposite struct { - ID int64 - KindIDs []int16 + // ID is the database identifier of the decoded node. + ID int64 + + // KindIDs contains the PostgreSQL int2 identifiers of the node's kinds. + KindIDs []int16 + + // Properties contains the node's decoded JSON property values. Properties map[string]any } +// ScanNull rejects a null node because the owned scalar representation has no null state. +func (s *nodeComposite) ScanNull() error { + return fmt.Errorf("cannot scan NULL into %T", s) +} + +// ScanIndex returns the destination for a PostgreSQL node field in schema order. +func (s *nodeComposite) ScanIndex(index int) any { + switch index { + case 0: + return &s.ID + case 1: + return &s.KindIDs + case 2: + return &s.Properties + default: + return fmt.Errorf("%T only has 3 fields: index %d is out of bounds", s, index) + } +} + +// nodeCompositeFromRaw accepts an owned node value, pointer, or pgx fallback map. +func nodeCompositeFromRaw(raw any) (nodeComposite, bool) { + switch typedRaw := raw.(type) { + case nodeComposite: + return typedRaw, true + case *nodeComposite: + if typedRaw != nil { + return *typedRaw, true + } + case map[string]any: + var node nodeComposite + if node.TryMap(typedRaw) { + return node, true + } + } + + return nodeComposite{}, false +} + func (s *nodeComposite) TryMap(compositeMap map[string]any) bool { return s.FromMap(compositeMap) == nil } @@ -256,21 +349,62 @@ func (s *nodeComposite) ToNode(ctx context.Context, kindMapper KindMapper, node return nil } +// pathComposite is the ordered Go representation of PostgreSQL's path composite type. type pathComposite struct { + // Nodes contains the path's decoded nodes in traversal order. Nodes []nodeComposite + + // Edges contains the path's decoded relationships in traversal order. Edges []edgeComposite } +// ScanNull rejects a null path because the owned scalar representation has no null state. +func (s *pathComposite) ScanNull() error { + return fmt.Errorf("cannot scan NULL into %T", s) +} + +// ScanIndex returns the destination for a PostgreSQL path field in schema order. +func (s *pathComposite) ScanIndex(index int) any { + switch index { + case 0: + return &s.Nodes + case 1: + return &s.Edges + default: + return fmt.Errorf("%T only has 2 fields: index %d is out of bounds", s, index) + } +} + +// pathCompositeFromRaw accepts an owned path value, pointer, or pgx fallback map. +func pathCompositeFromRaw(raw any) (pathComposite, bool) { + switch typedRaw := raw.(type) { + case pathComposite: + return typedRaw, true + case *pathComposite: + if typedRaw != nil { + return *typedRaw, true + } + case map[string]any: + var path pathComposite + if path.TryMap(typedRaw) { + return path, true + } + } + + return pathComposite{}, false +} + func (s *pathComposite) TryMap(compositeMap map[string]any) bool { return s.FromMap(compositeMap) == nil } +// FromMap populates a path composite from pgx's fallback map representation of its node and edge arrays. func (s *pathComposite) FromMap(compositeMap map[string]any) error { if rawNodes, hasNodes := compositeMap["nodes"]; hasNodes { if nodes, err := nodeCompositesFromRaw(rawNodes); err != nil { return err } else { - s.Nodes = append(s.Nodes, nodes...) + s.Nodes = nodes } } @@ -278,7 +412,7 @@ func (s *pathComposite) FromMap(compositeMap map[string]any) error { if edges, err := edgeCompositesFromRaw(rawEdges); err != nil { return err } else { - s.Edges = append(s.Edges, edges...) + s.Edges = edges } } diff --git a/integration/cypher_template_test.go b/integration/cypher_template_test.go index 6fd5b5f6..4fd534ff 100644 --- a/integration/cypher_template_test.go +++ b/integration/cypher_template_test.go @@ -31,42 +31,97 @@ import ( "github.com/specterops/dawgs/graph" "github.com/specterops/dawgs/opengraph" + "github.com/specterops/dawgs/testutil" ) +// cypherTemplateFile describes the ordinary and metamorphic query families loaded from one template JSON file. type cypherTemplateFile struct { - Families []cypherTemplateFamily `json:"families,omitempty"` + // Families contains independently asserted query-template families. + Families []cypherTemplateFamily `json:"families,omitempty"` + + // Metamorphic contains families whose query variants must produce equivalent results. Metamorphic []cypherMetamorphicFamily `json:"metamorphic,omitempty"` - path string + + // path records the source file for subtest naming and diagnostics. + path string } +// cypherTemplateFamily combines a fixture and query template with the variants asserted against it. type cypherTemplateFamily struct { - Name string `json:"name"` - Fixture *opengraph.Graph `json:"fixture"` - Template string `json:"template"` - Params map[string]any `json:"params,omitempty"` + // Name identifies the family in test output. + Name string `json:"name"` + + // Fixture is loaded transactionally for every variant. + Fixture *opengraph.Graph `json:"fixture"` + + // Template is the Cypher source rendered with each variant's Vars. + Template string `json:"template"` + + // Params supplies parameters shared by every variant. + Params testutil.Params `json:"params,omitempty"` + + // NodeParams maps shared parameter names to fixture node identifiers. + NodeParams map[string]string `json:"node_params,omitempty"` + + // NodeListParams maps shared parameter names to lists of fixture node identifiers. + NodeListParams map[string][]string `json:"node_list_params,omitempty"` + + // Variants enumerates template substitutions and expected results. Variants []cypherTemplateVariant `json:"variants"` } +// cypherTemplateVariant supplies one rendering and assertion for a query-template family. type cypherTemplateVariant struct { - Name string `json:"name"` - Vars map[string]string `json:"vars,omitempty"` - Params map[string]any `json:"params,omitempty"` - Assert json.RawMessage `json:"assert"` + // Name identifies the variant in test output. + Name string `json:"name"` + + // Vars contains text substitutions applied to the Cypher template. + Vars map[string]string `json:"vars,omitempty"` + + // Params augments or overrides family-level query parameters. + Params testutil.Params `json:"params,omitempty"` + + // NodeParams augments or overrides family-level fixture-node parameters. + NodeParams map[string]string `json:"node_params,omitempty"` + + // NodeListParams augments or overrides family-level fixture-node-list parameters. + NodeListParams map[string][]string `json:"node_list_params,omitempty"` + + // Assert encodes the expected primary query result. + Assert json.RawMessage `json:"assert"` + + // PostAssertions contains state checks run after the primary query drains. + PostAssertions []stateAssertion `json:"post_assertions,omitempty"` } +// cypherMetamorphicFamily describes queries that must agree under the selected comparison modes. type cypherMetamorphicFamily struct { - Name string `json:"name"` - Fixture *opengraph.Graph `json:"fixture"` - Compare comparisonModes `json:"compare"` + // Name identifies the family in test output. + Name string `json:"name"` + + // Fixture is loaded once for the family's equivalence comparison. + Fixture *opengraph.Graph `json:"fixture"` + + // Compare selects the result dimensions used to establish equivalence. + Compare comparisonModes `json:"compare"` + + // Queries contains the query variants compared with the baseline. Queries []cypherMetamorphicQuery `json:"queries"` } +// cypherMetamorphicQuery is one named Cypher statement and parameter set in an equivalence family. type cypherMetamorphicQuery struct { - Name string `json:"name"` - Cypher string `json:"cypher"` - Params map[string]any `json:"params,omitempty"` + // Name identifies the query in test output. + Name string `json:"name"` + + // Cypher is the statement executed for this variant. + Cypher string `json:"cypher"` + + // Params contains the statement's query parameters. + Params testutil.Params `json:"params,omitempty"` } +// TestCypherTemplates renders every template variant and verifies its query and post-state assertions against the shared fixture. func TestCypherTemplates(t *testing.T) { templateFiles := loadCypherTemplateFiles(t) nodeKinds, edgeKinds := cypherTemplateKinds(templateFiles) @@ -85,10 +140,13 @@ func TestCypherTemplates(t *testing.T) { cypher = renderCypherTemplate(t, family.Template, variant.Vars) check = parseAssertion(t, variant.Assert) tc = testCase{ - Name: variant.Name, - Cypher: cypher, - Params: mergeParams(family.Params, variant.Params), - Fixture: family.Fixture, + Name: variant.Name, + Cypher: cypher, + Params: mergeParams(family.Params, variant.Params), + NodeParams: mergeStringMap(family.NodeParams, variant.NodeParams), + NodeListParams: mergeStringListMap(family.NodeListParams, variant.NodeListParams), + Fixture: family.Fixture, + PostAssertions: variant.PostAssertions, } ) @@ -107,6 +165,7 @@ func TestCypherTemplates(t *testing.T) { } } +// loadCypherTemplateFiles reads and decodes every JSON template file, preserving each source path for diagnostics. func loadCypherTemplateFiles(t *testing.T) []cypherTemplateFile { t.Helper() @@ -137,6 +196,8 @@ func loadCypherTemplateFiles(t *testing.T) []cypherTemplateFile { return templateFiles } +// cypherTemplateKinds collects the node and relationship kinds used by every +// inline template and metamorphic fixture. func cypherTemplateKinds(templateFiles []cypherTemplateFile) (graph.Kinds, graph.Kinds) { var nodeKinds, edgeKinds graph.Kinds @@ -161,6 +222,8 @@ func cypherTemplateKinds(templateFiles []cypherTemplateFile) (graph.Kinds, graph return nodeKinds, edgeKinds } +// renderCypherTemplate replaces named placeholders and fails if any placeholder +// remains unresolved. func renderCypherTemplate(t *testing.T, template string, vars map[string]string) string { t.Helper() @@ -176,6 +239,7 @@ func renderCypherTemplate(t *testing.T, template string, vars map[string]string) return rendered } +// mergeParams returns a copy of base with overrides taking precedence. func mergeParams(base, overrides map[string]any) map[string]any { if len(base) == 0 && len(overrides) == 0 { return nil @@ -192,6 +256,42 @@ func mergeParams(base, overrides map[string]any) map[string]any { return merged } +// mergeStringMap returns a copy of base with string overrides taking +// precedence. +func mergeStringMap(base, overrides map[string]string) map[string]string { + if len(base) == 0 && len(overrides) == 0 { + return nil + } + + merged := make(map[string]string, len(base)+len(overrides)) + for key, value := range base { + merged[key] = value + } + for key, value := range overrides { + merged[key] = value + } + return merged +} + +// mergeStringListMap returns a deep-enough copy of base with list overrides +// taking precedence. +func mergeStringListMap(base, overrides map[string][]string) map[string][]string { + if len(base) == 0 && len(overrides) == 0 { + return nil + } + + merged := make(map[string][]string, len(base)+len(overrides)) + for key, value := range base { + merged[key] = append([]string(nil), value...) + } + for key, value := range overrides { + merged[key] = append([]string(nil), value...) + } + return merged +} + +// runWithTemplateFixture executes a rendered case against its inline fixture, +// checks the query result and postconditions, and rolls the transaction back. func runWithTemplateFixture(t *testing.T, ctx context.Context, db graph.Database, tc testCase, assertion caseAssertion) { t.Helper() @@ -200,16 +300,21 @@ func runWithTemplateFixture(t *testing.T, ctx context.Context, db graph.Database } queryErrorObserved := false - session := &Session{DB: db, Ctx: ctx} + session := &Session{ + DB: db, + Ctx: ctx, + } err := session.WithRollbackFixture(t, tc.Fixture, false, func(tx graph.Transaction, idMap opengraph.IDMap) error { - result := tx.Query(tc.Cypher, tc.Params) - defer result.Close() + params := resolveFixtureParams(t, tc.Params, tc.NodeParams, tc.NodeListParams, idMap) + result := tx.Query(tc.Cypher, params) assertion.checkResult(t, result, newAssertionContext(idMap)) + result.Close() if assertion.expectQueryError { queryErrorObserved = true + return nil } - return nil + return runStateAssertions(t, tx, idMap, tc.PostAssertions) }) if assertion.expectQueryError && queryErrorObserved && err != nil { @@ -221,6 +326,8 @@ func runWithTemplateFixture(t *testing.T, ctx context.Context, db graph.Database } } +// runMetamorphicFamily executes every query over one fixture and requires their +// selected comparison signatures to match the first query. func runMetamorphicFamily(t *testing.T, ctx context.Context, db graph.Database, family cypherMetamorphicFamily) { t.Helper() @@ -232,7 +339,10 @@ func runMetamorphicFamily(t *testing.T, ctx context.Context, db graph.Database, t.Fatal("metamorphic cases must define at least two queries") } - session := &Session{DB: db, Ctx: ctx} + session := &Session{ + DB: db, + Ctx: ctx, + } err := session.WithRollbackFixture(t, family.Fixture, false, func(tx graph.Transaction, idMap opengraph.IDMap) error { assertCtx := newAssertionContext(idMap) var baselineName string @@ -277,8 +387,10 @@ func runMetamorphicFamily(t *testing.T, ctx context.Context, db graph.Database, } } +// comparisonModes accepts either one comparison-mode string or a list in template JSON. type comparisonModes []string +// UnmarshalJSON accepts either a single comparison mode or a list of modes. func (s *comparisonModes) UnmarshalJSON(raw []byte) error { var mode string if err := json.Unmarshal(raw, &mode); err == nil { @@ -295,10 +407,13 @@ func (s *comparisonModes) UnmarshalJSON(raw []byte) error { return nil } +// String joins comparison modes for use in generated subtest names. func (s comparisonModes) String() string { return strings.Join(s, ",") } +// comparisonSignature computes each requested comparison mode for a collected +// result in declaration order. func comparisonSignature(t *testing.T, result queryResult, ctx assertionContext, modes comparisonModes) []string { t.Helper() @@ -314,6 +429,8 @@ func comparisonSignature(t *testing.T, result queryResult, ctx assertionContext, return signature } +// comparisonModeSignature canonicalizes a collected result according to one +// supported metamorphic comparison mode. func comparisonModeSignature(t *testing.T, result queryResult, ctx assertionContext, mode string) string { t.Helper() @@ -346,6 +463,12 @@ func comparisonModeSignature(t *testing.T, result queryResult, ctx assertionCont signatures = append(signatures, pathEdgeKindSignature(t, path)) } signature = sortedSignatures(signatures) + case "path_relationship_records": + signatures := make([]string, 0, len(result.rows)) + for _, path := range collectPaths(t, result) { + signatures = append(signatures, pathRelationshipRecordSignature(t, path, ctx)) + } + signature = sortedSignatures(signatures) default: t.Fatalf("unknown metamorphic comparison mode %q", mode) } @@ -358,6 +481,8 @@ func comparisonModeSignature(t *testing.T, result queryResult, ctx assertionCont return mode + ":" + string(encoded) } +// firstScalarSignatures returns the canonical signature of each row's first +// projected value. func firstScalarSignatures(t *testing.T, result queryResult) []string { t.Helper() @@ -373,6 +498,7 @@ func firstScalarSignatures(t *testing.T, result queryResult) []string { return signatures } +// rowScalarSignatures renders every result row into a deterministic scalar signature. func rowScalarSignatures(result queryResult) []string { signatures := make([]string, 0, len(result.rows)) for _, row := range result.rows { @@ -382,6 +508,7 @@ func rowScalarSignatures(result queryResult) []string { return signatures } +// sortedSignatures returns a sorted copy without modifying its input. func sortedSignatures(signatures []string) []string { sorted := append([]string(nil), signatures...) sort.Strings(sorted) diff --git a/integration/cypher_test.go b/integration/cypher_test.go index b9adfc96..4ae03bae 100644 --- a/integration/cypher_test.go +++ b/integration/cypher_test.go @@ -32,25 +32,64 @@ import ( "github.com/specterops/dawgs/graph" "github.com/specterops/dawgs/opengraph" + "github.com/specterops/dawgs/testutil" ) // caseFile represents one JSON test case file. type caseFile struct { - Dataset string `json:"dataset"` - Cases []testCase `json:"cases"` + // Dataset selects the fixture dataset loaded before executing Cases. + Dataset string `json:"dataset"` + + // Cases contains the queries and assertions decoded from this file. + Cases []testCase `json:"cases"` } // testCase is a single test: a Cypher query and an assertion on its result. // Cases with a "fixture" field run in a write transaction that rolls back, // so the inline data doesn't persist. type testCase struct { - Name string `json:"name"` - Cypher string `json:"cypher"` - Params map[string]any `json:"params,omitempty"` - Assert json.RawMessage `json:"assert"` + // Name identifies the case in test output. + Name string `json:"name"` + + // Cypher is the statement executed by the case. + Cypher string `json:"cypher"` + + // Params contains literal and generated query parameters. + Params testutil.Params `json:"params,omitempty"` + + // NodeParams maps parameter names to fixture node identifiers. + NodeParams map[string]string `json:"node_params,omitempty"` + + // NodeListParams maps parameter names to lists of fixture node identifiers. + NodeListParams map[string][]string `json:"node_list_params,omitempty"` + + // Assert encodes the expected primary result assertion. + Assert json.RawMessage `json:"assert"` + + // PostAssertions contains state checks executed after the primary result drains. + PostAssertions []stateAssertion `json:"post_assertions,omitempty"` + + // Fixture optionally supplies inline graph data loaded in a rollback transaction. Fixture *opengraph.Graph `json:"fixture,omitempty"` } +// stateAssertion runs after the primary query has been fully drained. It is +// executed in the same transaction and against the same fixture ID map. +type stateAssertion struct { + // Name optionally identifies the assertion in diagnostics. + Name string `json:"name,omitempty"` + + // Cypher is the state-inspection query executed after the primary query. + Cypher string `json:"cypher"` + + // Params contains parameters for the state-inspection query. + Params testutil.Params `json:"params,omitempty"` + + // Assert encodes the expected state-inspection result. + Assert json.RawMessage `json:"assert"` +} + +// TestCypher executes every fixture-backed case, grouping cases by dataset so each group shares one loaded graph. func TestCypher(t *testing.T) { files, err := filepath.Glob("testdata/cases/*.json") if err != nil { @@ -60,10 +99,13 @@ func TestCypher(t *testing.T) { t.Fatal("no case files found in testdata/cases/") } - // Parse all case files and group by dataset. + // group collects case files that share one fixture dataset. type group struct { + // dataset names the fixture dataset shared by files. dataset string - files []caseFile + + // files contains the parsed cases in the dataset group. + files []caseFile } var ( groups = map[string]*group{} @@ -141,11 +183,15 @@ func TestCypher(t *testing.T) { // {"contains_edge": {start,end,kind,props}} — some row/path has a relationship matching all listed fields // {"node_ids": ["a", "b"]} — exact multiset of returned fixture node IDs, order-independent // {"node_id_set": ["a", "b"]} — exact set of returned fixture node IDs, order-independent +// {"node_records": [{id,kinds,props}]} — exact returned nodes, including kinds and properties +// {"relationship_triples": [{start,end,kind}]} — exact returned relationship triples +// {"relationship_records": [{start,end,kind,props}]} — exact returned relationships and properties // {"ordered_node_ids": ["a", "b"]} — first returned node ID per row, preserving row order // {"node_list_ids": [["a", "b"]]} — exact multiset of returned node-list ID sequences // {"path_node_ids": [["a", "b"]]} — exact multiset of returned path node ID sequences // {"path_lengths": [N...]} — exact multiset of returned path edge counts // {"path_edge_kinds": [["K"...]]} — exact multiset of returned path edge kind sequences +// {"path_relationship_records": [[{start,end,kind,props}...]]} — exact ordered relationships for every returned path // {"relationship_list_kinds": [["K"...]]} — exact multiset of returned relationship-list kind sequences // // Object assertions may combine multiple keys; every assertion must pass. @@ -218,6 +264,15 @@ func parseAssertion(t *testing.T, raw json.RawMessage) caseAssertion { case "node_id_set": assertions = append(assertions, assertNodeIDs(decodeAssertionValue[[]string](t, key, val), true)) + case "node_records": + assertions = append(assertions, assertNodeRecords(decodeAssertionValue[[]nodeExpectation](t, key, val))) + + case "relationship_triples": + assertions = append(assertions, assertRelationshipRecords(decodeAssertionValue[[]edgeExpectation](t, key, val), false)) + + case "relationship_records": + assertions = append(assertions, assertRelationshipRecords(decodeAssertionValue[[]edgeExpectation](t, key, val), true)) + case "ordered_node_ids": assertions = append(assertions, assertOrderedNodeIDs(decodeAssertionValue[[]string](t, key, val))) @@ -233,6 +288,9 @@ func parseAssertion(t *testing.T, raw json.RawMessage) caseAssertion { case "path_edge_kinds": assertions = append(assertions, assertPathEdgeKinds(decodeAssertionValue[[][]string](t, key, val))) + case "path_relationship_records": + assertions = append(assertions, assertPathRelationshipRecords(decodeAssertionValue[[][]edgeExpectation](t, key, val))) + case "relationship_list_kinds": assertions = append(assertions, assertRelationshipListKinds(decodeAssertionValue[[][]string](t, key, val))) @@ -262,8 +320,9 @@ func runReadOnly(t *testing.T, ctx context.Context, db graph.Database, idMap ope var ( queryErrorObserved = false + params = resolveFixtureParams(t, tc.Params, tc.NodeParams, tc.NodeListParams, idMap) err = db.ReadTransaction(ctx, func(tx graph.Transaction) error { - result := tx.Query(tc.Cypher, tc.Params) + result := tx.Query(tc.Cypher, params) defer result.Close() assertion.checkResult(t, result, newAssertionContext(idMap)) if assertion.expectQueryError { @@ -288,16 +347,21 @@ func runWithFixture(t *testing.T, ctx context.Context, db graph.Database, tc tes t.Helper() queryErrorObserved := false - session := &Session{DB: db, Ctx: ctx} + session := &Session{ + DB: db, + Ctx: ctx, + } err := session.WithRollbackFixture(t, tc.Fixture, true, func(tx graph.Transaction, idMap opengraph.IDMap) error { - result := tx.Query(tc.Cypher, tc.Params) - defer result.Close() + params := resolveFixtureParams(t, tc.Params, tc.NodeParams, tc.NodeListParams, idMap) + result := tx.Query(tc.Cypher, params) assertion.checkResult(t, result, newAssertionContext(idMap)) + result.Close() if assertion.expectQueryError { queryErrorObserved = true + return nil } - return nil + return runStateAssertions(t, tx, idMap, tc.PostAssertions) }) if assertion.expectQueryError && queryErrorObserved && err != nil { @@ -309,15 +373,90 @@ func runWithFixture(t *testing.T, ctx context.Context, db graph.Database, tc tes } } +// resolveFixtureParams copies literal parameters and replaces fixture node +// references with their backend database IDs. +func resolveFixtureParams( + t *testing.T, + params map[string]any, + nodeParams map[string]string, + nodeListParams map[string][]string, + idMap opengraph.IDMap, +) map[string]any { + t.Helper() + + resolved := make(map[string]any, len(params)+len(nodeParams)+len(nodeListParams)) + for name, value := range params { + resolved[name] = value + } + + for paramName, fixtureID := range nodeParams { + id, found := idMap[fixtureID] + if !found { + t.Fatalf("node parameter %q references unknown fixture ID %q", paramName, fixtureID) + } + resolved[paramName] = id.Int64() + } + + for paramName, fixtureIDs := range nodeListParams { + ids := make([]int64, len(fixtureIDs)) + for idx, fixtureID := range fixtureIDs { + id, found := idMap[fixtureID] + if !found { + t.Fatalf("node list parameter %q references unknown fixture ID %q", paramName, fixtureID) + } + ids[idx] = id.Int64() + } + resolved[paramName] = ids + } + + if len(resolved) == 0 { + return nil + } + return resolved +} + +// runStateAssertions executes and checks each postcondition query in the +// fixture's transaction. +func runStateAssertions(t *testing.T, tx graph.Transaction, idMap opengraph.IDMap, assertions []stateAssertion) error { + t.Helper() + + for idx, spec := range assertions { + name := spec.Name + if name == "" { + name = fmt.Sprintf("post assertion %d", idx+1) + } + if spec.Cypher == "" { + t.Fatalf("%s has no Cypher query", name) + } + + check := parseAssertion(t, spec.Assert) + if check.expectQueryError { + t.Fatalf("%s may not expect a query error", name) + } + + result := tx.Query(spec.Cypher, spec.Params) + check.checkResult(t, result, newAssertionContext(idMap)) + result.Close() + } + + return nil +} + // --- Assertion implementations --- +// caseAssertion selects either a normalized result check or an expected query-error check. type caseAssertion struct { - check resultAssertion + // check validates a successfully drained result. + check resultAssertion + + // expectQueryError selects the error path instead of invoking check. expectQueryError bool } +// resultAssertion validates a normalized query result using fixture-aware identity mapping. type resultAssertion func(*testing.T, queryResult, assertionContext) +// checkResult dispatches to the expected error path or the configured successful-result assertion. func (s caseAssertion) checkResult(t *testing.T, result graph.Result, ctx assertionContext) { t.Helper() @@ -333,10 +472,13 @@ func (s caseAssertion) checkResult(t *testing.T, result graph.Result, ctx assert s.check(t, collectResult(t, result), ctx) } +// assertionContext translates backend database IDs back to stable fixture identifiers. type assertionContext struct { + // fixtureIDByID maps database node IDs to their source fixture IDs. fixtureIDByID map[graph.ID]string } +// newAssertionContext reverses a fixture ID map for result assertions. func newAssertionContext(idMap opengraph.IDMap) assertionContext { ctx := assertionContext{ fixtureIDByID: make(map[graph.ID]string, len(idMap)), @@ -349,6 +491,7 @@ func newAssertionContext(idMap opengraph.IDMap) assertionContext { return ctx } +// fixtureID returns the stable fixture identifier for dbID and fails the current test if it is unknown. func (s assertionContext) fixtureID(t *testing.T, dbID graph.ID) string { t.Helper() @@ -360,16 +503,26 @@ func (s assertionContext) fixtureID(t *testing.T, dbID graph.ID) string { return "" } +// resultRow is an owned snapshot of one backend result row and its column names. type resultRow struct { - keys []string + // keys contains the row's projected column names. + keys []string + + // values contains an owned copy of the row's projected values. values []any } +// queryResult contains drained rows and the backend mapper needed to decode graph values. type queryResult struct { - rows []resultRow + // rows contains every drained row in result order. + rows []resultRow + + // mapper converts backend-specific values to graph-native representations. mapper graph.ValueMapper } +// collectResult drains a backend result into owned rows and retains its value +// mapper for graph-value assertions. func collectResult(t *testing.T, result graph.Result) queryResult { t.Helper() @@ -391,6 +544,7 @@ func collectResult(t *testing.T, result graph.Result) queryResult { return collected } +// assertQueryError drains result and requires the backend to report an execution error. func assertQueryError(t *testing.T, result graph.Result) { t.Helper() @@ -402,6 +556,7 @@ func assertQueryError(t *testing.T, result graph.Result) { } } +// decodeAssertionValue decodes assertion JSON into T and fails the current test with the assertion key on error. func decodeAssertionValue[T any](t *testing.T, key string, raw json.RawMessage) T { t.Helper() @@ -413,6 +568,7 @@ func decodeAssertionValue[T any](t *testing.T, key string, raw json.RawMessage) return value } +// assertNonEmpty requires at least one result row. func assertNonEmpty(t *testing.T, result queryResult, _ assertionContext) { t.Helper() if len(result.rows) == 0 { @@ -420,6 +576,7 @@ func assertNonEmpty(t *testing.T, result queryResult, _ assertionContext) { } } +// assertEmpty requires a result set with no rows. func assertEmpty(t *testing.T, result queryResult, _ assertionContext) { t.Helper() if len(result.rows) > 0 { @@ -427,10 +584,12 @@ func assertEmpty(t *testing.T, result queryResult, _ assertionContext) { } } +// assertNoError accepts any successfully collected result without imposing a row-shape assertion. func assertNoError(t *testing.T, _ queryResult, _ assertionContext) { t.Helper() } +// assertKeys requires every result row to expose exactly the expected projection keys in order. func assertKeys(expected []string) resultAssertion { return func(t *testing.T, result queryResult, _ assertionContext) { t.Helper() @@ -448,6 +607,7 @@ func assertKeys(expected []string) resultAssertion { } } +// assertRowCount requires exactly n result rows. func assertRowCount(n int) resultAssertion { return func(t *testing.T, result queryResult, _ assertionContext) { t.Helper() @@ -457,6 +617,7 @@ func assertRowCount(n int) resultAssertion { } } +// assertAtLeastInt64 requires the first scalar result to be an integer no smaller than min. func assertAtLeastInt64(min int64) resultAssertion { return func(t *testing.T, result queryResult, _ assertionContext) { t.Helper() @@ -474,6 +635,7 @@ func assertAtLeastInt64(min int64) resultAssertion { } } +// assertExactInt64 requires one row whose first scalar is exactly expected. func assertExactInt64(expected int64) resultAssertion { return func(t *testing.T, result queryResult, _ assertionContext) { t.Helper() @@ -491,6 +653,7 @@ func assertExactInt64(expected int64) resultAssertion { } } +// assertScalarValues compares each row's first scalar with expected, optionally preserving row order. func assertScalarValues(expected []any, ordered bool) resultAssertion { return func(t *testing.T, result queryResult, _ assertionContext) { t.Helper() @@ -519,6 +682,7 @@ func assertScalarValues(expected []any, ordered bool) resultAssertion { } } +// assertRowValues compares complete scalar rows with expected, optionally preserving row order. func assertRowValues(expected [][]any, ordered bool) resultAssertion { return func(t *testing.T, result queryResult, _ assertionContext) { t.Helper() @@ -543,6 +707,8 @@ func assertRowValues(expected [][]any, ordered bool) resultAssertion { } } +// firstScalarValue returns the first projected value and fails for an empty +// result or row. func firstScalarValue(t *testing.T, result queryResult) any { t.Helper() @@ -557,6 +723,7 @@ func firstScalarValue(t *testing.T, result queryResult) any { return result.rows[0].values[0] } +// asInt64 converts supported integer representations to int64 without accepting non-integral values. func asInt64(value any) (int64, bool) { switch typedValue := value.(type) { case int: @@ -596,6 +763,7 @@ func asInt64(value any) (int64, bool) { return 0, false } +// rowScalarSignature joins deterministic scalar signatures for one projected row. func rowScalarSignature(values []any) string { parts := make([]string, len(values)) for idx, value := range values { @@ -610,6 +778,8 @@ func rowScalarSignature(values []any) string { return string(encoded) } +// scalarSignature canonicalizes nil, numeric, string, boolean, and JSON-backed +// values for backend-independent comparisons. func scalarSignature(value any) string { if value == nil { return "null:" @@ -637,6 +807,8 @@ func scalarSignature(value any) string { } } +// jsonNumberSignature recognizes a JSON number and returns its canonical +// numeric signature. func jsonNumberSignature(encoded []byte) (string, bool) { decoder := json.NewDecoder(strings.NewReader(string(encoded))) decoder.UseNumber() @@ -659,6 +831,7 @@ func jsonNumberSignature(encoded []byte) (string, bool) { return fmt.Sprintf("number:%g", value), true } +// assertContainsNodeWithProp requires any returned node to contain key with the expected string value. func assertContainsNodeWithProp(key, expected string) resultAssertion { return func(t *testing.T, result queryResult, _ assertionContext) { t.Helper() @@ -676,6 +849,7 @@ func assertContainsNodeWithProp(key, expected string) resultAssertion { } } +// assertContainsNodeWithProps requires any returned node to contain the expected property subset. func assertContainsNodeWithProps(expected map[string]any) resultAssertion { return func(t *testing.T, result queryResult, _ assertionContext) { t.Helper() @@ -702,13 +876,34 @@ func assertContainsNodeWithProps(expected map[string]any) resultAssertion { } } +// edgeExpectation describes the stable identity, kind, and optional properties required of a relationship result. type edgeExpectation struct { - Start string `json:"start,omitempty"` - End string `json:"end,omitempty"` - Kind string `json:"kind,omitempty"` + // Start is the expected fixture ID of the relationship's start node. + Start string `json:"start,omitempty"` + + // End is the expected fixture ID of the relationship's end node. + End string `json:"end,omitempty"` + + // Kind is the expected relationship kind. + Kind string `json:"kind,omitempty"` + + // Props contains the expected relationship property subset. + Props map[string]any `json:"props,omitempty"` +} + +// nodeExpectation describes the stable identity, kinds, and optional properties required of a node result. +type nodeExpectation struct { + // ID is the expected fixture node identifier. + ID string `json:"id"` + + // Kinds contains the expected node kinds independent of order. + Kinds []string `json:"kinds,omitempty"` + + // Props contains the expected node property subset. Props map[string]any `json:"props,omitempty"` } +// assertContainsEdge requires any returned relationship to match expected endpoints, kind, and properties. func assertContainsEdge(expected edgeExpectation) resultAssertion { return func(t *testing.T, result queryResult, ctx assertionContext) { t.Helper() @@ -723,6 +918,7 @@ func assertContainsEdge(expected edgeExpectation) resultAssertion { } } +// assertNodeIDs compares collected fixture node IDs as a multiset, optionally deduplicating them first. func assertNodeIDs(expected []string, unique bool) resultAssertion { return func(t *testing.T, result queryResult, ctx assertionContext) { t.Helper() @@ -732,6 +928,74 @@ func assertNodeIDs(expected []string, unique bool) resultAssertion { } } +// assertNodeRecords compares returned nodes with expected fixture IDs, kinds, and property subsets independent of order. +func assertNodeRecords(expected []nodeExpectation) resultAssertion { + return func(t *testing.T, result queryResult, ctx assertionContext) { + t.Helper() + + got := make([]string, 0, len(expected)) + for _, row := range result.rows { + for _, rawValue := range row.values { + var node graph.Node + if result.mapper.Map(rawValue, &node) { + got = append(got, nodeRecordSignature(t, node, ctx)) + } + } + } + + want := make([]string, len(expected)) + for idx, node := range expected { + want[idx] = expectedNodeRecordSignature(node) + } + + assertStringMultiset(t, got, want, "node records") + } +} + +// assertRelationshipRecords compares returned relationships with expected records, optionally including properties. +func assertRelationshipRecords(expected []edgeExpectation, includeProperties bool) resultAssertion { + return func(t *testing.T, result queryResult, ctx assertionContext) { + t.Helper() + + relationships := collectRelationships(t, result) + got := make([]string, len(relationships)) + for idx, relationship := range relationships { + got[idx] = relationshipRecordSignature(t, relationship, ctx, includeProperties) + } + + want := make([]string, len(expected)) + for idx, relationship := range expected { + want[idx] = expectedRelationshipRecordSignature(relationship, includeProperties) + } + + assertStringMultiset(t, got, want, "relationship records") + } +} + +// assertPathRelationshipRecords compares the ordered relationship record sequence in each returned path. +func assertPathRelationshipRecords(expected [][]edgeExpectation) resultAssertion { + return func(t *testing.T, result queryResult, ctx assertionContext) { + t.Helper() + + paths := collectPaths(t, result) + got := make([]string, len(paths)) + for idx, path := range paths { + got[idx] = pathRelationshipRecordSignature(t, path, ctx) + } + want := make([]string, len(expected)) + for pathIdx, relationships := range expected { + parts := make([]string, len(relationships)) + for relationshipIdx, relationship := range relationships { + parts[relationshipIdx] = expectedRelationshipRecordSignature(relationship, true) + } + want[pathIdx] = strings.Join(parts, "\x02") + } + + assertStringMultiset(t, got, want, "ordered path relationship records") + } +} + +// assertOrderedNodeIDs compares fixture node IDs in result-row order. func assertOrderedNodeIDs(expected []string) resultAssertion { return func(t *testing.T, result queryResult, ctx assertionContext) { t.Helper() @@ -759,6 +1023,7 @@ func assertOrderedNodeIDs(expected []string) resultAssertion { } } +// assertNodeListIDs compares each returned node-list projection by its ordered fixture IDs. func assertNodeListIDs(expected [][]string) resultAssertion { return func(t *testing.T, result queryResult, ctx assertionContext) { t.Helper() @@ -782,6 +1047,8 @@ func assertNodeListIDs(expected [][]string) resultAssertion { } } +// collectNodeIDs maps every returned node to a fixture ID, optionally removing +// duplicates while preserving first occurrence order. func collectNodeIDs(t *testing.T, result queryResult, ctx assertionContext, unique bool) []string { t.Helper() @@ -810,6 +1077,7 @@ func collectNodeIDs(t *testing.T, result queryResult, ctx assertionContext, uniq return ids } +// nodeListIDSignature renders a node slice as an ordered fixture-ID sequence. func nodeListIDSignature(t *testing.T, nodes []*graph.Node, ctx assertionContext) string { t.Helper() @@ -825,6 +1093,7 @@ func nodeListIDSignature(t *testing.T, nodes []*graph.Node, ctx assertionContext return strings.Join(nodeIDs, "->") } +// assertPathNodeIDs compares each returned path by its ordered fixture node IDs. func assertPathNodeIDs(expected [][]string) resultAssertion { return func(t *testing.T, result queryResult, ctx assertionContext) { t.Helper() @@ -848,6 +1117,7 @@ func assertPathNodeIDs(expected [][]string) resultAssertion { } } +// assertPathLengths compares the relationship count of every returned path. func assertPathLengths(expected []int) resultAssertion { return func(t *testing.T, result queryResult, _ assertionContext) { t.Helper() @@ -866,6 +1136,7 @@ func assertPathLengths(expected []int) resultAssertion { } } +// assertPathEdgeKinds compares each returned path by its ordered relationship kinds. func assertPathEdgeKinds(expected [][]string) resultAssertion { return func(t *testing.T, result queryResult, _ assertionContext) { t.Helper() @@ -884,6 +1155,7 @@ func assertPathEdgeKinds(expected [][]string) resultAssertion { } } +// assertRelationshipListKinds compares each relationship-list projection by ordered kind names. func assertRelationshipListKinds(expected [][]string) resultAssertion { return func(t *testing.T, result queryResult, _ assertionContext) { t.Helper() @@ -913,6 +1185,7 @@ func assertRelationshipListKinds(expected [][]string) resultAssertion { } } +// pathNodeIDSignature renders a path as an ordered fixture-node-ID sequence. func pathNodeIDSignature(t *testing.T, path graph.Path, ctx assertionContext) string { t.Helper() @@ -928,6 +1201,7 @@ func pathNodeIDSignature(t *testing.T, path graph.Path, ctx assertionContext) st return strings.Join(nodeIDs, "->") } +// pathEdgeKindSignature renders a path as an ordered relationship-kind sequence. func pathEdgeKindSignature(t *testing.T, path graph.Path) string { t.Helper() @@ -947,6 +1221,7 @@ func pathEdgeKindSignature(t *testing.T, path graph.Path) string { return strings.Join(edgeKinds, "->") } +// relationshipListKindSignature renders a relationship-pointer slice as an ordered kind sequence. func relationshipListKindSignature(t *testing.T, relationships []*graph.Relationship) string { t.Helper() @@ -966,6 +1241,7 @@ func relationshipListKindSignature(t *testing.T, relationships []*graph.Relation return strings.Join(edgeKinds, "->") } +// relationshipValueListKindSignature renders a relationship-value slice as an ordered kind sequence. func relationshipValueListKindSignature(t *testing.T, relationships []graph.Relationship) string { t.Helper() @@ -981,6 +1257,7 @@ func relationshipValueListKindSignature(t *testing.T, relationships []graph.Rela return strings.Join(edgeKinds, "->") } +// collectPaths maps every path-valued result cell into a graph path. func collectPaths(t *testing.T, result queryResult) []graph.Path { t.Helper() @@ -997,6 +1274,8 @@ func collectPaths(t *testing.T, result queryResult) []graph.Path { return paths } +// collectRelationships maps standalone relationships and relationships nested +// in returned paths into one slice. func collectRelationships(t *testing.T, result queryResult) []graph.Relationship { t.Helper() @@ -1022,6 +1301,96 @@ func collectRelationships(t *testing.T, result queryResult) []graph.Relationship return relationships } +// nodeRecordSignature renders a returned node into a stable fixture ID, sorted kinds, and property signature. +func nodeRecordSignature(t *testing.T, node graph.Node, ctx assertionContext) string { + t.Helper() + + kinds := node.Kinds.Strings() + sort.Strings(kinds) + + return strings.Join([]string{ + ctx.fixtureID(t, node.ID), + strings.Join(kinds, ","), + propertyMapSignature(node.Properties.MapOrEmpty()), + }, "\x00") +} + +// expectedNodeRecordSignature renders a node expectation in the same canonical form as a returned node. +func expectedNodeRecordSignature(node nodeExpectation) string { + kinds := append([]string(nil), node.Kinds...) + sort.Strings(kinds) + + return strings.Join([]string{ + node.ID, + strings.Join(kinds, ","), + propertyMapSignature(node.Props), + }, "\x00") +} + +// relationshipRecordSignature renders a returned relationship into canonical fixture endpoints, kind, and optional properties. +func relationshipRecordSignature(t *testing.T, relationship graph.Relationship, ctx assertionContext, includeProperties bool) string { + t.Helper() + + kind := "" + if relationship.Kind != nil { + kind = relationship.Kind.String() + } + + parts := []string{ + ctx.fixtureID(t, relationship.StartID), + ctx.fixtureID(t, relationship.EndID), + kind, + } + if includeProperties { + parts = append(parts, propertyMapSignature(relationship.Properties.MapOrEmpty())) + } + + return strings.Join(parts, "\x00") +} + +// pathRelationshipRecordSignature renders a path's ordered relationships into one canonical comparison value. +func pathRelationshipRecordSignature(t *testing.T, path graph.Path, ctx assertionContext) string { + t.Helper() + + parts := make([]string, 0, len(path.Edges)) + for _, relationship := range path.Edges { + if relationship == nil { + t.Fatal("path contains a nil relationship") + } + parts = append(parts, relationshipRecordSignature(t, *relationship, ctx, true)) + } + return strings.Join(parts, "\x02") +} + +// expectedRelationshipRecordSignature renders a relationship expectation in the same canonical form as a returned relationship. +func expectedRelationshipRecordSignature(relationship edgeExpectation, includeProperties bool) string { + parts := []string{relationship.Start, relationship.End, relationship.Kind} + if includeProperties { + parts = append(parts, propertyMapSignature(relationship.Props)) + } + + return strings.Join(parts, "\x00") +} + +// propertyMapSignature renders properties in key order using canonical scalar +// signatures. +func propertyMapSignature(properties map[string]any) string { + keys := make([]string, 0, len(properties)) + for key := range properties { + keys = append(keys, key) + } + sort.Strings(keys) + + parts := make([]string, len(keys)) + for idx, key := range keys { + parts[idx] = key + "=" + scalarSignature(properties[key]) + } + + return strings.Join(parts, "\x01") +} + +// relationshipMatches reports whether a relationship satisfies the expected +// fixture endpoints, kind, and property subset. func relationshipMatches(t *testing.T, relationship graph.Relationship, expected edgeExpectation, ctx assertionContext) bool { t.Helper() @@ -1042,6 +1411,7 @@ func relationshipMatches(t *testing.T, relationship graph.Relationship, expected return propertiesMatch(relationship.Properties, expected.Props) } +// propertiesMatch reports whether properties contains every expected key with an equivalent value. func propertiesMatch(properties *graph.Properties, expected map[string]any) bool { if len(expected) == 0 { return true @@ -1061,6 +1431,7 @@ func propertiesMatch(properties *graph.Properties, expected map[string]any) bool return true } +// valuesEqual compares numeric values across concrete widths and delegates all other values to deep equality. func valuesEqual(actual, expected any) bool { if actualNumber, actualIsNumber := asFloat64(actual); actualIsNumber { if expectedNumber, expectedIsNumber := asFloat64(expected); expectedIsNumber { @@ -1071,6 +1442,7 @@ func valuesEqual(actual, expected any) bool { return reflect.DeepEqual(actual, expected) } +// asFloat64 converts supported numeric representations to a common comparison value. func asFloat64(value any) (float64, bool) { switch typedValue := value.(type) { case int: @@ -1102,6 +1474,7 @@ func asFloat64(value any) (float64, bool) { } } +// assertStringMultiset compares string collections after sorting copies and reports label on mismatch. func assertStringMultiset(t *testing.T, got, expected []string, label string) { t.Helper() diff --git a/integration/delegated_enrollment_legacy_builder_test.go b/integration/delegated_enrollment_legacy_builder_test.go new file mode 100644 index 00000000..99bfb7e3 --- /dev/null +++ b/integration/delegated_enrollment_legacy_builder_test.go @@ -0,0 +1,125 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +//go:build manual_integration + +package integration + +import ( + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/opengraph" + "github.com/specterops/dawgs/ops" + "github.com/specterops/dawgs/query" + "github.com/stretchr/testify/require" +) + +// TestLegacyBuilderDelegatedEnrollmentDiscovery verifies legacy criteria preserve delegated-enrollment discovery results across backends. +func TestLegacyBuilderDelegatedEnrollmentDiscovery(t *testing.T) { + fixture := delegatedEnrollmentFixture() + nodeKinds, edgeKinds := fixture.Kinds() + db, ctx := SetupDBWithKindsNoGraphCleanup(t, nodeKinds, edgeKinds) + ClearGraph(t, db, ctx) + + WithLegacyRelationshipQuery(t, &Session{ + DB: db, + Ctx: ctx, + }, fixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.In(query.EndProperty("objectid"), []string{"ca-a", "ca-b"}), + query.Kind(query.Relationship(), graph.StringKind("PublishedTo")), + query.Kind(query.Start(), graph.StringKind("CertTemplate")), + ) + }, func(relationshipQuery graph.RelationshipQuery, idMap opengraph.IDMap) error { + relationships, err := ops.FetchRelationships(relationshipQuery) + require.NoError(t, err) + require.Len(t, relationships, 3, "raw relationship results must retain duplicate paths to one template") + + nodes, err := ops.FetchStartNodes(relationshipQuery) + require.NoError(t, err) + require.Equal(t, 2, nodes.Len(), "FetchStartNodes must de-duplicate repeated start nodes") + require.True(t, nodes.ContainsID(idMap["template-a"])) + require.True(t, nodes.ContainsID(idMap["template-b"])) + return nil + }) +} + +// delegatedEnrollmentFixture builds templates, enrollment endpoints, and +// duplicate paths used by the delegated-enrollment regression cases. +func delegatedEnrollmentFixture() *opengraph.Graph { + return &opengraph.Graph{ + Nodes: []opengraph.Node{ + { + ID: "template-a", + Kinds: []string{"CertTemplate"}, + Properties: map[string]any{"objectid": "template-a"}, + }, + { + ID: "template-b", + Kinds: []string{"CertTemplate"}, + Properties: map[string]any{"objectid": "template-b"}, + }, + { + ID: "wrong-start", + Kinds: []string{"OtherTemplate"}, + Properties: map[string]any{"objectid": "wrong-start"}, + }, + { + ID: "ca-a", + Kinds: []string{"EnterpriseCA"}, + Properties: map[string]any{"objectid": "ca-a"}, + }, + { + ID: "ca-b", + Kinds: []string{"EnterpriseCA"}, + Properties: map[string]any{"objectid": "ca-b"}, + }, + }, + Edges: []opengraph.Edge{ + { + StartID: "template-a", + EndID: "ca-a", + Kind: "PublishedTo", + Properties: map[string]any{"marker": "published-a"}, + }, + { + StartID: "template-a", + EndID: "ca-b", + Kind: "PublishedTo", + Properties: map[string]any{"marker": "published-b"}, + }, + { + StartID: "template-b", + EndID: "ca-a", + Kind: "PublishedTo", + Properties: map[string]any{"marker": "published-c"}, + }, + { + StartID: "wrong-start", + EndID: "ca-a", + Kind: "PublishedTo", + Properties: map[string]any{"marker": "wrong-start"}, + }, + { + StartID: "template-a", + EndID: "ca-a", + Kind: "OtherPublication", + Properties: map[string]any{"marker": "wrong-edge"}, + }, + }, + } +} diff --git a/integration/direct_write_mutations_test.go b/integration/direct_write_mutations_test.go new file mode 100644 index 00000000..6dba96f2 --- /dev/null +++ b/integration/direct_write_mutations_test.go @@ -0,0 +1,1139 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +//go:build manual_integration + +package integration + +import ( + "context" + "fmt" + "math" + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/opengraph" + "github.com/specterops/dawgs/ops" + "github.com/specterops/dawgs/query" + "github.com/specterops/dawgs/testutil" + "github.com/stretchr/testify/require" +) + +const ( + // directWriteObjectID is the identity property used by node selectors and upserts. + directWriteObjectID = "objectid" + + // directWriteLastSeen is the mutable timestamp property used to verify update semantics. + directWriteLastSeen = "lastseen" +) + +var ( + // directWriteDeleteRelationshipKind identifies relationships targeted by direct delete tests. + directWriteDeleteRelationshipKind = graph.StringKind("WriteDeleteRelationship") + + // directWriteCreateRelationshipKind identifies relationships created and conflict-merged by batch tests. + directWriteCreateRelationshipKind = graph.StringKind("WriteCreateRelationship") + + // directWriteCreateRelationshipOther identifies non-target relationships that must survive create tests. + directWriteCreateRelationshipOther = graph.StringKind("WriteCreateRelationshipOther") + + // directWriteUpsertNodeKind identifies nodes targeted by identity-based upserts. + directWriteUpsertNodeKind = graph.StringKind("WriteUpsertNode") + + // directWriteUpsertNodeKindA is the first kind used to verify multi-kind node updates. + directWriteUpsertNodeKindA = graph.StringKind("WriteUpsertNodeA") + + // directWriteUpsertNodeKindB is the second kind used to verify multi-kind node updates. + directWriteUpsertNodeKindB = graph.StringKind("WriteUpsertNodeB") + + // directWriteUpsertNodeKindC is the replacement kind used to verify kind-set mutation. + directWriteUpsertNodeKindC = graph.StringKind("WriteUpsertNodeC") + + // directWriteUpsertRelationshipKind identifies relationships targeted by identity-based upserts. + directWriteUpsertRelationshipKind = graph.StringKind("WriteUpsertRelationship") + + // directWriteUpsertRelationshipOther identifies non-target relationships that must survive upserts. + directWriteUpsertRelationshipOther = graph.StringKind("WriteUpsertRelationshipOther") + + // directWriteEnsureRelationshipKind identifies relationships created or updated by read-then-write tests. + directWriteEnsureRelationshipKind = graph.StringKind("WriteEnsureRelationship") + + // directWriteEntityKind is the common base kind assigned to direct-write fixture nodes. + directWriteEntityKind = graph.StringKind("Entity") + + // directWriteGroupKind identifies group nodes used by get-or-create tests. + directWriteGroupKind = graph.StringKind("Group") + + // directWriteUnrelatedKind marks nodes that selectors must not mutate. + directWriteUnrelatedKind = graph.StringKind("WriteUnrelated") + + // directWriteSuffixKind marks nodes used to exercise suffix-selector updates. + directWriteSuffixKind = graph.StringKind("WriteSuffix") + + // directWriteMissingKind is intentionally absent from the fixture for miss-path assertions. + directWriteMissingKind = graph.StringKind("WriteMissing") + + // directWriteScanKind identifies nodes used by kind-scan selectors. + directWriteScanKind = graph.StringKind("WriteKindScan") + + // directWriteEndpointKind identifies relationship endpoint nodes in mutation fixtures. + directWriteEndpointKind = graph.StringKind("WriteEndpoint") + + // directWriteBoundarySizes exercises empty, exact, adjacent, and repeated batch-flush thresholds. + directWriteBoundarySizes = []int{0, 1, 1_000, 1_999, 2_000, 2_001, 4_001, 8_001} +) + +// TestDirectWriteDeleteRelationshipBoundariesAndSurvivors verifies batched relationship deletion at flush boundaries preserves non-target edges. +func TestDirectWriteDeleteRelationshipBoundariesAndSurvivors(t *testing.T) { + db, ctx := directWriteSetup(t) + + for _, size := range directWriteBoundarySizes { + t.Run(fmt.Sprintf("WRITE-01 size %d", size), func(t *testing.T) { + _, _ = directWriteLoadDirectWriteFixture(t, ctx, db, size) + ids := directWriteFetchRelationshipIDs(t, ctx, db, func() graph.Criteria { + return query.And( + query.Kind(query.Relationship(), directWriteDeleteRelationshipKind), + query.Equals(query.RelationshipProperty("deletebatch"), true), + ) + }) + require.Len(t, ids, size) + + require.NoError(t, db.BatchOperation(ctx, func(batch graph.Batch) error { + for _, id := range ids { + if err := batch.DeleteRelationship(id); err != nil { + return err + } + } + return nil + }, graph.WithBatchSize(2_000))) + + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH ()-[r:WriteDeleteRelationship]->() RETURN count(r)")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH ()-[r:WriteDeleteRelationship]->() WHERE r.marker = 'same-kind-survivor' RETURN count(r)")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH ()-[r:WriteSurvivor]->() RETURN count(r)")) + require.Equal(t, int64(size), countByCypher(t, ctx, db, "MATCH ()-[r:WriteUpdateRelationship]->() RETURN count(r)")) + require.Equal(t, directWriteIncidentCount(size), countByCypher(t, ctx, db, "MATCH ()-[r:WriteIncident]->() RETURN count(r)")) + }) + } + + t.Run("WRITE-01 duplicate and missing IDs are harmless", func(t *testing.T) { + directWriteLoadDirectWriteFixture(t, ctx, db, 3) + ids := directWriteFetchRelationshipIDs(t, ctx, db, func() graph.Criteria { + return query.And( + query.Kind(query.Relationship(), directWriteDeleteRelationshipKind), + query.Equals(query.RelationshipProperty("deletebatch"), true), + ) + }) + require.Len(t, ids, 3) + + require.NoError(t, db.BatchOperation(ctx, func(batch graph.Batch) error { + for _, id := range []graph.ID{ids[0], ids[0], graph.ID(math.MaxInt64 - 7)} { + if err := batch.DeleteRelationship(id); err != nil { + return err + } + } + return nil + }, graph.WithBatchSize(2))) + + require.Equal(t, int64(3), countByCypher(t, ctx, db, "MATCH ()-[r:WriteDeleteRelationship]->() RETURN count(r)")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH ()-[r:WriteDeleteRelationship]->() WHERE r.marker = 'same-kind-survivor' RETURN count(r)")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH ()-[r:WriteSurvivor]->() RETURN count(r)")) + }) +} + +// TestDirectWriteDeleteNodeBoundariesAndCascades verifies batched node deletion removes incident edges and preserves unrelated nodes. +func TestDirectWriteDeleteNodeBoundariesAndCascades(t *testing.T) { + db, ctx := directWriteSetup(t) + + for _, size := range directWriteBoundarySizes { + t.Run(fmt.Sprintf("WRITE-02 size %d", size), func(t *testing.T) { + _, idMap := directWriteLoadDirectWriteFixture(t, ctx, db, size) + ids := make([]graph.ID, 0, size) + for _, targetName := range testutil.FixtureNames("write-target", size) { + ids = append(ids, idMap[targetName]) + } + + require.NoError(t, db.BatchOperation(ctx, func(batch graph.Batch) error { + for _, id := range ids { + if err := batch.DeleteNode(id); err != nil { + return err + } + } + return nil + }, graph.WithBatchSize(2_000))) + + require.Equal(t, int64(2), countByCypher(t, ctx, db, "MATCH (n:WriteEndpoint) RETURN count(n)")) + require.Equal(t, int64(0), countByCypher(t, ctx, db, "MATCH (n:WriteDeleteNode) RETURN count(n)")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH ()-[r:WriteDeleteRelationship]->() RETURN count(r)")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH ()-[r:WriteSurvivor]->() RETURN count(r)")) + }) + } + + t.Run("WRITE-02 duplicate missing isolated self low high and mixed directions", func(t *testing.T) { + _, idMap := directWriteLoadDirectWriteFixture(t, ctx, db, 8) + targetIDs := testutil.FixtureNames("write-target", 8) + isolated := directWriteCreateNode(t, ctx, db, directWriteProperties(directWriteObjectID, "write-isolated"), graph.StringKind("WriteDeleteNode")) + + require.NoError(t, db.BatchOperation(ctx, func(batch graph.Batch) error { + for _, targetName := range targetIDs { + if err := batch.DeleteNode(idMap[targetName]); err != nil { + return err + } + } + if err := batch.DeleteNode(isolated.ID); err != nil { + return err + } + if err := batch.DeleteNode(idMap[targetIDs[0]]); err != nil { + return err + } + return batch.DeleteNode(graph.ID(math.MaxInt64 - 11)) + }, graph.WithBatchSize(3))) + + require.Equal(t, int64(2), countByCypher(t, ctx, db, "MATCH (n:WriteEndpoint) RETURN count(n)")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH ()-[r:WriteDeleteRelationship]->() RETURN count(r)")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH ()-[r:WriteSurvivor]->() RETURN count(r)")) + require.Equal(t, int64(0), countByCypher(t, ctx, db, "MATCH ()-[r:WriteIncident]->() RETURN count(r)")) + }) +} + +// TestDirectWriteCreateRelationshipConflictMerge verifies duplicate relationship keys merge properties without colliding with distinct endpoint tuples. +func TestDirectWriteCreateRelationshipConflictMerge(t *testing.T) { + db, ctx := directWriteSetup(t) + ClearGraph(t, db, ctx) + a, b, c := directWriteCreateEndpoints(t, ctx, db, "create-a", "create-b", "create-c") + + require.NoError(t, db.BatchOperation(ctx, func(batch graph.Batch) error { + updates := []struct { + // start is the relationship start node ID. + start graph.ID + + // end is the relationship end node ID. + end graph.ID + + // kind is the relationship kind to create or merge. + kind graph.Kind + + // properties supplies the values merged into the relationship. + properties *graph.Properties + }{ + { + start: a.ID, + end: b.ID, + kind: directWriteCreateRelationshipKind, + properties: directWriteProperties("firstseen", "2026-01-01T00:00:00Z", "custom", "first", "preserved", "yes"), + }, + { + start: a.ID, + end: b.ID, + kind: directWriteCreateRelationshipKind, + properties: directWriteProperties("lastseen", "2026-01-02T00:00:00Z", "custom", "within"), + }, + { + start: a.ID, + end: b.ID, + kind: directWriteCreateRelationshipKind, + properties: directWriteProperties("custom", "last", "nullable", nil), + }, + { + start: b.ID, + end: a.ID, + kind: directWriteCreateRelationshipKind, + properties: directWriteProperties("marker", "reverse"), + }, + { + start: a.ID, + end: b.ID, + kind: directWriteCreateRelationshipOther, + properties: directWriteProperties("marker", "other-kind"), + }, + { + start: a.ID, + end: c.ID, + kind: directWriteCreateRelationshipKind, + properties: graph.NewProperties(), + }, + } + for _, update := range updates { + if err := batch.CreateRelationshipByIDs(update.start, update.end, update.kind, update.properties); err != nil { + return err + } + } + return nil + }, graph.WithBatchSize(2))) + + primary := directWriteFetchRelationship(t, ctx, db, a.ID, b.ID, directWriteCreateRelationshipKind) + require.Equal(t, "2026-01-01T00:00:00Z", directWriteStringProperty(t, primary.Properties, "firstseen")) + require.Equal(t, "2026-01-02T00:00:00Z", directWriteStringProperty(t, primary.Properties, directWriteLastSeen)) + require.Equal(t, "last", directWriteStringProperty(t, primary.Properties, "custom")) + require.Equal(t, "yes", directWriteStringProperty(t, primary.Properties, "preserved")) + // Neo4j removes a property set to null while PostgreSQL retains a JSONB null + // key. The shared graph API exposes nil in both cases. + require.Nil(t, primary.Properties.Get("nullable").Any()) + require.NotNil(t, directWriteFetchRelationship(t, ctx, db, b.ID, a.ID, directWriteCreateRelationshipKind)) + require.NotNil(t, directWriteFetchRelationship(t, ctx, db, a.ID, b.ID, directWriteCreateRelationshipOther)) + require.Empty(t, directWriteFetchRelationship(t, ctx, db, a.ID, c.ID, directWriteCreateRelationshipKind).Properties.MapOrEmpty()) + require.Equal(t, int64(3), countByCypher(t, ctx, db, "MATCH ()-[r:WriteCreateRelationship]->() RETURN count(r)")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH ()-[r:WriteCreateRelationshipOther]->() RETURN count(r)")) + + require.NoError(t, db.BatchOperation(ctx, func(batch graph.Batch) error { + return batch.CreateRelationshipByIDs(a.ID, b.ID, directWriteCreateRelationshipKind, directWriteProperties( + directWriteLastSeen, "2026-01-03T00:00:00Z", + "retry", "yes", + )) + })) + primary = directWriteFetchRelationship(t, ctx, db, a.ID, b.ID, directWriteCreateRelationshipKind) + require.Equal(t, "2026-01-03T00:00:00Z", directWriteStringProperty(t, primary.Properties, directWriteLastSeen)) + require.Equal(t, "last", directWriteStringProperty(t, primary.Properties, "custom")) + require.Equal(t, "yes", directWriteStringProperty(t, primary.Properties, "retry")) + require.Equal(t, int64(3), countByCypher(t, ctx, db, "MATCH ()-[r:WriteCreateRelationship]->() RETURN count(r)")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH ()-[r:WriteCreateRelationshipOther]->() RETURN count(r)")) +} + +// TestDirectWriteUpdateNodeBySemanticsAndBoundaries verifies identity-based node updates, replacements, and misses across flush boundaries. +func TestDirectWriteUpdateNodeBySemanticsAndBoundaries(t *testing.T) { + db, ctx := directWriteSetup(t) + + for _, size := range []int{1_000, 1_999, 2_000, 2_001} { + t.Run(fmt.Sprintf("WRITE-04 size %d", size), func(t *testing.T) { + ClearGraph(t, db, ctx) + require.NoError(t, db.BatchOperation(ctx, func(batch graph.Batch) error { + for idx := range size { + if err := batch.UpdateNodeBy(directWriteNodeUpdate( + fmt.Sprintf("node-boundary-%04d", idx), + directWriteUpsertNodeKind, + directWriteProperties(directWriteLastSeen, "2026-01-02T00:00:00Z", "ordinal", idx), + )); err != nil { + return err + } + } + return nil + }, graph.WithBatchSize(2_000))) + require.Equal(t, int64(size), countByCypher(t, ctx, db, "MATCH (n:WriteUpsertNode) RETURN count(n)")) + first := directWriteFetchNodeByObjectID(t, ctx, db, "node-boundary-0000") + require.Equal(t, "2026-01-02T00:00:00Z", directWriteStringProperty(t, first.Properties, directWriteLastSeen)) + }) + } + + t.Run("WRITE-04 insert update duplicates retry lastseen and kind merge", func(t *testing.T) { + ClearGraph(t, db, ctx) + existing := directWriteCreateNode(t, ctx, db, directWriteProperties( + directWriteObjectID, "node-existing", + directWriteLastSeen, "2026-01-01T00:00:00Z", + "preserved", "yes", + ), directWriteUpsertNodeKindA) + + require.NoError(t, db.BatchOperation(ctx, func(batch graph.Batch) error { + updates := []graph.NodeUpdate{ + directWriteNodeUpdate("node-new", directWriteUpsertNodeKindA, directWriteProperties(directWriteLastSeen, "2026-01-01T00:00:00Z", "custom", "first")), + directWriteNodeUpdate("node-new", directWriteUpsertNodeKindB, directWriteProperties(directWriteLastSeen, "2026-01-02T00:00:00Z", "custom", "within")), + directWriteNodeUpdate("node-new", directWriteUpsertNodeKindC, directWriteProperties(directWriteLastSeen, "2026-01-03T00:00:00Z", "custom", "last")), + directWriteNodeUpdate("node-existing", directWriteUpsertNodeKindB, directWriteProperties(directWriteLastSeen, "2026-01-02T00:00:00Z", "changed", true)), + } + for _, update := range updates { + if err := batch.UpdateNodeBy(update); err != nil { + return err + } + } + return nil + }, graph.WithBatchSize(2))) + + inserted := directWriteFetchNodeByObjectID(t, ctx, db, "node-new") + require.Equal(t, "2026-01-03T00:00:00Z", directWriteStringProperty(t, inserted.Properties, directWriteLastSeen)) + require.Equal(t, "last", directWriteStringProperty(t, inserted.Properties, "custom")) + require.True(t, inserted.Kinds.ContainsOneOf(directWriteUpsertNodeKindA)) + require.True(t, inserted.Kinds.ContainsOneOf(directWriteUpsertNodeKindB)) + require.True(t, inserted.Kinds.ContainsOneOf(directWriteUpsertNodeKindC)) + + updated := directWriteFetchNodeByObjectID(t, ctx, db, "node-existing") + require.Equal(t, existing.ID, updated.ID) + require.Equal(t, "yes", directWriteStringProperty(t, updated.Properties, "preserved")) + require.True(t, updated.Properties.Get("changed").Any().(bool)) + + require.NoError(t, db.BatchOperation(ctx, func(batch graph.Batch) error { + return batch.UpdateNodeBy(directWriteNodeUpdate("node-new", directWriteUpsertNodeKind, directWriteProperties( + directWriteLastSeen, "2026-01-04T00:00:00Z", + "retry", "yes", + ))) + })) + inserted = directWriteFetchNodeByObjectID(t, ctx, db, "node-new") + require.Equal(t, "2026-01-04T00:00:00Z", directWriteStringProperty(t, inserted.Properties, directWriteLastSeen)) + require.Equal(t, "yes", directWriteStringProperty(t, inserted.Properties, "retry")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH (n) WHERE n.objectid = 'node-new' RETURN count(n)")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH (n) WHERE n.objectid = 'node-existing' RETURN count(n)")) + }) +} + +// TestDirectWriteUpdateRelationshipBySemanticsAndBoundaries verifies relationship upsert semantics and survivor isolation across flush boundaries. +func TestDirectWriteUpdateRelationshipBySemanticsAndBoundaries(t *testing.T) { + db, ctx := directWriteSetup(t) + + for _, size := range []int{1_000, 1_999, 2_000, 2_001} { + t.Run(fmt.Sprintf("WRITE-05 size %d", size), func(t *testing.T) { + ClearGraph(t, db, ctx) + require.NoError(t, db.BatchOperation(ctx, func(batch graph.Batch) error { + for idx := range size { + if err := batch.UpdateRelationshipBy(directWriteRelationshipUpdate( + fmt.Sprintf("rel-source-%04d", idx), + fmt.Sprintf("rel-target-%04d", idx), + directWriteUpsertRelationshipKind, + directWriteProperties(directWriteLastSeen, "2026-01-02T00:00:00Z", "ordinal", idx), + )); err != nil { + return err + } + } + return nil + }, graph.WithBatchSize(2_000))) + require.Equal(t, int64(size*2), countByCypher(t, ctx, db, "MATCH (n:WriteEndpoint) RETURN count(n)")) + require.Equal(t, int64(size), countByCypher(t, ctx, db, "MATCH ()-[r:WriteUpsertRelationship]->() RETURN count(r)")) + }) + } + + t.Run("WRITE-05 endpoint upsert duplicate retry reverse kind and property merge", func(t *testing.T) { + ClearGraph(t, db, ctx) + a := directWriteCreateNode(t, ctx, db, directWriteProperties(directWriteObjectID, "rel-a", "preserved", "start"), directWriteEndpointKind) + b := directWriteCreateNode(t, ctx, db, directWriteProperties(directWriteObjectID, "rel-b", "preserved", "end"), directWriteEndpointKind) + + require.NoError(t, db.BatchOperation(ctx, func(batch graph.Batch) error { + updates := []graph.RelationshipUpdate{ + directWriteRelationshipUpdate("rel-a", "rel-b", directWriteUpsertRelationshipKind, directWriteProperties(directWriteLastSeen, "2026-01-01T00:00:00Z", "custom", "first", "preserved", "yes")), + directWriteRelationshipUpdate("rel-a", "rel-b", directWriteUpsertRelationshipKind, directWriteProperties(directWriteLastSeen, "2026-01-02T00:00:00Z", "custom", "within")), + directWriteRelationshipUpdate("rel-a", "rel-b", directWriteUpsertRelationshipKind, directWriteProperties(directWriteLastSeen, "2026-01-03T00:00:00Z", "custom", "last")), + directWriteRelationshipUpdate("rel-b", "rel-a", directWriteUpsertRelationshipKind, directWriteProperties("marker", "reverse")), + directWriteRelationshipUpdate("rel-a", "rel-b", directWriteUpsertRelationshipOther, directWriteProperties("marker", "other-kind")), + directWriteRelationshipUpdate("rel-missing-a", "rel-missing-b", directWriteUpsertRelationshipKind, directWriteProperties("marker", "missing-endpoints")), + } + for _, update := range updates { + if err := batch.UpdateRelationshipBy(update); err != nil { + return err + } + } + return nil + }, graph.WithBatchSize(2))) + + primary := directWriteFetchRelationship(t, ctx, db, a.ID, b.ID, directWriteUpsertRelationshipKind) + require.Equal(t, "2026-01-03T00:00:00Z", directWriteStringProperty(t, primary.Properties, directWriteLastSeen)) + require.Equal(t, "last", directWriteStringProperty(t, primary.Properties, "custom")) + require.Equal(t, "yes", directWriteStringProperty(t, primary.Properties, "preserved")) + require.NotNil(t, directWriteFetchRelationship(t, ctx, db, b.ID, a.ID, directWriteUpsertRelationshipKind)) + require.NotNil(t, directWriteFetchRelationship(t, ctx, db, a.ID, b.ID, directWriteUpsertRelationshipOther)) + missingStart := directWriteFetchNodeByObjectID(t, ctx, db, "rel-missing-a") + missingEnd := directWriteFetchNodeByObjectID(t, ctx, db, "rel-missing-b") + require.NotNil(t, directWriteFetchRelationship(t, ctx, db, missingStart.ID, missingEnd.ID, directWriteUpsertRelationshipKind)) + require.Equal(t, int64(4), countByCypher(t, ctx, db, "MATCH (n:WriteEndpoint) RETURN count(n)")) + require.Equal(t, int64(3), countByCypher(t, ctx, db, "MATCH ()-[r:WriteUpsertRelationship]->() RETURN count(r)")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH ()-[r:WriteUpsertRelationshipOther]->() RETURN count(r)")) + require.Equal(t, "start", directWriteStringProperty(t, directWriteFetchNodeByObjectID(t, ctx, db, "rel-a").Properties, "preserved")) + require.Equal(t, "end", directWriteStringProperty(t, directWriteFetchNodeByObjectID(t, ctx, db, "rel-b").Properties, "preserved")) + + require.NoError(t, db.BatchOperation(ctx, func(batch graph.Batch) error { + return batch.UpdateRelationshipBy(directWriteRelationshipUpdate("rel-a", "rel-b", directWriteUpsertRelationshipKind, directWriteProperties( + directWriteLastSeen, "2026-01-04T00:00:00Z", + "retry", "yes", + ))) + })) + primary = directWriteFetchRelationship(t, ctx, db, a.ID, b.ID, directWriteUpsertRelationshipKind) + require.Equal(t, "2026-01-04T00:00:00Z", directWriteStringProperty(t, primary.Properties, directWriteLastSeen)) + require.Equal(t, "last", directWriteStringProperty(t, primary.Properties, "custom")) + require.Equal(t, "yes", directWriteStringProperty(t, primary.Properties, "retry")) + require.Equal(t, int64(3), countByCypher(t, ctx, db, "MATCH ()-[r:WriteUpsertRelationship]->() RETURN count(r)")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH ()-[r:WriteUpsertRelationshipOther]->() RETURN count(r)")) + }) +} + +// TestDirectWriteReadThenCreateOrUpdateRelationship verifies the read-then-write path updates an existing edge or creates the missing edge exactly once. +func TestDirectWriteReadThenCreateOrUpdateRelationship(t *testing.T) { + db, ctx := directWriteSetup(t) + ClearGraph(t, db, ctx) + a, b, _ := directWriteCreateEndpoints(t, ctx, db, "ensure-a", "ensure-b", "ensure-unused") + + // A reverse-direction relationship is a decoy, not an existing exact key. + require.NoError(t, db.WriteTransaction(ctx, func(tx graph.Transaction) error { + _, err := tx.CreateRelationshipByIDs(b.ID, a.ID, directWriteEnsureRelationshipKind, directWriteProperties("marker", "reverse")) + return err + })) + + createdID, created, err := directWriteEnsureRelationship(ctx, db, a.ID, b.ID, directWriteEnsureRelationshipKind, directWriteProperties( + directWriteLastSeen, "2026-01-01T00:00:00Z", + "custom", "created", + )) + require.NoError(t, err) + require.True(t, created) + require.Equal(t, int64(2), countByCypher(t, ctx, db, "MATCH ()-[r:WriteEnsureRelationship]->() RETURN count(r)")) + + updatedID, created, err := directWriteEnsureRelationship(ctx, db, a.ID, b.ID, directWriteEnsureRelationshipKind, directWriteProperties( + directWriteLastSeen, "2026-01-02T00:00:00Z", + "custom", "updated", + "newproperty", "yes", + )) + require.NoError(t, err) + require.False(t, created) + require.Equal(t, createdID, updatedID) + + repeatedID, created, err := directWriteEnsureRelationship(ctx, db, a.ID, b.ID, directWriteEnsureRelationshipKind, directWriteProperties( + directWriteLastSeen, "2026-01-02T00:00:00Z", + "custom", "updated", + "newproperty", "yes", + )) + require.NoError(t, err) + require.False(t, created) + require.Equal(t, createdID, repeatedID) + require.Equal(t, int64(2), countByCypher(t, ctx, db, "MATCH ()-[r:WriteEnsureRelationship]->() RETURN count(r)")) + + relationship := directWriteFetchRelationship(t, ctx, db, a.ID, b.ID, directWriteEnsureRelationshipKind) + require.Equal(t, "2026-01-02T00:00:00Z", directWriteStringProperty(t, relationship.Properties, directWriteLastSeen)) + require.Equal(t, "updated", directWriteStringProperty(t, relationship.Properties, "custom")) + require.Equal(t, "yes", directWriteStringProperty(t, relationship.Properties, "newproperty")) + reverse := directWriteFetchRelationship(t, ctx, db, b.ID, a.ID, directWriteEnsureRelationshipKind) + require.Equal(t, "reverse", directWriteStringProperty(t, reverse.Properties, "marker")) +} + +// TestDirectWriteFullNodeUpdateAfterSelectors verifies selector results can be fully replaced without mutating unmatched nodes. +func TestDirectWriteFullNodeUpdateAfterSelectors(t *testing.T) { + db, ctx := directWriteSetup(t) + ClearGraph(t, db, ctx) + + suffix := directWriteCreateNode(t, ctx, db, directWriteProperties( + directWriteObjectID, "S-1-5-21-512", + "name", "old suffix name", + "preserved", "suffix", + ), directWriteEntityKind, directWriteSuffixKind, directWriteUnrelatedKind) + missing := directWriteCreateNode(t, ctx, db, directWriteProperties( + directWriteObjectID, "missing-name", + "preserved", "missing", + ), directWriteEntityKind, directWriteMissingKind, directWriteUnrelatedKind) + scan := directWriteCreateNode(t, ctx, db, directWriteProperties( + directWriteObjectID, "kind-scan", + "name", "old scan name", + "preserved", "scan", + ), directWriteEntityKind, directWriteScanKind, directWriteUnrelatedKind) + directWriteCreateNode(t, ctx, db, directWriteProperties( + directWriteObjectID, "S-1-5-21-513", + "name", "decoy", + ), directWriteEntityKind, directWriteUnrelatedKind) + + require.NoError(t, db.WriteTransaction(ctx, func(tx graph.Transaction) error { + selectedSuffix, err := tx.Nodes().Filterf(func() graph.Criteria { + return query.And( + query.Kind(query.Node(), directWriteSuffixKind), + query.StringEndsWith(query.NodeProperty(directWriteObjectID), "-512"), + ) + }).First() + if err != nil { + return err + } + selectedSuffix.Properties.Set("name", "new suffix name") + if err := tx.UpdateNode(selectedSuffix); err != nil { + return err + } + + selectedMissing, err := tx.Nodes().Filterf(func() graph.Criteria { + return query.And( + query.Kind(query.Node(), directWriteMissingKind), + query.Not(query.Exists(query.NodeProperty("name"))), + ) + }).First() + if err != nil { + return err + } + selectedMissing.AddKinds(directWriteGroupKind) + if err := tx.UpdateNode(selectedMissing); err != nil { + return err + } + + selectedScan, err := tx.Nodes().Filterf(func() graph.Criteria { + return query.Kind(query.Node(), directWriteScanKind) + }).First() + if err != nil { + return err + } + selectedScan.Properties.Set("name", "new scan name") + selectedScan.AddKinds(directWriteGroupKind) + return tx.UpdateNode(selectedScan) + })) + + updatedSuffix := directWriteFetchNodeByID(t, ctx, db, suffix.ID) + require.Equal(t, "new suffix name", directWriteStringProperty(t, updatedSuffix.Properties, "name")) + require.Equal(t, "suffix", directWriteStringProperty(t, updatedSuffix.Properties, "preserved")) + require.True(t, updatedSuffix.Kinds.ContainsOneOf(directWriteUnrelatedKind)) + require.False(t, updatedSuffix.Kinds.ContainsOneOf(directWriteGroupKind)) + + updatedMissing := directWriteFetchNodeByID(t, ctx, db, missing.ID) + require.False(t, updatedMissing.Properties.Exists("name")) + require.Equal(t, "missing", directWriteStringProperty(t, updatedMissing.Properties, "preserved")) + require.True(t, updatedMissing.Kinds.ContainsOneOf(directWriteGroupKind)) + require.True(t, updatedMissing.Kinds.ContainsOneOf(directWriteUnrelatedKind)) + + updatedScan := directWriteFetchNodeByID(t, ctx, db, scan.ID) + require.Equal(t, "new scan name", directWriteStringProperty(t, updatedScan.Properties, "name")) + require.Equal(t, "scan", directWriteStringProperty(t, updatedScan.Properties, "preserved")) + require.True(t, updatedScan.Kinds.ContainsOneOf(directWriteGroupKind)) + require.True(t, updatedScan.Kinds.ContainsOneOf(directWriteUnrelatedKind)) +} + +// TestDirectWriteExactKeyMissThenCreateNode verifies an exact-key miss followed by creation yields one correctly keyed node. +func TestDirectWriteExactKeyMissThenCreateNode(t *testing.T) { + db, ctx := directWriteSetup(t) + ClearGraph(t, db, ctx) + + _, err := directWriteFindNodeByObjectID(ctx, db, "well-known-new") + require.Error(t, err) + require.True(t, graph.IsErrNotFound(err), "selector must report an exact-key miss before the driver create") + + completeProperties := directWriteProperties( + directWriteObjectID, "well-known-new", + "name", "Well Known Group", + "domainsid", "S-1-5-21", + "domainfqdn", "example.test", + directWriteLastSeen, "2026-01-01T00:00:00Z", + ) + created, wasCreated, err := directWriteGetOrCreateGroup(ctx, db, completeProperties) + require.NoError(t, err) + require.True(t, wasCreated) + require.True(t, created.Kinds.ContainsOneOf(directWriteEntityKind)) + require.True(t, created.Kinds.ContainsOneOf(directWriteGroupKind)) + require.Equal(t, "Well Known Group", directWriteStringProperty(t, created.Properties, "name")) + require.Equal(t, "S-1-5-21", directWriteStringProperty(t, created.Properties, "domainsid")) + require.Equal(t, "example.test", directWriteStringProperty(t, created.Properties, "domainfqdn")) + + selectorHit, err := directWriteFindNodeByObjectID(ctx, db, "well-known-new") + require.NoError(t, err) + require.Equal(t, created.ID, selectorHit.ID) + + repeated, wasCreated, err := directWriteGetOrCreateGroup(ctx, db, completeProperties) + require.NoError(t, err) + require.False(t, wasCreated) + require.Equal(t, created.ID, repeated.ID) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH (n) WHERE n.objectid = 'well-known-new' RETURN count(n)")) + + existing := directWriteCreateNode(t, ctx, db, directWriteProperties( + directWriteObjectID, "well-known-existing", + "name", "Existing", + "preserved", "yes", + ), directWriteEntityKind, directWriteUnrelatedKind) + existingResult, wasCreated, err := directWriteGetOrCreateGroup(ctx, db, directWriteProperties( + directWriteObjectID, "well-known-existing", + "name", "replacement ignored", + )) + require.NoError(t, err) + require.False(t, wasCreated) + require.Equal(t, existing.ID, existingResult.ID) + require.True(t, existingResult.Kinds.ContainsOneOf(directWriteGroupKind)) + require.True(t, existingResult.Kinds.ContainsOneOf(directWriteUnrelatedKind)) + require.Equal(t, "yes", directWriteStringProperty(t, existingResult.Properties, "preserved")) + require.Equal(t, "Existing", directWriteStringProperty(t, existingResult.Properties, "name")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH (n) WHERE n.objectid = 'well-known-existing' RETURN count(n)")) +} + +// BenchmarkMutationSafeDirectWrites measures guarded direct-write workloads across representative batch sizes. +func BenchmarkMutationSafeDirectWrites(b *testing.B) { + session := Open(b, Options{ + Schema: directWriteSchema(), + CleanupMode: CleanupGraph, + }) + + for _, size := range []int{1_000, 2_000, 2_001} { + b.Run(fmt.Sprintf("size-%d", size), func(b *testing.B) { + b.Run("WRITE-01 DeleteRelationship", func(b *testing.B) { + b.ReportAllocs() + for range b.N { + b.StopTimer() + directWriteClearBenchmarkGraph(b, session) + if _, err := opengraph.WriteGraph(session.Ctx, session.DB, testutil.NewDirectWriteScaleFixture(size)); err != nil { + b.Fatalf("load fixture: %v", err) + } + ids, err := directWriteRelationshipIDs(session.Ctx, session.DB, func() graph.Criteria { + return query.And( + query.Kind(query.Relationship(), directWriteDeleteRelationshipKind), + query.Equals(query.RelationshipProperty("deletebatch"), true), + ) + }) + if err != nil { + b.Fatalf("select relationship IDs: %v", err) + } + b.StartTimer() + if err := session.DB.BatchOperation(session.Ctx, func(batch graph.Batch) error { + for _, id := range ids { + if err := batch.DeleteRelationship(id); err != nil { + return err + } + } + return nil + }, graph.WithBatchSize(2_000)); err != nil { + b.Fatalf("delete relationships: %v", err) + } + b.StopTimer() + if remaining, err := directWriteCount(session.Ctx, session.DB, "MATCH ()-[r:WriteDeleteRelationship]->() RETURN count(r)"); err != nil || remaining != 1 { + b.Fatalf("remaining relationships: got %d, err %v", remaining, err) + } + } + }) + + b.Run("WRITE-02 DeleteNode cascade", func(b *testing.B) { + b.ReportAllocs() + for range b.N { + b.StopTimer() + directWriteClearBenchmarkGraph(b, session) + idMap, err := opengraph.WriteGraph(session.Ctx, session.DB, testutil.NewDirectWriteScaleFixture(size)) + if err != nil { + b.Fatalf("load fixture: %v", err) + } + ids := make([]graph.ID, 0, size) + for _, name := range testutil.FixtureNames("write-target", size) { + ids = append(ids, idMap[name]) + } + b.StartTimer() + if err := session.DB.BatchOperation(session.Ctx, func(batch graph.Batch) error { + for _, id := range ids { + if err := batch.DeleteNode(id); err != nil { + return err + } + } + return nil + }, graph.WithBatchSize(2_000)); err != nil { + b.Fatalf("delete nodes: %v", err) + } + b.StopTimer() + if remaining, err := directWriteCount(session.Ctx, session.DB, "MATCH (n:WriteDeleteNode) RETURN count(n)"); err != nil || remaining != 0 { + b.Fatalf("remaining nodes: got %d, err %v", remaining, err) + } + if survivors, err := directWriteCount(session.Ctx, session.DB, "MATCH ()-[r:WriteSurvivor]->() RETURN count(r)"); err != nil || survivors != 1 { + b.Fatalf("survivor relationships: got %d, err %v", survivors, err) + } + } + }) + + b.Run("WRITE-03 CreateRelationship conflict merge", func(b *testing.B) { + b.ReportAllocs() + for range b.N { + b.StopTimer() + directWriteClearBenchmarkGraph(b, session) + idMap, err := opengraph.WriteGraph(session.Ctx, session.DB, testutil.NewDirectWriteScaleFixture(size)) + if err != nil { + b.Fatalf("load fixture: %v", err) + } + rootID := idMap["write-root"] + b.StartTimer() + if err := session.DB.BatchOperation(session.Ctx, func(batch graph.Batch) error { + for idx, name := range testutil.FixtureNames("write-target", size) { + if err := batch.CreateRelationshipByIDs(rootID, idMap[name], directWriteCreateRelationshipKind, directWriteProperties("ordinal", idx, "custom", "first")); err != nil { + return err + } + if err := batch.CreateRelationshipByIDs(rootID, idMap[name], directWriteCreateRelationshipKind, directWriteProperties("custom", "last")); err != nil { + return err + } + } + return nil + }, graph.WithBatchSize(2_000)); err != nil { + b.Fatalf("create relationships: %v", err) + } + b.StopTimer() + if created, err := directWriteCount(session.Ctx, session.DB, "MATCH ()-[r:WriteCreateRelationship]->() RETURN count(r)"); err != nil || created != int64(size) { + b.Fatalf("created relationships: got %d, want %d, err %v", created, size, err) + } + if merged, err := directWriteCount(session.Ctx, session.DB, "MATCH ()-[r:WriteCreateRelationship]->() WHERE r.custom = 'last' RETURN count(r)"); err != nil || merged != int64(size) { + b.Fatalf("merged relationships: got %d, want %d, err %v", merged, size, err) + } + } + }) + + b.Run("WRITE-04 UpdateNodeBy", func(b *testing.B) { + b.ReportAllocs() + for range b.N { + b.StopTimer() + directWriteClearBenchmarkGraph(b, session) + b.StartTimer() + if err := session.DB.BatchOperation(session.Ctx, func(batch graph.Batch) error { + for idx := range size { + if err := batch.UpdateNodeBy(directWriteNodeUpdate(fmt.Sprintf("bench-node-%04d", idx), directWriteUpsertNodeKind, directWriteProperties("ordinal", idx))); err != nil { + return err + } + } + return nil + }, graph.WithBatchSize(2_000)); err != nil { + b.Fatalf("update nodes: %v", err) + } + b.StopTimer() + if updated, err := directWriteCount(session.Ctx, session.DB, "MATCH (n:WriteUpsertNode) RETURN count(n)"); err != nil || updated != int64(size) { + b.Fatalf("updated nodes: got %d, want %d, err %v", updated, size, err) + } + } + }) + + b.Run("WRITE-05 UpdateRelationshipBy", func(b *testing.B) { + b.ReportAllocs() + for range b.N { + b.StopTimer() + directWriteClearBenchmarkGraph(b, session) + b.StartTimer() + if err := session.DB.BatchOperation(session.Ctx, func(batch graph.Batch) error { + for idx := range size { + if err := batch.UpdateRelationshipBy(directWriteRelationshipUpdate( + fmt.Sprintf("bench-source-%04d", idx), + fmt.Sprintf("bench-target-%04d", idx), + directWriteUpsertRelationshipKind, + directWriteProperties("ordinal", idx), + )); err != nil { + return err + } + } + return nil + }, graph.WithBatchSize(2_000)); err != nil { + b.Fatalf("update relationships: %v", err) + } + b.StopTimer() + if updated, err := directWriteCount(session.Ctx, session.DB, "MATCH ()-[r:WriteUpsertRelationship]->() RETURN count(r)"); err != nil || updated != int64(size) { + b.Fatalf("updated relationships: got %d, want %d, err %v", updated, size, err) + } + } + }) + }) + } +} + +// directWriteSetup opens a guarded integration session with the mutation fixture schema and returns its database context. +func directWriteSetup(t *testing.T) (graph.Database, context.Context) { + t.Helper() + session := Open(t, Options{ + Schema: directWriteSchema(), + CleanupMode: CleanupGraph, + }) + return session.DB, session.Ctx +} + +// directWriteSchema returns the graph schema containing every kind used by direct-write fixtures and assertions. +func directWriteSchema() *graph.Schema { + nodeKinds, edgeKinds := directWriteKinds() + graphSchema := graph.Graph{ + Name: "integration_test", + Nodes: nodeKinds, + Edges: edgeKinds, + NodeConstraints: []graph.Constraint{{ + Field: directWriteObjectID, + Type: graph.BTreeIndex, + }}, + } + return &graph.Schema{ + Graphs: []graph.Graph{graphSchema}, + DefaultGraph: graphSchema, + } +} + +// directWriteKinds returns every node and relationship kind required by the +// direct-write fixture and mutation cases. +func directWriteKinds() (graph.Kinds, graph.Kinds) { + fixtureNodeKinds, fixtureEdgeKinds := testutil.NewDirectWriteScaleFixture(2).Kinds() + nodeKinds := fixtureNodeKinds.Add( + directWriteUpsertNodeKind, + directWriteUpsertNodeKindA, + directWriteUpsertNodeKindB, + directWriteUpsertNodeKindC, + directWriteEntityKind, + directWriteGroupKind, + directWriteUnrelatedKind, + directWriteSuffixKind, + directWriteMissingKind, + directWriteScanKind, + ) + edgeKinds := fixtureEdgeKinds.Add( + directWriteCreateRelationshipKind, + directWriteCreateRelationshipOther, + directWriteUpsertRelationshipKind, + directWriteUpsertRelationshipOther, + directWriteEnsureRelationshipKind, + ) + return nodeKinds, edgeKinds +} + +// directWriteLoadDirectWriteFixture clears the database, loads a generated +// direct-write graph, and returns both the fixture and its database ID map. +func directWriteLoadDirectWriteFixture(t *testing.T, ctx context.Context, db graph.Database, size int) (*opengraph.Graph, opengraph.IDMap) { + t.Helper() + ClearGraph(t, db, ctx) + fixture := testutil.NewDirectWriteScaleFixture(size) + idMap, err := opengraph.WriteGraph(ctx, db, fixture) + require.NoError(t, err) + return fixture, idMap +} + +// directWriteCreateEndpoints creates the three endpoint nodes required by relationship mutation cases. +func directWriteCreateEndpoints(t *testing.T, ctx context.Context, db graph.Database, objectIDs ...string) (*graph.Node, *graph.Node, *graph.Node) { + t.Helper() + require.Len(t, objectIDs, 3) + created := make([]*graph.Node, 0, len(objectIDs)) + require.NoError(t, db.WriteTransaction(ctx, func(tx graph.Transaction) error { + for _, objectID := range objectIDs { + node, err := tx.CreateNode(directWriteProperties(directWriteObjectID, objectID), directWriteEndpointKind) + if err != nil { + return err + } + created = append(created, node) + } + return nil + })) + return created[0], created[1], created[2] +} + +// directWriteCreateNode creates one node in a committed transaction and returns its database-assigned identity. +func directWriteCreateNode(t *testing.T, ctx context.Context, db graph.Database, properties *graph.Properties, kinds ...graph.Kind) *graph.Node { + t.Helper() + var created *graph.Node + require.NoError(t, db.WriteTransaction(ctx, func(tx graph.Transaction) error { + var err error + created, err = tx.CreateNode(properties, kinds...) + return err + })) + return created +} + +// directWriteProperties constructs a property bag from alternating string keys and values. +func directWriteProperties(keyValues ...any) *graph.Properties { + properties := graph.NewProperties() + for idx := 0; idx < len(keyValues); idx += 2 { + properties.Set(keyValues[idx].(string), keyValues[idx+1]) + } + return properties +} + +// directWriteIncidentCount returns the expected number of fixture relationships incident to targets nodes. +func directWriteIncidentCount(targets int) int64 { + switch targets { + case 0: + return 0 + case 1: + return 1 + default: + return int64(targets + 1) + } +} + +// directWriteNodeUpdate builds an identity-property node upsert while preserving objectID in the replacement properties. +func directWriteNodeUpdate(objectID string, kind graph.Kind, properties *graph.Properties) graph.NodeUpdate { + properties = properties.Clone().Set(directWriteObjectID, objectID) + return graph.NodeUpdate{ + Node: graph.PrepareNode(properties, kind), + IdentityProperties: []string{directWriteObjectID}, + } +} + +// directWriteRelationshipUpdate builds a relationship upsert whose endpoints are selected by objectID. +func directWriteRelationshipUpdate(startObjectID, endObjectID string, kind graph.Kind, properties *graph.Properties) graph.RelationshipUpdate { + return graph.RelationshipUpdate{ + Start: graph.PrepareNode( + directWriteProperties(directWriteObjectID, startObjectID), + directWriteEndpointKind, + ), + StartIdentityProperties: []string{directWriteObjectID}, + End: graph.PrepareNode( + directWriteProperties(directWriteObjectID, endObjectID), + directWriteEndpointKind, + ), + EndIdentityProperties: []string{directWriteObjectID}, + Relationship: graph.PrepareRelationship(properties, kind), + } +} + +// directWriteFetchRelationshipIDs returns matching relationship IDs and fails the current test on query error. +func directWriteFetchRelationshipIDs(t *testing.T, ctx context.Context, db graph.Database, criteria graph.CriteriaProvider) []graph.ID { + t.Helper() + ids, err := directWriteRelationshipIDs(ctx, db, criteria) + require.NoError(t, err) + return ids +} + +// directWriteRelationshipIDs queries the IDs of relationships matching criteria in a read transaction. +func directWriteRelationshipIDs(ctx context.Context, db graph.Database, criteria graph.CriteriaProvider) ([]graph.ID, error) { + var ids []graph.ID + if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { + var err error + ids, err = ops.FetchRelationshipIDs(tx.Relationships().Filterf(criteria)) + return err + }); err != nil { + return nil, err + } + + return ids, nil +} + +// directWriteFetchRelationship returns the relationship with the exact endpoints and kind, failing the current test when absent. +func directWriteFetchRelationship(t *testing.T, ctx context.Context, db graph.Database, startID, endID graph.ID, kind graph.Kind) *graph.Relationship { + t.Helper() + var relationship *graph.Relationship + require.NoError(t, db.ReadTransaction(ctx, func(tx graph.Transaction) error { + var err error + relationship, err = tx.Relationships().Filterf(func() graph.Criteria { + return query.And( + query.Equals(query.StartID(), startID), + query.Equals(query.EndID(), endID), + query.Kind(query.Relationship(), kind), + ) + }).First() + return err + })) + return relationship +} + +// directWriteFetchNodeByObjectID returns the node selected by objectID and fails the current test on lookup error. +func directWriteFetchNodeByObjectID(t *testing.T, ctx context.Context, db graph.Database, objectID string) *graph.Node { + t.Helper() + node, err := directWriteFindNodeByObjectID(ctx, db, objectID) + require.NoError(t, err) + return node +} + +// directWriteFindNodeByObjectID queries the single node selected by objectID. +func directWriteFindNodeByObjectID(ctx context.Context, db graph.Database, objectID string) (*graph.Node, error) { + var node *graph.Node + if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { + var err error + node, err = tx.Nodes().Filterf(func() graph.Criteria { + return query.Equals(query.NodeProperty(directWriteObjectID), objectID) + }).First() + return err + }); err != nil { + return nil, err + } + + return node, nil +} + +// directWriteFetchNodeByID returns the node selected by database ID and fails the current test on lookup error. +func directWriteFetchNodeByID(t *testing.T, ctx context.Context, db graph.Database, id graph.ID) *graph.Node { + t.Helper() + var node *graph.Node + require.NoError(t, db.ReadTransaction(ctx, func(tx graph.Transaction) error { + var err error + node, err = tx.Nodes().Filter(query.Equals(query.NodeID(), id)).First() + return err + })) + return node +} + +// directWriteStringProperty reads key as a string and fails the current test when the value is absent or incompatible. +func directWriteStringProperty(t *testing.T, properties *graph.Properties, key string) string { + t.Helper() + value, err := properties.Get(key).String() + require.NoError(t, err) + return value +} + +// directWriteEnsureRelationship updates the exact relationship when present or creates it when absent, reporting which path ran. +func directWriteEnsureRelationship(ctx context.Context, db graph.Database, startID, endID graph.ID, kind graph.Kind, properties *graph.Properties) (graph.ID, bool, error) { + var ( + id graph.ID + created bool + ) + if err := db.WriteTransaction(ctx, func(tx graph.Transaction) error { + if relationship, err := tx.Relationships().Filterf(func() graph.Criteria { + return query.And( + query.Equals(query.StartID(), startID), + query.Equals(query.EndID(), endID), + query.Kind(query.Relationship(), kind), + ) + }).First(); err != nil { + if !graph.IsErrNotFound(err) { + return err + } + + if createdRelationship, err := tx.CreateRelationshipByIDs(startID, endID, kind, properties); err != nil { + return err + } else { + id = createdRelationship.ID + created = true + return nil + } + } else { + relationship.Properties.Merge(properties) + id = relationship.ID + return tx.UpdateRelationship(relationship) + } + }); err != nil { + return 0, false, err + } + + return id, created, nil +} + +// directWriteGetOrCreateGroup returns the group selected by objectID or creates it atomically when missing. +func directWriteGetOrCreateGroup(ctx context.Context, db graph.Database, properties *graph.Properties) (*graph.Node, bool, error) { + objectID, err := properties.Get(directWriteObjectID).String() + if err != nil { + return nil, false, err + } + + var ( + result *graph.Node + created bool + ) + if err := db.WriteTransaction(ctx, func(tx graph.Transaction) error { + if existing, err := tx.Nodes().Filterf(func() graph.Criteria { + return query.Equals(query.NodeProperty(directWriteObjectID), objectID) + }).First(); err != nil { + if !graph.IsErrNotFound(err) { + return err + } + + if createdNode, err := tx.CreateNode(properties.Clone(), directWriteEntityKind, directWriteGroupKind); err != nil { + return err + } else { + result = createdNode + created = true + return nil + } + } else { + result = existing + if !result.Kinds.ContainsOneOf(directWriteGroupKind) { + result.AddKinds(directWriteGroupKind) + return tx.UpdateNode(result) + } + + return nil + } + }); err != nil { + return nil, false, err + } + + return result, created, nil +} + +// directWriteClearBenchmarkGraph removes every benchmark node and its incident relationships before the next iteration. +func directWriteClearBenchmarkGraph(b *testing.B, session *Session) { + b.Helper() + if err := session.DB.WriteTransaction(session.Ctx, func(tx graph.Transaction) error { + return tx.Nodes().Delete() + }); err != nil { + b.Fatalf("clear benchmark graph: %v", err) + } +} + +// directWriteCount executes a scalar Cypher count query and returns its first value. +func directWriteCount(ctx context.Context, db graph.Database, cypher string) (int64, error) { + var count int64 + if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { + result := tx.Query(cypher, nil) + defer result.Close() + + if !result.Next() { + return result.Error() + } + if err := result.Scan(&count); err != nil { + return err + } + return result.Error() + }); err != nil { + return 0, err + } + + return count, nil +} diff --git a/integration/harness.go b/integration/harness.go index fa568613..4f964731 100644 --- a/integration/harness.go +++ b/integration/harness.go @@ -29,6 +29,7 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "github.com/specterops/dawgs" + "github.com/specterops/dawgs/databaseguard" "github.com/specterops/dawgs/drivers/neo4j" "github.com/specterops/dawgs/drivers/pg" "github.com/specterops/dawgs/graph" @@ -36,17 +37,24 @@ import ( "github.com/specterops/dawgs/util/size" ) +// ConnectionStringEnv names the default environment variable read by integration sessions. const ConnectionStringEnv = "CONNECTION_STRING" var ( - localDatasetFlag = flag.String("local-dataset", "", "name of a local dataset to test (e.g. local/phantom)") + // localDatasetFlag optionally restricts the integration harness to one local dataset. + localDatasetFlag = flag.String("local-dataset", "", "name of a local dataset to test (e.g. local/phantom)") + + // errFixtureRollback is the sentinel returned to force successful fixture transactions to roll back. errFixtureRollback = errors.New("fixture rollback") ) type CleanupMode int const ( + // CleanupGraph removes graph data when an integration session closes. CleanupGraph CleanupMode = iota + + // CloseOnly closes an integration session without deleting graph data. CloseOnly ) @@ -90,7 +98,8 @@ func DriverFromConnectionString(connStr string) (string, error) { } } -func Open(t *testing.T, opts Options) *Session { +// Open validates the configured disposable target, initializes its schema, and returns an integration session registered for cleanup. +func Open(t testing.TB, opts Options) *Session { t.Helper() ctx := context.Background() @@ -106,6 +115,9 @@ func Open(t *testing.T, opts Options) *Session { } t.Fatalf("%s env var is not set", connEnv) } + if err := databaseguard.ValidateEnvironment(connStr); err != nil { + t.Fatalf("integration database safety check failed: %v", err) + } driver, err := DriverFromConnectionString(connStr) if err != nil { @@ -226,6 +238,7 @@ func (s *Session) WithRollback(t *testing.T, delegate func(tx graph.Transaction) return s.withRollback(t, delegate) } +// withRollback runs delegate in a write transaction and converts the fixture rollback sentinel into success. func (s *Session) withRollback(t *testing.T, delegate func(tx graph.Transaction) error) error { t.Helper() @@ -243,7 +256,8 @@ func (s *Session) withRollback(t *testing.T, delegate func(tx graph.Transaction) return err } -func buildSchema(t *testing.T, opts Options) *graph.Schema { +// buildSchema combines kinds discovered from selected datasets with explicitly requested kinds. +func buildSchema(t testing.TB, opts Options) *graph.Schema { t.Helper() nodeKinds, edgeKinds := collectKinds(t, opts.Datasets, opts.datasetPath()) @@ -270,7 +284,7 @@ func buildSchema(t *testing.T, opts Options) *graph.Schema { } // collectKinds parses the given datasets and returns the union of all node and edge kinds. -func collectKinds(t *testing.T, datasets []string, datasetPath func(name string) string) (graph.Kinds, graph.Kinds) { +func collectKinds(t testing.TB, datasets []string, datasetPath func(name string) string) (graph.Kinds, graph.Kinds) { t.Helper() var nodeKinds, edgeKinds graph.Kinds @@ -295,6 +309,7 @@ func collectKinds(t *testing.T, datasets []string, datasetPath func(name string) return nodeKinds, edgeKinds } +// datasetPath returns the configured dataset resolver or the repository testdata resolver. func (s *Options) datasetPath() func(name string) string { if s.DatasetPath != nil { return s.DatasetPath @@ -305,6 +320,8 @@ func (s *Options) datasetPath() func(name string) string { } } +// graphQueryMemoryLimit returns the backend's configured query memory limit, +// defaulting to unlimited when the driver does not expose one. func (s Options) graphQueryMemoryLimit() size.Size { if s.GraphQueryMemoryLimit == 0 { return size.Gibibyte diff --git a/integration/legacy_query_harness.go b/integration/legacy_query_harness.go new file mode 100644 index 00000000..f53244dc --- /dev/null +++ b/integration/legacy_query_harness.go @@ -0,0 +1,75 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +//go:build manual_integration + +package integration + +import ( + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/opengraph" +) + +// WithLegacyNodeQuery executes legacy query-builder criteria directly through +// the selected backend and keeps fixture setup, execution, and assertions in a +// single rollback transaction. +func WithLegacyNodeQuery( + t *testing.T, + session *Session, + fixture *opengraph.Graph, + criteriaProvider func(idMap opengraph.IDMap) graph.Criteria, + delegate func(query graph.NodeQuery, idMap opengraph.IDMap) error, +) { + t.Helper() + + err := session.WithRollbackFixture(t, fixture, false, func(tx graph.Transaction, idMap opengraph.IDMap) error { + query := tx.Nodes() + if criteriaProvider != nil { + query = query.Filter(criteriaProvider(idMap)) + } + + return delegate(query, idMap) + }) + if err != nil { + t.Fatalf("legacy node query failed: %v", err) + } +} + +// WithLegacyRelationshipQuery is the relationship-query counterpart to +// WithLegacyNodeQuery. +func WithLegacyRelationshipQuery( + t *testing.T, + session *Session, + fixture *opengraph.Graph, + criteriaProvider func(idMap opengraph.IDMap) graph.Criteria, + delegate func(query graph.RelationshipQuery, idMap opengraph.IDMap) error, +) { + t.Helper() + + err := session.WithRollbackFixture(t, fixture, false, func(tx graph.Transaction, idMap opengraph.IDMap) error { + query := tx.Relationships() + if criteriaProvider != nil { + query = query.Filter(criteriaProvider(idMap)) + } + + return delegate(query, idMap) + }) + if err != nil { + t.Fatalf("legacy relationship query failed: %v", err) + } +} diff --git a/integration/logical_forms_legacy_builder_test.go b/integration/logical_forms_legacy_builder_test.go new file mode 100644 index 00000000..d1517606 --- /dev/null +++ b/integration/logical_forms_legacy_builder_test.go @@ -0,0 +1,441 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +//go:build manual_integration + +package integration + +import ( + "sort" + "testing" + "time" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/opengraph" + "github.com/specterops/dawgs/query" + "github.com/stretchr/testify/require" +) + +// TestLegacyBuilderLogicalForms verifies legacy logical predicates preserve grouping, precedence, and result identity. +func TestLegacyBuilderLogicalForms(t *testing.T) { + logicFixture := logicalFormsFixture() + projectionFixture := logicalProjectionFixture() + logicNodeKinds, logicEdgeKinds := logicFixture.Kinds() + projectionNodeKinds, projectionEdgeKinds := projectionFixture.Kinds() + + db, ctx := SetupDBWithKindsNoGraphCleanup( + t, + logicNodeKinds.Add(projectionNodeKinds...), + logicEdgeKinds.Add(projectionEdgeKinds...), + ) + ClearGraph(t, db, ctx) + session := &Session{ + DB: db, + Ctx: ctx, + } + + t.Run("LOGIC-01 branch-local relationship kinds", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, logicFixture, func(idMap opengraph.IDMap) graph.Criteria { + forwardID := idMap["direction-forward"] + reverseID := idMap["direction-reverse"] + return query.And( + query.Kind(query.Start(), graph.StringKind("LogicDomain")), + query.Kind(query.End(), graph.StringKind("LogicDomain")), + query.Or( + query.And( + query.Equals(query.StartID(), forwardID), + query.Equals(query.EndID(), reverseID), + query.KindIn(query.Relationship(), graph.StringKind("LogicKindA")), + ), + query.And( + query.Equals(query.StartID(), reverseID), + query.Equals(query.EndID(), forwardID), + query.KindIn(query.Relationship(), graph.StringKind("LogicKindB")), + ), + ), + ) + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + var ids []graph.ID + err := relationshipQuery.FetchIDs(func(cursor graph.Cursor[graph.ID]) error { + for id := range cursor.Chan() { + ids = append(ids, id) + } + return cursor.Error() + }) + require.NoError(t, err) + require.Len(t, ids, 2, "both invalid kind/direction combinations must remain excluded") + require.NotEqual(t, ids[0], ids[1]) + return nil + }) + }) + + t.Run("LOGIC-02 cross-binding temporal disjunction", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, logicFixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Kind(query.Relationship(), graph.StringKind("LogicStaleTrust")), + query.Or( + query.BeforeGraphQuery(query.RelationshipProperty("lastseen"), query.StartProperty("lastcollected")), + query.BeforeGraphQuery(query.RelationshipProperty("lastseen"), query.EndProperty("lastcollected")), + ), + ) + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + var markers []string + err := relationshipQuery.Fetch(func(cursor graph.Cursor[*graph.Relationship]) error { + for relationship := range cursor.Chan() { + marker, err := relationship.Properties.Get("marker").String() + require.NoError(t, err) + markers = append(markers, marker) + } + return cursor.Error() + }) + require.NoError(t, err) + sort.Strings(markers) + require.Equal(t, []string{"older-both", "older-end-only", "older-start-only"}, markers) + return nil + }) + }) + + t.Run("LOGIC-03 scoped negation and null-aware age predicate", func(t *testing.T) { + threshold := time.Date(2026, time.January, 3, 0, 0, 0, 0, time.UTC) + WithLegacyNodeQuery(t, session, logicFixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Not(query.KindIn(query.Node(), graph.StringKind("LogicProtected"))), + query.Or( + query.Not(query.Exists(query.NodeProperty("lastseen"))), + query.Before(query.NodeProperty("lastseen"), threshold), + ), + ) + }, func(nodeQuery graph.NodeQuery, idMap opengraph.IDMap) error { + var fixtureIDs []string + err := nodeQuery.FetchIDs(func(cursor graph.Cursor[graph.ID]) error { + for id := range cursor.Chan() { + fixtureIDs = append(fixtureIDs, regressionFixtureID(t, idMap, id)) + } + return cursor.Error() + }) + require.NoError(t, err) + sort.Strings(fixtureIDs) + require.Equal(t, []string{"candidate-missing", "candidate-null", "candidate-older", "direction-forward", "direction-reverse", "early-a", "early-b", "equal-a", "equal-b", "late-a", "late-b"}, fixtureIDs) + return nil + }) + }) + + t.Run("LOGIC-05 projection order and Go result types", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, projectionFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.Kind(query.Relationship(), graph.StringKind("LogicProjectionEdge")), + query.Equals(query.StartID(), idMap["projection-start"]), + ) + }, func(relationshipQuery graph.RelationshipQuery, idMap opengraph.IDMap) error { + err := relationshipQuery.FetchDirection(graph.DirectionInbound, func(cursor graph.Cursor[graph.DirectionalResult]) error { + results := make([]graph.DirectionalResult, 0, 1) + for result := range cursor.Chan() { + results = append(results, result) + } + require.NoError(t, cursor.Error()) + require.Len(t, results, 1) + require.IsType(t, &graph.Relationship{}, results[0].Relationship) + require.IsType(t, &graph.Node{}, results[0].Node) + require.Equal(t, idMap["projection-end"], results[0].Node.ID) + return nil + }) + require.NoError(t, err) + + err = relationshipQuery.Query(func(result graph.Result) error { + require.True(t, result.Next()) + var ( + nodeID, relationshipID graph.ID + nodeKinds graph.Kinds + relationshipKind graph.Kind + ) + + require.NoError(t, result.Scan(&nodeID, &nodeKinds, &relationshipID, &relationshipKind)) + require.Equal(t, idMap["projection-end"], nodeID) + require.Equal(t, graph.StringKind("LogicProjectionEdge"), relationshipKind) + require.Contains(t, nodeKinds, graph.StringKind("LogicProjectionEnd")) + require.NotZero(t, relationshipID) + require.False(t, result.Next()) + return result.Error() + }, query.Returning( + query.EndID(), + query.KindsOf(query.End()), + query.RelationshipID(), + query.KindsOf(query.Relationship()), + )) + require.NoError(t, err) + + err = relationshipQuery.FetchTriples(func(cursor graph.Cursor[graph.RelationshipTripleResult]) error { + triples := make([]graph.RelationshipTripleResult, 0, 1) + for triple := range cursor.Chan() { + triples = append(triples, triple) + } + require.NoError(t, cursor.Error()) + require.Len(t, triples, 1) + require.Equal(t, []graph.RelationshipTripleResult{{ + ID: triples[0].ID, + StartID: idMap["projection-start"], + EndID: idMap["projection-end"], + }}, triples) + return nil + }) + require.NoError(t, err) + + err = relationshipQuery.FetchIDs(func(cursor graph.Cursor[graph.ID]) error { + ids := make([]graph.ID, 0, 1) + for id := range cursor.Chan() { + ids = append(ids, id) + } + require.NoError(t, cursor.Error()) + require.Len(t, ids, 1) + return nil + }) + require.NoError(t, err) + + err = relationshipQuery.Fetch(func(cursor graph.Cursor[*graph.Relationship]) error { + relationships := make([]*graph.Relationship, 0, 1) + for relationship := range cursor.Chan() { + relationships = append(relationships, relationship) + } + require.NoError(t, cursor.Error()) + require.Len(t, relationships, 1) + require.IsType(t, &graph.Relationship{}, relationships[0]) + return nil + }) + require.NoError(t, err) + return nil + }) + }) +} + +// logicalFormsFixture builds direction, null, time, and boolean property cases +// for logical criteria regressions. +func logicalFormsFixture() *opengraph.Graph { + day := func(day int) time.Time { + return time.Date(2026, time.January, day, 0, 0, 0, 0, time.UTC) + } + + return &opengraph.Graph{ + Nodes: []opengraph.Node{ + { + ID: "direction-forward", + Kinds: []string{"LogicDomain"}, + Properties: map[string]any{"name": "forward"}, + }, + { + ID: "direction-reverse", + Kinds: []string{"LogicDomain"}, + Properties: map[string]any{"name": "reverse"}, + }, + { + ID: "early-a", + Kinds: []string{"LogicDomain"}, + Properties: map[string]any{"lastcollected": day(2)}, + }, + { + ID: "early-b", + Kinds: []string{"LogicDomain"}, + Properties: map[string]any{"lastcollected": day(2)}, + }, + { + ID: "equal-a", + Kinds: []string{"LogicDomain"}, + Properties: map[string]any{"lastcollected": day(3)}, + }, + { + ID: "equal-b", + Kinds: []string{"LogicDomain"}, + Properties: map[string]any{"lastcollected": day(3)}, + }, + { + ID: "late-a", + Kinds: []string{"LogicDomain"}, + Properties: map[string]any{"lastcollected": day(4)}, + }, + { + ID: "late-b", + Kinds: []string{"LogicDomain"}, + Properties: map[string]any{"lastcollected": day(4)}, + }, + { + ID: "late-b-newer", + Kinds: []string{"LogicDomain"}, + Properties: map[string]any{"lastcollected": day(4), "lastseen": day(4)}, + }, + { + ID: "late-b-missing", + Kinds: []string{"LogicDomain"}, + Properties: map[string]any{"lastcollected": day(4), "lastseen": day(4)}, + }, + { + ID: "late-b-null", + Kinds: []string{"LogicDomain"}, + Properties: map[string]any{"lastcollected": day(4), "lastseen": day(4)}, + }, + { + ID: "candidate-missing", + Kinds: []string{"LogicCandidate"}, + Properties: map[string]any{}, + }, + { + ID: "candidate-null", + Kinds: []string{"LogicCandidate"}, + Properties: map[string]any{"lastseen": nil}, + }, + { + ID: "candidate-older", + Kinds: []string{"LogicCandidate"}, + Properties: map[string]any{"lastseen": day(2)}, + }, + { + ID: "candidate-equal", + Kinds: []string{"LogicCandidate"}, + Properties: map[string]any{"lastseen": day(3)}, + }, + { + ID: "candidate-newer", + Kinds: []string{"LogicCandidate"}, + Properties: map[string]any{"lastseen": day(4)}, + }, + { + ID: "protected-missing", + Kinds: []string{"LogicProtected"}, + Properties: map[string]any{}, + }, + { + ID: "protected-null", + Kinds: []string{"LogicProtected"}, + Properties: map[string]any{"lastseen": nil}, + }, + { + ID: "protected-older", + Kinds: []string{"LogicProtected"}, + Properties: map[string]any{"lastseen": day(2)}, + }, + { + ID: "multi-kind-protected", + Kinds: []string{"LogicCandidate", "LogicProtected"}, + Properties: map[string]any{"lastseen": day(2)}, + }, + }, + Edges: []opengraph.Edge{ + { + StartID: "direction-forward", + EndID: "direction-reverse", + Kind: "LogicKindA", + Properties: map[string]any{"marker": "valid-forward"}, + }, + { + StartID: "direction-reverse", + EndID: "direction-forward", + Kind: "LogicKindB", + Properties: map[string]any{"marker": "valid-reverse"}, + }, + { + StartID: "direction-forward", + EndID: "direction-reverse", + Kind: "LogicKindB", + Properties: map[string]any{"marker": "invalid-forward-kind"}, + }, + { + StartID: "direction-reverse", + EndID: "direction-forward", + Kind: "LogicKindA", + Properties: map[string]any{"marker": "invalid-reverse-kind"}, + }, + { + StartID: "late-a", + EndID: "early-a", + Kind: "LogicStaleTrust", + Properties: map[string]any{"lastseen": day(3), "marker": "older-start-only"}, + }, + { + StartID: "early-a", + EndID: "late-a", + Kind: "LogicStaleTrust", + Properties: map[string]any{"lastseen": day(3), "marker": "older-end-only"}, + }, + { + StartID: "late-a", + EndID: "late-b", + Kind: "LogicStaleTrust", + Properties: map[string]any{"lastseen": day(3), "marker": "older-both"}, + }, + { + StartID: "equal-a", + EndID: "equal-b", + Kind: "LogicStaleTrust", + Properties: map[string]any{"lastseen": day(3), "marker": "equal"}, + }, + { + StartID: "late-a", + EndID: "late-b-newer", + Kind: "LogicStaleTrust", + Properties: map[string]any{"lastseen": day(5), "marker": "newer"}, + }, + { + StartID: "late-a", + EndID: "late-b-missing", + Kind: "LogicStaleTrust", + Properties: map[string]any{"marker": "missing"}, + }, + { + StartID: "late-a", + EndID: "late-b-null", + Kind: "LogicStaleTrust", + Properties: map[string]any{"lastseen": nil, "marker": "null"}, + }, + }, + } +} + +// logicalProjectionFixture builds the single relationship used to verify +// projection and fetch behavior for logical criteria. +func logicalProjectionFixture() *opengraph.Graph { + return &opengraph.Graph{ + Nodes: []opengraph.Node{ + { + ID: "projection-start", + Kinds: []string{"LogicProjectionStart"}, + Properties: map[string]any{"name": "start"}, + }, + { + ID: "projection-end", + Kinds: []string{"LogicProjectionEnd", "LogicProjectionEntity"}, + Properties: map[string]any{"name": "end"}, + }, + }, + Edges: []opengraph.Edge{ + { + StartID: "projection-start", + EndID: "projection-end", + Kind: "LogicProjectionEdge", + Properties: map[string]any{"marker": "projection"}, + }, + }, + } +} + +// regressionFixtureID resolves a database node ID back to its stable fixture identifier and fails when unmapped. +func regressionFixtureID(t *testing.T, idMap opengraph.IDMap, id graph.ID) string { + t.Helper() + for fixtureID, databaseID := range idMap { + if databaseID == id { + return fixtureID + } + } + t.Fatalf("database ID %d is absent from fixture ID map", id) + return "" +} diff --git a/integration/pgsql_delete_by_kind_test.go b/integration/pgsql_delete_by_kind_test.go new file mode 100644 index 00000000..08d910d6 --- /dev/null +++ b/integration/pgsql_delete_by_kind_test.go @@ -0,0 +1,167 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +//go:build manual_integration + +package integration + +import ( + "context" + "os" + "testing" + + "github.com/specterops/dawgs/drivers/pg" + "github.com/specterops/dawgs/graph" +) + +// nodesByKindDeleter mirrors the capability the BloodHound delete path detects on the PostgreSQL driver. +type nodesByKindDeleter interface { + // DeleteNodesByKinds deletes nodes matching any included kind unless they match an excluded kind. + DeleteNodesByKinds(ctx context.Context, includeAny graph.Kinds, excludeAny graph.Kinds) error +} + +// TestPostgreSQLDeleteNodesByKinds verifies the server-side, set-based node delete: includeAny restricts the delete to +// nodes carrying one of the listed kinds, excludeAny protects nodes carrying one of the listed kinds, undefined include +// kinds are a safe no-op while undefined exclude kinds fail closed, and deleting nodes cascades incident edges. +func TestPostgreSQLDeleteNodesByKinds(t *testing.T) { + connStr := os.Getenv("CONNECTION_STRING") + if connStr == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + + driver, err := DriverFromConnectionString(connStr) + if err != nil { + t.Fatalf("failed to detect driver: %v", err) + } + if driver != pg.DriverName { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + var ( + kindA = graph.StringKind("DeleteByKindA") + kindB = graph.StringKind("DeleteByKindB") + edgeKind = graph.StringKind("DeleteByKindEdge") + missing = graph.StringKind("DeleteByKindMissing") + db, ctx = SetupDBWithKinds(t, CleanupGraph, graph.Kinds{kindA, kindB}, graph.Kinds{edgeKind}) + ) + + deleter, hasCapability := graph.AsDriver[nodesByKindDeleter](db) + if !hasCapability { + t.Fatal("PostgreSQL driver does not implement DeleteNodesByKinds") + } + + // fixture creates two kindA nodes, two kindB nodes, and an A->B edge that must cascade when its start is deleted. + createFixture := func() { + if err := db.WriteTransaction(ctx, func(tx graph.Transaction) error { + a0, err := tx.CreateNode(graph.NewProperties(), kindA) + if err != nil { + return err + } + if _, err := tx.CreateNode(graph.NewProperties(), kindA); err != nil { + return err + } + + b0, err := tx.CreateNode(graph.NewProperties(), kindB) + if err != nil { + return err + } + if _, err := tx.CreateNode(graph.NewProperties(), kindB); err != nil { + return err + } + + _, err = tx.CreateRelationshipByIDs(a0.ID, b0.ID, edgeKind, graph.NewProperties()) + return err + }); err != nil { + t.Fatalf("failed to create delete-by-kind fixture: %v", err) + } + } + + t.Run("includeAny deletes matching kinds and cascades edges", func(t *testing.T) { + createFixture() + + if err := deleter.DeleteNodesByKinds(ctx, graph.Kinds{kindA}, nil); err != nil { + t.Fatalf("DeleteNodesByKinds(include kindA) failed: %v", err) + } + + if count := countByCypher(t, ctx, db, "MATCH (n:DeleteByKindA) RETURN count(n)"); count != 0 { + t.Fatalf("kindA node count: got %d, want 0", count) + } + if count := countByCypher(t, ctx, db, "MATCH (n:DeleteByKindB) RETURN count(n)"); count != 2 { + t.Fatalf("kindB node count: got %d, want 2", count) + } + if count := countByCypher(t, ctx, db, "MATCH ()-[r:DeleteByKindEdge]->() RETURN count(r)"); count != 0 { + t.Fatalf("edge count after cascade: got %d, want 0", count) + } + + cleanupAll(t, ctx, deleter) + }) + + t.Run("excludeAny protects matching kinds", func(t *testing.T) { + createFixture() + + // Delete every node except those carrying kindB. + if err := deleter.DeleteNodesByKinds(ctx, nil, graph.Kinds{kindB}); err != nil { + t.Fatalf("DeleteNodesByKinds(exclude kindB) failed: %v", err) + } + + if count := countByCypher(t, ctx, db, "MATCH (n:DeleteByKindA) RETURN count(n)"); count != 0 { + t.Fatalf("kindA node count: got %d, want 0", count) + } + if count := countByCypher(t, ctx, db, "MATCH (n:DeleteByKindB) RETURN count(n)"); count != 2 { + t.Fatalf("kindB node count: got %d, want 2", count) + } + + cleanupAll(t, ctx, deleter) + }) + + t.Run("undefined include kinds are a safe no-op", func(t *testing.T) { + createFixture() + + if err := deleter.DeleteNodesByKinds(ctx, graph.Kinds{missing}, nil); err != nil { + t.Fatalf("DeleteNodesByKinds(include missing) failed: %v", err) + } + + if count := countByCypher(t, ctx, db, "MATCH (n) RETURN count(n)"); count != 4 { + t.Fatalf("node count after no-op delete: got %d, want 4", count) + } + + cleanupAll(t, ctx, deleter) + }) + + t.Run("undefined exclude kinds fail closed and delete nothing", func(t *testing.T) { + createFixture() + + // An unresolved exclusion would otherwise collapse to an unguarded delete; the driver must refuse instead. + if err := deleter.DeleteNodesByKinds(ctx, nil, graph.Kinds{missing}); err == nil { + t.Fatal("DeleteNodesByKinds(exclude missing) succeeded, want error") + } + + if count := countByCypher(t, ctx, db, "MATCH (n) RETURN count(n)"); count != 4 { + t.Fatalf("node count after failed delete: got %d, want 4", count) + } + + cleanupAll(t, ctx, deleter) + }) +} + +// cleanupAll removes every node between subtests so each starts from an empty graph. +func cleanupAll(t *testing.T, ctx context.Context, deleter nodesByKindDeleter) { + t.Helper() + + if err := deleter.DeleteNodesByKinds(ctx, nil, nil); err != nil { + t.Fatalf("failed to clean up nodes between subtests: %v", err) + } +} diff --git a/integration/pgsql_delete_relationships_by_kind_test.go b/integration/pgsql_delete_relationships_by_kind_test.go new file mode 100644 index 00000000..81601b24 --- /dev/null +++ b/integration/pgsql_delete_relationships_by_kind_test.go @@ -0,0 +1,151 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +//go:build manual_integration + +package integration + +import ( + "context" + "os" + "testing" + + "github.com/specterops/dawgs/drivers/pg" + "github.com/specterops/dawgs/graph" +) + +// relationshipsByKindDeleter mirrors the capability the BloodHound delete path detects on the PostgreSQL driver. +type relationshipsByKindDeleter interface { + // DeleteRelationshipsByKinds deletes relationships matching any supplied kind. + DeleteRelationshipsByKinds(ctx context.Context, kinds graph.Kinds) error +} + +// TestPostgreSQLDeleteRelationshipsByKinds verifies the server-side, set-based relationship delete: the listed kinds +// restrict the delete to relationships carrying one of those kinds, nodes are left intact, multiple kinds are unioned, +// undefined kinds are a safe no-op, and an empty request deletes nothing. +func TestPostgreSQLDeleteRelationshipsByKinds(t *testing.T) { + connStr := os.Getenv("CONNECTION_STRING") + if connStr == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + + driver, err := DriverFromConnectionString(connStr) + if err != nil { + t.Fatalf("failed to detect driver: %v", err) + } + if driver != pg.DriverName { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + var ( + nodeKind = graph.StringKind("RelByKindNode") + edgeKindA = graph.StringKind("RelByKindEdgeA") + edgeKindB = graph.StringKind("RelByKindEdgeB") + missing = graph.StringKind("RelByKindMissing") + db, ctx = SetupDBWithKinds(t, CleanupGraph, graph.Kinds{nodeKind}, graph.Kinds{edgeKindA, edgeKindB}) + ) + + deleter, hasCapability := graph.AsDriver[relationshipsByKindDeleter](db) + if !hasCapability { + t.Fatal("PostgreSQL driver does not implement DeleteRelationshipsByKinds") + } + + // fixture creates three nodes joined by two edgeKindA edges and one edgeKindB edge. Nodes must survive every delete. + createFixture := func() { + if err := db.WriteTransaction(ctx, func(tx graph.Transaction) error { + n0, err := tx.CreateNode(graph.NewProperties(), nodeKind) + if err != nil { + return err + } + n1, err := tx.CreateNode(graph.NewProperties(), nodeKind) + if err != nil { + return err + } + n2, err := tx.CreateNode(graph.NewProperties(), nodeKind) + if err != nil { + return err + } + + if _, err := tx.CreateRelationshipByIDs(n0.ID, n1.ID, edgeKindA, graph.NewProperties()); err != nil { + return err + } + if _, err := tx.CreateRelationshipByIDs(n1.ID, n2.ID, edgeKindA, graph.NewProperties()); err != nil { + return err + } + _, err = tx.CreateRelationshipByIDs(n0.ID, n2.ID, edgeKindB, graph.NewProperties()) + return err + }); err != nil { + t.Fatalf("failed to create delete-relationships-by-kind fixture: %v", err) + } + } + + t.Run("deletes relationships of the given kind and leaves nodes", func(t *testing.T) { + createFixture() + + if err := deleter.DeleteRelationshipsByKinds(ctx, graph.Kinds{edgeKindA}); err != nil { + t.Fatalf("DeleteRelationshipsByKinds(edgeKindA) failed: %v", err) + } + + if count := countByCypher(t, ctx, db, "MATCH ()-[r:RelByKindEdgeA]->() RETURN count(r)"); count != 0 { + t.Fatalf("edgeKindA count: got %d, want 0", count) + } + if count := countByCypher(t, ctx, db, "MATCH ()-[r:RelByKindEdgeB]->() RETURN count(r)"); count != 1 { + t.Fatalf("edgeKindB count: got %d, want 1", count) + } + if count := countByCypher(t, ctx, db, "MATCH (n:RelByKindNode) RETURN count(n)"); count != 3 { + t.Fatalf("node count: got %d, want 3", count) + } + + ClearGraph(t, db, ctx) + }) + + t.Run("multiple kinds delete every matching relationship", func(t *testing.T) { + createFixture() + + if err := deleter.DeleteRelationshipsByKinds(ctx, graph.Kinds{edgeKindA, edgeKindB}); err != nil { + t.Fatalf("DeleteRelationshipsByKinds(edgeKindA, edgeKindB) failed: %v", err) + } + + if count := countByCypher(t, ctx, db, "MATCH ()-[r]->() RETURN count(r)"); count != 0 { + t.Fatalf("edge count: got %d, want 0", count) + } + if count := countByCypher(t, ctx, db, "MATCH (n:RelByKindNode) RETURN count(n)"); count != 3 { + t.Fatalf("node count: got %d, want 3", count) + } + + ClearGraph(t, db, ctx) + }) + + t.Run("undefined and empty kinds are a safe no-op", func(t *testing.T) { + createFixture() + + if err := deleter.DeleteRelationshipsByKinds(ctx, graph.Kinds{missing}); err != nil { + t.Fatalf("DeleteRelationshipsByKinds(missing) failed: %v", err) + } + if err := deleter.DeleteRelationshipsByKinds(ctx, graph.Kinds{}); err != nil { + t.Fatalf("DeleteRelationshipsByKinds(empty) failed: %v", err) + } + if err := deleter.DeleteRelationshipsByKinds(ctx, nil); err != nil { + t.Fatalf("DeleteRelationshipsByKinds(nil) failed: %v", err) + } + + if count := countByCypher(t, ctx, db, "MATCH ()-[r]->() RETURN count(r)"); count != 3 { + t.Fatalf("edge count after no-op delete: got %d, want 3", count) + } + + ClearGraph(t, db, ctx) + }) +} diff --git a/integration/pgsql_inline_asp_test.go b/integration/pgsql_inline_asp_test.go new file mode 100644 index 00000000..139817bd --- /dev/null +++ b/integration/pgsql_inline_asp_test.go @@ -0,0 +1,580 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//go:build manual_integration + +package integration + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "testing" + + "github.com/jackc/pgx/v5" + "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/specterops/dawgs/drivers/pg" + "github.com/specterops/dawgs/graph" +) + +var ( + inlineASPNodeKind = graph.StringKind("InlineASPNode") + inlineASPEdgeOne = graph.StringKind("InlineASPEdgeOne") + inlineASPEdgeTwo = graph.StringKind("InlineASPEdgeTwo") +) + +const inlineASPCypher = ` + MATCH p = allShortestPaths((s)-[:InlineASPEdgeOne|InlineASPEdgeTwo*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p +` + +// TestPostgreSQLInlineASPMatchesA1AndFallsBackWithoutPartialRows exercises the +// typed guarded statement at the real PostgreSQL boundary. A tiny state cap +// must select exact A1 and return the same complete relationship-distinct bag. +func TestPostgreSQLInlineASPMatchesA1AndFallsBackWithoutPartialRows(t *testing.T) { + session := Open(t, Options{ + RequireDriver: pg.DriverName, + SkipIfNoConnection: true, + SkipIfDriverMismatch: true, + CleanupMode: CleanupGraph, + ExtraNodeKinds: graph.Kinds{inlineASPNodeKind}, + ExtraEdgeKinds: graph.Kinds{inlineASPEdgeOne, inlineASPEdgeTwo}, + }) + + var startID, endID, disconnectedID, deepStartID, deepEndID graph.ID + if err := session.DB.WriteTransaction(session.Ctx, func(tx graph.Transaction) error { + start, err := tx.CreateNode(graph.NewProperties(), inlineASPNodeKind) + if err != nil { + return err + } + left, err := tx.CreateNode(graph.NewProperties(), inlineASPNodeKind) + if err != nil { + return err + } + right, err := tx.CreateNode(graph.NewProperties(), inlineASPNodeKind) + if err != nil { + return err + } + end, err := tx.CreateNode(graph.NewProperties(), inlineASPNodeKind) + if err != nil { + return err + } + startID, endID = start.ID, end.ID + disconnected, err := tx.CreateNode(graph.NewProperties(), inlineASPNodeKind) + if err != nil { + return err + } + disconnectedID = disconnected.ID + deepStart, err := tx.CreateNode(graph.NewProperties(), inlineASPNodeKind) + if err != nil { + return err + } + deepMiddleOne, err := tx.CreateNode(graph.NewProperties(), inlineASPNodeKind) + if err != nil { + return err + } + deepMiddleTwo, err := tx.CreateNode(graph.NewProperties(), inlineASPNodeKind) + if err != nil { + return err + } + deepEnd, err := tx.CreateNode(graph.NewProperties(), inlineASPNodeKind) + if err != nil { + return err + } + deepStartID, deepEndID = deepStart.ID, deepEnd.ID + for _, edge := range []struct { + start graph.ID + end graph.ID + kind graph.Kind + }{ + {start.ID, left.ID, inlineASPEdgeOne}, + {left.ID, end.ID, inlineASPEdgeOne}, + {start.ID, right.ID, inlineASPEdgeTwo}, + {right.ID, end.ID, inlineASPEdgeTwo}, + {left.ID, left.ID, inlineASPEdgeOne}, + {left.ID, start.ID, inlineASPEdgeTwo}, + {deepStart.ID, deepMiddleOne.ID, inlineASPEdgeOne}, + {deepMiddleOne.ID, deepMiddleTwo.ID, inlineASPEdgeOne}, + {deepMiddleTwo.ID, deepEnd.ID, inlineASPEdgeOne}, + } { + if _, err := tx.CreateRelationshipByIDs(edge.start, edge.end, edge.kind, graph.NewProperties()); err != nil { + return err + } + } + return nil + }); err != nil { + t.Fatalf("load inline ASP fixture: %v", err) + } + + pgDriver, ok := session.DB.(*pg.Driver) + if !ok { + t.Fatalf("expected PostgreSQL driver, found %T", session.DB) + } + defaultGraph, ok := pgDriver.DefaultGraph() + if !ok { + t.Fatal("PostgreSQL default graph is not set") + } + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), inlineASPCypher) + if err != nil { + t.Fatalf("parse inline ASP query: %v", err) + } + parameters := map[string]any{"start_id": int64(startID), "end_id": int64(endID)} + + a1, err := translate.Translate(session.Ctx, regularQuery, pgDriver.KindMapper(), parameters, defaultGraph.ID) + if err != nil { + t.Fatalf("translate A1: %v", err) + } + i1, err := translate.TranslateForTool(session.Ctx, regularQuery, pgDriver.KindMapper(), parameters, defaultGraph.ID, + translate.ToolOptions{ForceShortestPathExecutor: optimize.ShortestPathExecutorASPI1DAG}) + if err != nil { + t.Fatalf("translate I1: %v", err) + } + fallback, err := translate.TranslateWithProductionOptions(session.Ctx, regularQuery, pgDriver.KindMapper(), parameters, defaultGraph.ID, + translate.ProductionOptions{ + ShortestPathExecutor: optimize.ShortestPathExecutorASPI1DAG, + ShortestPathCaps: &translate.ProductionShortestPathCaps{ + StateLimit: 100, PredecessorLimit: 100, EnumerationLimit: 1, OutputBytesLimit: 1 << 20, + }, + AuthorizedBucket: &translate.ProductionTraversalBucket{ + Direction: "outbound", ObservationMode: "all_paths", MinimumDepth: 1, MaximumDepth: 4, + RelationshipKindCount: 2, UntypedRelationship: false, + }, + SelectorVersion: "asp-i1-integration-fallback-v1", + }) + if err != nil { + t.Fatalf("translate I1 fallback: %v", err) + } + + a1Rows := executeInlineASPTranslation(t, session, a1) + i1Rows, candidateReceipt := executeInlineASPTranslationWithReceipt(t, session, i1, "inline-asp-candidate") + fallbackRows, fallbackReceipt := executeInlineASPTranslationWithReceipt(t, session, fallback, "inline-asp-fallback") + if len(a1Rows) != 2 { + t.Fatalf("expected two relationship-distinct shortest paths, got %d: %v", len(a1Rows), a1Rows) + } + if fmt.Sprint(a1Rows) != fmt.Sprint(i1Rows) { + t.Fatalf("inline I1 differs from A1: A1=%v I1=%v", a1Rows, i1Rows) + } + if !containsAll(candidateReceipt, "ASP-I1-U-DAG+MAT-M0", "inline_predecessor_dag", "false", "1") { + t.Fatalf("candidate runtime receipt is incomplete: %s", candidateReceipt) + } + if fmt.Sprint(a1Rows) != fmt.Sprint(fallbackRows) { + t.Fatalf("guarded fallback differs from A1: A1=%v fallback=%v", a1Rows, fallbackRows) + } + if !containsAll(fallbackReceipt, "ASP-A1-DAG", "exact_a1_fallback", "true", "1") { + t.Fatalf("fallback runtime receipt is incomplete: %s", fallbackReceipt) + } + + // ASP-A1 reaches its spd_* predecessor workspace only beyond the two-hop + // preflight. Prove that a fresh stable-snapshot session can execute it. + session.PGPool.Reset() + freshA1Rows, freshA1Receipt := executeDriverCypherWithReceipt(t, session, inlineASPCypher, + map[string]any{"start_id": int64(deepStartID), "end_id": int64(deepEndID)}, + "inline-asp-fresh-repeatable-a1", optimize.ShortestPathExecutorASPA1DAG, + pg.OptionSetTransactionIsolation(pgx.RepeatableRead)) + if len(freshA1Rows) != 1 || !containsAll(freshA1Receipt, "ASP-A1-DAG") { + t.Fatalf("fresh repeatable-read session did not execute recursive A1: rows=%v receipt=%s", freshA1Rows, freshA1Receipt) + } + candidatePlan := explainInlineASPTranslation(t, session, i1) + requireOrientationSubplanMetric(t, candidatePlan, "asp_i1_fallback_rows", "Actual Rows", 0) + fallbackPlan := explainInlineASPTranslation(t, session, fallback) + requireOrientationSubplanMetric(t, fallbackPlan, "asp_i1_candidate_rows", "Actual Rows", 0) + + for _, testCase := range []struct { + name string + query string + parameters map[string]any + }{ + { + name: "inbound", + query: `MATCH p = allShortestPaths((e)<-[:InlineASPEdgeOne|InlineASPEdgeTwo*1..4]-(s)) + WHERE id(s) = $start_id AND id(e) = $end_id RETURN p`, + parameters: parameters, + }, + { + name: "no path", + query: inlineASPCypher, + parameters: map[string]any{"start_id": int64(startID), "end_id": int64(disconnectedID)}, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + query, err := frontend.ParseCypher(frontend.NewContext(), testCase.query) + if err != nil { + t.Fatalf("parse query: %v", err) + } + a1Translation, err := translate.Translate(session.Ctx, query, pgDriver.KindMapper(), testCase.parameters, defaultGraph.ID) + if err != nil { + t.Fatalf("translate A1: %v", err) + } + i1Translation, err := translate.TranslateForTool(session.Ctx, query, pgDriver.KindMapper(), testCase.parameters, defaultGraph.ID, + translate.ToolOptions{ForceShortestPathExecutor: optimize.ShortestPathExecutorASPI1DAG}) + if err != nil { + t.Fatalf("translate I1: %v", err) + } + expected := executeInlineASPTranslation(t, session, a1Translation) + actual := executeInlineASPTranslation(t, session, i1Translation) + if testCase.name == "no path" { + var receipt string + actual, receipt = executeInlineASPTranslationWithReceipt(t, session, i1Translation, "inline-asp-no-path") + if !containsAll(receipt, "ASP-I1-U-DAG+MAT-M0", "inline_no_path", "false", "1") { + t.Fatalf("no-path runtime receipt is incomplete: %s", receipt) + } + } + if fmt.Sprint(expected) != fmt.Sprint(actual) { + t.Fatalf("I1 differs from A1: A1=%v I1=%v", expected, actual) + } + }) + } + + t.Run("driver policy requires stable snapshot and rolls back immediately", func(t *testing.T) { + policy := inlineASPTraversalPolicy(t, inlineASPCypher) + if err := pgDriver.SetTraversalPolicy(policy); err != nil { + t.Fatalf("set inline ASP policy: %v", err) + } + t.Cleanup(func() { _ = pgDriver.SetTraversalPolicy(pg.TraversalPolicy{}) }) + + readCommittedRows, readCommittedReceipt := executeDriverCypherWithReceipt(t, session, inlineASPCypher, parameters, + "inline-asp-policy-read-committed", optimize.ShortestPathExecutorASPA1DAG) + if fmt.Sprint(a1Rows) != fmt.Sprint(readCommittedRows) || !containsAll(readCommittedReceipt, "ASP-A1-DAG") { + t.Fatalf("read-committed policy did not preserve A1: rows=%v receipt=%s", readCommittedRows, readCommittedReceipt) + } + + // Force the stable-snapshot execution onto a fresh PostgreSQL session. + // Its incumbent fallback workspace must be created before BEGIN READ ONLY. + session.PGPool.Reset() + repeatableRows, repeatableReceipt := executeDriverCypherWithReceipt(t, session, inlineASPCypher, parameters, + "inline-asp-policy-repeatable", optimize.ShortestPathExecutorASPI1DAG, pg.OptionSetTransactionIsolation(pgx.RepeatableRead)) + if fmt.Sprint(a1Rows) != fmt.Sprint(repeatableRows) || !containsAll(repeatableReceipt, "ASP-I1-U-DAG+MAT-M0", "inline_predecessor_dag") { + t.Fatalf("repeatable-read policy did not execute I1: rows=%v receipt=%s", repeatableRows, repeatableReceipt) + } + + if err := pgDriver.SetTraversalPolicy(pg.TraversalPolicy{Generation: policy.Generation + 1, DisableInlineASPDAG: true}); err != nil { + t.Fatalf("activate inline ASP rollback: %v", err) + } + rollbackRows, rollbackReceipt := executeDriverCypherWithReceipt(t, session, inlineASPCypher, parameters, + "inline-asp-policy-rollback", optimize.ShortestPathExecutorASPA1DAG, pg.OptionSetTransactionIsolation(pgx.RepeatableRead)) + if fmt.Sprint(a1Rows) != fmt.Sprint(rollbackRows) || !containsAll(rollbackReceipt, "ASP-A1-DAG") || strings.Contains(rollbackReceipt, "ASP-I1-U-DAG+MAT-M0") { + t.Fatalf("rollback did not immediately restore A1: rows=%v receipt=%s", rollbackRows, rollbackReceipt) + } + }) + + t.Run("canonical inline witness falls back to S4 before exposing rows", func(t *testing.T) { + const shortestCypher = `MATCH p = shortestPath((s)<-[:InlineASPEdgeOne*1..64]-(e)) + WHERE id(s) = $start_id AND id(e) = $end_id RETURN p` + query, err := frontend.ParseCypher(frontend.NewContext(), shortestCypher) + if err != nil { + t.Fatalf("parse canonical shortest query: %v", err) + } + deepParameters := map[string]any{"start_id": int64(deepEndID), "end_id": int64(deepStartID)} + incumbent, err := translate.Translate(session.Ctx, query, pgDriver.KindMapper(), deepParameters, defaultGraph.ID) + if err != nil { + t.Fatalf("translate shortest incumbent: %v", err) + } + candidate, err := translate.TranslateWithProductionOptions(session.Ctx, query, pgDriver.KindMapper(), deepParameters, defaultGraph.ID, + translate.ProductionOptions{ + ShortestPathExecutor: optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, + ShortestPathCaps: &translate.ProductionShortestPathCaps{ + StateLimit: 1, PredecessorLimit: 100, EnumerationLimit: 100, OutputBytesLimit: 1 << 20, + }, + AuthorizedBucket: &translate.ProductionTraversalBucket{ + Direction: "inbound", ObservationMode: "one_path", MinimumDepth: 1, MaximumDepth: 64, RelationshipKindCount: 1, + }, + SelectorVersion: optimize.ShortestPathSelectorStaticV6, + }) + if err != nil { + t.Fatalf("translate canonical shortest candidate: %v", err) + } + expected := executeInlineASPTranslation(t, session, incumbent) + actual, receipt := executeInlineASPTranslationWithReceipt(t, session, candidate, "sp-i1-s4-fallback", optimize.ShortestPathExecutorI1CanonicalPredecessorWitness) + if fmt.Sprint(expected) != fmt.Sprint(actual) { + t.Fatalf("canonical fallback differs from incumbent: incumbent=%v candidate=%v", expected, actual) + } + if !containsAll(receipt, "exact_s4_fallback", "SP-S4-C-WE+MAT-M0", "exact_relationship_trail_fallback", "SP-S3-U-E+MAT-M0", "2") { + t.Fatalf("canonical fallback receipt does not contain the complete event chain: %s", receipt) + } + }) + + t.Run("canonical driver policy requires stable snapshot and rolls back immediately", func(t *testing.T) { + const shortestCypher = `MATCH p = shortestPath((s)<-[:InlineASPEdgeOne*1..64]-(e)) + WHERE id(s) = $start_id AND id(e) = $end_id RETURN p` + parameters := map[string]any{"start_id": int64(deepEndID), "end_id": int64(deepStartID)} + policy := inlineCanonicalSPTraversalPolicy(t, shortestCypher) + if err := pgDriver.SetTraversalPolicy(policy); err != nil { + t.Fatalf("set canonical SP policy: %v", err) + } + t.Cleanup(func() { _ = pgDriver.SetTraversalPolicy(pg.TraversalPolicy{}) }) + + incumbentRows, incumbentReceipt := executeDriverCypherWithReceipt(t, session, shortestCypher, parameters, + "sp-i1-policy-read-committed", optimize.ShortestPathExecutorS4CanonicalWitness) + if len(incumbentRows) != 1 || !containsAll(incumbentReceipt, "SP-S4-C-WE+MAT-M0", "compact_workspace_witness") || + strings.Contains(incumbentReceipt, "SP-I1-C-WE+MAT-M0") { + t.Fatalf("read-committed policy did not preserve the S4 incumbent: rows=%v receipt=%s", incumbentRows, incumbentReceipt) + } + + // Exercise admission on a fresh connection so all session-local fallback + // workspace is initialized before the stable-snapshot transaction begins. + session.PGPool.Reset() + candidateRows, candidateReceipt := executeDriverCypherWithReceipt(t, session, shortestCypher, parameters, + "sp-i1-policy-repeatable", optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, + pg.OptionSetTransactionIsolation(pgx.RepeatableRead)) + if fmt.Sprint(incumbentRows) != fmt.Sprint(candidateRows) || + !containsAll(candidateReceipt, "SP-I1-C-WE+MAT-M0", "inline_canonical_witness") || + strings.Contains(candidateReceipt, "SP-S4-C-WE+MAT-M0") { + t.Fatalf("repeatable-read policy did not execute canonical I1: rows=%v receipt=%s", candidateRows, candidateReceipt) + } + + if err := pgDriver.SetTraversalPolicy(pg.TraversalPolicy{Generation: policy.Generation + 1, DisableInlineSPWitness: true}); err != nil { + t.Fatalf("activate canonical SP rollback: %v", err) + } + rollbackRows, rollbackReceipt := executeDriverCypherWithReceipt(t, session, shortestCypher, parameters, + "sp-i1-policy-rollback", optimize.ShortestPathExecutorS4CanonicalWitness, + pg.OptionSetTransactionIsolation(pgx.RepeatableRead)) + if fmt.Sprint(incumbentRows) != fmt.Sprint(rollbackRows) || + !containsAll(rollbackReceipt, "SP-S4-C-WE+MAT-M0", "compact_workspace_witness") || + strings.Contains(rollbackReceipt, "SP-I1-C-WE+MAT-M0") { + t.Fatalf("canonical SP rollback did not immediately restore S4: rows=%v receipt=%s", rollbackRows, rollbackReceipt) + } + }) +} + +func inlineASPTraversalPolicy(t *testing.T, query string) pg.TraversalPolicy { + t.Helper() + queryDigest := pg.TraversalPolicyQuerySHA256(query) + evidence := map[string]map[string]string{} + for _, role := range []string{"aa", "confirmation", "performance", "resource", "reference_closure", "operational"} { + evidence[role] = map[string]string{"sha256": strings.Repeat("01", sha256.Size)} + } + raw, err := json.Marshal(map[string]any{ + "version": 2, "candidate": string(optimize.ShortestPathExecutorASPI1DAG), "selector_version": "asp-i1-driver-integration-v1", + "source_commit": "integration", "source_sha256": strings.Repeat("0", 64), + "binary_sha256": strings.Repeat("0", 64), "corpus_sha256": strings.Repeat("0", 64), + "execution_boundary": "guarded_dual_arm", "fallback_executor": string(optimize.ShortestPathExecutorASPA1DAG), + "caps": map[string]int64{"state_limit": 1000, "predecessor_limit": 1000, "enumeration_limit": 1000, "output_bytes_limit": 1 << 20}, + "buckets": []map[string]any{{ + "query_sha256": []string{queryDigest}, "qualification_split": []string{"training", "holdout"}, + "direction": "outbound", "observation_mode": "all_paths", "minimum_depth": 1, "maximum_depth": 4, + "relationship_kind_count": 2, "untyped_relationship": false, + }}, + "evidence": evidence, + }) + if err != nil { + t.Fatalf("encode inline ASP policy: %v", err) + } + digest := sha256.Sum256(raw) + return pg.TraversalPolicy{ + Generation: 1, PromotionManifestSHA256: hex.EncodeToString(digest[:]), PromotionManifestJSON: raw, + QuerySHA256Allowlist: []string{queryDigest}, ShortestPathExecutor: optimize.ShortestPathExecutorASPI1DAG, + } +} + +func inlineCanonicalSPTraversalPolicy(t *testing.T, query string) pg.TraversalPolicy { + t.Helper() + queryDigest := pg.TraversalPolicyQuerySHA256(query) + evidence := map[string]map[string]string{} + for _, role := range []string{"aa", "confirmation", "performance", "resource", "reference_closure", "operational"} { + evidence[role] = map[string]string{"sha256": strings.Repeat("01", sha256.Size)} + } + raw, err := json.Marshal(map[string]any{ + "version": 2, "candidate": string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), "selector_version": optimize.ShortestPathSelectorStaticV6, + "source_commit": "integration", "source_sha256": strings.Repeat("0", 64), + "binary_sha256": strings.Repeat("0", 64), "corpus_sha256": strings.Repeat("0", 64), + "execution_boundary": "guarded_dual_arm", "fallback_executor": string(optimize.ShortestPathExecutorS4CanonicalWitness), + "caps": map[string]int64{"state_limit": 1000, "predecessor_limit": 1000, "enumeration_limit": 1000, "output_bytes_limit": 1 << 20}, + "buckets": []map[string]any{{ + "query_sha256": []string{queryDigest}, "qualification_split": []string{"training", "holdout"}, + "direction": "inbound", "observation_mode": "one_path", "minimum_depth": 1, "maximum_depth": 64, + "relationship_kind_count": 1, "untyped_relationship": false, + }}, + "evidence": evidence, + }) + if err != nil { + t.Fatalf("encode canonical SP policy: %v", err) + } + digest := sha256.Sum256(raw) + return pg.TraversalPolicy{ + Generation: 2, PromotionManifestSHA256: hex.EncodeToString(digest[:]), PromotionManifestJSON: raw, + QuerySHA256Allowlist: []string{queryDigest}, ShortestPathExecutor: optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, + } +} + +func explainInlineASPTranslation(t *testing.T, session *Session, translation translate.Result) any { + t.Helper() + sqlQuery, err := translate.Translated(translation) + if err != nil { + t.Fatalf("render translated query: %v", err) + } + var plan any + if err := session.DB.ReadTransaction(session.Ctx, func(tx graph.Transaction) error { + result := tx.Raw("explain (analyze, timing off, summary off, format json) "+sqlQuery, translation.Parameters) + defer result.Close() + if !result.Next() { + if err := result.Error(); err != nil { + return err + } + return errors.New("PostgreSQL EXPLAIN returned no rows") + } + values := result.Values() + if len(values) == 0 { + return errors.New("PostgreSQL EXPLAIN returned an empty row") + } + parsed, err := normalizeExplainPlan(values[0]) + if err != nil { + return err + } + plan = parsed + return result.Error() + }); err != nil { + t.Fatalf("explain inline ASP query: %v", err) + } + return plan +} + +func executeInlineASPTranslationWithReceipt(t *testing.T, session *Session, translation translate.Result, invocation string, requested ...optimize.ShortestPathExecutor) ([]string, string) { + t.Helper() + requestedIdentity := optimize.ShortestPathExecutorASPI1DAG + if len(requested) > 0 { + requestedIdentity = requested[0] + } + sqlQuery, err := translate.Translated(translation) + if err != nil { + t.Fatalf("render translated query: %v", err) + } + var rows []string + var receipt string + if err := session.DB.ReadTransaction(session.Ctx, func(tx graph.Transaction) error { + arm := tx.Raw("select public.begin_traversal_runtime_attestation_v1(@invocation, @requested)", map[string]any{ + "invocation": invocation, "requested": string(requestedIdentity), + }) + for arm.Next() { + } + if err := arm.Error(); err != nil { + arm.Close() + return err + } + arm.Close() + + result := tx.Raw(sqlQuery, translation.Parameters) + for result.Next() { + rows = append(rows, fmt.Sprint(result.Values())) + } + if err := result.Error(); err != nil { + result.Close() + return err + } + result.Close() + + read := tx.Raw(`select + coalesce(document ->> 'runtime_identity', ''), + coalesce(document ->> 'runtime_branch', ''), + coalesce(document ->> 'fallback_executed', ''), + coalesce(document ->> 'record_count', ''), + coalesce(document ->> 'events', '') + from (select public.read_traversal_runtime_attestation_v1(@invocation) document) receipt`, map[string]any{"invocation": invocation}) + if read.Next() { + receipt = fmt.Sprint(read.Values()) + } + if err := read.Error(); err != nil { + read.Close() + return err + } + read.Close() + clear := tx.Raw("select public.clear_traversal_runtime_attestation_v1(@invocation)", map[string]any{"invocation": invocation}) + for clear.Next() { + } + err := clear.Error() + clear.Close() + return err + }); err != nil { + t.Fatalf("execute translated query with receipt: %v\nSQL: %s", err, sqlQuery) + } + sort.Strings(rows) + return rows, receipt +} + +func executeDriverCypherWithReceipt(t *testing.T, session *Session, cypher string, parameters map[string]any, invocation string, + requested optimize.ShortestPathExecutor, options ...graph.TransactionOption) ([]string, string) { + t.Helper() + var rows []string + var receipt string + if err := session.DB.ReadTransaction(session.Ctx, func(tx graph.Transaction) error { + arm := tx.Raw("select public.begin_traversal_runtime_attestation_v1(@invocation, @requested)", map[string]any{ + "invocation": invocation, "requested": string(requested), + }) + for arm.Next() { + } + if err := arm.Error(); err != nil { + arm.Close() + return err + } + arm.Close() + + result := tx.Query(cypher, parameters) + for result.Next() { + rows = append(rows, fmt.Sprint(result.Values())) + } + if err := result.Error(); err != nil { + result.Close() + return err + } + result.Close() + + read := tx.Raw("select coalesce(public.read_traversal_runtime_attestation_v1(@invocation)::text, '')", map[string]any{"invocation": invocation}) + if read.Next() { + values := read.Values() + if len(values) > 0 { + receipt = fmt.Sprint(values[0]) + } + } + if err := read.Error(); err != nil { + read.Close() + return err + } + read.Close() + clear := tx.Raw("select public.clear_traversal_runtime_attestation_v1(@invocation)", map[string]any{"invocation": invocation}) + for clear.Next() { + } + err := clear.Error() + clear.Close() + return err + }, append(options, pg.OptionInitializeTraversalRuntimeAttestation())...); err != nil { + t.Fatalf("execute driver Cypher with receipt: %v", err) + } + sort.Strings(rows) + return rows, receipt +} + +func containsAll(value string, fragments ...string) bool { + for _, fragment := range fragments { + if !strings.Contains(value, fragment) { + return false + } + } + return true +} + +func executeInlineASPTranslation(t *testing.T, session *Session, translation translate.Result) []string { + t.Helper() + sqlQuery, err := translate.Translated(translation) + if err != nil { + t.Fatalf("render translated query: %v", err) + } + var rows []string + if err := session.DB.ReadTransaction(session.Ctx, func(tx graph.Transaction) error { + result := tx.Raw(sqlQuery, translation.Parameters) + defer result.Close() + for result.Next() { + rows = append(rows, fmt.Sprint(result.Values())) + } + return result.Error() + }); err != nil { + t.Fatalf("execute translated query: %v", err) + } + sort.Strings(rows) + return rows +} diff --git a/integration/pgsql_orientation_execution_plan_test.go b/integration/pgsql_orientation_execution_plan_test.go new file mode 100644 index 00000000..c65d534e --- /dev/null +++ b/integration/pgsql_orientation_execution_plan_test.go @@ -0,0 +1,727 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +//go:build manual_integration + +package integration + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/specterops/dawgs/drivers/pg" + "github.com/specterops/dawgs/graph" +) + +const orientationExecutionPlanCypher = ` + MATCH (root:ExpansionRoot) + WHERE root.root_key = $root_key + MATCH path = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(:SuffixTerminal) + RETURN path +` + +var ( + orientationRootKind = graph.StringKind("ExpansionRoot") + orientationExpansionKind = graph.StringKind("ExpansionNode") + orientationSuffixHeadKind = graph.StringKind("SuffixHead") + orientationSuffixMidKind = graph.StringKind("SuffixMiddle") + orientationSuffixEndKind = graph.StringKind("SuffixTerminal") + orientationExpandEdge = graph.StringKind("Expand") + orientationSuffixEdgeOne = graph.StringKind("EnterSuffix") + orientationSuffixEdgeTwo = graph.StringKind("ContinueSuffix") + orientationSuffixEdgeThree = graph.StringKind("CompleteSuffix") +) + +// TestPostgreSQLGuardedOrientationInactiveArmLoops proves the emitted +// marker-first LATERAL dependencies at the PostgreSQL execution boundary. The +// forward case must leave reverse recursion uninitialized; the reverse case +// must leave the exact materialized incumbent uninitialized. +func TestPostgreSQLGuardedOrientationInactiveArmLoops(t *testing.T) { + session := Open(t, Options{ + RequireDriver: pg.DriverName, + SkipIfNoConnection: true, + SkipIfDriverMismatch: true, + CleanupMode: CleanupGraph, + ExtraNodeKinds: graph.Kinds{ + orientationRootKind, + orientationExpansionKind, + orientationSuffixHeadKind, + orientationSuffixMidKind, + orientationSuffixEndKind, + }, + ExtraEdgeKinds: graph.Kinds{ + orientationExpandEdge, + orientationSuffixEdgeOne, + orientationSuffixEdgeTwo, + orientationSuffixEdgeThree, + }, + }) + + for _, testCase := range []struct { + name string + reverseDominates bool + expectedReverseLoops int64 + expectedIncumbentLoops int64 + expectedCandidateMarkers int64 + expectedIncumbentMarkers int64 + }{ + { + name: "forward policy does not initialize reverse recursion", + reverseDominates: false, + expectedReverseLoops: 0, + expectedIncumbentLoops: 1, + expectedCandidateMarkers: 0, + expectedIncumbentMarkers: 1, + }, + { + name: "reverse policy does not initialize exact incumbent", + reverseDominates: true, + expectedReverseLoops: 1, + expectedIncumbentLoops: 0, + expectedCandidateMarkers: 1, + expectedIncumbentMarkers: 0, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + session.ClearGraph(t) + loadOrientationExecutionFixture(t, session, testCase.reverseDominates) + + plan := explainGuardedOrientation(t, session) + requireOrientationSubplanMetric(t, plan, "_orientation_executed_candidate", "Actual Rows", testCase.expectedCandidateMarkers) + requireOrientationSubplanMetric(t, plan, "_orientation_executed_incumbent", "Actual Rows", testCase.expectedIncumbentMarkers) + requireOrientationSubplanMetric(t, plan, "_orientation_reverse", "Actual Loops", testCase.expectedReverseLoops) + requireOrientationSubplanMetric(t, plan, "_orientation_incumbent", "Actual Loops", testCase.expectedIncumbentLoops) + }) + } +} + +// TestPostgreSQLOrientationProbeV2ChangesOnlyItsVersionedDecision proves the +// depth-weighted v2 formula at the real PostgreSQL boundary while retaining +// v1's frozen choice for the same graph and statement. +func TestPostgreSQLOrientationProbeV2ChangesOnlyItsVersionedDecision(t *testing.T) { + session := Open(t, Options{ + RequireDriver: pg.DriverName, + SkipIfNoConnection: true, + SkipIfDriverMismatch: true, + CleanupMode: CleanupGraph, + ExtraNodeKinds: graph.Kinds{ + orientationRootKind, + orientationExpansionKind, + orientationSuffixHeadKind, + orientationSuffixMidKind, + orientationSuffixEndKind, + }, + ExtraEdgeKinds: graph.Kinds{ + orientationExpandEdge, + orientationSuffixEdgeOne, + orientationSuffixEdgeTwo, + orientationSuffixEdgeThree, + }, + }) + + loadOrientationV2CrossoverFixture(t, session) + for _, testCase := range []struct { + policy optimize.ExpansionSearchPolicy + expectedCandidateMarkers int64 + expectedIncumbentMarkers int64 + }{ + {policy: optimize.ExpansionSearchPolicyOrientationProbeV1, expectedIncumbentMarkers: 1}, + {policy: optimize.ExpansionSearchPolicyOrientationProbeV2, expectedCandidateMarkers: 1}, + } { + t.Run(string(testCase.policy), func(t *testing.T) { + plan := explainGuardedOrientationPolicy(t, session, testCase.policy) + requireOrientationSubplanMetric(t, plan, "_orientation_executed_candidate", "Actual Rows", testCase.expectedCandidateMarkers) + requireOrientationSubplanMetric(t, plan, "_orientation_executed_incumbent", "Actual Rows", testCase.expectedIncumbentMarkers) + }) + } +} + +// TestPostgreSQLShadowOrientationAttestsEmptyIncumbent proves the shadow +// statement records its only executable arm even when that arm returns no +// rows. The marker must be outside the incumbent LATERAL boundary or an empty +// result would leave the timed receipt unprovable. +func TestPostgreSQLShadowOrientationAttestsEmptyIncumbent(t *testing.T) { + session := Open(t, Options{ + RequireDriver: pg.DriverName, + SkipIfNoConnection: true, + SkipIfDriverMismatch: true, + CleanupMode: CleanupGraph, + ExtraNodeKinds: graph.Kinds{ + orientationRootKind, + orientationExpansionKind, + orientationSuffixHeadKind, + orientationSuffixMidKind, + orientationSuffixEndKind, + }, + ExtraEdgeKinds: graph.Kinds{ + orientationExpandEdge, + orientationSuffixEdgeOne, + orientationSuffixEdgeTwo, + orientationSuffixEdgeThree, + }, + }) + + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), orientationExecutionPlanCypher) + if err != nil { + t.Fatalf("parse shadow orientation query: %v", err) + } + pgDriver, ok := session.DB.(*pg.Driver) + if !ok { + t.Fatalf("expected PostgreSQL driver, found %T", session.DB) + } + defaultGraph, ok := pgDriver.DefaultGraph() + if !ok { + t.Fatal("PostgreSQL default graph is not set") + } + translation, err := translate.TranslateForTool( + session.Ctx, + regularQuery, + pgDriver.KindMapper(), + map[string]any{"root_key": "missing-orientation-plan-root"}, + defaultGraph.ID, + translate.ToolOptions{EnableExpansionOrientationShadow: true}, + ) + if err != nil { + t.Fatalf("translate shadow orientation query: %v", err) + } + sqlQuery, err := translate.Translated(translation) + if err != nil { + t.Fatalf("render shadow orientation query: %v", err) + } + + const invocation = "shadow-orientation-empty-incumbent" + var ( + rowCount int + receipt string + ) + if err := session.DB.ReadTransaction(session.Ctx, func(tx graph.Transaction) error { + arm := tx.Raw("select public.begin_traversal_runtime_attestation_v1(@invocation, @requested)", map[string]any{ + "invocation": invocation, + "requested": "EXPANSION-SUFFIX-SEEDED-REVERSE", + }) + for arm.Next() { + } + if err := arm.Error(); err != nil { + arm.Close() + return err + } + arm.Close() + + result := tx.Raw(sqlQuery, translation.Parameters) + for result.Next() { + rowCount++ + } + if err := result.Error(); err != nil { + result.Close() + return err + } + result.Close() + + read := tx.Raw("select coalesce(public.read_traversal_runtime_attestation_v1(@invocation)::text, '')", map[string]any{"invocation": invocation}) + if read.Next() && len(read.Values()) > 0 { + receipt = fmt.Sprint(read.Values()[0]) + } + if err := read.Error(); err != nil { + read.Close() + return err + } + read.Close() + + clear := tx.Raw("select public.clear_traversal_runtime_attestation_v1(@invocation)", map[string]any{"invocation": invocation}) + for clear.Next() { + } + err := clear.Error() + clear.Close() + return err + }); err != nil { + t.Fatalf("execute empty shadow orientation query: %v\nSQL: %s", err, sqlQuery) + } + if rowCount != 0 { + t.Fatalf("empty shadow incumbent returned %d rows", rowCount) + } + for _, fragment := range []string{`"runtime_identity": "EXPANSION-STEPWISE-FORWARD"`, `"runtime_branch": "shadow_incumbent"`, `"fallback_executed": false`, `"record_count": 1`} { + if !strings.Contains(receipt, fragment) { + t.Fatalf("empty shadow incumbent receipt lacks %q: %s", fragment, receipt) + } + } +} + +// TestPostgreSQLGuardedOrientationFallbackReceipts proves both cap+1 fallback +// paths produce one truthful incumbent receipt. Probe overflow skips reverse +// recursion entirely; state overflow performs only the bounded reverse +// admission probe before executing the exact forward fallback. +func TestPostgreSQLGuardedOrientationFallbackReceipts(t *testing.T) { + session := Open(t, Options{ + RequireDriver: pg.DriverName, + SkipIfNoConnection: true, + SkipIfDriverMismatch: true, + CleanupMode: CleanupGraph, + ExtraNodeKinds: graph.Kinds{ + orientationRootKind, + orientationExpansionKind, + orientationSuffixHeadKind, + orientationSuffixMidKind, + orientationSuffixEndKind, + }, + ExtraEdgeKinds: graph.Kinds{ + orientationExpandEdge, + orientationSuffixEdgeOne, + orientationSuffixEdgeTwo, + orientationSuffixEdgeThree, + }, + }) + + for _, testCase := range []struct { + name string + rootKey string + load func(*testing.T, *Session) + expectedRows int + }{ + { + name: "probe overflow", + rootKey: "orientation-probe-overflow-root", + load: loadOrientationProbeOverflowFixture, + expectedRows: 0, + }, + { + name: "state overflow", + rootKey: "orientation-state-overflow-root", + load: loadOrientationStateOverflowFixture, + expectedRows: 4096, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + session.ClearGraph(t) + testCase.load(t, session) + rowCount, receipt := executeGuardedOrientationWithReceipt(t, session, testCase.rootKey) + if rowCount != testCase.expectedRows { + t.Fatalf("guarded orientation returned %d rows, want %d", rowCount, testCase.expectedRows) + } + for _, fragment := range []string{`"runtime_identity": "EXPANSION-STEPWISE-FORWARD"`, `"runtime_branch": "exact_forward_incumbent"`, `"fallback_executed": true`, `"record_count": 1`} { + if !strings.Contains(receipt, fragment) { + t.Fatalf("guarded orientation receipt lacks %q: %s", fragment, receipt) + } + } + }) + } +} + +func executeGuardedOrientationWithReceipt(t *testing.T, session *Session, rootKey string) (int, string) { + t.Helper() + + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), orientationExecutionPlanCypher) + if err != nil { + t.Fatalf("parse guarded orientation query: %v", err) + } + pgDriver, ok := session.DB.(*pg.Driver) + if !ok { + t.Fatalf("expected PostgreSQL driver, found %T", session.DB) + } + defaultGraph, ok := pgDriver.DefaultGraph() + if !ok { + t.Fatal("PostgreSQL default graph is not set") + } + translation, err := translate.TranslateForTool( + session.Ctx, + regularQuery, + pgDriver.KindMapper(), + map[string]any{"root_key": rootKey}, + defaultGraph.ID, + translate.ToolOptions{EnableExpansionOrientationTournament: true}, + ) + if err != nil { + t.Fatalf("translate guarded orientation query: %v", err) + } + sqlQuery, err := translate.Translated(translation) + if err != nil { + t.Fatalf("render guarded orientation query: %v", err) + } + + invocation := "guarded-" + rootKey + var ( + rowCount int + receipt string + ) + if err := session.DB.ReadTransaction(session.Ctx, func(tx graph.Transaction) error { + arm := tx.Raw("select public.begin_traversal_runtime_attestation_v1(@invocation, @requested)", map[string]any{ + "invocation": invocation, + "requested": "EXPANSION-SUFFIX-SEEDED-REVERSE", + }) + for arm.Next() { + } + if err := arm.Error(); err != nil { + arm.Close() + return err + } + arm.Close() + + result := tx.Raw(sqlQuery, translation.Parameters) + for result.Next() { + rowCount++ + } + if err := result.Error(); err != nil { + result.Close() + return err + } + result.Close() + + read := tx.Raw("select coalesce(public.read_traversal_runtime_attestation_v1(@invocation)::text, '')", map[string]any{"invocation": invocation}) + if read.Next() && len(read.Values()) > 0 { + receipt = fmt.Sprint(read.Values()[0]) + } + if err := read.Error(); err != nil { + read.Close() + return err + } + read.Close() + + clear := tx.Raw("select public.clear_traversal_runtime_attestation_v1(@invocation)", map[string]any{"invocation": invocation}) + for clear.Next() { + } + err := clear.Error() + clear.Close() + return err + }); err != nil { + t.Fatalf("execute guarded orientation query: %v\nSQL: %s", err, sqlQuery) + } + return rowCount, receipt +} + +func loadOrientationProbeOverflowFixture(t *testing.T, session *Session) { + t.Helper() + + if err := session.DB.WriteTransaction(session.Ctx, func(tx graph.Transaction) error { + if _, err := tx.CreateNode(graph.AsProperties(map[string]any{"root_key": "orientation-probe-overflow-root"}), orientationRootKind); err != nil { + return err + } + for index := 0; index <= 512; index++ { + if _, err := createOrientationSuffix(tx); err != nil { + return err + } + } + return nil + }); err != nil { + t.Fatalf("load orientation probe-overflow fixture: %v", err) + } +} + +func loadOrientationStateOverflowFixture(t *testing.T, session *Session) { + t.Helper() + + if err := session.DB.WriteTransaction(session.Ctx, func(tx graph.Transaction) error { + root, err := tx.CreateNode(graph.AsProperties(map[string]any{"root_key": "orientation-state-overflow-root"}), orientationRootKind) + if err != nil { + return err + } + boundary, err := createOrientationSuffix(tx) + if err != nil { + return err + } + first, err := createOrientationNodes(tx, 16) + if err != nil { + return err + } + second, err := createOrientationNodes(tx, 32) + if err != nil { + return err + } + third, err := createOrientationNodes(tx, 8) + if err != nil { + return err + } + for _, node := range first { + if _, err := tx.CreateRelationshipByIDs(root.ID, node.ID, orientationExpandEdge, graph.NewProperties()); err != nil { + return err + } + } + if err := connectOrientationLayers(tx, first, second); err != nil { + return err + } + if err := connectOrientationLayers(tx, second, third); err != nil { + return err + } + for _, node := range third { + if _, err := tx.CreateRelationshipByIDs(node.ID, boundary.ID, orientationExpandEdge, graph.NewProperties()); err != nil { + return err + } + } + return nil + }); err != nil { + t.Fatalf("load orientation state-overflow fixture: %v", err) + } +} + +func createOrientationSuffix(tx graph.Transaction) (*graph.Node, error) { + boundary, err := tx.CreateNode(graph.NewProperties(), orientationExpansionKind) + if err != nil { + return nil, err + } + head, err := tx.CreateNode(graph.NewProperties(), orientationSuffixHeadKind) + if err != nil { + return nil, err + } + middle, err := tx.CreateNode(graph.NewProperties(), orientationSuffixMidKind) + if err != nil { + return nil, err + } + terminal, err := tx.CreateNode(graph.NewProperties(), orientationSuffixEndKind) + if err != nil { + return nil, err + } + for _, edge := range []struct { + start, end graph.ID + kind graph.Kind + }{ + {boundary.ID, head.ID, orientationSuffixEdgeOne}, + {head.ID, middle.ID, orientationSuffixEdgeTwo}, + {middle.ID, terminal.ID, orientationSuffixEdgeThree}, + } { + if _, err := tx.CreateRelationshipByIDs(edge.start, edge.end, edge.kind, graph.NewProperties()); err != nil { + return nil, err + } + } + return boundary, nil +} + +func createOrientationNodes(tx graph.Transaction, count int) ([]*graph.Node, error) { + nodes := make([]*graph.Node, 0, count) + for index := 0; index < count; index++ { + node, err := tx.CreateNode(graph.NewProperties(), orientationExpansionKind) + if err != nil { + return nil, err + } + nodes = append(nodes, node) + } + return nodes, nil +} + +func connectOrientationLayers(tx graph.Transaction, left, right []*graph.Node) error { + for _, start := range left { + for _, end := range right { + if _, err := tx.CreateRelationshipByIDs(start.ID, end.ID, orientationExpandEdge, graph.NewProperties()); err != nil { + return err + } + } + } + return nil +} + +func loadOrientationExecutionFixture(t *testing.T, session *Session, reverseDominates bool) { + t.Helper() + + if err := session.DB.WriteTransaction(session.Ctx, func(tx graph.Transaction) error { + root, err := tx.CreateNode(graph.AsProperties(map[string]any{"root_key": "orientation-plan-root"}), orientationRootKind) + if err != nil { + return err + } + + addSuffix := func(connectRoot bool) error { + boundary, err := tx.CreateNode(graph.NewProperties(), orientationExpansionKind) + if err != nil { + return err + } + head, err := tx.CreateNode(graph.NewProperties(), orientationSuffixHeadKind) + if err != nil { + return err + } + middle, err := tx.CreateNode(graph.NewProperties(), orientationSuffixMidKind) + if err != nil { + return err + } + terminal, err := tx.CreateNode(graph.NewProperties(), orientationSuffixEndKind) + if err != nil { + return err + } + if connectRoot { + if _, err := tx.CreateRelationshipByIDs(root.ID, boundary.ID, orientationExpandEdge, graph.NewProperties()); err != nil { + return err + } + } + for _, edge := range []struct { + start, end graph.ID + kind graph.Kind + }{ + {boundary.ID, head.ID, orientationSuffixEdgeOne}, + {head.ID, middle.ID, orientationSuffixEdgeTwo}, + {middle.ID, terminal.ID, orientationSuffixEdgeThree}, + } { + if _, err := tx.CreateRelationshipByIDs(edge.start, edge.end, edge.kind, graph.NewProperties()); err != nil { + return err + } + } + return nil + } + + if err := addSuffix(true); err != nil { + return err + } + if reverseDominates { + // One reverse seed but many typed forward neighbors makes reverse + // strictly dominate orientation-probe-v1's 4:3 hysteresis rule. + for index := 0; index < 24; index++ { + decoy, err := tx.CreateNode(graph.NewProperties(), orientationExpansionKind) + if err != nil { + return err + } + if _, err := tx.CreateRelationshipByIDs(root.ID, decoy.ID, orientationExpandEdge, graph.NewProperties()); err != nil { + return err + } + } + } else { + // Many disconnected suffix seeds overwhelm the one useful forward + // neighbor, so the incumbent wins decisively. + for index := 0; index < 20; index++ { + if err := addSuffix(false); err != nil { + return err + } + } + } + return nil + }); err != nil { + t.Fatalf("load guarded orientation fixture: %v", err) + } +} + +func loadOrientationV2CrossoverFixture(t *testing.T, session *Session) { + t.Helper() + + if err := session.DB.WriteTransaction(session.Ctx, func(tx graph.Transaction) error { + root, err := tx.CreateNode(graph.AsProperties(map[string]any{"root_key": "orientation-plan-root"}), orientationRootKind) + if err != nil { + return err + } + for range 4 { + boundary, err := createOrientationSuffix(tx) + if err != nil { + return err + } + if _, err := tx.CreateRelationshipByIDs(root.ID, boundary.ID, orientationExpandEdge, graph.NewProperties()); err != nil { + return err + } + } + return nil + }); err != nil { + t.Fatalf("load orientation v2 crossover fixture: %v", err) + } +} + +func explainGuardedOrientation(t *testing.T, session *Session) any { + return explainGuardedOrientationPolicy(t, session, optimize.ExpansionSearchPolicyOrientationProbeV1) +} + +func explainGuardedOrientationPolicy(t *testing.T, session *Session, policy optimize.ExpansionSearchPolicy) any { + t.Helper() + + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), orientationExecutionPlanCypher) + if err != nil { + t.Fatalf("parse guarded orientation query: %v", err) + } + pgDriver, ok := session.DB.(*pg.Driver) + if !ok { + t.Fatalf("expected PostgreSQL driver, found %T", session.DB) + } + defaultGraph, ok := pgDriver.DefaultGraph() + if !ok { + t.Fatal("PostgreSQL default graph is not set") + } + translation, err := translate.TranslateForTool( + session.Ctx, + regularQuery, + pgDriver.KindMapper(), + map[string]any{"root_key": "orientation-plan-root"}, + defaultGraph.ID, + translate.ToolOptions{ + ExpansionOrientationPolicy: policy, + EnableExpansionOrientationTournament: true, + }, + ) + if err != nil { + t.Fatalf("translate guarded orientation query: %v", err) + } + sqlQuery, err := translate.Translated(translation) + if err != nil { + t.Fatalf("render guarded orientation query: %v", err) + } + + var plan any + if err := session.DB.ReadTransaction(session.Ctx, func(tx graph.Transaction) error { + result := tx.Raw("explain (analyze, timing off, summary off, format json) "+sqlQuery, translation.Parameters) + defer result.Close() + if !result.Next() { + if err := result.Error(); err != nil { + return err + } + return errors.New("PostgreSQL EXPLAIN returned no rows") + } + values := result.Values() + if len(values) == 0 { + return errors.New("PostgreSQL EXPLAIN returned an empty row") + } + parsed, err := normalizeExplainPlan(values[0]) + if err != nil { + return err + } + plan = parsed + return result.Error() + }); err != nil { + t.Fatalf("explain guarded orientation query: %v", err) + } + return plan +} + +func requireOrientationSubplanMetric(t *testing.T, plan any, suffix, metric string, expected int64) { + t.Helper() + + subplan, found := findOrientationSubplan(plan, suffix) + if !found { + t.Fatalf("PostgreSQL JSON plan has no subplan ending in %q", suffix) + } + actual, ok := postgresPlanInt64(subplan[metric]) + if !ok { + t.Fatalf("orientation subplan %q has no numeric %s", subplan["Subplan Name"], metric) + } + if actual != expected { + t.Fatalf("orientation subplan %q %s: got %d, want %d", subplan["Subplan Name"], metric, actual, expected) + } +} + +func findOrientationSubplan(value any, suffix string) (map[string]any, bool) { + switch typed := value.(type) { + case []any: + for _, child := range typed { + if found, ok := findOrientationSubplan(child, suffix); ok { + return found, true + } + } + case map[string]any: + if name, ok := typed["Subplan Name"].(string); ok && strings.HasSuffix(name, suffix) { + return typed, true + } + for _, child := range typed { + if found, ok := findOrientationSubplan(child, suffix); ok { + return found, true + } + } + } + return nil, false +} + +func postgresPlanInt64(value any) (int64, bool) { + switch typed := value.(type) { + case float64: + return int64(typed), typed == float64(int64(typed)) + case int64: + return typed, true + case int: + return int64(typed), true + default: + return 0, false + } +} diff --git a/integration/regression_fixture.go b/integration/regression_fixture.go new file mode 100644 index 00000000..030072ed --- /dev/null +++ b/integration/regression_fixture.go @@ -0,0 +1,146 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package integration + +import ( + "fmt" + + "github.com/specterops/dawgs/opengraph" +) + +// defaultRegressionFanout is the relationship fanout used when a regression +// fixture does not request an explicit size. +const defaultRegressionFanout = 32 + +// FixtureNames returns deterministic fixture identifiers without committing +// large handwritten lists to the corpus. +func FixtureNames(prefix string, count int) []string { + if count < 0 { + count = 0 + } + + width := len(fmt.Sprintf("%d", max(count-1, 0))) + if width < 2 { + width = 2 + } + + values := make([]string, count) + for idx := range count { + values[idx] = fmt.Sprintf("%s-%0*d", prefix, width, idx) + } + + return values +} + +// FixtureKinds returns deterministic synthetic kind names for list-cardinality +// tests. +func FixtureKinds(count int) []string { + if count < 0 { + count = 0 + } + + kinds := make([]string, count) + for idx := range count { + kinds[idx] = fmt.Sprintf("RegressionKind%02d", idx+1) + } + + return kinds +} + +// NewReconciliationFixture builds the reusable reconciliation fixture. It includes +// typed and multi-kind endpoints, duplicate relationship kinds, missing and +// explicit-null properties, timestamps, both directions, and a deterministic +// high-degree anchor. A non-positive fanout selects a small production-like +// default. +func NewReconciliationFixture(fanout int) *opengraph.Graph { + if fanout <= 0 { + fanout = defaultRegressionFanout + } + + fixture := &opengraph.Graph{ + Nodes: []opengraph.Node{ + { + ID: "anchor", + Kinds: []string{"ADEntity", "Computer", "Entity"}, + Properties: map[string]any{"objectid": "anchor-id", "lastcollected": "2026-01-02T00:00:00Z", "name": "anchor"}, + }, + { + ID: "typed-end", + Kinds: []string{"ADEntity", "Group", "Entity"}, + Properties: map[string]any{"objectid": "typed-end-id", "lastcollected": "2026-01-03T00:00:00Z", "name": "typed-end"}, + }, + { + ID: "missing-lastseen", + Kinds: []string{"ADEntity", "Entity"}, + Properties: map[string]any{"objectid": "missing-id"}, + }, + { + ID: "null-lastseen", + Kinds: []string{"ADEntity", "Entity"}, + Properties: map[string]any{"objectid": "null-id", "lastseen": nil}, + }, + }, + Edges: []opengraph.Edge{ + { + StartID: "anchor", + EndID: "typed-end", + Kind: "MemberOf", + Properties: map[string]any{"lastseen": "2026-01-01T00:00:00Z", "isprimarygroup": false, "marker": "duplicate-a"}, + }, + { + StartID: "anchor", + EndID: "typed-end", + Kind: "MemberOf", + Properties: map[string]any{"lastseen": "2026-01-04T00:00:00Z", "isprimarygroup": true, "marker": "duplicate-b"}, + }, + { + StartID: "typed-end", + EndID: "anchor", + Kind: "MemberOf", + Properties: map[string]any{"marker": "reverse"}, + }, + { + StartID: "anchor", + EndID: "missing-lastseen", + Kind: "HasSession", + Properties: map[string]any{"marker": "missing-lastseen"}, + }, + { + StartID: "anchor", + EndID: "null-lastseen", + Kind: "HasSession", + Properties: map[string]any{"lastseen": nil, "marker": "null-lastseen"}, + }, + }, + } + + for _, fixtureID := range FixtureNames("fanout", fanout) { + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: fixtureID, + Kinds: []string{"ADEntity", "Entity", "User"}, + Properties: map[string]any{"objectid": fixtureID, "name": fixtureID}, + }) + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: "anchor", + EndID: fixtureID, + Kind: "FanoutEdge", + Properties: map[string]any{"lastseen": "2026-01-01T00:00:00Z"}, + }) + } + + return fixture +} diff --git a/integration/regression_fixture_test.go b/integration/regression_fixture_test.go new file mode 100644 index 00000000..bcc960d8 --- /dev/null +++ b/integration/regression_fixture_test.go @@ -0,0 +1,48 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package integration + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestFixtureNamesAreDeterministic verifies fixture identifiers are stable and zero-padded for a given prefix and count. +func TestFixtureNamesAreDeterministic(t *testing.T) { + require.Equal(t, []string{"id-00", "id-01", "id-02"}, FixtureNames("id", 3)) + require.Equal(t, []string{"RegressionKind01", "RegressionKind02"}, FixtureKinds(2)) + require.Equal(t, FixtureNames("id", 1_000), FixtureNames("id", 1_000)) + require.Empty(t, FixtureNames("id", -1)) +} + +// TestNewReconciliationFixtureIncludesRequiredShapes verifies the reconciliation fixture contains every typed, null, directional, and fanout shape required by regressions. +func TestNewReconciliationFixtureIncludesRequiredShapes(t *testing.T) { + fixture := NewReconciliationFixture(4) + require.Len(t, fixture.Nodes, 8) + require.Len(t, fixture.Edges, 9) + require.Equal(t, "fanout-00", fixture.Nodes[4].ID) + require.Equal(t, "fanout-03", fixture.Nodes[7].ID) + require.Equal(t, "FanoutEdge", fixture.Edges[5].Kind) + require.Equal(t, "fanout-03", fixture.Edges[8].EndID) + + nodeKinds, edgeKinds := fixture.Kinds() + require.Contains(t, nodeKinds.Strings(), "Computer") + require.Contains(t, nodeKinds.Strings(), "Group") + require.Contains(t, edgeKinds.Strings(), "MemberOf") + require.Contains(t, edgeKinds.Strings(), "HasSession") +} diff --git a/integration/relationship_scans_node_lookups_legacy_builder_test.go b/integration/relationship_scans_node_lookups_legacy_builder_test.go new file mode 100644 index 00000000..9d4a179c --- /dev/null +++ b/integration/relationship_scans_node_lookups_legacy_builder_test.go @@ -0,0 +1,557 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +//go:build manual_integration + +package integration + +import ( + "sort" + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/opengraph" + "github.com/specterops/dawgs/ops" + "github.com/specterops/dawgs/query" + "github.com/stretchr/testify/require" +) + +// TestLegacyBuilderRelationshipScansAndNodeLookups verifies legacy scan and lookup forms preserve expected records and ordering. +func TestLegacyBuilderRelationshipScansAndNodeLookups(t *testing.T) { + wideFixture := regressionTemplateFixture(t, "SCAN-01 through SCAN-04 wide relationship filters") + anchoredFixture := regressionTemplateFixture(t, "SCAN-05 through SCAN-08 anchored scans and projections") + basicFixture := regressionTemplateFixture(t, "LOOKUP-01 through LOOKUP-08 node predicates and projections") + advancedFixture := regressionTemplateFixture(t, "LOOKUP-09 through LOOKUP-14 and LOOKUP-16 advanced lookups") + countFixture := regressionTemplateFixture(t, "LOOKUP-15 dense graph counts") + + var nodeKinds, edgeKinds graph.Kinds + for _, fixture := range []*opengraph.Graph{wideFixture, anchoredFixture, basicFixture, advancedFixture, countFixture} { + nextNodeKinds, nextEdgeKinds := fixture.Kinds() + nodeKinds = nodeKinds.Add(nextNodeKinds...) + edgeKinds = edgeKinds.Add(nextEdgeKinds...) + } + db, ctx := SetupDBWithKindsNoGraphCleanup(t, nodeKinds, edgeKinds) + ClearGraph(t, db, ctx) + session := &Session{ + DB: db, + Ctx: ctx, + } + + t.Run("SCAN-01 base endpoints and relationship IDs", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, wideFixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.KindIn(query.Start(), graph.StringKind("ADBase"), graph.StringKind("AZBase")), + query.Kind(query.Relationship(), graph.StringKind("PostProcessed")), + query.KindIn(query.End(), graph.StringKind("ADBase"), graph.StringKind("AZBase")), + ) + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + ids, err := ops.FetchRelationshipIDs(relationshipQuery) + require.NoError(t, err) + require.Len(t, ids, 4) + return nil + }) + }) + + t.Run("SCAN-02 non-Meta relationship hydration", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, wideFixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Not(query.KindIn(query.Start(), graph.StringKind("Meta"), graph.StringKind("MetaDetail"))), + query.KindIn(query.Relationship(), graph.StringKind("TrackerA"), graph.StringKind("TrackerB")), + query.Not(query.KindIn(query.End(), graph.StringKind("Meta"), graph.StringKind("MetaDetail"))), + ) + }, assertStandaloneHopRelationshipMarkers(t, []string{"tracker-a", "tracker-b"})) + }) + + t.Run("SCAN-03 present lastseen relationship IDs", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, wideFixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Not(query.KindIn(query.Start(), graph.StringKind("Meta"), graph.StringKind("MetaDetail"))), + query.Kind(query.Relationship(), graph.StringKind("MigratedEdge")), + query.Exists(query.RelationshipProperty("lastseen")), + query.Not(query.KindIn(query.End(), graph.StringKind("Meta"), graph.StringKind("MetaDetail"))), + ) + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + ids, err := ops.FetchRelationshipIDs(relationshipQuery) + require.NoError(t, err) + require.Len(t, ids, 1) + return nil + }) + }) + + t.Run("SCAN-04 raw ownership representatives", func(t *testing.T) { + for kind, expected := range map[string]string{"OwnsRaw": "owns", "WriteOwnerRaw": "write-owner"} { + t.Run(kind, func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, wideFixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Kind(query.Relationship(), graph.StringKind(kind)), + query.Kind(query.Start(), graph.StringKind("Entity")), + ) + }, assertStandaloneHopRelationshipMarkers(t, []string{expected})) + }) + } + }) + + t.Run("SCAN-05 consolidated nine-kind inbound scan", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, anchoredFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.Kind(query.Start(), graph.StringKind("Entity")), + query.KindIn(query.Relationship(), scanLookupNineKinds()...), + query.Equals(query.EndID(), idMap["target"]), + ) + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + seenKinds := map[graph.Kind]int{} + err := ops.ForEachStartNode(relationshipQuery, func(relationship *graph.Relationship, node *graph.Node) error { + require.True(t, node.Kinds.ContainsOneOf(graph.StringKind("Entity"))) + seenKinds[relationship.Kind]++ + return nil + }) + require.NoError(t, err) + require.Len(t, seenKinds, 9) + for _, count := range seenKinds { + require.Equal(t, 1, count) + } + return nil + }) + }) + + t.Run("SCAN-06 FetchKinds contract", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, anchoredFixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Kind(query.Relationship(), graph.StringKind("LocalToComputer")), + query.Kind(query.End(), graph.StringKind("Computer")), + ) + }, func(relationshipQuery graph.RelationshipQuery, idMap opengraph.IDMap) error { + return relationshipQuery.FetchKinds(func(cursor graph.Cursor[graph.RelationshipKindsResult]) error { + var results []graph.RelationshipKindsResult + for result := range cursor.Chan() { + results = append(results, result) + } + require.NoError(t, cursor.Error()) + require.Len(t, results, 1) + require.Equal(t, idMap["source-01"], results[0].StartID) + require.Equal(t, idMap["target"], results[0].EndID) + require.Equal(t, graph.StringKind("LocalToComputer"), results[0].Kind) + require.NotZero(t, results[0].ID) + return nil + }) + }) + }) + + t.Run("SCAN-07 directed endpoint pairs", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, anchoredFixture, func(opengraph.IDMap) graph.Criteria { + return query.KindIn(query.Relationship(), graph.StringKind("MemberOf"), graph.StringKind("MemberOfLocalGroup")) + }, func(relationshipQuery graph.RelationshipQuery, idMap opengraph.IDMap) error { + return relationshipQuery.FetchTriples(func(cursor graph.Cursor[graph.RelationshipTripleResult]) error { + count := 0 + duplicatePairCount := 0 + for result := range cursor.Chan() { + count++ + if result.StartID == idMap["source-01"] && result.EndID == idMap["target"] { + duplicatePairCount++ + } + } + require.NoError(t, cursor.Error()) + require.Equal(t, 3, count) + require.Equal(t, 2, duplicatePairCount) + return nil + }) + }) + }) + + t.Run("SCAN-08 both ESC scenarios", func(t *testing.T) { + for _, testCase := range []struct { + // name identifies the ESC scenario subtest. + name string + + // scenarioB selects the alternate endpoint exclusion criteria. + scenarioB bool + + // expected is the number of relationships the scenario should return. + expected int + }{ + { + name: "scenario A", + expected: 3, + }, + { + name: "scenario B", + scenarioB: true, + expected: 2, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, anchoredFixture, func(idMap opengraph.IDMap) graph.Criteria { + criteria := []graph.Criteria{ + query.KindIn(query.Start(), graph.StringKind("Group"), graph.StringKind("User"), graph.StringKind("Computer")), + query.InIDs(query.EndID(), idMap["victim-computer"], idMap["victim-other"], idMap["victim-unused"]), + } + if testCase.scenarioB { + criteria = append(criteria, + query.Kind(query.End(), graph.StringKind("Computer")), + query.KindIn(query.Relationship(), graph.StringKind("GenericAll"), graph.StringKind("GenericWrite"), graph.StringKind("Owns"), graph.StringKind("WriteOwner"), graph.StringKind("WriteDACL")), + ) + } else { + criteria = append(criteria, query.KindIn(query.Relationship(), graph.StringKind("GenericAll"), graph.StringKind("GenericWrite"), graph.StringKind("Owns"), graph.StringKind("WriteOwner"), graph.StringKind("WriteDACL"), graph.StringKind("WritePublicInformation"))) + } + return query.And(criteria...) + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + ids, err := ops.FetchStartNodeIDs(relationshipQuery) + require.NoError(t, err) + require.Len(t, ids, testCase.expected) + return nil + }) + }) + } + }) + + t.Run("LOOKUP-01 kind scans and hydration", func(t *testing.T) { + WithLegacyNodeQuery(t, session, basicFixture, func(opengraph.IDMap) graph.Criteria { + return query.KindIn(query.Node(), graph.StringKind("Group"), graph.StringKind("User")) + }, func(nodeQuery graph.NodeQuery, _ opengraph.IDMap) error { + nodes, err := ops.FetchNodes(nodeQuery) + require.NoError(t, err) + require.Len(t, nodes, 8) + return nil + }) + }) + + t.Run("LOOKUP-02 equality First", func(t *testing.T) { + WithLegacyNodeQuery(t, session, basicFixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Kind(query.Node(), graph.StringKind("Computer")), + query.Equals(query.NodeProperty("objectid"), "S-1-5-21-100"), + ) + }, func(nodeQuery graph.NodeQuery, _ opengraph.IDMap) error { + node, err := nodeQuery.Limit(1).First() + require.NoError(t, err) + require.NotNil(t, node) + return nil + }) + }) + + t.Run("LOOKUP-03 boolean projection order and type", func(t *testing.T) { + WithLegacyNodeQuery(t, session, basicFixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Kind(query.Node(), graph.StringKind("Computer")), + query.Equals(query.NodeProperty("hasura"), true), + ) + }, func(nodeQuery graph.NodeQuery, _ opengraph.IDMap) error { + return nodeQuery.Query(func(results graph.Result) error { + count := 0 + for results.Next() { + var ( + id graph.ID + hasURA bool + ) + + require.NoError(t, results.Scan(&id, &hasURA)) + require.NotZero(t, id) + require.True(t, hasURA) + count++ + } + require.NoError(t, results.Error()) + require.Equal(t, 1, count) + return nil + }, query.Returning(query.NodeID(), query.NodeProperty("hasura"))) + }) + }) + + t.Run("LOOKUP-04 case-sensitive prefix", func(t *testing.T) { + assertScanLookupNodeIDs(t, session, basicFixture, []string{"adminsdholder"}, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Kind(query.Node(), graph.StringKind("Container")), + query.StringStartsWith(query.NodeProperty("distinguishedname"), "CN=ADMINSDHOLDER,CN=SYSTEM,"), + query.Equals(query.NodeProperty("domainsid"), "S-1-5-21"), + ) + }) + }) + + t.Run("LOOKUP-05 case-insensitive contains candidates", func(t *testing.T) { + assertScanLookupNodeIDs(t, session, basicFixture, []string{"ci-contains-exact", "ci-contains-substring"}, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Kind(query.Node(), graph.StringKind("Entity")), + query.CaseInsensitiveStringContains(query.NodeProperty("objectid"), "Approver_GUID"), + ) + }) + }) + + t.Run("LOOKUP-06 required and excluded kinds", func(t *testing.T) { + assertScanLookupNodeIDs(t, session, basicFixture, []string{"entity-only"}, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Kind(query.Node(), graph.StringKind("Entity")), + query.Not(query.KindIn(query.Node(), graph.StringKind("Group"), graph.StringKind("LocalGroup"))), + query.StringEndsWith(query.NodeProperty("objectid"), "-512"), + ) + }) + }) + + t.Run("LOOKUP-07 missing and null properties", func(t *testing.T) { + assertScanLookupNodeIDs(t, session, basicFixture, []string{"name-missing", "name-null"}, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Kind(query.Node(), graph.StringKind("Lookup")), + query.Not(query.Exists(query.NodeProperty("name"))), + ) + }) + }) + + t.Run("LOOKUP-08 nullable approver disjunction", func(t *testing.T) { + assertScanLookupNodeIDs(t, session, basicFixture, []string{"role-both", "role-group", "role-user"}, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Kind(query.Node(), graph.StringKind("AZRole")), + query.Equals(query.NodeProperty("tenantid"), "tenant-1"), + query.Equals(query.NodeProperty("approvalrequired"), true), + query.Or( + query.IsNotNull(query.NodeProperty("userapprovers")), + query.IsNotNull(query.NodeProperty("groupapprovers")), + ), + ) + }) + }) + + t.Run("LOOKUP-09 duplicate ID list hydration", func(t *testing.T) { + WithLegacyNodeQuery(t, session, advancedFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.InIDs(query.NodeID(), idMap["hydrate-a"], idMap["hydrate-a"], idMap["hydrate-b"]) + }, func(nodeQuery graph.NodeQuery, idMap opengraph.IDMap) error { + nodes, err := ops.FetchNodes(nodeQuery) + require.NoError(t, err) + require.Equal(t, []string{"hydrate-a", "hydrate-b"}, scanLookupFixtureIDs(t, idMap, scanLookupNodeIDs(nodes))) + return nil + }) + }) + + t.Run("LOOKUP-10 nested negated account flags", func(t *testing.T) { + WithLegacyNodeQuery(t, session, advancedFixture, func(idMap opengraph.IDMap) graph.Criteria { + ids := make([]graph.ID, 0, 16) + for _, first := range []string{"m", "n", "f", "t"} { + for _, second := range []string{"m", "n", "f", "t"} { + ids = append(ids, idMap["flags-"+first+second]) + } + } + return query.And( + query.Kind(query.Node(), graph.StringKind("User")), + query.Not(query.And(query.Exists(query.NodeProperty("gmsa")), query.Equals(query.NodeProperty("gmsa"), true))), + query.Not(query.And(query.Exists(query.NodeProperty("msa")), query.Equals(query.NodeProperty("msa"), true))), + query.InIDs(query.NodeID(), ids...), + ) + }, func(nodeQuery graph.NodeQuery, _ opengraph.IDMap) error { + nodes, err := ops.FetchNodes(nodeQuery) + require.NoError(t, err) + require.Len(t, nodes, 9) + return nil + }) + }) + + t.Run("LOOKUP-11 tenant adjacency property list", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, advancedFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.Equals(query.StartID(), idMap["tenant"]), + query.Kind(query.Relationship(), graph.StringKind("Contains")), + query.KindIn(query.End(), graph.StringKind("AZRole"), graph.StringKind("AZServicePrincipal")), + query.In(query.EndProperty("roletemplateid"), []string{"role-a", "role-b", "role-multi"}), + ) + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + nodes, err := ops.FetchEndNodes(relationshipQuery) + require.NoError(t, err) + require.Equal(t, 3, nodes.Len()) + return nil + }) + }) + + t.Run("LOOKUP-12 exact edge key First", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, advancedFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.Equals(query.StartID(), idMap["edge-start"]), + query.Equals(query.EndID(), idMap["edge-end"]), + query.Kind(query.Relationship(), graph.StringKind("MemberOf")), + ) + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + relationship, err := relationshipQuery.Limit(1).First() + require.NoError(t, err) + marker, err := relationship.Properties.Get("marker").String() + require.NoError(t, err) + require.Equal(t, "exact-edge", marker) + return nil + }) + }) + + t.Run("LOOKUP-13 suffix and bound endpoint", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, advancedFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.StringEndsWith(query.StartProperty("objectid"), "-555"), + query.Kind(query.Relationship(), graph.StringKind("LocalToComputer")), + query.Equals(query.EndID(), idMap["local-target"]), + ) + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + nodes, err := ops.FetchStartNodes(relationshipQuery) + require.NoError(t, err) + require.Equal(t, 2, nodes.Len()) + return nil + }) + }) + + t.Run("LOOKUP-14 descending node property", func(t *testing.T) { + WithLegacyNodeQuery(t, session, advancedFixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Kind(query.Node(), graph.StringKind("Domain")), + query.Exists(query.NodeProperty("name")), + ) + }, func(nodeQuery graph.NodeQuery, _ opengraph.IDMap) error { + var names []string + err := nodeQuery.OrderBy(query.Order(query.NodeProperty("name"), query.Descending())).Fetch(func(cursor graph.Cursor[*graph.Node]) error { + for node := range cursor.Chan() { + name, err := node.Properties.Get("name").String() + require.NoError(t, err) + names = append(names, name) + } + return cursor.Error() + }) + require.NoError(t, err) + require.Equal(t, []string{"Gamma", "Beta", "Beta", "Alpha"}, names) + return nil + }) + }) + + t.Run("LOOKUP-15 direct sequential counts", func(t *testing.T) { + for _, testCase := range []struct { + // family names the template fixture used by the count subtest. + family string + + // expectedNodes is the fixture's expected node count. + expectedNodes int64 + + // expectedEdges is the fixture's expected relationship count. + expectedEdges int64 + }{ + {family: "LOOKUP-15 empty graph counts"}, + { + family: "LOOKUP-15 node-only graph counts", + expectedNodes: 3, + }, + { + family: "LOOKUP-15 edge-bearing graph counts", + expectedNodes: 2, + expectedEdges: 1, + }, + { + family: "LOOKUP-15 dense graph counts", + expectedNodes: 4, + expectedEdges: 6, + }, + } { + t.Run(testCase.family, func(t *testing.T) { + fixture := regressionTemplateFixture(t, testCase.family) + err := session.WithRollbackFixture(t, fixture, false, func(tx graph.Transaction, _ opengraph.IDMap) error { + nodeCount, err := tx.Nodes().Count() + require.NoError(t, err) + edgeCount, err := tx.Relationships().Count() + require.NoError(t, err) + require.Equal(t, testCase.expectedNodes, nodeCount) + require.Equal(t, testCase.expectedEdges, edgeCount) + return nil + }) + require.NoError(t, err) + }) + } + }) + + t.Run("LOOKUP-16 four-property LDAP and LDAPS forms", func(t *testing.T) { + for _, testCase := range []struct { + // name identifies the LDAP or LDAPS property combination. + name string + + // kind optionally restricts the matched endpoint kind. + kind graph.Kind + + // available names the property that records protocol availability. + available string + + // protection names the protocol protection property. + protection string + + // expected is the object ID of the endpoint that should match. + expected string + }{ + { + name: "typed LDAP", + kind: graph.StringKind("Computer"), + available: "ldapavailable", + protection: "ldapsigning", + expected: "ntlm-ldap-good", + }, + { + name: "untyped LDAPS", + available: "ldapsavailable", + protection: "epa", + expected: "ntlm-ldaps-good", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + assertScanLookupNodeIDs(t, session, advancedFixture, []string{testCase.expected}, func(opengraph.IDMap) graph.Criteria { + criteria := []graph.Criteria{ + query.Equals(query.NodeProperty("domainsid"), "S-1-5-21"), + query.Equals(query.NodeProperty("isdc"), true), + query.Equals(query.NodeProperty(testCase.available), true), + query.Equals(query.NodeProperty(testCase.protection), false), + } + if testCase.kind != nil { + criteria = append([]graph.Criteria{query.Kind(query.Node(), testCase.kind)}, criteria...) + } + return query.And(criteria...) + }) + }) + } + }) +} + +// scanLookupNineKinds returns the nine synthetic relationship kinds used by wide-kind scan cases. +func scanLookupNineKinds() graph.Kinds { + kinds := make(graph.Kinds, 9) + for idx := range kinds { + kinds[idx] = graph.StringKind("ScanEdge0" + string(rune('1'+idx))) + } + return kinds +} + +// scanLookupNodeIDs extracts database IDs from a node result slice without reordering it. +func scanLookupNodeIDs(nodes []*graph.Node) []graph.ID { + ids := make([]graph.ID, len(nodes)) + for idx, node := range nodes { + ids[idx] = node.ID + } + return ids +} + +// scanLookupFixtureIDs maps database IDs to fixture IDs and sorts them for stable comparison. +func scanLookupFixtureIDs(t *testing.T, idMap opengraph.IDMap, ids []graph.ID) []string { + t.Helper() + fixtureIDs := make([]string, len(ids)) + for idx, id := range ids { + fixtureIDs[idx] = regressionFixtureID(t, idMap, id) + } + sort.Strings(fixtureIDs) + return fixtureIDs +} + +// assertScanLookupNodeIDs executes criteria through the legacy node query and compares the resulting fixture IDs. +func assertScanLookupNodeIDs(t *testing.T, session *Session, fixture *opengraph.Graph, expected []string, criteria func(opengraph.IDMap) graph.Criteria) { + t.Helper() + WithLegacyNodeQuery(t, session, fixture, criteria, func(nodeQuery graph.NodeQuery, idMap opengraph.IDMap) error { + ids, err := ops.FetchNodeIDs(nodeQuery) + require.NoError(t, err) + require.Equal(t, expected, scanLookupFixtureIDs(t, idMap, ids)) + return nil + }) +} diff --git a/integration/standalone_hops_legacy_builder_test.go b/integration/standalone_hops_legacy_builder_test.go new file mode 100644 index 00000000..b669ad81 --- /dev/null +++ b/integration/standalone_hops_legacy_builder_test.go @@ -0,0 +1,309 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +//go:build manual_integration + +package integration + +import ( + "fmt" + "sort" + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/opengraph" + "github.com/specterops/dawgs/ops" + "github.com/specterops/dawgs/query" + "github.com/stretchr/testify/require" +) + +// TestLegacyBuilderStandaloneHops verifies legacy one-hop queries preserve direction, kinds, and endpoint projections. +func TestLegacyBuilderStandaloneHops(t *testing.T) { + anchorFixture := regressionTemplateFixture(t, "HOP-01 through HOP-03 anchored direction and relationship-kind cardinality") + idFixture := regressionTemplateFixture(t, "HOP-04 and HOP-05 endpoint kinds and ID constraints") + predicateFixture := regressionTemplateFixture(t, "HOP-06 through HOP-08 scalar nested and collection endpoint predicates") + projectionFixture := regressionTemplateFixture(t, "HOP-09 and HOP-10 two-sided sets and directional projections") + + var nodeKinds, edgeKinds graph.Kinds + for _, fixture := range []*opengraph.Graph{anchorFixture, idFixture, predicateFixture, projectionFixture} { + nextNodeKinds, nextEdgeKinds := fixture.Kinds() + nodeKinds = nodeKinds.Add(nextNodeKinds...) + edgeKinds = edgeKinds.Add(nextEdgeKinds...) + } + db, ctx := SetupDBWithKindsNoGraphCleanup(t, nodeKinds, edgeKinds) + ClearGraph(t, db, ctx) + session := &Session{ + DB: db, + Ctx: ctx, + } + + t.Run("HOP-01 outbound full direction", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, anchorFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.Equals(query.StartID(), idMap["out-one"]), + query.Kind(query.Relationship(), graph.StringKind("HopKind01")), + ) + }, func(relationshipQuery graph.RelationshipQuery, idMap opengraph.IDMap) error { + return relationshipQuery.FetchDirection(graph.DirectionInbound, func(cursor graph.Cursor[graph.DirectionalResult]) error { + results := standaloneHopDirectionalResults(t, cursor) + require.Len(t, results, 1) + require.Equal(t, idMap["out-one-target"], results[0].Node.ID) + return nil + }) + }) + }) + + t.Run("HOP-02 inbound full direction", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, anchorFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.Equals(query.EndID(), idMap["in-one"]), + query.Kind(query.Relationship(), graph.StringKind("HopKind01")), + ) + }, func(relationshipQuery graph.RelationshipQuery, idMap opengraph.IDMap) error { + return relationshipQuery.FetchDirection(graph.DirectionOutbound, func(cursor graph.Cursor[graph.DirectionalResult]) error { + results := standaloneHopDirectionalResults(t, cursor) + require.Len(t, results, 1) + require.Equal(t, idMap["in-one-source"], results[0].Node.ID) + return nil + }) + }) + }) + + t.Run("HOP-03 thirty kinds preserve anchor orientation", func(t *testing.T) { + kinds := make(graph.Kinds, 30) + for idx := range kinds { + kinds[idx] = graph.StringKind(fmt.Sprintf("HopKind%02d", idx+1)) + } + WithLegacyRelationshipQuery(t, session, anchorFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.InIDs(query.StartID(), idMap["kind-center"]), + query.KindIn(query.Relationship(), kinds...), + ) + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + relationships, err := ops.FetchRelationships(relationshipQuery) + require.NoError(t, err) + require.Len(t, relationships, 30) + require.NotContains(t, standaloneHopRelationshipMarkers(t, relationships), "out-disallowed") + return nil + }) + }) + + t.Run("HOP-04 endpoint kind disjunction", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, idFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.InIDs(query.StartID(), idMap["root"]), + query.Kind(query.Relationship(), graph.StringKind("HopTypedEdge")), + query.KindIn(query.End(), graph.StringKind("HopEndA"), graph.StringKind("HopEndB")), + ) + }, func(relationshipQuery graph.RelationshipQuery, idMap opengraph.IDMap) error { + nodes, err := ops.FetchEndNodes(relationshipQuery) + require.NoError(t, err) + require.Equal(t, 3, nodes.Len()) + require.True(t, nodes.ContainsID(idMap["typed-a"])) + require.True(t, nodes.ContainsID(idMap["typed-b"])) + require.True(t, nodes.ContainsID(idMap["typed-multi"])) + return nil + }) + }) + + t.Run("HOP-05 endpoint IDs and traversal anchor contradiction", func(t *testing.T) { + for _, testCase := range []struct { + // name identifies whether the root constraint agrees with the path. + name string + + // allowedRoot is the fixture ID admitted by the root constraint. + allowedRoot string + + // expected lists the endpoint object IDs returned by the query. + expected []string + }{ + { + name: "matching", + allowedRoot: "root", + expected: []string{"id-a", "id-b"}, + }, + { + name: "contradictory", + allowedRoot: "other-root", + expected: nil, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, idFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.Equals(query.StartID(), idMap["root"]), + query.InIDs(query.Start(), idMap[testCase.allowedRoot]), + query.InIDs(query.EndID(), idMap["id-a"], idMap["id-b"]), + query.Kind(query.Relationship(), graph.StringKind("HopIDEdge")), + ) + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + relationships, err := ops.FetchRelationships(relationshipQuery) + require.NoError(t, err) + require.ElementsMatch(t, testCase.expected, standaloneHopRelationshipMarkers(t, relationships)) + return nil + }) + }) + } + }) + + t.Run("HOP-06 scalar property", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, predicateFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.Equals(query.StartID(), idMap["root"]), + query.Kind(query.Relationship(), graph.StringKind("HopPropertyEdge")), + query.Equals(query.EndProperty("enabled"), true), + query.Equals(query.EndProperty("score"), 7), + query.Equals(query.EndProperty("value"), "alpha"), + query.Equals(query.EndProperty("isassignabletorole"), "true"), + ) + }, assertStandaloneHopRelationshipMarkers(t, []string{"scalar-match"})) + }) + + t.Run("HOP-07 nested branch-local predicate", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, predicateFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.Equals(query.StartID(), idMap["root"]), + query.Kind(query.Relationship(), graph.StringKind("HopNestedEdge")), + query.Kind(query.End(), graph.StringKind("HopTemplate")), + query.Or( + query.And( + query.Equals(query.EndProperty("requiresmanagerapproval"), false), + query.GreaterThan(query.EndProperty("schemaversion"), 1), + query.Equals(query.EndProperty("authorizedsignatures"), 0), + query.Equals(query.EndProperty("authenticationenabled"), true), + ), + query.And( + query.Equals(query.EndProperty("requiresmanagerapproval"), false), + query.Equals(query.EndProperty("schemaversion"), 1), + query.Equals(query.EndProperty("authenticationenabled"), true), + ), + ), + ) + }, assertStandaloneHopRelationshipMarkers(t, []string{"nested-v1", "nested-v2"})) + }) + + t.Run("HOP-08 collection OR scalar predicate", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, predicateFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.Equals(query.StartID(), idMap["root"]), + query.Kind(query.Relationship(), graph.StringKind("HopCollectionEdge")), + query.Or( + query.Equals(query.EndProperty("schannelauthenticationenabled"), true), + query.Equals(query.Size(query.EndProperty("effectiveekus")), 0), + query.InInverted(query.EndProperty("effectiveekus"), "1.3.6.1.5.5.7.3.2"), + ), + ) + }, assertStandaloneHopRelationshipMarkers(t, []string{"collection-client", "collection-empty", "collection-scalar"})) + }) + + t.Run("HOP-09 two-sided ID lists", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, projectionFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.InIDs(query.StartID(), idMap["s1"], idMap["s2"]), + query.InIDs(query.EndID(), idMap["e1"], idMap["e2"]), + query.Kind(query.Relationship(), graph.StringKind("HopSetEdge")), + ) + }, assertStandaloneHopRelationshipMarkers(t, []string{"s1-e1", "s1-e2", "s2-e1", "s2-e2"})) + }) + + t.Run("HOP-10 both full directional projections", func(t *testing.T) { + t.Run("outbound", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, projectionFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.InIDs(query.StartID(), idMap["s1"]), + query.Kind(query.Relationship(), graph.StringKind("HopProjectionEdge")), + query.Kind(query.End(), graph.StringKind("HopProjectionEnd")), + query.Equals(query.EndProperty("active"), true), + ) + }, func(relationshipQuery graph.RelationshipQuery, idMap opengraph.IDMap) error { + return relationshipQuery.FetchDirection(graph.DirectionInbound, func(cursor graph.Cursor[graph.DirectionalResult]) error { + results := standaloneHopDirectionalResults(t, cursor) + require.Len(t, results, 1) + require.Equal(t, idMap["e1"], results[0].Node.ID) + return nil + }) + }) + }) + + t.Run("inbound", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, projectionFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.InIDs(query.EndID(), idMap["e1"]), + query.Kind(query.Relationship(), graph.StringKind("HopProjectionEdge")), + query.Kind(query.Start(), graph.StringKind("HopProjectionStart")), + query.Equals(query.StartProperty("active"), true), + ) + }, func(relationshipQuery graph.RelationshipQuery, idMap opengraph.IDMap) error { + return relationshipQuery.FetchDirection(graph.DirectionOutbound, func(cursor graph.Cursor[graph.DirectionalResult]) error { + results := standaloneHopDirectionalResults(t, cursor) + require.Len(t, results, 1) + require.Equal(t, idMap["s1"], results[0].Node.ID) + return nil + }) + }) + }) + }) +} + +// regressionTemplateFixture returns the inline fixture belonging to the named +// Cypher template family. +func regressionTemplateFixture(t *testing.T, familyName string) *opengraph.Graph { + t.Helper() + for _, templateFile := range loadCypherTemplateFiles(t) { + for _, family := range templateFile.Families { + if family.Name == familyName { + return family.Fixture + } + } + } + t.Fatalf("template family %q not found", familyName) + return nil +} + +// standaloneHopDirectionalResults drains a directional cursor and fails the current test on cursor error. +func standaloneHopDirectionalResults(t *testing.T, cursor graph.Cursor[graph.DirectionalResult]) []graph.DirectionalResult { + t.Helper() + var results []graph.DirectionalResult + for result := range cursor.Chan() { + results = append(results, result) + } + require.NoError(t, cursor.Error()) + return results +} + +// standaloneHopRelationshipMarkers returns sorted marker properties from a relationship slice. +func standaloneHopRelationshipMarkers(t *testing.T, relationships []*graph.Relationship) []string { + t.Helper() + markers := make([]string, 0, len(relationships)) + for _, relationship := range relationships { + marker, err := relationship.Properties.Get("marker").String() + require.NoError(t, err) + markers = append(markers, marker) + } + sort.Strings(markers) + return markers +} + +// assertStandaloneHopRelationshipMarkers returns a legacy-query assertion that compares sorted relationship markers. +func assertStandaloneHopRelationshipMarkers(t *testing.T, expected []string) func(graph.RelationshipQuery, opengraph.IDMap) error { + t.Helper() + return func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + relationships, err := ops.FetchRelationships(relationshipQuery) + require.NoError(t, err) + require.Equal(t, expected, standaloneHopRelationshipMarkers(t, relationships)) + return nil + } +} diff --git a/integration/testdata/README.md b/integration/testdata/README.md new file mode 100644 index 00000000..dea35272 --- /dev/null +++ b/integration/testdata/README.md @@ -0,0 +1,73 @@ +# Integration Corpus + +Files under `cases/` execute one Cypher query per case. Files under `templates/` +share a fixture and query template across variants. Fixture-backed cases run in +a write transaction that is always rolled back. + +Mutation cases use `assert: "no_error"` (or another primary result assertion) +and one or more `post_assertions`. The primary mutation result is fully drained +and checked before post-state queries run in the same transaction. Each +post-state entry contains `cypher`, optional `params`, and `assert`. + +The assertion vocabulary includes exact fixture-backed state checks: + +- `node_id_set` for exact surviving node IDs; +- `node_records` for exact node IDs, kinds, and complete property maps; +- `relationship_triples` for exact directed start/end/kind triples; +- `relationship_records` for exact triples and complete property maps; +- `exact_int` and `row_count` for counts. + +Every new reconciliation or post-processing mutation fixture must contain a +positive match and applicable decoys for direction, kind, property, fixture ID, +missing/null property state, and relationship property. Reuse +`NewReconciliationFixture`, `FixtureNames`, and `FixtureKinds` from the +`integration` package for deterministic Go integration cases and large +cardinality lists. + +Tagged datetime parameters decode to `time.Time`: + +```json +{ + "params": { + "threshold": { + "$type": "datetime", + "value": "2026-01-02T03:04:05Z" + } + } +} +``` + +Raw Cypher may instead use an explicit conversion such as +`datetime($threshold)`. Legacy query-builder cases must pass `time.Time` +directly. + +Large string-list parameters use the same tagged parameter decoder without a +large handwritten JSON array: + +```json +{ + "params": { + "object_ids": { + "$type": "string_list", + "prefix": "missing", + "count": 1000, + "include": ["target-id"] + } + } +} +``` + +Fixture-backed cases and template variants can bind database IDs without +hard-coding them. `node_params` maps a query parameter to one fixture node ID; +`node_list_params` maps a parameter to an ordered list of fixture node IDs: + +```json +{ + "node_params": {"start_id": "start"}, + "node_list_params": {"end_ids": ["end-a", "end-b"]} +} +``` + +The integration runner and `cmd/plancorpus` resolve these fields after loading +the fixture, so semantic execution and plan capture use the same ID-anchored +query shape. diff --git a/integration/testdata/adcs_fanout.json b/integration/testdata/adcs_fanout.json deleted file mode 100644 index dafbb835..00000000 --- a/integration/testdata/adcs_fanout.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "graph": { - "nodes": [ - {"id": "n", "kinds": ["Group"], "properties": {"objectid": "S-1-5-21-2643190041-1319121918-239771340-513"}}, - {"id": "p1-a", "kinds": ["Group"]}, - {"id": "p1-b", "kinds": ["Group"]}, - {"id": "p1-c", "kinds": ["Group"]}, - {"id": "p2-good", "kinds": ["Group"]}, - {"id": "p2-disabled", "kinds": ["Group"]}, - {"id": "p2-wrong-ca", "kinds": ["Group"]}, - {"id": "ca", "kinds": ["EnterpriseCA"]}, - {"id": "other-ca", "kinds": ["EnterpriseCA"]}, - {"id": "store", "kinds": ["NTAuthStore"]}, - {"id": "domain", "kinds": ["Domain"]}, - {"id": "other-domain", "kinds": ["Domain"]}, - {"id": "template-good", "kinds": ["CertTemplate"], "properties": {"authenticationenabled": true, "requiresmanagerapproval": false, "enrolleesuppliessubject": true, "schemaversion": 1, "authorizedsignatures": 1}}, - {"id": "template-alt", "kinds": ["CertTemplate"], "properties": {"authenticationenabled": true, "requiresmanagerapproval": false, "enrolleesuppliessubject": true, "schemaversion": 2, "authorizedsignatures": 0}}, - {"id": "template-disabled", "kinds": ["CertTemplate"], "properties": {"authenticationenabled": false, "requiresmanagerapproval": true, "enrolleesuppliessubject": false, "schemaversion": 2, "authorizedsignatures": 1}}, - {"id": "template-wrong-ca", "kinds": ["CertTemplate"], "properties": {"authenticationenabled": true, "requiresmanagerapproval": false, "enrolleesuppliessubject": true, "schemaversion": 1, "authorizedsignatures": 1}}, - {"id": "root", "kinds": ["RootCA"]}, - {"id": "other-root", "kinds": ["RootCA"]} - ], - "edges": [ - {"start_id": "n", "end_id": "p1-a", "kind": "MemberOf"}, - {"start_id": "n", "end_id": "p1-b", "kind": "MemberOf"}, - {"start_id": "p1-b", "end_id": "p1-c", "kind": "MemberOf"}, - {"start_id": "n", "end_id": "p2-good", "kind": "MemberOf"}, - {"start_id": "n", "end_id": "p2-disabled", "kind": "MemberOf"}, - {"start_id": "n", "end_id": "p2-wrong-ca", "kind": "MemberOf"}, - {"start_id": "n", "end_id": "ca", "kind": "Enroll"}, - {"start_id": "p1-a", "end_id": "ca", "kind": "Enroll"}, - {"start_id": "p1-b", "end_id": "ca", "kind": "Enroll"}, - {"start_id": "p1-c", "end_id": "ca", "kind": "Enroll"}, - {"start_id": "ca", "end_id": "store", "kind": "TrustedForNTAuth"}, - {"start_id": "store", "end_id": "domain", "kind": "NTAuthStoreFor"}, - {"start_id": "p2-good", "end_id": "template-good", "kind": "GenericAll"}, - {"start_id": "p2-good", "end_id": "template-alt", "kind": "Enroll"}, - {"start_id": "p2-disabled", "end_id": "template-disabled", "kind": "AllExtendedRights"}, - {"start_id": "p2-wrong-ca", "end_id": "template-wrong-ca", "kind": "GenericAll"}, - {"start_id": "template-good", "end_id": "ca", "kind": "PublishedTo"}, - {"start_id": "template-alt", "end_id": "ca", "kind": "PublishedTo"}, - {"start_id": "template-disabled", "end_id": "ca", "kind": "PublishedTo"}, - {"start_id": "template-wrong-ca", "end_id": "other-ca", "kind": "PublishedTo"}, - {"start_id": "ca", "end_id": "root", "kind": "IssuedSignedBy"}, - {"start_id": "ca", "end_id": "other-root", "kind": "EnterpriseCAFor"}, - {"start_id": "root", "end_id": "domain", "kind": "RootCAFor"}, - {"start_id": "other-root", "end_id": "other-domain", "kind": "RootCAFor"} - ] - } -} diff --git a/integration/testdata/bed8967.json b/integration/testdata/bed8967.json new file mode 100644 index 00000000..6d915e7b --- /dev/null +++ b/integration/testdata/bed8967.json @@ -0,0 +1,39 @@ +{ + "graph": { + "nodes": [ + { + "id": "alpha", + "kinds": ["BacktickNode"], + "properties": { + "name": "alpha", + "a-aaa": "alpha-hyphen", + "has`tick": "alpha-backtick", + " ": "alpha-whitespace", + "a\u20dd": "alpha-enclosing" + } + }, + { + "id": "beta", + "kinds": ["BacktickNode"], + "properties": { + "name": "beta", + "a-aaa": "beta-hyphen", + "has`tick": "beta-backtick", + " ": "beta-whitespace", + "a\u20dd": "beta-enclosing" + } + } + ], + "edges": [ + { + "start_id": "alpha", + "end_id": "beta", + "kind": "BacktickEdge", + "properties": { + "edge-key": "edge-hyphen", + "has`tick": "edge-backtick" + } + } + ] + } +} diff --git a/integration/testdata/cases/bed8967-backtick_property_keys.json b/integration/testdata/cases/bed8967-backtick_property_keys.json new file mode 100644 index 00000000..d2cfec05 --- /dev/null +++ b/integration/testdata/cases/bed8967-backtick_property_keys.json @@ -0,0 +1,52 @@ +{ + "dataset": "bed8967", + "cases": [ + { + "name": "BED-8967 read escaped node property keys", + "cypher": "match (n:BacktickNode) return n.`a-aaa`, n.`has``tick`, n.` `, n.`a\u20dd` order by n.name", + "assert": { + "ordered_row_values": [ + ["alpha-hyphen", "alpha-backtick", "alpha-whitespace", "alpha-enclosing"], + ["beta-hyphen", "beta-backtick", "beta-whitespace", "beta-enclosing"] + ] + } + }, + { + "name": "BED-8967 reject empty escaped property key", + "cypher": "match (n:BacktickNode) return n.``", + "assert": "query_error" + }, + { + "name": "BED-8967 filter node using escaped pattern property key", + "cypher": "match (n:BacktickNode {`a-aaa`: 'beta-hyphen'}) return n.name", + "assert": {"scalar_values": ["beta"]} + }, + { + "name": "BED-8967 read escaped relationship property keys", + "cypher": "match (:BacktickNode {name: 'alpha'})-[r:BacktickEdge]->(:BacktickNode {name: 'beta'}) return r.`edge-key`, r.`has``tick`", + "assert": {"row_values": [["edge-hyphen", "edge-backtick"]]} + }, + { + "name": "BED-8967 set escaped node property keys", + "cypher": "match (n:BacktickNode {name: 'mutable'}) set n.`set-key` = 'set-value', n.`has``tick` = 'updated-backtick' return n.`set-key`, n.`has``tick`", + "fixture": { + "nodes": [ + {"id": "mutable", "kinds": ["BacktickNode"], "properties": {"name": "mutable", "has`tick": "old-backtick"}} + ], + "edges": [] + }, + "assert": {"row_values": [["set-value", "updated-backtick"]]} + }, + { + "name": "BED-8967 remove escaped node property key", + "cypher": "match (n:BacktickNode {name: 'removable'}) remove n.`remove-key` return n.`remove-key`", + "fixture": { + "nodes": [ + {"id": "removable", "kinds": ["BacktickNode"], "properties": {"name": "removable", "remove-key": "remove-me"}} + ], + "edges": [] + }, + "assert": {"scalar_values": [null]} + } + ] +} diff --git a/integration/testdata/cases/expand_into.json b/integration/testdata/cases/expand_into.json new file mode 100644 index 00000000..4f7a8638 --- /dev/null +++ b/integration/testdata/cases/expand_into.json @@ -0,0 +1,90 @@ +{ + "dataset": "expand_into", + "cases": [ + { + "name": "fixed one-hop ExpandInto preserves typed relationship identity", + "cypher": "MATCH (s:ExpandIntoSource), (e:ExpandIntoTarget) WHERE s.name = 'source' AND e.name = 'target' MATCH (s)-[r:ExpandIntoKindA]->(e) RETURN type(r), r.slot", + "assert": {"ordered_row_values": [["ExpandIntoKindA", "a"]]} + }, + { + "name": "fixed one-hop ExpandInto preserves wildcard cross-kind multiplicity", + "cypher": "MATCH (s:ExpandIntoSource), (e:ExpandIntoTarget) WHERE s.name = 'source' AND e.name = 'target' MATCH (s)-[r]->(e) RETURN type(r), r.slot ORDER BY type(r)", + "assert": {"ordered_row_values": [["ExpandIntoKindA", "a"], ["ExpandIntoKindB", "b"]]} + }, + { + "name": "fixed one-hop ExpandInto preserves multi-kind relationship rows", + "cypher": "MATCH (s:ExpandIntoSource), (e:ExpandIntoTarget) WHERE s.name = 'source' AND e.name = 'target' MATCH (s)-[r:ExpandIntoKindA|ExpandIntoKindB]->(e) RETURN type(r), r.slot ORDER BY type(r)", + "assert": {"ordered_row_values": [["ExpandIntoKindA", "a"], ["ExpandIntoKindB", "b"]]} + }, + { + "name": "fixed one-hop ExpandInto reapplies duplicate outer pair multiplicity", + "cypher": "MATCH (s:ExpandIntoSource), (e:ExpandIntoTarget) WHERE s.name = 'source' AND e.name = 'target' WITH s, e, [1, 2] AS copies UNWIND copies AS copy MATCH (s)-[r:ExpandIntoKindA|ExpandIntoKindB]->(e) RETURN copy, type(r) ORDER BY copy, type(r)", + "assert": {"ordered_row_values": [[1, "ExpandIntoKindA"], [1, "ExpandIntoKindB"], [2, "ExpandIntoKindA"], [2, "ExpandIntoKindB"]]} + }, + { + "name": "fixed one-hop ExpandInto recognizes node endpoints introduced by UNWIND", + "cypher": "MATCH (s:ExpandIntoSource), (e:ExpandIntoTarget) WHERE s.name = 'source' AND e.name = 'target' WITH collect(s) AS sources, e UNWIND sources AS source MATCH (source)-[r:ExpandIntoKindA]->(e) RETURN r.slot", + "assert": {"ordered_row_values": [["a"]]} + }, + { + "name": "fixed one-hop ExpandInto preserves self loop", + "cypher": "MATCH (s:ExpandIntoSource), (e:ExpandIntoTarget) WHERE s.name = 'loop' AND e.name = 'loop' MATCH (s)-[r:ExpandIntoKindA]->(e) RETURN type(r), r.slot", + "assert": {"ordered_row_values": [["ExpandIntoKindA", "loop"]]} + }, + { + "name": "fixed one-hop ExpandInto missing pair is empty", + "cypher": "MATCH (s:ExpandIntoSource), (e:ExpandIntoTarget) WHERE s.name = 'missing' AND e.name = 'target' MATCH (s)-[r]->(e) RETURN r", + "assert": "empty" + }, + { + "name": "fixed one-hop ExpandInto preserves a pair when the source has lower degree", + "cypher": "MATCH (s:ExpandIntoSource), (e:ExpandIntoTarget) WHERE s.name = 'low-source' AND e.name = 'high-target' MATCH (s)-[r:ExpandIntoKindA]->(e) RETURN r.slot", + "assert": {"ordered_row_values": [["low-source-match"]]} + }, + { + "name": "fixed one-hop ExpandInto preserves a pair when the target has lower degree", + "cypher": "MATCH (s:ExpandIntoSource), (e:ExpandIntoTarget) WHERE s.name = 'high-source' AND e.name = 'low-target' MATCH (s)-[r:ExpandIntoKindA]->(e) RETURN r.slot", + "assert": {"ordered_row_values": [["low-target-match"]]} + }, + { + "name": "fixed one-hop ExpandInto preserves reversed directionless cross-kind multiplicity", + "cypher": "MATCH (s:ExpandIntoTarget), (e:ExpandIntoSource) WHERE s.name = 'target' AND e.name = 'source' MATCH (s)-[r:ExpandIntoKindA|ExpandIntoKindB]-(e) RETURN type(r), r.slot ORDER BY type(r)", + "assert": {"ordered_row_values": [["ExpandIntoKindA", "a"], ["ExpandIntoKindB", "b"]]} + }, + { + "name": "fixed one-hop ExpandInto preserves inbound cross-kind multiplicity", + "cypher": "MATCH (s:ExpandIntoTarget), (e:ExpandIntoSource) WHERE s.name = 'target' AND e.name = 'source' MATCH (s)<-[r:ExpandIntoKindA|ExpandIntoKindB]-(e) RETURN type(r), r.slot ORDER BY type(r)", + "assert": {"ordered_row_values": [["ExpandIntoKindA", "a"], ["ExpandIntoKindB", "b"]]} + }, + { + "name": "fixed one-hop ExpandInto emits a directionless self loop once", + "cypher": "MATCH (s:ExpandIntoSource), (e:ExpandIntoTarget) WHERE s.name = 'loop' AND e.name = 'loop' MATCH (s)-[r:ExpandIntoKindA]-(e) RETURN type(r), r.slot", + "assert": {"ordered_row_values": [["ExpandIntoKindA", "loop"]]} + }, + { + "name": "fixed one-hop directionless traversal emits an unbound self loop once", + "cypher": "MATCH (s:ExpandIntoSource)-[r:ExpandIntoKindA]-(e:ExpandIntoTarget) WHERE r.slot = 'loop' RETURN type(r), r.slot", + "assert": {"ordered_row_values": [["ExpandIntoKindA", "loop"]]} + }, + { + "name": "fixed one-hop directionless traversal emits a single-bound self loop once", + "cypher": "MATCH (s:ExpandIntoSource) WHERE s.name = 'loop' MATCH (s)-[r:ExpandIntoKindA]-(e:ExpandIntoTarget) RETURN type(r), r.slot", + "assert": {"ordered_row_values": [["ExpandIntoKindA", "loop"]]} + }, + { + "name": "fixed one-hop ExpandInto preserves arbitrary varying bound pairs", + "cypher": "MATCH (s:ExpandIntoSource), (e:ExpandIntoTarget) WHERE (s.name = 'source' AND e.name = 'target') OR (s.name = 'low-source' AND e.name = 'high-target') MATCH (s)-[r:ExpandIntoKindA]->(e) RETURN s.name, e.name, r.slot ORDER BY s.name", + "assert": {"ordered_row_values": [["low-source", "high-target", "low-source-match"], ["source", "target", "a"]]} + }, + { + "name": "optional fixed one-hop ExpandInto preserves null relationship rows", + "cypher": "MATCH (s:ExpandIntoSource), (e:ExpandIntoTarget) WHERE s.name = 'missing' AND e.name = 'target' OPTIONAL MATCH (s)-[r:ExpandIntoKindA]->(e) RETURN s.name, e.name, r.slot", + "assert": {"ordered_row_values": [["missing", "target", null]]} + }, + { + "name": "fixed one-hop ExpandInto composes with multiple path bindings", + "cypher": "MATCH (s:ExpandIntoSource), (e:ExpandIntoTarget) WHERE s.name = 'source' AND e.name = 'target' MATCH p = (s)-[r:ExpandIntoKindA]->(e) MATCH q = (s)-[r2:ExpandIntoKindB]->(e) RETURN length(p), length(q), r.slot, r2.slot", + "assert": {"ordered_row_values": [[1, 1, "a", "b"]]} + } + ] +} diff --git a/integration/testdata/cases/mutation_post_state_inline.json b/integration/testdata/cases/mutation_post_state_inline.json new file mode 100644 index 00000000..6dffb02d --- /dev/null +++ b/integration/testdata/cases/mutation_post_state_inline.json @@ -0,0 +1,67 @@ +{ + "cases": [ + { + "name": "relationship mutation assertions preserve every decoy", + "cypher": "MATCH (s:NodeKind1)-[r:EdgeKind1]->(e:NodeKind2) WHERE e.objectid = $object_id AND r.shoulddelete = $should_delete DELETE r", + "params": { + "object_id": "target-id", + "should_delete": true + }, + "fixture": { + "nodes": [ + {"id": "source", "kinds": ["NodeKind1"], "properties": {"name": "source"}}, + {"id": "source-opposite", "kinds": ["NodeKind1"], "properties": {"name": "source-opposite"}}, + {"id": "source-missing", "kinds": ["NodeKind1"], "properties": {"name": "source-missing"}}, + {"id": "target", "kinds": ["NodeKind2"], "properties": {"objectid": "target-id"}}, + {"id": "wrong-kind", "kinds": ["NodeKind1"], "properties": {"objectid": "target-id"}}, + {"id": "wrong-id", "kinds": ["NodeKind2"], "properties": {"objectid": "decoy-id"}} + ], + "edges": [ + {"start_id": "source", "end_id": "target", "kind": "EdgeKind1", "properties": {"shoulddelete": true, "marker": "target"}}, + {"start_id": "source-opposite", "end_id": "target", "kind": "EdgeKind1", "properties": {"shoulddelete": false, "marker": "opposite-property"}}, + {"start_id": "source-missing", "end_id": "target", "kind": "EdgeKind1", "properties": {"marker": "missing-property"}}, + {"start_id": "source", "end_id": "target", "kind": "EdgeKind2", "properties": {"shoulddelete": true, "marker": "wrong-edge-kind"}}, + {"start_id": "source", "end_id": "wrong-kind", "kind": "EdgeKind1", "properties": {"shoulddelete": true, "marker": "wrong-node-kind"}}, + {"start_id": "source", "end_id": "wrong-id", "kind": "EdgeKind1", "properties": {"shoulddelete": true, "marker": "wrong-object-id"}}, + {"start_id": "target", "end_id": "source", "kind": "EdgeKind1", "properties": {"shoulddelete": true, "marker": "reverse-direction"}} + ] + }, + "assert": "no_error", + "post_assertions": [ + { + "name": "exact surviving nodes and properties", + "cypher": "MATCH (n) RETURN n", + "assert": { + "node_records": [ + {"id": "source", "kinds": ["NodeKind1"], "props": {"name": "source"}}, + {"id": "source-opposite", "kinds": ["NodeKind1"], "props": {"name": "source-opposite"}}, + {"id": "source-missing", "kinds": ["NodeKind1"], "props": {"name": "source-missing"}}, + {"id": "target", "kinds": ["NodeKind2"], "props": {"objectid": "target-id"}}, + {"id": "wrong-kind", "kinds": ["NodeKind1"], "props": {"objectid": "target-id"}}, + {"id": "wrong-id", "kinds": ["NodeKind2"], "props": {"objectid": "decoy-id"}} + ] + } + }, + { + "name": "exact surviving relationships and properties", + "cypher": "MATCH ()-[r]->() RETURN r", + "assert": { + "relationship_records": [ + {"start": "source-opposite", "end": "target", "kind": "EdgeKind1", "props": {"shoulddelete": false, "marker": "opposite-property"}}, + {"start": "source-missing", "end": "target", "kind": "EdgeKind1", "props": {"marker": "missing-property"}}, + {"start": "source", "end": "target", "kind": "EdgeKind2", "props": {"shoulddelete": true, "marker": "wrong-edge-kind"}}, + {"start": "source", "end": "wrong-kind", "kind": "EdgeKind1", "props": {"shoulddelete": true, "marker": "wrong-node-kind"}}, + {"start": "source", "end": "wrong-id", "kind": "EdgeKind1", "props": {"shoulddelete": true, "marker": "wrong-object-id"}}, + {"start": "target", "end": "source", "kind": "EdgeKind1", "props": {"shoulddelete": true, "marker": "reverse-direction"}} + ] + } + }, + { + "name": "exact surviving relationship count", + "cypher": "MATCH ()-[r]->() RETURN count(r)", + "assert": {"exact_int": 6} + } + ] + } + ] +} diff --git a/integration/testdata/cases/optimizer_inline.json b/integration/testdata/cases/optimizer_inline.json index 96c9a2a7..adb4f3b8 100644 --- a/integration/testdata/cases/optimizer_inline.json +++ b/integration/testdata/cases/optimizer_inline.json @@ -1,189 +1,194 @@ { "cases": [ { - "name": "return two ADCS-style paths with shared CA and domain endpoints", - "cypher": "MATCH (n:Group) WHERE n.objectid = 'S-1-5-21-2643190041-1319121918-239771340-513' MATCH p1 = (n)-[:MemberOf*0..]->()-[:Enroll]->(ca:EnterpriseCA)-[:TrustedForNTAuth]->(:NTAuthStore)-[:NTAuthStoreFor]->(d:Domain) MATCH p2 = (n)-[:MemberOf*0..]->()-[:GenericAll|Enroll|AllExtendedRights]->(ct:CertTemplate)-[:PublishedTo]->(ca)-[:IssuedSignedBy|EnterpriseCAFor*1..]->(:RootCA)-[:RootCAFor]->(d) WHERE ct.authenticationenabled = true AND ct.requiresmanagerapproval = false AND ct.enrolleesuppliessubject = true AND (ct.schemaversion = 1 OR ct.authorizedsignatures = 0) RETURN p1, p2", + "name": "return two fixed-suffix expansion paths with shared endpoints", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = 'fixed-suffix-shared-endpoints-root' MATCH direct_path = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) MATCH predicate_path = (root)-[:Expand*0..16]->()-[:OptionA|OptionB|OptionC]->(predicate:PredicateNode)-[:JoinSuffix]->(head)-[:HeadToBridge|HeadToAlternateBridge*1..16]->(:BridgeNode)-[:ReachTerminal]->(terminal) WHERE predicate.eligible = true AND predicate.requires_review = false AND predicate.allows_direct = true AND (predicate.version = 1 OR predicate.required_approvals = 0) RETURN direct_path, predicate_path", "fixture": { "nodes": [ - {"id": "n", "kinds": ["Group"], "properties": {"objectid": "S-1-5-21-2643190041-1319121918-239771340-513"}}, - {"id": "p1-mid", "kinds": ["Group"]}, - {"id": "p2-mid", "kinds": ["Group"]}, - {"id": "ca", "kinds": ["EnterpriseCA"]}, - {"id": "store", "kinds": ["NTAuthStore"]}, - {"id": "domain", "kinds": ["Domain"]}, - {"id": "template", "kinds": ["CertTemplate"], "properties": {"authenticationenabled": true, "requiresmanagerapproval": false, "enrolleesuppliessubject": true, "schemaversion": 1, "authorizedsignatures": 1}}, - {"id": "root", "kinds": ["RootCA"]}, - {"id": "unused-root", "kinds": ["RootCA"]}, - {"id": "unused-template", "kinds": ["CertTemplate"], "properties": {"authenticationenabled": false, "requiresmanagerapproval": true, "enrolleesuppliessubject": false, "schemaversion": 2, "authorizedsignatures": 1}} + {"id": "root", "kinds": ["ExpansionRoot"], "properties": {"root_key": "fixed-suffix-shared-endpoints-root"}}, + {"id": "direct-mid", "kinds": ["ExpansionNode"]}, + {"id": "predicate-mid", "kinds": ["ExpansionNode"]}, + {"id": "suffix-head", "kinds": ["SuffixHead"]}, + {"id": "suffix-middle", "kinds": ["SuffixMiddle"]}, + {"id": "suffix-terminal", "kinds": ["SuffixTerminal"]}, + {"id": "predicate", "kinds": ["PredicateNode"], "properties": {"eligible": true, "requires_review": false, "allows_direct": true, "version": 1, "required_approvals": 1}}, + {"id": "bridge", "kinds": ["BridgeNode"]}, + {"id": "unused-bridge", "kinds": ["BridgeNode"]}, + {"id": "unused-predicate", "kinds": ["PredicateNode"], "properties": {"eligible": false, "requires_review": true, "allows_direct": false, "version": 2, "required_approvals": 1}} ], "edges": [ - {"start_id": "n", "end_id": "p1-mid", "kind": "MemberOf"}, - {"start_id": "p1-mid", "end_id": "ca", "kind": "Enroll"}, - {"start_id": "ca", "end_id": "store", "kind": "TrustedForNTAuth"}, - {"start_id": "store", "end_id": "domain", "kind": "NTAuthStoreFor"}, - {"start_id": "n", "end_id": "p2-mid", "kind": "MemberOf"}, - {"start_id": "p2-mid", "end_id": "template", "kind": "GenericAll"}, - {"start_id": "template", "end_id": "ca", "kind": "PublishedTo"}, - {"start_id": "ca", "end_id": "root", "kind": "IssuedSignedBy"}, - {"start_id": "root", "end_id": "domain", "kind": "RootCAFor"}, - {"start_id": "ca", "end_id": "unused-root", "kind": "EnterpriseCAFor"}, - {"start_id": "p2-mid", "end_id": "unused-template", "kind": "AllExtendedRights"} + {"start_id": "root", "end_id": "direct-mid", "kind": "Expand"}, + {"start_id": "direct-mid", "end_id": "suffix-head", "kind": "EnterSuffix"}, + {"start_id": "suffix-head", "end_id": "suffix-middle", "kind": "ContinueSuffix"}, + {"start_id": "suffix-middle", "end_id": "suffix-terminal", "kind": "CompleteSuffix"}, + {"start_id": "root", "end_id": "predicate-mid", "kind": "Expand"}, + {"start_id": "predicate-mid", "end_id": "predicate", "kind": "OptionA"}, + {"start_id": "predicate", "end_id": "suffix-head", "kind": "JoinSuffix"}, + {"start_id": "suffix-head", "end_id": "bridge", "kind": "HeadToBridge"}, + {"start_id": "bridge", "end_id": "suffix-terminal", "kind": "ReachTerminal"}, + {"start_id": "suffix-head", "end_id": "unused-bridge", "kind": "HeadToAlternateBridge"}, + {"start_id": "predicate-mid", "end_id": "unused-predicate", "kind": "OptionB"}, + {"start_id": "predicate-mid", "end_id": "unused-predicate", "kind": "OptionC"} ] }, "assert": { - "keys": ["p1", "p2"], + "keys": ["direct_path", "predicate_path"], "row_count": 1, "path_lengths": [4, 5], "path_node_ids": [ - ["n", "p1-mid", "ca", "store", "domain"], - ["n", "p2-mid", "template", "ca", "root", "domain"] + ["root", "direct-mid", "suffix-head", "suffix-middle", "suffix-terminal"], + ["root", "predicate-mid", "predicate", "suffix-head", "bridge", "suffix-terminal"] ], "path_edge_kinds": [ - ["MemberOf", "Enroll", "TrustedForNTAuth", "NTAuthStoreFor"], - ["MemberOf", "GenericAll", "PublishedTo", "IssuedSignedBy", "RootCAFor"] + ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], + ["Expand", "OptionA", "JoinSuffix", "HeadToBridge", "ReachTerminal"] ], - "contains_node_with_props": {"objectid": "S-1-5-21-2643190041-1319121918-239771340-513"}, - "contains_edge": {"start": "template", "end": "ca", "kind": "PublishedTo"} + "contains_node_with_props": {"root_key": "fixed-suffix-shared-endpoints-root"}, + "contains_edge": {"start": "predicate", "end": "suffix-head", "kind": "JoinSuffix"} } }, { - "name": "ADCS template predicate accepts both OR branches and rejects false alternatives", - "cypher": "MATCH (n:Group) WHERE n.objectid = 'optimizer-or-source' MATCH p = (n)-[:MemberOf*0..]->()-[:GenericAll|Enroll|AllExtendedRights]->(ct:CertTemplate)-[:PublishedTo]->(ca:EnterpriseCA)-[:IssuedSignedBy|EnterpriseCAFor*1..]->(:RootCA)-[:RootCAFor]->(d:Domain) WHERE ct.authenticationenabled = true AND ct.requiresmanagerapproval = false AND ct.enrolleesuppliessubject = true AND (ct.schemaversion = 1 OR ct.authorizedsignatures = 0) RETURN p", + "name": "fixed-suffix predicate accepts both OR branches and rejects false alternatives", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = 'fixed-suffix-predicate-root' MATCH predicate_path = (root)-[:Expand*0..16]->()-[:OptionA|OptionB|OptionC]->(predicate:PredicateNode)-[:JoinSuffix]->(head:SuffixHead)-[:HeadToBridge|HeadToAlternateBridge*1..16]->(:BridgeNode)-[:ReachTerminal]->(terminal:SuffixTerminal) WHERE predicate.eligible = true AND predicate.requires_review = false AND predicate.allows_direct = true AND (predicate.version = 1 OR predicate.required_approvals = 0) RETURN predicate_path", "fixture": { "nodes": [ - {"id": "n", "kinds": ["Group"], "properties": {"objectid": "optimizer-or-source"}}, - {"id": "mid-v1", "kinds": ["Group"]}, - {"id": "mid-sig", "kinds": ["Group"]}, - {"id": "mid-bad", "kinds": ["Group"]}, - {"id": "template-v1", "kinds": ["CertTemplate"], "properties": {"authenticationenabled": true, "requiresmanagerapproval": false, "enrolleesuppliessubject": true, "schemaversion": 1, "authorizedsignatures": 2}}, - {"id": "template-sig", "kinds": ["CertTemplate"], "properties": {"authenticationenabled": true, "requiresmanagerapproval": false, "enrolleesuppliessubject": true, "schemaversion": 2, "authorizedsignatures": 0}}, - {"id": "template-bad", "kinds": ["CertTemplate"], "properties": {"authenticationenabled": true, "requiresmanagerapproval": false, "enrolleesuppliessubject": true, "schemaversion": 2, "authorizedsignatures": 1}}, - {"id": "ca", "kinds": ["EnterpriseCA"]}, - {"id": "root", "kinds": ["RootCA"]}, - {"id": "unused-root", "kinds": ["RootCA"]}, - {"id": "domain", "kinds": ["Domain"]} + {"id": "root", "kinds": ["ExpansionRoot"], "properties": {"root_key": "fixed-suffix-predicate-root"}}, + {"id": "mid-version", "kinds": ["ExpansionNode"]}, + {"id": "mid-approval", "kinds": ["ExpansionNode"]}, + {"id": "mid-rejected", "kinds": ["ExpansionNode"]}, + {"id": "predicate-version", "kinds": ["PredicateNode"], "properties": {"eligible": true, "requires_review": false, "allows_direct": true, "version": 1, "required_approvals": 2}}, + {"id": "predicate-approval", "kinds": ["PredicateNode"], "properties": {"eligible": true, "requires_review": false, "allows_direct": true, "version": 2, "required_approvals": 0}}, + {"id": "predicate-rejected", "kinds": ["PredicateNode"], "properties": {"eligible": true, "requires_review": false, "allows_direct": true, "version": 2, "required_approvals": 1}}, + {"id": "suffix-head", "kinds": ["SuffixHead"]}, + {"id": "bridge", "kinds": ["BridgeNode"]}, + {"id": "unused-bridge", "kinds": ["BridgeNode"]}, + {"id": "suffix-terminal", "kinds": ["SuffixTerminal"]} ], "edges": [ - {"start_id": "n", "end_id": "mid-v1", "kind": "MemberOf"}, - {"start_id": "mid-v1", "end_id": "template-v1", "kind": "GenericAll"}, - {"start_id": "n", "end_id": "mid-sig", "kind": "MemberOf"}, - {"start_id": "mid-sig", "end_id": "template-sig", "kind": "Enroll"}, - {"start_id": "n", "end_id": "mid-bad", "kind": "MemberOf"}, - {"start_id": "mid-bad", "end_id": "template-bad", "kind": "AllExtendedRights"}, - {"start_id": "template-v1", "end_id": "ca", "kind": "PublishedTo"}, - {"start_id": "template-sig", "end_id": "ca", "kind": "PublishedTo"}, - {"start_id": "template-bad", "end_id": "ca", "kind": "PublishedTo"}, - {"start_id": "ca", "end_id": "root", "kind": "IssuedSignedBy"}, - {"start_id": "ca", "end_id": "unused-root", "kind": "EnterpriseCAFor"}, - {"start_id": "root", "end_id": "domain", "kind": "RootCAFor"} + {"start_id": "root", "end_id": "mid-version", "kind": "Expand"}, + {"start_id": "mid-version", "end_id": "predicate-version", "kind": "OptionA"}, + {"start_id": "root", "end_id": "mid-approval", "kind": "Expand"}, + {"start_id": "mid-approval", "end_id": "predicate-approval", "kind": "OptionB"}, + {"start_id": "root", "end_id": "mid-rejected", "kind": "Expand"}, + {"start_id": "mid-rejected", "end_id": "predicate-rejected", "kind": "OptionC"}, + {"start_id": "predicate-version", "end_id": "suffix-head", "kind": "JoinSuffix"}, + {"start_id": "predicate-approval", "end_id": "suffix-head", "kind": "JoinSuffix"}, + {"start_id": "predicate-rejected", "end_id": "suffix-head", "kind": "JoinSuffix"}, + {"start_id": "suffix-head", "end_id": "bridge", "kind": "HeadToBridge"}, + {"start_id": "suffix-head", "end_id": "unused-bridge", "kind": "HeadToAlternateBridge"}, + {"start_id": "bridge", "end_id": "suffix-terminal", "kind": "ReachTerminal"} ] }, "assert": { "row_count": 2, "path_node_ids": [ - ["n", "mid-v1", "template-v1", "ca", "root", "domain"], - ["n", "mid-sig", "template-sig", "ca", "root", "domain"] + ["root", "mid-version", "predicate-version", "suffix-head", "bridge", "suffix-terminal"], + ["root", "mid-approval", "predicate-approval", "suffix-head", "bridge", "suffix-terminal"] ], "path_edge_kinds": [ - ["MemberOf", "GenericAll", "PublishedTo", "IssuedSignedBy", "RootCAFor"], - ["MemberOf", "Enroll", "PublishedTo", "IssuedSignedBy", "RootCAFor"] + ["Expand", "OptionA", "JoinSuffix", "HeadToBridge", "ReachTerminal"], + ["Expand", "OptionB", "JoinSuffix", "HeadToBridge", "ReachTerminal"] ] } }, { - "name": "ADCS fanout returns every p1 and p2 path pair without endpoint collapse", - "cypher": "MATCH (n:Group) WHERE n.objectid = 'optimizer-fanout-source' MATCH p1 = (n)-[:MemberOf*0..]->()-[:Enroll]->(ca:EnterpriseCA)-[:TrustedForNTAuth]->(:NTAuthStore)-[:NTAuthStoreFor]->(d:Domain) MATCH p2 = (n)-[:MemberOf*0..]->()-[:GenericAll|Enroll|AllExtendedRights]->(ct:CertTemplate)-[:PublishedTo]->(ca)-[:IssuedSignedBy|EnterpriseCAFor*1..]->(:RootCA)-[:RootCAFor]->(d) WHERE ct.authenticationenabled = true AND ct.requiresmanagerapproval = false AND ct.enrolleesuppliessubject = true AND (ct.schemaversion = 1 OR ct.authorizedsignatures = 0) RETURN p1, p2", + "name": "fixed-suffix fanout returns every direct and predicate path pair without endpoint collapse", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = 'fixed-suffix-fanout-root' MATCH direct_path = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) MATCH predicate_path = (root)-[:Expand*0..16]->()-[:OptionA|OptionB|OptionC]->(predicate:PredicateNode)-[:JoinSuffix]->(head)-[:HeadToBridge|HeadToAlternateBridge*1..16]->(:BridgeNode)-[:ReachTerminal]->(terminal) WHERE predicate.eligible = true AND predicate.requires_review = false AND predicate.allows_direct = true AND (predicate.version = 1 OR predicate.required_approvals = 0) RETURN direct_path, predicate_path", "fixture": { "nodes": [ - {"id": "n", "kinds": ["Group"], "properties": {"objectid": "optimizer-fanout-source"}}, - {"id": "p1-a", "kinds": ["Group"]}, - {"id": "p1-b", "kinds": ["Group"]}, - {"id": "p2-a", "kinds": ["Group"]}, - {"id": "p2-b", "kinds": ["Group"]}, - {"id": "template-a", "kinds": ["CertTemplate"], "properties": {"authenticationenabled": true, "requiresmanagerapproval": false, "enrolleesuppliessubject": true, "schemaversion": 1, "authorizedsignatures": 1}}, - {"id": "template-b", "kinds": ["CertTemplate"], "properties": {"authenticationenabled": true, "requiresmanagerapproval": false, "enrolleesuppliessubject": true, "schemaversion": 2, "authorizedsignatures": 0}}, - {"id": "ca", "kinds": ["EnterpriseCA"]}, - {"id": "store", "kinds": ["NTAuthStore"]}, - {"id": "domain", "kinds": ["Domain"]}, - {"id": "root", "kinds": ["RootCA"]}, - {"id": "unused-root", "kinds": ["RootCA"]} + {"id": "root", "kinds": ["ExpansionRoot"], "properties": {"root_key": "fixed-suffix-fanout-root"}}, + {"id": "direct-mid-a", "kinds": ["ExpansionNode"]}, + {"id": "direct-mid-b", "kinds": ["ExpansionNode"]}, + {"id": "predicate-mid-a", "kinds": ["ExpansionNode"]}, + {"id": "predicate-mid-b", "kinds": ["ExpansionNode"]}, + {"id": "predicate-a", "kinds": ["PredicateNode"], "properties": {"eligible": true, "requires_review": false, "allows_direct": true, "version": 1, "required_approvals": 1}}, + {"id": "predicate-b", "kinds": ["PredicateNode"], "properties": {"eligible": true, "requires_review": false, "allows_direct": true, "version": 2, "required_approvals": 0}}, + {"id": "predicate-unused", "kinds": ["PredicateNode"], "properties": {"eligible": false, "requires_review": true, "allows_direct": false, "version": 2, "required_approvals": 1}}, + {"id": "suffix-head", "kinds": ["SuffixHead"]}, + {"id": "suffix-middle", "kinds": ["SuffixMiddle"]}, + {"id": "suffix-terminal", "kinds": ["SuffixTerminal"]}, + {"id": "bridge", "kinds": ["BridgeNode"]}, + {"id": "unused-bridge", "kinds": ["BridgeNode"]} ], "edges": [ - {"start_id": "n", "end_id": "p1-a", "kind": "MemberOf"}, - {"start_id": "p1-a", "end_id": "ca", "kind": "Enroll"}, - {"start_id": "n", "end_id": "p1-b", "kind": "MemberOf"}, - {"start_id": "p1-b", "end_id": "ca", "kind": "Enroll"}, - {"start_id": "ca", "end_id": "store", "kind": "TrustedForNTAuth"}, - {"start_id": "store", "end_id": "domain", "kind": "NTAuthStoreFor"}, - {"start_id": "n", "end_id": "p2-a", "kind": "MemberOf"}, - {"start_id": "p2-a", "end_id": "template-a", "kind": "GenericAll"}, - {"start_id": "n", "end_id": "p2-b", "kind": "MemberOf"}, - {"start_id": "p2-b", "end_id": "template-b", "kind": "AllExtendedRights"}, - {"start_id": "template-a", "end_id": "ca", "kind": "PublishedTo"}, - {"start_id": "template-b", "end_id": "ca", "kind": "PublishedTo"}, - {"start_id": "ca", "end_id": "root", "kind": "IssuedSignedBy"}, - {"start_id": "ca", "end_id": "unused-root", "kind": "EnterpriseCAFor"}, - {"start_id": "root", "end_id": "domain", "kind": "RootCAFor"} + {"start_id": "root", "end_id": "direct-mid-a", "kind": "Expand"}, + {"start_id": "direct-mid-a", "end_id": "suffix-head", "kind": "EnterSuffix"}, + {"start_id": "root", "end_id": "direct-mid-b", "kind": "Expand"}, + {"start_id": "direct-mid-b", "end_id": "suffix-head", "kind": "EnterSuffix"}, + {"start_id": "suffix-head", "end_id": "suffix-middle", "kind": "ContinueSuffix"}, + {"start_id": "suffix-middle", "end_id": "suffix-terminal", "kind": "CompleteSuffix"}, + {"start_id": "root", "end_id": "predicate-mid-a", "kind": "Expand"}, + {"start_id": "predicate-mid-a", "end_id": "predicate-a", "kind": "OptionA"}, + {"start_id": "root", "end_id": "predicate-mid-b", "kind": "Expand"}, + {"start_id": "predicate-mid-b", "end_id": "predicate-b", "kind": "OptionC"}, + {"start_id": "predicate-mid-a", "end_id": "predicate-unused", "kind": "OptionB"}, + {"start_id": "predicate-a", "end_id": "suffix-head", "kind": "JoinSuffix"}, + {"start_id": "predicate-b", "end_id": "suffix-head", "kind": "JoinSuffix"}, + {"start_id": "suffix-head", "end_id": "bridge", "kind": "HeadToBridge"}, + {"start_id": "suffix-head", "end_id": "unused-bridge", "kind": "HeadToAlternateBridge"}, + {"start_id": "bridge", "end_id": "suffix-terminal", "kind": "ReachTerminal"} ] }, "assert": { "row_count": 4, "path_node_ids": [ - ["n", "p1-a", "ca", "store", "domain"], - ["n", "p1-a", "ca", "store", "domain"], - ["n", "p1-b", "ca", "store", "domain"], - ["n", "p1-b", "ca", "store", "domain"], - ["n", "p2-a", "template-a", "ca", "root", "domain"], - ["n", "p2-a", "template-a", "ca", "root", "domain"], - ["n", "p2-b", "template-b", "ca", "root", "domain"], - ["n", "p2-b", "template-b", "ca", "root", "domain"] + ["root", "direct-mid-a", "suffix-head", "suffix-middle", "suffix-terminal"], + ["root", "direct-mid-a", "suffix-head", "suffix-middle", "suffix-terminal"], + ["root", "direct-mid-b", "suffix-head", "suffix-middle", "suffix-terminal"], + ["root", "direct-mid-b", "suffix-head", "suffix-middle", "suffix-terminal"], + ["root", "predicate-mid-a", "predicate-a", "suffix-head", "bridge", "suffix-terminal"], + ["root", "predicate-mid-a", "predicate-a", "suffix-head", "bridge", "suffix-terminal"], + ["root", "predicate-mid-b", "predicate-b", "suffix-head", "bridge", "suffix-terminal"], + ["root", "predicate-mid-b", "predicate-b", "suffix-head", "bridge", "suffix-terminal"] ], "path_edge_kinds": [ - ["MemberOf", "Enroll", "TrustedForNTAuth", "NTAuthStoreFor"], - ["MemberOf", "Enroll", "TrustedForNTAuth", "NTAuthStoreFor"], - ["MemberOf", "Enroll", "TrustedForNTAuth", "NTAuthStoreFor"], - ["MemberOf", "Enroll", "TrustedForNTAuth", "NTAuthStoreFor"], - ["MemberOf", "GenericAll", "PublishedTo", "IssuedSignedBy", "RootCAFor"], - ["MemberOf", "GenericAll", "PublishedTo", "IssuedSignedBy", "RootCAFor"], - ["MemberOf", "AllExtendedRights", "PublishedTo", "IssuedSignedBy", "RootCAFor"], - ["MemberOf", "AllExtendedRights", "PublishedTo", "IssuedSignedBy", "RootCAFor"] + ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], + ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], + ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], + ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], + ["Expand", "OptionA", "JoinSuffix", "HeadToBridge", "ReachTerminal"], + ["Expand", "OptionA", "JoinSuffix", "HeadToBridge", "ReachTerminal"], + ["Expand", "OptionC", "JoinSuffix", "HeadToBridge", "ReachTerminal"], + ["Expand", "OptionC", "JoinSuffix", "HeadToBridge", "ReachTerminal"] ] } }, { - "name": "ADCS fanout endpoint projection preserves row multiplicity", - "cypher": "MATCH (n:Group) WHERE n.objectid = 'optimizer-endpoint-fanout-source' MATCH p1 = (n)-[:MemberOf*0..]->()-[:Enroll]->(ca:EnterpriseCA)-[:TrustedForNTAuth]->(:NTAuthStore)-[:NTAuthStoreFor]->(d:Domain) MATCH p2 = (n)-[:MemberOf*0..]->()-[:GenericAll|Enroll|AllExtendedRights]->(ct:CertTemplate)-[:PublishedTo]->(ca)-[:IssuedSignedBy|EnterpriseCAFor*1..]->(:RootCA)-[:RootCAFor]->(d) WHERE ct.authenticationenabled = true AND ct.requiresmanagerapproval = false AND ct.enrolleesuppliessubject = true AND (ct.schemaversion = 1 OR ct.authorizedsignatures = 0) RETURN count(*) AS rows, count(distinct id(ca)) AS ca_count, count(distinct id(d)) AS domain_count, count(distinct id(ct)) AS template_count", + "name": "fixed-suffix fanout endpoint projection preserves row multiplicity", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = 'fixed-suffix-endpoint-fanout-root' MATCH direct_path = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) MATCH predicate_path = (root)-[:Expand*0..16]->()-[:OptionA|OptionB|OptionC]->(predicate:PredicateNode)-[:JoinSuffix]->(head)-[:HeadToBridge|HeadToAlternateBridge*1..16]->(:BridgeNode)-[:ReachTerminal]->(terminal) WHERE predicate.eligible = true AND predicate.requires_review = false AND predicate.allows_direct = true AND (predicate.version = 1 OR predicate.required_approvals = 0) RETURN count(*) AS rows, count(distinct id(head)) AS head_count, count(distinct id(terminal)) AS terminal_count, count(distinct id(predicate)) AS predicate_count", "fixture": { "nodes": [ - {"id": "n", "kinds": ["Group"], "properties": {"objectid": "optimizer-endpoint-fanout-source"}}, - {"id": "p1-a", "kinds": ["Group"]}, - {"id": "p1-b", "kinds": ["Group"]}, - {"id": "p2-a", "kinds": ["Group"]}, - {"id": "p2-b", "kinds": ["Group"]}, - {"id": "template-a", "kinds": ["CertTemplate"], "properties": {"authenticationenabled": true, "requiresmanagerapproval": false, "enrolleesuppliessubject": true, "schemaversion": 1, "authorizedsignatures": 1}}, - {"id": "template-b", "kinds": ["CertTemplate"], "properties": {"authenticationenabled": true, "requiresmanagerapproval": false, "enrolleesuppliessubject": true, "schemaversion": 2, "authorizedsignatures": 0}}, - {"id": "ca", "kinds": ["EnterpriseCA"]}, - {"id": "store", "kinds": ["NTAuthStore"]}, - {"id": "domain", "kinds": ["Domain"]}, - {"id": "root", "kinds": ["RootCA"]}, - {"id": "unused-root", "kinds": ["RootCA"]} + {"id": "root", "kinds": ["ExpansionRoot"], "properties": {"root_key": "fixed-suffix-endpoint-fanout-root"}}, + {"id": "direct-mid-a", "kinds": ["ExpansionNode"]}, + {"id": "direct-mid-b", "kinds": ["ExpansionNode"]}, + {"id": "predicate-mid-a", "kinds": ["ExpansionNode"]}, + {"id": "predicate-mid-b", "kinds": ["ExpansionNode"]}, + {"id": "predicate-a", "kinds": ["PredicateNode"], "properties": {"eligible": true, "requires_review": false, "allows_direct": true, "version": 1, "required_approvals": 1}}, + {"id": "predicate-b", "kinds": ["PredicateNode"], "properties": {"eligible": true, "requires_review": false, "allows_direct": true, "version": 2, "required_approvals": 0}}, + {"id": "predicate-unused", "kinds": ["PredicateNode"], "properties": {"eligible": false, "requires_review": true, "allows_direct": false, "version": 2, "required_approvals": 1}}, + {"id": "suffix-head", "kinds": ["SuffixHead"]}, + {"id": "suffix-middle", "kinds": ["SuffixMiddle"]}, + {"id": "suffix-terminal", "kinds": ["SuffixTerminal"]}, + {"id": "bridge", "kinds": ["BridgeNode"]}, + {"id": "unused-bridge", "kinds": ["BridgeNode"]} ], "edges": [ - {"start_id": "n", "end_id": "p1-a", "kind": "MemberOf"}, - {"start_id": "p1-a", "end_id": "ca", "kind": "Enroll"}, - {"start_id": "n", "end_id": "p1-b", "kind": "MemberOf"}, - {"start_id": "p1-b", "end_id": "ca", "kind": "Enroll"}, - {"start_id": "ca", "end_id": "store", "kind": "TrustedForNTAuth"}, - {"start_id": "store", "end_id": "domain", "kind": "NTAuthStoreFor"}, - {"start_id": "n", "end_id": "p2-a", "kind": "MemberOf"}, - {"start_id": "p2-a", "end_id": "template-a", "kind": "GenericAll"}, - {"start_id": "n", "end_id": "p2-b", "kind": "MemberOf"}, - {"start_id": "p2-b", "end_id": "template-b", "kind": "AllExtendedRights"}, - {"start_id": "template-a", "end_id": "ca", "kind": "PublishedTo"}, - {"start_id": "template-b", "end_id": "ca", "kind": "PublishedTo"}, - {"start_id": "ca", "end_id": "root", "kind": "IssuedSignedBy"}, - {"start_id": "ca", "end_id": "unused-root", "kind": "EnterpriseCAFor"}, - {"start_id": "root", "end_id": "domain", "kind": "RootCAFor"} + {"start_id": "root", "end_id": "direct-mid-a", "kind": "Expand"}, + {"start_id": "direct-mid-a", "end_id": "suffix-head", "kind": "EnterSuffix"}, + {"start_id": "root", "end_id": "direct-mid-b", "kind": "Expand"}, + {"start_id": "direct-mid-b", "end_id": "suffix-head", "kind": "EnterSuffix"}, + {"start_id": "suffix-head", "end_id": "suffix-middle", "kind": "ContinueSuffix"}, + {"start_id": "suffix-middle", "end_id": "suffix-terminal", "kind": "CompleteSuffix"}, + {"start_id": "root", "end_id": "predicate-mid-a", "kind": "Expand"}, + {"start_id": "predicate-mid-a", "end_id": "predicate-a", "kind": "OptionA"}, + {"start_id": "root", "end_id": "predicate-mid-b", "kind": "Expand"}, + {"start_id": "predicate-mid-b", "end_id": "predicate-b", "kind": "OptionC"}, + {"start_id": "predicate-mid-a", "end_id": "predicate-unused", "kind": "OptionB"}, + {"start_id": "predicate-a", "end_id": "suffix-head", "kind": "JoinSuffix"}, + {"start_id": "predicate-b", "end_id": "suffix-head", "kind": "JoinSuffix"}, + {"start_id": "suffix-head", "end_id": "bridge", "kind": "HeadToBridge"}, + {"start_id": "suffix-head", "end_id": "unused-bridge", "kind": "HeadToAlternateBridge"}, + {"start_id": "bridge", "end_id": "suffix-terminal", "kind": "ReachTerminal"} ] }, "assert": {"row_values": [[4, 1, 1, 2]]} diff --git a/integration/testdata/cases/shortest_bound.json b/integration/testdata/cases/shortest_bound.json new file mode 100644 index 00000000..199021df --- /dev/null +++ b/integration/testdata/cases/shortest_bound.json @@ -0,0 +1,82 @@ +{ + "dataset": "shortest_bound", + "cases": [ + { + "name": "bound pair shortest prefers direct edge and hydrates in order", + "cypher": "MATCH p = shortestPath((s)-[*1..]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p LIMIT 1", + "node_params": {"start_id": "start", "end_id": "direct"}, + "assert": { + "row_count": 1, + "path_node_ids": [["start", "direct"]], + "path_edge_kinds": [["BoundEdge"]], + "contains_edge": {"start": "start", "end": "direct", "kind": "BoundEdge", "props": {"route": "direct"}} + } + }, + { + "name": "bound pair shortest disconnected endpoints return empty", + "cypher": "MATCH p = shortestPath((s)-[*1..]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p LIMIT 1", + "node_params": {"start_id": "start", "end_id": "disconnected"}, + "assert": "empty" + }, + { + "name": "bound pair shortest respects direction", + "cypher": "MATCH p = shortestPath((s)-[*1..]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p LIMIT 1", + "node_params": {"start_id": "start", "end_id": "wrong-direction"}, + "assert": "empty" + }, + { + "name": "bound pair shortest respects relationship kind", + "cypher": "MATCH p = shortestPath((s)-[:BoundEdge*1..]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p LIMIT 1", + "node_params": {"start_id": "start", "end_id": "typed-end"}, + "assert": "empty" + }, + { + "name": "bound pair shortest respects maximum depth", + "cypher": "MATCH p = shortestPath((s)-[:BoundEdge*1..2]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p LIMIT 1", + "node_params": {"start_id": "start", "end_id": "cycle-end"}, + "assert": "empty" + }, + { + "name": "bound pair shortest handles cycles without relationship reuse", + "cypher": "MATCH p = shortestPath((s)-[:BoundEdge*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p LIMIT 1", + "node_params": {"start_id": "start", "end_id": "cycle-end"}, + "assert": { + "row_count": 1, + "path_node_ids": [["start", "cycle-a", "cycle-b", "cycle-end"]], + "path_edge_kinds": [["BoundEdge", "BoundEdge", "BoundEdge"]] + } + }, + { + "name": "bound pair shortest missing endpoint id returns empty", + "cypher": "MATCH p = shortestPath((s)-[*1..]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p LIMIT 1", + "params": {"start_id": -1}, + "node_params": {"end_id": "direct"}, + "assert": "empty" + }, + { + "name": "bound pair shortest null endpoint parameter returns empty", + "cypher": "MATCH p = shortestPath((s)-[*1..]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p LIMIT 1", + "params": {"start_id": null}, + "node_params": {"end_id": "direct"}, + "assert": "empty" + }, + { + "name": "bound pair shortest same endpoint keeps error contract", + "cypher": "MATCH p = shortestPath((s)-[*1..]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p LIMIT 1", + "node_params": {"start_id": "start", "end_id": "start"}, + "assert": "query_error" + }, + { + "name": "bound pair zero depth returns the same endpoint", + "cypher": "MATCH p = shortestPath((s)-[*0..0]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p LIMIT 1", + "node_params": {"start_id": "start", "end_id": "start"}, + "assert": {"row_count": 1, "path_node_ids": [["start"]], "path_edge_kinds": [[]]} + }, + { + "name": "bound pair unbounded zero minimum returns the same endpoint", + "cypher": "MATCH p = shortestPath((s)-[*0..]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p LIMIT 1", + "node_params": {"start_id": "start", "end_id": "start"}, + "assert": {"row_count": 1, "path_node_ids": [["start"]], "path_edge_kinds": [[]]} + } + ] +} diff --git a/integration/testdata/expand_into.json b/integration/testdata/expand_into.json new file mode 100644 index 00000000..4eec0f82 --- /dev/null +++ b/integration/testdata/expand_into.json @@ -0,0 +1,39 @@ +{ + "graph": { + "nodes": [ + {"id": "pair-source", "kinds": ["ExpandIntoSource"], "properties": {"name": "source"}}, + {"id": "pair-target", "kinds": ["ExpandIntoTarget"], "properties": {"name": "target"}}, + {"id": "pair-missing", "kinds": ["ExpandIntoSource"], "properties": {"name": "missing"}}, + {"id": "pair-loop", "kinds": ["ExpandIntoSource", "ExpandIntoTarget"], "properties": {"name": "loop"}}, + {"id": "low-source", "kinds": ["ExpandIntoSource"], "properties": {"name": "low-source"}}, + {"id": "high-target", "kinds": ["ExpandIntoTarget"], "properties": {"name": "high-target"}}, + {"id": "high-source", "kinds": ["ExpandIntoSource"], "properties": {"name": "high-source"}}, + {"id": "low-target", "kinds": ["ExpandIntoTarget"], "properties": {"name": "low-target"}}, + {"id": "source-decoy-1", "kinds": ["ExpandIntoSource"], "properties": {"name": "source-decoy-1"}}, + {"id": "source-decoy-2", "kinds": ["ExpandIntoSource"], "properties": {"name": "source-decoy-2"}}, + {"id": "source-decoy-3", "kinds": ["ExpandIntoSource"], "properties": {"name": "source-decoy-3"}}, + {"id": "source-decoy-4", "kinds": ["ExpandIntoSource"], "properties": {"name": "source-decoy-4"}}, + {"id": "target-decoy-1", "kinds": ["ExpandIntoTarget"], "properties": {"name": "target-decoy-1"}}, + {"id": "target-decoy-2", "kinds": ["ExpandIntoTarget"], "properties": {"name": "target-decoy-2"}}, + {"id": "target-decoy-3", "kinds": ["ExpandIntoTarget"], "properties": {"name": "target-decoy-3"}}, + {"id": "target-decoy-4", "kinds": ["ExpandIntoTarget"], "properties": {"name": "target-decoy-4"}} + ], + "edges": [ + {"start_id": "pair-source", "end_id": "pair-target", "kind": "ExpandIntoKindA", "properties": {"slot": "a"}}, + {"start_id": "pair-source", "end_id": "pair-target", "kind": "ExpandIntoKindB", "properties": {"slot": "b"}}, + {"start_id": "pair-loop", "end_id": "pair-loop", "kind": "ExpandIntoKindA", "properties": {"slot": "loop"}}, + {"start_id": "pair-source", "end_id": "pair-loop", "kind": "ExpandIntoDecoy", "properties": {"slot": "decoy-out"}}, + {"start_id": "pair-loop", "end_id": "pair-target", "kind": "ExpandIntoDecoy", "properties": {"slot": "decoy-in"}}, + {"start_id": "low-source", "end_id": "high-target", "kind": "ExpandIntoKindA", "properties": {"slot": "low-source-match"}}, + {"start_id": "source-decoy-1", "end_id": "high-target", "kind": "ExpandIntoKindA", "properties": {"slot": "high-target-1"}}, + {"start_id": "source-decoy-2", "end_id": "high-target", "kind": "ExpandIntoKindA", "properties": {"slot": "high-target-2"}}, + {"start_id": "source-decoy-3", "end_id": "high-target", "kind": "ExpandIntoKindA", "properties": {"slot": "high-target-3"}}, + {"start_id": "source-decoy-4", "end_id": "high-target", "kind": "ExpandIntoKindA", "properties": {"slot": "high-target-4"}}, + {"start_id": "high-source", "end_id": "low-target", "kind": "ExpandIntoKindA", "properties": {"slot": "low-target-match"}}, + {"start_id": "high-source", "end_id": "target-decoy-1", "kind": "ExpandIntoKindA", "properties": {"slot": "high-source-1"}}, + {"start_id": "high-source", "end_id": "target-decoy-2", "kind": "ExpandIntoKindA", "properties": {"slot": "high-source-2"}}, + {"start_id": "high-source", "end_id": "target-decoy-3", "kind": "ExpandIntoKindA", "properties": {"slot": "high-source-3"}}, + {"start_id": "high-source", "end_id": "target-decoy-4", "kind": "ExpandIntoKindA", "properties": {"slot": "high-source-4"}} + ] + } +} diff --git a/integration/testdata/fixed_suffix_expansion_adversarial.json b/integration/testdata/fixed_suffix_expansion_adversarial.json new file mode 100644 index 00000000..7aec947c --- /dev/null +++ b/integration/testdata/fixed_suffix_expansion_adversarial.json @@ -0,0 +1,73 @@ +{ + "graph": { + "nodes": [ + {"id":"boundary-terminal","kinds":["SuffixTerminal"]}, + {"id":"boundary-root","kinds":["ExpansionRoot"],"properties":{"root_key":"suffix-overflow-adversarial-root"}}, + {"id":"boundary-boundary-9001","kinds":["ExpansionNode"]}, + {"id":"boundary-head-a","kinds":["SuffixHead"]}, + {"id":"boundary-middle-a","kinds":["SuffixMiddle"]}, + {"id":"boundary-head-b","kinds":["SuffixHead"]}, + {"id":"boundary-middle-b","kinds":["SuffixMiddle"]}, + {"id":"boundary-lane-0001","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0002","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0003","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0004","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0005","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0006","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0007","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0008","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0009","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0010","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0011","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0012","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0013","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0014","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0015","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0016","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0017","kinds":["ExpansionNode"]} + ], + "edges": [ + {"start_id":"boundary-root","end_id":"boundary-lane-0001","kind":"Expand","properties":{"ordinal":1}}, + {"start_id":"boundary-lane-0001","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":101}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0002","kind":"Expand","properties":{"ordinal":2}}, + {"start_id":"boundary-lane-0002","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":102}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0003","kind":"Expand","properties":{"ordinal":3}}, + {"start_id":"boundary-lane-0003","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":103}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0004","kind":"Expand","properties":{"ordinal":4}}, + {"start_id":"boundary-lane-0004","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":104}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0005","kind":"Expand","properties":{"ordinal":5}}, + {"start_id":"boundary-lane-0005","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":105}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0006","kind":"Expand","properties":{"ordinal":6}}, + {"start_id":"boundary-lane-0006","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":106}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0007","kind":"Expand","properties":{"ordinal":7}}, + {"start_id":"boundary-lane-0007","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":107}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0008","kind":"Expand","properties":{"ordinal":8}}, + {"start_id":"boundary-lane-0008","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":108}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0009","kind":"Expand","properties":{"ordinal":9}}, + {"start_id":"boundary-lane-0009","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":109}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0010","kind":"Expand","properties":{"ordinal":10}}, + {"start_id":"boundary-lane-0010","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":110}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0011","kind":"Expand","properties":{"ordinal":11}}, + {"start_id":"boundary-lane-0011","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":111}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0012","kind":"Expand","properties":{"ordinal":12}}, + {"start_id":"boundary-lane-0012","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":112}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0013","kind":"Expand","properties":{"ordinal":13}}, + {"start_id":"boundary-lane-0013","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":113}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0014","kind":"Expand","properties":{"ordinal":14}}, + {"start_id":"boundary-lane-0014","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":114}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0015","kind":"Expand","properties":{"ordinal":15}}, + {"start_id":"boundary-lane-0015","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":115}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0016","kind":"Expand","properties":{"ordinal":16}}, + {"start_id":"boundary-lane-0016","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":116}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0017","kind":"Expand","properties":{"ordinal":17}}, + {"start_id":"boundary-lane-0017","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":117}}, + {"start_id":"boundary-boundary-9001","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":201}}, + {"start_id":"boundary-boundary-9001","end_id":"boundary-head-a","kind":"EnterSuffix","properties":{"ordinal":301}}, + {"start_id":"boundary-head-a","end_id":"boundary-middle-a","kind":"ContinueSuffix","properties":{"ordinal":302}}, + {"start_id":"boundary-middle-a","end_id":"boundary-terminal","kind":"CompleteSuffix","properties":{"ordinal":303}}, + {"start_id":"boundary-boundary-9001","end_id":"boundary-head-b","kind":"EnterSuffix","properties":{"ordinal":401}}, + {"start_id":"boundary-head-b","end_id":"boundary-middle-b","kind":"ContinueSuffix","properties":{"ordinal":402}}, + {"start_id":"boundary-middle-b","end_id":"boundary-terminal","kind":"CompleteSuffix","properties":{"ordinal":403}} + ] + } +} diff --git a/integration/testdata/fixed_suffix_expansion_fanout.json b/integration/testdata/fixed_suffix_expansion_fanout.json new file mode 100644 index 00000000..6f93a829 --- /dev/null +++ b/integration/testdata/fixed_suffix_expansion_fanout.json @@ -0,0 +1,50 @@ +{ + "graph": { + "nodes": [ + {"id": "fse-root", "kinds": ["ExpansionRoot"], "properties": {"root_key": "fixed-suffix-fanout-root"}}, + {"id": "fse-expansion-a", "kinds": ["ExpansionNode"]}, + {"id": "fse-expansion-b", "kinds": ["ExpansionNode"]}, + {"id": "fse-expansion-c", "kinds": ["ExpansionNode"]}, + {"id": "fse-option-good", "kinds": ["ExpansionNode"]}, + {"id": "fse-option-disabled", "kinds": ["ExpansionNode"]}, + {"id": "fse-option-wrong-head", "kinds": ["ExpansionNode"]}, + {"id": "fse-head", "kinds": ["SuffixHead"]}, + {"id": "fse-other-head", "kinds": ["SuffixHead"]}, + {"id": "fse-middle", "kinds": ["SuffixMiddle"]}, + {"id": "fse-terminal", "kinds": ["SuffixTerminal"]}, + {"id": "fse-other-terminal", "kinds": ["SuffixTerminal"]}, + {"id": "fse-predicate-good", "kinds": ["PredicateNode"], "properties": {"eligible": true, "requires_review": false, "allows_direct": true, "version": 1, "required_approvals": 1}}, + {"id": "fse-predicate-alt", "kinds": ["PredicateNode"], "properties": {"eligible": true, "requires_review": false, "allows_direct": true, "version": 2, "required_approvals": 0}}, + {"id": "fse-predicate-disabled", "kinds": ["PredicateNode"], "properties": {"eligible": false, "requires_review": true, "allows_direct": false, "version": 2, "required_approvals": 1}}, + {"id": "fse-predicate-wrong-head", "kinds": ["PredicateNode"], "properties": {"eligible": true, "requires_review": false, "allows_direct": true, "version": 1, "required_approvals": 1}}, + {"id": "fse-bridge", "kinds": ["BridgeNode"]}, + {"id": "fse-other-bridge", "kinds": ["BridgeNode"]} + ], + "edges": [ + {"start_id": "fse-root", "end_id": "fse-expansion-a", "kind": "Expand"}, + {"start_id": "fse-root", "end_id": "fse-expansion-b", "kind": "Expand"}, + {"start_id": "fse-expansion-b", "end_id": "fse-expansion-c", "kind": "Expand"}, + {"start_id": "fse-root", "end_id": "fse-option-good", "kind": "Expand"}, + {"start_id": "fse-root", "end_id": "fse-option-disabled", "kind": "Expand"}, + {"start_id": "fse-root", "end_id": "fse-option-wrong-head", "kind": "Expand"}, + {"start_id": "fse-root", "end_id": "fse-head", "kind": "EnterSuffix"}, + {"start_id": "fse-expansion-a", "end_id": "fse-head", "kind": "EnterSuffix"}, + {"start_id": "fse-expansion-b", "end_id": "fse-head", "kind": "EnterSuffix"}, + {"start_id": "fse-expansion-c", "end_id": "fse-head", "kind": "EnterSuffix"}, + {"start_id": "fse-head", "end_id": "fse-middle", "kind": "ContinueSuffix"}, + {"start_id": "fse-middle", "end_id": "fse-terminal", "kind": "CompleteSuffix"}, + {"start_id": "fse-option-good", "end_id": "fse-predicate-good", "kind": "OptionA"}, + {"start_id": "fse-option-good", "end_id": "fse-predicate-alt", "kind": "OptionB"}, + {"start_id": "fse-option-disabled", "end_id": "fse-predicate-disabled", "kind": "OptionC"}, + {"start_id": "fse-option-wrong-head", "end_id": "fse-predicate-wrong-head", "kind": "OptionA"}, + {"start_id": "fse-predicate-good", "end_id": "fse-head", "kind": "JoinSuffix"}, + {"start_id": "fse-predicate-alt", "end_id": "fse-head", "kind": "JoinSuffix"}, + {"start_id": "fse-predicate-disabled", "end_id": "fse-head", "kind": "JoinSuffix"}, + {"start_id": "fse-predicate-wrong-head", "end_id": "fse-other-head", "kind": "JoinSuffix"}, + {"start_id": "fse-head", "end_id": "fse-bridge", "kind": "HeadToBridge"}, + {"start_id": "fse-head", "end_id": "fse-other-bridge", "kind": "HeadToAlternateBridge"}, + {"start_id": "fse-bridge", "end_id": "fse-terminal", "kind": "ReachTerminal"}, + {"start_id": "fse-other-bridge", "end_id": "fse-other-terminal", "kind": "ReachTerminal"} + ] + } +} diff --git a/integration/testdata/shortest_bound.json b/integration/testdata/shortest_bound.json new file mode 100644 index 00000000..0ce18a91 --- /dev/null +++ b/integration/testdata/shortest_bound.json @@ -0,0 +1,33 @@ +{ + "graph": { + "nodes": [ + {"id": "start", "kinds": ["BoundNode"], "properties": {"name": "start"}}, + {"id": "direct", "kinds": ["BoundNode"], "properties": {"name": "direct"}}, + {"id": "long-mid", "kinds": ["BoundNode"]}, + {"id": "diamond-left", "kinds": ["BoundNode"]}, + {"id": "diamond-right", "kinds": ["BoundNode"]}, + {"id": "diamond-end", "kinds": ["BoundNode"]}, + {"id": "cycle-a", "kinds": ["BoundNode"]}, + {"id": "cycle-b", "kinds": ["BoundNode"]}, + {"id": "cycle-end", "kinds": ["BoundNode"]}, + {"id": "typed-end", "kinds": ["BoundNode"]}, + {"id": "disconnected", "kinds": ["BoundNode"]}, + {"id": "wrong-direction", "kinds": ["BoundNode"]} + ], + "edges": [ + {"start_id": "start", "end_id": "direct", "kind": "BoundEdge", "properties": {"route": "direct"}}, + {"start_id": "start", "end_id": "long-mid", "kind": "BoundEdge"}, + {"start_id": "long-mid", "end_id": "direct", "kind": "BoundEdge"}, + {"start_id": "start", "end_id": "diamond-left", "kind": "BoundEdge"}, + {"start_id": "start", "end_id": "diamond-right", "kind": "BoundEdge"}, + {"start_id": "diamond-left", "end_id": "diamond-end", "kind": "BoundEdge"}, + {"start_id": "diamond-right", "end_id": "diamond-end", "kind": "BoundEdge"}, + {"start_id": "start", "end_id": "cycle-a", "kind": "BoundEdge"}, + {"start_id": "cycle-a", "end_id": "cycle-b", "kind": "BoundEdge"}, + {"start_id": "cycle-b", "end_id": "cycle-a", "kind": "BoundEdge"}, + {"start_id": "cycle-b", "end_id": "cycle-end", "kind": "BoundEdge"}, + {"start_id": "start", "end_id": "typed-end", "kind": "OtherBoundEdge"}, + {"start_id": "wrong-direction", "end_id": "start", "kind": "BoundEdge"} + ] + } +} diff --git a/integration/testdata/templates/advanced_lookup_shapes.json b/integration/testdata/templates/advanced_lookup_shapes.json new file mode 100644 index 00000000..cdc137bc --- /dev/null +++ b/integration/testdata/templates/advanced_lookup_shapes.json @@ -0,0 +1,93 @@ +{ + "families": [ + { + "name": "LOOKUP-09 through LOOKUP-14 and LOOKUP-16 advanced lookups", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "hydrate-a", "kinds": ["Hydrate"], "properties": {"name": "hydrate-a", "value": 1}}, + {"id": "hydrate-b", "kinds": ["Hydrate"], "properties": {"name": "hydrate-b", "value": 2}}, + {"id": "hydrate-c", "kinds": ["Hydrate"], "properties": {"name": "hydrate-c", "value": 3}}, + {"id": "flags-mm", "kinds": ["User"], "properties": {"name": "flags-mm"}}, + {"id": "flags-mn", "kinds": ["User"], "properties": {"name": "flags-mn", "msa": null}}, + {"id": "flags-mf", "kinds": ["User"], "properties": {"name": "flags-mf", "msa": false}}, + {"id": "flags-mt", "kinds": ["User"], "properties": {"name": "flags-mt", "msa": true}}, + {"id": "flags-nm", "kinds": ["User"], "properties": {"name": "flags-nm", "gmsa": null}}, + {"id": "flags-nn", "kinds": ["User"], "properties": {"name": "flags-nn", "gmsa": null, "msa": null}}, + {"id": "flags-nf", "kinds": ["User"], "properties": {"name": "flags-nf", "gmsa": null, "msa": false}}, + {"id": "flags-nt", "kinds": ["User"], "properties": {"name": "flags-nt", "gmsa": null, "msa": true}}, + {"id": "flags-fm", "kinds": ["User"], "properties": {"name": "flags-fm", "gmsa": false}}, + {"id": "flags-fn", "kinds": ["User"], "properties": {"name": "flags-fn", "gmsa": false, "msa": null}}, + {"id": "flags-ff", "kinds": ["User"], "properties": {"name": "flags-ff", "gmsa": false, "msa": false}}, + {"id": "flags-ft", "kinds": ["User"], "properties": {"name": "flags-ft", "gmsa": false, "msa": true}}, + {"id": "flags-tm", "kinds": ["User"], "properties": {"name": "flags-tm", "gmsa": true}}, + {"id": "flags-tn", "kinds": ["User"], "properties": {"name": "flags-tn", "gmsa": true, "msa": null}}, + {"id": "flags-tf", "kinds": ["User"], "properties": {"name": "flags-tf", "gmsa": true, "msa": false}}, + {"id": "flags-tt", "kinds": ["User"], "properties": {"name": "flags-tt", "gmsa": true, "msa": true}}, + {"id": "tenant", "kinds": ["Tenant"], "properties": {"name": "tenant", "objectid": "tenant-1"}}, + {"id": "role-a", "kinds": ["AZRole"], "properties": {"name": "role-a", "roletemplateid": "role-a", "enabled": true, "state": "active"}}, + {"id": "role-b", "kinds": ["AZServicePrincipal"], "properties": {"name": "role-b", "roletemplateid": "role-b", "enabled": false, "state": "inactive"}}, + {"id": "role-multi", "kinds": ["AZRole", "AZServicePrincipal"], "properties": {"name": "role-multi", "roletemplateid": "role-multi", "enabled": true, "state": "active"}}, + {"id": "role-wrong-kind", "kinds": ["Other"], "properties": {"name": "role-wrong-kind", "roletemplateid": "role-a", "enabled": true, "state": "active"}}, + {"id": "edge-start", "kinds": ["Entity"], "properties": {"name": "edge-start"}}, + {"id": "edge-end", "kinds": ["Entity"], "properties": {"name": "edge-end"}}, + {"id": "local-good", "kinds": ["LocalGroup", "Entity"], "properties": {"name": "local-good", "objectid": "S-1-5-21-555"}}, + {"id": "local-good-2", "kinds": ["LocalGroup", "Entity"], "properties": {"name": "local-good-2", "objectid": "OTHER-555"}}, + {"id": "local-wrong-suffix", "kinds": ["LocalGroup", "Entity"], "properties": {"name": "local-wrong-suffix", "objectid": "S-1-5-21-556"}}, + {"id": "local-target", "kinds": ["Computer"], "properties": {"name": "local-target"}}, + {"id": "local-other-target", "kinds": ["Computer"], "properties": {"name": "local-other-target"}}, + {"id": "domain-missing", "kinds": ["Domain"], "properties": {"objectid": "domain-missing"}}, + {"id": "domain-alpha", "kinds": ["Domain"], "properties": {"name": "Alpha"}}, + {"id": "domain-beta-a", "kinds": ["Domain"], "properties": {"name": "Beta"}}, + {"id": "domain-beta-b", "kinds": ["Domain"], "properties": {"name": "Beta"}}, + {"id": "domain-multi", "kinds": ["Domain", "Other"], "properties": {"name": "Gamma"}}, + {"id": "ntlm-ldap-good", "kinds": ["Computer"], "properties": {"name": "ntlm-ldap-good", "domainsid": "S-1-5-21", "isdc": true, "ldapavailable": true, "ldapsigning": false}}, + {"id": "ntlm-ldap-domain", "kinds": ["Computer"], "properties": {"name": "ntlm-ldap-domain", "domainsid": "S-1-5-99", "isdc": true, "ldapavailable": true, "ldapsigning": false}}, + {"id": "ntlm-ldap-isdc", "kinds": ["Computer"], "properties": {"name": "ntlm-ldap-isdc", "domainsid": "S-1-5-21", "isdc": false, "ldapavailable": true, "ldapsigning": false}}, + {"id": "ntlm-ldap-available", "kinds": ["Computer"], "properties": {"name": "ntlm-ldap-available", "domainsid": "S-1-5-21", "isdc": true, "ldapavailable": false, "ldapsigning": false}}, + {"id": "ntlm-ldap-signing", "kinds": ["Computer"], "properties": {"name": "ntlm-ldap-signing", "domainsid": "S-1-5-21", "isdc": true, "ldapavailable": true, "ldapsigning": true}}, + {"id": "ntlm-ldaps-good", "kinds": ["Other"], "properties": {"name": "ntlm-ldaps-good", "domainsid": "S-1-5-21", "isdc": true, "ldapsavailable": true, "epa": false}}, + {"id": "ntlm-ldaps-domain", "kinds": ["Other"], "properties": {"name": "ntlm-ldaps-domain", "domainsid": "S-1-5-99", "isdc": true, "ldapsavailable": true, "epa": false}}, + {"id": "ntlm-ldaps-isdc", "kinds": ["Other"], "properties": {"name": "ntlm-ldaps-isdc", "domainsid": "S-1-5-21", "isdc": false, "ldapsavailable": true, "epa": false}}, + {"id": "ntlm-ldaps-available", "kinds": ["Other"], "properties": {"name": "ntlm-ldaps-available", "domainsid": "S-1-5-21", "isdc": true, "ldapsavailable": false, "epa": false}}, + {"id": "ntlm-ldaps-epa", "kinds": ["Other"], "properties": {"name": "ntlm-ldaps-epa", "domainsid": "S-1-5-21", "isdc": true, "ldapsavailable": true, "epa": true}} + ], + "edges": [ + {"start_id": "tenant", "end_id": "role-a", "kind": "Contains", "properties": {"marker": "contains-role-a"}}, + {"start_id": "tenant", "end_id": "role-b", "kind": "Contains", "properties": {"marker": "contains-role-b"}}, + {"start_id": "tenant", "end_id": "role-multi", "kind": "Contains", "properties": {"marker": "contains-role-multi"}}, + {"start_id": "tenant", "end_id": "role-wrong-kind", "kind": "Contains", "properties": {"marker": "contains-wrong-kind"}}, + {"start_id": "edge-start", "end_id": "edge-end", "kind": "MemberOf", "properties": {"marker": "exact-edge"}}, + {"start_id": "edge-end", "end_id": "edge-start", "kind": "MemberOf", "properties": {"marker": "reverse-edge"}}, + {"start_id": "edge-start", "end_id": "edge-end", "kind": "WrongEdge", "properties": {"marker": "wrong-edge"}}, + {"start_id": "local-good", "end_id": "local-target", "kind": "LocalToComputer", "properties": {"marker": "local-good"}}, + {"start_id": "local-good-2", "end_id": "local-target", "kind": "LocalToComputer", "properties": {"marker": "local-good-2"}}, + {"start_id": "local-wrong-suffix", "end_id": "local-target", "kind": "LocalToComputer", "properties": {"marker": "local-wrong-suffix"}}, + {"start_id": "local-good", "end_id": "local-other-target", "kind": "LocalToComputer", "properties": {"marker": "local-wrong-end"}}, + {"start_id": "local-good", "end_id": "local-target", "kind": "WrongLocal", "properties": {"marker": "local-wrong-kind"}} + ] + }, + "variants": [ + {"name": "LOOKUP-09 empty ID list", "vars": {"query": "MATCH (n) WHERE id(n) IN $ids RETURN n"}, "node_list_params": {"ids": []}, "assert": "empty"}, + {"name": "LOOKUP-09 single ID full hydration", "vars": {"query": "MATCH (n) WHERE id(n) IN $ids RETURN n"}, "node_list_params": {"ids": ["hydrate-a"]}, "assert": {"node_records": [{"id": "hydrate-a", "kinds": ["Hydrate"], "props": {"name": "hydrate-a", "value": 1}}]}}, + {"name": "LOOKUP-09 duplicate IDs do not duplicate nodes", "vars": {"query": "MATCH (n) WHERE id(n) IN $ids RETURN n"}, "node_list_params": {"ids": ["hydrate-a", "hydrate-a", "hydrate-b", "hydrate-a"]}, "assert": {"node_id_set": ["hydrate-a", "hydrate-b"], "row_count": 2}}, + {"name": "LOOKUP-09 thirty-two-entry sparse list", "vars": {"query": "MATCH (n) WHERE id(n) IN $ids RETURN n"}, "node_list_params": {"ids": ["hydrate-a", "hydrate-b", "hydrate-c", "hydrate-a", "hydrate-b", "hydrate-c", "hydrate-a", "hydrate-b", "hydrate-c", "hydrate-a", "hydrate-b", "hydrate-c", "hydrate-a", "hydrate-b", "hydrate-c", "hydrate-a", "hydrate-b", "hydrate-c", "hydrate-a", "hydrate-b", "hydrate-c", "hydrate-a", "hydrate-b", "hydrate-c", "hydrate-a", "hydrate-b", "hydrate-c", "hydrate-a", "hydrate-b", "hydrate-c", "hydrate-a", "hydrate-b"]}, "assert": {"node_id_set": ["hydrate-a", "hydrate-b", "hydrate-c"], "row_count": 3}}, + {"name": "LOOKUP-10 all missing null and boolean flag combinations", "vars": {"query": "MATCH (n:User) WHERE NOT (n.gmsa IS NOT NULL AND n.gmsa = true) AND NOT (n.msa IS NOT NULL AND n.msa = true) AND id(n) IN $ids RETURN n"}, "node_list_params": {"ids": ["flags-mm", "flags-mn", "flags-mf", "flags-mt", "flags-nm", "flags-nn", "flags-nf", "flags-nt", "flags-fm", "flags-fn", "flags-ff", "flags-ft", "flags-tm", "flags-tn", "flags-tf", "flags-tt"]}, "assert": {"node_id_set": ["flags-mm", "flags-mn", "flags-mf", "flags-nm", "flags-nn", "flags-nf", "flags-fm", "flags-fn", "flags-ff"]}}, + {"name": "LOOKUP-11 empty role-template list", "vars": {"query": "MATCH (s)-[:Contains]->(e) WHERE id(s) = $tenant AND (e:AZRole OR e:AZServicePrincipal) AND e.roletemplateid IN $roles RETURN e"}, "node_params": {"tenant": "tenant"}, "params": {"roles": []}, "assert": "empty"}, + {"name": "LOOKUP-11 single role kind and single role-template ID", "vars": {"query": "MATCH (s)-[:Contains]->(e:AZRole) WHERE id(s) = $tenant AND e.roletemplateid IN $roles RETURN e"}, "node_params": {"tenant": "tenant"}, "params": {"roles": ["role-a"]}, "assert": {"node_id_set": ["role-a"]}}, + {"name": "LOOKUP-11 thousand-entry role-template list", "vars": {"query": "MATCH (s)-[:Contains]->(e) WHERE id(s) = $tenant AND (e:AZRole OR e:AZServicePrincipal) AND e.roletemplateid IN $roles RETURN e"}, "node_params": {"tenant": "tenant"}, "params": {"roles": {"$type": "string_list", "prefix": "missing-role-", "count": 1000, "include": ["role-a", "role-b", "role-multi"]}}, "assert": {"node_id_set": ["role-a", "role-b", "role-multi"]}}, + {"name": "LOOKUP-11 endpoint boolean equality", "vars": {"query": "MATCH (s)-[:Contains]->(e) WHERE id(s) = $tenant AND (e:AZRole OR e:AZServicePrincipal) AND e.enabled = true RETURN e"}, "node_params": {"tenant": "tenant"}, "assert": {"node_id_set": ["role-a", "role-multi"]}}, + {"name": "LOOKUP-11 endpoint string equality", "vars": {"query": "MATCH (s)-[:Contains]->(e) WHERE id(s) = $tenant AND (e:AZRole OR e:AZServicePrincipal) AND e.state = $state RETURN e"}, "node_params": {"tenant": "tenant"}, "params": {"state": "active"}, "assert": {"node_id_set": ["role-a", "role-multi"]}}, + {"name": "LOOKUP-12 exact edge key First hit", "vars": {"query": "MATCH (s)-[r:MemberOf]->(e) WHERE id(s) = $start_id AND id(e) = $end_id RETURN r LIMIT 1"}, "node_params": {"start_id": "edge-start", "end_id": "edge-end"}, "assert": {"relationship_records": [{"start": "edge-start", "end": "edge-end", "kind": "MemberOf", "props": {"marker": "exact-edge"}}]}}, + {"name": "LOOKUP-12 exact edge key no hit", "vars": {"query": "MATCH (s)-[r:MemberOf]->(e) WHERE id(s) = $start_id AND id(e) = $end_id RETURN r LIMIT 1"}, "node_params": {"start_id": "edge-start", "end_id": "hydrate-a"}, "assert": "empty"}, + {"name": "LOOKUP-13 full start node suffix and bound end", "vars": {"query": "MATCH (s)-[:LocalToComputer]->(e) WHERE s.objectid ENDS WITH $suffix AND id(e) = $end_id RETURN s"}, "params": {"suffix": "-555"}, "node_params": {"end_id": "local-target"}, "assert": {"node_id_set": ["local-good", "local-good-2"]}}, + {"name": "LOOKUP-13 start ID suffix and bound end", "vars": {"query": "MATCH (s)-[:LocalToComputer]->(e) WHERE s.objectid ENDS WITH $suffix AND id(e) = $end_id RETURN id(s)"}, "params": {"suffix": "-555"}, "node_params": {"end_id": "local-target"}, "assert": {"keys": ["id(s)"], "row_count": 2}}, + {"name": "LOOKUP-14 descending order includes missing equal distinct and multi-kind", "vars": {"query": "MATCH (n:Domain) RETURN n ORDER BY n.name DESC"}, "assert": {"node_id_set": ["domain-missing", "domain-alpha", "domain-beta-a", "domain-beta-b", "domain-multi"], "row_count": 5}}, + {"name": "LOOKUP-14 secondary ID key makes equal-property ties deterministic", "vars": {"query": "MATCH (n:Domain) WHERE n.name IS NOT NULL RETURN n ORDER BY n.name DESC, id(n) ASC"}, "assert": {"ordered_node_ids": ["domain-multi", "domain-beta-a", "domain-beta-b", "domain-alpha"]}}, + {"name": "LOOKUP-16 typed LDAP ID projection with one decoy per leaf", "vars": {"query": "MATCH (n:Computer) WHERE n.domainsid = $domain AND n.isdc = true AND n.ldapavailable = true AND n.ldapsigning = false RETURN id(n)"}, "params": {"domain": "S-1-5-21"}, "assert": {"keys": ["id(n)"], "row_count": 1}}, + {"name": "LOOKUP-16 typed LDAP full-node projection", "vars": {"query": "MATCH (n:Computer) WHERE n.domainsid = $domain AND n.isdc = true AND n.ldapavailable = true AND n.ldapsigning = false RETURN n"}, "params": {"domain": "S-1-5-21"}, "assert": {"node_id_set": ["ntlm-ldap-good"]}}, + {"name": "LOOKUP-16 untyped LDAPS full-node projection with one decoy per leaf", "vars": {"query": "MATCH (n) WHERE n.domainsid = $domain AND n.isdc = true AND n.ldapsavailable = true AND n.epa = false RETURN n"}, "params": {"domain": "S-1-5-21"}, "assert": {"node_id_set": ["ntlm-ldaps-good"]}} + ] + } + ] +} diff --git a/integration/testdata/templates/basic_lookup_shapes.json b/integration/testdata/templates/basic_lookup_shapes.json new file mode 100644 index 00000000..4935b1dd --- /dev/null +++ b/integration/testdata/templates/basic_lookup_shapes.json @@ -0,0 +1,72 @@ +{ + "families": [ + { + "name": "LOOKUP-01 through LOOKUP-08 node predicates and projections", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "group", "kinds": ["Group", "Entity"], "properties": {"name": "group", "objectid": "S-1-5-21-512", "domainsid": "S-1-5-21"}}, + {"id": "user", "kinds": ["User", "Entity"], "properties": {"name": "user", "objectid": "S-1-5-21-513", "domainsid": "S-1-5-21"}}, + {"id": "multi", "kinds": ["Group", "User", "Entity"], "properties": {"name": "multi", "objectid": "S-1-5-21-514", "domainsid": "S-1-5-21"}}, + {"id": "local-group", "kinds": ["Group", "LocalGroup", "Entity"], "properties": {"name": "local-group", "objectid": "S-1-5-21-512", "domainsid": "S-1-5-21"}}, + {"id": "entity-only", "kinds": ["Entity"], "properties": {"name": "entity-only", "objectid": "S-1-5-21-512", "domainsid": "S-1-5-21"}}, + {"id": "tenant", "kinds": ["Tenant"], "properties": {"name": "tenant", "objectid": "tenant-1"}}, + {"id": "computer-hit-a", "kinds": ["Computer"], "properties": {"name": "dc.example.test", "objectid": "S-1-5-21-100", "enabled": true}}, + {"id": "computer-hit-b", "kinds": ["Computer", "Entity"], "properties": {"name": "dc.example.test", "objectid": "S-1-5-21-100", "enabled": true}}, + {"id": "computer-disabled", "kinds": ["Computer"], "properties": {"name": "dc.example.test", "objectid": "S-1-5-21-101", "enabled": false}}, + {"id": "objectid-untyped", "kinds": ["Other"], "properties": {"name": "dc.example.test", "objectid": "S-1-5-21-100", "enabled": true}}, + {"id": "ura-true", "kinds": ["Computer"], "properties": {"name": "ura-true", "hasura": true}}, + {"id": "ura-false", "kinds": ["Computer"], "properties": {"name": "ura-false", "hasura": false}}, + {"id": "ura-null", "kinds": ["Computer"], "properties": {"name": "ura-null", "hasura": null}}, + {"id": "ura-missing", "kinds": ["Computer"], "properties": {"name": "ura-missing"}}, + {"id": "adminsdholder", "kinds": ["Container"], "properties": {"name": "admin", "distinguishedname": "CN=ADMINSDHOLDER,CN=SYSTEM,DC=EXAMPLE,DC=TEST", "domainsid": "S-1-5-21"}}, + {"id": "admin-wrong-case", "kinds": ["Container"], "properties": {"name": "admin-case", "distinguishedname": "cn=adminsdholder,CN=SYSTEM,DC=EXAMPLE,DC=TEST", "domainsid": "S-1-5-21"}}, + {"id": "admin-wrong-domain", "kinds": ["Container"], "properties": {"name": "admin-domain", "distinguishedname": "CN=ADMINSDHOLDER,CN=SYSTEM,DC=OTHER", "domainsid": "S-1-5-99"}}, + {"id": "suffix-a", "kinds": ["Group"], "properties": {"name": "suffix-a", "objectid": "OBJECT-S-1"}}, + {"id": "suffix-b", "kinds": ["Group"], "properties": {"name": "suffix-b", "objectid": "OBJECT-S-2"}}, + {"id": "suffix-case", "kinds": ["Group"], "properties": {"name": "suffix-case", "objectid": "OBJECT-s-1"}}, + {"id": "suffix-wrong-kind", "kinds": ["User"], "properties": {"name": "suffix-user", "objectid": "OBJECT-S-1"}}, + {"id": "ci-prefix-exact", "kinds": ["Lookup"], "properties": {"name": "Remote Desktop Users Alpha"}}, + {"id": "ci-prefix-mixed", "kinds": ["Lookup"], "properties": {"name": "rEmOtE dEsKtOp UsErS Beta"}}, + {"id": "ci-prefix-literal", "kinds": ["Lookup"], "properties": {"name": "Remote%_Desktop Literal"}}, + {"id": "ci-prefix-wild-decoy", "kinds": ["Lookup"], "properties": {"name": "RemoteXXDesktop Decoy"}}, + {"id": "ci-contains-exact", "kinds": ["Entity"], "properties": {"name": "approver-exact", "objectid": "Approver_GUID"}}, + {"id": "ci-contains-substring", "kinds": ["Entity"], "properties": {"name": "approver-substring", "objectid": "prefix-APPROVER_guid-suffix"}}, + {"id": "ci-contains-decoy", "kinds": ["Entity"], "properties": {"name": "approver-decoy", "objectid": "different-guid"}}, + {"id": "name-missing", "kinds": ["Lookup"], "properties": {"objectid": "missing"}}, + {"id": "name-null", "kinds": ["Lookup"], "properties": {"name": null, "objectid": "null"}}, + {"id": "name-empty", "kinds": ["Lookup"], "properties": {"name": "", "objectid": "empty"}}, + {"id": "name-populated", "kinds": ["Lookup"], "properties": {"name": "populated", "objectid": "populated"}}, + {"id": "role-user", "kinds": ["AZRole"], "properties": {"name": "role-user", "tenantid": "tenant-1", "approvalrequired": true, "userapprovers": ["u1"]}}, + {"id": "role-group", "kinds": ["AZRole"], "properties": {"name": "role-group", "tenantid": "tenant-1", "approvalrequired": true, "groupapprovers": ["g1"]}}, + {"id": "role-both", "kinds": ["AZRole"], "properties": {"name": "role-both", "tenantid": "tenant-1", "approvalrequired": true, "userapprovers": ["u1"], "groupapprovers": ["g1"]}}, + {"id": "role-neither", "kinds": ["AZRole"], "properties": {"name": "role-neither", "tenantid": "tenant-1", "approvalrequired": true}}, + {"id": "role-null", "kinds": ["AZRole"], "properties": {"name": "role-null", "tenantid": "tenant-1", "approvalrequired": true, "userapprovers": null, "groupapprovers": null}}, + {"id": "role-wrong-tenant", "kinds": ["AZRole"], "properties": {"name": "role-wrong-tenant", "tenantid": "tenant-2", "approvalrequired": true, "userapprovers": ["u1"]}}, + {"id": "role-not-required", "kinds": ["AZRole"], "properties": {"name": "role-not-required", "tenantid": "tenant-1", "approvalrequired": false, "userapprovers": ["u1"]}} + ] + }, + "variants": [ + {"name": "LOOKUP-01 one kind ID projection", "vars": {"query": "MATCH (n:Group) RETURN id(n)"}, "assert": {"keys": ["id(n)"], "row_count": 6}}, + {"name": "LOOKUP-01 many kinds include multi-kind node once", "vars": {"query": "MATCH (n) WHERE n:Group OR n:User RETURN n"}, "assert": {"node_id_set": ["group", "user", "multi", "local-group", "suffix-a", "suffix-b", "suffix-case", "suffix-wrong-kind"]}}, + {"name": "LOOKUP-01 exact kind full hydration", "vars": {"query": "MATCH (n:Tenant) RETURN n"}, "assert": {"node_records": [{"id": "tenant", "kinds": ["Tenant"], "props": {"name": "tenant", "objectid": "tenant-1"}}]}}, + {"name": "LOOKUP-02 indexed kind and object ID First with multiple hits", "vars": {"query": "MATCH (n:Computer) WHERE n.objectid = $objectid RETURN n LIMIT 1"}, "params": {"objectid": "S-1-5-21-100"}, "assert": {"row_count": 1}}, + {"name": "LOOKUP-02 indexed kind no hit", "vars": {"query": "MATCH (n:Computer) WHERE n.objectid = $objectid RETURN n LIMIT 1"}, "params": {"objectid": "missing"}, "assert": "empty"}, + {"name": "LOOKUP-02 no-kind object ID includes untyped node", "vars": {"query": "MATCH (n) WHERE n.objectid = $objectid RETURN n"}, "params": {"objectid": "S-1-5-21-100"}, "assert": {"node_id_set": ["computer-hit-a", "computer-hit-b", "objectid-untyped"]}}, + {"name": "LOOKUP-02 two equalities string and boolean", "vars": {"query": "MATCH (n) WHERE n.name = $name AND n.enabled = $enabled RETURN id(n)"}, "params": {"name": "dc.example.test", "enabled": true}, "assert": {"keys": ["id(n)"], "row_count": 3}}, + {"name": "LOOKUP-03 true boolean and two-column projection", "vars": {"query": "MATCH (n:Computer) WHERE n.hasura = $value RETURN id(n), n.hasura"}, "params": {"value": true}, "assert": {"keys": ["id(n)", "n.hasura"], "row_count": 1}}, + {"name": "LOOKUP-03 false excludes null and missing", "vars": {"query": "MATCH (n:Computer) WHERE n.hasura = $value RETURN id(n), n.hasura"}, "params": {"value": false}, "assert": {"keys": ["id(n)", "n.hasura"], "row_count": 1}}, + {"name": "LOOKUP-04 case-sensitive AdminSDHolder prefix and domain", "vars": {"query": "MATCH (n:Container) WHERE n.distinguishedname STARTS WITH $prefix AND n.domainsid = $domain RETURN n"}, "params": {"prefix": "CN=ADMINSDHOLDER,CN=SYSTEM,", "domain": "S-1-5-21"}, "assert": {"node_id_set": ["adminsdholder"]}}, + {"name": "LOOKUP-04 OR of two suffixes is case-sensitive", "vars": {"query": "MATCH (n:Group) WHERE n.objectid ENDS WITH $a OR n.objectid ENDS WITH $b RETURN n"}, "params": {"a": "-S-1", "b": "-S-2"}, "assert": {"node_id_set": ["suffix-a", "suffix-b"]}}, + {"name": "LOOKUP-05 case-insensitive prefix exact and mixed case", "vars": {"query": "MATCH (n:Lookup) WHERE toLower(n.name) STARTS WITH $prefix RETURN n"}, "params": {"prefix": "remote desktop users"}, "assert": {"node_id_set": ["ci-prefix-exact", "ci-prefix-mixed"]}}, + {"name": "LOOKUP-05 percent and underscore remain literal", "vars": {"query": "MATCH (n:Lookup) WHERE toLower(n.name) STARTS WITH $prefix RETURN n"}, "params": {"prefix": "remote%_"}, "assert": {"node_id_set": ["ci-prefix-literal"]}}, + {"name": "LOOKUP-05 contains retains substring candidate", "vars": {"query": "MATCH (n:Entity) WHERE toLower(n.objectid) CONTAINS $fragment RETURN n"}, "params": {"fragment": "approver_guid"}, "assert": {"node_id_set": ["ci-contains-exact", "ci-contains-substring"]}}, + {"name": "LOOKUP-06 included kind group plus Entity suffix and domain", "vars": {"query": "MATCH (n) WHERE (n:Group OR n:User) AND n:Entity AND n.objectid ENDS WITH $suffix AND n.domainsid = $domain RETURN n"}, "params": {"suffix": "-512", "domain": "S-1-5-21"}, "assert": {"node_id_set": ["group", "local-group"]}}, + {"name": "LOOKUP-06 Entity excluding Group and LocalGroup", "vars": {"query": "MATCH (n:Entity) WHERE NOT (n:Group OR n:LocalGroup) AND n.objectid ENDS WITH $suffix RETURN n"}, "params": {"suffix": "-512"}, "assert": {"node_id_set": ["entity-only"]}}, + {"name": "LOOKUP-07 missing and explicit null names", "vars": {"query": "MATCH (n:Lookup) WHERE n.name IS NULL RETURN n"}, "assert": {"node_id_set": ["name-missing", "name-null"]}}, + {"name": "LOOKUP-07 empty and populated names are present", "vars": {"query": "MATCH (n:Lookup) WHERE n.name IS NOT NULL RETURN n"}, "assert": {"node_id_set": ["ci-prefix-exact", "ci-prefix-mixed", "ci-prefix-literal", "ci-prefix-wild-decoy", "name-empty", "name-populated"]}}, + {"name": "LOOKUP-08 either or both approver properties present", "vars": {"query": "MATCH (n:AZRole) WHERE n.tenantid = $tenant AND n.approvalrequired = true AND (n.userapprovers IS NOT NULL OR n.groupapprovers IS NOT NULL) RETURN n"}, "params": {"tenant": "tenant-1"}, "assert": {"node_id_set": ["role-user", "role-group", "role-both"]}} + ] + } + ] +} diff --git a/integration/testdata/templates/count_shapes.json b/integration/testdata/templates/count_shapes.json new file mode 100644 index 00000000..8db95c23 --- /dev/null +++ b/integration/testdata/templates/count_shapes.json @@ -0,0 +1,70 @@ +{ + "families": [ + { + "name": "LOOKUP-15 empty graph counts", + "template": "{{query}}", + "fixture": {"nodes": [], "edges": []}, + "variants": [ + {"name": "LOOKUP-15 empty node count", "vars": {"query": "MATCH (n) RETURN count(n)"}, "assert": {"exact_int": 0}}, + {"name": "LOOKUP-15 empty relationship count", "vars": {"query": "MATCH ()-[r]->() RETURN count(r)"}, "assert": {"exact_int": 0}} + ] + }, + { + "name": "LOOKUP-15 node-only graph counts", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "node-a", "kinds": ["CountNode"], "properties": {"name": "a"}}, + {"id": "node-b", "kinds": ["CountNode"], "properties": {"name": "b"}}, + {"id": "node-c", "kinds": ["CountNode"], "properties": {"name": "c"}} + ], + "edges": [] + }, + "variants": [ + {"name": "LOOKUP-15 node-only node count", "vars": {"query": "MATCH (n) RETURN count(n)"}, "assert": {"exact_int": 3}}, + {"name": "LOOKUP-15 node-only relationship count", "vars": {"query": "MATCH ()-[r]->() RETURN count(r)"}, "assert": {"exact_int": 0}} + ] + }, + { + "name": "LOOKUP-15 edge-bearing graph counts", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "node-a", "kinds": ["CountNode"], "properties": {"name": "a"}}, + {"id": "node-b", "kinds": ["CountNode"], "properties": {"name": "b"}} + ], + "edges": [ + {"start_id": "node-a", "end_id": "node-b", "kind": "CountEdge", "properties": {"marker": "edge"}} + ] + }, + "variants": [ + {"name": "LOOKUP-15 edge-bearing node count", "vars": {"query": "MATCH (n) RETURN count(n)"}, "assert": {"exact_int": 2}}, + {"name": "LOOKUP-15 edge-bearing relationship count", "vars": {"query": "MATCH ()-[r]->() RETURN count(r)"}, "assert": {"exact_int": 1}} + ] + }, + { + "name": "LOOKUP-15 dense graph counts", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "node-a", "kinds": ["CountNode"], "properties": {"name": "a"}}, + {"id": "node-b", "kinds": ["CountNode"], "properties": {"name": "b"}}, + {"id": "node-c", "kinds": ["CountNode"], "properties": {"name": "c"}}, + {"id": "node-d", "kinds": ["CountNode"], "properties": {"name": "d"}} + ], + "edges": [ + {"start_id": "node-a", "end_id": "node-b", "kind": "CountEdge", "properties": {"marker": "a-b"}}, + {"start_id": "node-a", "end_id": "node-c", "kind": "CountEdge", "properties": {"marker": "a-c"}}, + {"start_id": "node-a", "end_id": "node-d", "kind": "CountEdge", "properties": {"marker": "a-d"}}, + {"start_id": "node-b", "end_id": "node-a", "kind": "CountEdge", "properties": {"marker": "b-a"}}, + {"start_id": "node-c", "end_id": "node-a", "kind": "CountEdge", "properties": {"marker": "c-a"}}, + {"start_id": "node-d", "end_id": "node-a", "kind": "CountEdge", "properties": {"marker": "d-a"}} + ] + }, + "variants": [ + {"name": "LOOKUP-15 dense node count", "vars": {"query": "MATCH (n) RETURN count(n)"}, "assert": {"exact_int": 4}}, + {"name": "LOOKUP-15 dense relationship count", "vars": {"query": "MATCH ()-[r]->() RETURN count(r)"}, "assert": {"exact_int": 6}} + ] + } + ] +} diff --git a/integration/testdata/templates/fixed_suffix_expansion_shapes.json b/integration/testdata/templates/fixed_suffix_expansion_shapes.json new file mode 100644 index 00000000..8bb338d5 --- /dev/null +++ b/integration/testdata/templates/fixed_suffix_expansion_shapes.json @@ -0,0 +1,48 @@ +{ + "families": [ + { + "name": "Bounded fixed-suffix expansion semantics", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "fse-root", "kinds": ["ExpansionRoot"], "properties": {"root_key": "semantic-fse-root"}}, + {"id": "fse-mid", "kinds": ["ExpansionNode"], "properties": {"name": "mid"}}, + {"id": "fse-boundary-a", "kinds": ["ExpansionNode"], "properties": {"enabled": true}}, + {"id": "fse-boundary-b", "kinds": ["ExpansionNode"], "properties": {"enabled": true}}, + {"id": "fse-head", "kinds": ["SuffixHead"], "properties": {"name": "head"}}, + {"id": "fse-middle", "kinds": ["SuffixMiddle"], "properties": {"name": "middle"}}, + {"id": "fse-terminal", "kinds": ["SuffixTerminal"], "properties": {"name": "terminal"}}, + {"id": "fse-decoy-head", "kinds": ["SuffixHead"], "properties": {"name": "decoy"}} + ], + "edges": [ + {"start_id": "fse-root", "end_id": "fse-mid", "kind": "Expand", "properties": {"ordinal": 1}}, + {"start_id": "fse-mid", "end_id": "fse-mid", "kind": "Expand", "properties": {"ordinal": 2}}, + {"start_id": "fse-mid", "end_id": "fse-boundary-a", "kind": "Expand", "properties": {"ordinal": 3}}, + {"start_id": "fse-mid", "end_id": "fse-boundary-b", "kind": "Expand", "properties": {"ordinal": 4}}, + {"start_id": "fse-boundary-a", "end_id": "fse-head", "kind": "EnterSuffix", "properties": {"ordinal": 5}}, + {"start_id": "fse-boundary-b", "end_id": "fse-head", "kind": "EnterSuffix", "properties": {"ordinal": 6}}, + {"start_id": "fse-head", "end_id": "fse-middle", "kind": "ContinueSuffix", "properties": {"ordinal": 7}}, + {"start_id": "fse-middle", "end_id": "fse-terminal", "kind": "CompleteSuffix", "properties": {"ordinal": 8}}, + {"start_id": "fse-boundary-a", "end_id": "fse-decoy-head", "kind": "WrongEnterSuffix", "properties": {"ordinal": 9}} + ] + }, + "variants": [ + { + "name": "Bounded endpoint observation preserves physical suffix multiplicity", + "vars": {"query": "MATCH (root:ExpansionRoot) WHERE root.root_key = 'semantic-fse-root' MATCH (root)-[:Expand*0..2]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)"}, + "assert": {"keys": ["id(head)", "id(terminal)"], "row_count": 2} + }, + { + "name": "Bounded full path observation retains relationship-distinct paths", + "vars": {"query": "MATCH (root:ExpansionRoot) WHERE root.root_key = 'semantic-fse-root' MATCH p = (root)-[:Expand*0..2]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(:SuffixTerminal) RETURN p"}, + "assert": {"keys": ["p"], "row_count": 2} + }, + { + "name": "Bounded downstream WITH aggregation preserves bag semantics", + "vars": {"query": "MATCH (root:ExpansionRoot) WHERE root.root_key = 'semantic-fse-root' MATCH (root)-[:Expand*0..2]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) WITH head, terminal, count(*) AS trails RETURN trails"}, + "assert": {"keys": ["trails"], "row_values": [[2]], "row_count": 1} + } + ] + } + ] +} diff --git a/integration/testdata/templates/lowering_regression_shapes.json b/integration/testdata/templates/lowering_regression_shapes.json new file mode 100644 index 00000000..89410c43 --- /dev/null +++ b/integration/testdata/templates/lowering_regression_shapes.json @@ -0,0 +1,108 @@ +{ + "families": [ + { + "name": "greedy projection and zero-depth shortest path retain full values", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "common", "kinds": ["LoweringNode"], "properties": {"root": true, "terminal": true}}, + {"id": "other-root", "kinds": ["LoweringNode"], "properties": {"root": true}}, + {"id": "target", "kinds": ["LoweringNode"], "properties": {"terminal": true}} + ], + "edges": [ + {"start_id": "common", "end_id": "target", "kind": "LoweringEdge", "properties": {"route": "common-target"}}, + {"start_id": "other-root", "end_id": "target", "kind": "LoweringEdge", "properties": {"route": "other-target"}} + ] + }, + "variants": [ + { + "name": "RETURN star materializes nodes and paths including the equal endpoint pair", + "vars": {"query": "MATCH p = shortestPath((s:LoweringNode)-[:LoweringEdge*0..4]->(e:LoweringNode)) WHERE s.root = true AND e.terminal = true RETURN *"}, + "assert": { + "row_count": 3, + "node_id_set": ["common", "other-root", "target"], + "path_node_ids": [["common"], ["common", "target"], ["other-root", "target"]] + } + }, + { + "name": "WITH star observes the full path before a scalar final projection", + "vars": {"query": "MATCH p = shortestPath((s:LoweringNode)-[:LoweringEdge*0..4]->(e:LoweringNode)) WHERE s.root = true AND e.terminal = true WITH * RETURN p"}, + "assert": { + "row_count": 3, + "path_node_ids": [["common"], ["common", "target"], ["other-root", "target"]] + } + } + ] + }, + { + "name": "guarded endpoint seeded expansion preserves complete trail semantics", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "excluded-computer", "kinds": ["Computer"], "properties": {"name": "excluded"}}, + {"id": "good-computer", "kinds": ["Computer"], "properties": {"name": "good"}}, + {"id": "excluded-user", "kinds": ["User"], "properties": {"name": "excluded-user"}}, + {"id": "good-user", "kinds": ["User"], "properties": {"name": "good-user"}}, + {"id": "middle", "kinds": ["Group"], "properties": {"objectid": "MIDDLE"}}, + {"id": "alternate-middle", "kinds": ["Group"], "properties": {"objectid": "ALTERNATE-MIDDLE"}}, + {"id": "excluded-group", "kinds": ["Group"], "properties": {"objectid": "S-1-5-21-516"}}, + {"id": "terminal", "kinds": ["Group"], "properties": {"objectid": "S-1-5-21-512"}}, + {"id": "decoy", "kinds": ["Group"], "properties": {"objectid": "S-1-5-21-513"}} + ], + "edges": [ + {"start_id": "excluded-computer", "end_id": "excluded-group", "kind": "MemberOf", "properties": {"marker": "exclude"}}, + {"start_id": "excluded-computer", "end_id": "excluded-user", "kind": "HasSession", "properties": {"marker": "session-excluded"}}, + {"start_id": "good-computer", "end_id": "good-user", "kind": "HasSession", "properties": {"marker": "session-good"}}, + {"start_id": "good-user", "end_id": "good-user", "kind": "MemberOf", "properties": {"marker": "loop"}}, + {"start_id": "good-user", "end_id": "middle", "kind": "MemberOf", "properties": {"marker": "first"}}, + {"start_id": "middle", "end_id": "terminal", "kind": "MemberOf", "properties": {"marker": "second"}}, + {"start_id": "good-user", "end_id": "alternate-middle", "kind": "MemberOf", "properties": {"marker": "alternate-first"}}, + {"start_id": "alternate-middle", "end_id": "terminal", "kind": "MemberOf", "properties": {"marker": "alternate-second"}}, + {"start_id": "good-user", "end_id": "terminal", "kind": "MemberOf", "properties": {"marker": "direct"}}, + {"start_id": "good-user", "end_id": "decoy", "kind": "MemberOf", "properties": {"marker": "decoy"}} + ] + }, + "variants": [ + { + "name": "compound exclusion query returns ordered hydrated relationships", + "vars": {"query": "MATCH (s)-[:MemberOf*0..]->(excluded:Group) WHERE excluded.objectid ENDS WITH '-516' WITH COLLECT(s) AS exclude MATCH p = (c:Computer)-[:HasSession]->(:User)-[:MemberOf*1..]->(g:Group) WHERE g.objectid ENDS WITH '-512' AND NOT c IN exclude RETURN p LIMIT 1000"}, + "assert": { + "row_count": 6, + "path_relationship_records": [ + [{"start":"good-computer","end":"good-user","kind":"HasSession","props":{"marker":"session-good"}},{"start":"good-user","end":"terminal","kind":"MemberOf","props":{"marker":"direct"}}], + [{"start":"good-computer","end":"good-user","kind":"HasSession","props":{"marker":"session-good"}},{"start":"good-user","end":"middle","kind":"MemberOf","props":{"marker":"first"}},{"start":"middle","end":"terminal","kind":"MemberOf","props":{"marker":"second"}}], + [{"start":"good-computer","end":"good-user","kind":"HasSession","props":{"marker":"session-good"}},{"start":"good-user","end":"alternate-middle","kind":"MemberOf","props":{"marker":"alternate-first"}},{"start":"alternate-middle","end":"terminal","kind":"MemberOf","props":{"marker":"alternate-second"}}], + [{"start":"good-computer","end":"good-user","kind":"HasSession","props":{"marker":"session-good"}},{"start":"good-user","end":"good-user","kind":"MemberOf","props":{"marker":"loop"}},{"start":"good-user","end":"terminal","kind":"MemberOf","props":{"marker":"direct"}}], + [{"start":"good-computer","end":"good-user","kind":"HasSession","props":{"marker":"session-good"}},{"start":"good-user","end":"good-user","kind":"MemberOf","props":{"marker":"loop"}},{"start":"good-user","end":"middle","kind":"MemberOf","props":{"marker":"first"}},{"start":"middle","end":"terminal","kind":"MemberOf","props":{"marker":"second"}}], + [{"start":"good-computer","end":"good-user","kind":"HasSession","props":{"marker":"session-good"}},{"start":"good-user","end":"good-user","kind":"MemberOf","props":{"marker":"loop"}},{"start":"good-user","end":"alternate-middle","kind":"MemberOf","props":{"marker":"alternate-first"}},{"start":"alternate-middle","end":"terminal","kind":"MemberOf","props":{"marker":"alternate-second"}}] + ] + } + } + ] + } + ], + "metamorphic": [ + { + "name": "guarded endpoint lowering matches incumbent traversal", + "fixture": { + "nodes": [ + {"id": "computer", "kinds": ["Computer"], "properties": {}}, + {"id": "user", "kinds": ["User"], "properties": {}}, + {"id": "middle", "kinds": ["Group"], "properties": {}}, + {"id": "terminal", "kinds": ["Group"], "properties": {"objectid": "S-1-5-21-512"}} + ], + "edges": [ + {"start_id": "computer", "end_id": "user", "kind": "HasSession", "properties": {"marker": "session"}}, + {"start_id": "user", "end_id": "user", "kind": "MemberOf", "properties": {"marker": "loop"}}, + {"start_id": "user", "end_id": "middle", "kind": "MemberOf", "properties": {"marker": "first"}}, + {"start_id": "middle", "end_id": "terminal", "kind": "MemberOf", "properties": {"marker": "second"}} + ] + }, + "compare": ["path_node_ids", "path_relationship_records"], + "queries": [ + {"name": "guarded endpoint seeded", "cypher": "MATCH p = (c:Computer)-[:HasSession]->(:User)-[:MemberOf*1..]->(g:Group) WHERE g.objectid ENDS WITH '-512' RETURN p"}, + {"name": "incumbent relationship variable", "cypher": "MATCH p = (c:Computer)-[:HasSession]->(:User)-[rels:MemberOf*1..]->(g:Group) WHERE g.objectid ENDS WITH '-512' RETURN p"} + ] + } + ] +} diff --git a/integration/testdata/templates/mutation_post_state_shapes.json b/integration/testdata/templates/mutation_post_state_shapes.json new file mode 100644 index 00000000..f5a4d67b --- /dev/null +++ b/integration/testdata/templates/mutation_post_state_shapes.json @@ -0,0 +1,67 @@ +{ + "families": [ + { + "name": "mutation rollback restores the original fixture", + "template": "MATCH (n:DeleteTarget) WHERE n.objectid = $object_id DETACH DELETE n", + "params": {"object_id": "delete-me"}, + "fixture": { + "nodes": [ + {"id": "victim", "kinds": ["DeleteTarget", "Entity"], "properties": {"objectid": "delete-me", "marker": "victim"}}, + {"id": "survivor", "kinds": ["Entity"], "properties": {"objectid": "keep-me", "marker": "survivor"}}, + {"id": "kind-decoy", "kinds": ["Entity"], "properties": {"objectid": "delete-me", "marker": "wrong-kind"}}, + {"id": "property-decoy", "kinds": ["DeleteTarget"], "properties": {"objectid": "keep-me", "marker": "wrong-property"}} + ], + "edges": [ + {"start_id": "survivor", "end_id": "victim", "kind": "Incident", "properties": {"direction": "inbound"}}, + {"start_id": "victim", "end_id": "survivor", "kind": "Incident", "properties": {"direction": "outbound"}}, + {"start_id": "victim", "end_id": "victim", "kind": "Incident", "properties": {"direction": "self"}}, + {"start_id": "survivor", "end_id": "property-decoy", "kind": "Survives", "properties": {"marker": "keep-edge"}} + ] + }, + "variants": [ + { + "name": "first execution", + "assert": "no_error", + "post_assertions": [ + { + "cypher": "MATCH (n) RETURN n", + "assert": { + "node_records": [ + {"id": "survivor", "kinds": ["Entity"], "props": {"objectid": "keep-me", "marker": "survivor"}}, + {"id": "kind-decoy", "kinds": ["Entity"], "props": {"objectid": "delete-me", "marker": "wrong-kind"}}, + {"id": "property-decoy", "kinds": ["DeleteTarget"], "props": {"objectid": "keep-me", "marker": "wrong-property"}} + ] + } + }, + { + "cypher": "MATCH ()-[r]->() RETURN r", + "assert": {"relationship_records": [{"start": "survivor", "end": "property-decoy", "kind": "Survives", "props": {"marker": "keep-edge"}}]} + }, + { + "cypher": "MATCH (n) RETURN count(n)", + "assert": {"exact_int": 3} + } + ] + }, + { + "name": "identical execution after rollback", + "assert": "no_error", + "post_assertions": [ + { + "cypher": "MATCH (n) RETURN n", + "assert": {"node_id_set": ["survivor", "kind-decoy", "property-decoy"]} + }, + { + "cypher": "MATCH ()-[r]->() RETURN r", + "assert": {"relationship_triples": [{"start": "survivor", "end": "property-decoy", "kind": "Survives"}]} + }, + { + "cypher": "MATCH ()-[r]->() RETURN count(r)", + "assert": {"exact_int": 1} + } + ] + } + ] + } + ] +} diff --git a/integration/testdata/templates/post_processing_hop_shapes.json b/integration/testdata/templates/post_processing_hop_shapes.json new file mode 100644 index 00000000..211db629 --- /dev/null +++ b/integration/testdata/templates/post_processing_hop_shapes.json @@ -0,0 +1,280 @@ +{ + "families": [ + { + "name": "HOP-01 through HOP-03 anchored direction and relationship-kind cardinality", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "out-zero", "kinds": ["HopAnchor"], "properties": {"name": "out-zero"}}, + {"id": "out-one", "kinds": ["HopAnchor"], "properties": {"name": "out-one"}}, + {"id": "out-high", "kinds": ["HopAnchor"], "properties": {"name": "out-high"}}, + {"id": "in-zero", "kinds": ["HopAnchor"], "properties": {"name": "in-zero"}}, + {"id": "in-one", "kinds": ["HopAnchor"], "properties": {"name": "in-one"}}, + {"id": "in-high", "kinds": ["HopAnchor"], "properties": {"name": "in-high"}}, + {"id": "out-one-target", "kinds": ["HopEndpoint"], "properties": {"name": "out-one-target"}}, + {"id": "in-one-source", "kinds": ["HopEndpoint"], "properties": {"name": "in-one-source"}}, + {"id": "out-high-01", "kinds": ["HopEndpoint"], "properties": {"name": "out-high-01"}}, + {"id": "out-high-02", "kinds": ["HopEndpoint"], "properties": {"name": "out-high-02"}}, + {"id": "out-high-03", "kinds": ["HopEndpoint"], "properties": {"name": "out-high-03"}}, + {"id": "out-high-04", "kinds": ["HopEndpoint"], "properties": {"name": "out-high-04"}}, + {"id": "out-high-05", "kinds": ["HopEndpoint"], "properties": {"name": "out-high-05"}}, + {"id": "out-high-06", "kinds": ["HopEndpoint"], "properties": {"name": "out-high-06"}}, + {"id": "out-high-07", "kinds": ["HopEndpoint"], "properties": {"name": "out-high-07"}}, + {"id": "out-high-08", "kinds": ["HopEndpoint"], "properties": {"name": "out-high-08"}}, + {"id": "in-high-01", "kinds": ["HopEndpoint"], "properties": {"name": "in-high-01"}}, + {"id": "in-high-02", "kinds": ["HopEndpoint"], "properties": {"name": "in-high-02"}}, + {"id": "in-high-03", "kinds": ["HopEndpoint"], "properties": {"name": "in-high-03"}}, + {"id": "in-high-04", "kinds": ["HopEndpoint"], "properties": {"name": "in-high-04"}}, + {"id": "in-high-05", "kinds": ["HopEndpoint"], "properties": {"name": "in-high-05"}}, + {"id": "in-high-06", "kinds": ["HopEndpoint"], "properties": {"name": "in-high-06"}}, + {"id": "in-high-07", "kinds": ["HopEndpoint"], "properties": {"name": "in-high-07"}}, + {"id": "in-high-08", "kinds": ["HopEndpoint"], "properties": {"name": "in-high-08"}}, + {"id": "kind-center", "kinds": ["HopAnchor"], "properties": {"name": "kind-center"}}, + {"id": "kind-peer", "kinds": ["HopEndpoint"], "properties": {"name": "kind-peer"}} + ], + "edges": [ + {"start_id": "out-one", "end_id": "out-one-target", "kind": "HopKind01", "properties": {"marker": "out-one"}}, + {"start_id": "out-high", "end_id": "out-high-01", "kind": "HopKind01", "properties": {"marker": "out-high-01"}}, + {"start_id": "out-high", "end_id": "out-high-02", "kind": "HopKind01", "properties": {"marker": "out-high-02"}}, + {"start_id": "out-high", "end_id": "out-high-03", "kind": "HopKind01", "properties": {"marker": "out-high-03"}}, + {"start_id": "out-high", "end_id": "out-high-04", "kind": "HopKind01", "properties": {"marker": "out-high-04"}}, + {"start_id": "out-high", "end_id": "out-high-05", "kind": "HopKind01", "properties": {"marker": "out-high-05"}}, + {"start_id": "out-high", "end_id": "out-high-06", "kind": "HopKind01", "properties": {"marker": "out-high-06"}}, + {"start_id": "out-high", "end_id": "out-high-07", "kind": "HopKind01", "properties": {"marker": "out-high-07"}}, + {"start_id": "out-high", "end_id": "out-high-08", "kind": "HopKind01", "properties": {"marker": "out-high-08"}}, + {"start_id": "in-one-source", "end_id": "in-one", "kind": "HopKind01", "properties": {"marker": "in-one"}}, + {"start_id": "in-high-01", "end_id": "in-high", "kind": "HopKind01", "properties": {"marker": "in-high-01"}}, + {"start_id": "in-high-02", "end_id": "in-high", "kind": "HopKind01", "properties": {"marker": "in-high-02"}}, + {"start_id": "in-high-03", "end_id": "in-high", "kind": "HopKind01", "properties": {"marker": "in-high-03"}}, + {"start_id": "in-high-04", "end_id": "in-high", "kind": "HopKind01", "properties": {"marker": "in-high-04"}}, + {"start_id": "in-high-05", "end_id": "in-high", "kind": "HopKind01", "properties": {"marker": "in-high-05"}}, + {"start_id": "in-high-06", "end_id": "in-high", "kind": "HopKind01", "properties": {"marker": "in-high-06"}}, + {"start_id": "in-high-07", "end_id": "in-high", "kind": "HopKind01", "properties": {"marker": "in-high-07"}}, + {"start_id": "in-high-08", "end_id": "in-high", "kind": "HopKind01", "properties": {"marker": "in-high-08"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind01", "properties": {"marker": "out-kind-01"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind02", "properties": {"marker": "out-kind-02"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind03", "properties": {"marker": "out-kind-03"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind04", "properties": {"marker": "out-kind-04"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind05", "properties": {"marker": "out-kind-05"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind06", "properties": {"marker": "out-kind-06"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind07", "properties": {"marker": "out-kind-07"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind08", "properties": {"marker": "out-kind-08"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind09", "properties": {"marker": "out-kind-09"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind10", "properties": {"marker": "out-kind-10"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind11", "properties": {"marker": "out-kind-11"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind12", "properties": {"marker": "out-kind-12"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind13", "properties": {"marker": "out-kind-13"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind14", "properties": {"marker": "out-kind-14"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind15", "properties": {"marker": "out-kind-15"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind16", "properties": {"marker": "out-kind-16"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind17", "properties": {"marker": "out-kind-17"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind18", "properties": {"marker": "out-kind-18"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind19", "properties": {"marker": "out-kind-19"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind20", "properties": {"marker": "out-kind-20"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind21", "properties": {"marker": "out-kind-21"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind22", "properties": {"marker": "out-kind-22"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind23", "properties": {"marker": "out-kind-23"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind24", "properties": {"marker": "out-kind-24"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind25", "properties": {"marker": "out-kind-25"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind26", "properties": {"marker": "out-kind-26"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind27", "properties": {"marker": "out-kind-27"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind28", "properties": {"marker": "out-kind-28"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind29", "properties": {"marker": "out-kind-29"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind30", "properties": {"marker": "out-kind-30"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopDisallowed", "properties": {"marker": "out-disallowed"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind01", "properties": {"marker": "in-kind-01"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind02", "properties": {"marker": "in-kind-02"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind03", "properties": {"marker": "in-kind-03"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind04", "properties": {"marker": "in-kind-04"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind05", "properties": {"marker": "in-kind-05"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind06", "properties": {"marker": "in-kind-06"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind07", "properties": {"marker": "in-kind-07"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind08", "properties": {"marker": "in-kind-08"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind09", "properties": {"marker": "in-kind-09"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind10", "properties": {"marker": "in-kind-10"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind11", "properties": {"marker": "in-kind-11"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind12", "properties": {"marker": "in-kind-12"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind13", "properties": {"marker": "in-kind-13"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind14", "properties": {"marker": "in-kind-14"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind15", "properties": {"marker": "in-kind-15"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind16", "properties": {"marker": "in-kind-16"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind17", "properties": {"marker": "in-kind-17"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind18", "properties": {"marker": "in-kind-18"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind19", "properties": {"marker": "in-kind-19"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind20", "properties": {"marker": "in-kind-20"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind21", "properties": {"marker": "in-kind-21"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind22", "properties": {"marker": "in-kind-22"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind23", "properties": {"marker": "in-kind-23"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind24", "properties": {"marker": "in-kind-24"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind25", "properties": {"marker": "in-kind-25"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind26", "properties": {"marker": "in-kind-26"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind27", "properties": {"marker": "in-kind-27"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind28", "properties": {"marker": "in-kind-28"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind29", "properties": {"marker": "in-kind-29"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind30", "properties": {"marker": "in-kind-30"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopDisallowed", "properties": {"marker": "in-disallowed"}} + ] + }, + "variants": [ + {"name": "HOP-01 exact anchor zero fanout", "vars": {"query": "MATCH (s)-[r:HopKind01]->(e) WHERE id(s) = $anchor RETURN r, e"}, "node_params": {"anchor": "out-zero"}, "assert": "empty"}, + {"name": "HOP-01 exact anchor one fanout full hydration", "vars": {"query": "MATCH (s)-[r:HopKind01]->(e) WHERE id(s) = $anchor RETURN r, e"}, "node_params": {"anchor": "out-one"}, "assert": {"keys": ["r", "e"], "row_count": 1, "node_id_set": ["out-one-target"], "relationship_records": [{"start": "out-one", "end": "out-one-target", "kind": "HopKind01", "props": {"marker": "out-one"}}]}}, + {"name": "HOP-01 one-element IN high fanout", "vars": {"query": "MATCH (s)-[r:HopKind01]->(e) WHERE id(s) IN $anchors RETURN r, e"}, "node_list_params": {"anchors": ["out-high"]}, "assert": {"keys": ["r", "e"], "row_count": 8, "node_id_set": ["out-high-01", "out-high-02", "out-high-03", "out-high-04", "out-high-05", "out-high-06", "out-high-07", "out-high-08"]}}, + {"name": "HOP-02 exact anchor zero inbound fanout", "vars": {"query": "MATCH (s)-[r:HopKind01]->(e) WHERE id(e) = $anchor RETURN r, s"}, "node_params": {"anchor": "in-zero"}, "assert": "empty"}, + {"name": "HOP-02 exact anchor one inbound fanout full hydration", "vars": {"query": "MATCH (s)-[r:HopKind01]->(e) WHERE id(e) = $anchor RETURN r, s"}, "node_params": {"anchor": "in-one"}, "assert": {"keys": ["r", "s"], "row_count": 1, "node_id_set": ["in-one-source"], "relationship_records": [{"start": "in-one-source", "end": "in-one", "kind": "HopKind01", "props": {"marker": "in-one"}}]}}, + {"name": "HOP-02 one-element IN high inbound fanout", "vars": {"query": "MATCH (s)-[r:HopKind01]->(e) WHERE id(e) IN $anchors RETURN r, s"}, "node_list_params": {"anchors": ["in-high"]}, "assert": {"keys": ["r", "s"], "row_count": 8, "node_id_set": ["in-high-01", "in-high-02", "in-high-03", "in-high-04", "in-high-05", "in-high-06", "in-high-07", "in-high-08"]}}, + {"name": "HOP-03 outbound two kinds", "vars": {"query": "MATCH (s)-[r:HopKind01|HopKind02]->() WHERE id(s) = $anchor RETURN r.marker"}, "node_params": {"anchor": "kind-center"}, "assert": {"scalar_values": ["out-kind-01", "out-kind-02"]}}, + {"name": "HOP-03 outbound five kinds", "vars": {"query": "MATCH (s)-[r:HopKind01|HopKind02|HopKind03|HopKind04|HopKind05]->() WHERE id(s) = $anchor RETURN r.marker"}, "node_params": {"anchor": "kind-center"}, "assert": {"scalar_values": ["out-kind-01", "out-kind-02", "out-kind-03", "out-kind-04", "out-kind-05"]}}, + {"name": "HOP-03 outbound nine kinds", "vars": {"query": "MATCH (s)-[r:HopKind01|HopKind02|HopKind03|HopKind04|HopKind05|HopKind06|HopKind07|HopKind08|HopKind09]->() WHERE id(s) = $anchor RETURN r.marker"}, "node_params": {"anchor": "kind-center"}, "assert": {"scalar_values": ["out-kind-01", "out-kind-02", "out-kind-03", "out-kind-04", "out-kind-05", "out-kind-06", "out-kind-07", "out-kind-08", "out-kind-09"]}}, + {"name": "HOP-03 outbound thirty kinds full direction", "vars": {"query": "MATCH (s)-[r:HopKind01|HopKind02|HopKind03|HopKind04|HopKind05|HopKind06|HopKind07|HopKind08|HopKind09|HopKind10|HopKind11|HopKind12|HopKind13|HopKind14|HopKind15|HopKind16|HopKind17|HopKind18|HopKind19|HopKind20|HopKind21|HopKind22|HopKind23|HopKind24|HopKind25|HopKind26|HopKind27|HopKind28|HopKind29|HopKind30]->(e) WHERE id(s) = $anchor RETURN r, e"}, "node_params": {"anchor": "kind-center"}, "assert": {"keys": ["r", "e"], "row_count": 30, "contains_edge": {"start": "kind-center", "end": "kind-peer", "kind": "HopKind30", "props": {"marker": "out-kind-30"}}}}, + {"name": "HOP-03 inbound two kinds", "vars": {"query": "MATCH ()-[r:HopKind01|HopKind02]->(e) WHERE id(e) = $anchor RETURN r.marker"}, "node_params": {"anchor": "kind-center"}, "assert": {"scalar_values": ["in-kind-01", "in-kind-02"]}}, + {"name": "HOP-03 inbound five kinds", "vars": {"query": "MATCH ()-[r:HopKind01|HopKind02|HopKind03|HopKind04|HopKind05]->(e) WHERE id(e) = $anchor RETURN r.marker"}, "node_params": {"anchor": "kind-center"}, "assert": {"scalar_values": ["in-kind-01", "in-kind-02", "in-kind-03", "in-kind-04", "in-kind-05"]}}, + {"name": "HOP-03 inbound nine kinds", "vars": {"query": "MATCH ()-[r:HopKind01|HopKind02|HopKind03|HopKind04|HopKind05|HopKind06|HopKind07|HopKind08|HopKind09]->(e) WHERE id(e) = $anchor RETURN r.marker"}, "node_params": {"anchor": "kind-center"}, "assert": {"scalar_values": ["in-kind-01", "in-kind-02", "in-kind-03", "in-kind-04", "in-kind-05", "in-kind-06", "in-kind-07", "in-kind-08", "in-kind-09"]}}, + {"name": "HOP-03 inbound thirty kinds full direction", "vars": {"query": "MATCH (s)-[r:HopKind01|HopKind02|HopKind03|HopKind04|HopKind05|HopKind06|HopKind07|HopKind08|HopKind09|HopKind10|HopKind11|HopKind12|HopKind13|HopKind14|HopKind15|HopKind16|HopKind17|HopKind18|HopKind19|HopKind20|HopKind21|HopKind22|HopKind23|HopKind24|HopKind25|HopKind26|HopKind27|HopKind28|HopKind29|HopKind30]->(e) WHERE id(e) = $anchor RETURN r, s"}, "node_params": {"anchor": "kind-center"}, "assert": {"keys": ["r", "s"], "row_count": 30, "contains_edge": {"start": "kind-peer", "end": "kind-center", "kind": "HopKind30", "props": {"marker": "in-kind-30"}}}} + ] + }, + { + "name": "HOP-04 and HOP-05 endpoint kinds and ID constraints", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "root", "kinds": ["HopAnchor"], "properties": {"name": "root"}}, + {"id": "other-root", "kinds": ["HopAnchor"], "properties": {"name": "other-root"}}, + {"id": "typed-a", "kinds": ["HopEndA"], "properties": {"name": "typed-a"}}, + {"id": "typed-b", "kinds": ["HopEndB"], "properties": {"name": "typed-b"}}, + {"id": "typed-multi", "kinds": ["HopEndA", "HopEndB"], "properties": {"name": "typed-multi"}}, + {"id": "typed-wrong", "kinds": ["HopWrongEnd"], "properties": {"name": "typed-wrong"}}, + {"id": "id-a", "kinds": ["HopEndpoint"], "properties": {"name": "id-a"}}, + {"id": "id-b", "kinds": ["HopEndpoint"], "properties": {"name": "id-b"}}, + {"id": "id-decoy", "kinds": ["HopEndpoint"], "properties": {"name": "id-decoy"}} + ], + "edges": [ + {"start_id": "root", "end_id": "typed-a", "kind": "HopTypedEdge", "properties": {"marker": "typed-a"}}, + {"start_id": "root", "end_id": "typed-b", "kind": "HopTypedEdge", "properties": {"marker": "typed-b"}}, + {"start_id": "root", "end_id": "typed-multi", "kind": "HopTypedEdge", "properties": {"marker": "typed-multi"}}, + {"start_id": "root", "end_id": "typed-wrong", "kind": "HopTypedEdge", "properties": {"marker": "typed-wrong"}}, + {"start_id": "root", "end_id": "typed-a", "kind": "HopWrongEdge", "properties": {"marker": "wrong-edge"}}, + {"start_id": "typed-a", "end_id": "root", "kind": "HopTypedEdge", "properties": {"marker": "wrong-direction"}}, + {"start_id": "root", "end_id": "id-a", "kind": "HopIDEdge", "properties": {"marker": "id-a"}}, + {"start_id": "root", "end_id": "id-b", "kind": "HopIDEdge", "properties": {"marker": "id-b"}}, + {"start_id": "root", "end_id": "id-decoy", "kind": "HopIDEdge", "properties": {"marker": "id-decoy"}}, + {"start_id": "other-root", "end_id": "id-a", "kind": "HopIDEdge", "properties": {"marker": "wrong-root"}} + ] + }, + "variants": [ + {"name": "HOP-04 single endpoint kind includes multi-kind node", "vars": {"query": "MATCH (s)-[r:HopTypedEdge]->(e:HopEndA) WHERE id(s) = $root RETURN r.marker"}, "node_params": {"root": "root"}, "assert": {"scalar_values": ["typed-a", "typed-multi"]}}, + {"name": "HOP-04 endpoint kind disjunction excludes wrong kind edge and direction", "vars": {"query": "MATCH (s)-[r:HopTypedEdge]->(e) WHERE id(s) = $root AND (e:HopEndA OR e:HopEndB) RETURN r, e"}, "node_params": {"root": "root"}, "assert": {"keys": ["r", "e"], "row_count": 3, "node_id_set": ["typed-a", "typed-b", "typed-multi"]}}, + {"name": "HOP-05 empty end ID list", "vars": {"query": "MATCH (s)-[r:HopIDEdge]->(e) WHERE id(s) = $root AND id(e) IN $end_ids RETURN r.marker"}, "node_params": {"root": "root"}, "node_list_params": {"end_ids": []}, "assert": "empty"}, + {"name": "HOP-05 single end ID equality", "vars": {"query": "MATCH (s)-[r:HopIDEdge]->(e) WHERE id(s) = $root AND id(e) = $end_id RETURN r.marker"}, "node_params": {"root": "root", "end_id": "id-a"}, "assert": {"scalar_values": ["id-a"]}}, + {"name": "HOP-05 duplicate end IDs do not duplicate relationship rows", "vars": {"query": "MATCH (s)-[r:HopIDEdge]->(e) WHERE id(s) = $root AND id(e) IN $end_ids RETURN r.marker"}, "node_params": {"root": "root"}, "node_list_params": {"end_ids": ["id-a", "id-a"]}, "assert": {"scalar_values": ["id-a"]}}, + {"name": "HOP-05 small matching end ID list", "vars": {"query": "MATCH (s)-[r:HopIDEdge]->(e) WHERE id(s) = $root AND id(e) IN $end_ids RETURN r.marker"}, "node_params": {"root": "root"}, "node_list_params": {"end_ids": ["id-a", "id-b"]}, "assert": {"scalar_values": ["id-a", "id-b"]}}, + {"name": "HOP-05 matching traversal anchor ID constraint", "vars": {"query": "MATCH (s)-[r:HopIDEdge]->(e) WHERE id(s) = $root AND id(s) IN $allowed_roots AND id(e) IN $end_ids RETURN r.marker"}, "node_params": {"root": "root"}, "node_list_params": {"allowed_roots": ["root"], "end_ids": ["id-a", "id-b"]}, "assert": {"scalar_values": ["id-a", "id-b"]}}, + {"name": "HOP-05 contradictory traversal anchor ID constraint", "vars": {"query": "MATCH (s)-[r:HopIDEdge]->(e) WHERE id(s) = $root AND id(s) IN $allowed_roots AND id(e) IN $end_ids RETURN r.marker"}, "node_params": {"root": "root"}, "node_list_params": {"allowed_roots": ["other-root"], "end_ids": ["id-a", "id-b"]}, "assert": "empty"} + ] + }, + { + "name": "HOP-06 through HOP-08 scalar nested and collection endpoint predicates", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "root", "kinds": ["HopAnchor"], "properties": {"name": "root"}}, + {"id": "scalar-match", "kinds": ["HopPropertyEnd"], "properties": {"enabled": true, "score": 7, "value": "alpha", "isassignabletorole": "true"}}, + {"id": "scalar-false", "kinds": ["HopPropertyEnd"], "properties": {"enabled": false, "score": 0, "value": "", "isassignabletorole": "false"}}, + {"id": "scalar-missing", "kinds": ["HopPropertyEnd"], "properties": {}}, + {"id": "scalar-null", "kinds": ["HopPropertyEnd"], "properties": {"enabled": null, "score": null, "value": null, "isassignabletorole": null}}, + {"id": "nested-v2", "kinds": ["HopTemplate"], "properties": {"requiresmanagerapproval": false, "schemaversion": 2, "authorizedsignatures": 0, "authenticationenabled": true}}, + {"id": "nested-v1", "kinds": ["HopTemplate"], "properties": {"requiresmanagerapproval": false, "schemaversion": 1, "authorizedsignatures": 9, "authenticationenabled": true}}, + {"id": "nested-manager", "kinds": ["HopTemplate"], "properties": {"requiresmanagerapproval": true, "schemaversion": 2, "authorizedsignatures": 0, "authenticationenabled": true}}, + {"id": "nested-signatures", "kinds": ["HopTemplate"], "properties": {"requiresmanagerapproval": false, "schemaversion": 2, "authorizedsignatures": 1, "authenticationenabled": true}}, + {"id": "nested-auth", "kinds": ["HopTemplate"], "properties": {"requiresmanagerapproval": false, "schemaversion": 2, "authorizedsignatures": 0, "authenticationenabled": false}}, + {"id": "nested-cross", "kinds": ["HopTemplate"], "properties": {"requiresmanagerapproval": false, "schemaversion": 1, "authorizedsignatures": 0, "authenticationenabled": false}}, + {"id": "nested-wrong-kind", "kinds": ["HopWrongEnd"], "properties": {"requiresmanagerapproval": false, "schemaversion": 2, "authorizedsignatures": 0, "authenticationenabled": true}}, + {"id": "collection-empty", "kinds": ["HopCollectionEnd"], "properties": {"schannelauthenticationenabled": false, "effectiveekus": []}}, + {"id": "collection-client", "kinds": ["HopCollectionEnd"], "properties": {"schannelauthenticationenabled": false, "effectiveekus": ["1.3.6.1.5.5.7.3.2"]}}, + {"id": "collection-scalar", "kinds": ["HopCollectionEnd"], "properties": {"schannelauthenticationenabled": true, "effectiveekus": ["other"]}}, + {"id": "collection-other", "kinds": ["HopCollectionEnd"], "properties": {"schannelauthenticationenabled": false, "effectiveekus": ["other"]}}, + {"id": "collection-missing", "kinds": ["HopCollectionEnd"], "properties": {"schannelauthenticationenabled": false}}, + {"id": "collection-null", "kinds": ["HopCollectionEnd"], "properties": {"schannelauthenticationenabled": false, "effectiveekus": null}} + ], + "edges": [ + {"start_id": "root", "end_id": "scalar-match", "kind": "HopPropertyEdge", "properties": {"marker": "scalar-match"}}, + {"start_id": "root", "end_id": "scalar-false", "kind": "HopPropertyEdge", "properties": {"marker": "scalar-false"}}, + {"start_id": "root", "end_id": "scalar-missing", "kind": "HopPropertyEdge", "properties": {"marker": "scalar-missing"}}, + {"start_id": "root", "end_id": "scalar-null", "kind": "HopPropertyEdge", "properties": {"marker": "scalar-null"}}, + {"start_id": "root", "end_id": "nested-v2", "kind": "HopNestedEdge", "properties": {"marker": "nested-v2"}}, + {"start_id": "root", "end_id": "nested-v1", "kind": "HopNestedEdge", "properties": {"marker": "nested-v1"}}, + {"start_id": "root", "end_id": "nested-manager", "kind": "HopNestedEdge", "properties": {"marker": "nested-manager"}}, + {"start_id": "root", "end_id": "nested-signatures", "kind": "HopNestedEdge", "properties": {"marker": "nested-signatures"}}, + {"start_id": "root", "end_id": "nested-auth", "kind": "HopNestedEdge", "properties": {"marker": "nested-auth"}}, + {"start_id": "root", "end_id": "nested-cross", "kind": "HopNestedEdge", "properties": {"marker": "nested-cross"}}, + {"start_id": "root", "end_id": "nested-wrong-kind", "kind": "HopNestedEdge", "properties": {"marker": "nested-wrong-kind"}}, + {"start_id": "root", "end_id": "nested-v2", "kind": "HopWrongEdge", "properties": {"marker": "nested-wrong-edge"}}, + {"start_id": "root", "end_id": "collection-empty", "kind": "HopCollectionEdge", "properties": {"marker": "collection-empty"}}, + {"start_id": "root", "end_id": "collection-client", "kind": "HopCollectionEdge", "properties": {"marker": "collection-client"}}, + {"start_id": "root", "end_id": "collection-scalar", "kind": "HopCollectionEdge", "properties": {"marker": "collection-scalar"}}, + {"start_id": "root", "end_id": "collection-other", "kind": "HopCollectionEdge", "properties": {"marker": "collection-other"}}, + {"start_id": "root", "end_id": "collection-missing", "kind": "HopCollectionEdge", "properties": {"marker": "collection-missing"}}, + {"start_id": "root", "end_id": "collection-null", "kind": "HopCollectionEdge", "properties": {"marker": "collection-null"}} + ] + }, + "variants": [ + {"name": "HOP-06 boolean true excludes false missing and null", "vars": {"query": "MATCH (s)-[r:HopPropertyEdge]->(e) WHERE id(s) = $root AND e.enabled = true RETURN r.marker"}, "node_params": {"root": "root"}, "assert": {"scalar_values": ["scalar-match"]}}, + {"name": "HOP-06 boolean false", "vars": {"query": "MATCH (s)-[r:HopPropertyEdge]->(e) WHERE id(s) = $root AND e.enabled = false RETURN r.marker"}, "node_params": {"root": "root"}, "assert": {"scalar_values": ["scalar-false"]}}, + {"name": "HOP-06 numeric equality", "vars": {"query": "MATCH (s)-[r:HopPropertyEdge]->(e) WHERE id(s) = $root AND e.score = 7 RETURN r.marker"}, "node_params": {"root": "root"}, "assert": {"scalar_values": ["scalar-match"]}}, + {"name": "HOP-06 string equality", "vars": {"query": "MATCH (s)-[r:HopPropertyEdge]->(e) WHERE id(s) = $root AND e.value = 'alpha' RETURN r.marker"}, "node_params": {"root": "root"}, "assert": {"scalar_values": ["scalar-match"]}}, + {"name": "HOP-06 production string true value", "vars": {"query": "MATCH (s)-[r:HopPropertyEdge]->(e) WHERE id(s) = $root AND e.isassignabletorole = 'true' RETURN r, e"}, "node_params": {"root": "root"}, "assert": {"keys": ["r", "e"], "row_count": 1, "node_id_set": ["scalar-match"]}}, + {"name": "HOP-07 exact branch-local nested truth table", "vars": {"query": "MATCH (s)-[r:HopNestedEdge]->(e:HopTemplate) WHERE id(s) = $root AND ((e.requiresmanagerapproval = false AND e.schemaversion > 1 AND e.authorizedsignatures = 0 AND e.authenticationenabled = true) OR (e.requiresmanagerapproval = false AND e.schemaversion = 1 AND e.authenticationenabled = true)) RETURN r.marker"}, "node_params": {"root": "root"}, "assert": {"scalar_values": ["nested-v2", "nested-v1"]}}, + {"name": "HOP-07 full directional hydration", "vars": {"query": "MATCH (s)-[r:HopNestedEdge]->(e:HopTemplate) WHERE id(s) = $root AND ((e.requiresmanagerapproval = false AND e.schemaversion > 1 AND e.authorizedsignatures = 0 AND e.authenticationenabled = true) OR (e.requiresmanagerapproval = false AND e.schemaversion = 1 AND e.authenticationenabled = true)) RETURN r, e"}, "node_params": {"root": "root"}, "assert": {"keys": ["r", "e"], "row_count": 2, "node_id_set": ["nested-v2", "nested-v1"]}}, + {"name": "HOP-08 empty collection", "vars": {"query": "MATCH (s)-[r:HopCollectionEdge]->(e) WHERE id(s) = $root AND size(e.effectiveekus) = 0 RETURN r.marker"}, "node_params": {"root": "root"}, "assert": {"scalar_values": ["collection-empty"]}}, + {"name": "HOP-08 value membership", "vars": {"query": "MATCH (s)-[r:HopCollectionEdge]->(e) WHERE id(s) = $root AND $eku IN e.effectiveekus RETURN r.marker"}, "node_params": {"root": "root"}, "params": {"eku": "1.3.6.1.5.5.7.3.2"}, "assert": {"scalar_values": ["collection-client"]}}, + {"name": "HOP-08 nested collection OR scalar predicate", "vars": {"query": "MATCH (s)-[r:HopCollectionEdge]->(e) WHERE id(s) = $root AND (e.schannelauthenticationenabled = true OR size(e.effectiveekus) = 0 OR $eku IN e.effectiveekus) RETURN r, e"}, "node_params": {"root": "root"}, "params": {"eku": "1.3.6.1.5.5.7.3.2"}, "assert": {"keys": ["r", "e"], "row_count": 3, "node_id_set": ["collection-empty", "collection-client", "collection-scalar"]}} + ] + }, + { + "name": "HOP-09 and HOP-10 two-sided sets and directional projections", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "s1", "kinds": ["HopProjectionStart"], "properties": {"name": "s1", "active": true}}, + {"id": "s2", "kinds": ["HopProjectionStart"], "properties": {"name": "s2", "active": true}}, + {"id": "s3", "kinds": ["HopProjectionStart"], "properties": {"name": "s3", "active": false}}, + {"id": "e1", "kinds": ["HopProjectionEnd"], "properties": {"name": "e1", "active": true}}, + {"id": "e2", "kinds": ["HopProjectionEnd"], "properties": {"name": "e2", "active": true}}, + {"id": "e3", "kinds": ["HopProjectionEnd"], "properties": {"name": "e3", "active": false}}, + {"id": "common", "kinds": ["HopProjectionStart", "HopProjectionEnd"], "properties": {"name": "common", "active": true}}, + {"id": "wrong-kind-start", "kinds": ["HopWrongStart"], "properties": {"name": "wrong-kind-start", "active": true}}, + {"id": "wrong-kind-end", "kinds": ["HopWrongEnd"], "properties": {"name": "wrong-kind-end", "active": true}} + ], + "edges": [ + {"start_id": "s1", "end_id": "e1", "kind": "HopSetEdge", "properties": {"marker": "s1-e1"}}, + {"start_id": "s1", "end_id": "e2", "kind": "HopSetEdge", "properties": {"marker": "s1-e2"}}, + {"start_id": "s2", "end_id": "e1", "kind": "HopSetEdge", "properties": {"marker": "s2-e1"}}, + {"start_id": "s2", "end_id": "e2", "kind": "HopSetEdge", "properties": {"marker": "s2-e2"}}, + {"start_id": "s3", "end_id": "e3", "kind": "HopSetEdge", "properties": {"marker": "s3-e3"}}, + {"start_id": "common", "end_id": "common", "kind": "HopSetEdge", "properties": {"marker": "common-self"}}, + {"start_id": "s1", "end_id": "e1", "kind": "HopWrongEdge", "properties": {"marker": "wrong-edge"}}, + {"start_id": "e1", "end_id": "s1", "kind": "HopSetEdge", "properties": {"marker": "wrong-direction"}}, + {"start_id": "s1", "end_id": "e1", "kind": "HopProjectionEdge", "properties": {"marker": "projection-out"}}, + {"start_id": "s2", "end_id": "e2", "kind": "HopProjectionEdge", "properties": {"marker": "projection-second"}}, + {"start_id": "s3", "end_id": "e3", "kind": "HopProjectionEdge", "properties": {"marker": "projection-inactive"}}, + {"start_id": "wrong-kind-start", "end_id": "e1", "kind": "HopProjectionEdge", "properties": {"marker": "projection-wrong-start"}}, + {"start_id": "s1", "end_id": "wrong-kind-end", "kind": "HopProjectionEdge", "properties": {"marker": "projection-wrong-end"}} + ] + }, + "variants": [ + {"name": "HOP-09 empty start list", "vars": {"query": "MATCH (s)-[r:HopSetEdge]->(e) WHERE id(s) IN $start_ids AND id(e) IN $end_ids RETURN r.marker"}, "node_list_params": {"start_ids": [], "end_ids": ["e1", "e2"]}, "assert": "empty"}, + {"name": "HOP-09 empty end list", "vars": {"query": "MATCH (s)-[r:HopSetEdge]->(e) WHERE id(s) IN $start_ids AND id(e) IN $end_ids RETURN r.marker"}, "node_list_params": {"start_ids": ["s1", "s2"], "end_ids": []}, "assert": "empty"}, + {"name": "HOP-09 singleton sets", "vars": {"query": "MATCH (s)-[r:HopSetEdge]->(e) WHERE id(s) IN $start_ids AND id(e) IN $end_ids RETURN r.marker"}, "node_list_params": {"start_ids": ["s1"], "end_ids": ["e1"]}, "assert": {"scalar_values": ["s1-e1"]}}, + {"name": "HOP-09 duplicate IDs do not duplicate rows", "vars": {"query": "MATCH (s)-[r:HopSetEdge]->(e) WHERE id(s) IN $start_ids AND id(e) IN $end_ids RETURN r.marker"}, "node_list_params": {"start_ids": ["s1", "s1"], "end_ids": ["e1", "e1"]}, "assert": {"scalar_values": ["s1-e1"]}}, + {"name": "HOP-09 dense small bipartite sets", "vars": {"query": "MATCH (s)-[r:HopSetEdge]->(e) WHERE id(s) IN $start_ids AND id(e) IN $end_ids RETURN r, e"}, "node_list_params": {"start_ids": ["s1", "s2"], "end_ids": ["e1", "e2"]}, "assert": {"keys": ["r", "e"], "row_count": 4, "node_id_set": ["e1", "e2"]}}, + {"name": "HOP-09 overlapping start and end sets retain self edge", "vars": {"query": "MATCH (s)-[r:HopSetEdge]->(e) WHERE id(s) IN $start_ids AND id(e) IN $end_ids RETURN r.marker"}, "node_list_params": {"start_ids": ["common"], "end_ids": ["common"]}, "assert": {"scalar_values": ["common-self"]}}, + {"name": "HOP-10 outbound full relationship and endpoint", "vars": {"query": "MATCH (s)-[r:HopProjectionEdge]->(e:HopProjectionEnd) WHERE id(s) = $start_id AND e.active = true RETURN r, e"}, "node_params": {"start_id": "s1"}, "assert": {"keys": ["r", "e"], "row_count": 1, "node_id_set": ["e1"], "relationship_records": [{"start": "s1", "end": "e1", "kind": "HopProjectionEdge", "props": {"marker": "projection-out"}}]}}, + {"name": "HOP-10 outbound endpoint node projection", "vars": {"query": "MATCH (s)-[r:HopProjectionEdge]->(e:HopProjectionEnd) WHERE id(s) = $start_id AND e.active = true RETURN e"}, "node_params": {"start_id": "s1"}, "assert": {"node_id_set": ["e1"]}}, + {"name": "HOP-10 outbound endpoint ID projection", "vars": {"query": "MATCH (s)-[r:HopProjectionEdge]->(e:HopProjectionEnd) WHERE id(s) = $start_id AND e.active = true RETURN id(e)"}, "node_params": {"start_id": "s1"}, "assert": {"keys": ["id(e)"], "row_count": 1}}, + {"name": "HOP-10 relationship-only projection", "vars": {"query": "MATCH (s)-[r:HopProjectionEdge]->(e:HopProjectionEnd) WHERE id(s) = $start_id AND e.active = true RETURN r"}, "node_params": {"start_id": "s1"}, "assert": {"relationship_records": [{"start": "s1", "end": "e1", "kind": "HopProjectionEdge", "props": {"marker": "projection-out"}}]}}, + {"name": "HOP-10 inbound full relationship and endpoint", "vars": {"query": "MATCH (s:HopProjectionStart)-[r:HopProjectionEdge]->(e) WHERE id(e) = $end_id AND s.active = true RETURN r, s"}, "node_params": {"end_id": "e1"}, "assert": {"keys": ["r", "s"], "row_count": 1, "node_id_set": ["s1"], "relationship_records": [{"start": "s1", "end": "e1", "kind": "HopProjectionEdge", "props": {"marker": "projection-out"}}]}}, + {"name": "HOP-10 inbound endpoint node projection", "vars": {"query": "MATCH (s:HopProjectionStart)-[r:HopProjectionEdge]->(e) WHERE id(e) = $end_id AND s.active = true RETURN s"}, "node_params": {"end_id": "e1"}, "assert": {"node_id_set": ["s1"]}}, + {"name": "HOP-10 inbound endpoint ID projection", "vars": {"query": "MATCH (s:HopProjectionStart)-[r:HopProjectionEdge]->(e) WHERE id(e) = $end_id AND s.active = true RETURN id(s)"}, "node_params": {"end_id": "e1"}, "assert": {"keys": ["id(s)"], "row_count": 1}} + ] + } + ] +} diff --git a/integration/testdata/templates/post_processing_shapes.json b/integration/testdata/templates/post_processing_shapes.json new file mode 100644 index 00000000..0076e1cd --- /dev/null +++ b/integration/testdata/templates/post_processing_shapes.json @@ -0,0 +1,143 @@ +{ + "families": [ + { + "name": "LOGIC-03 scoped kind negation with null-aware age predicate", + "template": "MATCH (n) WHERE NOT n:LogicProtected AND (n.lastseen IS NULL OR datetime(n.lastseen) < datetime($threshold)) RETURN n", + "params": {"threshold": "2026-01-03T00:00:00Z"}, + "fixture": { + "nodes": [ + {"id": "missing", "kinds": ["LogicCandidate"], "properties": {"name": "missing"}}, + {"id": "null", "kinds": ["LogicCandidate"], "properties": {"name": "null", "lastseen": null}}, + {"id": "older", "kinds": ["LogicCandidate"], "properties": {"name": "older", "lastseen": "2026-01-02T00:00:00Z"}}, + {"id": "equal", "kinds": ["LogicCandidate"], "properties": {"name": "equal", "lastseen": "2026-01-03T00:00:00Z"}}, + {"id": "newer", "kinds": ["LogicCandidate"], "properties": {"name": "newer", "lastseen": "2026-01-04T00:00:00Z"}}, + {"id": "protected-missing", "kinds": ["LogicProtected"], "properties": {"name": "protected-missing"}}, + {"id": "protected-null", "kinds": ["LogicProtected"], "properties": {"name": "protected-null", "lastseen": null}}, + {"id": "protected-older", "kinds": ["LogicProtected"], "properties": {"name": "protected-older", "lastseen": "2026-01-02T00:00:00Z"}}, + {"id": "multi-kind-protected", "kinds": ["LogicCandidate", "LogicProtected"], "properties": {"name": "multi-kind-protected", "lastseen": "2026-01-02T00:00:00Z"}} + ] + }, + "variants": [ + { + "name": "missing null older equal newer and protected truth table", + "assert": {"node_id_set": ["missing", "null", "older"]} + } + ] + }, + { + "name": "PRUNE-01 stale relationship selection with protected kinds", + "template": "MATCH ()-[r]->() WHERE {{excluded}} AND datetime(r.lastseen) < datetime($threshold) RETURN {{projection}}", + "params": {"threshold": "2026-01-03T00:00:00Z"}, + "fixture": { + "nodes": [ + {"id": "a", "kinds": ["PruneEndpoint"], "properties": {"name": "a"}}, + {"id": "b", "kinds": ["PruneEndpoint"], "properties": {"name": "b"}}, + {"id": "b-equal", "kinds": ["PruneEndpoint"], "properties": {"name": "b-equal"}}, + {"id": "b-new", "kinds": ["PruneEndpoint"], "properties": {"name": "b-new"}}, + {"id": "b-missing", "kinds": ["PruneEndpoint"], "properties": {"name": "b-missing"}}, + {"id": "b-null", "kinds": ["PruneEndpoint"], "properties": {"name": "b-null"}} + ], + "edges": [ + {"start_id": "a", "end_id": "b", "kind": "CandidateRel", "properties": {"lastseen": "2026-01-02T00:00:00Z", "marker": "candidate-old"}}, + {"start_id": "a", "end_id": "b-equal", "kind": "CandidateRel", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "candidate-equal"}}, + {"start_id": "a", "end_id": "b-new", "kind": "CandidateRel", "properties": {"lastseen": "2026-01-04T00:00:00Z", "marker": "candidate-new"}}, + {"start_id": "a", "end_id": "b-missing", "kind": "CandidateRel", "properties": {"marker": "candidate-missing"}}, + {"start_id": "a", "end_id": "b-null", "kind": "CandidateRel", "properties": {"lastseen": null, "marker": "candidate-null"}}, + {"start_id": "a", "end_id": "b", "kind": "HasSession", "properties": {"lastseen": "2026-01-02T00:00:00Z", "marker": "session-old"}}, + {"start_id": "a", "end_id": "b", "kind": "MetaIncludes", "properties": {"lastseen": "2026-01-02T00:00:00Z", "marker": "meta-includes-old"}} + ] + }, + "variants": [ + { + "name": "one excluded kind leaves other old kinds eligible", + "vars": {"excluded": "NOT r:HasSession", "projection": "r.marker"}, + "assert": {"scalar_values": ["candidate-old", "meta-includes-old"]} + }, + { + "name": "several excluded kinds select only old candidate relationship IDs", + "vars": {"excluded": "NOT (r:HasSession OR r:MetaIncludes)", "projection": "id(r)"}, + "assert": {"keys": ["id(r)"], "row_count": 1} + }, + { + "name": "several excluded kinds exact old equal new missing and null matrix", + "vars": {"excluded": "NOT (r:HasSession OR r:MetaIncludes)", "projection": "r.marker"}, + "assert": {"scalar_values": ["candidate-old"]} + } + ] + }, + { + "name": "PRUNE-02 stale or unobserved HasSession selection", + "template": "MATCH ()-[r:HasSession]->() WHERE r.lastseen IS NULL OR datetime(r.lastseen) < datetime($threshold) RETURN {{projection}}", + "params": {"threshold": "2026-01-03T00:00:00Z"}, + "fixture": { + "nodes": [ + {"id": "a", "kinds": ["PruneEndpoint"], "properties": {"name": "a"}}, + {"id": "b", "kinds": ["PruneEndpoint"], "properties": {"name": "b"}}, + {"id": "b-null", "kinds": ["PruneEndpoint"], "properties": {"name": "b-null"}}, + {"id": "b-old", "kinds": ["PruneEndpoint"], "properties": {"name": "b-old"}}, + {"id": "b-equal", "kinds": ["PruneEndpoint"], "properties": {"name": "b-equal"}}, + {"id": "b-new", "kinds": ["PruneEndpoint"], "properties": {"name": "b-new"}} + ], + "edges": [ + {"start_id": "a", "end_id": "b", "kind": "HasSession", "properties": {"marker": "session-missing"}}, + {"start_id": "a", "end_id": "b-null", "kind": "HasSession", "properties": {"lastseen": null, "marker": "session-null"}}, + {"start_id": "a", "end_id": "b-old", "kind": "HasSession", "properties": {"lastseen": "2026-01-02T00:00:00Z", "marker": "session-old"}}, + {"start_id": "a", "end_id": "b-equal", "kind": "HasSession", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "session-equal"}}, + {"start_id": "a", "end_id": "b-new", "kind": "HasSession", "properties": {"lastseen": "2026-01-04T00:00:00Z", "marker": "session-new"}}, + {"start_id": "a", "end_id": "b", "kind": "OtherSession", "properties": {"marker": "wrong-kind-missing"}}, + {"start_id": "a", "end_id": "b-old", "kind": "OtherSession", "properties": {"lastseen": "2026-01-02T00:00:00Z", "marker": "wrong-kind-old"}} + ] + }, + "variants": [ + {"name": "returns missing null and old HasSession relationship IDs", "vars": {"projection": "id(r)"}, "assert": {"keys": ["id(r)"], "row_count": 3}}, + {"name": "exact missing null old equal newer and wrong-kind matrix", "vars": {"projection": "r.marker"}, "assert": {"scalar_values": ["session-missing", "session-null", "session-old"]}} + ] + }, + { + "name": "PRUNE-03 stale or unobserved node selection with protected kinds", + "template": "MATCH (n) WHERE NOT (n:Domain OR n:Tenant OR n:Meta OR n:MetaIncludes OR n:MigrationData) AND (n.lastseen IS NULL OR datetime(n.lastseen) < datetime($threshold)) RETURN {{projection}}", + "params": {"threshold": "2026-01-03T00:00:00Z"}, + "fixture": { + "nodes": [ + {"id": "candidate-missing", "kinds": ["CandidateNode"], "properties": {"name": "candidate-missing"}}, + {"id": "candidate-null", "kinds": ["CandidateNode"], "properties": {"name": "candidate-null", "lastseen": null}}, + {"id": "candidate-old", "kinds": ["CandidateNode"], "properties": {"name": "candidate-old", "lastseen": "2026-01-02T00:00:00Z"}}, + {"id": "candidate-equal", "kinds": ["CandidateNode"], "properties": {"name": "candidate-equal", "lastseen": "2026-01-03T00:00:00Z"}}, + {"id": "candidate-new", "kinds": ["CandidateNode"], "properties": {"name": "candidate-new", "lastseen": "2026-01-04T00:00:00Z"}}, + {"id": "domain-missing", "kinds": ["Domain"], "properties": {"name": "domain-missing"}}, + {"id": "tenant-old", "kinds": ["Tenant"], "properties": {"name": "tenant-old", "lastseen": "2026-01-02T00:00:00Z"}}, + {"id": "meta-null", "kinds": ["Meta"], "properties": {"name": "meta-null", "lastseen": null}}, + {"id": "meta-includes-old", "kinds": ["MetaIncludes"], "properties": {"name": "meta-includes-old", "lastseen": "2026-01-02T00:00:00Z"}}, + {"id": "migration-old", "kinds": ["MigrationData"], "properties": {"name": "migration-old", "lastseen": "2026-01-02T00:00:00Z"}}, + {"id": "multi-kind-protected", "kinds": ["CandidateNode", "Domain"], "properties": {"name": "multi-kind-protected", "lastseen": "2026-01-02T00:00:00Z"}} + ] + }, + "variants": [ + {"name": "returns only candidate missing null and old node IDs", "vars": {"projection": "id(n)"}, "assert": {"keys": ["id(n)"], "row_count": 3}}, + {"name": "exact protected multi-kind and age matrix", "vars": {"projection": "n"}, "assert": {"node_id_set": ["candidate-missing", "candidate-null", "candidate-old"]}} + ] + }, + { + "name": "PRUNE-04 orphan SID node selection", + "template": "MATCH (n) WHERE NOT (n:Domain OR n:Tenant OR n:Meta OR n:MetaIncludes OR n:MigrationData) AND n.name IS NULL AND n.objectid STARTS WITH $sid_prefix RETURN {{projection}}", + "params": {"sid_prefix": "S-1-5"}, + "fixture": { + "nodes": [ + {"id": "missing-name", "kinds": ["CandidateNode"], "properties": {"objectid": "S-1-5-100"}}, + {"id": "null-name", "kinds": ["CandidateNode"], "properties": {"name": null, "objectid": "S-1-5-101"}}, + {"id": "empty-name", "kinds": ["CandidateNode"], "properties": {"name": "", "objectid": "S-1-5-102"}}, + {"id": "named", "kinds": ["CandidateNode"], "properties": {"name": "named", "objectid": "S-1-5-103"}}, + {"id": "wrong-prefix", "kinds": ["CandidateNode"], "properties": {"objectid": "X-1-5-104"}}, + {"id": "missing-objectid", "kinds": ["CandidateNode"], "properties": {}}, + {"id": "domain", "kinds": ["Domain"], "properties": {"objectid": "S-1-5-105"}}, + {"id": "tenant", "kinds": ["Tenant"], "properties": {"name": null, "objectid": "S-1-5-106"}}, + {"id": "multi-kind-protected", "kinds": ["CandidateNode", "MigrationData"], "properties": {"objectid": "S-1-5-107"}} + ] + }, + "variants": [ + {"name": "returns only missing and null name SID node IDs", "vars": {"projection": "id(n)"}, "assert": {"keys": ["id(n)"], "row_count": 2}}, + {"name": "exact prefix name and protected-kind matrix", "vars": {"projection": "n"}, "assert": {"node_id_set": ["missing-name", "null-name"]}} + ] + } + ] +} diff --git a/integration/testdata/templates/reconciliation_shapes.json b/integration/testdata/templates/reconciliation_shapes.json new file mode 100644 index 00000000..d11ca7e6 --- /dev/null +++ b/integration/testdata/templates/reconciliation_shapes.json @@ -0,0 +1,815 @@ +{ + "families": [ + { + "name": "LOGIC-01 branch-local relationship kinds", + "template": "MATCH (s:LogicDomain)-[r]->(e:LogicDomain) WHERE (id(s) = $forward_start AND id(e) = $forward_end AND r:LogicKindA) OR (id(s) = $forward_end AND id(e) = $forward_start AND r:LogicKindB) RETURN r.marker", + "node_params": {"forward_start": "forward", "forward_end": "reverse"}, + "fixture": { + "nodes": [ + {"id": "forward", "kinds": ["LogicDomain"], "properties": {"name": "forward"}}, + {"id": "reverse", "kinds": ["LogicDomain"], "properties": {"name": "reverse"}} + ], + "edges": [ + {"start_id": "forward", "end_id": "reverse", "kind": "LogicKindA", "properties": {"marker": "valid-forward"}}, + {"start_id": "reverse", "end_id": "forward", "kind": "LogicKindB", "properties": {"marker": "valid-reverse"}}, + {"start_id": "forward", "end_id": "reverse", "kind": "LogicKindB", "properties": {"marker": "invalid-forward-kind"}}, + {"start_id": "reverse", "end_id": "forward", "kind": "LogicKindA", "properties": {"marker": "invalid-reverse-kind"}} + ] + }, + "variants": [ + { + "name": "both valid combinations exclude both invalid cross-combinations", + "assert": {"scalar_values": ["valid-forward", "valid-reverse"]} + } + ] + }, + { + "name": "LOGIC-02 cross-binding temporal disjunction", + "template": "MATCH (s:LogicDomain)-[r:LogicStaleTrust]->(e:LogicDomain) WHERE r.lastseen < s.lastcollected OR r.lastseen < e.lastcollected RETURN r.marker", + "fixture": { + "nodes": [ + {"id": "early-a", "kinds": ["LogicDomain"], "properties": {"lastcollected": "2026-01-02T00:00:00Z"}}, + {"id": "early-b", "kinds": ["LogicDomain"], "properties": {"lastcollected": "2026-01-02T00:00:00Z"}}, + {"id": "equal-a", "kinds": ["LogicDomain"], "properties": {"lastcollected": "2026-01-03T00:00:00Z"}}, + {"id": "equal-b", "kinds": ["LogicDomain"], "properties": {"lastcollected": "2026-01-03T00:00:00Z"}}, + {"id": "late-a", "kinds": ["LogicDomain"], "properties": {"lastcollected": "2026-01-04T00:00:00Z"}}, + {"id": "late-b", "kinds": ["LogicDomain"], "properties": {"lastcollected": "2026-01-04T00:00:00Z"}}, + {"id": "late-b-newer", "kinds": ["LogicDomain"], "properties": {"lastcollected": "2026-01-04T00:00:00Z"}}, + {"id": "late-b-missing-relationship", "kinds": ["LogicDomain"], "properties": {"lastcollected": "2026-01-04T00:00:00Z"}}, + {"id": "late-b-null-relationship", "kinds": ["LogicDomain"], "properties": {"lastcollected": "2026-01-04T00:00:00Z"}}, + {"id": "missing-a", "kinds": ["LogicDomain"], "properties": {}}, + {"id": "missing-b", "kinds": ["LogicDomain"], "properties": {}}, + {"id": "null-a", "kinds": ["LogicDomain"], "properties": {"lastcollected": null}}, + {"id": "null-b", "kinds": ["LogicDomain"], "properties": {"lastcollected": null}} + ], + "edges": [ + {"start_id": "late-a", "end_id": "early-a", "kind": "LogicStaleTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "older-start-only"}}, + {"start_id": "early-a", "end_id": "late-a", "kind": "LogicStaleTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "older-end-only"}}, + {"start_id": "late-a", "end_id": "late-b", "kind": "LogicStaleTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "older-both"}}, + {"start_id": "equal-a", "end_id": "equal-b", "kind": "LogicStaleTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "equal"}}, + {"start_id": "late-a", "end_id": "late-b-newer", "kind": "LogicStaleTrust", "properties": {"lastseen": "2026-01-05T00:00:00Z", "marker": "newer"}}, + {"start_id": "late-a", "end_id": "late-b-missing-relationship", "kind": "LogicStaleTrust", "properties": {"marker": "missing-relationship"}}, + {"start_id": "late-a", "end_id": "late-b-null-relationship", "kind": "LogicStaleTrust", "properties": {"lastseen": null, "marker": "null-relationship"}}, + {"start_id": "missing-a", "end_id": "late-a", "kind": "LogicStaleTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "missing-start-valid-end"}}, + {"start_id": "null-a", "end_id": "late-a", "kind": "LogicStaleTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "null-start-valid-end"}}, + {"start_id": "late-a", "end_id": "missing-b", "kind": "LogicStaleTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "missing-end-valid-start"}}, + {"start_id": "late-a", "end_id": "null-b", "kind": "LogicStaleTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "null-end-valid-start"}}, + {"start_id": "missing-a", "end_id": "null-b", "kind": "LogicStaleTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "missing-and-null-endpoints"}} + ] + }, + "variants": [ + { + "name": "older equal newer missing and null truth table", + "assert": { + "scalar_values": [ + "older-start-only", + "older-end-only", + "older-both", + "missing-start-valid-end", + "null-start-valid-end", + "missing-end-valid-start", + "null-end-valid-start" + ] + } + } + ] + }, + { + "name": "LOGIC-04 filtered relationship delete", + "template": "MATCH (s:LogicDeleteSource)-[r:LogicDeleteEdge]->(e:LogicDeleteTarget) WHERE e.objectid = $object_id AND r.shoulddelete = $should_delete DELETE r", + "params": {"object_id": "delete-edge", "should_delete": true}, + "fixture": { + "nodes": [ + {"id": "source", "kinds": ["LogicDeleteSource"], "properties": {"name": "source"}}, + {"id": "source-property-decoy", "kinds": ["LogicDeleteSource"], "properties": {"name": "source-property-decoy"}}, + {"id": "target", "kinds": ["LogicDeleteTarget"], "properties": {"objectid": "delete-edge"}}, + {"id": "decoy-target", "kinds": ["LogicDeleteTarget"], "properties": {"objectid": "keep-edge"}} + ], + "edges": [ + {"start_id": "source", "end_id": "target", "kind": "LogicDeleteEdge", "properties": {"shoulddelete": true, "marker": "delete"}}, + {"start_id": "source-property-decoy", "end_id": "target", "kind": "LogicDeleteEdge", "properties": {"shoulddelete": false, "marker": "property-decoy"}}, + {"start_id": "source", "end_id": "target", "kind": "LogicSurvivorEdge", "properties": {"shoulddelete": true, "marker": "kind-decoy"}}, + {"start_id": "source", "end_id": "decoy-target", "kind": "LogicDeleteEdge", "properties": {"shoulddelete": true, "marker": "endpoint-decoy"}}, + {"start_id": "target", "end_id": "source", "kind": "LogicDeleteEdge", "properties": {"shoulddelete": true, "marker": "direction-decoy"}} + ] + }, + "variants": [ + { + "name": "selected relationship binding is deleted and every decoy survives", + "assert": "no_error", + "post_assertions": [ + { + "cypher": "MATCH ()-[r]->() RETURN r", + "assert": { + "relationship_records": [ + {"start": "source-property-decoy", "end": "target", "kind": "LogicDeleteEdge", "props": {"shoulddelete": false, "marker": "property-decoy"}}, + {"start": "source", "end": "target", "kind": "LogicSurvivorEdge", "props": {"shoulddelete": true, "marker": "kind-decoy"}}, + {"start": "source", "end": "decoy-target", "kind": "LogicDeleteEdge", "props": {"shoulddelete": true, "marker": "endpoint-decoy"}}, + {"start": "target", "end": "source", "kind": "LogicDeleteEdge", "props": {"shoulddelete": true, "marker": "direction-decoy"}} + ] + } + } + ] + } + ] + }, + { + "name": "LOGIC-04 filtered detach node delete", + "template": "MATCH (n:LogicDeleteNode) WHERE n.objectid = $object_id DETACH DELETE n", + "params": {"object_id": "delete-node"}, + "fixture": { + "nodes": [ + {"id": "victim", "kinds": ["LogicDeleteNode"], "properties": {"objectid": "delete-node"}}, + {"id": "survivor", "kinds": ["LogicSurvivorNode"], "properties": {"objectid": "keep-node"}}, + {"id": "kind-decoy", "kinds": ["LogicSurvivorNode"], "properties": {"objectid": "delete-node"}}, + {"id": "property-decoy", "kinds": ["LogicDeleteNode"], "properties": {"objectid": "keep-node"}} + ], + "edges": [ + {"start_id": "survivor", "end_id": "victim", "kind": "LogicIncident", "properties": {"marker": "inbound"}}, + {"start_id": "victim", "end_id": "survivor", "kind": "LogicIncident", "properties": {"marker": "outbound"}}, + {"start_id": "victim", "end_id": "victim", "kind": "LogicIncident", "properties": {"marker": "self"}}, + {"start_id": "survivor", "end_id": "property-decoy", "kind": "LogicSurvivorEdge", "properties": {"marker": "survives"}} + ] + }, + "variants": [ + { + "name": "selected node binding cascades only its incident relationships", + "assert": "no_error", + "post_assertions": [ + {"cypher": "MATCH (n) RETURN n", "assert": {"node_id_set": ["survivor", "kind-decoy", "property-decoy"]}}, + {"cypher": "MATCH ()-[r]->() RETURN r", "assert": {"relationship_records": [{"start": "survivor", "end": "property-decoy", "kind": "LogicSurvivorEdge", "props": {"marker": "survives"}}]}} + ] + } + ] + }, + { + "name": "LOGIC-05 directional projection order", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "start", "kinds": ["LogicProjectionStart"], "properties": {"name": "start"}}, + {"id": "end", "kinds": ["LogicProjectionEnd", "LogicProjectionEntity"], "properties": {"name": "end"}} + ], + "edges": [ + {"start_id": "start", "end_id": "end", "kind": "LogicProjectionEdge", "properties": {"marker": "projection"}} + ] + }, + "variants": [ + { + "name": "full opposite node plus relationship", + "vars": {"query": "MATCH ()-[r:LogicProjectionEdge]->(e) RETURN r, e"}, + "assert": {"keys": ["r", "e"], "row_count": 1, "contains_edge": {"start": "start", "end": "end", "kind": "LogicProjectionEdge", "props": {"marker": "projection"}}} + }, + { + "name": "opposite ID kinds and relationship ID kind", + "vars": {"query": "MATCH ()-[r:LogicProjectionEdge]->(e) RETURN id(e), labels(e), id(r), type(r)"}, + "assert": {"keys": ["id(e)", "labels(e)", "id(r)", "type(r)"], "row_count": 1} + }, + { + "name": "start relationship end triple", + "vars": {"query": "MATCH (s)-[r:LogicProjectionEdge]->(e) RETURN s, r, e"}, + "assert": {"keys": ["s", "r", "e"], "row_count": 1, "contains_edge": {"start": "start", "end": "end", "kind": "LogicProjectionEdge"}} + }, + { + "name": "relationship ID only", + "vars": {"query": "MATCH ()-[r:LogicProjectionEdge]->() RETURN id(r)"}, + "assert": {"keys": ["id(r)"], "row_count": 1} + }, + { + "name": "full relationship", + "vars": {"query": "MATCH ()-[r:LogicProjectionEdge]->() RETURN r"}, + "assert": {"keys": ["r"], "row_count": 1, "contains_edge": {"start": "start", "end": "end", "kind": "LogicProjectionEdge", "props": {"marker": "projection"}}} + } + ] + }, + { + "name": "REC-01 inbound structure reconciliation delete", + "template": "MATCH ()-[r:{{relationship_kinds}}]->(e:ADEntity) WHERE e.objectid = $object_id DELETE r", + "fixture": { + "nodes": [ + {"id": "source-a", "kinds": ["Source"], "properties": {"name": "source-a"}}, + {"id": "source-b", "kinds": ["Source"], "properties": {"name": "source-b"}}, + {"id": "target", "kinds": ["ADEntity", "Group"], "properties": {"objectid": "target-in"}}, + {"id": "one-target", "kinds": ["ADEntity"], "properties": {"objectid": "one-in"}}, + {"id": "wrong-kind", "kinds": ["OtherEntity"], "properties": {"objectid": "target-in"}}, + {"id": "wrong-property", "kinds": ["ADEntity"], "properties": {"objectid": "other-in"}} + ], + "edges": [ + {"start_id": "source-a", "end_id": "target", "kind": "RecKind01", "properties": {"marker": "k01-a"}}, + {"start_id": "source-b", "end_id": "target", "kind": "RecKind01", "properties": {"marker": "k01-b"}}, + {"start_id": "source-a", "end_id": "target", "kind": "RecKind02", "properties": {"marker": "k02"}}, + {"start_id": "source-a", "end_id": "target", "kind": "RecKind09", "properties": {"marker": "k09"}}, + {"start_id": "source-a", "end_id": "target", "kind": "RecKind30", "properties": {"marker": "k30"}}, + {"start_id": "source-a", "end_id": "target", "kind": "RecKind31", "properties": {"marker": "wrong-edge"}}, + {"start_id": "source-a", "end_id": "wrong-kind", "kind": "RecKind01", "properties": {"marker": "wrong-end-kind"}}, + {"start_id": "source-a", "end_id": "wrong-property", "kind": "RecKind01", "properties": {"marker": "wrong-end-property"}}, + {"start_id": "target", "end_id": "source-a", "kind": "RecKind01", "properties": {"marker": "wrong-direction"}}, + {"start_id": "source-a", "end_id": "one-target", "kind": "RecKind01", "properties": {"marker": "one-match"}} + ] + }, + "variants": [ + { + "name": "one kind deletes many exact matches", + "vars": {"relationship_kinds": "RecKind01"}, + "params": {"object_id": "target-in"}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["k02", "k09", "k30", "wrong-edge", "wrong-end-kind", "wrong-end-property", "wrong-direction", "one-match"]}}] + }, + { + "name": "two kinds preserve nonselected kinds and decoys", + "vars": {"relationship_kinds": "RecKind01|RecKind02"}, + "params": {"object_id": "target-in"}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["k09", "k30", "wrong-edge", "wrong-end-kind", "wrong-end-property", "wrong-direction", "one-match"]}}] + }, + { + "name": "nine kinds include the ninth kind", + "vars": {"relationship_kinds": "RecKind01|RecKind02|RecKind03|RecKind04|RecKind05|RecKind06|RecKind07|RecKind08|RecKind09"}, + "params": {"object_id": "target-in"}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["k30", "wrong-edge", "wrong-end-kind", "wrong-end-property", "wrong-direction", "one-match"]}}] + }, + { + "name": "thirty kinds include the thirtieth kind", + "vars": {"relationship_kinds": "RecKind01|RecKind02|RecKind03|RecKind04|RecKind05|RecKind06|RecKind07|RecKind08|RecKind09|RecKind10|RecKind11|RecKind12|RecKind13|RecKind14|RecKind15|RecKind16|RecKind17|RecKind18|RecKind19|RecKind20|RecKind21|RecKind22|RecKind23|RecKind24|RecKind25|RecKind26|RecKind27|RecKind28|RecKind29|RecKind30"}, + "params": {"object_id": "target-in"}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["wrong-edge", "wrong-end-kind", "wrong-end-property", "wrong-direction", "one-match"]}}] + }, + { + "name": "single exact match", + "vars": {"relationship_kinds": "RecKind01"}, + "params": {"object_id": "one-in"}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["k01-a", "k01-b", "k02", "k09", "k30", "wrong-edge", "wrong-end-kind", "wrong-end-property", "wrong-direction"]}}] + }, + { + "name": "zero matches preserve every relationship", + "vars": {"relationship_kinds": "RecKind01"}, + "params": {"object_id": "missing-in"}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["k01-a", "k01-b", "k02", "k09", "k30", "wrong-edge", "wrong-end-kind", "wrong-end-property", "wrong-direction", "one-match"]}}] + } + ] + }, + { + "name": "REC-01 and REC-02 thirty-kind schema registry", + "template": "MATCH ()-[r]->() RETURN count(r)", + "fixture": { + "nodes": [ + {"id": "start", "kinds": ["RegistryStart"], "properties": {"name": "start"}}, + {"id": "end", "kinds": ["RegistryEnd"], "properties": {"name": "end"}} + ], + "edges": [ + {"start_id": "start", "end_id": "end", "kind": "RecKind03"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind04"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind05"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind06"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind07"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind08"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind10"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind11"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind12"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind13"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind14"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind15"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind16"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind17"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind18"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind19"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind20"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind21"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind22"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind23"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind24"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind25"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind26"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind27"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind28"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind29"} + ] + }, + "variants": [ + {"name": "register every nonmatching relationship kind", "assert": {"exact_int": 26}} + ] + }, + { + "name": "REC-02 outbound structure reconciliation delete", + "template": "MATCH (s:ADEntity)-[r:{{relationship_kinds}}]->() WHERE s.objectid = $object_id DELETE r", + "fixture": { + "nodes": [ + {"id": "target", "kinds": ["ADEntity", "Computer"], "properties": {"objectid": "target-out"}}, + {"id": "one-target", "kinds": ["ADEntity"], "properties": {"objectid": "one-out"}}, + {"id": "wrong-kind", "kinds": ["OtherEntity"], "properties": {"objectid": "target-out"}}, + {"id": "wrong-property", "kinds": ["ADEntity"], "properties": {"objectid": "other-out"}}, + {"id": "end-a", "kinds": ["Destination"], "properties": {"name": "end-a"}}, + {"id": "end-b", "kinds": ["Destination"], "properties": {"name": "end-b"}} + ], + "edges": [ + {"start_id": "target", "end_id": "end-a", "kind": "RecKind01", "properties": {"marker": "out-k01-a"}}, + {"start_id": "target", "end_id": "end-b", "kind": "RecKind01", "properties": {"marker": "out-k01-b"}}, + {"start_id": "target", "end_id": "end-a", "kind": "RecKind02", "properties": {"marker": "out-k02"}}, + {"start_id": "target", "end_id": "end-a", "kind": "RecKind09", "properties": {"marker": "out-k09"}}, + {"start_id": "target", "end_id": "end-a", "kind": "RecKind30", "properties": {"marker": "out-k30"}}, + {"start_id": "target", "end_id": "end-a", "kind": "RecKind31", "properties": {"marker": "out-wrong-edge"}}, + {"start_id": "wrong-kind", "end_id": "end-a", "kind": "RecKind01", "properties": {"marker": "out-wrong-start-kind"}}, + {"start_id": "wrong-property", "end_id": "end-a", "kind": "RecKind01", "properties": {"marker": "out-wrong-start-property"}}, + {"start_id": "end-a", "end_id": "target", "kind": "RecKind01", "properties": {"marker": "out-wrong-direction"}}, + {"start_id": "one-target", "end_id": "end-a", "kind": "RecKind01", "properties": {"marker": "out-one-match"}} + ] + }, + "variants": [ + { + "name": "one kind deletes many exact outbound matches", + "vars": {"relationship_kinds": "RecKind01"}, + "params": {"object_id": "target-out"}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["out-k02", "out-k09", "out-k30", "out-wrong-edge", "out-wrong-start-kind", "out-wrong-start-property", "out-wrong-direction", "out-one-match"]}}] + }, + { + "name": "two outbound kinds", + "vars": {"relationship_kinds": "RecKind01|RecKind02"}, + "params": {"object_id": "target-out"}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["out-k09", "out-k30", "out-wrong-edge", "out-wrong-start-kind", "out-wrong-start-property", "out-wrong-direction", "out-one-match"]}}] + }, + { + "name": "nine outbound kinds", + "vars": {"relationship_kinds": "RecKind01|RecKind02|RecKind03|RecKind04|RecKind05|RecKind06|RecKind07|RecKind08|RecKind09"}, + "params": {"object_id": "target-out"}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["out-k30", "out-wrong-edge", "out-wrong-start-kind", "out-wrong-start-property", "out-wrong-direction", "out-one-match"]}}] + }, + { + "name": "thirty outbound kinds", + "vars": {"relationship_kinds": "RecKind01|RecKind02|RecKind03|RecKind04|RecKind05|RecKind06|RecKind07|RecKind08|RecKind09|RecKind10|RecKind11|RecKind12|RecKind13|RecKind14|RecKind15|RecKind16|RecKind17|RecKind18|RecKind19|RecKind20|RecKind21|RecKind22|RecKind23|RecKind24|RecKind25|RecKind26|RecKind27|RecKind28|RecKind29|RecKind30"}, + "params": {"object_id": "target-out"}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["out-wrong-edge", "out-wrong-start-kind", "out-wrong-start-property", "out-wrong-direction", "out-one-match"]}}] + }, + { + "name": "single outbound exact match", + "vars": {"relationship_kinds": "RecKind01"}, + "params": {"object_id": "one-out"}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["out-k01-a", "out-k01-b", "out-k02", "out-k09", "out-k30", "out-wrong-edge", "out-wrong-start-kind", "out-wrong-start-property", "out-wrong-direction"]}}] + }, + { + "name": "zero outbound matches preserve every relationship", + "vars": {"relationship_kinds": "RecKind01"}, + "params": {"object_id": "missing-out"}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["out-k01-a", "out-k01-b", "out-k02", "out-k09", "out-k30", "out-wrong-edge", "out-wrong-start-kind", "out-wrong-start-property", "out-wrong-direction", "out-one-match"]}}] + } + ] + }, + { + "name": "REC-03 primary group reconciliation delete", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "user", "kinds": ["ADEntity", "User"], "properties": {"objectid": "user-id"}}, + {"id": "user-opposite", "kinds": ["ADEntity", "User"], "properties": {"objectid": "user-opposite-id"}}, + {"id": "user-missing", "kinds": ["ADEntity", "User"], "properties": {"objectid": "user-missing-id"}}, + {"id": "group", "kinds": ["ADEntity", "Group"], "properties": {"objectid": "group-id"}}, + {"id": "computer", "kinds": ["ADEntity", "Computer"], "properties": {"objectid": "computer-id"}}, + {"id": "other", "kinds": ["ADEntity", "Group"], "properties": {"objectid": "other-id"}}, + {"id": "out-opposite-target", "kinds": ["ADEntity", "Group"], "properties": {"objectid": "out-opposite-id"}}, + {"id": "out-missing-target", "kinds": ["ADEntity", "Group"], "properties": {"objectid": "out-missing-id"}} + ], + "edges": [ + {"start_id": "user", "end_id": "group", "kind": "MemberOf", "properties": {"isprimarygroup": false, "marker": "in-false"}}, + {"start_id": "user-opposite", "end_id": "group", "kind": "MemberOf", "properties": {"isprimarygroup": true, "marker": "in-opposite"}}, + {"start_id": "user-missing", "end_id": "group", "kind": "MemberOf", "properties": {"marker": "in-missing"}}, + {"start_id": "user", "end_id": "group", "kind": "OtherMembership", "properties": {"isprimarygroup": false, "marker": "in-wrong-kind"}}, + {"start_id": "computer", "end_id": "group", "kind": "MemberOf", "properties": {"isprimarygroup": true, "marker": "out-true-a"}}, + {"start_id": "computer", "end_id": "other", "kind": "MemberOf", "properties": {"isprimarygroup": true, "marker": "out-true-b"}}, + {"start_id": "computer", "end_id": "out-opposite-target", "kind": "MemberOf", "properties": {"isprimarygroup": false, "marker": "out-opposite"}}, + {"start_id": "computer", "end_id": "out-missing-target", "kind": "MemberOf", "properties": {"marker": "out-missing"}}, + {"start_id": "computer", "end_id": "group", "kind": "OtherMembership", "properties": {"isprimarygroup": true, "marker": "out-wrong-kind"}}, + {"start_id": "group", "end_id": "computer", "kind": "MemberOf", "properties": {"isprimarygroup": true, "marker": "out-wrong-direction"}} + ] + }, + "variants": [ + { + "name": "inbound false deletes only the matching MemberOf edge", + "vars": {"query": "MATCH ()-[r:MemberOf]->(e:Group) WHERE e.objectid = $object_id AND r.isprimarygroup = $flag DELETE r"}, + "params": {"object_id": "group-id", "flag": false}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["in-opposite", "in-missing", "in-wrong-kind", "out-true-a", "out-true-b", "out-opposite", "out-missing", "out-wrong-kind", "out-wrong-direction"]}}] + }, + { + "name": "outbound true deletes all exact MemberOf edges", + "vars": {"query": "MATCH (s:Computer)-[r:MemberOf]->() WHERE s.objectid = $object_id AND r.isprimarygroup = $flag DELETE r"}, + "params": {"object_id": "computer-id", "flag": true}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["in-false", "in-opposite", "in-missing", "in-wrong-kind", "out-opposite", "out-missing", "out-wrong-kind", "out-wrong-direction"]}}] + } + ] + }, + { + "name": "REC-04 endpoint object ID list reconciliation delete", + "template": "MATCH ()-[r:{{relationship_kind}}]->(e:{{entity_kind}}) WHERE e.objectid IN $object_ids DELETE r", + "fixture": { + "nodes": [ + {"id": "source", "kinds": ["Source"], "properties": {"name": "source"}}, + {"id": "source-duplicate", "kinds": ["Source"], "properties": {"name": "source-duplicate"}}, + {"id": "ad-a", "kinds": ["ADEntity", "Computer"], "properties": {"objectid": "ad-a"}}, + {"id": "ad-b", "kinds": ["ADEntity", "User"], "properties": {"objectid": "ad-b"}}, + {"id": "az-a", "kinds": ["AZEntity", "AZUser"], "properties": {"objectid": "az-a"}}, + {"id": "az-b", "kinds": ["AZEntity", "AZGroup"], "properties": {"objectid": "az-b"}}, + {"id": "wrong-kind", "kinds": ["OtherEntity"], "properties": {"objectid": "ad-a"}}, + {"id": "wrong-property", "kinds": ["ADEntity"], "properties": {"objectid": "other"}} + ], + "edges": [ + {"start_id": "source", "end_id": "ad-a", "kind": "ADReconcile", "properties": {"marker": "ad-a-1"}}, + {"start_id": "source-duplicate", "end_id": "ad-a", "kind": "ADReconcile", "properties": {"marker": "ad-a-2"}}, + {"start_id": "source", "end_id": "ad-b", "kind": "ADReconcile", "properties": {"marker": "ad-b"}}, + {"start_id": "source", "end_id": "az-a", "kind": "AZReconcile", "properties": {"marker": "az-a"}}, + {"start_id": "source", "end_id": "az-b", "kind": "AZReconcile", "properties": {"marker": "az-b"}}, + {"start_id": "source", "end_id": "wrong-kind", "kind": "ADReconcile", "properties": {"marker": "wrong-kind-end"}}, + {"start_id": "source", "end_id": "wrong-property", "kind": "ADReconcile", "properties": {"marker": "wrong-property"}}, + {"start_id": "source", "end_id": "ad-a", "kind": "OtherReconcile", "properties": {"marker": "wrong-edge"}}, + {"start_id": "ad-a", "end_id": "source", "kind": "ADReconcile", "properties": {"marker": "wrong-direction"}} + ] + }, + "variants": [ + { + "name": "empty AD list preserves every relationship", + "vars": {"relationship_kind": "ADReconcile", "entity_kind": "ADEntity"}, + "params": {"object_ids": []}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["ad-a-1", "ad-a-2", "ad-b", "az-a", "az-b", "wrong-kind-end", "wrong-property", "wrong-edge", "wrong-direction"]}}] + }, + { + "name": "singleton AD list deletes all duplicate matches", + "vars": {"relationship_kind": "ADReconcile", "entity_kind": "ADEntity"}, + "params": {"object_ids": ["ad-a"]}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["ad-b", "az-a", "az-b", "wrong-kind-end", "wrong-property", "wrong-edge", "wrong-direction"]}}] + }, + { + "name": "duplicate AD IDs do not widen the delete", + "vars": {"relationship_kind": "ADReconcile", "entity_kind": "ADEntity"}, + "params": {"object_ids": ["ad-a", "ad-a"]}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["ad-b", "az-a", "az-b", "wrong-kind-end", "wrong-property", "wrong-edge", "wrong-direction"]}}] + }, + { + "name": "small AD list deletes both selected endpoints", + "vars": {"relationship_kind": "ADReconcile", "entity_kind": "ADEntity"}, + "params": {"object_ids": ["ad-a", "ad-b"]}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["az-a", "az-b", "wrong-kind-end", "wrong-property", "wrong-edge", "wrong-direction"]}}] + }, + { + "name": "one thousand AD IDs preserve exact selection", + "vars": {"relationship_kind": "ADReconcile", "entity_kind": "ADEntity"}, + "params": {"object_ids": {"$type": "string_list", "prefix": "missing-ad", "count": 998, "include": ["ad-a", "ad-b"]}}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["az-a", "az-b", "wrong-kind-end", "wrong-property", "wrong-edge", "wrong-direction"]}}] + }, + { + "name": "large AD list preserves exact selection", + "vars": {"relationship_kind": "ADReconcile", "entity_kind": "ADEntity"}, + "params": {"object_ids": {"$type": "string_list", "prefix": "large-ad", "count": 1999, "include": ["ad-a", "ad-b"]}}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["az-a", "az-b", "wrong-kind-end", "wrong-property", "wrong-edge", "wrong-direction"]}}] + }, + { + "name": "one thousand no-match IDs preserve every relationship", + "vars": {"relationship_kind": "ADReconcile", "entity_kind": "ADEntity"}, + "params": {"object_ids": {"$type": "string_list", "prefix": "no-match-ad", "count": 1000}}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["ad-a-1", "ad-a-2", "ad-b", "az-a", "az-b", "wrong-kind-end", "wrong-property", "wrong-edge", "wrong-direction"]}}] + }, + { + "name": "Azure base kind and relationship kind remain isolated", + "vars": {"relationship_kind": "AZReconcile", "entity_kind": "AZEntity"}, + "params": {"object_ids": ["az-a", "az-b"]}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["ad-a-1", "ad-a-2", "ad-b", "wrong-kind-end", "wrong-property", "wrong-edge", "wrong-direction"]}}] + } + ] + }, + { + "name": "REC-05 delegated enrollment discovery", + "template": "MATCH (s:CertTemplate)-[r:PublishedTo]->(e) WHERE e.objectid IN $ca_ids RETURN r, s", + "fixture": { + "nodes": [ + {"id": "template-a", "kinds": ["CertTemplate"], "properties": {"objectid": "template-a"}}, + {"id": "template-b", "kinds": ["CertTemplate"], "properties": {"objectid": "template-b"}}, + {"id": "wrong-start", "kinds": ["OtherTemplate"], "properties": {"objectid": "wrong-start"}}, + {"id": "ca-a", "kinds": ["EnterpriseCA"], "properties": {"objectid": "ca-a"}}, + {"id": "ca-b", "kinds": ["EnterpriseCA"], "properties": {"objectid": "ca-b"}}, + {"id": "wrong-property", "kinds": ["EnterpriseCA"], "properties": {"objectid": "other-ca"}} + ], + "edges": [ + {"start_id": "template-a", "end_id": "ca-a", "kind": "PublishedTo", "properties": {"marker": "published-a"}}, + {"start_id": "template-a", "end_id": "ca-b", "kind": "PublishedTo", "properties": {"marker": "published-b"}}, + {"start_id": "template-b", "end_id": "ca-a", "kind": "PublishedTo", "properties": {"marker": "published-c"}}, + {"start_id": "wrong-start", "end_id": "ca-a", "kind": "PublishedTo", "properties": {"marker": "wrong-start"}}, + {"start_id": "template-a", "end_id": "ca-a", "kind": "OtherPublication", "properties": {"marker": "wrong-edge"}}, + {"start_id": "template-a", "end_id": "wrong-property", "kind": "PublishedTo", "properties": {"marker": "wrong-property"}} + ] + }, + "variants": [ + {"name": "empty CA list", "params": {"ca_ids": []}, "assert": {"row_count": 0}}, + { + "name": "single CA retains every raw relationship row", + "params": {"ca_ids": ["ca-a"]}, + "assert": {"row_count": 2, "node_id_set": ["template-a", "template-b"], "relationship_records": [ + {"start": "template-a", "end": "ca-a", "kind": "PublishedTo", "props": {"marker": "published-a"}}, + {"start": "template-b", "end": "ca-a", "kind": "PublishedTo", "props": {"marker": "published-c"}} + ]} + }, + { + "name": "duplicate paths retain rows and expose a deduplicated node set", + "params": {"ca_ids": ["ca-a", "ca-b"]}, + "assert": {"row_count": 3, "node_id_set": ["template-a", "template-b"], "relationship_records": [ + {"start": "template-a", "end": "ca-a", "kind": "PublishedTo", "props": {"marker": "published-a"}}, + {"start": "template-a", "end": "ca-b", "kind": "PublishedTo", "props": {"marker": "published-b"}}, + {"start": "template-b", "end": "ca-a", "kind": "PublishedTo", "props": {"marker": "published-c"}} + ]} + }, + { + "name": "large CA list retains the same exact raw rows", + "params": {"ca_ids": {"$type": "string_list", "prefix": "missing-ca", "count": 1999, "include": ["ca-a", "ca-b"]}}, + "assert": {"row_count": 3, "node_id_set": ["template-a", "template-b"], "relationship_records": [ + {"start": "template-a", "end": "ca-a", "kind": "PublishedTo", "props": {"marker": "published-a"}}, + {"start": "template-a", "end": "ca-b", "kind": "PublishedTo", "props": {"marker": "published-b"}}, + {"start": "template-b", "end": "ca-a", "kind": "PublishedTo", "props": {"marker": "published-c"}} + ]} + } + ] + }, + { + "name": "REC-06 delegated enrollment relationship delete", + "template": "MATCH ()-[r:DelegatedEnrollmentAgent]->(e:CertTemplate) WHERE id(e) IN $template_ids DELETE r", + "fixture": { + "nodes": [ + {"id": "agent", "kinds": ["ADEntity"], "properties": {"objectid": "agent"}}, + {"id": "agent-duplicate", "kinds": ["ADEntity"], "properties": {"objectid": "agent-duplicate"}}, + {"id": "template-a", "kinds": ["CertTemplate"], "properties": {"objectid": "template-a"}}, + {"id": "template-b", "kinds": ["CertTemplate"], "properties": {"objectid": "template-b"}}, + {"id": "wrong-end", "kinds": ["OtherTemplate"], "properties": {"objectid": "wrong-end"}} + ], + "edges": [ + {"start_id": "agent", "end_id": "template-a", "kind": "DelegatedEnrollmentAgent", "properties": {"marker": "dea-a-1"}}, + {"start_id": "agent-duplicate", "end_id": "template-a", "kind": "DelegatedEnrollmentAgent", "properties": {"marker": "dea-a-2"}}, + {"start_id": "agent", "end_id": "template-b", "kind": "DelegatedEnrollmentAgent", "properties": {"marker": "dea-b"}}, + {"start_id": "template-a", "end_id": "agent", "kind": "DelegatedEnrollmentAgent", "properties": {"marker": "wrong-direction"}}, + {"start_id": "agent", "end_id": "wrong-end", "kind": "DelegatedEnrollmentAgent", "properties": {"marker": "wrong-end-kind"}}, + {"start_id": "agent", "end_id": "template-a", "kind": "OtherDelegation", "properties": {"marker": "wrong-edge-kind"}} + ] + }, + "variants": [ + { + "name": "empty template ID list preserves every relationship", + "node_list_params": {"template_ids": []}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["dea-a-1", "dea-a-2", "dea-b", "wrong-direction", "wrong-end-kind", "wrong-edge-kind"]}}] + }, + { + "name": "single template ID deletes every exact relationship", + "node_list_params": {"template_ids": ["template-a"]}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["dea-b", "wrong-direction", "wrong-end-kind", "wrong-edge-kind"]}}] + }, + { + "name": "duplicate template IDs do not widen the delete", + "node_list_params": {"template_ids": ["template-a", "template-a"]}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["dea-b", "wrong-direction", "wrong-end-kind", "wrong-edge-kind"]}}] + }, + { + "name": "small template ID list deletes both endpoints", + "node_list_params": {"template_ids": ["template-a", "template-b"]}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["wrong-direction", "wrong-end-kind", "wrong-edge-kind"]}}] + } + ] + }, + { + "name": "REC-07 HostsCAService reconciliation delete", + "template": "MATCH ()-[r:HostsCAService]->(e:EnterpriseCA) WHERE e.objectid = $object_id DELETE r", + "fixture": { + "nodes": [ + {"id": "host-a", "kinds": ["Computer"], "properties": {"objectid": "host-a"}}, + {"id": "host-b", "kinds": ["Computer"], "properties": {"objectid": "host-b"}}, + {"id": "ca", "kinds": ["EnterpriseCA"], "properties": {"objectid": "ca-id"}}, + {"id": "wrong-kind", "kinds": ["OtherCA"], "properties": {"objectid": "ca-id"}}, + {"id": "wrong-property", "kinds": ["EnterpriseCA"], "properties": {"objectid": "other-ca"}} + ], + "edges": [ + {"start_id": "host-a", "end_id": "ca", "kind": "HostsCAService", "properties": {"marker": "hosts-a"}}, + {"start_id": "host-b", "end_id": "ca", "kind": "HostsCAService", "properties": {"marker": "hosts-b"}}, + {"start_id": "host-a", "end_id": "wrong-kind", "kind": "HostsCAService", "properties": {"marker": "wrong-ca-kind"}}, + {"start_id": "host-a", "end_id": "wrong-property", "kind": "HostsCAService", "properties": {"marker": "wrong-object-id"}}, + {"start_id": "host-a", "end_id": "ca", "kind": "OtherCAService", "properties": {"marker": "wrong-edge-kind"}} + ] + }, + "variants": [ + { + "name": "exact CA hit deletes duplicate matching edges", + "params": {"object_id": "ca-id"}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["wrong-ca-kind", "wrong-object-id", "wrong-edge-kind"]}}] + }, + { + "name": "no CA hit preserves every relationship", + "params": {"object_id": "missing-ca"}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["hosts-a", "hosts-b", "wrong-ca-kind", "wrong-object-id", "wrong-edge-kind"]}}] + } + ] + }, + { + "name": "REC-08 AD entity detach delete", + "template": "MATCH (n:ADEntity) WHERE n.objectid IN $object_ids DETACH DELETE n", + "fixture": { + "nodes": [ + {"id": "isolated", "kinds": ["ADEntity", "User"], "properties": {"objectid": "isolated"}}, + {"id": "low", "kinds": ["ADEntity", "Computer"], "properties": {"objectid": "low"}}, + {"id": "high", "kinds": ["ADEntity", "Group"], "properties": {"objectid": "high"}}, + {"id": "kind-decoy", "kinds": ["OtherEntity"], "properties": {"objectid": "isolated"}}, + {"id": "property-decoy", "kinds": ["ADEntity"], "properties": {"objectid": "other"}}, + {"id": "neighbor-a", "kinds": ["ADEntity"], "properties": {"objectid": "neighbor-a"}}, + {"id": "neighbor-b", "kinds": ["ADEntity"], "properties": {"objectid": "neighbor-b"}}, + {"id": "neighbor-c", "kinds": ["ADEntity"], "properties": {"objectid": "neighbor-c"}} + ], + "edges": [ + {"start_id": "neighbor-a", "end_id": "low", "kind": "Incident", "properties": {"marker": "low-in"}}, + {"start_id": "low", "end_id": "neighbor-b", "kind": "Incident", "properties": {"marker": "low-out"}}, + {"start_id": "low", "end_id": "low", "kind": "Incident", "properties": {"marker": "low-self"}}, + {"start_id": "neighbor-a", "end_id": "high", "kind": "Incident", "properties": {"marker": "high-in-a"}}, + {"start_id": "neighbor-b", "end_id": "high", "kind": "Incident", "properties": {"marker": "high-in-b"}}, + {"start_id": "high", "end_id": "neighbor-a", "kind": "Incident", "properties": {"marker": "high-out-a"}}, + {"start_id": "high", "end_id": "neighbor-c", "kind": "Incident", "properties": {"marker": "high-out-c"}}, + {"start_id": "high", "end_id": "high", "kind": "Incident", "properties": {"marker": "high-self"}}, + {"start_id": "kind-decoy", "end_id": "property-decoy", "kind": "Survivor", "properties": {"marker": "survivor"}} + ] + }, + "variants": [ + { + "name": "empty object ID list preserves all nodes and relationships", + "params": {"object_ids": []}, + "assert": "no_error", + "post_assertions": [ + {"cypher": "MATCH (n) RETURN n", "assert": {"node_id_set": ["isolated", "low", "high", "kind-decoy", "property-decoy", "neighbor-a", "neighbor-b", "neighbor-c"]}}, + {"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["low-in", "low-out", "low-self", "high-in-a", "high-in-b", "high-out-a", "high-out-c", "high-self", "survivor"]}} + ] + }, + { + "name": "isolated target deletes exactly one node", + "params": {"object_ids": ["isolated"]}, + "assert": "no_error", + "post_assertions": [ + {"cypher": "MATCH (n) RETURN n", "assert": {"node_id_set": ["low", "high", "kind-decoy", "property-decoy", "neighbor-a", "neighbor-b", "neighbor-c"]}}, + {"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["low-in", "low-out", "low-self", "high-in-a", "high-in-b", "high-out-a", "high-out-c", "high-self", "survivor"]}} + ] + }, + { + "name": "low degree target cascades inbound outbound and self edges", + "params": {"object_ids": ["low"]}, + "assert": "no_error", + "post_assertions": [ + {"cypher": "MATCH (n) RETURN n", "assert": {"node_id_set": ["isolated", "high", "kind-decoy", "property-decoy", "neighbor-a", "neighbor-b", "neighbor-c"]}}, + {"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["high-in-a", "high-in-b", "high-out-a", "high-out-c", "high-self", "survivor"]}} + ] + }, + { + "name": "small list includes a high degree target", + "params": {"object_ids": ["low", "high"]}, + "assert": "no_error", + "post_assertions": [ + {"cypher": "MATCH (n) RETURN n", "assert": {"node_id_set": ["isolated", "kind-decoy", "property-decoy", "neighbor-a", "neighbor-b", "neighbor-c"]}}, + {"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["survivor"]}} + ] + }, + { + "name": "large object ID list preserves exact targets", + "params": {"object_ids": {"$type": "string_list", "prefix": "missing-node", "count": 1999, "include": ["low", "high"]}}, + "assert": "no_error", + "post_assertions": [ + {"cypher": "MATCH (n) RETURN n", "assert": {"node_id_set": ["isolated", "kind-decoy", "property-decoy", "neighbor-a", "neighbor-b", "neighbor-c"]}}, + {"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["survivor"]}} + ] + } + ] + }, + { + "name": "TRUST-01 and TRUST-02 stale trust temporal disjunction", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "early-a", "kinds": ["Domain"], "properties": {"lastcollected": "2026-01-02T00:00:00Z"}}, + {"id": "early-b", "kinds": ["Domain"], "properties": {"lastcollected": "2026-01-02T00:00:00Z"}}, + {"id": "equal-a", "kinds": ["Domain"], "properties": {"lastcollected": "2026-01-03T00:00:00Z"}}, + {"id": "equal-b", "kinds": ["Domain"], "properties": {"lastcollected": "2026-01-03T00:00:00Z"}}, + {"id": "late-a", "kinds": ["Domain"], "properties": {"lastcollected": "2026-01-04T00:00:00Z"}}, + {"id": "late-b", "kinds": ["Domain"], "properties": {"lastcollected": "2026-01-04T00:00:00Z"}}, + {"id": "late-b-newer", "kinds": ["Domain"], "properties": {"lastcollected": "2026-01-04T00:00:00Z"}}, + {"id": "late-b-missing-relationship", "kinds": ["Domain"], "properties": {"lastcollected": "2026-01-04T00:00:00Z"}}, + {"id": "late-b-null-relationship", "kinds": ["Domain"], "properties": {"lastcollected": "2026-01-04T00:00:00Z"}}, + {"id": "missing-a", "kinds": ["Domain"], "properties": {}}, + {"id": "missing-b", "kinds": ["Domain"], "properties": {}}, + {"id": "null-a", "kinds": ["Domain"], "properties": {"lastcollected": null}}, + {"id": "null-b", "kinds": ["Domain"], "properties": {"lastcollected": null}}, + {"id": "wrong-start", "kinds": ["Computer"], "properties": {"lastcollected": "2026-01-04T00:00:00Z"}}, + {"id": "wrong-end", "kinds": ["User"], "properties": {"lastcollected": "2026-01-04T00:00:00Z"}} + ], + "edges": [ + {"start_id": "late-a", "end_id": "early-a", "kind": "SameForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "same-older-start-only"}}, + {"start_id": "early-a", "end_id": "late-a", "kind": "SameForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "same-older-end-only"}}, + {"start_id": "late-a", "end_id": "late-b", "kind": "SameForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "same-older-both"}}, + {"start_id": "equal-a", "end_id": "equal-b", "kind": "SameForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "same-equal"}}, + {"start_id": "late-a", "end_id": "late-b-newer", "kind": "SameForestTrust", "properties": {"lastseen": "2026-01-05T00:00:00Z", "marker": "same-newer"}}, + {"start_id": "late-a", "end_id": "late-b-missing-relationship", "kind": "SameForestTrust", "properties": {"marker": "same-missing-relationship"}}, + {"start_id": "late-a", "end_id": "late-b-null-relationship", "kind": "SameForestTrust", "properties": {"lastseen": null, "marker": "same-null-relationship"}}, + {"start_id": "missing-a", "end_id": "late-a", "kind": "SameForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "same-missing-start-valid-end"}}, + {"start_id": "null-a", "end_id": "late-a", "kind": "SameForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "same-null-start-valid-end"}}, + {"start_id": "late-a", "end_id": "missing-b", "kind": "SameForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "same-missing-end-valid-start"}}, + {"start_id": "late-a", "end_id": "null-b", "kind": "SameForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "same-null-end-valid-start"}}, + {"start_id": "missing-a", "end_id": "null-b", "kind": "SameForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "same-missing-null-endpoints"}}, + {"start_id": "wrong-start", "end_id": "late-a", "kind": "SameForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "same-wrong-start-kind"}}, + {"start_id": "late-a", "end_id": "wrong-end", "kind": "SameForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "same-wrong-end-kind"}}, + {"start_id": "late-a", "end_id": "early-a", "kind": "CrossForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-older-start-only"}}, + {"start_id": "early-a", "end_id": "late-a", "kind": "CrossForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-older-end-only"}}, + {"start_id": "late-a", "end_id": "late-b", "kind": "CrossForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-older-both"}}, + {"start_id": "equal-a", "end_id": "equal-b", "kind": "CrossForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-equal"}}, + {"start_id": "late-a", "end_id": "late-b-newer", "kind": "CrossForestTrust", "properties": {"lastseen": "2026-01-05T00:00:00Z", "marker": "cross-newer"}}, + {"start_id": "late-a", "end_id": "late-b-missing-relationship", "kind": "CrossForestTrust", "properties": {"marker": "cross-missing-relationship"}}, + {"start_id": "late-a", "end_id": "late-b-null-relationship", "kind": "CrossForestTrust", "properties": {"lastseen": null, "marker": "cross-null-relationship"}}, + {"start_id": "missing-a", "end_id": "late-a", "kind": "CrossForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-missing-start-valid-end"}}, + {"start_id": "null-a", "end_id": "late-a", "kind": "CrossForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-null-start-valid-end"}}, + {"start_id": "late-a", "end_id": "missing-b", "kind": "CrossForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-missing-end-valid-start"}}, + {"start_id": "late-a", "end_id": "null-b", "kind": "CrossForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-null-end-valid-start"}}, + {"start_id": "missing-a", "end_id": "null-b", "kind": "CrossForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-missing-null-endpoints"}}, + {"start_id": "wrong-start", "end_id": "late-a", "kind": "CrossForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-wrong-start-kind"}}, + {"start_id": "late-a", "end_id": "wrong-end", "kind": "CrossForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-wrong-end-kind"}} + ] + }, + "variants": [ + { + "name": "TRUST-01 returns only stale SameForestTrust relationship IDs", + "vars": {"query": "MATCH (s:Domain)-[r:SameForestTrust]->(e:Domain) WHERE datetime(r.lastseen) < datetime(s.lastcollected) OR datetime(r.lastseen) < datetime(e.lastcollected) RETURN id(r)"}, + "assert": {"keys": ["id(r)"], "row_count": 7} + }, + { + "name": "TRUST-01 exact sparse truth and null matrix", + "vars": {"query": "MATCH (s:Domain)-[r:SameForestTrust]->(e:Domain) WHERE datetime(r.lastseen) < datetime(s.lastcollected) OR datetime(r.lastseen) < datetime(e.lastcollected) RETURN r.marker"}, + "assert": {"scalar_values": ["same-older-start-only", "same-older-end-only", "same-older-both", "same-missing-start-valid-end", "same-null-start-valid-end", "same-missing-end-valid-start", "same-null-end-valid-start"]} + }, + { + "name": "TRUST-02 returns and hydrates only stale CrossForestTrust relationships", + "vars": {"query": "MATCH (s:Domain)-[r:CrossForestTrust]->(e:Domain) WHERE datetime(r.lastseen) < datetime(s.lastcollected) OR datetime(r.lastseen) < datetime(e.lastcollected) RETURN r"}, + "assert": {"row_count": 7, "relationship_records": [ + {"start": "late-a", "end": "early-a", "kind": "CrossForestTrust", "props": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-older-start-only"}}, + {"start": "early-a", "end": "late-a", "kind": "CrossForestTrust", "props": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-older-end-only"}}, + {"start": "late-a", "end": "late-b", "kind": "CrossForestTrust", "props": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-older-both"}}, + {"start": "missing-a", "end": "late-a", "kind": "CrossForestTrust", "props": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-missing-start-valid-end"}}, + {"start": "null-a", "end": "late-a", "kind": "CrossForestTrust", "props": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-null-start-valid-end"}}, + {"start": "late-a", "end": "missing-b", "kind": "CrossForestTrust", "props": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-missing-end-valid-start"}}, + {"start": "late-a", "end": "null-b", "kind": "CrossForestTrust", "props": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-null-end-valid-start"}} + ]} + } + ] + }, + { + "name": "TRUST-03 directional stale trust derivation", + "template": "MATCH (s:Domain)-[r]->(e:Domain) WHERE (id(s) = $forward_start AND id(e) = $forward_end AND r:AbuseTGTDelegation) OR (id(s) = $forward_end AND id(e) = $forward_start AND r:SpoofSIDHistory) RETURN {{projection}}", + "node_params": {"forward_start": "forward", "forward_end": "reverse"}, + "fixture": { + "nodes": [ + {"id": "forward", "kinds": ["Domain"], "properties": {"name": "forward"}}, + {"id": "reverse", "kinds": ["Domain"], "properties": {"name": "reverse"}}, + {"id": "wrong-kind", "kinds": ["Computer"], "properties": {"name": "wrong-kind"}} + ], + "edges": [ + {"start_id": "forward", "end_id": "reverse", "kind": "AbuseTGTDelegation", "properties": {"marker": "valid-forward-abuse"}}, + {"start_id": "reverse", "end_id": "forward", "kind": "SpoofSIDHistory", "properties": {"marker": "valid-reverse-spoof"}}, + {"start_id": "forward", "end_id": "reverse", "kind": "SpoofSIDHistory", "properties": {"marker": "invalid-forward-spoof"}}, + {"start_id": "reverse", "end_id": "forward", "kind": "AbuseTGTDelegation", "properties": {"marker": "invalid-reverse-abuse"}}, + {"start_id": "forward", "end_id": "wrong-kind", "kind": "AbuseTGTDelegation", "properties": {"marker": "invalid-end-kind"}} + ] + }, + "variants": [ + {"name": "relationship ID projection preserves branch-local direction and kind", "vars": {"projection": "id(r)"}, "assert": {"keys": ["id(r)"], "row_count": 2}}, + {"name": "exact directional markers exclude both cross-combinations", "vars": {"projection": "r.marker"}, "assert": {"scalar_values": ["valid-forward-abuse", "valid-reverse-spoof"]}}, + { + "name": "reverse driving trust relationship preserves ID projection", + "vars": {"projection": "id(r)"}, + "node_params": {"forward_start": "reverse", "forward_end": "forward"}, + "assert": {"keys": ["id(r)"], "row_count": 2} + }, + { + "name": "reverse driving trust relationship swaps only the intended branch-local matches", + "vars": {"projection": "r.marker"}, + "node_params": {"forward_start": "reverse", "forward_end": "forward"}, + "assert": {"scalar_values": ["invalid-forward-spoof", "invalid-reverse-abuse"]} + } + ] + } + ] +} diff --git a/integration/testdata/templates/relationship_scan_shapes.json b/integration/testdata/templates/relationship_scan_shapes.json new file mode 100644 index 00000000..af9dec77 --- /dev/null +++ b/integration/testdata/templates/relationship_scan_shapes.json @@ -0,0 +1,119 @@ +{ + "families": [ + { + "name": "SCAN-01 through SCAN-04 wide relationship filters", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "ad-a", "kinds": ["ADBase"], "properties": {"name": "ad-a"}}, + {"id": "ad-b", "kinds": ["ADBase"], "properties": {"name": "ad-b"}}, + {"id": "az-a", "kinds": ["AZBase"], "properties": {"name": "az-a"}}, + {"id": "az-b", "kinds": ["AZBase"], "properties": {"name": "az-b"}}, + {"id": "plain-a", "kinds": ["Plain", "MissingPost"], "properties": {"name": "plain-a"}}, + {"id": "plain-b", "kinds": ["Plain"], "properties": {"name": "plain-b"}}, + {"id": "meta-start", "kinds": ["Meta", "Plain"], "properties": {"name": "meta-start"}}, + {"id": "meta-end", "kinds": ["MetaDetail", "Plain"], "properties": {"name": "meta-end"}}, + {"id": "meta-both", "kinds": ["Meta", "MetaDetail", "Plain"], "properties": {"name": "meta-both"}}, + {"id": "entity-a", "kinds": ["Entity"], "properties": {"name": "entity-a"}}, + {"id": "entity-b", "kinds": ["Entity"], "properties": {"name": "entity-b"}}, + {"id": "not-entity", "kinds": ["Other"], "properties": {"name": "not-entity"}} + ], + "edges": [ + {"start_id": "ad-a", "end_id": "ad-b", "kind": "PostProcessed", "properties": {"marker": "post-ad"}}, + {"start_id": "az-a", "end_id": "az-b", "kind": "PostProcessed", "properties": {"marker": "post-az"}}, + {"start_id": "ad-a", "end_id": "az-b", "kind": "PostProcessed", "properties": {"marker": "post-cross-a"}}, + {"start_id": "az-a", "end_id": "ad-b", "kind": "PostProcessed", "properties": {"marker": "post-cross-b"}}, + {"start_id": "plain-a", "end_id": "ad-b", "kind": "PostProcessed", "properties": {"marker": "post-wrong-start"}}, + {"start_id": "ad-a", "end_id": "plain-b", "kind": "PostProcessed", "properties": {"marker": "post-wrong-end"}}, + {"start_id": "ad-a", "end_id": "ad-b", "kind": "WrongPost", "properties": {"marker": "post-wrong-kind"}}, + {"start_id": "plain-a", "end_id": "plain-b", "kind": "TrackerA", "properties": {"marker": "tracker-a", "hydrated": true}}, + {"start_id": "plain-a", "end_id": "plain-b", "kind": "TrackerB", "properties": {"marker": "tracker-b", "hydrated": true}}, + {"start_id": "meta-start", "end_id": "plain-b", "kind": "TrackerA", "properties": {"marker": "tracker-meta-start"}}, + {"start_id": "plain-a", "end_id": "meta-end", "kind": "TrackerA", "properties": {"marker": "tracker-meta-end"}}, + {"start_id": "meta-both", "end_id": "meta-both", "kind": "TrackerB", "properties": {"marker": "tracker-meta-both"}}, + {"start_id": "plain-a", "end_id": "plain-b", "kind": "MigratedEdge", "properties": {"marker": "migration-present", "lastseen": "2026-01-03T00:00:00Z"}}, + {"start_id": "plain-b", "end_id": "plain-a", "kind": "MigratedEdge", "properties": {"marker": "migration-null", "lastseen": null}}, + {"start_id": "plain-a", "end_id": "entity-a", "kind": "MigratedEdge", "properties": {"marker": "migration-missing"}}, + {"start_id": "meta-start", "end_id": "plain-b", "kind": "MigratedEdge", "properties": {"marker": "migration-meta-start", "lastseen": "2026-01-03T00:00:00Z"}}, + {"start_id": "plain-a", "end_id": "meta-end", "kind": "MigratedEdge", "properties": {"marker": "migration-meta-end", "lastseen": "2026-01-03T00:00:00Z"}}, + {"start_id": "entity-a", "end_id": "entity-b", "kind": "OwnsRaw", "properties": {"marker": "owns", "hydrated": "yes"}}, + {"start_id": "entity-a", "end_id": "entity-b", "kind": "WriteOwnerRaw", "properties": {"marker": "write-owner", "hydrated": "yes"}}, + {"start_id": "not-entity", "end_id": "entity-b", "kind": "OwnsRaw", "properties": {"marker": "owns-wrong-start"}} + ] + }, + "variants": [ + {"name": "SCAN-01 AD and Azure bases exact relationship kind", "vars": {"query": "MATCH (s)-[r:PostProcessed]->(e) WHERE (s:ADBase OR s:AZBase) AND (e:ADBase OR e:AZBase) RETURN id(r)"}, "assert": {"keys": ["id(r)"], "row_count": 4}}, + {"name": "SCAN-01 wrong relationship kind is empty", "vars": {"query": "MATCH (s)-[r:MissingPost]->(e) WHERE (s:ADBase OR s:AZBase) AND (e:ADBase OR e:AZBase) RETURN id(r)"}, "assert": "empty"}, + {"name": "SCAN-02 one kind excludes every Meta endpoint position", "vars": {"query": "MATCH (s)-[r:TrackerA]->(e) WHERE NOT (s:Meta OR s:MetaDetail) AND NOT (e:Meta OR e:MetaDetail) RETURN r"}, "assert": {"relationship_records": [{"start": "plain-a", "end": "plain-b", "kind": "TrackerA", "props": {"marker": "tracker-a", "hydrated": true}}]}}, + {"name": "SCAN-02 many kinds hydrate complete relationships", "vars": {"query": "MATCH (s)-[r:TrackerA|TrackerB]->(e) WHERE NOT (s:Meta OR s:MetaDetail) AND NOT (e:Meta OR e:MetaDetail) RETURN r"}, "assert": {"relationship_records": [{"start": "plain-a", "end": "plain-b", "kind": "TrackerA", "props": {"marker": "tracker-a", "hydrated": true}}, {"start": "plain-a", "end": "plain-b", "kind": "TrackerB", "props": {"marker": "tracker-b", "hydrated": true}}]}}, + {"name": "SCAN-03 present lastseen only with Meta decoys", "vars": {"query": "MATCH (s)-[r:MigratedEdge]->(e) WHERE NOT (s:Meta OR s:MetaDetail) AND r.lastseen IS NOT NULL AND NOT (e:Meta OR e:MetaDetail) RETURN id(r)"}, "assert": {"keys": ["id(r)"], "row_count": 1}}, + {"name": "SCAN-04 OwnsRaw full hydration", "vars": {"query": "MATCH (s:Entity)-[r:OwnsRaw]->() RETURN r"}, "assert": {"relationship_records": [{"start": "entity-a", "end": "entity-b", "kind": "OwnsRaw", "props": {"marker": "owns", "hydrated": "yes"}}]}}, + {"name": "SCAN-04 WriteOwnerRaw representative", "vars": {"query": "MATCH (s:Entity)-[r:WriteOwnerRaw]->() RETURN r"}, "assert": {"relationship_records": [{"start": "entity-a", "end": "entity-b", "kind": "WriteOwnerRaw", "props": {"marker": "write-owner", "hydrated": "yes"}}]}} + ] + }, + { + "name": "SCAN-05 through SCAN-08 anchored scans and projections", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "target", "kinds": ["Computer"], "properties": {"name": "target"}}, + {"id": "zero-target", "kinds": ["Computer", "MissingMember"], "properties": {"name": "zero-target"}}, + {"id": "wrong-end", "kinds": ["Other"], "properties": {"name": "wrong-end"}}, + {"id": "source-01", "kinds": ["Entity", "Group"], "properties": {"name": "source-01", "objectid": "S-1-5-01"}}, + {"id": "source-02", "kinds": ["Entity", "User"], "properties": {"name": "source-02", "objectid": "S-1-5-02"}}, + {"id": "source-03", "kinds": ["Entity", "Computer"], "properties": {"name": "source-03", "objectid": "S-1-5-03"}}, + {"id": "source-04", "kinds": ["Entity"], "properties": {"name": "source-04"}}, + {"id": "source-05", "kinds": ["Entity"], "properties": {"name": "source-05"}}, + {"id": "source-06", "kinds": ["Entity"], "properties": {"name": "source-06"}}, + {"id": "source-07", "kinds": ["Entity"], "properties": {"name": "source-07"}}, + {"id": "source-08", "kinds": ["Entity"], "properties": {"name": "source-08"}}, + {"id": "source-09", "kinds": ["Entity"], "properties": {"name": "source-09"}}, + {"id": "not-entity", "kinds": ["Other"], "properties": {"name": "not-entity"}}, + {"id": "victim-computer", "kinds": ["Computer"], "properties": {"name": "victim-computer"}}, + {"id": "victim-other", "kinds": ["Other"], "properties": {"name": "victim-other"}}, + {"id": "victim-unused", "kinds": ["Computer"], "properties": {"name": "victim-unused"}}, + {"id": "attacker-group", "kinds": ["Group", "Entity"], "properties": {"name": "attacker-group"}}, + {"id": "attacker-user", "kinds": ["User", "Entity"], "properties": {"name": "attacker-user"}}, + {"id": "attacker-computer", "kinds": ["Computer", "Entity"], "properties": {"name": "attacker-computer"}}, + {"id": "attacker-wrong", "kinds": ["Other"], "properties": {"name": "attacker-wrong"}} + ], + "edges": [ + {"start_id": "source-01", "end_id": "target", "kind": "ScanEdge01", "properties": {"marker": "scan-01", "hydrated": true}}, + {"start_id": "source-02", "end_id": "target", "kind": "ScanEdge02", "properties": {"marker": "scan-02"}}, + {"start_id": "source-03", "end_id": "target", "kind": "ScanEdge03", "properties": {"marker": "scan-03"}}, + {"start_id": "source-04", "end_id": "target", "kind": "ScanEdge04", "properties": {"marker": "scan-04"}}, + {"start_id": "source-05", "end_id": "target", "kind": "ScanEdge05", "properties": {"marker": "scan-05"}}, + {"start_id": "source-06", "end_id": "target", "kind": "ScanEdge06", "properties": {"marker": "scan-06"}}, + {"start_id": "source-07", "end_id": "target", "kind": "ScanEdge07", "properties": {"marker": "scan-07"}}, + {"start_id": "source-08", "end_id": "target", "kind": "ScanEdge08", "properties": {"marker": "scan-08"}}, + {"start_id": "source-09", "end_id": "target", "kind": "ScanEdge09", "properties": {"marker": "scan-09"}}, + {"start_id": "not-entity", "end_id": "target", "kind": "ScanEdge01", "properties": {"marker": "scan-wrong-start"}}, + {"start_id": "source-01", "end_id": "wrong-end", "kind": "LocalToComputer", "properties": {"marker": "local-wrong-end"}}, + {"start_id": "source-01", "end_id": "target", "kind": "LocalToComputer", "properties": {"marker": "local-valid"}}, + {"start_id": "source-01", "end_id": "target", "kind": "MemberOf", "properties": {"marker": "member-01"}}, + {"start_id": "source-01", "end_id": "target", "kind": "MemberOfLocalGroup", "properties": {"marker": "member-local-01"}}, + {"start_id": "source-02", "end_id": "target", "kind": "MemberOf", "properties": {"marker": "member-02"}}, + {"start_id": "source-02", "end_id": "target", "kind": "WrongMember", "properties": {"marker": "member-wrong"}}, + {"start_id": "attacker-group", "end_id": "victim-computer", "kind": "GenericAll", "properties": {"marker": "esc-group"}}, + {"start_id": "attacker-user", "end_id": "victim-other", "kind": "WritePublicInformation", "properties": {"marker": "esc-user-a-only"}}, + {"start_id": "attacker-computer", "end_id": "victim-computer", "kind": "WriteDACL", "properties": {"marker": "esc-computer"}}, + {"start_id": "attacker-wrong", "end_id": "victim-computer", "kind": "GenericAll", "properties": {"marker": "esc-wrong-start"}}, + {"start_id": "attacker-group", "end_id": "victim-computer", "kind": "WrongEsc", "properties": {"marker": "esc-wrong-kind"}} + ] + }, + "variants": [ + {"name": "SCAN-05 zero inbound degree", "vars": {"query": "MATCH (s:Entity)-[r:ScanEdge01]->(e) WHERE id(e) = $target RETURN r, s"}, "node_params": {"target": "zero-target"}, "assert": "empty"}, + {"name": "SCAN-05 one kind one match full hydration", "vars": {"query": "MATCH (s:Entity)-[r:ScanEdge01]->(e) WHERE id(e) = $target RETURN r, s"}, "node_params": {"target": "target"}, "assert": {"keys": ["r", "s"], "node_id_set": ["source-01"], "relationship_records": [{"start": "source-01", "end": "target", "kind": "ScanEdge01", "props": {"marker": "scan-01", "hydrated": true}}]}}, + {"name": "SCAN-05 nine kinds high inbound degree", "vars": {"query": "MATCH (s:Entity)-[r:ScanEdge01|ScanEdge02|ScanEdge03|ScanEdge04|ScanEdge05|ScanEdge06|ScanEdge07|ScanEdge08|ScanEdge09]->(e) WHERE id(e) = $target RETURN r, s"}, "node_params": {"target": "target"}, "assert": {"keys": ["r", "s"], "row_count": 9, "node_id_set": ["source-01", "source-02", "source-03", "source-04", "source-05", "source-06", "source-07", "source-08", "source-09"]}}, + {"name": "SCAN-06 exact FetchKinds projection", "vars": {"query": "MATCH (s)-[r:LocalToComputer]->(e:Computer) RETURN id(s), id(r), type(r), id(e)"}, "assert": {"keys": ["id(s)", "id(r)", "type(r)", "id(e)"], "row_count": 1}}, + {"name": "SCAN-07 one kind directed endpoint IDs", "vars": {"query": "MATCH (s)-[r:MemberOf]->(e) RETURN id(s), id(e)"}, "assert": {"keys": ["id(s)", "id(e)"], "row_count": 2}}, + {"name": "SCAN-07 many kinds retain duplicate endpoint pairs", "vars": {"query": "MATCH (s)-[r:MemberOf|MemberOfLocalGroup]->(e) RETURN id(s), id(e)"}, "assert": {"keys": ["id(s)", "id(e)"], "row_count": 3}}, + {"name": "SCAN-07 absent kind zero matches", "vars": {"query": "MATCH (s)-[r:MissingMember]->(e) RETURN id(s), id(e)"}, "assert": "empty"}, + {"name": "SCAN-08 scenario A empty victim list", "vars": {"query": "MATCH (s)-[r:GenericAll|GenericWrite|Owns|WriteOwner|WriteDACL|WritePublicInformation]->(e) WHERE (s:Group OR s:User OR s:Computer) AND id(e) IN $victims RETURN id(s)"}, "node_list_params": {"victims": []}, "assert": "empty"}, + {"name": "SCAN-08 scenario A single victim", "vars": {"query": "MATCH (s)-[r:GenericAll|GenericWrite|Owns|WriteOwner|WriteDACL|WritePublicInformation]->(e) WHERE (s:Group OR s:User OR s:Computer) AND id(e) IN $victims RETURN id(s)"}, "node_list_params": {"victims": ["victim-other"]}, "assert": {"keys": ["id(s)"], "row_count": 1}}, + {"name": "SCAN-08 scenario A thirty-two-entry victim list", "vars": {"query": "MATCH (s)-[r:GenericAll|GenericWrite|Owns|WriteOwner|WriteDACL|WritePublicInformation]->(e) WHERE (s:Group OR s:User OR s:Computer) AND id(e) IN $victims RETURN id(s)"}, "node_list_params": {"victims": ["victim-computer", "victim-other", "victim-unused", "victim-computer", "victim-other", "victim-unused", "victim-computer", "victim-other", "victim-unused", "victim-computer", "victim-other", "victim-unused", "victim-computer", "victim-other", "victim-unused", "victim-computer", "victim-other", "victim-unused", "victim-computer", "victim-other", "victim-unused", "victim-computer", "victim-other", "victim-unused", "victim-computer", "victim-other", "victim-unused", "victim-computer", "victim-other", "victim-unused", "victim-computer", "victim-other"]}, "assert": {"keys": ["id(s)"], "row_count": 3}}, + {"name": "SCAN-08 scenario B typed end and five kinds", "vars": {"query": "MATCH (s)-[r:GenericAll|GenericWrite|Owns|WriteOwner|WriteDACL]->(e:Computer) WHERE (s:Group OR s:User OR s:Computer) AND id(e) IN $victims RETURN id(s)"}, "node_list_params": {"victims": ["victim-computer", "victim-other", "victim-unused"]}, "assert": {"keys": ["id(s)"], "row_count": 2}} + ] + } + ] +} diff --git a/integration/trust_pruning_legacy_builder_test.go b/integration/trust_pruning_legacy_builder_test.go new file mode 100644 index 00000000..2f369b75 --- /dev/null +++ b/integration/trust_pruning_legacy_builder_test.go @@ -0,0 +1,854 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +//go:build manual_integration + +package integration + +import ( + "context" + "sort" + "testing" + "time" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/opengraph" + "github.com/specterops/dawgs/ops" + "github.com/specterops/dawgs/query" + "github.com/specterops/dawgs/testutil" + "github.com/stretchr/testify/require" +) + +// TestLegacyBuilderTrustAndPruningSelectors verifies legacy trust and pruning selectors retain their filtering semantics. +func TestLegacyBuilderTrustAndPruningSelectors(t *testing.T) { + fixture := trustPruningFixture() + nodeKinds, edgeKinds := fixture.Kinds() + db, ctx := SetupDBWithKindsNoGraphCleanup(t, nodeKinds, edgeKinds) + ClearGraph(t, db, ctx) + session := &Session{ + DB: db, + Ctx: ctx, + } + threshold := regressionDay(3) + + t.Run("TRUST-01 SameForestTrust IDs", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, fixture, func(opengraph.IDMap) graph.Criteria { + return trustPruningCriteria("SameForestTrust") + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + ids, err := ops.FetchRelationshipIDs(relationshipQuery) + require.NoError(t, err) + require.Len(t, ids, 1) + return nil + }) + }) + + t.Run("TRUST-02 CrossForestTrust hydration", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, fixture, func(opengraph.IDMap) graph.Criteria { + return trustPruningCriteria("CrossForestTrust") + }, func(relationshipQuery graph.RelationshipQuery, idMap opengraph.IDMap) error { + relationships, err := ops.FetchRelationships(relationshipQuery) + require.NoError(t, err) + require.Len(t, relationships, 1) + require.Equal(t, idMap["late-a"], relationships[0].StartID) + require.Equal(t, idMap["early"], relationships[0].EndID) + require.Equal(t, graph.StringKind("CrossForestTrust"), relationships[0].Kind) + marker, err := relationships[0].Properties.Get("marker").String() + require.NoError(t, err) + require.Equal(t, "cross-old", marker) + return nil + }) + }) + + t.Run("TRUST-03 directional derived IDs", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, fixture, func(idMap opengraph.IDMap) graph.Criteria { + return directionalTrustCriteria(idMap, "late-a", "late-b") + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + relationships, err := ops.FetchRelationships(relationshipQuery) + require.NoError(t, err) + require.Len(t, relationships, 2) + markers := make([]string, 0, len(relationships)) + for _, relationship := range relationships { + marker, err := relationship.Properties.Get("marker").String() + require.NoError(t, err) + markers = append(markers, marker) + } + sort.Strings(markers) + require.Equal(t, []string{"valid-forward-abuse", "valid-reverse-spoof"}, markers) + return nil + }) + }) + + t.Run("TRUST-03 reverse driving trust relationship", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, fixture, func(idMap opengraph.IDMap) graph.Criteria { + return directionalTrustCriteria(idMap, "late-b", "late-a") + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + relationships, err := ops.FetchRelationships(relationshipQuery) + require.NoError(t, err) + require.Len(t, relationships, 2) + require.Equal(t, []string{"invalid-forward-spoof", "invalid-reverse-abuse"}, trustPruningRelationshipMarkers(t, relationships)) + return nil + }) + }) + + t.Run("PRUNE-01 protected kinds and old relationships", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, fixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Not(query.KindIn(query.Relationship(), graph.StringKind("HasSession"), graph.StringKind("MetaIncludes"))), + query.Before(query.RelationshipProperty("lastseen"), threshold), + ) + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + relationships, err := ops.FetchRelationships(relationshipQuery) + require.NoError(t, err) + require.Equal(t, []string{"candidate-old"}, trustPruningRelationshipMarkers(t, relationships)) + return nil + }) + }) + + t.Run("PRUNE-02 HasSession missing null or old", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, fixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Kind(query.Relationship(), graph.StringKind("HasSession")), + query.Or( + query.Not(query.Exists(query.RelationshipProperty("lastseen"))), + query.Before(query.RelationshipProperty("lastseen"), threshold), + ), + ) + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + relationships, err := ops.FetchRelationships(relationshipQuery) + require.NoError(t, err) + require.Equal(t, []string{"session-missing", "session-null", "session-old"}, trustPruningRelationshipMarkers(t, relationships)) + return nil + }) + }) + + t.Run("PRUNE-03 protected kinds and missing null or old nodes", func(t *testing.T) { + WithLegacyNodeQuery(t, session, fixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Not(query.KindIn(query.Node(), pruningProtectedNodeKinds()...)), + query.Or( + query.Not(query.Exists(query.NodeProperty("lastseen"))), + query.Before(query.NodeProperty("lastseen"), threshold), + ), + ) + }, func(nodeQuery graph.NodeQuery, idMap opengraph.IDMap) error { + ids, err := ops.FetchNodeIDs(nodeQuery) + require.NoError(t, err) + require.Equal(t, []string{"candidate-missing", "candidate-null", "candidate-old", "orphan-empty", "orphan-missing", "orphan-null", "orphan-wrong-prefix"}, trustPruningFixtureIDs(t, idMap, ids)) + return nil + }) + }) + + t.Run("PRUNE-04 orphan SID nodes", func(t *testing.T) { + WithLegacyNodeQuery(t, session, fixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Not(query.KindIn(query.Node(), pruningProtectedNodeKinds()...)), + query.Not(query.Exists(query.NodeProperty("name"))), + query.StringStartsWith(query.NodeProperty("objectid"), "S-1-5"), + ) + }, func(nodeQuery graph.NodeQuery, idMap opengraph.IDMap) error { + ids, err := ops.FetchNodeIDs(nodeQuery) + require.NoError(t, err) + require.Equal(t, []string{"orphan-missing", "orphan-null"}, trustPruningFixtureIDs(t, idMap, ids)) + return nil + }) + }) +} + +// TestDirectBatchPruning verifies direct batch pruning removes selected nodes and relationships without affecting survivors. +func TestDirectBatchPruning(t *testing.T) { + fixture := batchPruningFixture(32) + nodeKinds, edgeKinds := fixture.Kinds() + db, ctx := SetupDBWithKinds(t, CleanupGraph, nodeKinds, edgeKinds) + + loadFixture := func(t *testing.T) opengraph.IDMap { + t.Helper() + ClearGraph(t, db, ctx) + idMap, err := opengraph.WriteGraph(ctx, db, fixture) + require.NoError(t, err) + return idMap + } + + t.Run("PRUNE-05 empty single and many relationships", func(t *testing.T) { + for _, testCase := range []struct { + // name identifies the relationship-pruning population. + name string + + // criteria selects relationships for deletion. + criteria graph.CriteriaProvider + + // expected is the number of accepted delete attempts. + expected int + + // remaining is the expected PruneDelete relationship count. + remaining int64 + }{ + { + name: "empty", + criteria: func() graph.Criteria { return query.Equals(query.RelationshipProperty("marker"), "absent") }, + expected: 0, + remaining: 3, + }, + { + name: "single", + criteria: func() graph.Criteria { return query.Equals(query.RelationshipProperty("marker"), "single") }, + expected: 1, + remaining: 2, + }, + { + name: "many", + criteria: func() graph.Criteria { return query.Kind(query.Relationship(), graph.StringKind("PruneDelete")) }, + expected: 3, + remaining: 0, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + loadFixture(t) + deleted, err := pruneRelationshipsInBatches(ctx, db, testCase.criteria, nil) + require.NoError(t, err) + require.Equal(t, testCase.expected, deleted) + require.Equal(t, testCase.remaining, countByCypher(t, ctx, db, "MATCH ()-[r:PruneDelete]->() RETURN count(r)")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH ()-[r:PruneSurvivor]->() RETURN count(r)")) + }) + } + }) + + t.Run("PRUNE-05 relationship absent after selection is harmless", func(t *testing.T) { + loadFixture(t) + deleted, err := pruneRelationshipsInBatches(ctx, db, func() graph.Criteria { + return query.Equals(query.RelationshipProperty("marker"), "single") + }, func(ids []graph.ID) error { + return db.BatchOperation(ctx, func(batch graph.Batch) error { + return batch.DeleteRelationship(ids[0]) + }) + }) + require.NoError(t, err) + require.Equal(t, 1, deleted, "the production workflow counts accepted delete attempts") + require.Equal(t, int64(2), countByCypher(t, ctx, db, "MATCH ()-[r:PruneDelete]->() RETURN count(r)")) + }) + + t.Run("PRUNE-06 empty single many and high-degree nodes", func(t *testing.T) { + for _, testCase := range []struct { + // name identifies the node-pruning population. + name string + + // criteria selects candidate nodes for deletion. + criteria graph.CriteriaProvider + + // expected is the number of accepted delete attempts. + expected int + + // expectedCandidates is the expected surviving candidate count. + expectedCandidates int64 + + // expectedIncidents is the expected surviving incident-edge count. + expectedIncidents int64 + }{ + { + name: "empty", + criteria: func() graph.Criteria { return query.Equals(query.NodeProperty("objectid"), "absent") }, + expected: 0, + expectedCandidates: 3, + expectedIncidents: 34, + }, + { + name: "single", + criteria: func() graph.Criteria { return query.Equals(query.NodeProperty("objectid"), "single") }, + expected: 1, + expectedCandidates: 2, + expectedIncidents: 34, + }, + { + name: "many including high degree", + criteria: func() graph.Criteria { return query.Equals(query.NodeProperty("remove"), true) }, + expected: 2, + expectedCandidates: 1, + expectedIncidents: 1, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + loadFixture(t) + deleted, err := pruneNodesInBatches(ctx, db, testCase.criteria, nil) + require.NoError(t, err) + require.Equal(t, testCase.expected, deleted) + require.Equal(t, testCase.expectedCandidates, countByCypher(t, ctx, db, "MATCH (n:PruneDeleteNode) RETURN count(n)")) + require.Equal(t, testCase.expectedIncidents, countByCypher(t, ctx, db, "MATCH ()-[r:PruneIncident]->() RETURN count(r)")) + }) + } + }) + + t.Run("PRUNE-06 node absent after selection is harmless", func(t *testing.T) { + loadFixture(t) + deleted, err := pruneNodesInBatches(ctx, db, func() graph.Criteria { + return query.Equals(query.NodeProperty("objectid"), "single") + }, func(ids []graph.ID) error { + return db.BatchOperation(ctx, func(batch graph.Batch) error { + return batch.DeleteNode(ids[0]) + }) + }) + require.NoError(t, err) + require.Equal(t, 1, deleted) + require.Equal(t, int64(2), countByCypher(t, ctx, db, "MATCH (n:PruneDeleteNode) RETURN count(n)")) + }) +} + +// BenchmarkDirectBatchPruning measures direct pruning across representative fixture sizes. +func BenchmarkDirectBatchPruning(b *testing.B) { + fixture := testutil.NewTrustPruningScaleFixture(2_000) + nodeKinds, edgeKinds := fixture.Kinds() + session := Open(b, Options{ + ExtraNodeKinds: nodeKinds, + ExtraEdgeKinds: edgeKinds, + CleanupMode: CloseOnly, + }) + + resetFixture := func(b *testing.B) { + b.Helper() + if err := session.DB.WriteTransaction(session.Ctx, func(tx graph.Transaction) error { + return tx.Nodes().Delete() + }); err != nil { + b.Fatalf("clear benchmark graph: %v", err) + } + if _, err := opengraph.WriteGraph(session.Ctx, session.DB, fixture); err != nil { + b.Fatalf("load benchmark fixture: %v", err) + } + } + + b.Run("PRUNE-05 relationship ID selection and batch delete", func(b *testing.B) { + b.ReportAllocs() + for idx := 0; idx < b.N; idx++ { + b.StopTimer() + resetFixture(b) + b.StartTimer() + deleted, err := pruneRelationshipsInBatches(session.Ctx, session.DB, func() graph.Criteria { + return query.Kind(query.Relationship(), graph.StringKind("PruneBatch")) + }, nil) + if err != nil { + b.Fatalf("prune relationships: %v", err) + } + if deleted != 2_000 { + b.Fatalf("deleted relationships: got %d, want 2000", deleted) + } + } + }) + + b.Run("PRUNE-06 node ID selection high-degree cascade and batch delete", func(b *testing.B) { + b.ReportAllocs() + for idx := 0; idx < b.N; idx++ { + b.StopTimer() + resetFixture(b) + b.StartTimer() + deleted, err := pruneNodesInBatches(session.Ctx, session.DB, func() graph.Criteria { + return query.Equals(query.NodeProperty("remove"), true) + }, nil) + if err != nil { + b.Fatalf("prune nodes: %v", err) + } + if deleted != 1_001 { + b.Fatalf("deleted nodes: got %d, want 1001", deleted) + } + } + }) +} + +// trustPruningCriteria selects domain-to-domain relationships of kind whose +// last-seen time predates either endpoint's collection time. +func trustPruningCriteria(kind string) graph.Criteria { + return query.And( + query.Kind(query.Start(), graph.StringKind("Domain")), + query.Kind(query.End(), graph.StringKind("Domain")), + query.KindIn(query.Relationship(), graph.StringKind(kind)), + query.Or( + query.BeforeGraphQuery(query.RelationshipProperty("lastseen"), query.StartProperty("lastcollected")), + query.BeforeGraphQuery(query.RelationshipProperty("lastseen"), query.EndProperty("lastcollected")), + ), + ) +} + +// directionalTrustCriteria selects the two directed trust kinds between the +// supplied fixture endpoints. +func directionalTrustCriteria(idMap opengraph.IDMap, forward, reverse string) graph.Criteria { + forwardID := idMap[forward] + reverseID := idMap[reverse] + return query.And( + query.Kind(query.Start(), graph.StringKind("Domain")), + query.Kind(query.End(), graph.StringKind("Domain")), + query.Or( + query.And( + query.Equals(query.StartID(), forwardID), + query.Equals(query.EndID(), reverseID), + query.KindIn(query.Relationship(), graph.StringKind("AbuseTGTDelegation")), + ), + query.And( + query.Equals(query.StartID(), reverseID), + query.Equals(query.EndID(), forwardID), + query.KindIn(query.Relationship(), graph.StringKind("SpoofSIDHistory")), + ), + ), + ) +} + +// pruningProtectedNodeKinds returns the labels whose nodes must survive trust-pruning regression queries even when their relationships are stale. +func pruningProtectedNodeKinds() graph.Kinds { + return graph.Kinds{ + graph.StringKind("Domain"), + graph.StringKind("Tenant"), + graph.StringKind("Meta"), + graph.StringKind("MetaIncludes"), + graph.StringKind("MigrationData"), + } +} + +// trustPruningRelationshipMarkers returns sorted marker properties from selected relationships. +func trustPruningRelationshipMarkers(t *testing.T, relationships []*graph.Relationship) []string { + t.Helper() + markers := make([]string, 0, len(relationships)) + for _, relationship := range relationships { + marker, err := relationship.Properties.Get("marker").String() + require.NoError(t, err) + markers = append(markers, marker) + } + sort.Strings(markers) + return markers +} + +// trustPruningFixtureIDs maps database IDs to sorted stable fixture identifiers. +func trustPruningFixtureIDs(t *testing.T, idMap opengraph.IDMap, ids []graph.ID) []string { + t.Helper() + fixtureIDs := make([]string, 0, len(ids)) + for _, id := range ids { + fixtureIDs = append(fixtureIDs, regressionFixtureID(t, idMap, id)) + } + sort.Strings(fixtureIDs) + return fixtureIDs +} + +// trustPruningFixture builds stale, current, null-timestamp, and decoy trust +// relationships for pruning regressions. +func trustPruningFixture() *opengraph.Graph { + return &opengraph.Graph{ + Nodes: []opengraph.Node{ + { + ID: "early", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": regressionDay(2)}, + }, + { + ID: "late-a", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": regressionDay(4)}, + }, + { + ID: "late-b", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": regressionDay(4)}, + }, + { + ID: "candidate-rel-equal", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": regressionDay(4)}, + }, + { + ID: "candidate-rel-new", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": regressionDay(4)}, + }, + { + ID: "candidate-rel-missing", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": regressionDay(4)}, + }, + { + ID: "candidate-rel-null", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": regressionDay(4)}, + }, + { + ID: "session-null", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": regressionDay(4)}, + }, + { + ID: "session-old", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": regressionDay(4)}, + }, + { + ID: "session-equal", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": regressionDay(4)}, + }, + { + ID: "session-new", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": regressionDay(4)}, + }, + { + ID: "equal-a", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": regressionDay(3)}, + }, + { + ID: "equal-b", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": regressionDay(3)}, + }, + { + ID: "wrong-end", + Kinds: []string{"Computer"}, + Properties: map[string]any{"lastcollected": regressionDay(4), "lastseen": regressionDay(4)}, + }, + { + ID: "candidate-missing", + Kinds: []string{"CandidateNode"}, + Properties: map[string]any{}, + }, + { + ID: "candidate-null", + Kinds: []string{"CandidateNode"}, + Properties: map[string]any{"lastseen": nil}, + }, + { + ID: "candidate-old", + Kinds: []string{"CandidateNode"}, + Properties: map[string]any{"lastseen": regressionDay(2)}, + }, + { + ID: "candidate-equal", + Kinds: []string{"CandidateNode"}, + Properties: map[string]any{"lastseen": regressionDay(3)}, + }, + { + ID: "candidate-new", + Kinds: []string{"CandidateNode"}, + Properties: map[string]any{"lastseen": regressionDay(4)}, + }, + { + ID: "orphan-missing", + Kinds: []string{"CandidateNode"}, + Properties: map[string]any{"objectid": "S-1-5-100"}, + }, + { + ID: "orphan-null", + Kinds: []string{"CandidateNode"}, + Properties: map[string]any{"name": nil, "objectid": "S-1-5-101"}, + }, + { + ID: "orphan-empty", + Kinds: []string{"CandidateNode"}, + Properties: map[string]any{"name": "", "objectid": "S-1-5-102"}, + }, + { + ID: "orphan-wrong-prefix", + Kinds: []string{"CandidateNode"}, + Properties: map[string]any{"objectid": "X-1-5-103"}, + }, + { + ID: "orphan-protected", + Kinds: []string{"CandidateNode", "Domain"}, + Properties: map[string]any{"objectid": "S-1-5-104"}, + }, + }, + Edges: []opengraph.Edge{ + { + StartID: "late-a", + EndID: "early", + Kind: "SameForestTrust", + Properties: map[string]any{"lastseen": regressionDay(3), "marker": "same-old"}, + }, + { + StartID: "equal-a", + EndID: "equal-b", + Kind: "SameForestTrust", + Properties: map[string]any{"lastseen": regressionDay(3), "marker": "same-equal"}, + }, + { + StartID: "late-a", + EndID: "wrong-end", + Kind: "SameForestTrust", + Properties: map[string]any{"lastseen": regressionDay(3), "marker": "same-wrong-end"}, + }, + { + StartID: "late-a", + EndID: "early", + Kind: "CrossForestTrust", + Properties: map[string]any{"lastseen": regressionDay(3), "marker": "cross-old"}, + }, + { + StartID: "equal-a", + EndID: "equal-b", + Kind: "CrossForestTrust", + Properties: map[string]any{"lastseen": regressionDay(3), "marker": "cross-equal"}, + }, + { + StartID: "late-a", + EndID: "late-b", + Kind: "AbuseTGTDelegation", + Properties: map[string]any{"marker": "valid-forward-abuse"}, + }, + { + StartID: "late-b", + EndID: "late-a", + Kind: "SpoofSIDHistory", + Properties: map[string]any{"marker": "valid-reverse-spoof"}, + }, + { + StartID: "late-a", + EndID: "late-b", + Kind: "SpoofSIDHistory", + Properties: map[string]any{"marker": "invalid-forward-spoof"}, + }, + { + StartID: "late-b", + EndID: "late-a", + Kind: "AbuseTGTDelegation", + Properties: map[string]any{"marker": "invalid-reverse-abuse"}, + }, + { + StartID: "late-a", + EndID: "late-b", + Kind: "CandidateRel", + Properties: map[string]any{"lastseen": regressionDay(2), "marker": "candidate-old"}, + }, + { + StartID: "late-a", + EndID: "candidate-rel-equal", + Kind: "CandidateRel", + Properties: map[string]any{"lastseen": regressionDay(3), "marker": "candidate-equal"}, + }, + { + StartID: "late-a", + EndID: "candidate-rel-new", + Kind: "CandidateRel", + Properties: map[string]any{"lastseen": regressionDay(4), "marker": "candidate-new"}, + }, + { + StartID: "late-a", + EndID: "candidate-rel-missing", + Kind: "CandidateRel", + Properties: map[string]any{"marker": "candidate-missing"}, + }, + { + StartID: "late-a", + EndID: "candidate-rel-null", + Kind: "CandidateRel", + Properties: map[string]any{"lastseen": nil, "marker": "candidate-null"}, + }, + { + StartID: "late-a", + EndID: "late-b", + Kind: "HasSession", + Properties: map[string]any{"marker": "session-missing"}, + }, + { + StartID: "late-a", + EndID: "session-null", + Kind: "HasSession", + Properties: map[string]any{"lastseen": nil, "marker": "session-null"}, + }, + { + StartID: "late-a", + EndID: "session-old", + Kind: "HasSession", + Properties: map[string]any{"lastseen": regressionDay(2), "marker": "session-old"}, + }, + { + StartID: "late-a", + EndID: "session-equal", + Kind: "HasSession", + Properties: map[string]any{"lastseen": regressionDay(3), "marker": "session-equal"}, + }, + { + StartID: "late-a", + EndID: "session-new", + Kind: "HasSession", + Properties: map[string]any{"lastseen": regressionDay(4), "marker": "session-new"}, + }, + { + StartID: "late-a", + EndID: "late-b", + Kind: "MetaIncludes", + Properties: map[string]any{"lastseen": regressionDay(2), "marker": "meta-includes-old"}, + }, + }, + } +} + +// batchPruningFixture builds removable relationships and a high-degree node +// population for batched pruning tests and benchmarks. +func batchPruningFixture(fanout int) *opengraph.Graph { + fixture := &opengraph.Graph{ + Nodes: []opengraph.Node{ + { + ID: "rel-a", + Kinds: []string{"PruneEndpoint"}, + Properties: map[string]any{"name": "rel-a"}, + }, + { + ID: "rel-b", + Kinds: []string{"PruneEndpoint"}, + Properties: map[string]any{"name": "rel-b"}, + }, + { + ID: "rel-c", + Kinds: []string{"PruneEndpoint"}, + Properties: map[string]any{"name": "rel-c"}, + }, + { + ID: "single", + Kinds: []string{"PruneDeleteNode"}, + Properties: map[string]any{"objectid": "single", "remove": true}, + }, + { + ID: "high", + Kinds: []string{"PruneDeleteNode"}, + Properties: map[string]any{"objectid": "high", "remove": true}, + }, + { + ID: "survivor", + Kinds: []string{"PruneDeleteNode"}, + Properties: map[string]any{"objectid": "survivor", "remove": false}, + }, + }, + Edges: []opengraph.Edge{ + { + StartID: "rel-a", + EndID: "rel-b", + Kind: "PruneDelete", + Properties: map[string]any{"marker": "single"}, + }, + { + StartID: "rel-a", + EndID: "rel-c", + Kind: "PruneDelete", + Properties: map[string]any{"marker": "many-a"}, + }, + { + StartID: "rel-b", + EndID: "rel-a", + Kind: "PruneDelete", + Properties: map[string]any{"marker": "many-b"}, + }, + { + StartID: "rel-a", + EndID: "rel-b", + Kind: "PruneSurvivor", + Properties: map[string]any{"marker": "survivor"}, + }, + { + StartID: "survivor", + EndID: "rel-a", + Kind: "PruneIncident", + Properties: map[string]any{"marker": "survivor-incident"}, + }, + { + StartID: "high", + EndID: "high", + Kind: "PruneIncident", + Properties: map[string]any{"marker": "high-self"}, + }, + }, + } + + for idx, neighborID := range FixtureNames("neighbor", fanout) { + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: neighborID, + Kinds: []string{"PruneNeighbor"}, + Properties: map[string]any{"name": neighborID}, + }) + startID, endID := "high", neighborID + if idx%2 == 0 { + startID, endID = neighborID, "high" + } + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: startID, + EndID: endID, + Kind: "PruneIncident", + Properties: map[string]any{"marker": neighborID}, + }) + } + return fixture +} + +// pruneRelationshipsInBatches snapshots matching relationship IDs, invokes the selection hook, and deletes those IDs in one batch. +func pruneRelationshipsInBatches(ctx context.Context, db graph.Database, criteria graph.CriteriaProvider, afterSelect func([]graph.ID) error) (int, error) { + var ids []graph.ID + if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { + var err error + ids, err = ops.FetchRelationshipIDs(tx.Relationships().Filterf(criteria)) + return err + }); err != nil { + return 0, err + } + if afterSelect != nil { + if err := afterSelect(ids); err != nil { + return 0, err + } + } + + deleted := 0 + if err := db.BatchOperation(ctx, func(batch graph.Batch) error { + for _, id := range ids { + if err := batch.DeleteRelationship(id); err != nil { + return err + } + + deleted++ + } + return nil + }); err != nil { + return 0, err + } + + return deleted, nil +} + +// pruneNodesInBatches snapshots matching node IDs, invokes the selection hook, and deletes those IDs in one batch. +func pruneNodesInBatches(ctx context.Context, db graph.Database, criteria graph.CriteriaProvider, afterSelect func([]graph.ID) error) (int, error) { + var ids []graph.ID + if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { + var err error + ids, err = ops.FetchNodeIDs(tx.Nodes().Filterf(criteria)) + return err + }); err != nil { + return 0, err + } + if afterSelect != nil { + if err := afterSelect(ids); err != nil { + return 0, err + } + } + + deleted := 0 + if err := db.BatchOperation(ctx, func(batch graph.Batch) error { + for _, id := range ids { + if err := batch.DeleteNode(id); err != nil { + return err + } + + deleted++ + } + return nil + }); err != nil { + return 0, err + } + + return deleted, nil +} + +// regressionDay returns midnight UTC on the requested January 2026 day for deterministic temporal fixtures. +func regressionDay(day int) time.Time { + return time.Date(2026, time.January, day, 0, 0, 0, 0, time.UTC) +} diff --git a/integration/wipe_graph_test.go b/integration/wipe_graph_test.go index 0e1390f2..574ccf75 100644 --- a/integration/wipe_graph_test.go +++ b/integration/wipe_graph_test.go @@ -11,16 +11,23 @@ import ( "github.com/stretchr/testify/require" ) -// WipeGraph is a Postgres-only bulk-delete primitive, so this suite is scoped to the pg driver and skips itself unless -// CONNECTION_STRING selects a Postgres backend. +// TestWipeGraph verifies the PostgreSQL-only bulk-delete primitive and skips unless CONNECTION_STRING selects that backend. func TestWipeGraph(t *testing.T) { var ( wipeNode = graph.StringKind("WipeNode") survivor = graph.StringKind("WipeSurvivor") wipeEdge = graph.StringKind("WIPE_EDGE") - defaultGraph = graph.Graph{Name: "wipe_default", Nodes: graph.Kinds{wipeNode, survivor}, Edges: graph.Kinds{wipeEdge}} - secondaryGraph = graph.Graph{Name: "wipe_secondary", Nodes: graph.Kinds{wipeNode, survivor}, Edges: graph.Kinds{wipeEdge}} + defaultGraph = graph.Graph{ + Name: "wipe_default", + Nodes: graph.Kinds{wipeNode, survivor}, + Edges: graph.Kinds{wipeEdge}, + } + secondaryGraph = graph.Graph{ + Name: "wipe_secondary", + Nodes: graph.Kinds{wipeNode, survivor}, + Edges: graph.Kinds{wipeEdge}, + } schema = graph.Schema{ Graphs: []graph.Graph{defaultGraph, secondaryGraph}, @@ -73,7 +80,7 @@ func TestWipeGraph(t *testing.T) { session.ClearGraph(t) seed(t) - require.Equal(t, int64(3), countNodes(t, ctx, db)) + require.Equal(t, int64(3), countNodes(t, ctx, db, defaultGraph, secondaryGraph)) require.Equal(t, int64(1), countEdges(t, ctx, db)) require.NoError(t, wiper.WipeGraph(ctx, func(tx graph.Transaction) error { @@ -81,7 +88,7 @@ func TestWipeGraph(t *testing.T) { return err })) - require.Equal(t, int64(1), countNodes(t, ctx, db)) + require.Equal(t, int64(1), countNodes(t, ctx, db, defaultGraph, secondaryGraph)) require.Equal(t, int64(0), countEdges(t, ctx, db)) require.NoError(t, db.ReadTransaction(ctx, func(tx graph.Transaction) error { @@ -113,7 +120,7 @@ func TestWipeGraph(t *testing.T) { require.ErrorIs(t, err, errRetain) // The transaction rolled back, so the seeded graph is left untouched. - require.Equal(t, int64(3), countNodes(t, ctx, db)) + require.Equal(t, int64(3), countNodes(t, ctx, db, defaultGraph, secondaryGraph)) require.Equal(t, int64(1), countEdges(t, ctx, db)) }) @@ -123,25 +130,32 @@ func TestWipeGraph(t *testing.T) { require.NoError(t, wiper.WipeGraph(ctx, nil)) - require.Equal(t, int64(0), countNodes(t, ctx, db)) + require.Equal(t, int64(0), countNodes(t, ctx, db, defaultGraph, secondaryGraph)) require.Equal(t, int64(0), countEdges(t, ctx, db)) }) } -func countNodes(t *testing.T, ctx context.Context, db graph.Database) int64 { +// countNodes returns the total node count across graphs. +func countNodes(t *testing.T, ctx context.Context, db graph.Database, graphs ...graph.Graph) int64 { t.Helper() var count int64 require.NoError(t, db.ReadTransaction(ctx, func(tx graph.Transaction) error { - result, err := tx.Nodes().Count() - count = result - return err + for _, targetGraph := range graphs { + result, err := tx.WithGraph(targetGraph).Nodes().Count() + if err != nil { + return err + } + count += result + } + return nil })) return count } +// countEdges returns the relationship count in the database's current graph. func countEdges(t *testing.T, ctx context.Context, db graph.Database) int64 { t.Helper() diff --git a/query/builder_test.go b/query/builder_test.go index 0a44e20d..091e57f2 100644 --- a/query/builder_test.go +++ b/query/builder_test.go @@ -78,6 +78,31 @@ func TestBuilderProjectionModifiersAreOrderIndependent(t *testing.T) { } } +// TestBuilderRendersRawPropertyKeys verifies the legacy builder preserves escaped property-key syntax in rendered Cypher. +func TestBuilderRendersRawPropertyKeys(t *testing.T) { + builder := query.NewBuilder(nil) + builder.Apply(query.Returning( + query.NodeProperty("a-aaa"), + query.Property(query.Node(), "has`tick"), + query.Property(query.Node(), " "), + )) + + regularQuery, err := builder.Build(false) + if err != nil { + t.Fatalf("build query: %v", err) + } + + var cypher bytes.Buffer + if err := cypherFormat.NewCypherEmitter(false).Write(regularQuery, &cypher); err != nil { + t.Fatalf("render Cypher: %v", err) + } + + expected := "match (n) return n.`a-aaa`, n.`has``tick`, n.` `" + if cypher.String() != expected { + t.Fatalf("expected %q, got %q", expected, cypher.String()) + } +} + func assertRetrieverProjection(t *testing.T, rendered string) { t.Helper() diff --git a/query/neo4j/neo4j_test.go b/query/neo4j/neo4j_test.go index 1305ab23..c2fa801b 100644 --- a/query/neo4j/neo4j_test.go +++ b/query/neo4j/neo4j_test.go @@ -14,27 +14,45 @@ import ( ) var ( + // SystemTags is the synthetic system-tags property used by query-builder tests. SystemTags = "system_tags" - User = graph.StringKind("User") - Domain = graph.StringKind("Domain") - Computer = graph.StringKind("Computer") - Group = graph.StringKind("Group") - HasSession = graph.StringKind("HasSession") + // User is the user node kind used by query-builder fixtures. + User = graph.StringKind("User") + + // Domain is the domain node kind used by query-builder fixtures. + Domain = graph.StringKind("Domain") + + // Computer is the computer node kind used by query-builder fixtures. + Computer = graph.StringKind("Computer") + + // Group is the group node kind used by query-builder fixtures. + Group = graph.StringKind("Group") + + // HasSession is the relationship kind used by session-path fixtures. + HasSession = graph.StringKind("HasSession") + + // GenericWrite is the relationship kind used by generic-write fixtures. GenericWrite = graph.StringKind("GenericWrite") ) +// QueryOutputAssertion contains one accepted query rendering and parameter map. type QueryOutputAssertion struct { - Query string + // Query is the expected rendered Cypher text. + Query string + + // Parameters contains the expected query parameters. Parameters map[string]any } +// expectAnalysisError returns an assertion that requires query preparation to report an analysis error. func expectAnalysisError(rawQuery *cypher.RegularQuery) func(t *testing.T) { return func(t *testing.T) { require.NotNil(t, neo4j.NewQueryBuilder(rawQuery).Prepare()) } } +// assertQueryShortestPathResult prepares a shortest-path query and compares its rendered text and optional parameters. func assertQueryShortestPathResult(rawQuery *cypher.RegularQuery, expectedOutput string, expectedParameters ...map[string]any) func(t *testing.T) { return func(t *testing.T) { builder := neo4j.NewQueryBuilder(rawQuery) @@ -53,6 +71,7 @@ func assertQueryShortestPathResult(rawQuery *cypher.RegularQuery, expectedOutput } } +// assertQueryResult prepares a query and compares its rendered text and optional parameters. func assertQueryResult(rawQuery *cypher.RegularQuery, expectedOutput string, expectedParameters ...map[string]any) func(t *testing.T) { return func(t *testing.T) { var ( @@ -76,6 +95,7 @@ func assertQueryResult(rawQuery *cypher.RegularQuery, expectedOutput string, exp } } +// assertOneOfQueryResult requires a prepared query to match one accepted rendering and parameter set. func assertOneOfQueryResult(rawQuery *cypher.RegularQuery, expectations []QueryOutputAssertion) func(t *testing.T) { return func(t *testing.T) { builder := neo4j.NewQueryBuilder(rawQuery) @@ -206,7 +226,612 @@ func TestQueryBuilderProjectionModifiersAreOrderIndependent(t *testing.T) { } } +// TestQueryBuilder_LOGIC01PreservesBranchLocalRelationshipKinds verifies disjunctive branches retain their own relationship-kind predicates. +func TestQueryBuilder_LOGIC01PreservesBranchLocalRelationshipKinds(t *testing.T) { + rawQuery := query.SinglePartQuery( + query.Where( + query.Or( + query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Equals(query.EndID(), graph.ID(202)), + query.KindIn(query.Relationship(), graph.StringKind("KindA")), + ), + query.And( + query.Equals(query.StartID(), graph.ID(202)), + query.Equals(query.EndID(), graph.ID(101)), + query.KindIn(query.Relationship(), graph.StringKind("KindB")), + ), + ), + ), + query.Returning(query.RelationshipID()), + ) + + assertQueryResult( + rawQuery, + "match (s)-[r]->(e) where (id(s) = $p0 and id(e) = $p1 and r:KindA or id(s) = $p2 and id(e) = $p3 and r:KindB) return id(r)", + map[string]any{ + "p0": graph.ID(101), + "p1": graph.ID(202), + "p2": graph.ID(202), + "p3": graph.ID(101), + }, + )(t) +} + +// TestQueryBuilder_LogicalForms verifies Neo4j rendering preserves supported logical expression shapes and precedence. +func TestQueryBuilder_LogicalForms(t *testing.T) { + temporalThreshold := time.Date(2026, time.January, 2, 3, 4, 5, 0, time.UTC) + + t.Run("LOGIC-02 cross-binding temporal disjunction", assertQueryResult( + query.SinglePartQuery( + query.Where( + query.Or( + query.BeforeGraphQuery(query.RelationshipProperty("lastseen"), query.StartProperty("lastcollected")), + query.BeforeGraphQuery(query.RelationshipProperty("lastseen"), query.EndProperty("lastcollected")), + ), + ), + query.Returning(query.RelationshipID()), + ), + "match (s)-[r]->(e) where (r.lastseen < s.lastcollected or r.lastseen < e.lastcollected) return id(r)", + )) + + t.Run("LOGIC-03 scoped negation and null-aware age predicate", assertQueryResult( + query.SinglePartQuery( + query.Where( + query.And( + query.Not(query.KindIn(query.Node(), graph.StringKind("Protected"))), + query.Or( + query.Not(query.Exists(query.NodeProperty("lastseen"))), + query.Before(query.NodeProperty("lastseen"), temporalThreshold), + ), + ), + ), + query.Returning(query.NodeID()), + ), + "match (n) where not (n:Protected) and (not (n.lastseen is not null) or n.lastseen < $p0) return id(n)", + map[string]any{"p0": temporalThreshold}, + )) +} + +// TestQueryBuilder_LOGIC05ProjectionOrder verifies projection ordering remains stable for the LOGIC-05 regression form. +func TestQueryBuilder_LOGIC05ProjectionOrder(t *testing.T) { + testCases := map[string]struct { + // projection is the return clause under test. + projection *cypher.Return + + // expected is the rendered Cypher query. + expected string + }{ + "full opposite node plus relationship": { + projection: query.Returning(query.Relationship(), query.End()), + expected: "match ()-[r]->(e) return r, e", + }, + "opposite ID and kinds plus relationship ID and kind": { + projection: query.Returning(query.EndID(), query.KindsOf(query.End()), query.RelationshipID(), query.KindsOf(query.Relationship())), + expected: "match ()-[r]->(e) return id(e), labels(e), id(r), type(r)", + }, + "start relationship end triple": { + projection: query.Returning(query.Start(), query.Relationship(), query.End()), + expected: "match (s)-[r]->(e) return s, r, e", + }, + "relationship ID only": { + projection: query.Returning(query.RelationshipID()), + expected: "match ()-[r]->() return id(r)", + }, + "full relationship": { + projection: query.Returning(query.Relationship()), + expected: "match ()-[r]->() return r", + }, + } + + for name, testCase := range testCases { + t.Run(name, assertQueryResult( + query.SinglePartQuery(testCase.projection), + testCase.expected, + )) + } +} + +// TestQueryBuilder_ReconciliationForms verifies reconciliation forms render the expected predicates and projections. +func TestQueryBuilder_ReconciliationForms(t *testing.T) { + reconciliationKinds := func(count int) graph.Kinds { + kinds := make(graph.Kinds, count) + for idx := range count { + kinds[idx] = graph.StringKind(fmt.Sprintf("ReconcileKind%02d", idx+1)) + } + return kinds + } + + for _, count := range []int{1, 2, 9, 30} { + kinds := reconciliationKinds(count) + renderedKinds := "ReconcileKind01" + for idx := 1; idx < count; idx++ { + renderedKinds += fmt.Sprintf("|ReconcileKind%02d", idx+1) + } + + t.Run(fmt.Sprintf("REC-01 inbound relationship delete with %d kinds", count), assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.End(), graph.StringKind("ADEntity")), + query.Equals(query.EndProperty("objectid"), "target-id"), + query.KindIn(query.Relationship(), kinds...), + )), + query.Delete(query.Relationship()), + ), + fmt.Sprintf("match ()-[r:%s]->(e) where e:ADEntity and e.objectid = $p0 delete r", renderedKinds), + map[string]any{"p0": "target-id"}, + )) + + t.Run(fmt.Sprintf("REC-02 outbound relationship delete with %d kinds", count), assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Start(), graph.StringKind("ADEntity")), + query.Equals(query.StartProperty("objectid"), "target-id"), + query.KindIn(query.Relationship(), kinds...), + )), + query.Delete(query.Relationship()), + ), + fmt.Sprintf("match (s)-[r:%s]->() where s:ADEntity and s.objectid = $p0 delete r", renderedKinds), + map[string]any{"p0": "target-id"}, + )) + } + + t.Run("REC-03 inbound primary-group relationship delete", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.End(), graph.StringKind("Group")), + query.Equals(query.EndProperty("objectid"), "group-id"), + query.Kind(query.Relationship(), graph.StringKind("MemberOf")), + query.Equals(query.RelationshipProperty("isprimarygroup"), false), + )), + query.Delete(query.Relationship()), + ), + "match ()-[r:MemberOf]->(e) where e:Group and e.objectid = $p0 and r.isprimarygroup = $p1 delete r", + map[string]any{"p0": "group-id", "p1": false}, + )) + + t.Run("REC-03 outbound primary-group relationship delete", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Start(), graph.StringKind("Computer")), + query.Equals(query.StartProperty("objectid"), "computer-id"), + query.Kind(query.Relationship(), graph.StringKind("MemberOf")), + query.Equals(query.RelationshipProperty("isprimarygroup"), true), + )), + query.Delete(query.Relationship()), + ), + "match (s)-[r:MemberOf]->() where s:Computer and s.objectid = $p0 and r.isprimarygroup = $p1 delete r", + map[string]any{"p0": "computer-id", "p1": true}, + )) + + t.Run("REC-04 endpoint object ID list relationship delete", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Relationship(), graph.StringKind("ReconcileKind01")), + query.Kind(query.End(), graph.StringKind("ADEntity")), + query.In(query.EndProperty("objectid"), []string{"target-1", "target-2"}), + )), + query.Delete(query.Relationship()), + ), + "match ()-[r:ReconcileKind01]->(e) where e:ADEntity and e.objectid in $p0 delete r", + map[string]any{"p0": []string{"target-1", "target-2"}}, + )) + + t.Run("REC-05 delegated enrollment discovery projection", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.In(query.EndProperty("objectid"), []string{"ca-1", "ca-2"}), + query.Kind(query.Relationship(), graph.StringKind("PublishedTo")), + query.Kind(query.Start(), graph.StringKind("CertTemplate")), + )), + query.Returning(query.Relationship(), query.Start()), + ), + "match (s)-[r:PublishedTo]->(e) where e.objectid in $p0 and s:CertTemplate return r, s", + map[string]any{"p0": []string{"ca-1", "ca-2"}}, + )) + + t.Run("REC-06 delegated enrollment relationship delete by end IDs", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.End(), graph.StringKind("CertTemplate")), + query.InIDs(query.EndID(), graph.ID(101), graph.ID(202)), + query.KindIn(query.Relationship(), graph.StringKind("DelegatedEnrollmentAgent")), + )), + query.Delete(query.Relationship()), + ), + "match ()-[r:DelegatedEnrollmentAgent]->(e) where e:CertTemplate and id(e) in $p0 delete r", + map[string]any{"p0": []graph.ID{101, 202}}, + )) + + t.Run("REC-07 HostsCAService relationship delete", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.End(), graph.StringKind("EnterpriseCA")), + query.Equals(query.EndProperty("objectid"), "ca-id"), + query.KindIn(query.Relationship(), graph.StringKind("HostsCAService")), + )), + query.Delete(query.Relationship()), + ), + "match ()-[r:HostsCAService]->(e) where e:EnterpriseCA and e.objectid = $p0 delete r", + map[string]any{"p0": "ca-id"}, + )) + + t.Run("REC-08 AD entity detach delete", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Node(), graph.StringKind("ADEntity")), + query.In(query.NodeProperty("objectid"), []string{"target-1", "target-2"}), + )), + query.Delete(query.Node()), + ), + "match (n) where n:ADEntity and n.objectid in $p0 detach delete n", + map[string]any{"p0": []string{"target-1", "target-2"}}, + )) +} + +// TestQueryBuilder_TrustAndPruningForms verifies trust and pruning forms preserve selector and mutation semantics. +func TestQueryBuilder_TrustAndPruningForms(t *testing.T) { + threshold := time.Date(2026, time.January, 3, 0, 0, 0, 0, time.UTC) + domain := graph.StringKind("Domain") + + t.Run("TRUST-01 SameForestTrust ID projection", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Start(), domain), + query.Kind(query.End(), domain), + query.Kind(query.Relationship(), graph.StringKind("SameForestTrust")), + query.Or( + query.BeforeGraphQuery(query.RelationshipProperty("lastseen"), query.StartProperty("lastcollected")), + query.BeforeGraphQuery(query.RelationshipProperty("lastseen"), query.EndProperty("lastcollected")), + ), + )), + query.Returning(query.RelationshipID()), + ), + "match (s)-[r:SameForestTrust]->(e) where s:Domain and e:Domain and (r.lastseen < s.lastcollected or r.lastseen < e.lastcollected) return id(r)", + )) + + t.Run("TRUST-02 CrossForestTrust full relationship projection", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Start(), domain), + query.Kind(query.End(), domain), + query.KindIn(query.Relationship(), graph.StringKind("CrossForestTrust")), + query.Or( + query.BeforeGraphQuery(query.RelationshipProperty("lastseen"), query.StartProperty("lastcollected")), + query.BeforeGraphQuery(query.RelationshipProperty("lastseen"), query.EndProperty("lastcollected")), + ), + )), + query.Returning(query.Relationship()), + ), + "match (s)-[r:CrossForestTrust]->(e) where s:Domain and e:Domain and (r.lastseen < s.lastcollected or r.lastseen < e.lastcollected) return r", + )) + + t.Run("TRUST-03 directional derived trust disjunction", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Start(), domain), + query.Kind(query.End(), domain), + query.Or( + query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Equals(query.EndID(), graph.ID(202)), + query.KindIn(query.Relationship(), graph.StringKind("AbuseTGTDelegation")), + ), + query.And( + query.Equals(query.StartID(), graph.ID(202)), + query.Equals(query.EndID(), graph.ID(101)), + query.KindIn(query.Relationship(), graph.StringKind("SpoofSIDHistory")), + ), + ), + )), + query.Returning(query.RelationshipID()), + ), + "match (s)-[r]->(e) where s:Domain and e:Domain and (id(s) = $p0 and id(e) = $p1 and r:AbuseTGTDelegation or id(s) = $p2 and id(e) = $p3 and r:SpoofSIDHistory) return id(r)", + map[string]any{"p0": graph.ID(101), "p1": graph.ID(202), "p2": graph.ID(202), "p3": graph.ID(101)}, + )) + + t.Run("PRUNE-01 relationship TTL excludes several kinds", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Not(query.KindIn(query.Relationship(), graph.StringKind("MetaIncludes"), graph.StringKind("HasSession"))), + query.Before(query.RelationshipProperty("lastseen"), threshold), + )), + query.Returning(query.RelationshipID()), + ), + "match ()-[r]->() where not ((r:MetaIncludes or r:HasSession)) and r.lastseen < $p0 return id(r)", + map[string]any{"p0": threshold}, + )) + + t.Run("PRUNE-02 HasSession missing or stale TTL", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.KindIn(query.Relationship(), graph.StringKind("HasSession")), + query.Or( + query.Not(query.Exists(query.RelationshipProperty("lastseen"))), + query.Before(query.RelationshipProperty("lastseen"), threshold), + ), + )), + query.Returning(query.RelationshipID()), + ), + "match ()-[r:HasSession]->() where (not (r.lastseen is not null) or r.lastseen < $p0) return id(r)", + map[string]any{"p0": threshold}, + )) + + t.Run("PRUNE-03 node TTL excludes several kinds", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Not(query.KindIn(query.Node(), graph.StringKind("Domain"), graph.StringKind("Tenant"), graph.StringKind("Meta"), graph.StringKind("MetaIncludes"), graph.StringKind("MigrationData"))), + query.Or( + query.Not(query.Exists(query.NodeProperty("lastseen"))), + query.Before(query.NodeProperty("lastseen"), threshold), + ), + )), + query.Returning(query.NodeID()), + ), + "match (n) where not ((n:Domain or n:Tenant or n:Meta or n:MetaIncludes or n:MigrationData)) and (not (n.lastseen is not null) or n.lastseen < $p0) return id(n)", + map[string]any{"p0": threshold}, + )) + + t.Run("PRUNE-04 orphan SID prefix", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Not(query.KindIn(query.Node(), graph.StringKind("Domain"), graph.StringKind("Tenant"), graph.StringKind("Meta"), graph.StringKind("MetaIncludes"), graph.StringKind("MigrationData"))), + query.Not(query.Exists(query.NodeProperty("name"))), + query.StringStartsWith(query.NodeProperty("objectid"), "S-1-5"), + )), + query.Returning(query.NodeID()), + ), + "match (n) where not ((n:Domain or n:Tenant or n:Meta or n:MetaIncludes or n:MigrationData)) and not (n.name is not null) and n.objectid starts with $p0 return id(n)", + map[string]any{"p0": "S-1-5"}, + )) +} + +// TestQueryBuilder_StandaloneHopForms verifies one-hop forms preserve direction, kinds, and endpoint projections. +func TestQueryBuilder_StandaloneHopForms(t *testing.T) { + hopKinds := func(count int) graph.Kinds { + kinds := make(graph.Kinds, count) + for idx := range count { + kinds[idx] = graph.StringKind(fmt.Sprintf("HopKind%02d", idx+1)) + } + return kinds + } + + t.Run("HOP-01 outbound exact start anchor with full directional projection", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("HopKind01")), + )), + query.Returning(query.Relationship(), query.End()), + ), + "match (s)-[r:HopKind01]->(e) where id(s) = $p0 return r, e", + map[string]any{"p0": graph.ID(101)}, + )) + + t.Run("HOP-01 outbound one-element start IN anchor", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.InIDs(query.StartID(), graph.ID(101)), + query.KindIn(query.Relationship(), graph.StringKind("HopKind01")), + )), + query.Returning(query.Relationship(), query.End()), + ), + "match (s)-[r:HopKind01]->(e) where id(s) in $p0 return r, e", + map[string]any{"p0": []graph.ID{101}}, + )) + + t.Run("HOP-02 inbound exact end anchor with full directional projection", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Equals(query.EndID(), graph.ID(202)), + query.Kind(query.Relationship(), graph.StringKind("HopKind01")), + )), + query.Returning(query.Relationship(), query.Start()), + ), + "match (s)-[r:HopKind01]->(e) where id(e) = $p0 return r, s", + map[string]any{"p0": graph.ID(202)}, + )) + + for _, count := range []int{2, 5, 9, 30} { + kinds := hopKinds(count) + renderedKinds := "HopKind01" + for idx := 1; idx < count; idx++ { + renderedKinds += fmt.Sprintf("|HopKind%02d", idx+1) + } + + t.Run(fmt.Sprintf("HOP-03 outbound %d relationship kinds", count), assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.InIDs(query.StartID(), graph.ID(101)), + query.KindIn(query.Relationship(), kinds...), + )), + query.Returning(query.Relationship(), query.End()), + ), + fmt.Sprintf("match (s)-[r:%s]->(e) where id(s) in $p0 return r, e", renderedKinds), + map[string]any{"p0": []graph.ID{101}}, + )) + + t.Run(fmt.Sprintf("HOP-03 inbound %d relationship kinds", count), assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.InIDs(query.EndID(), graph.ID(202)), + query.KindIn(query.Relationship(), kinds...), + )), + query.Returning(query.Relationship(), query.Start()), + ), + fmt.Sprintf("match (s)-[r:%s]->(e) where id(e) in $p0 return r, s", renderedKinds), + map[string]any{"p0": []graph.ID{202}}, + )) + } + + t.Run("HOP-04 opposite endpoint kind disjunction", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.InIDs(query.StartID(), graph.ID(101)), + query.KindIn(query.Relationship(), graph.StringKind("HopTypedEdge")), + query.KindIn(query.End(), graph.StringKind("HopEndA"), graph.StringKind("HopEndB")), + )), + query.Returning(query.Relationship(), query.End()), + ), + "match (s)-[r:HopTypedEdge]->(e) where id(s) in $p0 and (e:HopEndA or e:HopEndB) return r, e", + map[string]any{"p0": []graph.ID{101}}, + )) + + t.Run("HOP-05 endpoint IDs through variable spelling", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("HopIDEdge")), + query.InIDs(query.End(), graph.ID(202), graph.ID(303)), + )), + query.Returning(query.Relationship(), query.End()), + ), + "match (s)-[r:HopIDEdge]->(e) where id(s) = $p0 and id(e) in $p1 return r, e", + map[string]any{"p0": graph.ID(101), "p1": []graph.ID{202, 303}}, + )) + + t.Run("HOP-05 endpoint IDs through identity-function spelling", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.InIDs(query.Start(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("HopIDEdge")), + query.InIDs(query.EndID(), graph.ID(202), graph.ID(303)), + )), + query.Returning(query.Relationship(), query.End()), + ), + "match (s)-[r:HopIDEdge]->(e) where id(s) in $p0 and id(e) in $p1 return r, e", + map[string]any{"p0": []graph.ID{101}, "p1": []graph.ID{202, 303}}, + )) + + t.Run("HOP-06 opposite endpoint scalar properties", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("HopPropertyEdge")), + query.Equals(query.EndProperty("enabled"), true), + query.Equals(query.EndProperty("score"), 7), + query.Equals(query.EndProperty("name"), "target"), + query.Equals(query.EndProperty("isassignabletorole"), "true"), + )), + query.Returning(query.Relationship(), query.End()), + ), + "match (s)-[r:HopPropertyEdge]->(e) where id(s) = $p0 and e.enabled = $p1 and e.score = $p2 and e.name = $p3 and e.isassignabletorole = $p4 return r, e", + map[string]any{"p0": graph.ID(101), "p1": true, "p2": 7, "p3": "target", "p4": "true"}, + )) + + t.Run("HOP-07 nested production-style endpoint predicate", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.KindIn(query.Relationship(), graph.StringKind("HopNestedEdge")), + query.Kind(query.End(), graph.StringKind("HopTemplate")), + query.Or( + query.And( + query.Equals(query.EndProperty("requiresmanagerapproval"), false), + query.GreaterThan(query.EndProperty("schemaversion"), 1), + query.Equals(query.EndProperty("authorizedsignatures"), 0), + query.Equals(query.EndProperty("authenticationenabled"), true), + ), + query.And( + query.Equals(query.EndProperty("requiresmanagerapproval"), false), + query.Equals(query.EndProperty("schemaversion"), 1), + query.Equals(query.EndProperty("authenticationenabled"), true), + ), + ), + )), + query.Returning(query.Relationship(), query.End()), + ), + "match (s)-[r:HopNestedEdge]->(e) where id(s) = $p0 and e:HopTemplate and (e.requiresmanagerapproval = $p1 and e.schemaversion > $p2 and e.authorizedsignatures = $p3 and e.authenticationenabled = $p4 or e.requiresmanagerapproval = $p5 and e.schemaversion = $p6 and e.authenticationenabled = $p7) return r, e", + map[string]any{"p0": graph.ID(101), "p1": false, "p2": 1, "p3": 0, "p4": true, "p5": false, "p6": 1, "p7": true}, + )) + + t.Run("HOP-08 collection predicates nested with scalar fallback", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("HopCollectionEdge")), + query.Or( + query.Equals(query.EndProperty("schannelauthenticationenabled"), true), + query.Equals(query.Size(query.EndProperty("effectiveekus")), 0), + query.InInverted(query.EndProperty("effectiveekus"), "1.3.6.1.5.5.7.3.2"), + ), + )), + query.Returning(query.Relationship(), query.End()), + ), + "match (s)-[r:HopCollectionEdge]->(e) where id(s) = $p0 and (e.schannelauthenticationenabled = $p1 or size(e.effectiveekus) = $p2 or $p3 in e.effectiveekus) return r, e", + map[string]any{"p0": graph.ID(101), "p1": true, "p2": 0, "p3": "1.3.6.1.5.5.7.3.2"}, + )) + + t.Run("HOP-09 two-sided endpoint ID lists", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.InIDs(query.StartID(), graph.ID(101), graph.ID(202)), + query.InIDs(query.EndID(), graph.ID(303), graph.ID(404)), + query.Kind(query.Relationship(), graph.StringKind("HopSetEdge")), + )), + query.Returning(query.Relationship(), query.End()), + ), + "match (s)-[r:HopSetEdge]->(e) where id(s) in $p0 and id(e) in $p1 return r, e", + map[string]any{"p0": []graph.ID{101, 202}, "p1": []graph.ID{303, 404}}, + )) + + t.Run("HOP-10 outbound endpoint kind property and start anchor", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.InIDs(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("HopProjectionEdge")), + query.Kind(query.End(), graph.StringKind("HopProjectionEnd")), + query.Equals(query.EndProperty("active"), true), + )), + query.Returning(query.Relationship(), query.End()), + ), + "match (s)-[r:HopProjectionEdge]->(e) where id(s) in $p0 and e:HopProjectionEnd and e.active = $p1 return r, e", + map[string]any{"p0": []graph.ID{101}, "p1": true}, + )) + + t.Run("HOP-10 inbound endpoint kind property and end anchor", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.InIDs(query.EndID(), graph.ID(202)), + query.Kind(query.Relationship(), graph.StringKind("HopProjectionEdge")), + query.Kind(query.Start(), graph.StringKind("HopProjectionStart")), + query.Equals(query.StartProperty("active"), true), + )), + query.Returning(query.Relationship(), query.Start()), + ), + "match (s)-[r:HopProjectionEdge]->(e) where id(e) in $p0 and s:HopProjectionStart and s.active = $p1 return r, s", + map[string]any{"p0": []graph.ID{202}, "p1": true}, + )) + + t.Run("HOP-10 explicit start-node projection", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.InIDs(query.EndID(), graph.ID(202)), + query.Kind(query.Relationship(), graph.StringKind("HopProjectionEdge")), + )), + query.Returning(query.Start()), + ), + "match (s)-[r:HopProjectionEdge]->(e) where id(e) in $p0 return s", + map[string]any{"p0": []graph.ID{202}}, + )) + + t.Run("HOP-10 explicit end-ID and relationship projection", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.InIDs(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("HopProjectionEdge")), + )), + query.Returning(query.EndID(), query.Relationship()), + ), + "match (s)-[r:HopProjectionEdge]->(e) where id(s) in $p0 return id(e), r", + map[string]any{"p0": []graph.ID{101}}, + )) +} + +// TestQueryBuilder_Render verifies legacy query criteria render the expected Neo4j Cypher and parameters. func TestQueryBuilder_Render(t *testing.T) { + temporalThreshold := time.Date(2026, time.January, 2, 3, 4, 5, 0, time.UTC) + // Node Queries t.Run("Node Count", assertQueryResult(query.SinglePartQuery( query.Where( @@ -555,7 +1180,7 @@ func TestQueryBuilder_Render(t *testing.T) { t.Run("Node Datetime Before", assertQueryResult(query.SinglePartQuery( query.Where( query.And( - query.Before(query.NodeProperty("lastseen"), time.Now().UTC()), + query.Before(query.NodeProperty("lastseen"), temporalThreshold), query.In(query.NodeID(), []int{1, 2, 3, 4}), ), ), @@ -563,7 +1188,10 @@ func TestQueryBuilder_Render(t *testing.T) { query.Returning( query.Node(), ), - ), "match (n) where n.lastseen < $p0 and id(n) in $p1 return n")) + ), "match (n) where n.lastseen < $p0 and id(n) in $p1 return n", map[string]any{ + "p0": temporalThreshold, + "p1": []int{1, 2, 3, 4}, + })) t.Run("Node Datetime Before or Equal to", assertQueryResult(query.SinglePartQuery( query.Where( diff --git a/query/neo4j/relationship_scans_node_lookups_test.go b/query/neo4j/relationship_scans_node_lookups_test.go new file mode 100644 index 00000000..d84eaf82 --- /dev/null +++ b/query/neo4j/relationship_scans_node_lookups_test.go @@ -0,0 +1,419 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package neo4j_test + +import ( + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/query" +) + +// scanLookupKinds converts fixture kind names into the graph.Kinds accepted by query helpers. +func scanLookupKinds(names ...string) graph.Kinds { + kinds := make(graph.Kinds, len(names)) + for idx, name := range names { + kinds[idx] = graph.StringKind(name) + } + return kinds +} + +// TestQueryBuilder_RelationshipScans verifies relationship scan forms render kind, endpoint, property, and ordering constraints correctly. +func TestQueryBuilder_RelationshipScans(t *testing.T) { + t.Run("SCAN-01 base endpoints and relationship ID projection", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.KindIn(query.Start(), scanLookupKinds("ADBase", "AZBase")...), + query.Kind(query.Relationship(), graph.StringKind("PostProcessed")), + query.KindIn(query.End(), scanLookupKinds("ADBase", "AZBase")...), + )), + query.Returning(query.RelationshipID()), + ), + "match (s)-[r:PostProcessed]->(e) where (s:ADBase or s:AZBase) and (e:ADBase or e:AZBase) return id(r)", + )) + + t.Run("SCAN-02 excludes Meta endpoints and hydrates relationships", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Not(query.KindIn(query.Start(), scanLookupKinds("Meta", "MetaDetail")...)), + query.KindIn(query.Relationship(), scanLookupKinds("TrackerA", "TrackerB")...), + query.Not(query.KindIn(query.End(), scanLookupKinds("Meta", "MetaDetail")...)), + )), + query.Returning(query.Relationship()), + ), + "match (s)-[r:TrackerA|TrackerB]->(e) where not ((s:Meta or s:MetaDetail)) and not ((e:Meta or e:MetaDetail)) return r", + )) + + t.Run("SCAN-03 non-Meta lastseen relationship IDs", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Not(query.KindIn(query.Start(), scanLookupKinds("Meta", "MetaDetail")...)), + query.Kind(query.Relationship(), graph.StringKind("MigratedEdge")), + query.Exists(query.RelationshipProperty("lastseen")), + query.Not(query.KindIn(query.End(), scanLookupKinds("Meta", "MetaDetail")...)), + )), + query.Returning(query.RelationshipID()), + ), + "match (s)-[r:MigratedEdge]->(e) where not ((s:Meta or s:MetaDetail)) and r.lastseen is not null and not ((e:Meta or e:MetaDetail)) return id(r)", + )) + + t.Run("SCAN-04 raw ownership scan", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Relationship(), graph.StringKind("OwnsRaw")), + query.Kind(query.Start(), graph.StringKind("Entity")), + )), + query.Returning(query.Relationship()), + ), + "match (s)-[r:OwnsRaw]->() where s:Entity return r", + )) + + nineKinds := scanLookupKinds("ScanEdge01", "ScanEdge02", "ScanEdge03", "ScanEdge04", "ScanEdge05", "ScanEdge06", "ScanEdge07", "ScanEdge08", "ScanEdge09") + t.Run("SCAN-05 consolidated nine-kind inbound scan", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Start(), graph.StringKind("Entity")), + query.KindIn(query.Relationship(), nineKinds...), + query.Equals(query.EndID(), graph.ID(202)), + )), + query.Returning(query.Relationship(), query.Start()), + ), + "match (s)-[r:ScanEdge01|ScanEdge02|ScanEdge03|ScanEdge04|ScanEdge05|ScanEdge06|ScanEdge07|ScanEdge08|ScanEdge09]->(e) where s:Entity and id(e) = $p0 return r, s", + map[string]any{"p0": graph.ID(202)}, + )) + + t.Run("SCAN-06 FetchKinds projection order", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Relationship(), graph.StringKind("LocalToComputer")), + query.Kind(query.End(), graph.StringKind("Computer")), + )), + query.Returning(query.StartID(), query.RelationshipID(), query.KindsOf(query.Relationship()), query.EndID()), + ), + "match (s)-[r:LocalToComputer]->(e) where e:Computer return id(s), id(r), type(r), id(e)", + )) + + t.Run("SCAN-07 directed ID pair projection", assertQueryResult( + query.SinglePartQuery( + query.Where(query.KindIn(query.Relationship(), scanLookupKinds("MemberOf", "MemberOfLocalGroup")...)), + query.Returning(query.StartID(), query.EndID()), + ), + "match (s)-[r:MemberOf|MemberOfLocalGroup]->(e) return id(s), id(e)", + )) + + startKinds := scanLookupKinds("Group", "User", "Computer") + t.Run("SCAN-08 scenario A", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.KindIn(query.Start(), startKinds...), + query.InIDs(query.EndID(), graph.ID(202), graph.ID(303)), + query.KindIn(query.Relationship(), scanLookupKinds("GenericAll", "GenericWrite", "Owns", "WriteOwner", "WriteDACL", "WritePublicInformation")...), + )), + query.Returning(query.StartID()), + ), + "match (s)-[r:GenericAll|GenericWrite|Owns|WriteOwner|WriteDACL|WritePublicInformation]->(e) where (s:Group or s:User or s:Computer) and id(e) in $p0 return id(s)", + map[string]any{"p0": []graph.ID{202, 303}}, + )) + + t.Run("SCAN-08 scenario B", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.KindIn(query.Start(), startKinds...), + query.InIDs(query.EndID(), graph.ID(202), graph.ID(303)), + query.Kind(query.End(), graph.StringKind("Computer")), + query.KindIn(query.Relationship(), scanLookupKinds("GenericAll", "GenericWrite", "Owns", "WriteOwner", "WriteDACL")...), + )), + query.Returning(query.StartID()), + ), + "match (s)-[r:GenericAll|GenericWrite|Owns|WriteOwner|WriteDACL]->(e) where (s:Group or s:User or s:Computer) and id(e) in $p0 and e:Computer return id(s)", + map[string]any{"p0": []graph.ID{202, 303}}, + )) +} + +// TestQueryBuilder_NodeLookups verifies node lookup forms render identifiers, kind filters, and property predicates correctly. +func TestQueryBuilder_NodeLookups(t *testing.T) { + t.Run("LOOKUP-01 kind disjunction ID projection", assertQueryResult( + query.SinglePartQuery( + query.Where(query.KindIn(query.Node(), scanLookupKinds("Group", "User")...)), + query.Returning(query.NodeID()), + ), + "match (n) where (n:Group or n:User) return id(n)", + )) + + t.Run("LOOKUP-01 exact kind full hydration", assertQueryResult( + query.SinglePartQuery( + query.Where(query.Kind(query.Node(), graph.StringKind("Tenant"))), + query.Returning(query.Node()), + ), + "match (n) where n:Tenant return n", + )) + + t.Run("LOOKUP-02 indexed equality and LIMIT 1", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Node(), graph.StringKind("Computer")), + query.Equals(query.NodeProperty("objectid"), "S-1-5-21"), + )), + query.Returning(query.Node()), + query.Limit(1), + ), + "match (n) where n:Computer and n.objectid = $p0 return n limit 1", + map[string]any{"p0": "S-1-5-21"}, + )) + + t.Run("LOOKUP-02 no-kind two-property equality", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Equals(query.NodeProperty("name"), "dc.example.test"), + query.Equals(query.NodeProperty("enabled"), true), + )), + query.Returning(query.NodeID()), + ), + "match (n) where n.name = $p0 and n.enabled = $p1 return id(n)", + map[string]any{"p0": "dc.example.test", "p1": true}, + )) + + t.Run("LOOKUP-03 boolean property projection order", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Node(), graph.StringKind("Computer")), + query.Equals(query.NodeProperty("hasura"), true), + )), + query.Returning(query.NodeID(), query.NodeProperty("hasura")), + ), + "match (n) where n:Computer and n.hasura = $p0 return id(n), n.hasura", + map[string]any{"p0": true}, + )) + + t.Run("LOOKUP-04 prefix and domain equality", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Node(), graph.StringKind("Container")), + query.StringStartsWith(query.NodeProperty("distinguishedname"), "CN=ADMINSDHOLDER,CN=SYSTEM,"), + query.Equals(query.NodeProperty("domainsid"), "S-1-5-21"), + )), + query.Returning(query.Node()), + ), + "match (n) where n:Container and n.distinguishedname starts with $p0 and n.domainsid = $p1 return n", + map[string]any{"p0": "CN=ADMINSDHOLDER,CN=SYSTEM,", "p1": "S-1-5-21"}, + )) + + t.Run("LOOKUP-04 suffix disjunction", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Node(), graph.StringKind("Group")), + query.Or( + query.StringEndsWith(query.NodeProperty("objectid"), "-S-1"), + query.StringEndsWith(query.NodeProperty("objectid"), "-S-2"), + ), + )), + query.Returning(query.NodeID()), + ), + "match (n) where n:Group and (n.objectid ends with $p0 or n.objectid ends with $p1) return id(n)", + map[string]any{"p0": "-S-1", "p1": "-S-2"}, + )) + + t.Run("LOOKUP-05 case-insensitive prefix", assertQueryResult( + query.SinglePartQuery( + query.Where(query.CaseInsensitiveStringStartsWith(query.NodeProperty("name"), "Remote Desktop Users%_")), + query.Returning(query.NodeID()), + ), + "match (n) where toLower(n.name) starts with $p0 return id(n)", + map[string]any{"p0": "remote desktop users%_"}, + )) + + t.Run("LOOKUP-05 case-insensitive contains", assertQueryResult( + query.SinglePartQuery( + query.Where(query.CaseInsensitiveStringContains(query.NodeProperty("objectid"), "Approver_GUID")), + query.Returning(query.Node()), + ), + "match (n) where toLower(n.objectid) contains $p0 return n", + map[string]any{"p0": "approver_guid"}, + )) + + t.Run("LOOKUP-06 required kind groups and suffix", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.KindIn(query.Node(), scanLookupKinds("Group", "User")...), + query.Kind(query.Node(), graph.StringKind("Entity")), + query.StringEndsWith(query.NodeProperty("objectid"), "-512"), + query.Equals(query.NodeProperty("domainsid"), "S-1-5-21"), + )), + query.Returning(query.Node()), + ), + "match (n) where (n:Group or n:User) and n:Entity and n.objectid ends with $p0 and n.domainsid = $p1 return n", + map[string]any{"p0": "-512", "p1": "S-1-5-21"}, + )) + + t.Run("LOOKUP-06 required and excluded kinds", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Node(), graph.StringKind("Entity")), + query.Not(query.KindIn(query.Node(), scanLookupKinds("Group", "LocalGroup")...)), + query.StringEndsWith(query.NodeProperty("objectid"), "-512"), + )), + query.Returning(query.Node()), + ), + "match (n) where n:Entity and not ((n:Group or n:LocalGroup)) and n.objectid ends with $p0 return n", + map[string]any{"p0": "-512"}, + )) + + t.Run("LOOKUP-07 missing name", assertQueryResult( + query.SinglePartQuery( + query.Where(query.Not(query.Exists(query.NodeProperty("name")))), + query.Returning(query.Node()), + ), + "match (n) where not (n.name is not null) return n", + )) + + t.Run("LOOKUP-08 either approver property present", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Node(), graph.StringKind("AZRole")), + query.Equals(query.NodeProperty("tenantid"), "tenant-1"), + query.Equals(query.NodeProperty("approvalrequired"), true), + query.Or( + query.IsNotNull(query.NodeProperty("userapprovers")), + query.IsNotNull(query.NodeProperty("groupapprovers")), + ), + )), + query.Returning(query.Node()), + ), + "match (n) where n:AZRole and n.tenantid = $p0 and n.approvalrequired = $p1 and (n.userapprovers is not null or n.groupapprovers is not null) return n", + map[string]any{"p0": "tenant-1", "p1": true}, + )) + + t.Run("LOOKUP-09 ID list full hydration", assertQueryResult( + query.SinglePartQuery( + query.Where(query.InIDs(query.NodeID(), graph.ID(101), graph.ID(202), graph.ID(101))), + query.Returning(query.Node()), + ), + "match (n) where id(n) in $p0 return n", + map[string]any{"p0": []graph.ID{101, 202, 101}}, + )) + + t.Run("LOOKUP-10 nested negated account flags", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Node(), graph.StringKind("User")), + query.Not(query.And( + query.Exists(query.NodeProperty("gmsa")), + query.Equals(query.NodeProperty("gmsa"), true), + )), + query.Not(query.And( + query.Exists(query.NodeProperty("msa")), + query.Equals(query.NodeProperty("msa"), true), + )), + query.InIDs(query.NodeID(), graph.ID(101), graph.ID(202)), + )), + query.Returning(query.Node()), + ), + "match (n) where n:User and not (n.gmsa is not null and n.gmsa = $p0) and not (n.msa is not null and n.msa = $p1) and id(n) in $p2 return n", + map[string]any{"p0": true, "p1": true, "p2": []graph.ID{101, 202}}, + )) + + t.Run("LOOKUP-11 tenant adjacency with endpoint list property", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("Contains")), + query.KindIn(query.End(), scanLookupKinds("AZRole", "AZServicePrincipal")...), + query.In(query.EndProperty("roletemplateid"), []string{"role-a", "role-b"}), + )), + query.Returning(query.End()), + ), + "match (s)-[r:Contains]->(e) where id(s) = $p0 and (e:AZRole or e:AZServicePrincipal) and e.roletemplateid in $p1 return e", + map[string]any{"p0": graph.ID(101), "p1": []string{"role-a", "role-b"}}, + )) + + t.Run("LOOKUP-12 exact relationship key First", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Equals(query.EndID(), graph.ID(202)), + query.Kind(query.Relationship(), graph.StringKind("MemberOf")), + )), + query.Returning(query.Relationship()), + query.Limit(1), + ), + "match (s)-[r:MemberOf]->(e) where id(s) = $p0 and id(e) = $p1 return r limit 1", + map[string]any{"p0": graph.ID(101), "p1": graph.ID(202)}, + )) + + t.Run("LOOKUP-13 suffix and bound endpoint full start", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.StringEndsWith(query.StartProperty("objectid"), "-555"), + query.Kind(query.Relationship(), graph.StringKind("LocalToComputer")), + query.Equals(query.EndID(), graph.ID(202)), + )), + query.Returning(query.Start()), + ), + "match (s)-[r:LocalToComputer]->(e) where s.objectid ends with $p0 and id(e) = $p1 return s", + map[string]any{"p0": "-555", "p1": graph.ID(202)}, + )) + + t.Run("LOOKUP-13 suffix and bound endpoint start ID", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.StringEndsWith(query.StartProperty("objectid"), "-555"), + query.Kind(query.Relationship(), graph.StringKind("LocalToComputer")), + query.Equals(query.EndID(), graph.ID(202)), + )), + query.Returning(query.StartID()), + ), + "match (s)-[r:LocalToComputer]->(e) where s.objectid ends with $p0 and id(e) = $p1 return id(s)", + map[string]any{"p0": "-555", "p1": graph.ID(202)}, + )) + + t.Run("LOOKUP-14 descending property order", assertQueryResult( + query.SinglePartQuery( + query.Where(query.Kind(query.Node(), graph.StringKind("Domain"))), + query.Returning(query.Node()), + query.OrderBy(query.Order(query.NodeProperty("name"), query.Descending())), + ), + "match (n) where n:Domain return n order by n.name desc", + )) + + ntlmCriteria := query.And( + query.Kind(query.Node(), graph.StringKind("Computer")), + query.Equals(query.NodeProperty("domainsid"), "S-1-5-21"), + query.Equals(query.NodeProperty("isdc"), true), + query.Equals(query.NodeProperty("ldapavailable"), true), + query.Equals(query.NodeProperty("ldapsigning"), false), + ) + + t.Run("LOOKUP-16 typed NTLM ID projection", assertQueryResult( + query.SinglePartQuery(query.Where(ntlmCriteria), query.Returning(query.NodeID())), + "match (n) where n:Computer and n.domainsid = $p0 and n.isdc = $p1 and n.ldapavailable = $p2 and n.ldapsigning = $p3 return id(n)", + map[string]any{"p0": "S-1-5-21", "p1": true, "p2": true, "p3": false}, + )) + + t.Run("LOOKUP-16 untyped NTLM full hydration", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Equals(query.NodeProperty("domainsid"), "S-1-5-21"), + query.Equals(query.NodeProperty("isdc"), true), + query.Equals(query.NodeProperty("ldapsavailable"), true), + query.Equals(query.NodeProperty("epa"), false), + )), + query.Returning(query.Node()), + ), + "match (n) where n.domainsid = $p0 and n.isdc = $p1 and n.ldapsavailable = $p2 and n.epa = $p3 return n", + map[string]any{"p0": "S-1-5-21", "p1": true, "p2": true, "p3": false}, + )) +} diff --git a/query/neo4j/rewrite.go b/query/neo4j/rewrite.go index b99b06a9..27e1f272 100644 --- a/query/neo4j/rewrite.go +++ b/query/neo4j/rewrite.go @@ -20,10 +20,12 @@ func NewExpressionListRewriter() walk.Visitor[cypher.SyntaxNode] { } } +// pushExpression records a syntax node as the current ancestor during traversal. func (s *ExpressionListRewriter) pushExpression(expression cypher.SyntaxNode) { s.descentStack = append(s.descentStack, expression) } +// peekExpression returns the nearest ancestor syntax node without removing it. func (s *ExpressionListRewriter) peekExpression() (cypher.SyntaxNode, bool) { if len(s.descentStack) == 0 { return nil, false @@ -32,6 +34,7 @@ func (s *ExpressionListRewriter) peekExpression() (cypher.SyntaxNode, bool) { return s.descentStack[len(s.descentStack)-1], true } +// peekExpressionList returns the nearest ancestor when it supports list replacement operations. func (s *ExpressionListRewriter) peekExpressionList() (cypher.ExpressionList, bool) { if ancestorNode, hasPrevious := s.peekExpression(); hasPrevious { ancestorExpressionList, isExpressionList := ancestorNode.(cypher.ExpressionList) @@ -41,6 +44,7 @@ func (s *ExpressionListRewriter) peekExpressionList() (cypher.ExpressionList, bo return nil, false } +// hasNegationAncestor reports whether traversal is currently nested beneath a negation. func (s *ExpressionListRewriter) hasNegationAncestor() bool { for idx := len(s.descentStack) - 1; idx >= 0; idx-- { if _, isNegation := s.descentStack[idx].(*cypher.Negation); isNegation { @@ -51,10 +55,23 @@ func (s *ExpressionListRewriter) hasNegationAncestor() bool { return false } +// hasDisjunctionAncestor reports whether traversal is currently nested beneath a disjunction. +func (s *ExpressionListRewriter) hasDisjunctionAncestor() bool { + for idx := len(s.descentStack) - 1; idx >= 0; idx-- { + if _, isDisjunction := s.descentStack[idx].(*cypher.Disjunction); isDisjunction { + return true + } + } + + return false +} + +// popExpression removes the current node from the traversal ancestry stack. func (s *ExpressionListRewriter) popExpression() { s.descentStack = s.descentStack[:len(s.descentStack)-1] } +// unwrapParenthetical removes nested parentheses so rewrite rules can inspect the underlying syntax node. func unwrapParenthetical(expression cypher.SyntaxNode) cypher.SyntaxNode { cursor := expression @@ -71,6 +88,7 @@ func unwrapParenthetical(expression cypher.SyntaxNode) cypher.SyntaxNode { return cursor } +// rewriteStringNegation preserves null-inclusive semantics when Neo4j evaluates negated string comparisons. func (s *ExpressionListRewriter) rewriteStringNegation(negation *cypher.Negation) { if ancestorExpressionList, isExpressionList := s.peekExpressionList(); isExpressionList { switch typedNegatedExpression := unwrapParenthetical(negation.Expression).(type) { @@ -94,6 +112,7 @@ func (s *ExpressionListRewriter) rewriteStringNegation(negation *cypher.Negation } } +// peekLastMatch returns the nearest enclosing MATCH clause in the traversal stack. func (s *ExpressionListRewriter) peekLastMatch() (*cypher.Match, bool) { for idx := len(s.descentStack) - 1; idx >= 0; idx-- { if lastMatch, typeOK := s.descentStack[idx].(*cypher.Match); typeOK { @@ -109,6 +128,7 @@ func (s *ExpressionListRewriter) Enter(node cypher.SyntaxNode) { s.pushExpression(node) } +// Exit removes empty expression lists, folds eligible relationship kinds into MATCH, and normalizes negated or parenthesized expressions. func (s *ExpressionListRewriter) Exit(node cypher.SyntaxNode) { attemptSelfRemoval := func() { if ancestorNode, hasPrevious := s.peekExpression(); hasPrevious { @@ -131,7 +151,11 @@ func (s *ExpressionListRewriter) Exit(node cypher.SyntaxNode) { if variable, typeOK := typedNode.Reference.(*cypher.Variable); !typeOK { s.SetErrorf("expected a variable as the reference for a kind matcher but received: %T", node) } else if variable.Symbol == query.EdgeSymbol { - if s.hasNegationAncestor() { + // Relationship kinds can be folded into the match pattern only when + // doing so preserves their logical scope. A kind nested under a NOT or + // OR must remain in the WHERE expression; hoisting it would either + // invert the predicate or merge branch-local kinds into one pattern. + if s.hasNegationAncestor() || s.hasDisjunctionAncestor() { return } diff --git a/query/v2/backend_test.go b/query/v2/backend_test.go index f5524ff9..e4c8fc64 100644 --- a/query/v2/backend_test.go +++ b/query/v2/backend_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" ) +// testKindMapper returns an in-memory mapper populated in argument order. func testKindMapper(kinds ...graph.Kind) *pgutil.InMemoryKindMapper { mapper := pgutil.NewInMemoryKindMapper() @@ -169,6 +170,7 @@ func TestBackendParityNeo4jPrepare(t *testing.T) { } } +// TestBackendParityPGTranslateTraversalDepth verifies traversal-depth controls reach PostgreSQL's recursive path translation. func TestBackendParityPGTranslateTraversalDepth(t *testing.T) { edgeKind := graph.StringKind("MemberOf") mapper := testKindMapper(edgeKind) @@ -186,7 +188,7 @@ func TestBackendParityPGTranslateTraversalDepth(t *testing.T) { ), expectedSQLContains: []string{ "with recursive", - "ordered_edges_to_path", + "ordered_edge_ids_to_path", "n0.id = @pi0::int8", "e0.kind_id = any (array [1]::int2[])", "depth < 2", @@ -205,7 +207,7 @@ func TestBackendParityPGTranslateTraversalDepth(t *testing.T) { "n0.id = @pi0::int8", "e0.kind_id = any (array [1]::int2[])", "depth < 2", - "select (s0.n0).id, (s0.n1).id from s0", + "select s0.n0 as \"id(s)\", s0.n1 as \"id(e)\" from s0", }, }, } @@ -228,6 +230,7 @@ func TestBackendParityPGTranslateTraversalDepth(t *testing.T) { } } +// TestBackendParityPGTranslate verifies v2 builders produce stable PostgreSQL SQL and parameter bindings across query forms. func TestBackendParityPGTranslate(t *testing.T) { userKind := graph.StringKind("User") edgeKind := graph.StringKind("MemberOf") @@ -246,7 +249,7 @@ func TestBackendParityPGTranslate(t *testing.T) { v2.Node().ID(), v2.Node().Kinds(), ), - expectedSQL: "with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (n0.kind_ids operator (pg_catalog.&&) array [1]::int2[] and cypher_contains((n0.properties ->> 'name'), (@pi0::text)::text)::bool)) select (s0.n0).id, (array(select _kind.name from generate_subscripts((s0.n0).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s0.n0).kind_ids)[_kind_idx] order by _kind_idx))::text[] from s0;", + expectedSQL: "with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (n0.kind_ids operator (pg_catalog.&&) array [1]::int2[] and cypher_contains((n0.properties ->> 'name'), (@pi0::text)::text)::bool)) select (s0.n0).id as \"id(n)\", (array(select _kind.name from generate_subscripts((s0.n0).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s0.n0).kind_ids)[_kind_idx] order by _kind_idx))::text[] as \"labels(n)\" from s0;", expectedParams: map[string]any{"pi0": "admin"}, }, "relationship read": { @@ -258,7 +261,7 @@ func TestBackendParityPGTranslate(t *testing.T) { v2.Relationship().ID(), v2.End().ID(), ), - expectedSQL: "with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = @pi0::int8) and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [2]::int2[])) select (s0.n0).id, (s0.e0).id, (s0.n1).id from s0;", + expectedSQL: "with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, n1.id as n1 from edge e0 join node n0 on (n0.id = @pi0::int8) and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [2]::int2[])) select s0.n0 as \"id(s)\", (s0.e0).id as \"id(r)\", s0.n1 as \"id(e)\" from s0;", expectedParams: map[string]any{"pi0": 1}, }, "update node": { @@ -306,6 +309,7 @@ func TestBackendParityPGTranslate(t *testing.T) { } } +// TestBackendParityPGTranslateShortestPaths verifies shortest-path controls select the expected PostgreSQL search harness. func TestBackendParityPGTranslateShortestPaths(t *testing.T) { edgeKind := graph.StringKind("MemberOf") mapper := testKindMapper(edgeKind) @@ -332,7 +336,7 @@ func TestBackendParityPGTranslateShortestPaths(t *testing.T) { ).Return( v2.Path(), ), - expectedHarness: "bidirectional_asp_harness", + expectedHarness: "all_shortest_paths_dag", }, } @@ -347,18 +351,24 @@ func TestBackendParityPGTranslateShortestPaths(t *testing.T) { sql, err := translate.Translated(translation) require.NoError(t, err) require.Contains(t, sql, testCase.expectedHarness) - require.Contains(t, sql, "ordered_edges_to_path") - require.Contains(t, sql, "n0.id = 1") - require.Contains(t, sql, "n1.id = 2") + require.Contains(t, sql, "ordered_edge_ids_to_path") + require.Contains(t, sql, "n0.id = @pi0::int8") + require.Contains(t, sql, "n1.id = @pi1::int8") + require.Contains(t, sql, "singleton_endpoints") - serializedHarnessQueryHasKindConstraint := false - for _, parameterValue := range translation.Parameters { - if serializedQuery, typeOK := parameterValue.(string); typeOK && strings.Contains(serializedQuery, "array [1]::int2[]") { - serializedHarnessQueryHasKindConstraint = true - break + if name == "shortest path" { + serializedHarnessQueryHasKindConstraint := false + for _, parameterValue := range translation.Parameters { + if serializedQuery, typeOK := parameterValue.(string); typeOK && strings.Contains(serializedQuery, "array [1]::int2[]") { + serializedHarnessQueryHasKindConstraint = true + break + } } + require.True(t, serializedHarnessQueryHasKindConstraint, "expected serialized shortest-path harness query to contain edge kind constraint: %#v", translation.Parameters) + } else { + require.Contains(t, sql, "array [1]::int2[]") + require.NotContains(t, sql, "bidirectional_asp_harness") } - require.True(t, serializedHarnessQueryHasKindConstraint, "expected serialized shortest-path harness query to contain edge kind constraint: %#v", translation.Parameters) }) } } diff --git a/query/v2/query.go b/query/v2/query.go index a841c752..3c5c0117 100644 --- a/query/v2/query.go +++ b/query/v2/query.go @@ -99,6 +99,8 @@ func (s runtimeIdentifiers) End() *cypher.Variable { return cypher.NewVariableWithSymbol(s.end) } +// Identifiers exposes the canonical variables used for path, node, start, +// relationship, and end expressions. var Identifiers = runtimeIdentifiers{ path: "p", node: "n", @@ -373,7 +375,10 @@ func Or(operands ...cypher.SyntaxNode) cypher.SyntaxNode { type SortDirection int const ( + // SortAscending orders values from least to greatest. SortAscending SortDirection = iota + + // SortDescending orders values from greatest to least. SortDescending ) @@ -652,7 +657,16 @@ func (s *entity[T]) ID() IdentityContinuation { } } +// Property returns a comparison continuation for a validated property lookup or records an invalid-key error. func (s *entity[T]) Property(propertyName string) PropertyContinuation { + if err := cypher.ValidatePropertyKeyName(propertyName); err != nil { + return &propertyContinuation{ + comparisonContinuation: comparisonContinuation{ + qualifierExpression: invalidExpression(err), + }, + } + } + return &propertyContinuation{ comparisonContinuation: comparisonContinuation{ qualifierExpression: cypher.NewPropertyLookup(s.identifier.Symbol, propertyName), @@ -775,9 +789,16 @@ type QueryBuilder interface { type updatingClauseKind int const ( + // updatingClauseSet identifies a pending SET clause. updatingClauseSet updatingClauseKind = iota + + // updatingClauseRemove identifies a pending REMOVE clause. updatingClauseRemove + + // updatingClauseDelete identifies a pending DELETE clause. updatingClauseDelete + + // updatingClauseCreate identifies a pending CREATE clause. updatingClauseCreate ) diff --git a/query/v2/query_test.go b/query/v2/query_test.go index 188530fd..b4685eee 100644 --- a/query/v2/query_test.go +++ b/query/v2/query_test.go @@ -117,6 +117,26 @@ func TestCreateRelationshipWithExplicitEndpoints(t *testing.T) { }, preparedQuery.Parameters) } +// TestRawPropertyKeysRenderEscaped verifies raw property keys retain required Cypher escaping in prepared queries. +func TestRawPropertyKeysRenderEscaped(t *testing.T) { + preparedQuery, err := v2.New().Return( + v2.Node().Property("a-aaa"), + v2.Node().Property("has`tick"), + v2.Node().Property(" "), + ).Build() + require.NoError(t, err) + + require.Equal(t, "match (n) return n.`a-aaa`, n.`has``tick`, n.` `", renderPrepared(t, preparedQuery)) +} + +// TestEmptyPropertyKeyReturnsBuildError verifies an empty raw property key fails during query construction. +func TestEmptyPropertyKeyReturnsBuildError(t *testing.T) { + _, err := v2.New().Return( + v2.Node().Property(""), + ).Build() + require.ErrorIs(t, err, cypher.ErrEmptyPropertyKeyName) +} + func TestCreateSplitsDisjointNodePatterns(t *testing.T) { preparedQuery, err := v2.New().Create( v2.NodePattern(graph.Kinds{graph.StringKind("A")}, nil), diff --git a/query/v2/util.go b/query/v2/util.go index 03b5fb10..13f3d283 100644 --- a/query/v2/util.go +++ b/query/v2/util.go @@ -247,7 +247,12 @@ func variableReference(value any) (*cypher.Variable, error) { } } +// propertyLookupOrError constructs a property lookup or an error expression when its key or variable reference is invalid. func propertyLookupOrError(reference any, propertyName string) cypher.Expression { + if err := cypher.ValidatePropertyKeyName(propertyName); err != nil { + return invalidExpression(err) + } + if variable, err := variableReference(reference); err != nil { return invalidExpression(err) } else { diff --git a/regression_coverage_manifest.md b/regression_coverage_manifest.md new file mode 100644 index 00000000..54113f73 --- /dev/null +++ b/regression_coverage_manifest.md @@ -0,0 +1,352 @@ +# BloodHound Regression Coverage Manifest + +Baseline audit for the source-derived regression program, recorded when the +regression harness was established. The original delivery plan is archived +verbatim in [`learning.md`](learning.md). This file is the authoritative +coverage contract and gap map for the stable query-form IDs; update a cell when +a case is added, and link the exact test or generated case that changed it. + +## Coverage contract + +The corpus represents query shapes found in reviewed BHE and BHCE source; it +does not import application business logic or reproduce complete downstream +traversal algorithms. Normalize every discovered query into this tuple: + +```text +query target ++ direction ++ start/end ID anchor ++ start/end kind constraints ++ relationship kind constraints ++ node/relationship property predicates ++ logical grouping ++ projection ++ terminal operation +``` + +Two call sites may share a stable ID only when the entire tuple is equivalent. +Add a new ID for a new operator, grouping, direction, anchor location, +projection, mutation target, or execution path. Relationship names may share a +case, but kind-list and parameter-list cardinality remain test dimensions. +Audit existing primitive coverage before adding a production composition, +builder path, projection, cardinality, or scale case. + +| ID | Layer | Contract | +| --- | --- | --- | +| `QB` | Legacy query-builder pipeline | Preserve the AST and backend forms built from reviewed criteria; raw Cypher alone is insufficient for rewrite-sensitive forms. | +| `CY` | Cypher parser/mutation cases | Preserve accepted syntax, formatting, and mutation parsing. | +| `PG` | PostgreSQL translation goldens | Preserve SQL, parameters, correlation, projection, and mutation targets. | +| `IT` | Shared integration cases | Prove backend-equivalent observations and exact mutation effects. | +| `PC` | Plan corpus | Capture translated SQL, lowering metadata, and PostgreSQL plans for comparison. | +| `PI` | PostgreSQL plan-invariant tests | Assert stable index, orientation, filter, cardinality, or mutation-target properties. | +| `SC` | Scale/runtime corpus | Exercise representative cardinality and selectivity with repeatable fixtures. | +| `DR` | Driver integration/benchmark | Exercise direct driver and batch APIs that bypass Cypher translation. | + +Coverage rules: + +1. Every active form with a Cypher equivalent requires `PG` and `IT` coverage. +2. Every legacy-builder form requires `QB`; rewrite-sensitive forms also run + through the builder API in `IT` rather than only through an equivalent raw + query. +3. Every Cypher mutation requires `CY`, `PG`, and exact `IT` post-state; + direct-driver mutations require exact `DR` post-state instead. +4. High-cardinality or join-sensitive forms require `PC`; declared + representatives additionally require `SC` and stable plan-sensitive forms + require `PI`. +5. Direct batched mutations require semantic `DR` coverage across flush + boundaries. +6. Shared integration cases remain backend-equivalent. PostgreSQL-only plan, + resource, and runtime assertions stay in PostgreSQL-scoped tests or the + scale corpus. + +Status values: + +- `E` — existing coverage is equivalent to the complete normalized tuple. +- `P` — a primitive exists, but the production composition, projection, + cardinality, mutation target, or scale dimension is missing. +- `C` — production-complete coverage added by this regression project. +- `A` — absent. +- `—` — the layer is not required by this coverage contract. + +No active production ID was complete when the audit began. The following +references are the existing primitives used by the table; they are linked here +instead of being cloned under BloodHound-specific names: + +- `QB-PRED`: [`TestQueryBuilder_Render` predicate, temporal, kind, ID, string, + null, and mutation subtests](query/neo4j/neo4j_test.go). +- `QB-PROJ`: [`TestQueryBuilder_Render` relationship projection + subtests](query/neo4j/neo4j_test.go). +- `CY-MUT`: [Cypher create/update/delete parser cases](cypher/test/cases/mutation_tests.json). +- `PG-PRED`: [PostgreSQL node/predicate translation goldens](cypher/models/pgsql/test/translation_cases/nodes.sql). +- `PG-DEL`: [PostgreSQL delete translation goldens](cypher/models/pgsql/test/translation_cases/delete.sql). +- `PG-BIND`: [PostgreSQL binding and rewrite goldens](cypher/models/pgsql/test/translation_cases/pattern_binding.sql). +- `IT-PRED`: [backend-equivalent node predicate cases](integration/testdata/cases/nodes_inline.json). +- `IT-HOP`: [backend-equivalent directed one-hop template cases](integration/testdata/templates/pattern_shapes.json). +- `IT-MUT`: [primitive mutation cases](integration/testdata/cases/delete_inline.json) and + [the initial exact post-state harness sentinel](integration/testdata/cases/mutation_post_state_inline.json). +- `SC-HOP`: [`one_hop_typed_from_bound_id`](benchmark/testdata/scale/cases/traversal.json). +- `SC-LOOKUP`: [`objectid_exact_string_anchor` and + `boolean_property_filter`](benchmark/testdata/scale/cases/lookups.json). +- `SC-COUNT`: [`all_node_count`, `typed_node_count`, and + `typed_edge_count`](benchmark/testdata/scale/cases/counts.json). +- `DR-BATCH`: [`TestBatchTransaction_NodeUpdate`](drivers/neo4j/batch_integration_test.go#L48). +- `PI-IDX`: [`TestPostgreSQLPropertyIndexPlans`](integration/pgsql_property_index_plan_test.go#L58). +- `LOGIC-QB`: [`TestQueryBuilder_LOGIC01PreservesBranchLocalRelationshipKinds`, + `TestQueryBuilder_LogicalForms`, and + `TestQueryBuilder_LOGIC05ProjectionOrder`](query/neo4j/neo4j_test.go), plus + [`TestLegacyBuilderPostgreSQL_LogicalForms` and + `TestLegacyBuilderPostgreSQL_LOGIC05ProjectionOrder`](cypher/models/pgsql/test/logical_forms_legacy_builder_test.go). +- `LOGIC-CY`: [`LOGIC-04` filtered relationship and node delete parser + cases](cypher/test/cases/mutation_tests.json). +- `LOGIC-PG`: [`reconciliation.sql`](cypher/models/pgsql/test/translation_cases/reconciliation.sql) + and [`post_processing.sql`](cypher/models/pgsql/test/translation_cases/post_processing.sql). +- `LOGIC-IT`: [`TestLegacyBuilderLogicalForms`](integration/logical_forms_legacy_builder_test.go) + and the backend-equivalent [`reconciliation_shapes.json`](integration/testdata/templates/reconciliation_shapes.json) + and [`post_processing_shapes.json`](integration/testdata/templates/post_processing_shapes.json) corpora. +- `LOGIC-PC`: the `LOGIC-01`, `LOGIC-02`, and `LOGIC-04` families in + [`reconciliation_shapes.json`](integration/testdata/templates/reconciliation_shapes.json), + loaded directly by `cmd/plancorpus` with fixture-ID parameter resolution. +- `REC-QB`: [`TestQueryBuilder_ReconciliationForms`](query/neo4j/neo4j_test.go) + and [`TestLegacyBuilderPostgreSQL_ReconciliationForms`](cypher/models/pgsql/test/reconciliation_forms_legacy_builder_test.go). +- `REC-CY`: the `REC-01` through `REC-04` and `REC-06` through `REC-08` + mutation parser cases in [`mutation_tests.json`](cypher/test/cases/mutation_tests.json). +- `REC-PG`: the `REC-01` through `REC-08` PostgreSQL goldens in + [`reconciliation.sql`](cypher/models/pgsql/test/translation_cases/reconciliation.sql). +- `REC-IT`: the exact reconciliation semantic families in + [`reconciliation_shapes.json`](integration/testdata/templates/reconciliation_shapes.json) + and the [`FetchStartNodes` de-dup contract](integration/delegated_enrollment_legacy_builder_test.go). +- `REC-PC`: the `REC-01` through `REC-08` families loaded from + [`reconciliation_shapes.json`](integration/testdata/templates/reconciliation_shapes.json) + by `cmd/plancorpus`. +- `REC-SC`: the repeatable `REC-01`, `REC-02`, `REC-04`, `REC-06`, and + `REC-08` write scenarios in + [`reconciliation.json`](benchmark/testdata/scale/cases/reconciliation.json). +- `TRUST-PRUNE-QB`: [`TestQueryBuilder_TrustAndPruningForms`](query/neo4j/neo4j_test.go) + and [`TestLegacyBuilderPostgreSQL_TrustAndPruningForms`](cypher/models/pgsql/test/trust_pruning_forms_legacy_builder_test.go). +- `TRUST-PRUNE-PG`: the `TRUST-01` through `TRUST-03` and `PRUNE-01` through + `PRUNE-04` PostgreSQL goldens in + [`reconciliation.sql`](cypher/models/pgsql/test/translation_cases/reconciliation.sql) + and [`post_processing.sql`](cypher/models/pgsql/test/translation_cases/post_processing.sql). +- `TRUST-PRUNE-IT`: the exact truth/null and hydration families in + [`reconciliation_shapes.json`](integration/testdata/templates/reconciliation_shapes.json) + and [`post_processing_shapes.json`](integration/testdata/templates/post_processing_shapes.json), + plus [`TestLegacyBuilderTrustAndPruningSelectors`](integration/trust_pruning_legacy_builder_test.go). +- `TRUST-PRUNE-PC`: the `TRUST-01` through `TRUST-03` and `PRUNE-01` through + `PRUNE-04` families loaded from the shared template corpus by `cmd/plancorpus`. +- `TRUST-PRUNE-SC`: the dense trust reads, pruning selectors, and mutation-safe + batch-delete equivalents in + [`trust_pruning.json`](benchmark/testdata/scale/cases/trust_pruning.json), + backed by [`NewTrustPruningScaleFixture`](testutil/reconciliation_fixture.go). +- `PRUNE-DR`: [`TestDirectBatchPruning` and + `BenchmarkDirectBatchPruning`](integration/trust_pruning_legacy_builder_test.go), + including IDs absent at delete time and a mixed-direction high-degree cascade. +- `HOP-QB`: [`TestQueryBuilder_StandaloneHopForms`](query/neo4j/neo4j_test.go) + and [`TestLegacyBuilderPostgreSQL_StandaloneHopForms`](cypher/models/pgsql/test/standalone_hop_forms_legacy_builder_test.go). +- `HOP-PG`: the `HOP-01` through `HOP-10` PostgreSQL goldens in + [`stepwise_traversal.sql`](cypher/models/pgsql/test/translation_cases/stepwise_traversal.sql). +- `HOP-IT`: the backend-equivalent standalone-hop families in + [`post_processing_hop_shapes.json`](integration/testdata/templates/post_processing_hop_shapes.json), + plus [`TestLegacyBuilderStandaloneHops`](integration/standalone_hops_legacy_builder_test.go). +- `HOP-PC`: the `HOP-01` through `HOP-10` families loaded from + [`post_processing_hop_shapes.json`](integration/testdata/templates/post_processing_hop_shapes.json) + by `cmd/plancorpus`. +- `HOP-SC`: the repeatable standalone-hop scenarios in + [`hops.json`](benchmark/testdata/scale/cases/hops.json), backed by + [`NewHopScaleFixture`](testutil/reconciliation_fixture.go). +- `SCAN-LOOKUP-QB`: [`TestQueryBuilder_RelationshipScans` and + `TestQueryBuilder_NodeLookups`](query/neo4j/relationship_scans_node_lookups_test.go), plus + [`TestLegacyBuilderPostgreSQL_RelationshipScans` and + `TestLegacyBuilderPostgreSQL_NodeLookups`](cypher/models/pgsql/test/relationship_scans_node_lookups_legacy_builder_test.go). +- `SCAN-LOOKUP-PG`: the `SCAN-01` through `SCAN-08` and `LOOKUP-01` through + `LOOKUP-14`/`LOOKUP-16` PostgreSQL goldens in + [`relationship_scans_node_lookups.sql`](cypher/models/pgsql/test/translation_cases/relationship_scans_node_lookups.sql). +- `SCAN-LOOKUP-IT`: the backend-equivalent scan, lookup, and count families in + [`relationship_scan_shapes.json`](integration/testdata/templates/relationship_scan_shapes.json), + [`basic_lookup_shapes.json`](integration/testdata/templates/basic_lookup_shapes.json), + [`advanced_lookup_shapes.json`](integration/testdata/templates/advanced_lookup_shapes.json), + and [`count_shapes.json`](integration/testdata/templates/count_shapes.json), + plus [`TestLegacyBuilderRelationshipScansAndNodeLookups`](integration/relationship_scans_node_lookups_legacy_builder_test.go). +- `SCAN-LOOKUP-PC`: the `SCAN-*` and applicable `LOOKUP-*` families loaded from + the shared scan/lookup template corpus by `cmd/plancorpus`. +- `SCAN-LOOKUP-SC`: the required wide-scan, large-list, adjacency, count, and NTLM + scenarios in [`scans_lookups.json`](benchmark/testdata/scale/cases/scans_lookups.json), + backed by [`NewScanLookupScaleFixture`](testutil/reconciliation_fixture.go). +- `WRITE-DR`: [`TestDirectWriteDeleteRelationshipBoundariesAndSurvivors` through + `TestDirectWriteExactKeyMissThenCreateNode`](integration/direct_write_mutations_test.go), + covering direct batch and transactional APIs on the selected backend with the + shared [`NewDirectWriteScaleFixture`](testutil/reconciliation_fixture.go), plus + the PostgreSQL conflict-key/property-index regression in + [`batch_test.go`](drivers/pg/batch_test.go). +- `WRITE-IT`: the exact-key create/update, full-node update, and exact-key + miss/create workflows in [`direct_write_mutations_test.go`](integration/direct_write_mutations_test.go), + with selector and driver-operation assertions kept separate. +- `WRITE-SC`: the reset-per-iteration, post-state-checked + [`BenchmarkMutationSafeDirectWrites`](integration/direct_write_mutations_test.go) + at 1,000 items and across the 2,000-item DAWGS flush boundary. +- `SCALE-PI`: [`TestPostgreSQLScalePlanInvariants`](cmd/graphbench/postgresql_plan_invariants_integration_test.go) + executes every required Cypher scale representative through PostgreSQL with + `EXPLAIN ANALYZE`, exact read/write cardinality, rollback-isolated mutation + post-state, mutation-target, binding, and anchor-index assertions. The + backend-independent [`TestScaleCorpusRequiredRepresentativesDeclareCardinality`](cmd/graphbench/scale_corpus_contract_test.go) + prevents a required stable ID or its cardinality contract from disappearing. +- `SCALE-BASELINE`: `cmd/graphbench` captures translated SQL, lowering + metadata, plans, buffer/runtime metrics, and cardinalities for the complete + scale corpus; `cmd/plancorpus` captures the shared semantic corpus with source + metadata. Generated captures remain review artifacts under the ignored + `.coverage/` directory rather than committed machine-specific baselines. +- `DORMANT-GATE`: [`TestDormantFormsStayOutOfPlanCorpus`](cmd/plancorpus/dormant_forms_guard_test.go) + and [`TestDormantFormsStayOutOfScaleCorpus`](cmd/graphbench/dormant_forms_guard_test.go) + keep every `FUTURE-*` ID out of active semantic, plan, and scale gates. The + activation and ongoing source-review procedure is recorded in + [`regression_source_parity.md`](docs/regression_source_parity.md). +- `COMPLETION-SC`: the `SCAN-01` ID-only, `SCAN-06` shallow IDs/kind, + `SCAN-02` relationship hydration, and `LOOKUP-09` node hydration scale cases + are classified and enforced by + [`TestScaleCorpusDistinguishesProjectionClasses`](cmd/graphbench/scale_corpus_contract_test.go). +- `COMPLETION-GATE`: [`TestRegressionCoverageManifestClosesEveryActiveID`](regression_manifest_test.go) + requires all 64 stable active IDs to remain present without an `A` or `P` + layer while preserving `FUTURE-01` as non-production-complete. + +## Logical sentinels + +| ID | QB | CY | PG | IT | PC | PI | SC | DR | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| `LOGIC-01` | C (`LOGIC-QB`) | — | C (`LOGIC-PG`) | C (`LOGIC-IT`) | C (`LOGIC-PC`) | C (`SCALE-PI`) | — | — | +| `LOGIC-02` | C (`LOGIC-QB`) | — | C (`LOGIC-PG`) | C (`LOGIC-IT`) | C (`LOGIC-PC`) | C (`SCALE-PI`) | — | — | +| `LOGIC-03` | C (`LOGIC-QB`) | — | C (`LOGIC-PG`) | C (`LOGIC-IT`) | — | — | — | — | +| `LOGIC-04` | — | C (`LOGIC-CY`) | C (`LOGIC-PG`) | C (`LOGIC-IT`) | C (`LOGIC-PC`) | C (`SCALE-PI`) | — | — | +| `LOGIC-05` | C (`LOGIC-QB`) | — | C (`LOGIC-PG`) | C (`LOGIC-IT`) | — | — | — | — | + +## Reconciliation + +| ID | QB | CY | PG | IT | PC | PI | SC | DR | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| `REC-01` | C (`REC-QB`) | C (`REC-CY`) | C (`REC-PG`) | C (`REC-IT`) | C (`REC-PC`) | C (`SCALE-PI`) | C (`REC-SC`) | — | +| `REC-02` | C (`REC-QB`) | C (`REC-CY`) | C (`REC-PG`) | C (`REC-IT`) | C (`REC-PC`) | C (`SCALE-PI`) | C (`REC-SC`) | — | +| `REC-03` | C (`REC-QB`) | C (`REC-CY`) | C (`REC-PG`) | C (`REC-IT`) | C (`REC-PC`) | — | — | — | +| `REC-04` | C (`REC-QB`) | C (`REC-CY`) | C (`REC-PG`) | C (`REC-IT`) | C (`REC-PC`) | C (`SCALE-PI`) | C (`REC-SC`) | — | +| `REC-05` | C (`REC-QB`) | — | C (`REC-PG`) | C (`REC-IT`) | C (`REC-PC`) | — | — | — | +| `REC-06` | C (`REC-QB`) | C (`REC-CY`) | C (`REC-PG`) | C (`REC-IT`) | C (`REC-PC`) | C (`SCALE-PI`) | C (`REC-SC`) | — | +| `REC-07` | C (`REC-QB`) | C (`REC-CY`) | C (`REC-PG`) | C (`REC-IT`) | C (`REC-PC`) | — | — | — | +| `REC-08` | C (`REC-QB`) | C (`REC-CY`) | C (`REC-PG`) | C (`REC-IT`) | C (`REC-PC`) | C (`SCALE-PI`) | C (`REC-SC`) | — | + +## Trust, pruning, and aging + +| ID | QB | CY | PG | IT | PC | PI | SC | DR | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| `TRUST-01` | C (`TRUST-PRUNE-QB`) | — | C (`TRUST-PRUNE-PG`) | C (`TRUST-PRUNE-IT`) | C (`TRUST-PRUNE-PC`) | C (`SCALE-PI`) | C (`TRUST-PRUNE-SC`) | — | +| `TRUST-02` | C (`TRUST-PRUNE-QB`) | — | C (`TRUST-PRUNE-PG`) | C (`TRUST-PRUNE-IT`) | C (`TRUST-PRUNE-PC`) | C (`SCALE-PI`) | C (`TRUST-PRUNE-SC`) | — | +| `TRUST-03` | C (`TRUST-PRUNE-QB`) | — | C (`TRUST-PRUNE-PG`) | C (`TRUST-PRUNE-IT`) | C (`TRUST-PRUNE-PC`) | C (`SCALE-PI`) | — | — | +| `PRUNE-01` | C (`TRUST-PRUNE-QB`) | — | C (`TRUST-PRUNE-PG`) | C (`TRUST-PRUNE-IT`) | C (`TRUST-PRUNE-PC`) | C (`SCALE-PI`) | C (`TRUST-PRUNE-SC`) | — | +| `PRUNE-02` | C (`TRUST-PRUNE-QB`) | — | C (`TRUST-PRUNE-PG`) | C (`TRUST-PRUNE-IT`) | C (`TRUST-PRUNE-PC`) | C (`SCALE-PI`) | C (`TRUST-PRUNE-SC`) | — | +| `PRUNE-03` | C (`TRUST-PRUNE-QB`) | — | C (`TRUST-PRUNE-PG`) | C (`TRUST-PRUNE-IT`) | C (`TRUST-PRUNE-PC`) | C (`SCALE-PI`) | C (`TRUST-PRUNE-SC`) | — | +| `PRUNE-04` | C (`TRUST-PRUNE-QB`) | — | C (`TRUST-PRUNE-PG`) | C (`TRUST-PRUNE-IT`) | C (`TRUST-PRUNE-PC`) | C (`SCALE-PI`) | C (`TRUST-PRUNE-SC`) | — | +| `PRUNE-05` | — | — | — | — | — | — | C (`TRUST-PRUNE-SC`) | C (`PRUNE-DR`) | +| `PRUNE-06` | — | — | — | — | — | — | C (`TRUST-PRUNE-SC`) | C (`PRUNE-DR`) | + +## Standalone hops + +| ID | QB | CY | PG | IT | PC | PI | SC | DR | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| `HOP-01` | C (`HOP-QB`) | — | C (`HOP-PG`) | C (`HOP-IT`) | C (`HOP-PC`) | C (`SCALE-PI`) | C (`HOP-SC`) | — | +| `HOP-02` | C (`HOP-QB`) | — | C (`HOP-PG`) | C (`HOP-IT`) | C (`HOP-PC`) | C (`SCALE-PI`) | C (`HOP-SC`) | — | +| `HOP-03` | C (`HOP-QB`) | — | C (`HOP-PG`) | C (`HOP-IT`) | C (`HOP-PC`) | C (`SCALE-PI`) | C (`HOP-SC`) | — | +| `HOP-04` | C (`HOP-QB`) | — | C (`HOP-PG`) | C (`HOP-IT`) | C (`HOP-PC`) | C (`SCALE-PI`) | C (`HOP-SC`) | — | +| `HOP-05` | C (`HOP-QB`) | — | C (`HOP-PG`) | C (`HOP-IT`) | C (`HOP-PC`) | C (`SCALE-PI`) | C (`HOP-SC`) | — | +| `HOP-06` | C (`HOP-QB`) | — | C (`HOP-PG`) | C (`HOP-IT`) | C (`HOP-PC`) | — | — | — | +| `HOP-07` | C (`HOP-QB`) | — | C (`HOP-PG`) | C (`HOP-IT`) | C (`HOP-PC`) | C (`SCALE-PI`) | C (`HOP-SC`) | — | +| `HOP-08` | C (`HOP-QB`) | — | C (`HOP-PG`) | C (`HOP-IT`) | C (`HOP-PC`) | — | — | — | +| `HOP-09` | C (`HOP-QB`) | — | C (`HOP-PG`) | C (`HOP-IT`) | C (`HOP-PC`) | C (`SCALE-PI`) | C (`HOP-SC`) | — | +| `HOP-10` | C (`HOP-QB`) | — | C (`HOP-PG`) | C (`HOP-IT`) | C (`HOP-PC`) | — | — | — | + +## Relationship scans and node lookups + +| ID | QB | CY | PG | IT | PC | PI | SC | DR | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| `SCAN-01` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | +| `SCAN-02` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | +| `SCAN-03` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | +| `SCAN-04` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | +| `SCAN-05` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | +| `SCAN-06` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | — | C (`COMPLETION-SC`) | — | +| `SCAN-07` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | +| `SCAN-08` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | +| `LOOKUP-01` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | — | — | — | +| `LOOKUP-02` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | +| `LOOKUP-03` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | — | — | — | — | +| `LOOKUP-04` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | +| `LOOKUP-05` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | +| `LOOKUP-06` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | — | — | — | +| `LOOKUP-07` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | — | — | — | — | +| `LOOKUP-08` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | — | — | — | +| `LOOKUP-09` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | +| `LOOKUP-10` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | — | — | — | +| `LOOKUP-11` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | +| `LOOKUP-12` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | — | — | — | +| `LOOKUP-13` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | +| `LOOKUP-14` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | — | — | — | +| `LOOKUP-15` | — | — | — | C (`SCAN-LOOKUP-IT`) | — | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | +| `LOOKUP-16` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | + +## Direct writes + +| ID | QB | CY | PG | IT | PC | PI | SC | DR | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| `WRITE-01` | — | — | — | — | — | — | C (`WRITE-SC`) | C (`WRITE-DR`) | +| `WRITE-02` | — | — | — | — | — | — | C (`WRITE-SC`) | C (`WRITE-DR`) | +| `WRITE-03` | — | — | — | — | — | — | C (`WRITE-SC`) | C (`WRITE-DR`) | +| `WRITE-04` | — | — | — | — | — | — | C (`WRITE-SC`) | C (`WRITE-DR`) | +| `WRITE-05` | — | — | — | — | — | — | C (`WRITE-SC`) | C (`WRITE-DR`) | +| `WRITE-06` | — | — | — | C (`WRITE-IT`) | — | — | — | C (`WRITE-DR`) | +| `WRITE-07` | — | — | — | C (`WRITE-IT`) | — | — | — | C (`WRITE-DR`) | +| `WRITE-08` | — | — | — | C (`WRITE-IT`) | — | — | — | C (`WRITE-DR`) | + +## Dormant coverage + +`FUTURE-01` remains intentionally incomplete because its reviewed callers are +disabled. `DORMANT-GATE` protects that classification; it is not query coverage +and therefore does not change the primitive or absent cells below. + +| ID | QB | CY | PG | IT | PC | PI | SC | DR | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| `FUTURE-01` | P (`QB-PRED`) | P (`CY-MUT`) | P (`PG-DEL`) | P (`IT-MUT`) | A | — | A | — | + +## Completion audit + +The executable manifest gate and the following evidence close the coverage +contract: + +1. All 64 active stable IDs are present at their required layers without an + absent or primitive-only cell (`COMPLETION-GATE`). +2. Shared Cypher mutations and direct writes assert exact targets, survivors, + properties, and counts with rollback/reset isolation (`IT-MUT`, + `REC-IT`, `TRUST-PRUNE-IT`, and `WRITE-IT`). +3. The `LOGIC-01` branch-local direction/kind truth table executes through the + shared integration corpus on PostgreSQL and Neo4j (`LOGIC-IT`). +4. PostgreSQL translation and plan coverage exercises equality-anchored deletes + in both active endpoint orientations and every production-active list form + (`REC-PG`, `REC-PC`, and `SCALE-PI`). The only outbound tenant-list + form is disabled upstream and remains `FUTURE-01` as required. +5. Scale coverage explicitly separates ID-only, shallow IDs/kind, full + relationship, and full-node projections (`COMPLETION-SC`). +6. Direct-write coverage includes the 1,000-item application batch and the + 1,999/2,000/2,001 DAWGS flush boundary (`WRITE-DR` and `WRITE-SC`). +7. `HOP-*` semantic and scale cases remain standalone one-hop queries; no new + runner sequences BloodHound traversal behavior (`HOP-IT` and + `HOP-SC`). +8. Dormant IDs are rejected from active plan and scale corpora until their + callers are enabled (`DORMANT-GATE`). + +## Harness foundation + +These prerequisites are intentionally not marked `C` against production IDs: + +- Mutation post-state assertions: [standalone sentinel](integration/testdata/cases/mutation_post_state_inline.json) + and [template rollback/repeat sentinel](integration/testdata/templates/mutation_post_state_shapes.json). +- Reusable deterministic fixture and list/fanout generators: + [`NewReconciliationFixture`](integration/regression_fixture.go). +- Backend-selected legacy query execution: + [`WithLegacyNodeQuery` and `WithLegacyRelationshipQuery`](integration/legacy_query_harness.go). +- Mutation-safe scale execution and list-valued fixture IDs: + [`WriteScenario`](cmd/graphbench/types.go) and + [`resolveCaseParams`](cmd/graphbench/datasets.go). diff --git a/regression_manifest_test.go b/regression_manifest_test.go new file mode 100644 index 00000000..e28856c7 --- /dev/null +++ b/regression_manifest_test.go @@ -0,0 +1,97 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package dawgs + +import ( + "fmt" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestRegressionCoverageManifestClosesEveryActiveID verifies that every active +// query form has complete coverage while dormant forms remain unactivated. +func TestRegressionCoverageManifestClosesEveryActiveID(t *testing.T) { + raw, err := os.ReadFile("regression_coverage_manifest.md") + require.NoError(t, err) + + var ( + rows = parseRegressionManifestRows(string(raw)) + activeFamilies = map[string]int{ + "LOGIC": 5, + "REC": 8, + "TRUST": 3, + "PRUNE": 6, + "HOP": 10, + "SCAN": 8, + "LOOKUP": 16, + "WRITE": 8, + } + ) + + for family, count := range activeFamilies { + for idx := 1; idx <= count; idx++ { + id := fmt.Sprintf("%s-%02d", family, idx) + cells, found := rows[id] + require.True(t, found, "coverage manifest is missing active query form %s", id) + for _, cell := range cells { + status := strings.Fields(cell) + if len(status) > 0 { + require.NotContains(t, []string{"A", "P"}, status[0], + "active query form %s retains an unclosed layer: %s", id, cell) + } + } + } + } + + futureCells, found := rows["FUTURE-01"] + require.True(t, found, "coverage manifest is missing dormant query form FUTURE-01") + require.Contains(t, futureCells, "A", "FUTURE-01 must retain absent activation-only layers") + for _, cell := range futureCells { + status := strings.Fields(cell) + if len(status) > 0 { + require.NotEqual(t, "C", status[0], "FUTURE-01 must remain outside production-complete coverage") + } + } +} + +// parseRegressionManifestRows indexes the coverage cells in each manifest row +// by query-form identifier. +func parseRegressionManifestRows(manifest string) map[string][]string { + rows := map[string][]string{} + for _, line := range strings.Split(manifest, "\n") { + if !strings.HasPrefix(line, "| `") { + continue + } + + columns := strings.Split(line, "|") + if len(columns) < 11 { + continue + } + + id := strings.Trim(strings.TrimSpace(columns[1]), "`") + cells := make([]string, 0, len(columns)-3) + for _, column := range columns[2 : len(columns)-1] { + cells = append(cells, strings.TrimSpace(column)) + } + rows[id] = cells + } + + return rows +} diff --git a/testutil/metadata.go b/testutil/metadata.go new file mode 100644 index 00000000..bf7522d6 --- /dev/null +++ b/testutil/metadata.go @@ -0,0 +1,57 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import "runtime/debug" + +// BaselineMetadata records build identity needed to interpret a generated benchmark baseline. +type BaselineMetadata struct { + // DAWGSVersion identifies the DAWGS build that produced the baseline. + DAWGSVersion string `json:"dawgs_version"` +} + +// ResolveBaselineMetadata returns metadata for dawgsVersion, deriving the current build identity when it is empty. +func ResolveBaselineMetadata(dawgsVersion string) BaselineMetadata { + if dawgsVersion == "" { + dawgsVersion = currentDAWGSVersion() + } + + return BaselineMetadata{ + DAWGSVersion: dawgsVersion, + } +} + +// currentDAWGSVersion derives a module version and optional VCS revision from Go build information. +func currentDAWGSVersion() string { + buildInfo, ok := debug.ReadBuildInfo() + if !ok { + return "unknown" + } + + version := buildInfo.Main.Version + if version == "" { + version = "(devel)" + } + + for _, setting := range buildInfo.Settings { + if setting.Key == "vcs.revision" && setting.Value != "" { + return version + "@" + setting.Value + } + } + + return version +} diff --git a/testutil/metadata_test.go b/testutil/metadata_test.go new file mode 100644 index 00000000..48106cca --- /dev/null +++ b/testutil/metadata_test.go @@ -0,0 +1,34 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestResolveBaselineMetadata verifies an explicit version is preserved in generated baseline metadata. +func TestResolveBaselineMetadata(t *testing.T) { + metadata := ResolveBaselineMetadata("dawgs") + require.Equal(t, BaselineMetadata{ + DAWGSVersion: "dawgs", + }, metadata) + + defaults := ResolveBaselineMetadata("") + require.NotEmpty(t, defaults.DAWGSVersion) +} diff --git a/testutil/params.go b/testutil/params.go new file mode 100644 index 00000000..51e49a6a --- /dev/null +++ b/testutil/params.go @@ -0,0 +1,208 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +// Package testutil provides reusable corpus, fixture, and baseline helpers for +// DAWGS tests and diagnostic commands. +package testutil + +import ( + "encoding/json" + "fmt" + "time" +) + +const ( + // typeKey identifies the discriminator field in a tagged test parameter. + typeKey = "$type" + + // valueKey identifies the payload field in a tagged scalar parameter. + valueKey = "value" + + // prefixKey identifies the generated value prefix in a tagged string list. + prefixKey = "prefix" + + // countKey identifies the generated value count in a tagged string list. + countKey = "count" + + // includeKey identifies literal values prepended to a tagged string list. + includeKey = "include" +) + +// Params is a query parameter map that supports tagged generated and temporal +// values. A datetime is represented in JSON as: +// +// {"$type": "datetime", "value": "2026-01-02T03:04:05Z"} +// +// A deterministic string list is represented without committing a large +// handwritten array as: +// +// {"$type": "string_list", "prefix": "missing", "count": 1000, "include": ["target-id"]} +// +// Tagged values may also appear in nested maps and lists. +type Params map[string]any + +// UnmarshalJSON decodes plain JSON parameters and expands supported tagged +// values recursively. +func (s *Params) UnmarshalJSON(raw []byte) error { + var decoded map[string]any + if err := json.Unmarshal(raw, &decoded); err != nil { + return err + } + + converted, err := convertMap(decoded) + if err != nil { + return err + } + + *s = Params(converted) + return nil +} + +// convertMap recursively converts every value in a decoded parameter map. +func convertMap(values map[string]any) (map[string]any, error) { + converted := make(map[string]any, len(values)) + for key, value := range values { + typedValue, err := convertValue(value) + if err != nil { + return nil, fmt.Errorf("parameter %q: %w", key, err) + } + + converted[key] = typedValue + } + + return converted, nil +} + +// convertValue expands tagged maps and recursively converts nested maps and +// lists while preserving scalar values. +func convertValue(value any) (any, error) { + switch typedValue := value.(type) { + case map[string]any: + if typeName, tagged := typedValue[typeKey]; tagged { + return convertTaggedValue(typeName, typedValue) + } + + return convertMap(typedValue) + + case []any: + converted := make([]any, len(typedValue)) + for idx, item := range typedValue { + next, err := convertValue(item) + if err != nil { + return nil, fmt.Errorf("list item %d: %w", idx, err) + } + converted[idx] = next + } + + return converted, nil + + default: + return value, nil + } +} + +// convertTaggedValue validates and expands one supported tagged parameter. +func convertTaggedValue(rawType any, tagged map[string]any) (any, error) { + typeName, ok := rawType.(string) + if !ok { + return nil, fmt.Errorf("%s must be a string", typeKey) + } + + switch typeName { + case "datetime": + rawValue, found := tagged[valueKey] + if !found { + return nil, fmt.Errorf("datetime is missing %q", valueKey) + } + + value, ok := rawValue.(string) + if !ok { + return nil, fmt.Errorf("datetime %q must be a string", valueKey) + } + + parsed, err := time.Parse(time.RFC3339Nano, value) + if err != nil { + return nil, fmt.Errorf("parse datetime %q: %w", value, err) + } + + if len(tagged) != 2 { + return nil, fmt.Errorf("datetime must contain only %q and %q", typeKey, valueKey) + } + + return parsed, nil + + case "string_list": + return convertStringList(tagged) + + default: + return nil, fmt.Errorf("unsupported tagged parameter type %q", typeName) + } +} + +// convertStringList expands a tagged string-list specification into its +// literal and generated values. +func convertStringList(tagged map[string]any) ([]string, error) { + rawPrefix, found := tagged[prefixKey] + if !found { + return nil, fmt.Errorf("string_list is missing %q", prefixKey) + } + prefix, ok := rawPrefix.(string) + if !ok { + return nil, fmt.Errorf("string_list %q must be a string", prefixKey) + } + + rawCount, found := tagged[countKey] + if !found { + return nil, fmt.Errorf("string_list is missing %q", countKey) + } + countValue, ok := rawCount.(float64) + if !ok || countValue < 0 || countValue != float64(int(countValue)) { + return nil, fmt.Errorf("string_list %q must be a non-negative integer", countKey) + } + count := int(countValue) + + include := make([]string, 0) + if rawInclude, found := tagged[includeKey]; found { + values, ok := rawInclude.([]any) + if !ok { + return nil, fmt.Errorf("string_list %q must be a string list", includeKey) + } + include = make([]string, len(values)) + for idx, value := range values { + stringValue, ok := value.(string) + if !ok { + return nil, fmt.Errorf("string_list %q item %d must be a string", includeKey, idx) + } + include[idx] = stringValue + } + } + + if len(tagged) != 3 && !(len(tagged) == 4 && tagged[includeKey] != nil) { + return nil, fmt.Errorf("string_list must contain only %q, %q, %q, and optional %q", typeKey, prefixKey, countKey, includeKey) + } + + width := len(fmt.Sprintf("%d", max(count-1, 0))) + if width < 2 { + width = 2 + } + + values := make([]string, 0, len(include)+count) + values = append(values, include...) + for idx := range count { + values = append(values, fmt.Sprintf("%s-%0*d", prefix, width, idx)) + } + return values, nil +} diff --git a/testutil/params_test.go b/testutil/params_test.go new file mode 100644 index 00000000..287e7669 --- /dev/null +++ b/testutil/params_test.go @@ -0,0 +1,93 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestParamsDecodesTaggedDatetime verifies tagged datetime values are parsed +// recursively with nanosecond precision. +func TestParamsDecodesTaggedDatetime(t *testing.T) { + var values Params + require.NoError(t, json.Unmarshal([]byte(`{ + "threshold": {"$type": "datetime", "value": "2026-01-02T03:04:05.123456789Z"}, + "nested": [{"$type": "datetime", "value": "2025-02-03T04:05:06Z"}] + }`), &values)) + + require.Equal(t, time.Date(2026, time.January, 2, 3, 4, 5, 123456789, time.UTC), values["threshold"]) + require.Equal(t, []any{time.Date(2025, time.February, 3, 4, 5, 6, 0, time.UTC)}, values["nested"]) +} + +// TestParamsDecodesNestedObjectsAsStandardMaps verifies untagged objects remain +// ordinary nested parameter maps. +func TestParamsDecodesNestedObjectsAsStandardMaps(t *testing.T) { + var values Params + require.NoError(t, json.Unmarshal([]byte(`{ + "properties": {"name": "node", "nested": {"enabled": true}} + }`), &values)) + + properties, ok := values["properties"].(map[string]any) + require.True(t, ok) + require.Equal(t, "node", properties["name"]) + + nested, ok := properties["nested"].(map[string]any) + require.True(t, ok) + require.Equal(t, true, nested["enabled"]) +} + +// TestParamsRejectsUnknownTaggedType verifies unsupported tagged parameter +// discriminators fail decoding. +func TestParamsRejectsUnknownTaggedType(t *testing.T) { + var values Params + err := json.Unmarshal([]byte(`{"threshold":{"$type":"timestamp","value":"2026-01-02T03:04:05Z"}}`), &values) + require.ErrorContains(t, err, `unsupported tagged parameter type "timestamp"`) +} + +// TestParamsDecodesDeterministicStringList verifies literal inclusions precede +// deterministically numbered generated values. +func TestParamsDecodesDeterministicStringList(t *testing.T) { + var values Params + require.NoError(t, json.Unmarshal([]byte(`{ + "object_ids": {"$type": "string_list", "prefix": "missing", "count": 3, "include": ["target-a", "target-b"]} + }`), &values)) + + require.Equal(t, []string{"target-a", "target-b", "missing-00", "missing-01", "missing-02"}, values["object_ids"]) +} + +// TestParamsRejectsInvalidStringList verifies malformed string-list +// specifications fail decoding. +func TestParamsRejectsInvalidStringList(t *testing.T) { + testCases := []string{ + `{"ids":{"$type":"string_list","count":1}}`, + `{"ids":{"$type":"string_list","prefix":"x","count":-1}}`, + `{"ids":{"$type":"string_list","prefix":"x","count":1.5}}`, + `{"ids":{"$type":"string_list","prefix":"x","count":1,"include":[1]}}`, + `{"ids":{"$type":"string_list","prefix":"x","count":1,"extra":true}}`, + } + + for _, raw := range testCases { + t.Run(raw, func(t *testing.T) { + var values Params + require.Error(t, json.Unmarshal([]byte(raw), &values)) + }) + } +} diff --git a/testutil/perf_endpoint_seeded.go b/testutil/perf_endpoint_seeded.go new file mode 100644 index 00000000..d44e981b --- /dev/null +++ b/testutil/perf_endpoint_seeded.go @@ -0,0 +1,176 @@ +package testutil + +import ( + "fmt" + "strings" + + "github.com/specterops/dawgs/opengraph" +) + +// EndpointSeededExpansionScaleDataset identifies the generated endpoint-seeded +// expansion fixture. +const EndpointSeededExpansionScaleDataset = "generated_endpoint_seeded_expansion_v1" + +// EndpointSeededExpansionScaleConfig controls the endpoint populations and +// traversal lanes emitted by NewEndpointSeededExpansionScaleFixture. +type EndpointSeededExpansionScaleConfig struct { + // Depth sets the number of MemberOf hops in each lane. + Depth int + + // MatchingEndpoints sets the number of terminal groups whose object IDs + // satisfy the benchmark predicate. + MatchingEndpoints int + + // OtherEndpoints sets the number of terminal groups that do not satisfy the + // benchmark predicate. + OtherEndpoints int + + // MatchingEligibleLanes sets the number of session-backed lanes ending at a + // matching endpoint. + MatchingEligibleLanes int + + // OtherEligibleLanes sets the number of session-backed lanes ending at a + // nonmatching endpoint. + OtherEligibleLanes int + + // MatchingIneligibleLanes sets the number of lanes without a session edge + // that nevertheless end at a matching endpoint. + MatchingIneligibleLanes int + + // ParallelEdges sets the number of MemberOf edges emitted per lane hop. + ParallelEdges int + + // AddCycle adds a reverse MemberOf edge near the middle of every lane. + AddCycle bool + + // PropertyPayloadSize sets the length of synthetic payload properties. + PropertyPayloadSize int +} + +// ValidateEndpointSeededExpansionScaleConfig rejects fixture configurations +// that cannot produce a valid or uniquely keyed endpoint-seeded graph. +func ValidateEndpointSeededExpansionScaleConfig(config EndpointSeededExpansionScaleConfig) error { + if config.Depth < 1 || config.Depth > 64 { + return fmt.Errorf("depth must be between 1 and 64") + } + if config.MatchingEndpoints < 1 || config.OtherEndpoints < 0 || config.MatchingEligibleLanes < 1 || config.OtherEligibleLanes < 0 || config.MatchingIneligibleLanes < 0 { + return fmt.Errorf("endpoint and lane counts are invalid") + } + if config.ParallelEdges != 1 { + return fmt.Errorf("parallel edges must be exactly one because DAWGS graph storage uniquely keys edges by start, end, kind, and graph") + } + if config.PropertyPayloadSize < 0 { + return fmt.Errorf("property payload size must not be negative") + } + return nil +} + +// NewEndpointSeededExpansionScaleFixture creates terminal-selective expansion +// lanes with independently controlled productive and unproductive reverse work. +func NewEndpointSeededExpansionScaleFixture(config EndpointSeededExpansionScaleConfig) *opengraph.Graph { + if ValidateEndpointSeededExpansionScaleConfig(config) != nil { + return nil + } + payload := strings.Repeat("x", config.PropertyPayloadSize) + fixture := &opengraph.Graph{} + for idx := range config.MatchingEndpoints { + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: fmt.Sprintf("ese-match-%03d", idx), + Kinds: []string{"Group"}, + Properties: map[string]any{ + "objectid": fmt.Sprintf("S-1-5-21-%03d-512", idx), + "payload": payload, + }, + }) + } + for idx := range config.OtherEndpoints { + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: fmt.Sprintf("ese-other-%03d", idx), + Kinds: []string{"Group"}, + Properties: map[string]any{ + "objectid": fmt.Sprintf("S-1-5-21-%03d-513", idx), + "payload": payload, + }, + }) + } + + addLane := func(class string, lane int, endpoint string, eligible bool) { + user := fmt.Sprintf("ese-%s-user-%04d", class, lane) + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: user, + Kinds: []string{"User"}, + Properties: map[string]any{"payload": payload}, + }) + if eligible { + computer := fmt.Sprintf("ese-%s-computer-%04d", class, lane) + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: computer, + Kinds: []string{"Computer"}, + Properties: map[string]any{"payload": payload}, + }) + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: computer, + EndID: user, + Kind: "HasSession", + Properties: map[string]any{"logical_key": computer + "-session"}, + }) + } + previous := user + for level := 1; level <= config.Depth; level++ { + next := endpoint + if level < config.Depth { + next = fmt.Sprintf("ese-%s-lane-%04d-level-%02d", class, lane, level) + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: next, + Kinds: []string{"Group"}, + Properties: map[string]any{"payload": payload}, + }) + } + for parallel := range config.ParallelEdges { + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: previous, + EndID: next, + Kind: "MemberOf", + Properties: map[string]any{ + "logical_key": fmt.Sprintf("%s-%04d-%02d-%02d", class, lane, level, parallel), + }, + }) + } + if config.AddCycle && level == max(1, config.Depth/2) && previous != user { + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: next, + EndID: previous, + Kind: "MemberOf", + Properties: map[string]any{ + "logical_key": fmt.Sprintf("%s-%04d-cycle", class, lane), + }, + }) + } + previous = next + } + } + + for lane := range config.MatchingEligibleLanes { + addLane("matching", lane, fmt.Sprintf("ese-match-%03d", lane%config.MatchingEndpoints), true) + } + for lane := range config.OtherEligibleLanes { + endpoint := "ese-other-000" + if config.OtherEndpoints > 0 { + endpoint = fmt.Sprintf("ese-other-%03d", lane%config.OtherEndpoints) + } else { + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: endpoint, + Kinds: []string{"Group"}, + Properties: map[string]any{ + "objectid": "S-1-5-21-513", + "payload": payload, + }, + }) + } + addLane("other", lane, endpoint, true) + } + for lane := range config.MatchingIneligibleLanes { + addLane("ineligible", lane, fmt.Sprintf("ese-match-%03d", lane%config.MatchingEndpoints), false) + } + return fixture +} diff --git a/testutil/perf_fixtures.go b/testutil/perf_fixtures.go new file mode 100644 index 00000000..506552bb --- /dev/null +++ b/testutil/perf_fixtures.go @@ -0,0 +1,654 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import ( + "errors" + "fmt" + "strings" + + "github.com/specterops/dawgs/opengraph" +) + +const ( + // ShortestPathScaleDataset identifies the generated shortest-path fixture. + ShortestPathScaleDataset = "generated_shortest_paths" + + // FixedSuffixExpansionScaleDataset identifies the generated fixed-suffix + // expansion fixture. + FixedSuffixExpansionScaleDataset = "generated_fixed_suffix_expansion" + + // FixedSuffixExpansionScaleV3Dataset identifies the fixed-suffix fixture + // grammar with independent root, cycle, and self-loop controls. + FixedSuffixExpansionScaleV3Dataset = FixedSuffixExpansionScaleDataset + "_v3" +) + +// ShortestPathScaleConfig controls the depth and dead-end fanout of the +// generated shortest-path fixture. +type ShortestPathScaleConfig struct { + // Depth sets the length of the fixture's unique linear route. + Depth int + + // Fanout sets the number of dead ends attached to the route's start. + Fanout int +} + +// NewShortestPathScaleFixture builds deterministic linear, diamond, dead-end, +// cycle, parallel-edge, self-loop, wrong-direction, and disconnected shapes +// around a bound endpoint pair. Fanout controls parallel dead ends without +// changing the unique linear route's requested depth. +func NewShortestPathScaleFixture(config ShortestPathScaleConfig) *opengraph.Graph { + depth := max(config.Depth, 1) + fanout := max(config.Fanout, 1) + fixture := &opengraph.Graph{} + + fixture.Nodes = append(fixture.Nodes, + opengraph.Node{ + ID: "sp-start", + Kinds: []string{"ShortestNode"}, + Properties: map[string]any{"role": "start"}, + }, + opengraph.Node{ + ID: "sp-end", + Kinds: []string{"ShortestNode"}, + Properties: map[string]any{"role": "end"}, + }, + opengraph.Node{ + ID: "sp-disconnected", + Kinds: []string{"ShortestNode"}, + }, + opengraph.Node{ + ID: "sp-wrong-direction", + Kinds: []string{"ShortestNode"}, + }, + ) + + previous := "sp-start" + for level := 1; level < depth; level++ { + next := fmt.Sprintf("sp-linear-%02d", level) + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: next, + Kinds: []string{"ShortestNode"}, + }) + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: previous, + EndID: next, + Kind: "Traverse", + }) + previous = next + } + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: previous, + EndID: "sp-end", + Kind: "Traverse", + }) + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: "sp-end", + EndID: "sp-wrong-direction", + Kind: "Traverse", + }) + + for idx := range fanout { + deadEnd := fmt.Sprintf("sp-dead-%04d", idx) + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: deadEnd, + Kinds: []string{"ShortestNode"}, + }) + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: "sp-start", + EndID: deadEnd, + Kind: "Traverse", + }) + } + + fixture.Nodes = append(fixture.Nodes, + opengraph.Node{ + ID: "sp-diamond-left", + Kinds: []string{"ShortestNode"}, + }, + opengraph.Node{ + ID: "sp-diamond-right", + Kinds: []string{"ShortestNode"}, + }, + opengraph.Node{ + ID: "sp-diamond-end", + Kinds: []string{"ShortestNode"}, + }, + opengraph.Node{ + ID: "sp-cycle-a", + Kinds: []string{"ShortestNode"}, + }, + opengraph.Node{ + ID: "sp-cycle-b", + Kinds: []string{"ShortestNode"}, + }, + opengraph.Node{ + ID: "sp-parallel-end", + Kinds: []string{"ShortestNode"}, + }, + opengraph.Node{ + ID: "sp-self-loop", + Kinds: []string{"ShortestNode"}, + }, + opengraph.Node{ + ID: "sp-self-loop-exit", + Kinds: []string{"ShortestNode"}, + }, + ) + fixture.Edges = append(fixture.Edges, + opengraph.Edge{ + StartID: "sp-start", + EndID: "sp-diamond-left", + Kind: "Traverse", + }, + opengraph.Edge{ + StartID: "sp-start", + EndID: "sp-diamond-right", + Kind: "Traverse", + }, + opengraph.Edge{ + StartID: "sp-diamond-left", + EndID: "sp-diamond-end", + Kind: "TypedTraverse", + }, + opengraph.Edge{ + StartID: "sp-diamond-right", + EndID: "sp-diamond-end", + Kind: "TypedTraverse", + }, + opengraph.Edge{ + StartID: "sp-start", + EndID: "sp-cycle-a", + Kind: "Traverse", + }, + opengraph.Edge{ + StartID: "sp-cycle-a", + EndID: "sp-cycle-b", + Kind: "Traverse", + }, + opengraph.Edge{ + StartID: "sp-cycle-b", + EndID: "sp-cycle-a", + Kind: "Traverse", + }, + opengraph.Edge{ + StartID: "sp-start", + EndID: "sp-parallel-end", + Kind: "Traverse", + Properties: map[string]any{"logical_key": "sp-parallel-0"}, + }, + opengraph.Edge{ + StartID: "sp-start", + EndID: "sp-parallel-end", + Kind: "TypedTraverse", + Properties: map[string]any{"logical_key": "sp-parallel-1"}, + }, + opengraph.Edge{ + StartID: "sp-start", + EndID: "sp-self-loop", + Kind: "Traverse", + }, + opengraph.Edge{ + StartID: "sp-self-loop", + EndID: "sp-self-loop", + Kind: "Traverse", + }, + opengraph.Edge{ + StartID: "sp-self-loop", + EndID: "sp-self-loop-exit", + Kind: "Traverse", + }, + ) + + return fixture +} + +// FixedSuffixExpansionScaleConfig controls expansion work, suffix density, +// decoys, and payload size in a fixed-suffix fixture. +type FixedSuffixExpansionScaleConfig struct { + // ExpansionDepth sets the number of Expand hops in each branch. + ExpansionDepth int + + // Fanout sets the number of expansion branches rooted at the fixture root. + Fanout int + + // ValidSuffixEvery attaches a suffix to every nth legacy branch. + ValidSuffixEvery int + + // PropertyPayloadSize sets the length of synthetic payload properties. + PropertyPayloadSize int + + // ExactReachableSuffixSources decouples reachable suffix density from the + // legacy modulus control. Nil preserves ValidSuffixEvery behavior; zero is + // an exact zero and is therefore materially different from nil. + ExactReachableSuffixSources *int + + // ReachableSuffixDepths restricts suffix attachment to the listed expansion + // depths when nonempty. + ReachableSuffixDepths []int + + // DisconnectedSuffixSources sets the number of suffix sources unreachable + // from any expansion root. + DisconnectedSuffixSources int + + // ReverseFanIn sets the number of decoy Expand edges entering a productive + // branch boundary. + ReverseFanIn int + + // SuffixPathsPerBoundary sets the number of distinct suffix paths attached + // to each selected boundary. + SuffixPathsPerBoundary int + + // RootMatchCount sets the number of roots matching the fixture root key. + RootMatchCount int + + // RootHasZeroDepthSuffix controls whether the primary root has a suffix; + // nil preserves the enabled default. + RootHasZeroDepthSuffix *bool + + // AddProductiveBoundaryCycle adds a two-edge Expand cycle at the + // deterministic productive boundary. The two physical relationships have + // distinct endpoints and logical keys, so the cycle can be traversed once + // in either relationship-distinct expansion direction. + AddProductiveBoundaryCycle bool + + // AddProductiveBoundarySelfLoop adds one Expand self-loop at the + // deterministic productive boundary. + AddProductiveBoundarySelfLoop bool +} + +// ValidateFixedSuffixExpansionScaleV3Config rejects dimensions that cannot +// describe the exact v3 fixture grammar. V3 requires every population to be +// explicit; legacy and v2 callers retain their existing defaulting behavior. +func ValidateFixedSuffixExpansionScaleV3Config(config FixedSuffixExpansionScaleConfig) error { + values := []int{ + config.ExpansionDepth, config.Fanout, config.DisconnectedSuffixSources, + config.ReverseFanIn, config.SuffixPathsPerBoundary, config.RootMatchCount, + config.PropertyPayloadSize, + } + for _, value := range values { + if value < 0 { + return errors.New("fixed-suffix v3 configuration values must not be negative") + } + } + if config.ExpansionDepth > 64 { + return errors.New("fixed-suffix v3 depth must not exceed 64") + } + if config.Fanout < 1 { + return errors.New("fixed-suffix v3 fanout must be positive") + } + if config.ExactReachableSuffixSources == nil { + return errors.New("fixed-suffix v3 reachable suffix sources must be explicit") + } + reachable := *config.ExactReachableSuffixSources + if reachable < 0 || reachable > config.Fanout { + return errors.New("fixed-suffix v3 reachable suffix sources must be between zero and fanout") + } + if config.ExpansionDepth == 0 && reachable != 0 { + return errors.New("fixed-suffix v3 depth-zero fixtures cannot have reachable branch suffixes") + } + if config.SuffixPathsPerBoundary < 1 { + return errors.New("fixed-suffix v3 suffix path multiplicity must be positive") + } + if config.RootMatchCount < 1 { + return errors.New("fixed-suffix v3 root match count must be positive") + } + if config.RootHasZeroDepthSuffix == nil { + return errors.New("fixed-suffix v3 zero-depth suffix control must be explicit") + } + if config.ValidSuffixEvery != 0 || len(config.ReachableSuffixDepths) != 0 { + return errors.New("fixed-suffix v3 cannot mix legacy suffix-density controls with exact controls") + } + + hasProductiveBoundary := *config.RootHasZeroDepthSuffix || reachable > 0 + if !hasProductiveBoundary && config.ReverseFanIn != 0 { + return errors.New("fixed-suffix v3 reverse fan-in requires a productive boundary") + } + if !hasProductiveBoundary && (config.AddProductiveBoundaryCycle || config.AddProductiveBoundarySelfLoop) { + return errors.New("fixed-suffix v3 cycle and self-loop controls require a productive boundary") + } + return nil +} + +// NewFixedSuffixExpansionScaleFixture builds a deterministic expansion fanout +// feeding a shared fixed suffix. It also emits independent wrong-kind, +// wrong-direction, wrong-endpoint-kind, and disconnected suffix decoys. +func NewFixedSuffixExpansionScaleFixture(config FixedSuffixExpansionScaleConfig) *opengraph.Graph { + if config.ExactReachableSuffixSources == nil && len(config.ReachableSuffixDepths) == 0 && config.DisconnectedSuffixSources == 0 && config.ReverseFanIn == 0 && config.SuffixPathsPerBoundary == 0 && config.RootMatchCount == 0 && config.RootHasZeroDepthSuffix == nil && !config.AddProductiveBoundaryCycle && !config.AddProductiveBoundarySelfLoop { + return newLegacyFixedSuffixExpansionScaleFixture(config) + } + depth := max(config.ExpansionDepth, 0) + fanout := max(config.Fanout, 1) + validEvery := max(config.ValidSuffixEvery, 1) + reachableSources := -1 + if config.ExactReachableSuffixSources != nil { + reachableSources = min(max(*config.ExactReachableSuffixSources, 0), fanout) + } + suffixPaths := max(config.SuffixPathsPerBoundary, 1) + rootCount := max(config.RootMatchCount, 1) + rootHasSuffix := true + if config.RootHasZeroDepthSuffix != nil { + rootHasSuffix = *config.RootHasZeroDepthSuffix + } + payload := strings.Repeat("x", max(config.PropertyPayloadSize, 0)) + + fixture := &opengraph.Graph{ + Nodes: []opengraph.Node{ + { + ID: "fse-terminal", + Kinds: []string{"SuffixTerminal"}, + }, + { + ID: "fse-wrong-endpoint", + Kinds: []string{"ExpansionNode"}, + }, + }, + } + for rootIdx := range rootCount { + rootID := "fse-root" + if rootIdx > 0 { + rootID = fmt.Sprintf("fse-root-%02d", rootIdx) + } + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: rootID, + Kinds: []string{"ExpansionRoot"}, + Properties: map[string]any{"root_key": "generated-fse-root", "payload": payload}, + }) + } + addSuffix := func(source, key string) { + for pathIdx := range suffixPaths { + headID := fmt.Sprintf("fse-head-%s-%02d", key, pathIdx) + middleID := fmt.Sprintf("fse-middle-%s-%02d", key, pathIdx) + fixture.Nodes = append(fixture.Nodes, + opengraph.Node{ + ID: headID, + Kinds: []string{"SuffixHead"}, + Properties: map[string]any{"payload": payload}, + }, + opengraph.Node{ + ID: middleID, + Kinds: []string{"SuffixMiddle"}, + }, + ) + fixture.Edges = append(fixture.Edges, + opengraph.Edge{ + StartID: source, + EndID: headID, + Kind: "EnterSuffix", + Properties: map[string]any{"payload": payload, "logical_key": key + ":enter"}, + }, + opengraph.Edge{ + StartID: headID, + EndID: middleID, + Kind: "ContinueSuffix", + Properties: map[string]any{"logical_key": key + ":continue"}, + }, + opengraph.Edge{ + StartID: middleID, + EndID: "fse-terminal", + Kind: "CompleteSuffix", + Properties: map[string]any{"logical_key": key + ":complete"}, + }, + ) + } + } + if rootHasSuffix { + addSuffix("fse-root", "root") + } + + productiveBoundary := "fse-root" + if depth > 0 { + for branch := range fanout { + previous := "fse-root" + for level := 1; level <= depth; level++ { + next := fmt.Sprintf("fse-branch-%04d-level-%02d", branch, level) + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: next, + Kinds: []string{"ExpansionNode"}, + Properties: map[string]any{"payload": payload}, + }) + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: previous, + EndID: next, + Kind: "Expand", + Properties: map[string]any{"logical_key": fmt.Sprintf("branch-%04d-level-%02d", branch, level)}, + }) + previous = next + } + reachable := branch%validEvery == 0 + if reachableSources >= 0 { + reachable = branch < reachableSources + } + if reachable && (len(config.ReachableSuffixDepths) == 0 || containsInt(config.ReachableSuffixDepths, depth)) { + addSuffix(previous, fmt.Sprintf("branch-%04d-depth-%02d", branch, depth)) + if branch == 0 { + productiveBoundary = previous + } + } + } + } + for idx := range max(config.DisconnectedSuffixSources, 0) { + source := fmt.Sprintf("fse-disconnected-%05d", idx) + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: source, + Kinds: []string{"ExpansionNode"}, + }) + addSuffix(source, fmt.Sprintf("disconnected-%05d", idx)) + } + for idx := range max(config.ReverseFanIn, 0) { + source := fmt.Sprintf("fse-fanin-%05d", idx) + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: source, + Kinds: []string{"ExpansionNode"}, + }) + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: source, + EndID: productiveBoundary, + Kind: "Expand", + Properties: map[string]any{"logical_key": fmt.Sprintf("fanin-%05d", idx)}, + }) + } + if config.AddProductiveBoundaryCycle { + const cycleNode = "fse-productive-boundary-cycle" + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: cycleNode, + Kinds: []string{"ExpansionNode"}, + }) + fixture.Edges = append(fixture.Edges, + opengraph.Edge{ + StartID: productiveBoundary, + EndID: cycleNode, + Kind: "Expand", + Properties: map[string]any{ + "logical_key": "productive-boundary-cycle-enter", + }, + }, + opengraph.Edge{ + StartID: cycleNode, + EndID: productiveBoundary, + Kind: "Expand", + Properties: map[string]any{ + "logical_key": "productive-boundary-cycle-return", + }, + }, + ) + } + if config.AddProductiveBoundarySelfLoop { + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: productiveBoundary, + EndID: productiveBoundary, + Kind: "Expand", + Properties: map[string]any{ + "logical_key": "productive-boundary-self-loop", + }, + }) + } + + decoySource := "fse-root" + if depth > 0 { + decoySource = "fse-branch-0000-level-01" + } + fixture.Nodes = append(fixture.Nodes, + opengraph.Node{ + ID: "fse-decoy-head", + Kinds: []string{"SuffixHead"}, + }, + opengraph.Node{ + ID: "fse-decoy-middle", + Kinds: []string{"SuffixMiddle"}, + }, + ) + fixture.Edges = append(fixture.Edges, + opengraph.Edge{ + StartID: decoySource, + EndID: "fse-decoy-head", + Kind: "WrongEnterSuffix", + }, + opengraph.Edge{ + StartID: "fse-decoy-head", + EndID: decoySource, + Kind: "EnterSuffix", + }, + opengraph.Edge{ + StartID: decoySource, + EndID: "fse-wrong-endpoint", + Kind: "EnterSuffix", + }, + ) + + return fixture +} + +// newLegacyFixedSuffixExpansionScaleFixture builds the original shared-suffix +// topology used when no independent population controls are configured. +func newLegacyFixedSuffixExpansionScaleFixture(config FixedSuffixExpansionScaleConfig) *opengraph.Graph { + depth := max(config.ExpansionDepth, 0) + fanout := max(config.Fanout, 1) + validEvery := max(config.ValidSuffixEvery, 1) + payload := strings.Repeat("x", max(config.PropertyPayloadSize, 0)) + fixture := &opengraph.Graph{ + Nodes: []opengraph.Node{ + { + ID: "fse-root", + Kinds: []string{"ExpansionRoot"}, + Properties: map[string]any{"root_key": "generated-fse-root", "payload": payload}, + }, + { + ID: "fse-head", + Kinds: []string{"SuffixHead"}, + Properties: map[string]any{"payload": payload}, + }, + { + ID: "fse-middle", + Kinds: []string{"SuffixMiddle"}, + }, + { + ID: "fse-terminal", + Kinds: []string{"SuffixTerminal"}, + }, + { + ID: "fse-wrong-endpoint", + Kinds: []string{"ExpansionNode"}, + }, + { + ID: "fse-disconnected", + Kinds: []string{"ExpansionNode"}, + }, + }, + } + fixture.Edges = append(fixture.Edges, + opengraph.Edge{ + StartID: "fse-root", + EndID: "fse-head", + Kind: "EnterSuffix", + Properties: map[string]any{"payload": payload}, + }, + opengraph.Edge{ + StartID: "fse-head", + EndID: "fse-middle", + Kind: "ContinueSuffix", + }, + opengraph.Edge{ + StartID: "fse-middle", + EndID: "fse-terminal", + Kind: "CompleteSuffix", + }, + ) + if depth > 0 { + for branch := range fanout { + previous := "fse-root" + for level := 1; level <= depth; level++ { + next := fmt.Sprintf("fse-branch-%04d-level-%02d", branch, level) + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: next, + Kinds: []string{"ExpansionNode"}, + Properties: map[string]any{"payload": payload}, + }) + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: previous, + EndID: next, + Kind: "Expand", + }) + previous = next + } + if branch%validEvery == 0 { + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: previous, + EndID: "fse-head", + Kind: "EnterSuffix", + }) + } + } + } + decoySource := "fse-root" + if depth > 0 { + decoySource = "fse-branch-0000-level-01" + } + fixture.Edges = append(fixture.Edges, + opengraph.Edge{ + StartID: decoySource, + EndID: "fse-head", + Kind: "WrongEnterSuffix", + }, + opengraph.Edge{ + StartID: "fse-head", + EndID: decoySource, + Kind: "EnterSuffix", + }, + opengraph.Edge{ + StartID: decoySource, + EndID: "fse-wrong-endpoint", + Kind: "EnterSuffix", + }, + opengraph.Edge{ + StartID: "fse-disconnected", + EndID: "fse-head", + Kind: "EnterSuffix", + }, + ) + return fixture +} + +// containsInt reports whether target occurs in values. +func containsInt(values []int, target int) bool { + for _, value := range values { + if value == target { + return true + } + } + return false +} diff --git a/testutil/perf_fixtures_test.go b/testutil/perf_fixtures_test.go new file mode 100644 index 00000000..82c2cd43 --- /dev/null +++ b/testutil/perf_fixtures_test.go @@ -0,0 +1,325 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import ( + "encoding/json" + "slices" + "strings" + "testing" + + "github.com/specterops/dawgs/opengraph" + "github.com/stretchr/testify/require" +) + +// TestShortestPathScaleFixtureIsDeterministicAndCardinalityExact verifies the +// legacy shortest-path fixture is stable and emits the expected topology. +func TestShortestPathScaleFixtureIsDeterministicAndCardinalityExact(t *testing.T) { + config := ShortestPathScaleConfig{ + Depth: 16, + Fanout: 10, + } + first := NewShortestPathScaleFixture(config) + second := NewShortestPathScaleFixture(config) + firstJSON, err := json.Marshal(first) + require.NoError(t, err) + secondJSON, err := json.Marshal(second) + require.NoError(t, err) + require.Equal(t, firstJSON, secondJSON) + require.Len(t, first.Nodes, 4+(config.Depth-1)+config.Fanout+8) + require.Len(t, first.Edges, config.Depth+1+config.Fanout+12) + + var parallel, selfLoops int + for _, edge := range first.Edges { + if edge.StartID == "sp-start" && edge.EndID == "sp-parallel-end" { + parallel++ + } + if edge.StartID == "sp-self-loop" && edge.EndID == "sp-self-loop" { + selfLoops++ + } + } + require.Equal(t, 2, parallel) + require.Equal(t, 1, selfLoops) +} + +// TestEndpointSeededExpansionFixtureIsDeterministicAndSeparatesWorkClasses verifies productive, nonmatching, and ineligible lanes remain distinct. +func TestEndpointSeededExpansionFixtureIsDeterministicAndSeparatesWorkClasses(t *testing.T) { + config := EndpointSeededExpansionScaleConfig{ + Depth: 3, + MatchingEndpoints: 2, + OtherEndpoints: 1, + MatchingEligibleLanes: 2, + OtherEligibleLanes: 1, + MatchingIneligibleLanes: 1, + ParallelEdges: 1, + AddCycle: true, + PropertyPayloadSize: 8, + } + first := NewEndpointSeededExpansionScaleFixture(config) + second := NewEndpointSeededExpansionScaleFixture(config) + firstJSON, err := json.Marshal(first) + require.NoError(t, err) + secondJSON, err := json.Marshal(second) + require.NoError(t, err) + require.Equal(t, firstJSON, secondJSON) + require.NotEmpty(t, first.Nodes) + require.NotEmpty(t, first.Edges) + require.NoError(t, ValidateEndpointSeededExpansionScaleConfig(config)) + require.Error(t, ValidateEndpointSeededExpansionScaleConfig(EndpointSeededExpansionScaleConfig{})) + config.ParallelEdges = 2 + require.ErrorContains(t, ValidateEndpointSeededExpansionScaleConfig(config), "uniquely keys edges") +} + +// TestShortestPathScaleV2FixtureIsDeterministicAndTopologyExact verifies the +// configurable fixture is stable and assigns unique logical edge keys. +func TestShortestPathScaleV2FixtureIsDeterministicAndTopologyExact(t *testing.T) { + config := ShortestPathScaleV2Config{ + Depth: 3, + ForwardRootFanOut: 2, + ReverseRootFanIn: 2, + IntermediateFanOut: 1, + IntermediateReverseFanIn: 4, + FanInLevel: 2, + ParallelKindCount: 3, + ParallelTargetCount: 2, + DiamondWidth: 2, + DisconnectedWidth: 3, + PropertyPayloadSize: 8, + AddCycle: true, + AddSelfLoop: true, + } + first := NewShortestPathScaleV2Fixture(config) + second := NewShortestPathScaleV2Fixture(config) + firstJSON, err := json.Marshal(first) + require.NoError(t, err) + secondJSON, err := json.Marshal(second) + require.NoError(t, err) + require.Equal(t, firstJSON, secondJSON) + require.Len(t, first.Nodes, 32) + require.Len(t, first.Edges, 33) + + logicalKeys := map[string]bool{} + for _, edge := range first.Edges { + key, ok := edge.Properties["logical_key"].(string) + require.True(t, ok) + require.NotEmpty(t, key) + require.False(t, logicalKeys[key], key) + logicalKeys[key] = true + } +} + +// TestShortestPathScaleV2ConfigurationRejectsImpossibleShapes verifies invalid +// dimensions and inconsistent fan-in controls are rejected. +func TestShortestPathScaleV2ConfigurationRejectsImpossibleShapes(t *testing.T) { + for _, config := range []ShortestPathScaleV2Config{ + { + Depth: -1, + }, + { + Depth: 65, + }, + { + Depth: 3, + FanInLevel: 2, + }, + { + Depth: 3, + IntermediateReverseFanIn: 1, + FanInLevel: 3, + }, + { + ParallelKindCount: 1, + }, + { + ParallelTargetCount: 1, + }, + } { + require.Error(t, ValidateShortestPathScaleV2Config(config)) + } + require.NoError(t, ValidateShortestPathScaleV2Config(ShortestPathScaleV2Config{})) +} + +// TestFixedSuffixExpansionScaleFixtureIsDeterministicAndCoversDecoys verifies +// the legacy suffix topology remains stable and includes wrong-kind edges. +func TestFixedSuffixExpansionScaleFixtureIsDeterministicAndCoversDecoys(t *testing.T) { + config := FixedSuffixExpansionScaleConfig{ + ExpansionDepth: 4, + Fanout: 10, + ValidSuffixEvery: 2, + PropertyPayloadSize: 32, + } + first := NewFixedSuffixExpansionScaleFixture(config) + second := NewFixedSuffixExpansionScaleFixture(config) + firstJSON, err := json.Marshal(first) + require.NoError(t, err) + secondJSON, err := json.Marshal(second) + require.NoError(t, err) + require.Equal(t, firstJSON, secondJSON) + require.Len(t, first.Nodes, 6+config.ExpansionDepth*config.Fanout) + require.Len(t, first.Edges, 3+config.ExpansionDepth*config.Fanout+5+4) + + _, edgeKinds := first.Kinds() + require.Contains(t, edgeKinds.Strings(), "WrongEnterSuffix") +} + +// TestFixedSuffixExpansionScaleFixtureV2ControlsSuffixPopulationsIndependently verifies reachable, disconnected, and reverse-fan-in populations can vary +// without changing one another. +func TestFixedSuffixExpansionScaleFixtureV2ControlsSuffixPopulationsIndependently(t *testing.T) { + reachable := 0 + zeroDepth := false + fixture := NewFixedSuffixExpansionScaleFixture(FixedSuffixExpansionScaleConfig{ + ExpansionDepth: 2, + Fanout: 4, + ExactReachableSuffixSources: &reachable, + DisconnectedSuffixSources: 3, + ReverseFanIn: 2, + SuffixPathsPerBoundary: 2, + RootMatchCount: 1, + RootHasZeroDepthSuffix: &zeroDepth, + }) + + var enterSuffix, expand int + for _, edge := range fixture.Edges { + switch edge.Kind { + case "EnterSuffix": + enterSuffix++ + case "Expand": + expand++ + } + } + require.Equal(t, 8, enterSuffix) + require.Equal(t, 10, expand) + nodeIDs := make([]string, 0, len(fixture.Nodes)) + for _, node := range fixture.Nodes { + nodeIDs = append(nodeIDs, node.ID) + } + require.NotContains(t, nodeIDs, "fse-disconnected") + require.Contains(t, nodeIDs, "fse-disconnected-00002") +} + +// TestFixedSuffixExpansionScaleFixtureV3ControlsRootMultiplicity verifies that +// matching root rows vary without multiplying the primary root's fanout or +// suffix population. +func TestFixedSuffixExpansionScaleFixtureV3ControlsRootMultiplicity(t *testing.T) { + reachable := 1 + zeroDepth := false + config := FixedSuffixExpansionScaleConfig{ + ExpansionDepth: 2, + Fanout: 2, + ExactReachableSuffixSources: &reachable, + SuffixPathsPerBoundary: 1, + RootMatchCount: 3, + RootHasZeroDepthSuffix: &zeroDepth, + } + require.NoError(t, ValidateFixedSuffixExpansionScaleV3Config(config)) + + fixture := NewFixedSuffixExpansionScaleFixture(config) + matchingRoots := 0 + for _, node := range fixture.Nodes { + if slices.Contains(node.Kinds, "ExpansionRoot") && node.Properties["root_key"] == "generated-fse-root" { + matchingRoots++ + } + } + require.Equal(t, 3, matchingRoots) + + rootExpandEdges := 0 + for _, edge := range fixture.Edges { + if edge.Kind == "Expand" && edge.StartID == "fse-root" { + rootExpandEdges++ + } + } + require.Equal(t, 2, rootExpandEdges) +} + +// TestFixedSuffixExpansionScaleFixtureV3ProductiveBoundaryControls verifies +// all cycle/self-loop combinations and the stable, relationship-distinct +// topology emitted for each enabled control. +func TestFixedSuffixExpansionScaleFixtureV3ProductiveBoundaryControls(t *testing.T) { + for _, testCase := range []struct { + name string + cycle bool + selfLoop bool + wantEdges int + }{ + {name: "neither"}, + {name: "cycle", cycle: true, wantEdges: 2}, + {name: "self-loop", selfLoop: true, wantEdges: 1}, + {name: "both", cycle: true, selfLoop: true, wantEdges: 3}, + } { + t.Run(testCase.name, func(t *testing.T) { + reachable := 0 + zeroDepth := true + config := FixedSuffixExpansionScaleConfig{ + ExpansionDepth: 2, + Fanout: 1, + ExactReachableSuffixSources: &reachable, + SuffixPathsPerBoundary: 1, + RootMatchCount: 1, + RootHasZeroDepthSuffix: &zeroDepth, + AddProductiveBoundaryCycle: testCase.cycle, + AddProductiveBoundarySelfLoop: testCase.selfLoop, + } + require.NoError(t, ValidateFixedSuffixExpansionScaleV3Config(config)) + + fixture := NewFixedSuffixExpansionScaleFixture(config) + controlEdges := map[string]opengraph.Edge{} + for _, edge := range fixture.Edges { + logicalKey, _ := edge.Properties["logical_key"].(string) + if strings.HasPrefix(logicalKey, "productive-boundary-") { + controlEdges[logicalKey] = edge + } + } + require.Len(t, controlEdges, testCase.wantEdges) + if testCase.cycle { + require.Equal(t, "fse-productive-boundary-cycle", controlEdges["productive-boundary-cycle-enter"].EndID) + require.Equal(t, "fse-root", controlEdges["productive-boundary-cycle-return"].EndID) + } + if testCase.selfLoop { + selfLoop := controlEdges["productive-boundary-self-loop"] + require.Equal(t, "fse-root", selfLoop.StartID) + require.Equal(t, selfLoop.StartID, selfLoop.EndID) + } + }) + } +} + +// TestFixedSuffixExpansionScaleV3ConfigurationRejectsUnproductiveControls +// verifies that topology and fan-in controls cannot be attached to a boundary +// with no generated suffix. +func TestFixedSuffixExpansionScaleV3ConfigurationRejectsUnproductiveControls(t *testing.T) { + reachable := 0 + zeroDepth := false + base := FixedSuffixExpansionScaleConfig{ + ExpansionDepth: 2, + Fanout: 1, + ExactReachableSuffixSources: &reachable, + SuffixPathsPerBoundary: 1, + RootMatchCount: 1, + RootHasZeroDepthSuffix: &zeroDepth, + } + require.NoError(t, ValidateFixedSuffixExpansionScaleV3Config(base)) + + withCycle := base + withCycle.AddProductiveBoundaryCycle = true + require.Error(t, ValidateFixedSuffixExpansionScaleV3Config(withCycle)) + withSelfLoop := base + withSelfLoop.AddProductiveBoundarySelfLoop = true + require.Error(t, ValidateFixedSuffixExpansionScaleV3Config(withSelfLoop)) + withFanIn := base + withFanIn.ReverseFanIn = 1 + require.Error(t, ValidateFixedSuffixExpansionScaleV3Config(withFanIn)) +} diff --git a/testutil/perf_shortest_v2.go b/testutil/perf_shortest_v2.go new file mode 100644 index 00000000..d6fce0aa --- /dev/null +++ b/testutil/perf_shortest_v2.go @@ -0,0 +1,248 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import ( + "errors" + "fmt" + "strings" + + "github.com/specterops/dawgs/opengraph" +) + +// ShortestPathScaleV2Dataset identifies the second-generation generated +// shortest-path fixture. +const ShortestPathScaleV2Dataset = ShortestPathScaleDataset + "_v2" + +// ShortestPathScaleV2Config controls independent path, decoy, fanout, and +// payload dimensions in the second-generation shortest-path fixture. +type ShortestPathScaleV2Config struct { + // Depth sets the number of edges in each primary path. + Depth int + + // ForwardRootFanOut sets the number of forward dead ends at the primary + // start node. + ForwardRootFanOut int + + // ReverseRootFanIn sets the number of reverse dead ends entering the inbound + // root node. + ReverseRootFanIn int + + // IntermediateFanOut sets the number of forward dead ends at FanInLevel. + IntermediateFanOut int + + // IntermediateReverseFanIn sets the number of reverse dead ends entering + // the inbound path at FanInLevel. + IntermediateReverseFanIn int + + // FanInLevel selects the intermediate level used for fanout and reverse + // fan-in decoys. + FanInLevel int + + // ParallelKindCount sets the number of distinct relationship kinds between + // each parallel start and target pair. + ParallelKindCount int + + // ParallelTargetCount sets the number of targets in the parallel-edge + // subgraph. + ParallelTargetCount int + + // DiamondWidth sets the number of equal-length branches in the diamond + // subgraph. + DiamondWidth int + + // DisconnectedWidth sets the number of intermediate nodes in the + // disconnected path. + DisconnectedWidth int + + // PropertyPayloadSize sets the length of synthetic payload properties. + PropertyPayloadSize int + + // AddCycle includes a reachable two-node cycle. + AddCycle bool + + // AddSelfLoop includes a reachable self-loop. + AddSelfLoop bool +} + +// ValidateShortestPathScaleV2Config rejects negative, inconsistent, or +// unsupported fixture dimensions. +func ValidateShortestPathScaleV2Config(config ShortestPathScaleV2Config) error { + values := []int{ + config.Depth, config.ForwardRootFanOut, config.ReverseRootFanIn, + config.IntermediateFanOut, config.IntermediateReverseFanIn, + config.FanInLevel, config.ParallelKindCount, config.ParallelTargetCount, + config.DiamondWidth, config.DisconnectedWidth, config.PropertyPayloadSize, + } + for _, value := range values { + if value < 0 { + return errors.New("shortest-path v2 configuration values must not be negative") + } + } + if config.Depth > 64 { + return errors.New("shortest-path v2 depth must not exceed 64") + } + if config.IntermediateFanOut == 0 && config.IntermediateReverseFanIn == 0 { + if config.FanInLevel != 0 { + return errors.New("shortest-path v2 fan-in level must be zero without intermediate fanout or fan-in") + } + } else if config.FanInLevel < 1 || config.FanInLevel >= config.Depth { + return errors.New("shortest-path v2 fan-in level must identify an intermediate path level") + } + if (config.ParallelKindCount == 0) != (config.ParallelTargetCount == 0) { + return errors.New("shortest-path v2 parallel kind and target counts must both be zero or both be positive") + } + return nil +} + +// NewShortestPathScaleV2Fixture builds independent deterministic anchors for +// a primary path, hidden fan-in/fan-out, parallel kinds, diamonds, cycles, +// self-loops, and disconnected exhaustion. Every relationship has a stable +// logical_key so backend physical IDs are never required for path comparison. +func NewShortestPathScaleV2Fixture(config ShortestPathScaleV2Config) *opengraph.Graph { + if err := ValidateShortestPathScaleV2Config(config); err != nil { + panic(err) + } + + payload := strings.Repeat("x", config.PropertyPayloadSize) + fixture := &opengraph.Graph{} + addNode := func(id string, properties map[string]any) { + if properties == nil { + properties = map[string]any{} + } + if payload != "" { + properties["payload"] = payload + } + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: id, + Kinds: []string{"ShortestNode"}, + Properties: properties, + }) + } + addEdge := func(start, end, kind, key string) { + properties := map[string]any{"logical_key": key} + if payload != "" { + properties["payload"] = payload + } + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: start, + EndID: end, + Kind: kind, + Properties: properties, + }) + } + + addNode("sp-v2-start", map[string]any{"role": "start", "level": 0}) + addNode("sp-v2-end", map[string]any{"role": "end", "level": config.Depth}) + pathNodes := []string{"sp-v2-start"} + for level := 1; level < config.Depth; level++ { + id := fmt.Sprintf("sp-v2-linear-%02d", level) + addNode(id, map[string]any{"role": "path", "level": level}) + pathNodes = append(pathNodes, id) + } + if config.Depth > 0 { + pathNodes = append(pathNodes, "sp-v2-end") + for level := 1; level < len(pathNodes); level++ { + addEdge(pathNodes[level-1], pathNodes[level], "Traverse", fmt.Sprintf("primary-%02d", level)) + } + } + inboundPathNodes := []string{"sp-v2-inbound-end"} + addNode("sp-v2-inbound-end", map[string]any{"role": "inbound_terminal", "level": config.Depth}) + addNode("sp-v2-inbound-root", map[string]any{"role": "inbound_root", "level": 0}) + for level := config.Depth - 1; level >= 1; level-- { + id := fmt.Sprintf("sp-v2-inbound-linear-%02d", level) + addNode(id, map[string]any{"role": "inbound_path", "level": level}) + inboundPathNodes = append(inboundPathNodes, id) + } + if config.Depth > 0 { + inboundPathNodes = append(inboundPathNodes, "sp-v2-inbound-root") + for level := 1; level < len(inboundPathNodes); level++ { + addEdge(inboundPathNodes[level-1], inboundPathNodes[level], "Traverse", fmt.Sprintf("inbound-primary-%02d", level)) + } + } + + for idx := range config.ForwardRootFanOut { + id := fmt.Sprintf("sp-v2-root-out-%06d", idx) + addNode(id, map[string]any{"role": "root_forward_dead_end"}) + addEdge("sp-v2-start", id, "Traverse", fmt.Sprintf("root-out-%06d", idx)) + } + for idx := range config.ReverseRootFanIn { + id := fmt.Sprintf("sp-v2-root-in-%06d", idx) + addNode(id, map[string]any{"role": "root_reverse_dead_end"}) + addEdge(id, "sp-v2-inbound-root", "Traverse", fmt.Sprintf("root-in-%06d", idx)) + } + if config.FanInLevel > 0 { + boundary := pathNodes[config.FanInLevel] + for idx := range config.IntermediateFanOut { + id := fmt.Sprintf("sp-v2-level-%02d-out-%06d", config.FanInLevel, idx) + addNode(id, map[string]any{"role": "intermediate_forward_dead_end", "level": config.FanInLevel + 1}) + addEdge(boundary, id, "Traverse", fmt.Sprintf("level-%02d-out-%06d", config.FanInLevel, idx)) + } + for idx := range config.IntermediateReverseFanIn { + id := fmt.Sprintf("sp-v2-level-%02d-in-%06d", config.FanInLevel, idx) + addNode(id, map[string]any{"role": "intermediate_reverse_dead_end", "level": config.FanInLevel - 1}) + inboundBoundary := fmt.Sprintf("sp-v2-inbound-linear-%02d", config.FanInLevel) + addEdge(id, inboundBoundary, "Traverse", fmt.Sprintf("level-%02d-in-%06d", config.FanInLevel, idx)) + } + } + + if config.ParallelKindCount > 0 { + addNode("sp-v2-parallel-start", map[string]any{"role": "parallel_start"}) + for target := range config.ParallelTargetCount { + targetID := fmt.Sprintf("sp-v2-parallel-target-%06d", target) + addNode(targetID, map[string]any{"role": "parallel_target"}) + for kind := range config.ParallelKindCount { + addEdge("sp-v2-parallel-start", targetID, fmt.Sprintf("ParallelKind%02d", kind), fmt.Sprintf("parallel-k%02d-t%06d", kind, target)) + } + } + } + + if config.DiamondWidth > 0 { + addNode("sp-v2-diamond-start", map[string]any{"role": "diamond_start"}) + addNode("sp-v2-diamond-end", map[string]any{"role": "diamond_end"}) + for idx := range config.DiamondWidth { + middle := fmt.Sprintf("sp-v2-diamond-%06d", idx) + addNode(middle, map[string]any{"role": "diamond_middle"}) + addEdge("sp-v2-diamond-start", middle, "DiamondTraverse", fmt.Sprintf("diamond-%06d-a", idx)) + addEdge(middle, "sp-v2-diamond-end", "DiamondTraverse", fmt.Sprintf("diamond-%06d-b", idx)) + } + } + + addNode("sp-v2-disconnected-start", map[string]any{"role": "disconnected_start"}) + addNode("sp-v2-disconnected-end", map[string]any{"role": "disconnected_end"}) + previous := "sp-v2-disconnected-start" + for idx := range config.DisconnectedWidth { + next := fmt.Sprintf("sp-v2-disconnected-%06d", idx) + addNode(next, map[string]any{"role": "disconnected_state"}) + addEdge(previous, next, "Traverse", fmt.Sprintf("disconnected-%06d", idx)) + previous = next + } + if config.AddCycle { + addNode("sp-v2-cycle-a", map[string]any{"role": "cycle"}) + addNode("sp-v2-cycle-b", map[string]any{"role": "cycle"}) + addEdge("sp-v2-start", "sp-v2-cycle-a", "Traverse", "cycle-entry") + addEdge("sp-v2-cycle-a", "sp-v2-cycle-b", "Traverse", "cycle-a-b") + addEdge("sp-v2-cycle-b", "sp-v2-cycle-a", "Traverse", "cycle-b-a") + } + if config.AddSelfLoop { + addNode("sp-v2-self-loop", map[string]any{"role": "self_loop"}) + addEdge("sp-v2-start", "sp-v2-self-loop", "Traverse", "self-loop-entry") + addEdge("sp-v2-self-loop", "sp-v2-self-loop", "Traverse", "self-loop") + } + + return fixture +} diff --git a/testutil/reconciliation_fixture.go b/testutil/reconciliation_fixture.go new file mode 100644 index 00000000..ca93400d --- /dev/null +++ b/testutil/reconciliation_fixture.go @@ -0,0 +1,1051 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import ( + "fmt" + + "github.com/specterops/dawgs/opengraph" +) + +const ( + // ReconciliationScaleDataset identifies the generated reconciliation + // fixture. + ReconciliationScaleDataset = "generated_reconciliation" + + // TrustPruningScaleDataset identifies the generated trust-pruning fixture. + TrustPruningScaleDataset = "generated_trust_pruning" + + // HopScaleDataset identifies the generated relationship-hop fixture. + HopScaleDataset = "generated_hops" + + // ScanLookupScaleDataset identifies the generated scan-and-lookup fixture. + ScanLookupScaleDataset = "generated_scan_lookups" +) + +// GeneratedNodeListParam resolves optional fixture IDs followed by a +// deterministic prefix/count sequence. It keeps high-cardinality database-ID +// parameters out of handwritten JSON. +type GeneratedNodeListParam struct { + // Prefix is prepended to each generated node identifier. + Prefix string `json:"prefix"` + + // Count is the number of sequential identifiers to generate. + Count int `json:"count"` + + // Include lists literal identifiers to place before generated identifiers. + Include []string `json:"include,omitempty"` +} + +// FixtureNames returns stable, zero-padded fixture IDs. +func FixtureNames(prefix string, count int) []string { + if count < 0 { + count = 0 + } + + width := len(fmt.Sprintf("%d", max(count-1, 0))) + if width < 2 { + width = 2 + } + + values := make([]string, count) + for idx := range count { + values[idx] = fmt.Sprintf("%s-%0*d", prefix, width, idx) + } + return values +} + +// NewDirectWriteScaleFixture returns a deterministic graph for direct batch +// mutation tests. The requested number of target nodes is used exactly so that +// callers can exercise batch-flush boundaries without fixture rounding. +// +// Every target has one deletable relationship and one relationship-upsert +// baseline. Deletion directions alternate, while the first two targets (when +// present) provide self-connected and high-degree cascade shapes. Separate +// root-to-survivor relationships are never incident to a target, including a +// same-kind survivor for exact delete-set assertions. +func NewDirectWriteScaleFixture(targets int) *opengraph.Graph { + if targets < 0 { + targets = 0 + } + + fixture := &opengraph.Graph{ + Nodes: []opengraph.Node{ + { + ID: "write-root", + Kinds: []string{"WriteEndpoint"}, + Properties: map[string]any{"objectid": "write-root", "role": "root"}, + }, + { + ID: "write-survivor", + Kinds: []string{"WriteEndpoint"}, + Properties: map[string]any{"objectid": "write-survivor", "role": "survivor"}, + }, + }, + Edges: []opengraph.Edge{ + { + StartID: "write-root", + EndID: "write-survivor", + Kind: "WriteSurvivor", + Properties: map[string]any{"marker": "survivor"}, + }, + { + StartID: "write-root", + EndID: "write-survivor", + Kind: "WriteDeleteRelationship", + Properties: map[string]any{"deletebatch": false, "marker": "same-kind-survivor"}, + }, + }, + } + + targetIDs := FixtureNames("write-target", targets) + for idx, targetID := range targetIDs { + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: targetID, + Kinds: []string{"WriteDeleteNode", "WriteUpdateNode"}, + Properties: map[string]any{ + "objectid": targetID, + "deletebatch": true, + "lastseen": "2026-01-01T00:00:00Z", + "ordinal": idx, + }, + }) + + startID, endID := "write-root", targetID + if idx%2 == 1 { + startID, endID = targetID, "write-root" + } + fixture.Edges = append(fixture.Edges, + opengraph.Edge{ + StartID: startID, + EndID: endID, + Kind: "WriteDeleteRelationship", + Properties: map[string]any{ + "deletebatch": true, + "marker": targetID, + }, + }, + opengraph.Edge{ + StartID: "write-root", + EndID: targetID, + Kind: "WriteUpdateRelationship", + Properties: map[string]any{ + "lastseen": "2026-01-01T00:00:00Z", + "marker": targetID, + }, + }, + ) + } + + if targets > 0 { + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: targetIDs[0], + EndID: targetIDs[0], + Kind: "WriteIncident", + Properties: map[string]any{"marker": "self"}, + }) + } + if targets > 1 { + for idx, targetID := range targetIDs { + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: targetIDs[1], + EndID: targetID, + Kind: "WriteIncident", + Properties: map[string]any{"marker": fmt.Sprintf("high-%04d", idx)}, + }) + } + } + + return fixture +} + +// NewReconciliationScaleFixture returns the deterministic graphbench fixture +// for the ingestion reconciliation forms. fanout controls the degree of the +// REC-08 detach-delete target. +func NewReconciliationScaleFixture(fanout int) *opengraph.Graph { + if fanout < 1 { + fanout = 128 + } + + fixture := &opengraph.Graph{ + Nodes: []opengraph.Node{ + { + ID: "source", + Kinds: []string{"Source"}, + Properties: map[string]any{"objectid": "source"}, + }, + { + ID: "list-source-duplicate", + Kinds: []string{"Source"}, + Properties: map[string]any{"objectid": "list-source-duplicate"}, + }, + { + ID: "sink", + Kinds: []string{"Destination"}, + Properties: map[string]any{"objectid": "sink"}, + }, + { + ID: "inbound-target", + Kinds: []string{"ADEntity", "Group"}, + Properties: map[string]any{"objectid": "rec-in"}, + }, + { + ID: "outbound-target", + Kinds: []string{"ADEntity", "Computer"}, + Properties: map[string]any{"objectid": "rec-out"}, + }, + { + ID: "list-target", + Kinds: []string{"ADEntity", "User"}, + Properties: map[string]any{"objectid": "rec-list"}, + }, + { + ID: "template", + Kinds: []string{"CertTemplate"}, + Properties: map[string]any{"objectid": "template"}, + }, + { + ID: "agent", + Kinds: []string{"ADEntity", "User"}, + Properties: map[string]any{"objectid": "agent"}, + }, + { + ID: "agent-duplicate", + Kinds: []string{"ADEntity", "User"}, + Properties: map[string]any{"objectid": "agent-duplicate"}, + }, + { + ID: "delete-target", + Kinds: []string{"ADEntity", "Group"}, + Properties: map[string]any{"objectid": "delete-target"}, + }, + { + ID: "survivor", + Kinds: []string{"ADEntity", "User"}, + Properties: map[string]any{"objectid": "survivor"}, + }, + }, + Edges: []opengraph.Edge{ + { + StartID: "source", + EndID: "inbound-target", + Kind: "RecKind01", + Properties: map[string]any{"marker": "rec-01-a"}, + }, + { + StartID: "source", + EndID: "inbound-target", + Kind: "RecKind30", + Properties: map[string]any{"marker": "rec-01-b"}, + }, + { + StartID: "outbound-target", + EndID: "sink", + Kind: "RecKind01", + Properties: map[string]any{"marker": "rec-02-a"}, + }, + { + StartID: "outbound-target", + EndID: "sink", + Kind: "RecKind30", + Properties: map[string]any{"marker": "rec-02-b"}, + }, + { + StartID: "source", + EndID: "list-target", + Kind: "ADReconcile", + Properties: map[string]any{"marker": "rec-04-a"}, + }, + { + StartID: "list-source-duplicate", + EndID: "list-target", + Kind: "ADReconcile", + Properties: map[string]any{"marker": "rec-04-b"}, + }, + { + StartID: "agent", + EndID: "template", + Kind: "DelegatedEnrollmentAgent", + Properties: map[string]any{"marker": "rec-06-a"}, + }, + { + StartID: "agent-duplicate", + EndID: "template", + Kind: "DelegatedEnrollmentAgent", + Properties: map[string]any{"marker": "rec-06-b"}, + }, + { + StartID: "source", + EndID: "survivor", + Kind: "Survivor", + Properties: map[string]any{"marker": "survivor"}, + }, + }, + } + + for _, templateID := range FixtureNames("scale-template", 2_000) { + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: templateID, + Kinds: []string{"CertTemplate"}, + Properties: map[string]any{"objectid": templateID}, + }) + } + + // Ensure every relationship kind in the 30-kind disjunction is registered, + // while anchoring each decoy away from the REC-01/REC-02 target endpoints. + for idx := 2; idx < 30; idx++ { + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: "source", + EndID: "survivor", + Kind: fmt.Sprintf("RecKind%02d", idx), + Properties: map[string]any{"marker": fmt.Sprintf("kind-decoy-%02d", idx)}, + }) + } + + for idx := range fanout { + neighborID := fmt.Sprintf("detach-neighbor-%04d", idx) + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: neighborID, + Kinds: []string{"ADEntity"}, + Properties: map[string]any{"objectid": neighborID}, + }) + + startID, endID := "delete-target", neighborID + if idx%2 == 0 { + startID, endID = neighborID, "delete-target" + } + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: startID, + EndID: endID, + Kind: "Incident", + Properties: map[string]any{"marker": fmt.Sprintf("incident-%04d", idx)}, + }) + } + + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: "delete-target", + EndID: "delete-target", + Kind: "Incident", + Properties: map[string]any{"marker": "incident-self"}, + }) + return fixture +} + +// NewTrustPruningScaleFixture returns deterministic dense trust and pruning +// shapes without changing the cardinalities of the reconciliation fixture. +func NewTrustPruningScaleFixture(fanout int) *opengraph.Graph { + if fanout < 1 { + fanout = 128 + } + + fixture := &opengraph.Graph{ + Nodes: []opengraph.Node{ + { + ID: "trust-early", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": "2026-01-02T00:00:00Z"}, + }, + { + ID: "trust-late-a", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": "2026-01-04T00:00:00Z"}, + }, + { + ID: "trust-late-b", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": "2026-01-04T00:00:00Z"}, + }, + { + ID: "trust-equal-a", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": "2026-01-03T00:00:00Z"}, + }, + { + ID: "trust-equal-b", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": "2026-01-03T00:00:00Z"}, + }, + { + ID: "prune-a", + Kinds: []string{"PruneEndpoint"}, + Properties: map[string]any{"name": "a"}, + }, + { + ID: "prune-b", + Kinds: []string{"PruneEndpoint"}, + Properties: map[string]any{"name": "b"}, + }, + { + ID: "prune-missing", + Kinds: []string{"PruneCandidate"}, + Properties: map[string]any{"name": "missing"}, + }, + { + ID: "prune-null", + Kinds: []string{"PruneCandidate"}, + Properties: map[string]any{"name": "null", "lastseen": nil}, + }, + { + ID: "prune-protected", + Kinds: []string{"PruneCandidate", "Domain"}, + Properties: map[string]any{"name": "protected", "lastseen": "2026-01-02T00:00:00Z"}, + }, + { + ID: "orphan-missing", + Kinds: []string{"PruneCandidate"}, + Properties: map[string]any{"objectid": "S-1-5-100"}, + }, + { + ID: "orphan-null", + Kinds: []string{"PruneCandidate"}, + Properties: map[string]any{"name": nil, "objectid": "S-1-5-101"}, + }, + { + ID: "orphan-named", + Kinds: []string{"PruneCandidate"}, + Properties: map[string]any{"name": "named", "objectid": "S-1-5-102"}, + }, + { + ID: "orphan-wrong-prefix", + Kinds: []string{"PruneCandidate"}, + Properties: map[string]any{"objectid": "X-1-5-103"}, + }, + { + ID: "prune-batch-high", + Kinds: []string{"PruneBatchNode"}, + Properties: map[string]any{"remove": true}, + }, + { + ID: "prune-batch-survivor", + Kinds: []string{"PruneBatchNode"}, + Properties: map[string]any{"remove": false}, + }, + }, + Edges: []opengraph.Edge{ + { + StartID: "trust-equal-a", + EndID: "trust-equal-b", + Kind: "SameForestTrust", + Properties: map[string]any{"lastseen": "2026-01-03T00:00:00Z", "marker": "same-equal"}, + }, + { + StartID: "trust-equal-a", + EndID: "trust-equal-b", + Kind: "CrossForestTrust", + Properties: map[string]any{"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-equal"}, + }, + { + StartID: "trust-late-a", + EndID: "trust-late-b", + Kind: "SameForestTrust", + Properties: map[string]any{"lastseen": "2026-01-05T00:00:00Z", "marker": "same-new"}, + }, + { + StartID: "trust-late-a", + EndID: "trust-late-b", + Kind: "CrossForestTrust", + Properties: map[string]any{"lastseen": "2026-01-05T00:00:00Z", "marker": "cross-new"}, + }, + { + StartID: "trust-late-a", + EndID: "trust-late-b", + Kind: "AbuseTGTDelegation", + Properties: map[string]any{"marker": "valid-forward-abuse"}, + }, + { + StartID: "trust-late-b", + EndID: "trust-late-a", + Kind: "SpoofSIDHistory", + Properties: map[string]any{"marker": "valid-reverse-spoof"}, + }, + { + StartID: "trust-late-a", + EndID: "trust-late-b", + Kind: "SpoofSIDHistory", + Properties: map[string]any{"marker": "invalid-forward-spoof"}, + }, + { + StartID: "trust-late-b", + EndID: "trust-late-a", + Kind: "AbuseTGTDelegation", + Properties: map[string]any{"marker": "invalid-reverse-abuse"}, + }, + { + StartID: "prune-a", + EndID: "prune-b", + Kind: "PruneBatchSurvivor", + Properties: map[string]any{"remove": false}, + }, + { + StartID: "prune-a", + EndID: "prune-b", + Kind: "MetaIncludes", + Properties: map[string]any{"lastseen": "2026-01-02T00:00:00Z", "marker": "protected-meta-includes"}, + }, + }, + } + + for idx := range fanout { + var ( + suffix = fmt.Sprintf("%04d", idx) + trustEarlyID = "trust-early-" + suffix + oldNodeID = "prune-old-" + suffix + newNodeID = "prune-new-" + suffix + orphanNodeID = "orphan-scale-" + suffix + batchNodeID = "prune-batch-" + suffix + neighborID = "prune-neighbor-" + suffix + candidateOldTargetID = "candidate-old-target-" + suffix + candidateNewTargetID = "candidate-new-target-" + suffix + sessionMissingTargetID = "session-missing-target-" + suffix + sessionOldTargetID = "session-old-target-" + suffix + sessionEqualTargetID = "session-equal-target-" + suffix + batchEdgeTargetID = "prune-batch-edge-target-" + suffix + ) + + fixture.Nodes = append(fixture.Nodes, + opengraph.Node{ + ID: trustEarlyID, + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": "2026-01-02T00:00:00Z"}, + }, + opengraph.Node{ + ID: oldNodeID, + Kinds: []string{"PruneCandidate"}, + Properties: map[string]any{"name": oldNodeID, "lastseen": "2026-01-02T00:00:00Z"}, + }, + opengraph.Node{ + ID: newNodeID, + Kinds: []string{"PruneCandidate"}, + Properties: map[string]any{"name": newNodeID, "lastseen": "2026-01-04T00:00:00Z"}, + }, + opengraph.Node{ + ID: orphanNodeID, + Kinds: []string{"PruneCandidate"}, + Properties: map[string]any{"objectid": "S-1-5-" + suffix}, + }, + opengraph.Node{ + ID: batchNodeID, + Kinds: []string{"PruneBatchNode"}, + Properties: map[string]any{"remove": idx%2 == 0}, + }, + opengraph.Node{ + ID: neighborID, + Kinds: []string{"PruneNeighbor"}, + Properties: map[string]any{"name": neighborID}, + }, + opengraph.Node{ + ID: candidateOldTargetID, + Kinds: []string{"PruneEndpoint"}, + Properties: map[string]any{"name": candidateOldTargetID}, + }, + opengraph.Node{ + ID: candidateNewTargetID, + Kinds: []string{"PruneEndpoint"}, + Properties: map[string]any{"name": candidateNewTargetID}, + }, + opengraph.Node{ + ID: sessionMissingTargetID, + Kinds: []string{"PruneEndpoint"}, + Properties: map[string]any{"name": sessionMissingTargetID}, + }, + opengraph.Node{ + ID: sessionOldTargetID, + Kinds: []string{"PruneEndpoint"}, + Properties: map[string]any{"name": sessionOldTargetID}, + }, + opengraph.Node{ + ID: sessionEqualTargetID, + Kinds: []string{"PruneEndpoint"}, + Properties: map[string]any{"name": sessionEqualTargetID}, + }, + opengraph.Node{ + ID: batchEdgeTargetID, + Kinds: []string{"PruneEndpoint"}, + Properties: map[string]any{"name": batchEdgeTargetID}, + }, + ) + + fixture.Edges = append(fixture.Edges, + opengraph.Edge{ + StartID: "trust-late-a", + EndID: trustEarlyID, + Kind: "SameForestTrust", + Properties: map[string]any{"lastseen": "2026-01-03T00:00:00Z", "marker": "same-old-" + suffix}, + }, + opengraph.Edge{ + StartID: "trust-late-a", + EndID: trustEarlyID, + Kind: "CrossForestTrust", + Properties: map[string]any{"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-old-" + suffix}, + }, + opengraph.Edge{ + StartID: "prune-a", + EndID: candidateOldTargetID, + Kind: "CandidateRel", + Properties: map[string]any{"lastseen": "2026-01-02T00:00:00Z", "marker": "candidate-old-" + suffix}, + }, + opengraph.Edge{ + StartID: "prune-a", + EndID: candidateNewTargetID, + Kind: "CandidateRel", + Properties: map[string]any{"lastseen": "2026-01-04T00:00:00Z", "marker": "candidate-new-" + suffix}, + }, + opengraph.Edge{ + StartID: "prune-a", + EndID: sessionMissingTargetID, + Kind: "HasSession", + Properties: map[string]any{"marker": "session-missing-" + suffix}, + }, + opengraph.Edge{ + StartID: "prune-a", + EndID: sessionOldTargetID, + Kind: "HasSession", + Properties: map[string]any{"lastseen": "2026-01-02T00:00:00Z", "marker": "session-old-" + suffix}, + }, + opengraph.Edge{ + StartID: "prune-a", + EndID: sessionEqualTargetID, + Kind: "HasSession", + Properties: map[string]any{"lastseen": "2026-01-03T00:00:00Z", "marker": "session-equal-" + suffix}, + }, + opengraph.Edge{ + StartID: "prune-a", + EndID: batchEdgeTargetID, + Kind: "PruneBatch", + Properties: map[string]any{"remove": true, "marker": "batch-" + suffix}, + }, + opengraph.Edge{ + StartID: "prune-batch-high", + EndID: neighborID, + Kind: "PruneIncident", + Properties: map[string]any{"marker": "incident-" + suffix}, + }, + ) + } + + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: "prune-batch-high", + EndID: "prune-batch-high", + Kind: "PruneIncident", + Properties: map[string]any{"marker": "incident-self"}, + }) + return fixture +} + +// NewHopScaleFixture returns deterministic one-hop fanout, kind-cardinality, +// endpoint-list, predicate-selectivity, and two-sided set shapes. +func NewHopScaleFixture(fanout int) *opengraph.Graph { + if fanout < 30 { + fanout = 128 + } + + fixture := &opengraph.Graph{ + Nodes: []opengraph.Node{ + { + ID: "hop-out-root", + Kinds: []string{"HopAnchor"}, + Properties: map[string]any{"name": "hop-out-root"}, + }, + { + ID: "hop-in-root", + Kinds: []string{"HopAnchor"}, + Properties: map[string]any{"name": "hop-in-root"}, + }, + { + ID: "hop-kind-root", + Kinds: []string{"HopAnchor"}, + Properties: map[string]any{"name": "hop-kind-root"}, + }, + { + ID: "hop-decoy-root", + Kinds: []string{"HopAnchor"}, + Properties: map[string]any{"name": "hop-decoy-root"}, + }, + }, + } + + for idx := range fanout { + var ( + suffix = fmt.Sprintf("%04d", idx) + peerID = "hop-peer-" + suffix + sourceID = "hop-source-" + suffix + properties = map[string]any{ + "name": peerID, + "requiresmanagerapproval": false, + "authenticationenabled": true, + } + ) + + switch idx % 4 { + case 0: + properties["schemaversion"] = 2 + properties["authorizedsignatures"] = 0 + case 1: + properties["schemaversion"] = 1 + properties["authorizedsignatures"] = 9 + case 2: + properties["schemaversion"] = 2 + properties["authorizedsignatures"] = 1 + case 3: + properties["schemaversion"] = 2 + properties["authorizedsignatures"] = 0 + properties["authenticationenabled"] = false + } + + peerKinds := []string{"HopEndpoint", "HopEndA", "HopTemplate"} + if idx%2 == 0 { + peerKinds = append(peerKinds, "HopEndB") + } + fixture.Nodes = append(fixture.Nodes, + opengraph.Node{ + ID: peerID, + Kinds: peerKinds, + Properties: properties, + }, + opengraph.Node{ + ID: sourceID, + Kinds: []string{"HopSource"}, + Properties: map[string]any{"name": sourceID}, + }, + ) + fixture.Edges = append(fixture.Edges, + opengraph.Edge{ + StartID: "hop-out-root", + EndID: peerID, + Kind: "HopKind01", + Properties: map[string]any{"marker": "out-" + suffix}, + }, + opengraph.Edge{ + StartID: sourceID, + EndID: "hop-in-root", + Kind: "HopKind01", + Properties: map[string]any{"marker": "in-" + suffix}, + }, + opengraph.Edge{ + StartID: "hop-kind-root", + EndID: peerID, + Kind: fmt.Sprintf("HopKind%02d", idx%30+1), + Properties: map[string]any{"marker": "kind-" + suffix}, + }, + opengraph.Edge{ + StartID: "hop-out-root", + EndID: peerID, + Kind: "HopTypedEdge", + Properties: map[string]any{"marker": "typed-" + suffix}, + }, + opengraph.Edge{ + StartID: "hop-out-root", + EndID: peerID, + Kind: "HopNestedEdge", + Properties: map[string]any{"marker": "nested-" + suffix}, + }, + ) + } + + for idx, targetID := range FixtureNames("hop-id-target", 1_000) { + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: targetID, + Kinds: []string{"HopIDEndpoint"}, + Properties: map[string]any{"name": targetID}, + }) + if idx < fanout { + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: "hop-out-root", + EndID: targetID, + Kind: "HopIDEdge", + Properties: map[string]any{"marker": fmt.Sprintf("id-%04d", idx)}, + }) + } + } + + setStarts := FixtureNames("hop-set-start", 32) + setEnds := FixtureNames("hop-set-end", 32) + for _, startID := range setStarts { + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: startID, + Kinds: []string{"HopSetStart"}, + Properties: map[string]any{"name": startID}, + }) + } + for _, endID := range setEnds { + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: endID, + Kinds: []string{"HopSetEnd"}, + Properties: map[string]any{"name": endID}, + }) + } + for _, startID := range setStarts { + for _, endID := range setEnds { + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: startID, + EndID: endID, + Kind: "HopSetEdge", + Properties: map[string]any{"marker": startID + "-" + endID}, + }) + } + } + fixture.Edges = append(fixture.Edges, + opengraph.Edge{ + StartID: "hop-decoy-root", + EndID: "hop-peer-0000", + Kind: "HopTypedEdge", + Properties: map[string]any{"marker": "wrong-root"}, + }, + opengraph.Edge{ + StartID: "hop-peer-0000", + EndID: "hop-out-root", + Kind: "HopTypedEdge", + Properties: map[string]any{"marker": "wrong-direction"}, + }, + opengraph.Edge{ + StartID: setStarts[0], + EndID: setEnds[0], + Kind: "HopWrongSetEdge", + Properties: map[string]any{"marker": "wrong-set-kind"}, + }, + opengraph.Edge{ + StartID: setEnds[0], + EndID: setStarts[0], + Kind: "HopSetEdge", + Properties: map[string]any{"marker": "wrong-set-direction"}, + }, + ) + return fixture +} + +// NewScanLookupScaleFixture returns deterministic wide-scan, large lookup, +// adjacency, ordering, and count shapes for the scan/lookup regression corpus. +func NewScanLookupScaleFixture(fanout int) *opengraph.Graph { + if fanout < 9 { + fanout = 128 + } + + fixture := &opengraph.Graph{ + Nodes: []opengraph.Node{ + { + ID: "scan-base-root", + Kinds: []string{"ADBase"}, + Properties: map[string]any{"name": "scan-base-root"}, + }, + { + ID: "scan-tracker-root", + Kinds: []string{"Plain"}, + Properties: map[string]any{"name": "scan-tracker-root"}, + }, + { + ID: "scan-nine-kind-target", + Kinds: []string{"Computer"}, + Properties: map[string]any{"name": "scan-nine-kind-target"}, + }, + { + ID: "scan-local-target", + Kinds: []string{"Computer"}, + Properties: map[string]any{"name": "scan-local-target"}, + }, + { + ID: "lookup-tenant", + Kinds: []string{"Tenant"}, + Properties: map[string]any{"name": "lookup-tenant", "objectid": "tenant-scale"}, + }, + // The extra isolated labels make negative Meta/MetaDetail predicates + // translatable without changing any fixture cardinality. + { + ID: "lookup-local-target", + Kinds: []string{"Computer", "Meta", "MetaDetail"}, + Properties: map[string]any{"name": "lookup-local-target"}, + }, + }, + } + + escalationKinds := []string{"GenericAll", "GenericWrite", "Owns", "WriteOwner", "WriteDACL", "WritePublicInformation"} + victimIDs := FixtureNames("scan-victim", max(1_000, fanout)) + for idx := range fanout { + suffix := fmt.Sprintf("%04d", idx) + scanEndID := "scan-end-" + suffix + scanEntityID := "scan-entity-" + suffix + lookupObjectID := "lookup-object-" + suffix + lookupStringID := "lookup-string-" + suffix + lookupLocalID := "lookup-local-" + suffix + ntlmID := "lookup-ntlm-" + suffix + + entityKinds := []string{"Entity"} + switch idx % 3 { + case 0: + entityKinds = append(entityKinds, "Group") + case 1: + entityKinds = append(entityKinds, "User") + case 2: + entityKinds = append(entityKinds, "Computer") + } + + lookupObjectSuffix := "-513" + if idx%2 == 0 { + lookupObjectSuffix = "-512" + } + lookupName := fmt.Sprintf("Remote Desktop Users %04d", idx) + if idx%2 == 1 { + lookupName = fmt.Sprintf("rEmOtE dEsKtOp UsErS %04d", idx) + } + + fixture.Nodes = append(fixture.Nodes, + opengraph.Node{ + ID: scanEndID, + Kinds: []string{"AZBase", "Plain"}, + Properties: map[string]any{"name": scanEndID}, + }, + opengraph.Node{ + ID: scanEntityID, + Kinds: entityKinds, + Properties: map[string]any{"name": scanEntityID}, + }, + opengraph.Node{ + ID: lookupObjectID, + Kinds: []string{"Computer"}, + Properties: map[string]any{"name": lookupObjectID, "objectid": "S-1-5-21-scale", "enabled": true}, + }, + opengraph.Node{ + ID: lookupStringID, + Kinds: []string{"Group", "Entity"}, + Properties: map[string]any{"name": lookupName, "objectid": "S-1-5-21" + lookupObjectSuffix, "domainsid": "S-1-5-21"}, + }, + opengraph.Node{ + ID: lookupLocalID, + Kinds: []string{"LocalGroup", "Entity"}, + Properties: map[string]any{"name": lookupLocalID, "objectid": "S-1-5-21-555"}, + }, + opengraph.Node{ + ID: ntlmID, + Kinds: []string{"Computer"}, + Properties: map[string]any{"name": ntlmID, "domainsid": "S-1-5-21", "isdc": true, "ldapavailable": true, "ldapsigning": false}, + }, + ) + + migrationProperties := map[string]any{"marker": "migration-" + suffix} + if idx%2 == 0 { + migrationProperties["lastseen"] = "2026-01-03T00:00:00Z" + } else if idx%4 == 1 { + migrationProperties["lastseen"] = nil + } + + victimID := victimIDs[idx] + fixture.Edges = append(fixture.Edges, + opengraph.Edge{ + StartID: "scan-base-root", + EndID: scanEndID, + Kind: "ScanPostProcessed", + Properties: map[string]any{"marker": "post-" + suffix}, + }, + opengraph.Edge{ + StartID: "scan-tracker-root", + EndID: scanEndID, + Kind: "TrackerA", + Properties: map[string]any{"marker": "tracker-a-" + suffix}, + }, + opengraph.Edge{ + StartID: "scan-tracker-root", + EndID: scanEndID, + Kind: "TrackerB", + Properties: map[string]any{"marker": "tracker-b-" + suffix}, + }, + opengraph.Edge{ + StartID: "scan-tracker-root", + EndID: scanEndID, + Kind: "MigratedEdge", + Properties: migrationProperties, + }, + opengraph.Edge{ + StartID: scanEntityID, + EndID: scanEndID, + Kind: "OwnsRaw", + Properties: map[string]any{"marker": "owns-" + suffix}, + }, + opengraph.Edge{ + StartID: scanEntityID, + EndID: "scan-nine-kind-target", + Kind: fmt.Sprintf("ScanEdge%02d", idx%9+1), + Properties: map[string]any{"marker": "scan-" + suffix}, + }, + opengraph.Edge{ + StartID: scanEntityID, + EndID: "scan-local-target", + Kind: "LocalToComputer", + Properties: map[string]any{"marker": "scan-local-" + suffix}, + }, + opengraph.Edge{ + StartID: scanEntityID, + EndID: scanEndID, + Kind: "MemberOf", + Properties: map[string]any{"marker": "member-" + suffix}, + }, + opengraph.Edge{ + StartID: scanEntityID, + EndID: scanEndID, + Kind: "MemberOfLocalGroup", + Properties: map[string]any{"marker": "member-local-" + suffix}, + }, + opengraph.Edge{ + StartID: scanEntityID, + EndID: victimID, + Kind: escalationKinds[idx%len(escalationKinds)], + Properties: map[string]any{"marker": "esc-" + suffix}, + }, + opengraph.Edge{ + StartID: lookupLocalID, + EndID: "lookup-local-target", + Kind: "LocalToComputer", + Properties: map[string]any{"marker": "lookup-local-" + suffix}, + }, + ) + } + + for idx, victimID := range victimIDs { + victimKinds := []string{"Other"} + if idx%2 == 0 { + victimKinds = []string{"Computer"} + } + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: victimID, + Kinds: victimKinds, + Properties: map[string]any{"name": victimID}, + }) + } + + for _, targetID := range FixtureNames("lookup-id-target", 1_000) { + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: targetID, + Kinds: []string{"Hydrate"}, + Properties: map[string]any{"name": targetID}, + }) + } + + for idx, roleID := range FixtureNames("lookup-role", 1_000) { + roleTemplateID := fmt.Sprintf("role-template-%03d", idx) + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: roleID, + Kinds: []string{"AZRole"}, + Properties: map[string]any{"name": roleID, "roletemplateid": roleTemplateID, "enabled": true}, + }) + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: "lookup-tenant", + EndID: roleID, + Kind: "Contains", + Properties: map[string]any{"marker": roleID}, + }) + } + + return fixture +} diff --git a/testutil/reconciliation_fixture_test.go b/testutil/reconciliation_fixture_test.go new file mode 100644 index 00000000..62a10e59 --- /dev/null +++ b/testutil/reconciliation_fixture_test.go @@ -0,0 +1,153 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import ( + "fmt" + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/opengraph" + "github.com/stretchr/testify/require" +) + +// requireUniqueScaleEdgeKeys verifies a fixture does not contain duplicate +// start, end, and kind tuples rejected by PostgreSQL storage. +func requireUniqueScaleEdgeKeys(t *testing.T, fixture *opengraph.Graph) { + t.Helper() + + keys := map[string]struct{}{} + for _, edge := range fixture.Edges { + key := edge.StartID + "\x00" + edge.EndID + "\x00" + edge.Kind + require.NotContains(t, keys, key, "duplicate PostgreSQL edge key %s -> %s [%s]", edge.StartID, edge.EndID, edge.Kind) + keys[key] = struct{}{} + } +} + +// TestNewReconciliationScaleFixture verifies the reconciliation fixture's +// cardinality, edge-key uniqueness, and complete kind range. +func TestNewReconciliationScaleFixture(t *testing.T) { + fixture := NewReconciliationScaleFixture(8) + nodeKinds, edgeKinds := fixture.Kinds() + + require.Len(t, fixture.Nodes, 2_019) + require.Len(t, fixture.Edges, 46) + requireUniqueScaleEdgeKeys(t, fixture) + require.Contains(t, nodeKinds, graph.StringKind("ADEntity")) + for idx := 1; idx <= 30; idx++ { + require.Contains(t, edgeKinds, graph.StringKind(fmt.Sprintf("RecKind%02d", idx))) + } +} + +// TestFixtureNamesAreDeterministic verifies generated fixture identifiers are +// stable, padded, and empty for negative counts. +func TestFixtureNamesAreDeterministic(t *testing.T) { + require.Equal(t, []string{"item-00", "item-01", "item-02"}, FixtureNames("item", 3)) + require.Equal(t, FixtureNames("item", 2_000), FixtureNames("item", 2_000)) + require.Empty(t, FixtureNames("item", -1)) +} + +// TestNewDirectWriteScaleFixtureUsesExactBoundaryAndCascadeShape verifies exact +// target counts and the intended delete, update, and incident edge populations. +func TestNewDirectWriteScaleFixtureUsesExactBoundaryAndCascadeShape(t *testing.T) { + empty := NewDirectWriteScaleFixture(0) + require.Len(t, empty.Nodes, 2) + require.Len(t, empty.Edges, 2) + + fixture := NewDirectWriteScaleFixture(3) + require.Len(t, fixture.Nodes, 5) + require.Len(t, fixture.Edges, 12) + + var ( + deleteEdges int + updateEdges int + incidentEdges int + ) + for _, edge := range fixture.Edges { + switch edge.Kind { + case "WriteDeleteRelationship": + deleteEdges++ + case "WriteUpdateRelationship": + updateEdges++ + case "WriteIncident": + incidentEdges++ + } + } + require.Equal(t, 4, deleteEdges) + require.Equal(t, 3, updateEdges) + require.Equal(t, 4, incidentEdges) + + require.Equal(t, "write-target-00", fixture.Nodes[2].ID) + require.Equal(t, "write-target-02", fixture.Nodes[4].ID) + require.Equal(t, "write-root", fixture.Edges[2].StartID) + require.Equal(t, "write-target-01", fixture.Edges[4].StartID) +} + +// TestNewTrustPruningScaleFixtureIncludesDenseAndDecoyShapes verifies the +// fixture includes all node and relationship categories used by pruning cases. +func TestNewTrustPruningScaleFixtureIncludesDenseAndDecoyShapes(t *testing.T) { + fixture := NewTrustPruningScaleFixture(8) + nodeKinds, edgeKinds := fixture.Kinds() + + require.Len(t, fixture.Nodes, 112) + require.Len(t, fixture.Edges, 83) + requireUniqueScaleEdgeKeys(t, fixture) + require.Contains(t, nodeKinds, graph.StringKind("Domain")) + require.Contains(t, nodeKinds, graph.StringKind("PruneCandidate")) + require.Contains(t, nodeKinds, graph.StringKind("PruneBatchNode")) + require.Contains(t, edgeKinds, graph.StringKind("SameForestTrust")) + require.Contains(t, edgeKinds, graph.StringKind("CrossForestTrust")) + require.Contains(t, edgeKinds, graph.StringKind("HasSession")) + require.Contains(t, edgeKinds, graph.StringKind("PruneBatch")) + require.Contains(t, edgeKinds, graph.StringKind("MetaIncludes")) +} + +// TestNewHopScaleFixtureIncludesDenseAndLargeListShapes verifies dense hop +// topology, broad kind coverage, and large-list endpoints are present. +func TestNewHopScaleFixtureIncludesDenseAndLargeListShapes(t *testing.T) { + fixture := NewHopScaleFixture(32) + nodeKinds, edgeKinds := fixture.Kinds() + + require.Len(t, fixture.Nodes, 1_132) + require.Len(t, fixture.Edges, 1_220) + require.Contains(t, nodeKinds, graph.StringKind("HopTemplate")) + require.Contains(t, nodeKinds, graph.StringKind("HopIDEndpoint")) + for idx := 1; idx <= 30; idx++ { + require.Contains(t, edgeKinds, graph.StringKind(fmt.Sprintf("HopKind%02d", idx))) + } + require.Contains(t, edgeKinds, graph.StringKind("HopSetEdge")) +} + +// TestNewScanLookupScaleFixtureIncludesWideAndLargeListShapes verifies the +// fixture contains all scan, lookup, hydration, and relationship categories. +func TestNewScanLookupScaleFixtureIncludesWideAndLargeListShapes(t *testing.T) { + fixture := NewScanLookupScaleFixture(32) + nodeKinds, edgeKinds := fixture.Kinds() + + require.Len(t, fixture.Nodes, 3_198) + require.Len(t, fixture.Edges, 1_352) + require.Contains(t, nodeKinds, graph.StringKind("ADBase")) + require.Contains(t, nodeKinds, graph.StringKind("AZRole")) + require.Contains(t, nodeKinds, graph.StringKind("Hydrate")) + require.Contains(t, nodeKinds, graph.StringKind("Meta")) + require.Contains(t, nodeKinds, graph.StringKind("MetaDetail")) + require.Contains(t, edgeKinds, graph.StringKind("ScanPostProcessed")) + require.Contains(t, edgeKinds, graph.StringKind("Contains")) + for idx := 1; idx <= 9; idx++ { + require.Contains(t, edgeKinds, graph.StringKind(fmt.Sprintf("ScanEdge%02d", idx))) + } +} diff --git a/tools/dawgrun/README.md b/tools/dawgrun/README.md index c92676ab..03989d18 100644 --- a/tools/dawgrun/README.md +++ b/tools/dawgrun/README.md @@ -35,19 +35,13 @@ connection. ## Building -From a `DAWGS` checkout: +From a `DAWGS` checkout, run the tool directly: go tool dawgrun -With a customized `DAWGS` clone, for testing features, version differences, etc: +To build a local binary instead: - cd tools/dawgrun - just build-with-dawgs path/to/DAWGS - -To switch the build back to mainline: - - cd tools/dawgrun - just build-with-upstream + go build -o tools/dawgrun/dawgrun ./tools/dawgrun/cmd/dawgrun ## Running @@ -288,3 +282,20 @@ the `DAWGRUN_STYLE` environment variable. Any styles in [Chroma](https://github.com/alecthomas/chroma/tree/master/styles) are available for use as a syntax highlighting style. CLI mode disables all terminal styling, including syntax highlighting and styled warnings, when stdout is not a terminal. + +## Common Issues + +### Why does opening a new Postgres database fail with `open failed: could not set default graph: no rows in result set`? + +`open` selects the configured default graph, which is named `default` +unless `-default-graph` is provided. A new Postgres database may have a +DAWGS schema but no graph row yet, so selecting that default graph fails. + +For a database that should be initialized for dawgrun, open it with +`-init-graph`: + + dawgrun > open -init-graph local "postgres://postgres:password@localhost:32771/" + +If you use a non-default graph name, pass it with `-default-graph`: + + dawgrun > open -init-graph -default-graph mygraph local "postgres://postgres:password@localhost:32771/" diff --git a/tools/dawgrun/pkg/commands/cypher.go b/tools/dawgrun/pkg/commands/cypher.go index 2d2e05ca..2d5b1303 100644 --- a/tools/dawgrun/pkg/commands/cypher.go +++ b/tools/dawgrun/pkg/commands/cypher.go @@ -17,10 +17,14 @@ import ( ) const ( + // queryCypherOutputFormatTable selects tabular rendering for fetched rows. queryCypherOutputFormatTable = "table" - queryCypherOutputFormatJSON = "json" + + // queryCypherOutputFormatJSON selects JSON rendering for fetched rows. + queryCypherOutputFormatJSON = "json" ) +// parseCmd describes the command that parses Cypher and prints its AST. func parseCmd() CommandDesc { return CommandDesc{ args: []string{"<...query>"}, @@ -38,6 +42,8 @@ func parseCmd() CommandDesc { } } +// translateToPsqlCmd describes the command that translates Cypher into +// formatted PostgreSQL SQL. func translateToPsqlCmd() CommandDesc { flagSet := flag.NewFlagSet("translate-psql", flag.ContinueOnError) @@ -92,7 +98,7 @@ func translateToPsqlCmd() CommandDesc { // Certain queries will materialize parameters into the output when translated, so we need to build // an OutputBuilder so we can carry forward those params. - queryBuilder := format.NewOutputBuilder() + queryBuilder := format.NewOutputBuilder().WithTargetGraph(result.GraphID) if result.Parameters != nil { queryBuilder.WithMaterializedParameters(result.Parameters) } @@ -116,6 +122,7 @@ func translateToPsqlCmd() CommandDesc { } } +// explainAsPsqlCmd defines the interactive command that translates Cypher and asks PostgreSQL to explain the resulting SQL. func explainAsPsqlCmd() CommandDesc { return CommandDesc{ args: []string{"", "<...query>"}, @@ -153,7 +160,7 @@ func explainAsPsqlCmd() CommandDesc { // Certain queries will materialize parameters into the output when translated, so we need to build // an OutputBuilder so we can carry forward those params. - queryBuilder := format.NewOutputBuilder() + queryBuilder := format.NewOutputBuilder().WithTargetGraph(result.GraphID) if result.Parameters != nil { queryBuilder.WithMaterializedParameters(result.Parameters) } @@ -200,6 +207,8 @@ func explainAsPsqlCmd() CommandDesc { } } +// defaultGraphID returns a connection's configured default graph or the +// translator fallback when no PostgreSQL default is available. func defaultGraphID(ctx *CommandContext, connName string) int32 { if connName == "" { return translate.DefaultGraphID @@ -222,6 +231,8 @@ func defaultGraphID(ctx *CommandContext, connName string) int32 { return translate.DefaultGraphID } +// queryCypherCmd describes the command that executes Cypher and renders fetched +// rows as a table or JSON. func queryCypherCmd() CommandDesc { flagSet := flag.NewFlagSet("query-cypher", flag.ContinueOnError)