From c670529663310f35b8f6cfcbbf9643a4bdcf1f3c Mon Sep 17 00:00:00 2001 From: Charlie Le Date: Fri, 21 Aug 2026 11:49:43 -0700 Subject: [PATCH] fix(integration): skip `or` in cross-version query fuzz comparisons TestExpandedPostingsCacheFuzz compares the latest released Cortex image (cortex-1, resolved from the VERSION file, currently v1.21.1 -> Prometheus 3.8.1) against the current build (cortex-2 -> Prometheus 3.9.1 since #7535). The two embedded Prometheus versions disagree about what to do when a query result contains series that become identical after __name__ removal. Prometheus <= 3.8 failed the whole query in cleanupMetricLabels: if mat.ContainsSameLabelset() { ev.errorf("vector cannot contain metrics with the same labelset") } Prometheus 3.9 replaced that with mergeSeriesWithSameLabelset, which merges the colliding series when their timestamps do not overlap and only errors when they do. Its doc comment names the exact shape: "operations like OR combine series that originally had different names but end up with the same labelset after dropping the name". So the fuzzer generating something like -( label_replace(rate({__name__="test_series_6"}[4m]), "__promqlsmith_dst_label__", "$1", "__name__", "(.*)") or {__name__="test_series_6",test_label="test_label_value_2"} ) gets `execution: vector cannot contain metrics with the same labelset` from cortex-1 and no error at all from cortex-2, which the test reports as an error mismatch. sameErrorClass (#7550) cannot reconcile it because one side has no error. No release tag contains #7535 yet, so master and the newest released image are guaranteed to disagree here until the next release. The divergence follows the image, not the expanded-postings-cache flag: running the failing query against the release image *with* the cache enabled still errors, and against HEAD *without* the cache still succeeds. isValidQuery(expr, skipBackwardIncompat=true) already drops queries whose semantics changed across the embedded Prometheus versions (stddev, stdvar, quantile, predict_linear, atan2). Add `or` to that set, since `or` is the only operator that can union series carrying different __name__s into one result - the precondition for the collision. `and` and `unless` only ever return series from their left hand side and are left alone. Whether a given `or` actually collides can only be known by evaluating it, so the filter is syntactic; it is an AST walk for parser.LOR rather than a strings.Contains, so a label *value* containing "or" cannot accidentally drop a query. skipBackwardIncompat=true is only passed by the cross-version tests, so `or` remains fully covered by the fuzz tests that compare two instances of the same build (TestVerticalShardingFuzz, TestProtobufCodecFuzz, TestParquetFuzz, ...). Verified against the seed from the failing CI run, plus four other seeds: CORTEX_IMAGE= FUZZ_SEED=1787335629 go test -v \ -tags "integration,requires_docker,integration_query_fuzz" \ -timeout 2400s -count=1 ./integration/ \ -run '^TestExpandedPostingsCacheFuzz$' reproduces `case 453 error mismatch` before the change and passes after it. Fixes #7803 Signed-off-by: Charlie Le --- integration/query_fuzz_test.go | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/integration/query_fuzz_test.go b/integration/query_fuzz_test.go index 0d7484c6b0..6557b4236e 100644 --- a/integration/query_fuzz_test.go +++ b/integration/query_fuzz_test.go @@ -2510,10 +2510,43 @@ func isValidQuery(generatedQuery parser.Expr, skipBackwardIncompat bool) bool { if strings.Contains(queryStr, "atan2") { return false } + if containsLogicalOr(generatedQuery) { + // Prometheus 3.9 changed how a result whose series collide after __name__ + // removal is handled: cleanupMetricLabels used to fail the whole query with + // "vector cannot contain metrics with the same labelset", and now merges + // series that have non-overlapping timestamps instead (see + // mergeSeriesWithSameLabelset, vendored by #7535). + // + // `or` is what builds such a result, by unioning series that only differ by + // __name__ and are then name-dropped by an enclosing operation, e.g. + // `-(rate({__name__="a"}[4m]) or {__name__="a"})`. The older Prometheus in + // the latest released Cortex image errors where HEAD returns data, which is a + // legitimate cross-version difference and not a Cortex bug. Whether a given + // `or` actually collides can only be known by evaluating it, so skip `or` + // entirely for cross-version comparisons. `or` stays covered by the fuzz + // tests that compare two instances of the same build. + // + // See https://github.com/cortexproject/cortex/issues/7803. + return false + } } return isValid } +// containsLogicalOr reports whether the expression uses the `or` set operator anywhere. +// `and` and `unless` are excluded on purpose: they only ever return series taken from +// the left hand side, so they cannot union series that differ only by __name__. +func containsLogicalOr(expr parser.Expr) bool { + found := false + parser.Inspect(expr, func(node parser.Node, _ []parser.Node) error { + if n, ok := node.(*parser.BinaryExpr); ok && n.Op == parser.LOR { + found = true + } + return nil + }) + return found +} + func resultLength(x model.Value) int { vx, xvec := x.(model.Vector) if xvec {