diff --git a/src/ops/collection.c b/src/ops/collection.c index b9ac009d..6b20c4b2 100644 --- a/src/ops/collection.c +++ b/src/ops/collection.c @@ -1374,8 +1374,21 @@ ray_t* ray_in_fn(ray_t* val, ray_t* vec) { * the WHERE kernel uses null-matches-nothing semantics — the * two must not be conflated. NULL result = unsupported shape * (STR etc.): fall through to the hashset probe below. */ - if (!ray_vec_may_have_nulls(val) && - !ray_vec_may_have_nulls(vec)) { + /* Admission check, so it uses the EXACT ray_vec_has_nulls rather + * than the conservative row-kernel gate. ray_vec_may_have_nulls + * returns true unconditionally for SYM and STR — their null is a + * payload value, not an attribute bit — so gating this on it made + * the branch unreachable for every text column, including the SYM + * verdict-LUT the kernel carries specifically for them. Every + * `in` over a symbol column fell through to the generic per-row + * hashset probe: ~4.8 ms against ~40 us for the equivalent `==` + * on a 351k-row column (#593). vec.h says as much where it + * defines the two — "paths requiring null-free data use has_nulls + * below". The exact check costs one pass and is not on a per-row + * path; a null-bearing operand still falls through, because the + * kernel's null-matches-nothing semantics differ from this path's + * null-equals-null. */ + if (!ray_vec_has_nulls(val) && !ray_vec_has_nulls(vec)) { ray_t* fast = ray_in_vec_exec(val, vec, false); if (fast) return fast; } diff --git a/test/rfl/collection/in.rfl b/test/rfl/collection/in.rfl index fbead052..026417e4 100644 --- a/test/rfl/collection/in.rfl +++ b/test/rfl/collection/in.rfl @@ -84,3 +84,42 @@ (in [1h 2h 3h] [3h 2h 1h]) -- [true true true] ;; Test with empty arrays (in (list) [1h 2h]) -- (list) + +;; ========== TEXT (SYM/STR) NULL SEMANTICS (#593) ========== +;; `in` admits the typed verdict-LUT kernel only when BOTH sides are exactly +;; null-free, because that kernel uses null-matches-nothing semantics while +;; this path is null-equals-null. The gate used to ask +;; ray_vec_may_have_nulls, which is unconditionally true for SYM and STR +;; (their null is a payload value, not an attribute bit) — so the kernel was +;; unreachable for text columns and every symbol `in` fell through to the +;; generic per-row hashset probe. These pin the semantics on both sides of +;; that gate so the admission check cannot drift back. + +;; null in the COLUMN, null needle — null equals null +(in (as 'SYM ["a" "" "b"]) (as 'SYM [""])) -- [false true false] +(in ["a" "" "b"] [""]) -- [false true false] + +;; null in the column, non-null needle — the null matches nothing else +(in (as 'SYM ["a" "" "b"]) (as 'SYM ["a"])) -- [true false false] + +;; null only in the NEEDLES: the column is null-free but the gate must still +;; reject, since a null needle needs null-equals-null against nothing +(in (as 'SYM ["a" "b"]) (as 'SYM ["" "a"])) -- [true false] + +;; nulls on both sides +(in (as 'SYM ["a" ""]) (as 'SYM ["" "z"])) -- [false true] + +;; null-free text: this is the shape that now reaches the kernel +(in (as 'SYM ["a" "b" "c"]) (as 'SYM ["b" "c"])) -- [false true true] +(in ["a" "b" "c"] ["b" "c"]) -- [false true true] + +;; empty needle set over a null-free symbol column +(in (as 'SYM ["a" "b"]) (as 'SYM [])) -- [false false] + +;; a single needle is the degenerate case the issue reported: same answer as +;; the equality it should cost the same as +(in (as 'SYM ["a" "b" "a"]) (as 'SYM ["a"])) -- [true false true] +(== (as 'SYM ["a" "b" "a"]) 'a) -- [true false true] + +;; duplicate needles must not double-count or change the verdict +(in (as 'SYM ["a" "b"]) (as 'SYM ["a" "a" "a"])) -- [true false]