From 903a772bcf079fb669cd86a7702cc7c03fe58413 Mon Sep 17 00:00:00 2001 From: comphead Date: Thu, 16 Jul 2026 08:23:35 -0700 Subject: [PATCH 1/9] feat: fall back to Spark for collated predicate operands Comet's native equality, ordering, and hashing compare raw UTF-8 bytes, so a predicate operand carrying a non-UTF8_BINARY collation (Spark 4+) would produce wrong answers on the native path -- for example, `'a' = 'A'` under UNICODE_CI returns true in Spark but false byte-wise. Add a `ComparisonUtils.collationSupportLevel` shared helper and wire it into `getSupportLevel` for the six binary comparison serdes plus `In`/`InSet`. `CometNot`'s special-case rewrites for `Not(EqualTo|EqualNullSafe|In)` now guard on the same helper so they fall through to the generic path (and thus to Spark) when any operand is collated. The helper walks nested types via `hasNonDefaultStringCollation`, so collated strings inside array/map/struct operands are caught too. --- .../org/apache/comet/serde/predicates.scala | 79 ++++++++++++++++--- 1 file changed, 69 insertions(+), 10 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/predicates.scala b/spark/src/main/scala/org/apache/comet/serde/predicates.scala index 63b64fbcf2..68ca662d6c 100644 --- a/spark/src/main/scala/org/apache/comet/serde/predicates.scala +++ b/spark/src/main/scala/org/apache/comet/serde/predicates.scala @@ -35,25 +35,31 @@ object CometNot extends CometExpressionSerde[Not] { binding: Boolean): Option[ExprOuterClass.Expr] = { expr.child match { - case expr: EqualTo => + case inner: EqualTo if !ComparisonUtils.hasCollatedOperand(inner.left, inner.right) => createBinaryExpr( - expr, - expr.left, - expr.right, + inner, + inner.left, + inner.right, inputs, binding, (builder, binaryExpr) => builder.setNeq(binaryExpr)) - case expr: EqualNullSafe => + case inner: EqualNullSafe if !ComparisonUtils.hasCollatedOperand(inner.left, inner.right) => createBinaryExpr( - expr, - expr.left, - expr.right, + inner, + inner.left, + inner.right, inputs, binding, (builder, binaryExpr) => builder.setNeqNullSafe(binaryExpr)) - case expr: In => - ComparisonUtils.in(expr, expr.value, expr.list, inputs, binding, negate = true) + case inner: In if !ComparisonUtils.hasCollatedOperand(inner.value +: inner.list) => + ComparisonUtils.in(inner, inner.value, inner.list, inputs, binding, negate = true) case _ => + // Includes the collated variants of EqualTo / EqualNullSafe / In above: fall through so + // the child expression's own serde is consulted, which now returns `Unsupported` for + // non-UTF8_BINARY operands (see `CometEqualTo.getSupportLevel` and siblings). That makes + // `exprToProtoInternal` return None for the child, which cascades this Not to None and + // falls the enclosing operator back to Spark — the only way to honour collation-aware + // (in)equality without a native path. createUnaryExpr( expr, expr.child, @@ -104,6 +110,8 @@ object CometOr extends CometExpressionSerde[Or] { } object CometEqualTo extends CometExpressionSerde[EqualTo] { + override def getSupportLevel(expr: EqualTo): SupportLevel = + ComparisonUtils.collationSupportLevel(expr.left, expr.right) override def convert( expr: EqualTo, inputs: Seq[Attribute], @@ -119,6 +127,8 @@ object CometEqualTo extends CometExpressionSerde[EqualTo] { } object CometEqualNullSafe extends CometExpressionSerde[EqualNullSafe] { + override def getSupportLevel(expr: EqualNullSafe): SupportLevel = + ComparisonUtils.collationSupportLevel(expr.left, expr.right) override def convert( expr: EqualNullSafe, inputs: Seq[Attribute], @@ -134,6 +144,8 @@ object CometEqualNullSafe extends CometExpressionSerde[EqualNullSafe] { } object CometGreaterThan extends CometExpressionSerde[GreaterThan] { + override def getSupportLevel(expr: GreaterThan): SupportLevel = + ComparisonUtils.collationSupportLevel(expr.left, expr.right) override def convert( expr: GreaterThan, inputs: Seq[Attribute], @@ -149,6 +161,8 @@ object CometGreaterThan extends CometExpressionSerde[GreaterThan] { } object CometGreaterThanOrEqual extends CometExpressionSerde[GreaterThanOrEqual] { + override def getSupportLevel(expr: GreaterThanOrEqual): SupportLevel = + ComparisonUtils.collationSupportLevel(expr.left, expr.right) override def convert( expr: GreaterThanOrEqual, inputs: Seq[Attribute], @@ -164,6 +178,8 @@ object CometGreaterThanOrEqual extends CometExpressionSerde[GreaterThanOrEqual] } object CometLessThan extends CometExpressionSerde[LessThan] { + override def getSupportLevel(expr: LessThan): SupportLevel = + ComparisonUtils.collationSupportLevel(expr.left, expr.right) override def convert( expr: LessThan, inputs: Seq[Attribute], @@ -179,6 +195,8 @@ object CometLessThan extends CometExpressionSerde[LessThan] { } object CometLessThanOrEqual extends CometExpressionSerde[LessThanOrEqual] { + override def getSupportLevel(expr: LessThanOrEqual): SupportLevel = + ComparisonUtils.collationSupportLevel(expr.left, expr.right) override def convert( expr: LessThanOrEqual, inputs: Seq[Attribute], @@ -234,6 +252,15 @@ object CometIsNaN extends CometExpressionSerde[IsNaN] { } object CometIn extends CometExpressionSerde[In] { + + override def getSupportLevel(expr: In): SupportLevel = { + if (ComparisonUtils.hasCollatedOperand(expr.value +: expr.list)) { + Unsupported(Some(ComparisonUtils.nonDefaultCollationReason("In"))) + } else { + Compatible() + } + } + override def convert( expr: In, inputs: Seq[Attribute], @@ -243,6 +270,15 @@ object CometIn extends CometExpressionSerde[In] { } object CometInSet extends CometExpressionSerde[InSet] { + + override def getSupportLevel(expr: InSet): SupportLevel = { + if (ComparisonUtils.hasCollatedOperand(expr.child)) { + Unsupported(Some(ComparisonUtils.nonDefaultCollationReason("InSet"))) + } else { + Compatible() + } + } + override def convert( expr: InSet, inputs: Seq[Attribute], @@ -259,6 +295,29 @@ object CometInSet extends CometExpressionSerde[InSet] { object ComparisonUtils { + // Comet's native equality/ordering/hashing compare raw bytes, so any predicate operand carrying + // a non-UTF8_BINARY collation (Spark 4+) would produce wrong answers on the native path — e.g. + // `'a' = 'A'` under `UNICODE_CI` returns true in Spark but false byte-wise. Every binary + // comparison and `In`/`InSet` serde routes its `getSupportLevel` through here so that any + // collated operand triggers a clean fallback to Spark. `hasNonDefaultStringCollation` walks + // nested types too, so collated strings inside array/map/struct operands are also caught. + def nonDefaultCollationReason(exprName: String): String = + s"$exprName does not support non-UTF8_BINARY collated operands; " + + "native comparison is byte-wise and cannot honour collation semantics." + + def hasCollatedOperand(operands: Expression*): Boolean = + operands.exists(op => hasNonDefaultStringCollation(op.dataType)) + + def hasCollatedOperand(operands: Iterable[Expression]): Boolean = + operands.exists(op => hasNonDefaultStringCollation(op.dataType)) + + def collationSupportLevel(left: Expression, right: Expression): SupportLevel = + if (hasCollatedOperand(left, right)) { + Unsupported(Some(nonDefaultCollationReason("BinaryComparison"))) + } else { + Compatible() + } + def in( expr: Expression, value: Expression, From 53611d75b3ac779f61c1efe840c89b40e73ea384 Mon Sep 17 00:00:00 2001 From: comphead Date: Thu, 16 Jul 2026 09:36:54 -0700 Subject: [PATCH 2/9] feat: dispatch collated str_to_map through JVM codegen Mix CodegenDispatchFallback into CometStrToMap so that when the input string or a delimiter carries a non-UTF8_BINARY collation the expression is executed by Spark's own doGenCode inside the Comet kernel rather than falling the whole projection back to Spark. Update the str_to_map_collation.sql fixture to drop the expect_fallback(...) assertion accordingly. --- .../src/main/scala/org/apache/comet/serde/maps.scala | 11 ++++++++--- .../expressions/map/str_to_map_collation.sql | 9 ++++----- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/maps.scala b/spark/src/main/scala/org/apache/comet/serde/maps.scala index a5ae24d215..c8304440c3 100644 --- a/spark/src/main/scala/org/apache/comet/serde/maps.scala +++ b/spark/src/main/scala/org/apache/comet/serde/maps.scala @@ -166,12 +166,17 @@ object CometMapFromEntries } } -object CometStrToMap extends CometScalarFunction[StringToMap]("str_to_map") with CometTypeShim { +object CometStrToMap + extends CometScalarFunction[StringToMap]("str_to_map") + with CometTypeShim + with CodegenDispatchFallback { // Spark 4.1.1+ honours spark.sql.legacy.truncateForEmptyRegexSplit by truncating trailing // empty entries from the split result. Comet's native str_to_map always behaves as if the flag - // were false, so it is incompatible when legacy truncation is enabled. Read by string key so it - // resolves on older Spark versions where the config is not registered. + // were false. When the flag is true, mark this Incompatible so the CodegenDispatchFallback + // trait routes the expression through the JVM codegen dispatcher (Spark's own doGenCode inside + // the Comet kernel) rather than falling the entire projection back to Spark. Read by string + // key so it resolves on older Spark versions where the config is not registered. private val legacyTruncateConfig = "spark.sql.legacy.truncateForEmptyRegexSplit" private val legacyTruncateReason = diff --git a/spark/src/test/resources/sql-tests/expressions/map/str_to_map_collation.sql b/spark/src/test/resources/sql-tests/expressions/map/str_to_map_collation.sql index 7816a1442c..41594f612c 100644 --- a/spark/src/test/resources/sql-tests/expressions/map/str_to_map_collation.sql +++ b/spark/src/test/resources/sql-tests/expressions/map/str_to_map_collation.sql @@ -17,17 +17,16 @@ -- MinSparkVersion: 4.0 --- Spark 4.0+ applies collation-aware delimiter matching in str_to_map. Comet's native --- implementation matches raw delimiter bytes, so non-UTF8_BINARY collations must fall back. +-- Spark 4.0+ applies collation-aware delimiter matching in str_to_map. -- collated input string -query expect_fallback(`str_to_map` does not support non-UTF8_BINARY collations) +query SELECT str_to_map('a:1,b:2' COLLATE UTF8_LCASE, ',', ':') -- collated pair delimiter -query expect_fallback(`str_to_map` does not support non-UTF8_BINARY collations) +query SELECT str_to_map('aX1,bX2', 'x' COLLATE UTF8_LCASE, ':') -- collated key-value delimiter -query expect_fallback(`str_to_map` does not support non-UTF8_BINARY collations) +query SELECT str_to_map('aX1,bX2', ',', 'x' COLLATE UTF8_LCASE) From a8e4799e6c7f91ea64edd1d160a030d765982237 Mon Sep 17 00:00:00 2001 From: comphead Date: Thu, 16 Jul 2026 12:36:42 -0700 Subject: [PATCH 3/9] test: dispatch legacy-truncate str_to_map through codegen Update str_to_map_legacy_truncate.sql to drop the expect_fallback assertions -- with CodegenDispatchFallback on CometStrToMap the expression now runs via Spark's own doGenCode inside the Comet kernel instead of falling the projection back to Spark. --- .../map/str_to_map_legacy_truncate.sql | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/spark/src/test/resources/sql-tests/expressions/map/str_to_map_legacy_truncate.sql b/spark/src/test/resources/sql-tests/expressions/map/str_to_map_legacy_truncate.sql index a5b5eb9f16..8ab8058920 100644 --- a/spark/src/test/resources/sql-tests/expressions/map/str_to_map_legacy_truncate.sql +++ b/spark/src/test/resources/sql-tests/expressions/map/str_to_map_legacy_truncate.sql @@ -15,19 +15,22 @@ -- specific language governing permissions and limitations -- under the License. --- Tests that str_to_map falls back to Spark when --- spark.sql.legacy.truncateForEmptyRegexSplit is enabled. In legacy mode Spark truncates trailing --- empty entries from the split result, which Comet's native str_to_map does not honour. +-- Tests that str_to_map routes through the JVM codegen dispatcher when +-- spark.sql.legacy.truncateForEmptyRegexSplit is enabled. In legacy mode Spark truncates +-- trailing empty entries from the split result, which Comet's native str_to_map does not +-- honour. `CometStrToMap` marks the expression Incompatible when the flag is on and mixes in +-- [[CodegenDispatchFallback]], so the projection stays native (Spark's own `doGenCode` runs +-- inside the Comet kernel) while producing Spark-exact results. -- See https://github.com/apache/datafusion-comet/issues/4477 -- Config: spark.sql.legacy.truncateForEmptyRegexSplit=true --- trailing pair delimiter: legacy mode truncates the trailing empty entry, so Comet must fall --- back to Spark -query expect_fallback(truncateForEmptyRegexSplit) +-- trailing pair delimiter: legacy mode truncates the trailing empty entry; Comet delegates to +-- the codegen dispatcher. +query SELECT str_to_map('a:1,b:2,', ',', ':') --- column input also falls back +-- column input is also handled via the codegen dispatcher statement CREATE TABLE test_str_to_map_legacy(s STRING, pair_delim STRING, key_value_delim STRING) USING parquet @@ -37,5 +40,5 @@ INSERT INTO test_str_to_map_legacy VALUES ('x:1;y:2;', ';', ':'), (NULL, ',', ':') -query expect_fallback(truncateForEmptyRegexSplit) +query SELECT str_to_map(s, pair_delim, key_value_delim) FROM test_str_to_map_legacy From 4228c7c8b014c9446078032a928c9c6f1b8b7505 Mon Sep 17 00:00:00 2001 From: comphead Date: Thu, 16 Jul 2026 15:15:42 -0700 Subject: [PATCH 4/9] feat: fall back to Spark for collated Like/Contains/StartsWith/EndsWith Extends 903a772b6 to the string-predicate serdes flagged in review. CometLike, CometContains, CometStartsWith, and CometEndsWith now route their `getSupportLevel` through `ComparisonUtils.collationSupportLevel` so any non-UTF8_BINARY operand triggers a clean fallback. Previously Contains / StartsWith / EndsWith reached the native scalar-function bridge via `CometScalarFunction` with no collation guard, and Like's `getSupportLevel` only rejected custom escape characters. Native kernels compare raw UTF-8 bytes, so a case- or accent-insensitive collation would yield wrong answers (e.g. `'Hello' LIKE '%hello%'` returns true in Spark under UNICODE_CI but false byte-wise). `collationSupportLevel` grows an optional `exprName` so each predicate surfaces a fallback reason that names itself in EXPLAIN. Expands `sql-tests/.../collation.sql` with predicate coverage: the six binary comparisons under UTF8_LCASE and UNICODE_CI, `IN`/`NOT IN` (scalar and `InSet`-threshold), CometNot's `NOT (=)` / `NOT (<=>)` rewrites, `LIKE`, `Contains`, `StartsWith`, `EndsWith`, collated strings buried in `array`/`map`/`struct` operands (exercising `hasNonDefaultStringCollation`'s recursion), and NULL propagation through each of the above. --- .../apache/comet/serde/QueryPlanSerde.scala | 6 +- .../org/apache/comet/serde/predicates.scala | 7 +- .../org/apache/comet/serde/strings.scala | 40 ++++- .../expressions/string/collation.sql | 155 ++++++++++++++++++ 4 files changed, 199 insertions(+), 9 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 5eee0c5cad..284094de35 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -218,8 +218,8 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { classOf[Chr] -> CometScalarFunction("char"), classOf[ConcatWs] -> CometConcatWs, classOf[Concat] -> CometConcat, - classOf[Contains] -> CometScalarFunction("contains"), - classOf[EndsWith] -> CometScalarFunction("ends_with"), + classOf[Contains] -> CometContains, + classOf[EndsWith] -> CometEndsWith, classOf[GetJsonObject] -> CometGetJsonObject, classOf[InitCap] -> CometInitCap, classOf[Length] -> CometLength, @@ -233,7 +233,7 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { classOf[RegExpReplace] -> CometRegExpReplace, classOf[Reverse] -> CometReverse, classOf[RLike] -> CometRLike, - classOf[StartsWith] -> CometScalarFunction("starts_with"), + classOf[StartsWith] -> CometStartsWith, classOf[StringInstr] -> CometScalarFunction("instr"), classOf[StringRepeat] -> CometStringRepeat, classOf[StringReplace] -> CometStringReplace, diff --git a/spark/src/main/scala/org/apache/comet/serde/predicates.scala b/spark/src/main/scala/org/apache/comet/serde/predicates.scala index 68ca662d6c..8fb933a5ad 100644 --- a/spark/src/main/scala/org/apache/comet/serde/predicates.scala +++ b/spark/src/main/scala/org/apache/comet/serde/predicates.scala @@ -311,9 +311,12 @@ object ComparisonUtils { def hasCollatedOperand(operands: Iterable[Expression]): Boolean = operands.exists(op => hasNonDefaultStringCollation(op.dataType)) - def collationSupportLevel(left: Expression, right: Expression): SupportLevel = + def collationSupportLevel( + left: Expression, + right: Expression, + exprName: String = "BinaryComparison"): SupportLevel = if (hasCollatedOperand(left, right)) { - Unsupported(Some(nonDefaultCollationReason("BinaryComparison"))) + Unsupported(Some(nonDefaultCollationReason(exprName))) } else { Compatible() } diff --git a/spark/src/main/scala/org/apache/comet/serde/strings.scala b/spark/src/main/scala/org/apache/comet/serde/strings.scala index 07b3c0c99f..cbbce64a9e 100644 --- a/spark/src/main/scala/org/apache/comet/serde/strings.scala +++ b/spark/src/main/scala/org/apache/comet/serde/strings.scala @@ -19,7 +19,7 @@ package org.apache.comet.serde -import org.apache.spark.sql.catalyst.expressions.{Attribute, Base64, BitLength, Cast, Concat, ConcatWs, Elt, Empty2Null, Expression, FindInSet, FormatNumber, FormatString, GetJsonObject, InitCap, Left, Length, Levenshtein, Like, Literal, Lower, Mask, OctetLength, Overlay, RegExpExtract, RegExpExtractAll, RegExpInStr, RegExpReplace, Right, RLike, SoundEx, StringLocate, StringLPad, StringRepeat, StringReplace, StringRPad, StringSplit, StringTranslate, Substring, SubstringIndex, ToCharacter, ToNumber, TryToNumber, UnBase64, Upper} +import org.apache.spark.sql.catalyst.expressions.{Attribute, Base64, BitLength, Cast, Concat, ConcatWs, Contains, Elt, Empty2Null, EndsWith, Expression, FindInSet, FormatNumber, FormatString, GetJsonObject, InitCap, Left, Length, Levenshtein, Like, Literal, Lower, Mask, OctetLength, Overlay, RegExpExtract, RegExpExtractAll, RegExpInStr, RegExpReplace, Right, RLike, SoundEx, StartsWith, StringLocate, StringLPad, StringRepeat, StringReplace, StringRPad, StringSplit, StringTranslate, Substring, SubstringIndex, ToCharacter, ToNumber, TryToNumber, UnBase64, Upper} import org.apache.spark.sql.types.{BinaryType, DataTypes, LongType, StringType} import org.apache.comet.CometConf @@ -291,10 +291,12 @@ object CometConcatWs extends CometExpressionSerde[ConcatWs] { object CometLike extends CometExpressionSerde[Like] { override def getSupportLevel(expr: Like): SupportLevel = { - if (expr.escapeChar == '\\') { - Compatible() - } else { + if (ComparisonUtils.hasCollatedOperand(expr.left, expr.right)) { + Unsupported(Some(ComparisonUtils.nonDefaultCollationReason("Like"))) + } else if (expr.escapeChar != '\\') { Unsupported(Some(s"custom escape character ${expr.escapeChar} not supported in LIKE")) + } else { + Compatible() } } @@ -309,6 +311,36 @@ object CometLike extends CometExpressionSerde[Like] { } } +/** + * Serdes for `Contains` / `StartsWith` / `EndsWith` that guard on non-UTF8_BINARY collation and + * otherwise delegate to the generic `contains` / `starts_with` / `ends_with` scalar-function + * bridge. The native kernels compare raw bytes and cannot honour case- or accent-insensitive + * collations, so a collated operand must fall back to Spark. + */ +object CometContains extends CometExpressionSerde[Contains] { + override def getSupportLevel(expr: Contains): SupportLevel = + ComparisonUtils.collationSupportLevel(expr.left, expr.right, "Contains") + + override def convert(expr: Contains, inputs: Seq[Attribute], binding: Boolean): Option[Expr] = + CometScalarFunction[Contains]("contains").convert(expr, inputs, binding) +} + +object CometStartsWith extends CometExpressionSerde[StartsWith] { + override def getSupportLevel(expr: StartsWith): SupportLevel = + ComparisonUtils.collationSupportLevel(expr.left, expr.right, "StartsWith") + + override def convert(expr: StartsWith, inputs: Seq[Attribute], binding: Boolean): Option[Expr] = + CometScalarFunction[StartsWith]("starts_with").convert(expr, inputs, binding) +} + +object CometEndsWith extends CometExpressionSerde[EndsWith] { + override def getSupportLevel(expr: EndsWith): SupportLevel = + ComparisonUtils.collationSupportLevel(expr.left, expr.right, "EndsWith") + + override def convert(expr: EndsWith, inputs: Seq[Attribute], binding: Boolean): Option[Expr] = + CometScalarFunction[EndsWith]("ends_with").convert(expr, inputs, binding) +} + /** * `rlike` runs Spark's own implementation through the codegen dispatcher by default, for * byte-exact results. The native (rust) regexp engine is faster but has different semantics from diff --git a/spark/src/test/resources/sql-tests/expressions/string/collation.sql b/spark/src/test/resources/sql-tests/expressions/string/collation.sql index 89dd95b9e7..660640b275 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/collation.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/collation.sql @@ -51,3 +51,158 @@ SELECT concat('Hello' COLLATE UNICODE_CI, 'World' COLLATE UNICODE_CI) query SELECT reverse('Hello' COLLATE UNICODE_CI) + +-- ============================================================================ +-- Predicate fallback for non-UTF8_BINARY collated operands. +-- +-- Comet's native equality/ordering/hashing/substring kernels compare raw UTF-8 bytes, so any +-- predicate operand carrying a non-UTF8_BINARY collation would produce wrong answers on the +-- native path -- e.g. `'a' = 'A'` under UNICODE_CI returns true in Spark but false byte-wise. +-- Every binary comparison and `In`/`InSet`/`Like`/`Contains`/`StartsWith`/`EndsWith` serde now +-- guards its `getSupportLevel` so any collated operand triggers a clean fallback to Spark. +-- The queries below exercise that fallback; if the guard regresses, Comet's native answer +-- diverges from Spark's collation-aware answer and the diff surfaces here. +-- ============================================================================ + +statement +CREATE TABLE test_collated_predicates(id INT, s STRING) USING parquet + +statement +INSERT INTO test_collated_predicates VALUES + (1, 'a'), + (2, 'A'), + (3, 'hello'), + (4, 'HELLO'), + (5, 'World'), + (6, 'b'), + (7, NULL) + +-- ---------- EqualTo / EqualNullSafe / inequality --------------------------- + +-- UTF8_LCASE folds case, so 'a' and 'A' both match 'A'. +query +SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) = 'A' FROM test_collated_predicates ORDER BY id + +query +SELECT id FROM test_collated_predicates WHERE CAST(s AS STRING COLLATE UTF8_LCASE) = 'A' ORDER BY id + +-- UNICODE_CI is a distinct ICU collation, also case-insensitive. +query +SELECT id, CAST(s AS STRING COLLATE UNICODE_CI) = 'A' FROM test_collated_predicates ORDER BY id + +-- Not-equals should surface via `!=` (parsed as Not(EqualTo)) and `<>`. +query +SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) != 'A' FROM test_collated_predicates ORDER BY id + +query +SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) <> 'A' FROM test_collated_predicates ORDER BY id + +-- Null-safe equality: <=> treats two NULLs as equal. +query +SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) <=> 'A' FROM test_collated_predicates ORDER BY id + +query +SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) <=> CAST(NULL AS STRING COLLATE UTF8_LCASE) FROM test_collated_predicates ORDER BY id + +-- ---------- Ordering ------------------------------------------------------ + +query +SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) < 'b' FROM test_collated_predicates ORDER BY id + +query +SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) <= 'a' FROM test_collated_predicates ORDER BY id + +query +SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) > 'a' FROM test_collated_predicates ORDER BY id + +query +SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) >= 'A' FROM test_collated_predicates ORDER BY id + +query +SELECT id, CAST(s AS STRING COLLATE UNICODE_CI) < 'b' FROM test_collated_predicates ORDER BY id + +-- ---------- In / NotIn / InSet --------------------------------------------- + +-- Small list of literals -> `In`. +query +SELECT id FROM test_collated_predicates WHERE CAST(s AS STRING COLLATE UTF8_LCASE) IN ('A', 'HELLO') ORDER BY id + +-- `NOT IN` exercises CometNot's special-case rewrite for In: the collated variant must fall +-- through to the generic path so the child serde's Unsupported cascades to a Spark fallback. +query +SELECT id FROM test_collated_predicates WHERE CAST(s AS STRING COLLATE UTF8_LCASE) NOT IN ('A', 'HELLO') ORDER BY id + +-- Optimizer promotes a large literal list to `InSet`; force it by widening past the threshold +-- (spark.sql.optimizer.inSetConversionThreshold defaults to 10). +query +SELECT id FROM test_collated_predicates WHERE CAST(s AS STRING COLLATE UTF8_LCASE) IN ('A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'HELLO') ORDER BY id + +-- ---------- NOT (EqualTo / EqualNullSafe) rewrites ------------------------ + +-- Exercises CometNot's guard so the collated child falls through to Spark. +query +SELECT id FROM test_collated_predicates WHERE NOT (CAST(s AS STRING COLLATE UTF8_LCASE) = 'A') ORDER BY id + +query +SELECT id FROM test_collated_predicates WHERE NOT (CAST(s AS STRING COLLATE UTF8_LCASE) <=> 'A') ORDER BY id + +-- ---------- Like ----------------------------------------------------------- + +query +SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) LIKE '%LLO' FROM test_collated_predicates ORDER BY id + +query +SELECT id FROM test_collated_predicates WHERE CAST(s AS STRING COLLATE UTF8_LCASE) LIKE 'H%' ORDER BY id + +query +SELECT id FROM test_collated_predicates WHERE CAST(s AS STRING COLLATE UNICODE_CI) LIKE 'h%' ORDER BY id + +-- ---------- Contains / StartsWith / EndsWith ------------------------------- + +query +SELECT id, contains(CAST(s AS STRING COLLATE UTF8_LCASE), 'HE') FROM test_collated_predicates ORDER BY id + +query +SELECT id, startswith(CAST(s AS STRING COLLATE UTF8_LCASE), 'HE') FROM test_collated_predicates ORDER BY id + +query +SELECT id, endswith(CAST(s AS STRING COLLATE UTF8_LCASE), 'LO') FROM test_collated_predicates ORDER BY id + +-- Filter-side use, exercising the WHERE-clause pushdown path. +query +SELECT id FROM test_collated_predicates WHERE contains(CAST(s AS STRING COLLATE UTF8_LCASE), 'ELL') ORDER BY id + +query +SELECT id FROM test_collated_predicates WHERE startswith(CAST(s AS STRING COLLATE UNICODE_CI), 'h') ORDER BY id + +query +SELECT id FROM test_collated_predicates WHERE endswith(CAST(s AS STRING COLLATE UNICODE_CI), 'O') ORDER BY id + +-- ---------- Nested-collated types ------------------------------------------ + +-- hasNonDefaultStringCollation walks nested types, so a collated string buried inside an +-- array/map/struct operand must also trigger fallback. If the recursive check regressed, +-- Comet would attempt native array-equality with byte-wise element compares and diverge here. +query +SELECT id, array(CAST(s AS STRING COLLATE UTF8_LCASE)) = array('A' COLLATE UTF8_LCASE) FROM test_collated_predicates ORDER BY id + +query +SELECT id, struct(CAST(s AS STRING COLLATE UTF8_LCASE)) = struct('A' COLLATE UTF8_LCASE) FROM test_collated_predicates ORDER BY id + +-- Map with collated value type. +query +SELECT id, map('k', CAST(s AS STRING COLLATE UTF8_LCASE)) = map('k', 'A' COLLATE UTF8_LCASE) FROM test_collated_predicates ORDER BY id + +-- ---------- NULL propagation ------------------------------------------------ + +query +SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) = CAST(NULL AS STRING COLLATE UTF8_LCASE) FROM test_collated_predicates ORDER BY id + +query +SELECT id, CAST(NULL AS STRING COLLATE UTF8_LCASE) IN ('A', 'B') FROM test_collated_predicates ORDER BY id + +query +SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) LIKE CAST(NULL AS STRING COLLATE UTF8_LCASE) FROM test_collated_predicates ORDER BY id + +query +SELECT id, contains(CAST(s AS STRING COLLATE UTF8_LCASE), CAST(NULL AS STRING COLLATE UTF8_LCASE)) FROM test_collated_predicates ORDER BY id From ba70de9d3ff642c2a966c1471ff8171d975c762f Mon Sep 17 00:00:00 2001 From: comphead Date: Thu, 16 Jul 2026 16:43:00 -0700 Subject: [PATCH 5/9] test: expect fallback for collated-predicate SQL tests Every predicate query in `collation.sql` now uses `query expect_fallback(non-UTF8_BINARY collated operands)` instead of plain `query`. The predicate serdes correctly return `Unsupported` for collated operands, so the enclosing Project falls back to Spark -- which is the whole point of the fix -- but that violates the default `CheckCoverageAndAnswer` assertion mode's "only Comet native operators" check. `expect_fallback` still verifies the answer against Spark and additionally pins the shared fallback-reason substring, so a regressed guard shows up as either a wrong answer or a missing fallback. Also drops the map test: `CometCreateMap` routes through the JVM codegen dispatcher, so its interaction with the outer EqualTo's collation guard is not the clean path this file is meant to pin down. --- .../expressions/string/collation.sql | 78 +++++++++---------- 1 file changed, 37 insertions(+), 41 deletions(-) diff --git a/spark/src/test/resources/sql-tests/expressions/string/collation.sql b/spark/src/test/resources/sql-tests/expressions/string/collation.sql index 660640b275..cfd6e2af60 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/collation.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/collation.sql @@ -59,9 +59,9 @@ SELECT reverse('Hello' COLLATE UNICODE_CI) -- predicate operand carrying a non-UTF8_BINARY collation would produce wrong answers on the -- native path -- e.g. `'a' = 'A'` under UNICODE_CI returns true in Spark but false byte-wise. -- Every binary comparison and `In`/`InSet`/`Like`/`Contains`/`StartsWith`/`EndsWith` serde now --- guards its `getSupportLevel` so any collated operand triggers a clean fallback to Spark. --- The queries below exercise that fallback; if the guard regresses, Comet's native answer --- diverges from Spark's collation-aware answer and the diff surfaces here. +-- guards its `getSupportLevel` so any collated operand triggers a clean fallback to Spark with a +-- shared "non-UTF8_BINARY collated operands" reason. `expect_fallback(...)` below pins that +-- reason substring so a regressed guard surfaces as either a wrong answer or a missing fallback. -- ============================================================================ statement @@ -80,129 +80,125 @@ INSERT INTO test_collated_predicates VALUES -- ---------- EqualTo / EqualNullSafe / inequality --------------------------- -- UTF8_LCASE folds case, so 'a' and 'A' both match 'A'. -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) = 'A' FROM test_collated_predicates ORDER BY id -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id FROM test_collated_predicates WHERE CAST(s AS STRING COLLATE UTF8_LCASE) = 'A' ORDER BY id -- UNICODE_CI is a distinct ICU collation, also case-insensitive. -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, CAST(s AS STRING COLLATE UNICODE_CI) = 'A' FROM test_collated_predicates ORDER BY id -- Not-equals should surface via `!=` (parsed as Not(EqualTo)) and `<>`. -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) != 'A' FROM test_collated_predicates ORDER BY id -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) <> 'A' FROM test_collated_predicates ORDER BY id -- Null-safe equality: <=> treats two NULLs as equal. -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) <=> 'A' FROM test_collated_predicates ORDER BY id -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) <=> CAST(NULL AS STRING COLLATE UTF8_LCASE) FROM test_collated_predicates ORDER BY id -- ---------- Ordering ------------------------------------------------------ -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) < 'b' FROM test_collated_predicates ORDER BY id -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) <= 'a' FROM test_collated_predicates ORDER BY id -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) > 'a' FROM test_collated_predicates ORDER BY id -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) >= 'A' FROM test_collated_predicates ORDER BY id -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, CAST(s AS STRING COLLATE UNICODE_CI) < 'b' FROM test_collated_predicates ORDER BY id -- ---------- In / NotIn / InSet --------------------------------------------- -- Small list of literals -> `In`. -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id FROM test_collated_predicates WHERE CAST(s AS STRING COLLATE UTF8_LCASE) IN ('A', 'HELLO') ORDER BY id -- `NOT IN` exercises CometNot's special-case rewrite for In: the collated variant must fall -- through to the generic path so the child serde's Unsupported cascades to a Spark fallback. -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id FROM test_collated_predicates WHERE CAST(s AS STRING COLLATE UTF8_LCASE) NOT IN ('A', 'HELLO') ORDER BY id -- Optimizer promotes a large literal list to `InSet`; force it by widening past the threshold -- (spark.sql.optimizer.inSetConversionThreshold defaults to 10). -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id FROM test_collated_predicates WHERE CAST(s AS STRING COLLATE UTF8_LCASE) IN ('A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'HELLO') ORDER BY id -- ---------- NOT (EqualTo / EqualNullSafe) rewrites ------------------------ -- Exercises CometNot's guard so the collated child falls through to Spark. -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id FROM test_collated_predicates WHERE NOT (CAST(s AS STRING COLLATE UTF8_LCASE) = 'A') ORDER BY id -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id FROM test_collated_predicates WHERE NOT (CAST(s AS STRING COLLATE UTF8_LCASE) <=> 'A') ORDER BY id -- ---------- Like ----------------------------------------------------------- -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) LIKE '%LLO' FROM test_collated_predicates ORDER BY id -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id FROM test_collated_predicates WHERE CAST(s AS STRING COLLATE UTF8_LCASE) LIKE 'H%' ORDER BY id -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id FROM test_collated_predicates WHERE CAST(s AS STRING COLLATE UNICODE_CI) LIKE 'h%' ORDER BY id -- ---------- Contains / StartsWith / EndsWith ------------------------------- -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, contains(CAST(s AS STRING COLLATE UTF8_LCASE), 'HE') FROM test_collated_predicates ORDER BY id -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, startswith(CAST(s AS STRING COLLATE UTF8_LCASE), 'HE') FROM test_collated_predicates ORDER BY id -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, endswith(CAST(s AS STRING COLLATE UTF8_LCASE), 'LO') FROM test_collated_predicates ORDER BY id -- Filter-side use, exercising the WHERE-clause pushdown path. -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id FROM test_collated_predicates WHERE contains(CAST(s AS STRING COLLATE UTF8_LCASE), 'ELL') ORDER BY id -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id FROM test_collated_predicates WHERE startswith(CAST(s AS STRING COLLATE UNICODE_CI), 'h') ORDER BY id -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id FROM test_collated_predicates WHERE endswith(CAST(s AS STRING COLLATE UNICODE_CI), 'O') ORDER BY id -- ---------- Nested-collated types ------------------------------------------ -- hasNonDefaultStringCollation walks nested types, so a collated string buried inside an --- array/map/struct operand must also trigger fallback. If the recursive check regressed, --- Comet would attempt native array-equality with byte-wise element compares and diverge here. -query +-- array/struct operand must also trigger fallback. If the recursive check regressed, Comet would +-- attempt native array-/struct-equality with byte-wise element compares and diverge here. +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, array(CAST(s AS STRING COLLATE UTF8_LCASE)) = array('A' COLLATE UTF8_LCASE) FROM test_collated_predicates ORDER BY id -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, struct(CAST(s AS STRING COLLATE UTF8_LCASE)) = struct('A' COLLATE UTF8_LCASE) FROM test_collated_predicates ORDER BY id --- Map with collated value type. -query -SELECT id, map('k', CAST(s AS STRING COLLATE UTF8_LCASE)) = map('k', 'A' COLLATE UTF8_LCASE) FROM test_collated_predicates ORDER BY id - -- ---------- NULL propagation ------------------------------------------------ -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) = CAST(NULL AS STRING COLLATE UTF8_LCASE) FROM test_collated_predicates ORDER BY id -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, CAST(NULL AS STRING COLLATE UTF8_LCASE) IN ('A', 'B') FROM test_collated_predicates ORDER BY id -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) LIKE CAST(NULL AS STRING COLLATE UTF8_LCASE) FROM test_collated_predicates ORDER BY id -query +query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, contains(CAST(s AS STRING COLLATE UTF8_LCASE), CAST(NULL AS STRING COLLATE UTF8_LCASE)) FROM test_collated_predicates ORDER BY id From 9601fd7bec3fc146dc2e9c2ff1aa50a7adc79bc3 Mon Sep 17 00:00:00 2001 From: comphead Date: Thu, 16 Jul 2026 16:49:01 -0700 Subject: [PATCH 6/9] test: drop collation cases Spark folds or rejects at analysis Two classes of tests never reached the predicate serdes and so could not exercise the fix: - NULL-literal RHS: Spark's `NullPropagation` rewrites `x <=> NULL` to `IsNull(x)`, folds `x = NULL` and `NULL IN (...)` to `Literal(null)`, etc., so the predicate serde is never asked about a collated NULL operand. The row `(7, NULL)` in `test_collated_predicates` still exercises NULL propagation through every remaining predicate query. - UNICODE_CI + LIKE / Contains / StartsWith / EndsWith: Spark 4.0 restricts these to `StringTypeNonCSAICollation`, which rejects UNICODE_CI at analysis time with a DATATYPE_MISMATCH. UTF8_LCASE is accepted and already covers the same fallback path. --- .../expressions/string/collation.sql | 34 +++++++------------ 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/spark/src/test/resources/sql-tests/expressions/string/collation.sql b/spark/src/test/resources/sql-tests/expressions/string/collation.sql index cfd6e2af60..160cc2deb1 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/collation.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/collation.sql @@ -97,13 +97,13 @@ SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) != 'A' FROM test_collated_predic query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) <> 'A' FROM test_collated_predicates ORDER BY id --- Null-safe equality: <=> treats two NULLs as equal. +-- Null-safe equality: <=> treats two NULLs as equal. Note that `x <=> NULL` is folded by +-- Spark's `NullPropagation` to `IsNull(x)` -- which routes through a different serde -- so we +-- test the meaningful case here: a non-NULL RHS with the collated column, where the row's own +-- NULLs (id=7) still exercise NULL propagation through the predicate. query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) <=> 'A' FROM test_collated_predicates ORDER BY id -query expect_fallback(non-UTF8_BINARY collated operands) -SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) <=> CAST(NULL AS STRING COLLATE UTF8_LCASE) FROM test_collated_predicates ORDER BY id - -- ---------- Ordering ------------------------------------------------------ query expect_fallback(non-UTF8_BINARY collated operands) @@ -154,8 +154,8 @@ SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) LIKE '%LLO' FROM test_collated_p query expect_fallback(non-UTF8_BINARY collated operands) SELECT id FROM test_collated_predicates WHERE CAST(s AS STRING COLLATE UTF8_LCASE) LIKE 'H%' ORDER BY id -query expect_fallback(non-UTF8_BINARY collated operands) -SELECT id FROM test_collated_predicates WHERE CAST(s AS STRING COLLATE UNICODE_CI) LIKE 'h%' ORDER BY id +-- Spark 4.0 restricts LIKE / Contains / StartsWith / EndsWith to `StringTypeNonCSAICollation`, +-- which analyzes-out UNICODE_CI at planning time, so we cover only UTF8_LCASE for these. -- ---------- Contains / StartsWith / EndsWith ------------------------------- @@ -173,10 +173,10 @@ query expect_fallback(non-UTF8_BINARY collated operands) SELECT id FROM test_collated_predicates WHERE contains(CAST(s AS STRING COLLATE UTF8_LCASE), 'ELL') ORDER BY id query expect_fallback(non-UTF8_BINARY collated operands) -SELECT id FROM test_collated_predicates WHERE startswith(CAST(s AS STRING COLLATE UNICODE_CI), 'h') ORDER BY id +SELECT id FROM test_collated_predicates WHERE startswith(CAST(s AS STRING COLLATE UTF8_LCASE), 'HE') ORDER BY id query expect_fallback(non-UTF8_BINARY collated operands) -SELECT id FROM test_collated_predicates WHERE endswith(CAST(s AS STRING COLLATE UNICODE_CI), 'O') ORDER BY id +SELECT id FROM test_collated_predicates WHERE endswith(CAST(s AS STRING COLLATE UTF8_LCASE), 'O') ORDER BY id -- ---------- Nested-collated types ------------------------------------------ @@ -189,16 +189,8 @@ SELECT id, array(CAST(s AS STRING COLLATE UTF8_LCASE)) = array('A' COLLATE UTF8_ query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, struct(CAST(s AS STRING COLLATE UTF8_LCASE)) = struct('A' COLLATE UTF8_LCASE) FROM test_collated_predicates ORDER BY id --- ---------- NULL propagation ------------------------------------------------ - -query expect_fallback(non-UTF8_BINARY collated operands) -SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) = CAST(NULL AS STRING COLLATE UTF8_LCASE) FROM test_collated_predicates ORDER BY id - -query expect_fallback(non-UTF8_BINARY collated operands) -SELECT id, CAST(NULL AS STRING COLLATE UTF8_LCASE) IN ('A', 'B') FROM test_collated_predicates ORDER BY id - -query expect_fallback(non-UTF8_BINARY collated operands) -SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) LIKE CAST(NULL AS STRING COLLATE UTF8_LCASE) FROM test_collated_predicates ORDER BY id - -query expect_fallback(non-UTF8_BINARY collated operands) -SELECT id, contains(CAST(s AS STRING COLLATE UTF8_LCASE), CAST(NULL AS STRING COLLATE UTF8_LCASE)) FROM test_collated_predicates ORDER BY id +-- NOTE: NULL literal operands are omitted intentionally. Spark's `NullPropagation` rewrites +-- `x = NULL` to `Literal(null)`, `x <=> NULL` to `IsNull(x)`, `NULL IN (...)` to +-- `Literal(null)`, etc., so the predicate serdes never see them. NULL-in-data coverage is +-- exercised on every query above via the (7, NULL) row: Spark returns NULL for those rows and +-- the fallback-answer comparison would catch a divergence. From 96257c8b359c668ff511a539623a21628fc90ae0 Mon Sep 17 00:00:00 2001 From: comphead Date: Thu, 16 Jul 2026 16:53:14 -0700 Subject: [PATCH 7/9] test: tighten comments in collation.sql Drop the "what" narration and shrink each remaining note to a single sentence about *why* the case exists (NullPropagation folding, NonCSAICollation restriction, InSet threshold, etc.). No SQL changes. --- .../expressions/string/collation.sql | 45 +++++-------------- 1 file changed, 12 insertions(+), 33 deletions(-) diff --git a/spark/src/test/resources/sql-tests/expressions/string/collation.sql b/spark/src/test/resources/sql-tests/expressions/string/collation.sql index 160cc2deb1..a5af351a94 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/collation.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/collation.sql @@ -53,15 +53,9 @@ query SELECT reverse('Hello' COLLATE UNICODE_CI) -- ============================================================================ --- Predicate fallback for non-UTF8_BINARY collated operands. --- --- Comet's native equality/ordering/hashing/substring kernels compare raw UTF-8 bytes, so any --- predicate operand carrying a non-UTF8_BINARY collation would produce wrong answers on the --- native path -- e.g. `'a' = 'A'` under UNICODE_CI returns true in Spark but false byte-wise. --- Every binary comparison and `In`/`InSet`/`Like`/`Contains`/`StartsWith`/`EndsWith` serde now --- guards its `getSupportLevel` so any collated operand triggers a clean fallback to Spark with a --- shared "non-UTF8_BINARY collated operands" reason. `expect_fallback(...)` below pins that --- reason substring so a regressed guard surfaces as either a wrong answer or a missing fallback. +-- Predicate fallback for non-UTF8_BINARY collated operands. Comet's native +-- kernels are byte-wise, so any collated operand must fall back to Spark; +-- expect_fallback(...) pins the shared reason substring. -- ============================================================================ statement @@ -79,28 +73,23 @@ INSERT INTO test_collated_predicates VALUES -- ---------- EqualTo / EqualNullSafe / inequality --------------------------- --- UTF8_LCASE folds case, so 'a' and 'A' both match 'A'. query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) = 'A' FROM test_collated_predicates ORDER BY id query expect_fallback(non-UTF8_BINARY collated operands) SELECT id FROM test_collated_predicates WHERE CAST(s AS STRING COLLATE UTF8_LCASE) = 'A' ORDER BY id --- UNICODE_CI is a distinct ICU collation, also case-insensitive. +-- UNICODE_CI covers a second ICU collation to prove the guard is not UTF8_LCASE-only. query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, CAST(s AS STRING COLLATE UNICODE_CI) = 'A' FROM test_collated_predicates ORDER BY id --- Not-equals should surface via `!=` (parsed as Not(EqualTo)) and `<>`. query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) != 'A' FROM test_collated_predicates ORDER BY id query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) <> 'A' FROM test_collated_predicates ORDER BY id --- Null-safe equality: <=> treats two NULLs as equal. Note that `x <=> NULL` is folded by --- Spark's `NullPropagation` to `IsNull(x)` -- which routes through a different serde -- so we --- test the meaningful case here: a non-NULL RHS with the collated column, where the row's own --- NULLs (id=7) still exercise NULL propagation through the predicate. +-- `x <=> NULL` folds via NullPropagation, so we test only a non-NULL RHS. query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) <=> 'A' FROM test_collated_predicates ORDER BY id @@ -123,23 +112,19 @@ SELECT id, CAST(s AS STRING COLLATE UNICODE_CI) < 'b' FROM test_collated_predica -- ---------- In / NotIn / InSet --------------------------------------------- --- Small list of literals -> `In`. query expect_fallback(non-UTF8_BINARY collated operands) SELECT id FROM test_collated_predicates WHERE CAST(s AS STRING COLLATE UTF8_LCASE) IN ('A', 'HELLO') ORDER BY id --- `NOT IN` exercises CometNot's special-case rewrite for In: the collated variant must fall --- through to the generic path so the child serde's Unsupported cascades to a Spark fallback. +-- NOT IN exercises CometNot's collated fall-through into the generic path. query expect_fallback(non-UTF8_BINARY collated operands) SELECT id FROM test_collated_predicates WHERE CAST(s AS STRING COLLATE UTF8_LCASE) NOT IN ('A', 'HELLO') ORDER BY id --- Optimizer promotes a large literal list to `InSet`; force it by widening past the threshold --- (spark.sql.optimizer.inSetConversionThreshold defaults to 10). +-- 12 elements forces InSet (spark.sql.optimizer.inSetConversionThreshold=10 by default). query expect_fallback(non-UTF8_BINARY collated operands) SELECT id FROM test_collated_predicates WHERE CAST(s AS STRING COLLATE UTF8_LCASE) IN ('A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'HELLO') ORDER BY id -- ---------- NOT (EqualTo / EqualNullSafe) rewrites ------------------------ --- Exercises CometNot's guard so the collated child falls through to Spark. query expect_fallback(non-UTF8_BINARY collated operands) SELECT id FROM test_collated_predicates WHERE NOT (CAST(s AS STRING COLLATE UTF8_LCASE) = 'A') ORDER BY id @@ -154,8 +139,8 @@ SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) LIKE '%LLO' FROM test_collated_p query expect_fallback(non-UTF8_BINARY collated operands) SELECT id FROM test_collated_predicates WHERE CAST(s AS STRING COLLATE UTF8_LCASE) LIKE 'H%' ORDER BY id --- Spark 4.0 restricts LIKE / Contains / StartsWith / EndsWith to `StringTypeNonCSAICollation`, --- which analyzes-out UNICODE_CI at planning time, so we cover only UTF8_LCASE for these. +-- Spark 4.0 rejects UNICODE_CI for LIKE / Contains / StartsWith / EndsWith at analysis time +-- (StringTypeNonCSAICollation), so only UTF8_LCASE is covered for these. -- ---------- Contains / StartsWith / EndsWith ------------------------------- @@ -168,7 +153,6 @@ SELECT id, startswith(CAST(s AS STRING COLLATE UTF8_LCASE), 'HE') FROM test_coll query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, endswith(CAST(s AS STRING COLLATE UTF8_LCASE), 'LO') FROM test_collated_predicates ORDER BY id --- Filter-side use, exercising the WHERE-clause pushdown path. query expect_fallback(non-UTF8_BINARY collated operands) SELECT id FROM test_collated_predicates WHERE contains(CAST(s AS STRING COLLATE UTF8_LCASE), 'ELL') ORDER BY id @@ -180,17 +164,12 @@ SELECT id FROM test_collated_predicates WHERE endswith(CAST(s AS STRING COLLATE -- ---------- Nested-collated types ------------------------------------------ --- hasNonDefaultStringCollation walks nested types, so a collated string buried inside an --- array/struct operand must also trigger fallback. If the recursive check regressed, Comet would --- attempt native array-/struct-equality with byte-wise element compares and diverge here. +-- hasNonDefaultStringCollation recurses into array/struct element types. query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, array(CAST(s AS STRING COLLATE UTF8_LCASE)) = array('A' COLLATE UTF8_LCASE) FROM test_collated_predicates ORDER BY id query expect_fallback(non-UTF8_BINARY collated operands) SELECT id, struct(CAST(s AS STRING COLLATE UTF8_LCASE)) = struct('A' COLLATE UTF8_LCASE) FROM test_collated_predicates ORDER BY id --- NOTE: NULL literal operands are omitted intentionally. Spark's `NullPropagation` rewrites --- `x = NULL` to `Literal(null)`, `x <=> NULL` to `IsNull(x)`, `NULL IN (...)` to --- `Literal(null)`, etc., so the predicate serdes never see them. NULL-in-data coverage is --- exercised on every query above via the (7, NULL) row: Spark returns NULL for those rows and --- the fallback-answer comparison would catch a divergence. +-- NULL-literal operands are omitted: NullPropagation folds them before Comet sees them. The +-- (7, NULL) row covers NULL-in-data on every query above. From bb243220200609bc495c143cb9c21917745c3bf9 Mon Sep 17 00:00:00 2001 From: comphead Date: Thu, 16 Jul 2026 16:54:52 -0700 Subject: [PATCH 8/9] collation fallback --- .../org/apache/comet/serde/predicates.scala | 72 ++++++++----------- .../org/apache/comet/serde/strings.scala | 34 +++------ 2 files changed, 39 insertions(+), 67 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/predicates.scala b/spark/src/main/scala/org/apache/comet/serde/predicates.scala index 8fb933a5ad..162f5bea7f 100644 --- a/spark/src/main/scala/org/apache/comet/serde/predicates.scala +++ b/spark/src/main/scala/org/apache/comet/serde/predicates.scala @@ -21,7 +21,7 @@ package org.apache.comet.serde import scala.jdk.CollectionConverters._ -import org.apache.spark.sql.catalyst.expressions.{And, Attribute, EqualNullSafe, EqualTo, Expression, GreaterThan, GreaterThanOrEqual, In, InSet, IsNaN, IsNotNull, IsNull, LessThan, LessThanOrEqual, Literal, Not, Or} +import org.apache.spark.sql.catalyst.expressions.{And, Attribute, BinaryExpression, EqualNullSafe, EqualTo, Expression, GreaterThan, GreaterThanOrEqual, In, InSet, IsNaN, IsNotNull, IsNull, LessThan, LessThanOrEqual, Literal, Not, Or} import org.apache.spark.sql.types.BooleanType import org.apache.comet.CometSparkSessionExtensions.withFallbackReason @@ -34,8 +34,10 @@ object CometNot extends CometExpressionSerde[Not] { inputs: Seq[Attribute], binding: Boolean): Option[ExprOuterClass.Expr] = { + import ComparisonUtils.hasCollatedOperand + expr.child match { - case inner: EqualTo if !ComparisonUtils.hasCollatedOperand(inner.left, inner.right) => + case inner: EqualTo if !hasCollatedOperand(inner.left, inner.right) => createBinaryExpr( inner, inner.left, @@ -43,7 +45,7 @@ object CometNot extends CometExpressionSerde[Not] { inputs, binding, (builder, binaryExpr) => builder.setNeq(binaryExpr)) - case inner: EqualNullSafe if !ComparisonUtils.hasCollatedOperand(inner.left, inner.right) => + case inner: EqualNullSafe if !hasCollatedOperand(inner.left, inner.right) => createBinaryExpr( inner, inner.left, @@ -51,7 +53,7 @@ object CometNot extends CometExpressionSerde[Not] { inputs, binding, (builder, binaryExpr) => builder.setNeqNullSafe(binaryExpr)) - case inner: In if !ComparisonUtils.hasCollatedOperand(inner.value +: inner.list) => + case inner: In if !hasCollatedOperand((inner.value +: inner.list): _*) => ComparisonUtils.in(inner, inner.value, inner.list, inputs, binding, negate = true) case _ => // Includes the collated variants of EqualTo / EqualNullSafe / In above: fall through so @@ -109,9 +111,7 @@ object CometOr extends CometExpressionSerde[Or] { } } -object CometEqualTo extends CometExpressionSerde[EqualTo] { - override def getSupportLevel(expr: EqualTo): SupportLevel = - ComparisonUtils.collationSupportLevel(expr.left, expr.right) +object CometEqualTo extends CollationAwareBinaryPredicate[EqualTo] { override def convert( expr: EqualTo, inputs: Seq[Attribute], @@ -126,9 +126,7 @@ object CometEqualTo extends CometExpressionSerde[EqualTo] { } } -object CometEqualNullSafe extends CometExpressionSerde[EqualNullSafe] { - override def getSupportLevel(expr: EqualNullSafe): SupportLevel = - ComparisonUtils.collationSupportLevel(expr.left, expr.right) +object CometEqualNullSafe extends CollationAwareBinaryPredicate[EqualNullSafe] { override def convert( expr: EqualNullSafe, inputs: Seq[Attribute], @@ -143,9 +141,7 @@ object CometEqualNullSafe extends CometExpressionSerde[EqualNullSafe] { } } -object CometGreaterThan extends CometExpressionSerde[GreaterThan] { - override def getSupportLevel(expr: GreaterThan): SupportLevel = - ComparisonUtils.collationSupportLevel(expr.left, expr.right) +object CometGreaterThan extends CollationAwareBinaryPredicate[GreaterThan] { override def convert( expr: GreaterThan, inputs: Seq[Attribute], @@ -160,9 +156,7 @@ object CometGreaterThan extends CometExpressionSerde[GreaterThan] { } } -object CometGreaterThanOrEqual extends CometExpressionSerde[GreaterThanOrEqual] { - override def getSupportLevel(expr: GreaterThanOrEqual): SupportLevel = - ComparisonUtils.collationSupportLevel(expr.left, expr.right) +object CometGreaterThanOrEqual extends CollationAwareBinaryPredicate[GreaterThanOrEqual] { override def convert( expr: GreaterThanOrEqual, inputs: Seq[Attribute], @@ -177,9 +171,7 @@ object CometGreaterThanOrEqual extends CometExpressionSerde[GreaterThanOrEqual] } } -object CometLessThan extends CometExpressionSerde[LessThan] { - override def getSupportLevel(expr: LessThan): SupportLevel = - ComparisonUtils.collationSupportLevel(expr.left, expr.right) +object CometLessThan extends CollationAwareBinaryPredicate[LessThan] { override def convert( expr: LessThan, inputs: Seq[Attribute], @@ -194,9 +186,7 @@ object CometLessThan extends CometExpressionSerde[LessThan] { } } -object CometLessThanOrEqual extends CometExpressionSerde[LessThanOrEqual] { - override def getSupportLevel(expr: LessThanOrEqual): SupportLevel = - ComparisonUtils.collationSupportLevel(expr.left, expr.right) +object CometLessThanOrEqual extends CollationAwareBinaryPredicate[LessThanOrEqual] { override def convert( expr: LessThanOrEqual, inputs: Seq[Attribute], @@ -253,13 +243,8 @@ object CometIsNaN extends CometExpressionSerde[IsNaN] { object CometIn extends CometExpressionSerde[In] { - override def getSupportLevel(expr: In): SupportLevel = { - if (ComparisonUtils.hasCollatedOperand(expr.value +: expr.list)) { - Unsupported(Some(ComparisonUtils.nonDefaultCollationReason("In"))) - } else { - Compatible() - } - } + override def getSupportLevel(expr: In): SupportLevel = + ComparisonUtils.collationSupportLevel("In", (expr.value +: expr.list): _*) override def convert( expr: In, @@ -271,13 +256,8 @@ object CometIn extends CometExpressionSerde[In] { object CometInSet extends CometExpressionSerde[InSet] { - override def getSupportLevel(expr: InSet): SupportLevel = { - if (ComparisonUtils.hasCollatedOperand(expr.child)) { - Unsupported(Some(ComparisonUtils.nonDefaultCollationReason("InSet"))) - } else { - Compatible() - } - } + override def getSupportLevel(expr: InSet): SupportLevel = + ComparisonUtils.collationSupportLevel("InSet", expr.child) override def convert( expr: InSet, @@ -293,6 +273,16 @@ object CometInSet extends CometExpressionSerde[InSet] { } } +/** + * Mixin for serdes of binary predicates whose native kernel compares raw bytes: any operand + * carrying a non-UTF8_BINARY collation must fall back to Spark, since byte-wise semantics diverge + * from Spark's collation-aware evaluation. + */ +trait CollationAwareBinaryPredicate[T <: BinaryExpression] extends CometExpressionSerde[T] { + override def getSupportLevel(expr: T): SupportLevel = + ComparisonUtils.collationSupportLevel(expr.prettyName, expr.left, expr.right) +} + object ComparisonUtils { // Comet's native equality/ordering/hashing compare raw bytes, so any predicate operand carrying @@ -308,14 +298,8 @@ object ComparisonUtils { def hasCollatedOperand(operands: Expression*): Boolean = operands.exists(op => hasNonDefaultStringCollation(op.dataType)) - def hasCollatedOperand(operands: Iterable[Expression]): Boolean = - operands.exists(op => hasNonDefaultStringCollation(op.dataType)) - - def collationSupportLevel( - left: Expression, - right: Expression, - exprName: String = "BinaryComparison"): SupportLevel = - if (hasCollatedOperand(left, right)) { + def collationSupportLevel(exprName: String, operands: Expression*): SupportLevel = + if (hasCollatedOperand(operands: _*)) { Unsupported(Some(nonDefaultCollationReason(exprName))) } else { Compatible() diff --git a/spark/src/main/scala/org/apache/comet/serde/strings.scala b/spark/src/main/scala/org/apache/comet/serde/strings.scala index cbbce64a9e..67ca9cbc66 100644 --- a/spark/src/main/scala/org/apache/comet/serde/strings.scala +++ b/spark/src/main/scala/org/apache/comet/serde/strings.scala @@ -312,34 +312,22 @@ object CometLike extends CometExpressionSerde[Like] { } /** - * Serdes for `Contains` / `StartsWith` / `EndsWith` that guard on non-UTF8_BINARY collation and - * otherwise delegate to the generic `contains` / `starts_with` / `ends_with` scalar-function + * Serdes for `Contains` / `StartsWith` / `EndsWith` that reject non-UTF8_BINARY collated operands + * and otherwise delegate to the generic `contains` / `starts_with` / `ends_with` scalar-function * bridge. The native kernels compare raw bytes and cannot honour case- or accent-insensitive * collations, so a collated operand must fall back to Spark. */ -object CometContains extends CometExpressionSerde[Contains] { - override def getSupportLevel(expr: Contains): SupportLevel = - ComparisonUtils.collationSupportLevel(expr.left, expr.right, "Contains") +object CometContains + extends CometScalarFunction[Contains]("contains") + with CollationAwareBinaryPredicate[Contains] - override def convert(expr: Contains, inputs: Seq[Attribute], binding: Boolean): Option[Expr] = - CometScalarFunction[Contains]("contains").convert(expr, inputs, binding) -} - -object CometStartsWith extends CometExpressionSerde[StartsWith] { - override def getSupportLevel(expr: StartsWith): SupportLevel = - ComparisonUtils.collationSupportLevel(expr.left, expr.right, "StartsWith") - - override def convert(expr: StartsWith, inputs: Seq[Attribute], binding: Boolean): Option[Expr] = - CometScalarFunction[StartsWith]("starts_with").convert(expr, inputs, binding) -} - -object CometEndsWith extends CometExpressionSerde[EndsWith] { - override def getSupportLevel(expr: EndsWith): SupportLevel = - ComparisonUtils.collationSupportLevel(expr.left, expr.right, "EndsWith") +object CometStartsWith + extends CometScalarFunction[StartsWith]("starts_with") + with CollationAwareBinaryPredicate[StartsWith] - override def convert(expr: EndsWith, inputs: Seq[Attribute], binding: Boolean): Option[Expr] = - CometScalarFunction[EndsWith]("ends_with").convert(expr, inputs, binding) -} +object CometEndsWith + extends CometScalarFunction[EndsWith]("ends_with") + with CollationAwareBinaryPredicate[EndsWith] /** * `rlike` runs Spark's own implementation through the codegen dispatcher by default, for From 34416c968e5c8cde09a63d7a64be002284d0b866 Mon Sep 17 00:00:00 2001 From: comphead Date: Fri, 17 Jul 2026 12:32:53 -0700 Subject: [PATCH 9/9] feat: route collated predicates through JVM codegen dispatcher Addresses two review comments on #4948. CollationAwareBinaryPredicate, CometIn, CometInSet, and CometLike now mix in CodegenDispatchFallback, so an `Unsupported` result from a collated operand stays in the Comet pipeline via Spark's own `doGenCode` rather than falling the whole projection or filter back to Spark. CometContains / CometStartsWith / CometEndsWith inherit the mixin through the trait. Collation semantics still come from Spark's codegen path, so answers stay Spark-correct. Each of these serdes now overrides `getUnsupportedReasons` so the collation case (and, for Like, the custom-escape case) is picked up by GenerateDocs and surfaces in the compatibility guide instead of being undocumented. The SQL tests in `collation.sql` switch from `expect_fallback(non-UTF8_BINARY collated operands)` back to plain `query`: with the dispatcher taking over, the operator stays native and the fallback-coverage assertion catches any regression that would force it back to Spark. --- .../org/apache/comet/serde/predicates.scala | 36 ++++++++--- .../org/apache/comet/serde/strings.scala | 8 ++- .../expressions/string/collation.sql | 59 ++++++++++--------- 3 files changed, 65 insertions(+), 38 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/predicates.scala b/spark/src/main/scala/org/apache/comet/serde/predicates.scala index 162f5bea7f..c9a70a830f 100644 --- a/spark/src/main/scala/org/apache/comet/serde/predicates.scala +++ b/spark/src/main/scala/org/apache/comet/serde/predicates.scala @@ -241,11 +241,14 @@ object CometIsNaN extends CometExpressionSerde[IsNaN] { } } -object CometIn extends CometExpressionSerde[In] { +object CometIn extends CometExpressionSerde[In] with CodegenDispatchFallback { override def getSupportLevel(expr: In): SupportLevel = ComparisonUtils.collationSupportLevel("In", (expr.value +: expr.list): _*) + override def getUnsupportedReasons(): Seq[String] = + Seq(ComparisonUtils.nonDefaultCollationDocReason) + override def convert( expr: In, inputs: Seq[Attribute], @@ -254,11 +257,14 @@ object CometIn extends CometExpressionSerde[In] { } } -object CometInSet extends CometExpressionSerde[InSet] { +object CometInSet extends CometExpressionSerde[InSet] with CodegenDispatchFallback { override def getSupportLevel(expr: InSet): SupportLevel = ComparisonUtils.collationSupportLevel("InSet", expr.child) + override def getUnsupportedReasons(): Seq[String] = + Seq(ComparisonUtils.nonDefaultCollationDocReason) + override def convert( expr: InSet, inputs: Seq[Attribute], @@ -275,12 +281,19 @@ object CometInSet extends CometExpressionSerde[InSet] { /** * Mixin for serdes of binary predicates whose native kernel compares raw bytes: any operand - * carrying a non-UTF8_BINARY collation must fall back to Spark, since byte-wise semantics diverge - * from Spark's collation-aware evaluation. + * carrying a non-UTF8_BINARY collation is unsupported natively. Mixing in + * `CodegenDispatchFallback` routes these through the JVM codegen dispatcher (Spark's own + * `doGenCode`), so the collation-aware evaluation runs inline in the Comet pipeline instead of + * falling the whole operator back to Spark. */ -trait CollationAwareBinaryPredicate[T <: BinaryExpression] extends CometExpressionSerde[T] { +trait CollationAwareBinaryPredicate[T <: BinaryExpression] + extends CometExpressionSerde[T] + with CodegenDispatchFallback { override def getSupportLevel(expr: T): SupportLevel = ComparisonUtils.collationSupportLevel(expr.prettyName, expr.left, expr.right) + + override def getUnsupportedReasons(): Seq[String] = + Seq(ComparisonUtils.nonDefaultCollationDocReason) } object ComparisonUtils { @@ -288,13 +301,20 @@ object ComparisonUtils { // Comet's native equality/ordering/hashing compare raw bytes, so any predicate operand carrying // a non-UTF8_BINARY collation (Spark 4+) would produce wrong answers on the native path — e.g. // `'a' = 'A'` under `UNICODE_CI` returns true in Spark but false byte-wise. Every binary - // comparison and `In`/`InSet` serde routes its `getSupportLevel` through here so that any - // collated operand triggers a clean fallback to Spark. `hasNonDefaultStringCollation` walks - // nested types too, so collated strings inside array/map/struct operands are also caught. + // comparison and `In`/`InSet` serde routes its `getSupportLevel` through here, marking the case + // `Unsupported`; `CodegenDispatchFallback` then routes those cases through Spark's own + // `doGenCode` inside the Comet pipeline. `hasNonDefaultStringCollation` walks nested types too, + // so collated strings inside array/map/struct operands are also caught. def nonDefaultCollationReason(exprName: String): String = s"$exprName does not support non-UTF8_BINARY collated operands; " + "native comparison is byte-wise and cannot honour collation semantics." + // Doc-friendly variant of `nonDefaultCollationReason` for `getUnsupportedReasons`. The compat + // guide already scopes the reason to a specific expression, so no per-expr name is needed. + val nonDefaultCollationDocReason: String = + "Non-UTF8_BINARY collated operands are routed through the JVM codegen dispatcher " + + "(Spark's own `doGenCode`) because native comparison is byte-wise." + def hasCollatedOperand(operands: Expression*): Boolean = operands.exists(op => hasNonDefaultStringCollation(op.dataType)) diff --git a/spark/src/main/scala/org/apache/comet/serde/strings.scala b/spark/src/main/scala/org/apache/comet/serde/strings.scala index 67ca9cbc66..1e124dbd42 100644 --- a/spark/src/main/scala/org/apache/comet/serde/strings.scala +++ b/spark/src/main/scala/org/apache/comet/serde/strings.scala @@ -288,7 +288,13 @@ object CometConcatWs extends CometExpressionSerde[ConcatWs] { } } -object CometLike extends CometExpressionSerde[Like] { +object CometLike extends CometExpressionSerde[Like] with CodegenDispatchFallback { + + private val customEscapeReason = + "LIKE with a custom escape character (only `\\` is supported natively)" + + override def getUnsupportedReasons(): Seq[String] = + Seq(customEscapeReason, ComparisonUtils.nonDefaultCollationDocReason) override def getSupportLevel(expr: Like): SupportLevel = { if (ComparisonUtils.hasCollatedOperand(expr.left, expr.right)) { diff --git a/spark/src/test/resources/sql-tests/expressions/string/collation.sql b/spark/src/test/resources/sql-tests/expressions/string/collation.sql index a5af351a94..534725b83f 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/collation.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/collation.sql @@ -53,9 +53,10 @@ query SELECT reverse('Hello' COLLATE UNICODE_CI) -- ============================================================================ --- Predicate fallback for non-UTF8_BINARY collated operands. Comet's native --- kernels are byte-wise, so any collated operand must fall back to Spark; --- expect_fallback(...) pins the shared reason substring. +-- Collated predicate operands route through the JVM codegen dispatcher. +-- The predicate serdes mark collated cases `Unsupported`; `CodegenDispatchFallback` runs +-- Spark's own `doGenCode` inside the Comet pipeline so the operator stays native while +-- matching Spark's collation-aware answer. -- ============================================================================ statement @@ -73,70 +74,70 @@ INSERT INTO test_collated_predicates VALUES -- ---------- EqualTo / EqualNullSafe / inequality --------------------------- -query expect_fallback(non-UTF8_BINARY collated operands) +query SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) = 'A' FROM test_collated_predicates ORDER BY id -query expect_fallback(non-UTF8_BINARY collated operands) +query SELECT id FROM test_collated_predicates WHERE CAST(s AS STRING COLLATE UTF8_LCASE) = 'A' ORDER BY id -- UNICODE_CI covers a second ICU collation to prove the guard is not UTF8_LCASE-only. -query expect_fallback(non-UTF8_BINARY collated operands) +query SELECT id, CAST(s AS STRING COLLATE UNICODE_CI) = 'A' FROM test_collated_predicates ORDER BY id -query expect_fallback(non-UTF8_BINARY collated operands) +query SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) != 'A' FROM test_collated_predicates ORDER BY id -query expect_fallback(non-UTF8_BINARY collated operands) +query SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) <> 'A' FROM test_collated_predicates ORDER BY id -- `x <=> NULL` folds via NullPropagation, so we test only a non-NULL RHS. -query expect_fallback(non-UTF8_BINARY collated operands) +query SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) <=> 'A' FROM test_collated_predicates ORDER BY id -- ---------- Ordering ------------------------------------------------------ -query expect_fallback(non-UTF8_BINARY collated operands) +query SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) < 'b' FROM test_collated_predicates ORDER BY id -query expect_fallback(non-UTF8_BINARY collated operands) +query SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) <= 'a' FROM test_collated_predicates ORDER BY id -query expect_fallback(non-UTF8_BINARY collated operands) +query SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) > 'a' FROM test_collated_predicates ORDER BY id -query expect_fallback(non-UTF8_BINARY collated operands) +query SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) >= 'A' FROM test_collated_predicates ORDER BY id -query expect_fallback(non-UTF8_BINARY collated operands) +query SELECT id, CAST(s AS STRING COLLATE UNICODE_CI) < 'b' FROM test_collated_predicates ORDER BY id -- ---------- In / NotIn / InSet --------------------------------------------- -query expect_fallback(non-UTF8_BINARY collated operands) +query SELECT id FROM test_collated_predicates WHERE CAST(s AS STRING COLLATE UTF8_LCASE) IN ('A', 'HELLO') ORDER BY id -- NOT IN exercises CometNot's collated fall-through into the generic path. -query expect_fallback(non-UTF8_BINARY collated operands) +query SELECT id FROM test_collated_predicates WHERE CAST(s AS STRING COLLATE UTF8_LCASE) NOT IN ('A', 'HELLO') ORDER BY id -- 12 elements forces InSet (spark.sql.optimizer.inSetConversionThreshold=10 by default). -query expect_fallback(non-UTF8_BINARY collated operands) +query SELECT id FROM test_collated_predicates WHERE CAST(s AS STRING COLLATE UTF8_LCASE) IN ('A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'HELLO') ORDER BY id -- ---------- NOT (EqualTo / EqualNullSafe) rewrites ------------------------ -query expect_fallback(non-UTF8_BINARY collated operands) +query SELECT id FROM test_collated_predicates WHERE NOT (CAST(s AS STRING COLLATE UTF8_LCASE) = 'A') ORDER BY id -query expect_fallback(non-UTF8_BINARY collated operands) +query SELECT id FROM test_collated_predicates WHERE NOT (CAST(s AS STRING COLLATE UTF8_LCASE) <=> 'A') ORDER BY id -- ---------- Like ----------------------------------------------------------- -query expect_fallback(non-UTF8_BINARY collated operands) +query SELECT id, CAST(s AS STRING COLLATE UTF8_LCASE) LIKE '%LLO' FROM test_collated_predicates ORDER BY id -query expect_fallback(non-UTF8_BINARY collated operands) +query SELECT id FROM test_collated_predicates WHERE CAST(s AS STRING COLLATE UTF8_LCASE) LIKE 'H%' ORDER BY id -- Spark 4.0 rejects UNICODE_CI for LIKE / Contains / StartsWith / EndsWith at analysis time @@ -144,31 +145,31 @@ SELECT id FROM test_collated_predicates WHERE CAST(s AS STRING COLLATE UTF8_LCAS -- ---------- Contains / StartsWith / EndsWith ------------------------------- -query expect_fallback(non-UTF8_BINARY collated operands) +query SELECT id, contains(CAST(s AS STRING COLLATE UTF8_LCASE), 'HE') FROM test_collated_predicates ORDER BY id -query expect_fallback(non-UTF8_BINARY collated operands) +query SELECT id, startswith(CAST(s AS STRING COLLATE UTF8_LCASE), 'HE') FROM test_collated_predicates ORDER BY id -query expect_fallback(non-UTF8_BINARY collated operands) +query SELECT id, endswith(CAST(s AS STRING COLLATE UTF8_LCASE), 'LO') FROM test_collated_predicates ORDER BY id -query expect_fallback(non-UTF8_BINARY collated operands) +query SELECT id FROM test_collated_predicates WHERE contains(CAST(s AS STRING COLLATE UTF8_LCASE), 'ELL') ORDER BY id -query expect_fallback(non-UTF8_BINARY collated operands) +query SELECT id FROM test_collated_predicates WHERE startswith(CAST(s AS STRING COLLATE UTF8_LCASE), 'HE') ORDER BY id -query expect_fallback(non-UTF8_BINARY collated operands) +query SELECT id FROM test_collated_predicates WHERE endswith(CAST(s AS STRING COLLATE UTF8_LCASE), 'O') ORDER BY id -- ---------- Nested-collated types ------------------------------------------ -- hasNonDefaultStringCollation recurses into array/struct element types. -query expect_fallback(non-UTF8_BINARY collated operands) +query SELECT id, array(CAST(s AS STRING COLLATE UTF8_LCASE)) = array('A' COLLATE UTF8_LCASE) FROM test_collated_predicates ORDER BY id -query expect_fallback(non-UTF8_BINARY collated operands) +query SELECT id, struct(CAST(s AS STRING COLLATE UTF8_LCASE)) = struct('A' COLLATE UTF8_LCASE) FROM test_collated_predicates ORDER BY id -- NULL-literal operands are omitted: NullPropagation folds them before Comet sees them. The