From 768b3e90f261c7aea58bdb98dc698b90deeeae34 Mon Sep 17 00:00:00 2001 From: Kazantsev Maksim Date: Sun, 14 Dec 2025 16:24:01 +0400 Subject: [PATCH 1/6] impl map_from_entries --- native/core/src/execution/jni_api.rs | 2 + .../apache/comet/serde/QueryPlanSerde.scala | 3 +- .../scala/org/apache/comet/serde/maps.scala | 29 +++++++++++- .../comet/CometMapExpressionSuite.scala | 45 +++++++++++++++++++ 4 files changed, 77 insertions(+), 2 deletions(-) diff --git a/native/core/src/execution/jni_api.rs b/native/core/src/execution/jni_api.rs index a24d993059..4f53cea3e6 100644 --- a/native/core/src/execution/jni_api.rs +++ b/native/core/src/execution/jni_api.rs @@ -46,6 +46,7 @@ use datafusion_spark::function::datetime::date_add::SparkDateAdd; use datafusion_spark::function::datetime::date_sub::SparkDateSub; use datafusion_spark::function::hash::sha1::SparkSha1; use datafusion_spark::function::hash::sha2::SparkSha2; +use datafusion_spark::function::map::map_from_entries::MapFromEntries; use datafusion_spark::function::math::expm1::SparkExpm1; use datafusion_spark::function::string::char::CharFunc; use datafusion_spark::function::string::concat::SparkConcat; @@ -337,6 +338,7 @@ fn register_datafusion_spark_function(session_ctx: &SessionContext) { session_ctx.register_udf(ScalarUDF::new_from_impl(SparkSha1::default())); session_ctx.register_udf(ScalarUDF::new_from_impl(SparkConcat::default())); session_ctx.register_udf(ScalarUDF::new_from_impl(SparkBitwiseNot::default())); + session_ctx.register_udf(ScalarUDF::new_from_impl(MapFromEntries::default())); } /// Prepares arrow arrays for output. diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 54df2f1688..a99cf3824b 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -125,7 +125,8 @@ object QueryPlanSerde extends Logging with CometExprShim { classOf[MapKeys] -> CometMapKeys, classOf[MapEntries] -> CometMapEntries, classOf[MapValues] -> CometMapValues, - classOf[MapFromArrays] -> CometMapFromArrays) + classOf[MapFromArrays] -> CometMapFromArrays, + classOf[MapFromEntries] -> CometMapFromEntries) private val structExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = Map( classOf[CreateNamedStruct] -> CometCreateNamedStruct, diff --git a/spark/src/main/scala/org/apache/comet/serde/maps.scala b/spark/src/main/scala/org/apache/comet/serde/maps.scala index 2e217f6af0..498aa3594c 100644 --- a/spark/src/main/scala/org/apache/comet/serde/maps.scala +++ b/spark/src/main/scala/org/apache/comet/serde/maps.scala @@ -19,9 +19,12 @@ package org.apache.comet.serde +import scala.annotation.tailrec + import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.types.{ArrayType, MapType} +import org.apache.spark.sql.types.{ArrayType, BinaryType, DataType, MapType, StructType} +import org.apache.comet.serde.CometArrayReverse.containsBinary import org.apache.comet.serde.QueryPlanSerde.{exprToProtoInternal, optExprWithInfo, scalarFunctionExprToProto, scalarFunctionExprToProtoWithReturnType} object CometMapKeys extends CometExpressionSerde[MapKeys] { @@ -89,3 +92,27 @@ object CometMapFromArrays extends CometExpressionSerde[MapFromArrays] { optExprWithInfo(mapFromArraysExpr, expr, expr.children: _*) } } + +object CometMapFromEntries extends CometScalarFunction[MapFromEntries]("map_from_entries") { + val keyUnsupportedReason = "Using BinaryType as Map keys is not allowed in map_from_entries" + val valueUnsupportedReason = "Using BinaryType as Map values is not allowed in map_from_entries" + + private def containsBinary(dataType: DataType): Boolean = { + dataType match { + case BinaryType => true + case StructType(fields) => fields.exists(field => containsBinary(field.dataType)) + case ArrayType(elementType, _) => containsBinary(elementType) + case _ => false + } + } + + override def getSupportLevel(expr: MapFromEntries): SupportLevel = { + if (containsBinary(expr.dataType.keyType)) { + return Incompatible(Some(keyUnsupportedReason)) + } + if (containsBinary(expr.dataType.valueType)) { + return Incompatible(Some(valueUnsupportedReason)) + } + Compatible(None) + } +} diff --git a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala index 88c13391a6..01b9744ed6 100644 --- a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala @@ -25,7 +25,9 @@ import org.apache.hadoop.fs.Path import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.types.BinaryType +import org.apache.comet.serde.CometMapFromEntries import org.apache.comet.testing.{DataGenOptions, ParquetGenerator, SchemaGenOptions} class CometMapExpressionSuite extends CometTestBase { @@ -125,4 +127,47 @@ class CometMapExpressionSuite extends CometTestBase { } } + test("map_from_entries") { + withTempDir { dir => + val path = new Path(dir.toURI.toString, "test.parquet") + val filename = path.toString + val random = new Random(42) + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + val schemaGenOptions = + SchemaGenOptions( + generateArray = true, + generateStruct = true, + primitiveTypes = SchemaGenOptions.defaultPrimitiveTypes.filterNot(_ == BinaryType)) + val dataGenOptions = DataGenOptions(allowNull = false, generateNegativeZero = false) + ParquetGenerator.makeParquetFile( + random, + spark, + filename, + 100, + schemaGenOptions, + dataGenOptions) + } + val df = spark.read.parquet(filename) + df.createOrReplaceTempView("t1") + for (field <- df.schema.fieldNames) { + checkSparkAnswerAndOperator( + spark.sql(s"SELECT map_from_entries(array(struct($field as a, $field as b))) FROM t1")) + } + } + } + + test("map_from_entries - fallback for binary type") { + val table = "t2" + withTable(table) { + sql( + s"create table $table using parquet as select cast(array() as array) as c1 from range(10)") + checkSparkAnswerAndFallbackReason( + sql(s"select map_from_entries(array(struct(c1, 0))) from $table"), + CometMapFromEntries.keyUnsupportedReason) + checkSparkAnswerAndFallbackReason( + sql(s"select map_from_entries(array(struct(0, c1))) from $table"), + CometMapFromEntries.valueUnsupportedReason) + } + } + } From c68c3428676b5d991e7ba9e13464bf2ce1ec84e8 Mon Sep 17 00:00:00 2001 From: Kazantsev Maksim Date: Tue, 16 Dec 2025 16:10:43 +0400 Subject: [PATCH 2/6] Revert "impl map_from_entries" This reverts commit 768b3e90f261c7aea58bdb98dc698b90deeeae34. --- native/core/src/execution/jni_api.rs | 2 - .../apache/comet/serde/QueryPlanSerde.scala | 3 +- .../scala/org/apache/comet/serde/maps.scala | 29 +----------- .../comet/CometMapExpressionSuite.scala | 45 ------------------- 4 files changed, 2 insertions(+), 77 deletions(-) diff --git a/native/core/src/execution/jni_api.rs b/native/core/src/execution/jni_api.rs index 4f53cea3e6..a24d993059 100644 --- a/native/core/src/execution/jni_api.rs +++ b/native/core/src/execution/jni_api.rs @@ -46,7 +46,6 @@ use datafusion_spark::function::datetime::date_add::SparkDateAdd; use datafusion_spark::function::datetime::date_sub::SparkDateSub; use datafusion_spark::function::hash::sha1::SparkSha1; use datafusion_spark::function::hash::sha2::SparkSha2; -use datafusion_spark::function::map::map_from_entries::MapFromEntries; use datafusion_spark::function::math::expm1::SparkExpm1; use datafusion_spark::function::string::char::CharFunc; use datafusion_spark::function::string::concat::SparkConcat; @@ -338,7 +337,6 @@ fn register_datafusion_spark_function(session_ctx: &SessionContext) { session_ctx.register_udf(ScalarUDF::new_from_impl(SparkSha1::default())); session_ctx.register_udf(ScalarUDF::new_from_impl(SparkConcat::default())); session_ctx.register_udf(ScalarUDF::new_from_impl(SparkBitwiseNot::default())); - session_ctx.register_udf(ScalarUDF::new_from_impl(MapFromEntries::default())); } /// Prepares arrow arrays for output. diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index a99cf3824b..54df2f1688 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -125,8 +125,7 @@ object QueryPlanSerde extends Logging with CometExprShim { classOf[MapKeys] -> CometMapKeys, classOf[MapEntries] -> CometMapEntries, classOf[MapValues] -> CometMapValues, - classOf[MapFromArrays] -> CometMapFromArrays, - classOf[MapFromEntries] -> CometMapFromEntries) + classOf[MapFromArrays] -> CometMapFromArrays) private val structExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = Map( classOf[CreateNamedStruct] -> CometCreateNamedStruct, diff --git a/spark/src/main/scala/org/apache/comet/serde/maps.scala b/spark/src/main/scala/org/apache/comet/serde/maps.scala index 498aa3594c..2e217f6af0 100644 --- a/spark/src/main/scala/org/apache/comet/serde/maps.scala +++ b/spark/src/main/scala/org/apache/comet/serde/maps.scala @@ -19,12 +19,9 @@ package org.apache.comet.serde -import scala.annotation.tailrec - import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.types.{ArrayType, BinaryType, DataType, MapType, StructType} +import org.apache.spark.sql.types.{ArrayType, MapType} -import org.apache.comet.serde.CometArrayReverse.containsBinary import org.apache.comet.serde.QueryPlanSerde.{exprToProtoInternal, optExprWithInfo, scalarFunctionExprToProto, scalarFunctionExprToProtoWithReturnType} object CometMapKeys extends CometExpressionSerde[MapKeys] { @@ -92,27 +89,3 @@ object CometMapFromArrays extends CometExpressionSerde[MapFromArrays] { optExprWithInfo(mapFromArraysExpr, expr, expr.children: _*) } } - -object CometMapFromEntries extends CometScalarFunction[MapFromEntries]("map_from_entries") { - val keyUnsupportedReason = "Using BinaryType as Map keys is not allowed in map_from_entries" - val valueUnsupportedReason = "Using BinaryType as Map values is not allowed in map_from_entries" - - private def containsBinary(dataType: DataType): Boolean = { - dataType match { - case BinaryType => true - case StructType(fields) => fields.exists(field => containsBinary(field.dataType)) - case ArrayType(elementType, _) => containsBinary(elementType) - case _ => false - } - } - - override def getSupportLevel(expr: MapFromEntries): SupportLevel = { - if (containsBinary(expr.dataType.keyType)) { - return Incompatible(Some(keyUnsupportedReason)) - } - if (containsBinary(expr.dataType.valueType)) { - return Incompatible(Some(valueUnsupportedReason)) - } - Compatible(None) - } -} diff --git a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala index 01b9744ed6..88c13391a6 100644 --- a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala @@ -25,9 +25,7 @@ import org.apache.hadoop.fs.Path import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.BinaryType -import org.apache.comet.serde.CometMapFromEntries import org.apache.comet.testing.{DataGenOptions, ParquetGenerator, SchemaGenOptions} class CometMapExpressionSuite extends CometTestBase { @@ -127,47 +125,4 @@ class CometMapExpressionSuite extends CometTestBase { } } - test("map_from_entries") { - withTempDir { dir => - val path = new Path(dir.toURI.toString, "test.parquet") - val filename = path.toString - val random = new Random(42) - withSQLConf(CometConf.COMET_ENABLED.key -> "false") { - val schemaGenOptions = - SchemaGenOptions( - generateArray = true, - generateStruct = true, - primitiveTypes = SchemaGenOptions.defaultPrimitiveTypes.filterNot(_ == BinaryType)) - val dataGenOptions = DataGenOptions(allowNull = false, generateNegativeZero = false) - ParquetGenerator.makeParquetFile( - random, - spark, - filename, - 100, - schemaGenOptions, - dataGenOptions) - } - val df = spark.read.parquet(filename) - df.createOrReplaceTempView("t1") - for (field <- df.schema.fieldNames) { - checkSparkAnswerAndOperator( - spark.sql(s"SELECT map_from_entries(array(struct($field as a, $field as b))) FROM t1")) - } - } - } - - test("map_from_entries - fallback for binary type") { - val table = "t2" - withTable(table) { - sql( - s"create table $table using parquet as select cast(array() as array) as c1 from range(10)") - checkSparkAnswerAndFallbackReason( - sql(s"select map_from_entries(array(struct(c1, 0))) from $table"), - CometMapFromEntries.keyUnsupportedReason) - checkSparkAnswerAndFallbackReason( - sql(s"select map_from_entries(array(struct(0, c1))) from $table"), - CometMapFromEntries.valueUnsupportedReason) - } - } - } From e0a02bf32f85575ffbe95eb28096acc5710d5589 Mon Sep 17 00:00:00 2001 From: Kazantsev Maksim Date: Wed, 1 Apr 2026 21:07:39 +0400 Subject: [PATCH 3/6] Feat: support Infinity/-Infinity Nan values for numeric types --- native/spark-expr/src/json_funcs/to_json.rs | 10 ++- .../org/apache/comet/serde/structs.scala | 66 +++++++------------ .../comet/CometJsonExpressionSuite.scala | 2 +- 3 files changed, 35 insertions(+), 43 deletions(-) diff --git a/native/spark-expr/src/json_funcs/to_json.rs b/native/spark-expr/src/json_funcs/to_json.rs index 3cc827f210..b8153e78e3 100644 --- a/native/spark-expr/src/json_funcs/to_json.rs +++ b/native/spark-expr/src/json_funcs/to_json.rs @@ -235,7 +235,7 @@ fn struct_to_json(array: &StructArray, timezone: &str) -> Result { json.push_str("\":"); // value let string_value = string_arrays[col_index].value(row_index); - if is_string[col_index] { + if is_string[col_index] || is_infinity(string_value) || is_nan(string_value) { json.push('"'); json.push_str(&escape_string(string_value)); json.push('"'); @@ -252,6 +252,14 @@ fn struct_to_json(array: &StructArray, timezone: &str) -> Result { Ok(Arc::new(builder.finish())) } +fn is_infinity(input: &str) -> bool { + input == "Infinity" || input == "-Infinity" +} + +fn is_nan(input: &str) -> bool { + input == "NaN" +} + #[cfg(test)] mod test { use crate::json_funcs::to_json::struct_to_json; diff --git a/spark/src/main/scala/org/apache/comet/serde/structs.scala b/spark/src/main/scala/org/apache/comet/serde/structs.scala index 449d0fc5b9..e16a9a3a77 100644 --- a/spark/src/main/scala/org/apache/comet/serde/structs.scala +++ b/spark/src/main/scala/org/apache/comet/serde/structs.scala @@ -105,53 +105,37 @@ object CometGetArrayStructFields extends CometExpressionSerde[GetArrayStructFiel object CometStructsToJson extends CometExpressionSerde[StructsToJson] { - override def getSupportLevel(expr: StructsToJson): SupportLevel = - Incompatible( - Some( - "Does not support Infinity/-Infinity for numeric types" + - " (https://github.com/apache/datafusion-comet/issues/3016)")) + override def getSupportLevel(expr: StructsToJson): SupportLevel = { + if (expr.options.nonEmpty) { + Unsupported(Some("StructsToJson with options is not supported")) + } + val dataType = expr.child.dataType + if (!isSupportedType(dataType)) { + Unsupported(Some(s"Struct type: $dataType contains unsupported types")) + } + Compatible() + } override def convert( expr: StructsToJson, inputs: Seq[Attribute], binding: Boolean): Option[ExprOuterClass.Expr] = { - if (expr.options.nonEmpty) { - withInfo(expr, "StructsToJson with options is not supported") - None - } else { - val isSupported = expr.child.dataType match { - case s: StructType => - s.fields.forall(f => isSupportedType(f.dataType)) - case _: MapType | _: ArrayType => - // Spark supports map and array in StructsToJson but this is not yet - // implemented in Comet - false - case _ => - false - } - - if (isSupported) { - exprToProtoInternal(expr.child, inputs, binding) match { - case Some(p) => - val toJson = ExprOuterClass.ToJson - .newBuilder() - .setChild(p) - .setTimezone(expr.timeZoneId.getOrElse("UTC")) - .setIgnoreNullFields(true) - .build() - Some( - ExprOuterClass.Expr - .newBuilder() - .setToJson(toJson) - .build()) - case _ => - withInfo(expr, expr.child) - None - } - } else { - withInfo(expr, "Unsupported data type", expr.child) + exprToProtoInternal(expr.child, inputs, binding) match { + case Some(p) => + val toJson = ExprOuterClass.ToJson + .newBuilder() + .setChild(p) + .setTimezone(expr.timeZoneId.getOrElse("UTC")) + .setIgnoreNullFields(true) + .build() + Some( + ExprOuterClass.Expr + .newBuilder() + .setToJson(toJson) + .build()) + case _ => + withInfo(expr, expr.child) None - } } } diff --git a/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala index 64c330dbdd..9c54e81e75 100644 --- a/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala @@ -60,7 +60,7 @@ class CometJsonExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelpe filename, 100, SchemaGenOptions(generateArray = false, generateStruct = false, generateMap = false), - DataGenOptions(generateNaN = false, generateInfinity = false)) + DataGenOptions(generateNaN = true, generateInfinity = true)) } val table = spark.read.parquet(filename) val fieldsNames = table.schema.fields From c32201476952edd5c5cc4277ff307f39da4a8104 Mon Sep 17 00:00:00 2001 From: Kazantsev Maksim Date: Thu, 2 Apr 2026 19:21:36 +0400 Subject: [PATCH 4/6] Fix PR issues --- spark/src/main/scala/org/apache/comet/serde/structs.scala | 4 ++-- .../sql-tests/expressions/struct/structs_to_json.sql | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/structs.scala b/spark/src/main/scala/org/apache/comet/serde/structs.scala index e16a9a3a77..43b999a7d1 100644 --- a/spark/src/main/scala/org/apache/comet/serde/structs.scala +++ b/spark/src/main/scala/org/apache/comet/serde/structs.scala @@ -107,11 +107,11 @@ object CometStructsToJson extends CometExpressionSerde[StructsToJson] { override def getSupportLevel(expr: StructsToJson): SupportLevel = { if (expr.options.nonEmpty) { - Unsupported(Some("StructsToJson with options is not supported")) + return Unsupported(Some("StructsToJson with options is not supported")) } val dataType = expr.child.dataType if (!isSupportedType(dataType)) { - Unsupported(Some(s"Struct type: $dataType contains unsupported types")) + return Unsupported(Some(s"Struct type: $dataType contains unsupported types")) } Compatible() } diff --git a/spark/src/test/resources/sql-tests/expressions/struct/structs_to_json.sql b/spark/src/test/resources/sql-tests/expressions/struct/structs_to_json.sql index 7f2310f147..9107f77426 100644 --- a/spark/src/test/resources/sql-tests/expressions/struct/structs_to_json.sql +++ b/spark/src/test/resources/sql-tests/expressions/struct/structs_to_json.sql @@ -29,3 +29,9 @@ SELECT to_json(named_struct('a', a, 'b', b)) FROM test_to_json -- literal arguments query spark_answer_only SELECT to_json(named_struct('a', 1, 'b', 'hello')) + +query expect_fallback(StructsToJson with options is not supported) +SELECT to_json(named_struct('a', 1, 'b', 'hello'), map('timestampFormat', 'dd/MM/yyyy')) + +query expect_fallback(Struct type: StructType(StructField(a,ArrayType(IntegerType,false),false)) contains unsupported types) +SELECT to_json(named_struct('a', array(1))) From 2dce7274b28dc2ab0ba2f24e449288c7f755c1ae Mon Sep 17 00:00:00 2001 From: Kazantsev Maksim Date: Fri, 3 Apr 2026 21:34:05 +0400 Subject: [PATCH 5/6] Fix tests --- .../expressions/struct/structs_to_json.sql | 10 ++++++---- .../apache/comet/CometJsonExpressionSuite.scala | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/spark/src/test/resources/sql-tests/expressions/struct/structs_to_json.sql b/spark/src/test/resources/sql-tests/expressions/struct/structs_to_json.sql index 9107f77426..776d11241e 100644 --- a/spark/src/test/resources/sql-tests/expressions/struct/structs_to_json.sql +++ b/spark/src/test/resources/sql-tests/expressions/struct/structs_to_json.sql @@ -30,8 +30,10 @@ SELECT to_json(named_struct('a', a, 'b', b)) FROM test_to_json query spark_answer_only SELECT to_json(named_struct('a', 1, 'b', 'hello')) -query expect_fallback(StructsToJson with options is not supported) -SELECT to_json(named_struct('a', 1, 'b', 'hello'), map('timestampFormat', 'dd/MM/yyyy')) +-- query expect_fallback(StructsToJson with options is not supported) +query ignore("Need support Spark 4.0.0") +SELECT to_json(named_struct('a', a, 'b', b), map('timestampFormat', 'dd/MM/yyyy')) -query expect_fallback(Struct type: StructType(StructField(a,ArrayType(IntegerType,false),false)) contains unsupported types) -SELECT to_json(named_struct('a', array(1))) +-- query expect_fallback(Struct type: StructType(StructField(a,ArrayType(IntegerType,false),false)) contains unsupported types) +query ignore("Need support Spark 4.0.0") +SELECT to_json(named_struct(a, array(b))) diff --git a/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala index 9c54e81e75..67742c0cff 100644 --- a/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala @@ -72,6 +72,22 @@ class CometJsonExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelpe } } + test("to_json - fallback reasons") { + assume(!isSpark40Plus) + withTable("t") { + sql("CREATE TABLE t(a INT, b STRING) USING parquet") + sql("INSERT INTO t VALUES (1, 'hello')") + checkSparkAnswerAndFallbackReason( + "SELECT to_json(named_struct('a', a, 'b', b), map('timestampFormat', 'dd/MM/yyyy')) FROM t", + "StructsToJson with options is not supported") + checkSparkAnswerAndFallbackReason( + "SELECT to_json(named_struct('a', a, 'b', array(b))) FROM t", + "Struct type: StructType(StructField(a,IntegerType,true)," + + "StructField(b,ArrayType(StringType,true),false)) " + + "contains unsupported types") + } + } + test("from_json - basic primitives") { Seq(true, false).foreach { dictionaryEnabled => withParquetTable( From c7bf49f1a3565eb2c422134923c9d0464a6c64df Mon Sep 17 00:00:00 2001 From: Kazantsev Maksim Date: Fri, 3 Apr 2026 21:35:39 +0400 Subject: [PATCH 6/6] Fix tests --- .../scala/org/apache/comet/CometJsonExpressionSuite.scala | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala index 67742c0cff..50a3c0c79f 100644 --- a/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala @@ -81,10 +81,8 @@ class CometJsonExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelpe "SELECT to_json(named_struct('a', a, 'b', b), map('timestampFormat', 'dd/MM/yyyy')) FROM t", "StructsToJson with options is not supported") checkSparkAnswerAndFallbackReason( - "SELECT to_json(named_struct('a', a, 'b', array(b))) FROM t", - "Struct type: StructType(StructField(a,IntegerType,true)," + - "StructField(b,ArrayType(StringType,true),false)) " + - "contains unsupported types") + "SELECT to_json(named_struct('b', array(b))) FROM t", + "Struct type: StructType(StructField(b,ArrayType(StringType,true),false)) contains unsupported types") } }