chore: bump spark-4.2 profile to the released 4.2.0

pom.xml

@@ -732,13 +732,12 @@ under the License.
     </profile>
 
     <profile>
-      <!-- WIP: Spark 4.2 preview support. Targets the 4.2.0-preview4 release; tests are not
-           wired up yet, this profile only aims to compile main sources. -->
+      <!-- Experimental Spark 4.2 support, targeting the 4.2.0 release. -->
       <id>spark-4.2</id>
       <properties>
         <scala.version>2.13.18</scala.version>
         <scala.binary.version>2.13</scala.binary.version>
-        <spark.version>4.2.0-preview4</spark.version>
+        <spark.version>4.2.0</spark.version>
         <spark.version.short>4.2</spark.version.short>
         <parquet.version>1.17.0</parquet.version>
         <semanticdb.version>4.13.6</semanticdb.version>
@@ -1252,6 +1251,15 @@ under the License.
                         <ignoreClass>com.google.thirdparty.publicsuffix.PublicSuffixType</ignoreClass>
                       </ignoreClasses>
                     </dependency>
+                    <dependency>
+                      <groupId>org.apache.datasketches</groupId>
+                      <artifactId>datasketches-memory</artifactId>
+                      <ignoreClasses>
+                        <!-- Spark 4.2.0 bundles its own copy of this class in spark-catalyst,
+                             which collides with the transitive datasketches-memory jar. -->
+                        <ignoreClass>org.apache.datasketches.memory.internal.ResourceImpl</ignoreClass>
+                      </ignoreClasses>
+                    </dependency>
                   </dependencies>
                   <findAllDuplicates>true</findAllDuplicates>
                   <ignoreWhenIdentical>true</ignoreWhenIdentical>

spark/pom.xml

@@ -315,7 +315,9 @@ under the License.
           <version>1.10.0</version>
           <scope>test</scope>
         </dependency>
-        <!-- Jetty 11.x for Spark 4.2 (jakarta.servlet); matches Spark 4.2.0-preview4's jetty.version -->
+        <!-- Jetty 11.x for Spark 4.2 (jakarta.servlet). Spark 4.2.0 itself ships Jetty 12, but
+             the Iceberg REST catalog test helper needs jetty-servlet, which Jetty 12 replaced
+             with the jetty-ee10-servlet artifact, so the test classpath stays on 11.x. -->
         <dependency>
           <groupId>org.eclipse.jetty</groupId>
           <artifactId>jetty-server</artifactId>

spark/src/main/scala/org/apache/comet/codegen/CometArrayData.scala

@@ -35,8 +35,8 @@ import org.apache.comet.shims.CometInternalRowShim
  *
  * `ArrayData` and `InternalRow` are sibling abstract classes, so a base aimed at one cannot serve
  * the other. The shared `get(ordinal, dataType)` dispatch lives in
- * [[CometSpecializedGettersDispatch]]. Mixes in [[CometInternalRowShim]] so Spark 4.x's
- * `getVariant` / `getGeography` / `getGeometry` get throwing defaults.
+ * [[CometSpecializedGettersDispatch]]. Mixes in [[CometInternalRowShim]] so the getters Spark 4.x
+ * adds to `SpecializedGetters` get throwing defaults.
  */
 abstract class CometArrayData extends ArrayData with CometInternalRowShim {
 

spark/src/main/scala/org/apache/comet/serde/contraintExpressions.scala

@@ -27,9 +27,6 @@ import org.apache.comet.serde.QueryPlanSerde.{exprToProtoInternal, optExprWithFa
 object CometKnownFloatingPointNormalized
     extends CometExpressionSerde[KnownFloatingPointNormalized] {
 
-  override def getUnsupportedReasons(): Seq[String] = Seq(
-    "Only supports `NormalizeNaNAndZero` child expressions")
-
   override def getSupportLevel(expr: KnownFloatingPointNormalized): SupportLevel = {
     expr.child match {
       case normalized: NormalizeNaNAndZero =>
@@ -40,9 +37,10 @@ object CometKnownFloatingPointNormalized
           Unsupported(Some(s"Unsupported datatype ${wrapped.dataType}"))
         }
       case _ =>
-        Unsupported(
-          Some(
-            "KnownFloatingPointNormalized only supports NormalizeNaNAndZero child expressions"))
+        // Nested normalization (array / struct / map). KnownFloatingPointNormalized is a runtime
+        // no-op tag, so defer to the child's serde: convert serializes the child directly and
+        // falls back gracefully if it is unsupported.
+        Compatible()
     }
   }
 
@@ -51,18 +49,31 @@ object CometKnownFloatingPointNormalized
       inputs: Seq[Attribute],
       binding: Boolean): Option[ExprOuterClass.Expr] = {
 
-    val wrapped = expr.child.asInstanceOf[NormalizeNaNAndZero].child
+    expr.child match {
+      case normalize: NormalizeNaNAndZero =>
+        // Scalar float/double normalization: unwrap and emit the native NormalizeNaNAndZero node.
+        // getSupportLevel has already verified the datatype is serializable.
+        val wrapped = normalize.child
+        val dataType = serializeDataType(wrapped.dataType).get
+        val ex = exprToProtoInternal(wrapped, inputs, binding)
+        val optExpr = ex.map { child =>
+          val builder = ExprOuterClass.NormalizeNaNAndZero
+            .newBuilder()
+            .setChild(child)
+            .setDatatype(dataType)
+          ExprOuterClass.Expr.newBuilder().setNormalizeNanAndZero(builder).build()
+        }
+        optExprWithFallbackReason(optExpr, expr, wrapped)
 
-    val dataType = serializeDataType(wrapped.dataType).get
-    val ex = exprToProtoInternal(wrapped, inputs, binding)
-    val optExpr = ex.map { child =>
-      val builder = ExprOuterClass.NormalizeNaNAndZero
-        .newBuilder()
-        .setChild(child)
-        .setDatatype(dataType)
-      ExprOuterClass.Expr.newBuilder().setNormalizeNanAndZero(builder).build()
+      case child =>
+        // Nested normalization (array / struct / map). Spark 4.2 normalizes the inputs to
+        // array_distinct and the array set operations by wrapping them as, e.g.,
+        // `KnownFloatingPointNormalized(ArrayTransform(arr, x -> NormalizeNaNAndZero(x)))`.
+        // `KnownFloatingPointNormalized` is a runtime no-op tag, so serialize the child directly
+        // and let its serde (e.g. the ArrayTransform codegen dispatcher) carry the normalization.
+        val optExpr = exprToProtoInternal(child, inputs, binding)
+        optExprWithFallbackReason(optExpr, expr, child)
     }
-    optExprWithFallbackReason(optExpr, expr, wrapped)
   }
 }
 

spark/src/main/spark-3.x/org/apache/comet/shims/CometInternalRowShim.scala

@@ -21,7 +21,8 @@ package org.apache.comet.shims
 
 /**
  * Per-profile shim mixed into `CometInternalRow` and `CometArrayData`. Spark 4.x adds abstract
- * `SpecializedGetters` getters (`getVariant` in 4.0, `getGeography` and `getGeometry` in 4.1)
- * that subclasses must implement; Spark 3.x has none, so this trait is empty.
+ * `SpecializedGetters` getters (`getVariant` in 4.0, `getGeography` and `getGeometry` in 4.1,
+ * `getBinaryView` in 4.2) that subclasses must implement; Spark 3.x has none, so this trait is
+ * empty.
  */
 trait CometInternalRowShim

spark/src/main/spark-4.0/org/apache/comet/shims/CometInternalRowShim.scala

@@ -24,8 +24,8 @@ import org.apache.spark.unsafe.types.VariantVal
 /**
  * Throwing defaults for Spark 4.0 `SpecializedGetters` additions: `getVariant`. Mixed into
  * `CometInternalRow` and `CometArrayData` so the codegen kernel's subclasses satisfy the
- * abstract-method check at class-load time. 4.1 also adds `getGeography` / `getGeometry` (see the
- * spark-4.1 shim).
+ * abstract-method check at class-load time. 4.1 also adds `getGeography` / `getGeometry`, and 4.2
+ * replaces those with `getBinaryView` (see the spark-4.1 and spark-4.2 shims).
  */
 trait CometInternalRowShim {
   def getVariant(ordinal: Int): VariantVal =

spark/src/main/spark-4.2/org/apache/comet/shims/CometInternalRowShim.scala

@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.comet.shims
+
+import org.apache.spark.unsafe.types.{BinaryView, VariantVal}
+
+/**
+ * Throwing defaults for Spark 4.x `SpecializedGetters` additions: `getVariant` (4.0) and
+ * `getBinaryView` (4.2). Spark 4.2 dropped 4.1's `getGeography` / `getGeometry` in favour of the
+ * single `getBinaryView` accessor. Mixed into `CometInternalRow` and `CometArrayData`.
+ */
+trait CometInternalRowShim {
+  def getVariant(ordinal: Int): VariantVal =
+    throw new UnsupportedOperationException(
+      s"${getClass.getSimpleName}: getVariant not supported")
+
+  def getBinaryView(ordinal: Int): BinaryView =
+    throw new UnsupportedOperationException(
+      s"${getClass.getSimpleName}: getBinaryView not supported")
+}