From 382d06594530199e384fb5a99d1dce62b87de313 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 4 May 2026 09:00:44 -0600 Subject: [PATCH 01/14] build: Enable Spark SQL tests for Spark 4.2.0-preview4 Adds dev/diffs/4.2.0-preview4.diff so the Spark SQL test suite can run against Apache Spark 4.2.0-preview4 with Comet enabled, and wires the 4.2 profile into the spark_sql_test workflow matrix. The diff was seeded from 4.1.1.diff and reconciled against v4.2.0-preview4. --- .github/workflows/spark_sql_test.yml | 1 + dev/diffs/4.2.0-preview4.diff | 4311 ++++++++++++++++++++++++++ pom.xml | 3 +- 3 files changed, 4313 insertions(+), 2 deletions(-) create mode 100644 dev/diffs/4.2.0-preview4.diff diff --git a/.github/workflows/spark_sql_test.yml b/.github/workflows/spark_sql_test.yml index 980629174f..146c32c7fe 100644 --- a/.github/workflows/spark_sql_test.yml +++ b/.github/workflows/spark_sql_test.yml @@ -142,6 +142,7 @@ jobs: - {spark-short: '4.0', spark-full: '4.0.2', java: 17, scan-impl: 'auto'} - {spark-short: '4.0', spark-full: '4.0.2', java: 21, scan-impl: 'auto'} - {spark-short: '4.1', spark-full: '4.1.1', java: 17, scan-impl: 'auto'} + - {spark-short: '4.2', spark-full: '4.2.0-preview4', java: 17, scan-impl: 'auto'} fail-fast: false name: spark-sql-${{ matrix.config.scan-impl }}-${{ matrix.module.name }}/spark-${{ matrix.config.spark-full }} # Hive tests stay on the standard GitHub-hosted runner: HiveSparkSubmitSuite diff --git a/dev/diffs/4.2.0-preview4.diff b/dev/diffs/4.2.0-preview4.diff new file mode 100644 index 0000000000..824209ff97 --- /dev/null +++ b/dev/diffs/4.2.0-preview4.diff @@ -0,0 +1,4311 @@ +diff --git a/core/src/test/scala/org/apache/spark/storage/FallbackStorageSuite.scala b/core/src/test/scala/org/apache/spark/storage/FallbackStorageSuite.scala +index 6df8bc85b51..dabb75e2b75 100644 +--- a/core/src/test/scala/org/apache/spark/storage/FallbackStorageSuite.scala ++++ b/core/src/test/scala/org/apache/spark/storage/FallbackStorageSuite.scala +@@ -268,6 +268,11 @@ class FallbackStorageSuite extends SparkFunSuite with LocalSparkContext { + } + + test("Upload from all decommissioned executors") { ++ // Comet replaces Spark's shuffle with its own native shuffle, which is incompatible with ++ // the fallback storage migration path used by BlockManagerDecommissioner. ++ val cometEnv = System.getenv("ENABLE_COMET") ++ assume(cometEnv == null || cometEnv == "0" || cometEnv == "false", ++ "Skipped when Comet is enabled: incompatible with Comet native shuffle storage") + sc = new SparkContext(getSparkConf(2, 2)) + withSpark(sc) { sc => + TestUtils.waitUntilExecutorsUp(sc, 2, 60000) +@@ -298,6 +303,11 @@ class FallbackStorageSuite extends SparkFunSuite with LocalSparkContext { + } + + test("Upload multi stages") { ++ // Comet replaces Spark's shuffle with its own native shuffle, which is incompatible with ++ // the fallback storage migration path used by BlockManagerDecommissioner. ++ val cometEnv = System.getenv("ENABLE_COMET") ++ assume(cometEnv == null || cometEnv == "0" || cometEnv == "false", ++ "Skipped when Comet is enabled: incompatible with Comet native shuffle storage") + sc = new SparkContext(getSparkConf()) + withSpark(sc) { sc => + TestUtils.waitUntilExecutorsUp(sc, 1, 60000) +@@ -332,6 +342,11 @@ class FallbackStorageSuite extends SparkFunSuite with LocalSparkContext { + + CompressionCodec.shortCompressionCodecNames.keys.foreach { codec => + test(s"$codec - Newly added executors should access old data from remote storage") { ++ // Comet replaces Spark's shuffle with its own native shuffle, which is incompatible with ++ // the fallback storage migration path used by BlockManagerDecommissioner. ++ val cometEnv = System.getenv("ENABLE_COMET") ++ assume(cometEnv == null || cometEnv == "0" || cometEnv == "false", ++ "Skipped when Comet is enabled: incompatible with Comet native shuffle storage") + sc = new SparkContext(getSparkConf(2, 0).set(IO_COMPRESSION_CODEC, codec)) + withSpark(sc) { sc => + TestUtils.waitUntilExecutorsUp(sc, 2, 60000) +diff --git a/pom.xml b/pom.xml +index 1e7774b3ae6..b18f4a83d31 100644 +--- a/pom.xml ++++ b/pom.xml +@@ -152,6 +152,8 @@ + 4.0.3 + 2.5.3 + 2.0.8 ++ 4.1 ++ 0.16.0-SNAPSHOT + + + org.apache.datasketches +diff --git a/sql/core/pom.xml b/sql/core/pom.xml +index cd7b2fe7805..cbd987e29b8 100644 +--- a/sql/core/pom.xml ++++ b/sql/core/pom.xml +@@ -97,6 +97,10 @@ + org.apache.spark + spark-tags_${scala.binary.version} + ++ ++ org.apache.datafusion ++ comet-spark-spark${spark.version.short}_${scala.binary.version} ++ + + + spark-4.2 2.13.18 From 45273c0dbc59f843cdeddac5c068c910ab12d064 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 4 May 2026 09:22:54 -0600 Subject: [PATCH 02/14] fix: set spark.version.short to 4.2 in 4.2.0-preview4 diff Inherited from 4.1.1.diff during reconciliation. The Spark build was trying to resolve comet-spark-spark4.1_2.13 instead of the 4.2 artifact, causing the sql_hive jobs to fail before any tests ran. --- dev/diffs/4.2.0-preview4.diff | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/diffs/4.2.0-preview4.diff b/dev/diffs/4.2.0-preview4.diff index 824209ff97..059d914b35 100644 --- a/dev/diffs/4.2.0-preview4.diff +++ b/dev/diffs/4.2.0-preview4.diff @@ -39,14 +39,14 @@ index 6df8bc85b51..dabb75e2b75 100644 withSpark(sc) { sc => TestUtils.waitUntilExecutorsUp(sc, 2, 60000) diff --git a/pom.xml b/pom.xml -index 1e7774b3ae6..b18f4a83d31 100644 +index 1e7774b3ae6..6d36b281332 100644 --- a/pom.xml +++ b/pom.xml @@ -152,6 +152,8 @@ 4.0.3 2.5.3 2.0.8 -+ 4.1 ++ 4.2 + 0.16.0-SNAPSHOT + org.apache.datasketches.memory.internal.ResourceImpl + + true true diff --git a/spark/pom.xml b/spark/pom.xml index 26ca9f6dca..cc3f278ee3 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -315,7 +315,9 @@ under the License. 1.10.0 test - + org.eclipse.jetty jetty-server From c1e57f9b7e243a113930a4cd1e425f8c1e566f1f Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 16 Jul 2026 11:56:41 -0600 Subject: [PATCH 05/14] test: add Spark SQL test diff for released Spark 4.2.0 Reseed the Spark SQL test overrides from 4.1.2.diff against the v4.2.0 tag and drop the 4.2.0-preview4 diff. The notable reconciliation is the test harness refactor: Spark 4.2 shrinks SQLTestUtils to a deprecated empty alias and moves withSQLConf, stripSparkFilter and the test() override into QueryTest, with SharedSparkSession now extending QueryTest directly. Comet's hooks (isCometEnabled, the IgnoreComet skip and the stripSparkFilter Comet cases) move to QueryTest accordingly. isCometEnabled has to qualify classic.SparkSession there, since a bare SparkSession in that package resolves to the unified class. --- dev/diffs/{4.2.0-preview4.diff => 4.2.0.diff} | 1521 +++++++---------- 1 file changed, 639 insertions(+), 882 deletions(-) rename dev/diffs/{4.2.0-preview4.diff => 4.2.0.diff} (80%) diff --git a/dev/diffs/4.2.0-preview4.diff b/dev/diffs/4.2.0.diff similarity index 80% rename from dev/diffs/4.2.0-preview4.diff rename to dev/diffs/4.2.0.diff index 059d914b35..8b9c654192 100644 --- a/dev/diffs/4.2.0-preview4.diff +++ b/dev/diffs/4.2.0.diff @@ -39,19 +39,19 @@ index 6df8bc85b51..dabb75e2b75 100644 withSpark(sc) { sc => TestUtils.waitUntilExecutorsUp(sc, 2, 60000) diff --git a/pom.xml b/pom.xml -index 1e7774b3ae6..6d36b281332 100644 +index 46558134f41..e58691eef40 100644 --- a/pom.xml +++ b/pom.xml -@@ -152,6 +152,8 @@ +@@ -154,6 +154,8 @@ 4.0.3 2.5.3 2.0.8 + 4.2 -+ 0.16.0-SNAPSHOT ++ 1.0.0-SNAPSHOT + spark-4.2 2.13.18 From 51a0085f0b70fada1fe865b62de97dd69ab25288 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 16 Jul 2026 12:00:28 -0600 Subject: [PATCH 09/14] ci: allow run-spark-4.2-tests to gate the preflight job The preflight job only proceeds on a labeled event for known gating labels. Without run-spark-4.2-tests in that allowlist, labelling a PR to request the Spark 4.2 SQL tests skipped the whole pipeline instead of running them. --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ccde30da0..4abc420016 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,6 +58,7 @@ jobs: github.event.label.name == 'run-spark-3.4-tests' || github.event.label.name == 'run-spark-4.0-tests' || github.event.label.name == 'run-spark-4.1-tests' || + github.event.label.name == 'run-spark-4.2-tests' || github.event.label.name == 'run-iceberg-tests' steps: - uses: actions/checkout@v7 From db8b9a503b940c3cea31be8666ad8e5d7a463b94 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 17 Jul 2026 10:10:38 -0600 Subject: [PATCH 10/14] chore: bump spark-4.2 profile to the released 4.2.0 Spark 4.2.0 is now in Maven Central, so the spark-4.2 profile no longer needs to target a preview. Point spark.version at 4.2.0 and make the main sources compile against it. - Drop the WIP compile-only comment on the spark-4.2 profile. - Spark 4.2.0 bundles its own copy of org.apache.datasketches.memory.internal.ResourceImpl inside spark-catalyst, colliding with the transitive datasketches-memory jar and failing the BanDuplicateClasses enforcer rule; ignore that class. - Spark 4.2 removes GeographyVal / GeometryVal and replaces getGeography / getGeometry on SpecializedGetters with a single getBinaryView returning BinaryView. CometInternalRowShim can no longer be shared from the spark-4.1+ source root, so split it into per-version spark-4.1 and spark-4.2 copies. - Correct the stale Jetty comment that referenced preview4. This does not wire up the Spark SQL tests for 4.2; it only moves the profile onto the released version and keeps it compiling. --- pom.xml | 14 +++++-- spark/pom.xml | 4 +- .../apache/comet/codegen/CometArrayData.scala | 4 +- .../comet/shims/CometInternalRowShim.scala | 5 ++- .../comet/shims/CometInternalRowShim.scala | 4 +- .../comet/shims/CometInternalRowShim.scala | 0 .../comet/shims/CometInternalRowShim.scala | 37 +++++++++++++++++++ 7 files changed, 58 insertions(+), 10 deletions(-) rename spark/src/main/{spark-4.1+ => spark-4.1}/org/apache/comet/shims/CometInternalRowShim.scala (100%) create mode 100644 spark/src/main/spark-4.2/org/apache/comet/shims/CometInternalRowShim.scala diff --git a/pom.xml b/pom.xml index 6739c01f29..318e3ca51e 100644 --- a/pom.xml +++ b/pom.xml @@ -732,13 +732,12 @@ under the License. - + spark-4.2 2.13.18 2.13 - 4.2.0-preview4 + 4.2.0 4.2 1.17.0 4.13.6 @@ -1252,6 +1251,15 @@ under the License. com.google.thirdparty.publicsuffix.PublicSuffixType + + org.apache.datasketches + datasketches-memory + + + org.apache.datasketches.memory.internal.ResourceImpl + + true true diff --git a/spark/pom.xml b/spark/pom.xml index 26ca9f6dca..cc3f278ee3 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -315,7 +315,9 @@ under the License. 1.10.0 test - + org.eclipse.jetty jetty-server diff --git a/spark/src/main/scala/org/apache/comet/codegen/CometArrayData.scala b/spark/src/main/scala/org/apache/comet/codegen/CometArrayData.scala index 308d1e9d96..53d52ff2e0 100644 --- a/spark/src/main/scala/org/apache/comet/codegen/CometArrayData.scala +++ b/spark/src/main/scala/org/apache/comet/codegen/CometArrayData.scala @@ -35,8 +35,8 @@ import org.apache.comet.shims.CometInternalRowShim * * `ArrayData` and `InternalRow` are sibling abstract classes, so a base aimed at one cannot serve * the other. The shared `get(ordinal, dataType)` dispatch lives in - * [[CometSpecializedGettersDispatch]]. Mixes in [[CometInternalRowShim]] so Spark 4.x's - * `getVariant` / `getGeography` / `getGeometry` get throwing defaults. + * [[CometSpecializedGettersDispatch]]. Mixes in [[CometInternalRowShim]] so the getters Spark 4.x + * adds to `SpecializedGetters` get throwing defaults. */ abstract class CometArrayData extends ArrayData with CometInternalRowShim { diff --git a/spark/src/main/spark-3.x/org/apache/comet/shims/CometInternalRowShim.scala b/spark/src/main/spark-3.x/org/apache/comet/shims/CometInternalRowShim.scala index 18b3a4e6b3..ee85cdb8e9 100644 --- a/spark/src/main/spark-3.x/org/apache/comet/shims/CometInternalRowShim.scala +++ b/spark/src/main/spark-3.x/org/apache/comet/shims/CometInternalRowShim.scala @@ -21,7 +21,8 @@ package org.apache.comet.shims /** * Per-profile shim mixed into `CometInternalRow` and `CometArrayData`. Spark 4.x adds abstract - * `SpecializedGetters` getters (`getVariant` in 4.0, `getGeography` and `getGeometry` in 4.1) - * that subclasses must implement; Spark 3.x has none, so this trait is empty. + * `SpecializedGetters` getters (`getVariant` in 4.0, `getGeography` and `getGeometry` in 4.1, + * `getBinaryView` in 4.2) that subclasses must implement; Spark 3.x has none, so this trait is + * empty. */ trait CometInternalRowShim diff --git a/spark/src/main/spark-4.0/org/apache/comet/shims/CometInternalRowShim.scala b/spark/src/main/spark-4.0/org/apache/comet/shims/CometInternalRowShim.scala index b855fe3a91..253a7f8f3e 100644 --- a/spark/src/main/spark-4.0/org/apache/comet/shims/CometInternalRowShim.scala +++ b/spark/src/main/spark-4.0/org/apache/comet/shims/CometInternalRowShim.scala @@ -24,8 +24,8 @@ import org.apache.spark.unsafe.types.VariantVal /** * Throwing defaults for Spark 4.0 `SpecializedGetters` additions: `getVariant`. Mixed into * `CometInternalRow` and `CometArrayData` so the codegen kernel's subclasses satisfy the - * abstract-method check at class-load time. 4.1 also adds `getGeography` / `getGeometry` (see the - * spark-4.1 shim). + * abstract-method check at class-load time. 4.1 also adds `getGeography` / `getGeometry`, and 4.2 + * replaces those with `getBinaryView` (see the spark-4.1 and spark-4.2 shims). */ trait CometInternalRowShim { def getVariant(ordinal: Int): VariantVal = diff --git a/spark/src/main/spark-4.1+/org/apache/comet/shims/CometInternalRowShim.scala b/spark/src/main/spark-4.1/org/apache/comet/shims/CometInternalRowShim.scala similarity index 100% rename from spark/src/main/spark-4.1+/org/apache/comet/shims/CometInternalRowShim.scala rename to spark/src/main/spark-4.1/org/apache/comet/shims/CometInternalRowShim.scala diff --git a/spark/src/main/spark-4.2/org/apache/comet/shims/CometInternalRowShim.scala b/spark/src/main/spark-4.2/org/apache/comet/shims/CometInternalRowShim.scala new file mode 100644 index 0000000000..36ee7f4c79 --- /dev/null +++ b/spark/src/main/spark-4.2/org/apache/comet/shims/CometInternalRowShim.scala @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.shims + +import org.apache.spark.unsafe.types.{BinaryView, VariantVal} + +/** + * Throwing defaults for Spark 4.x `SpecializedGetters` additions: `getVariant` (4.0) and + * `getBinaryView` (4.2). Spark 4.2 dropped 4.1's `getGeography` / `getGeometry` in favour of the + * single `getBinaryView` accessor. Mixed into `CometInternalRow` and `CometArrayData`. + */ +trait CometInternalRowShim { + def getVariant(ordinal: Int): VariantVal = + throw new UnsupportedOperationException( + s"${getClass.getSimpleName}: getVariant not supported") + + def getBinaryView(ordinal: Int): BinaryView = + throw new UnsupportedOperationException( + s"${getClass.getSimpleName}: getBinaryView not supported") +} From 78049c56e398e22c8df6247e2e83d60e245a87a7 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 17 Jul 2026 10:18:38 -0600 Subject: [PATCH 11/14] chore: regenerate Spark 4.2 plan-stability golden files Regenerated with dev/regenerate-golden-files.sh --spark-version 4.2 against the released 4.2.0. The q2, q5, q54 (v1_4) and q5a (v2_7) goldens are pruned because their plans now match the shared fallback on 4.2.0, and q77a (v2_7) is added to record a degraded plan. --- .../q2/extended.txt | 60 --- .../q5/extended.txt | 107 ------ .../q54/extended.txt | 113 ------ .../q5a/extended.txt | 320 ---------------- .../q77a/extended.txt | 362 ++++++++++++++++++ 5 files changed, 362 insertions(+), 600 deletions(-) delete mode 100644 spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_2/q2/extended.txt delete mode 100644 spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_2/q5/extended.txt delete mode 100644 spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_2/q54/extended.txt delete mode 100644 spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_2/q5a/extended.txt create mode 100644 spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_2/q77a/extended.txt diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_2/q2/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_2/q2/extended.txt deleted file mode 100644 index 60c768385c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_2/q2/extended.txt +++ /dev/null @@ -1,60 +0,0 @@ -CometNativeColumnarToRow -+- CometSort - +- CometExchange - +- CometProject - +- CometBroadcastHashJoin - :- CometProject - : +- CometBroadcastHashJoin - : :- CometHashAggregate - : : +- CometExchange - : : +- CometHashAggregate - : : +- CometUnion - : : :- CometProject - : : : +- CometBroadcastHashJoin - : : : :- CometProject - : : : : +- CometNativeScan parquet spark_catalog.default.web_sales - : : : +- CometBroadcastExchange - : : : +- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : +- CometProject - : : +- CometBroadcastHashJoin - : : :- CometProject - : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales - : : +- CometBroadcastExchange - : : +- CometProject - : : +- CometFilter - : : +- CometNativeScan parquet spark_catalog.default.date_dim - : +- CometBroadcastExchange - : +- CometProject - : +- CometFilter - : +- CometNativeScan parquet spark_catalog.default.date_dim - +- CometBroadcastExchange - +- CometProject - +- CometBroadcastHashJoin - :- CometHashAggregate - : +- CometExchange - : +- CometHashAggregate - : +- CometUnion - : :- CometProject - : : +- CometBroadcastHashJoin - : : :- CometProject - : : : +- CometNativeScan parquet spark_catalog.default.web_sales - : : +- CometBroadcastExchange - : : +- CometProject - : : +- CometFilter - : : +- CometNativeScan parquet spark_catalog.default.date_dim - : +- CometProject - : +- CometBroadcastHashJoin - : :- CometProject - : : +- CometNativeScan parquet spark_catalog.default.catalog_sales - : +- CometBroadcastExchange - : +- CometProject - : +- CometFilter - : +- CometNativeScan parquet spark_catalog.default.date_dim - +- CometBroadcastExchange - +- CometProject - +- CometFilter - +- CometNativeScan parquet spark_catalog.default.date_dim - -Comet accelerated 57 out of 57 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_2/q5/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_2/q5/extended.txt deleted file mode 100644 index 16de14eac2..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_2/q5/extended.txt +++ /dev/null @@ -1,107 +0,0 @@ -CometNativeColumnarToRow -+- CometTakeOrderedAndProject - +- CometHashAggregate - +- CometExchange - +- CometHashAggregate - +- CometExpand - +- CometUnion - :- CometHashAggregate - : +- CometExchange - : +- CometHashAggregate - : +- CometProject - : +- CometBroadcastHashJoin - : :- CometUnion - : : :- CometProject - : : : +- CometBroadcastHashJoin - : : : :- CometProject - : : : : +- CometFilter - : : : : +- CometNativeScan parquet spark_catalog.default.store_sales - : : : : +- CometSubqueryBroadcast - : : : : +- CometBroadcastExchange - : : : : +- CometProject - : : : : +- CometFilter - : : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : : +- CometBroadcastExchange - : : : +- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : +- CometProject - : : +- CometBroadcastHashJoin - : : :- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.store_returns - : : : +- ReusedSubquery - : : +- CometBroadcastExchange - : : +- CometProject - : : +- CometFilter - : : +- CometNativeScan parquet spark_catalog.default.date_dim - : +- CometBroadcastExchange - : +- CometProject - : +- CometFilter - : +- CometNativeScan parquet spark_catalog.default.store - :- CometHashAggregate - : +- CometExchange - : +- CometHashAggregate - : +- CometProject - : +- CometBroadcastHashJoin - : :- CometUnion - : : :- CometProject - : : : +- CometBroadcastHashJoin - : : : :- CometProject - : : : : +- CometFilter - : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales - : : : : +- ReusedSubquery - : : : +- CometBroadcastExchange - : : : +- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : +- CometProject - : : +- CometBroadcastHashJoin - : : :- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns - : : : +- ReusedSubquery - : : +- CometBroadcastExchange - : : +- CometProject - : : +- CometFilter - : : +- CometNativeScan parquet spark_catalog.default.date_dim - : +- CometBroadcastExchange - : +- CometProject - : +- CometFilter - : +- CometNativeScan parquet spark_catalog.default.catalog_page - +- CometHashAggregate - +- CometExchange - +- CometHashAggregate - +- CometProject - +- CometBroadcastHashJoin - :- CometUnion - : :- CometProject - : : +- CometBroadcastHashJoin - : : :- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.web_sales - : : : +- ReusedSubquery - : : +- CometBroadcastExchange - : : +- CometProject - : : +- CometFilter - : : +- CometNativeScan parquet spark_catalog.default.date_dim - : +- CometProject - : +- CometBroadcastHashJoin - : :- CometProject - : : +- CometBroadcastHashJoin - : : :- CometBroadcastExchange - : : : +- CometNativeScan parquet spark_catalog.default.web_returns - : : : +- ReusedSubquery - : : +- CometProject - : : +- CometFilter - : : +- CometNativeScan parquet spark_catalog.default.web_sales - : +- CometBroadcastExchange - : +- CometProject - : +- CometFilter - : +- CometNativeScan parquet spark_catalog.default.date_dim - +- CometBroadcastExchange - +- CometProject - +- CometFilter - +- CometNativeScan parquet spark_catalog.default.web_site - -Comet accelerated 98 out of 104 eligible operators (94%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_2/q54/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_2/q54/extended.txt deleted file mode 100644 index 3cabbbdf79..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_2/q54/extended.txt +++ /dev/null @@ -1,113 +0,0 @@ -CometNativeColumnarToRow -+- CometTakeOrderedAndProject - +- CometHashAggregate - +- CometExchange - +- CometHashAggregate - +- CometHashAggregate - +- CometExchange - +- CometHashAggregate - +- CometProject - +- CometBroadcastHashJoin - :- CometProject - : +- CometBroadcastHashJoin - : :- CometProject - : : +- CometBroadcastHashJoin - : : :- CometProject - : : : +- CometBroadcastHashJoin - : : : :- CometHashAggregate - : : : : +- CometExchange - : : : : +- CometHashAggregate - : : : : +- CometProject - : : : : +- CometBroadcastHashJoin - : : : : :- CometProject - : : : : : +- CometBroadcastHashJoin - : : : : : :- CometUnion - : : : : : : :- CometProject - : : : : : : : +- CometBroadcastHashJoin - : : : : : : : :- CometProject - : : : : : : : : +- CometFilter - : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales - : : : : : : : : +- CometSubqueryBroadcast - : : : : : : : : +- CometBroadcastExchange - : : : : : : : : +- CometProject - : : : : : : : : +- CometFilter - : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : : : : : : +- CometBroadcastExchange - : : : : : : : +- CometProject - : : : : : : : +- CometFilter - : : : : : : : +- CometNativeScan parquet spark_catalog.default.item - : : : : : : +- CometProject - : : : : : : +- CometBroadcastHashJoin - : : : : : : :- CometProject - : : : : : : : +- CometFilter - : : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales - : : : : : : : +- ReusedSubquery - : : : : : : +- CometBroadcastExchange - : : : : : : +- CometProject - : : : : : : +- CometFilter - : : : : : : +- CometNativeScan parquet spark_catalog.default.item - : : : : : +- CometBroadcastExchange - : : : : : +- CometProject - : : : : : +- CometFilter - : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : : : +- CometBroadcastExchange - : : : : +- CometFilter - : : : : +- CometNativeScan parquet spark_catalog.default.customer - : : : +- CometBroadcastExchange - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.store_sales - : : : +- CometSubqueryBroadcast - : : : +- CometBroadcastExchange - : : : +- CometProject - : : : +- CometFilter - : : : : :- ReusedSubquery - : : : : +- ReusedSubquery - : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : : :- Subquery - : : : : +- CometNativeColumnarToRow - : : : : +- CometHashAggregate - : : : : +- CometExchange - : : : : +- CometHashAggregate - : : : : +- CometProject - : : : : +- CometFilter - : : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : : +- Subquery - : : : +- CometNativeColumnarToRow - : : : +- CometHashAggregate - : : : +- CometExchange - : : : +- CometHashAggregate - : : : +- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : +- CometBroadcastExchange - : : +- CometProject - : : +- CometFilter - : : +- CometNativeScan parquet spark_catalog.default.customer_address - : +- CometBroadcastExchange - : +- CometProject - : +- CometFilter - : +- CometNativeScan parquet spark_catalog.default.store - +- CometBroadcastExchange - +- CometProject - +- CometFilter - : :- ReusedSubquery - : +- ReusedSubquery - +- CometNativeScan parquet spark_catalog.default.date_dim - :- Subquery - : +- CometNativeColumnarToRow - : +- CometHashAggregate - : +- CometExchange - : +- CometHashAggregate - : +- CometProject - : +- CometFilter - : +- CometNativeScan parquet spark_catalog.default.date_dim - +- Subquery - +- CometNativeColumnarToRow - +- CometHashAggregate - +- CometExchange - +- CometHashAggregate - +- CometProject - +- CometFilter - +- CometNativeScan parquet spark_catalog.default.date_dim - -Comet accelerated 95 out of 106 eligible operators (89%). Final plan contains 5 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_2/q5a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_2/q5a/extended.txt deleted file mode 100644 index 5af7a1556c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_2/q5a/extended.txt +++ /dev/null @@ -1,320 +0,0 @@ -CometNativeColumnarToRow -+- CometTakeOrderedAndProject - +- CometHashAggregate - +- CometExchange - +- CometHashAggregate - +- CometUnion - :- CometHashAggregate - : +- CometExchange - : +- CometHashAggregate - : +- CometUnion - : :- CometHashAggregate - : : +- CometExchange - : : +- CometHashAggregate - : : +- CometProject - : : +- CometBroadcastHashJoin - : : :- CometUnion - : : : :- CometProject - : : : : +- CometBroadcastHashJoin - : : : : :- CometProject - : : : : : +- CometFilter - : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales - : : : : : +- CometSubqueryBroadcast - : : : : : +- CometBroadcastExchange - : : : : : +- CometProject - : : : : : +- CometFilter - : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : : : +- CometBroadcastExchange - : : : : +- CometProject - : : : : +- CometFilter - : : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : : +- CometProject - : : : +- CometBroadcastHashJoin - : : : :- CometProject - : : : : +- CometFilter - : : : : +- CometNativeScan parquet spark_catalog.default.store_returns - : : : : +- ReusedSubquery - : : : +- CometBroadcastExchange - : : : +- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : +- CometBroadcastExchange - : : +- CometProject - : : +- CometFilter - : : +- CometNativeScan parquet spark_catalog.default.store - : :- CometHashAggregate - : : +- CometExchange - : : +- CometHashAggregate - : : +- CometProject - : : +- CometBroadcastHashJoin - : : :- CometUnion - : : : :- CometProject - : : : : +- CometBroadcastHashJoin - : : : : :- CometProject - : : : : : +- CometFilter - : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales - : : : : : +- ReusedSubquery - : : : : +- CometBroadcastExchange - : : : : +- CometProject - : : : : +- CometFilter - : : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : : +- CometProject - : : : +- CometBroadcastHashJoin - : : : :- CometProject - : : : : +- CometFilter - : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns - : : : : +- ReusedSubquery - : : : +- CometBroadcastExchange - : : : +- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : +- CometBroadcastExchange - : : +- CometProject - : : +- CometFilter - : : +- CometNativeScan parquet spark_catalog.default.catalog_page - : +- CometHashAggregate - : +- CometExchange - : +- CometHashAggregate - : +- CometProject - : +- CometBroadcastHashJoin - : :- CometUnion - : : :- CometProject - : : : +- CometBroadcastHashJoin - : : : :- CometProject - : : : : +- CometFilter - : : : : +- CometNativeScan parquet spark_catalog.default.web_sales - : : : : +- ReusedSubquery - : : : +- CometBroadcastExchange - : : : +- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : +- CometProject - : : +- CometBroadcastHashJoin - : : :- CometProject - : : : +- CometBroadcastHashJoin - : : : :- CometBroadcastExchange - : : : : +- CometNativeScan parquet spark_catalog.default.web_returns - : : : : +- ReusedSubquery - : : : +- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.web_sales - : : +- CometBroadcastExchange - : : +- CometProject - : : +- CometFilter - : : +- CometNativeScan parquet spark_catalog.default.date_dim - : +- CometBroadcastExchange - : +- CometProject - : +- CometFilter - : +- CometNativeScan parquet spark_catalog.default.web_site - :- CometHashAggregate - : +- CometExchange - : +- CometHashAggregate - : +- CometHashAggregate - : +- CometExchange - : +- CometHashAggregate - : +- CometUnion - : :- CometHashAggregate - : : +- CometExchange - : : +- CometHashAggregate - : : +- CometProject - : : +- CometBroadcastHashJoin - : : :- CometUnion - : : : :- CometProject - : : : : +- CometBroadcastHashJoin - : : : : :- CometProject - : : : : : +- CometFilter - : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales - : : : : : +- CometSubqueryBroadcast - : : : : : +- CometBroadcastExchange - : : : : : +- CometProject - : : : : : +- CometFilter - : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : : : +- CometBroadcastExchange - : : : : +- CometProject - : : : : +- CometFilter - : : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : : +- CometProject - : : : +- CometBroadcastHashJoin - : : : :- CometProject - : : : : +- CometFilter - : : : : +- CometNativeScan parquet spark_catalog.default.store_returns - : : : : +- ReusedSubquery - : : : +- CometBroadcastExchange - : : : +- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : +- CometBroadcastExchange - : : +- CometProject - : : +- CometFilter - : : +- CometNativeScan parquet spark_catalog.default.store - : :- CometHashAggregate - : : +- CometExchange - : : +- CometHashAggregate - : : +- CometProject - : : +- CometBroadcastHashJoin - : : :- CometUnion - : : : :- CometProject - : : : : +- CometBroadcastHashJoin - : : : : :- CometProject - : : : : : +- CometFilter - : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales - : : : : : +- ReusedSubquery - : : : : +- CometBroadcastExchange - : : : : +- CometProject - : : : : +- CometFilter - : : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : : +- CometProject - : : : +- CometBroadcastHashJoin - : : : :- CometProject - : : : : +- CometFilter - : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns - : : : : +- ReusedSubquery - : : : +- CometBroadcastExchange - : : : +- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : +- CometBroadcastExchange - : : +- CometProject - : : +- CometFilter - : : +- CometNativeScan parquet spark_catalog.default.catalog_page - : +- CometHashAggregate - : +- CometExchange - : +- CometHashAggregate - : +- CometProject - : +- CometBroadcastHashJoin - : :- CometUnion - : : :- CometProject - : : : +- CometBroadcastHashJoin - : : : :- CometProject - : : : : +- CometFilter - : : : : +- CometNativeScan parquet spark_catalog.default.web_sales - : : : : +- ReusedSubquery - : : : +- CometBroadcastExchange - : : : +- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : +- CometProject - : : +- CometBroadcastHashJoin - : : :- CometProject - : : : +- CometBroadcastHashJoin - : : : :- CometBroadcastExchange - : : : : +- CometNativeScan parquet spark_catalog.default.web_returns - : : : : +- ReusedSubquery - : : : +- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.web_sales - : : +- CometBroadcastExchange - : : +- CometProject - : : +- CometFilter - : : +- CometNativeScan parquet spark_catalog.default.date_dim - : +- CometBroadcastExchange - : +- CometProject - : +- CometFilter - : +- CometNativeScan parquet spark_catalog.default.web_site - +- CometHashAggregate - +- CometExchange - +- CometHashAggregate - +- CometHashAggregate - +- CometExchange - +- CometHashAggregate - +- CometUnion - :- CometHashAggregate - : +- CometExchange - : +- CometHashAggregate - : +- CometProject - : +- CometBroadcastHashJoin - : :- CometUnion - : : :- CometProject - : : : +- CometBroadcastHashJoin - : : : :- CometProject - : : : : +- CometFilter - : : : : +- CometNativeScan parquet spark_catalog.default.store_sales - : : : : +- CometSubqueryBroadcast - : : : : +- CometBroadcastExchange - : : : : +- CometProject - : : : : +- CometFilter - : : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : : +- CometBroadcastExchange - : : : +- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : +- CometProject - : : +- CometBroadcastHashJoin - : : :- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.store_returns - : : : +- ReusedSubquery - : : +- CometBroadcastExchange - : : +- CometProject - : : +- CometFilter - : : +- CometNativeScan parquet spark_catalog.default.date_dim - : +- CometBroadcastExchange - : +- CometProject - : +- CometFilter - : +- CometNativeScan parquet spark_catalog.default.store - :- CometHashAggregate - : +- CometExchange - : +- CometHashAggregate - : +- CometProject - : +- CometBroadcastHashJoin - : :- CometUnion - : : :- CometProject - : : : +- CometBroadcastHashJoin - : : : :- CometProject - : : : : +- CometFilter - : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales - : : : : +- ReusedSubquery - : : : +- CometBroadcastExchange - : : : +- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : +- CometProject - : : +- CometBroadcastHashJoin - : : :- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns - : : : +- ReusedSubquery - : : +- CometBroadcastExchange - : : +- CometProject - : : +- CometFilter - : : +- CometNativeScan parquet spark_catalog.default.date_dim - : +- CometBroadcastExchange - : +- CometProject - : +- CometFilter - : +- CometNativeScan parquet spark_catalog.default.catalog_page - +- CometHashAggregate - +- CometExchange - +- CometHashAggregate - +- CometProject - +- CometBroadcastHashJoin - :- CometUnion - : :- CometProject - : : +- CometBroadcastHashJoin - : : :- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.web_sales - : : : +- ReusedSubquery - : : +- CometBroadcastExchange - : : +- CometProject - : : +- CometFilter - : : +- CometNativeScan parquet spark_catalog.default.date_dim - : +- CometProject - : +- CometBroadcastHashJoin - : :- CometProject - : : +- CometBroadcastHashJoin - : : :- CometBroadcastExchange - : : : +- CometNativeScan parquet spark_catalog.default.web_returns - : : : +- ReusedSubquery - : : +- CometProject - : : +- CometFilter - : : +- CometNativeScan parquet spark_catalog.default.web_sales - : +- CometBroadcastExchange - : +- CometProject - : +- CometFilter - : +- CometNativeScan parquet spark_catalog.default.date_dim - +- CometBroadcastExchange - +- CometProject - +- CometFilter - +- CometNativeScan parquet spark_catalog.default.web_site - -Comet accelerated 299 out of 317 eligible operators (94%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_2/q77a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_2/q77a/extended.txt new file mode 100644 index 0000000000..cf441b7592 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_2/q77a/extended.txt @@ -0,0 +1,362 @@ +CometNativeColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometColumnarExchange + +- HashAggregate + +- Union + :- HashAggregate [COMET: Comet aggregate that merges intermediate buffers requires a Comet child aggregate when the intermediate buffer formats are incompatible with Spark. Incompatible aggregate function(s): sum] + : +- Exchange + : +- HashAggregate + : +- Union + : :- CometNativeColumnarToRow + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometSubqueryBroadcast + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- CometBroadcastExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : +- ReusedSubquery + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : :- Project + : : +- BroadcastNestedLoopJoin + : : :- BroadcastExchange + : : : +- CometNativeColumnarToRow + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- ReusedSubquery + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- Project + : : : :- Subquery + : : : : +- CometNativeColumnarToRow + : : : : +- CometProject + : : : : +- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : : +- ReusedSubquery + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- ReusedSubquery + : : +- Scan OneRowRelation [COMET: Scan OneRowRelation is not supported] + : +- CometNativeColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- ReusedSubquery + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_page + : +- CometBroadcastExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : +- ReusedSubquery + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_page + :- HashAggregate [COMET: Comet aggregate that merges intermediate buffers requires a Comet child aggregate when the intermediate buffer formats are incompatible with Spark. Incompatible aggregate function(s): sum] + : +- Exchange + : +- HashAggregate + : +- HashAggregate [COMET: Comet aggregate that merges intermediate buffers requires a Comet child aggregate when the intermediate buffer formats are incompatible with Spark. Incompatible aggregate function(s): sum] + : +- Exchange + : +- HashAggregate + : +- Union + : :- CometNativeColumnarToRow + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometSubqueryBroadcast + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- CometBroadcastExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : +- ReusedSubquery + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : :- Project + : : +- BroadcastNestedLoopJoin + : : :- BroadcastExchange + : : : +- CometNativeColumnarToRow + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- ReusedSubquery + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- Project + : : : :- Subquery + : : : : +- CometNativeColumnarToRow + : : : : +- CometProject + : : : : +- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : : +- ReusedSubquery + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- ReusedSubquery + : : +- Scan OneRowRelation [COMET: Scan OneRowRelation is not supported] + : +- CometNativeColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- ReusedSubquery + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_page + : +- CometBroadcastExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : +- ReusedSubquery + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_page + +- HashAggregate [COMET: Comet aggregate that merges intermediate buffers requires a Comet child aggregate when the intermediate buffer formats are incompatible with Spark. Incompatible aggregate function(s): sum] + +- Exchange + +- HashAggregate + +- HashAggregate [COMET: Comet aggregate that merges intermediate buffers requires a Comet child aggregate when the intermediate buffer formats are incompatible with Spark. Incompatible aggregate function(s): sum] + +- Exchange + +- HashAggregate + +- Union + :- CometNativeColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometSubqueryBroadcast + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : +- ReusedSubquery + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + :- Project + : +- BroadcastNestedLoopJoin + : :- BroadcastExchange + : : +- CometNativeColumnarToRow + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- ReusedSubquery + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- Project + : : :- Subquery + : : : +- CometNativeColumnarToRow + : : : +- CometProject + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : +- ReusedSubquery + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- ReusedSubquery + : +- Scan OneRowRelation [COMET: Scan OneRowRelation is not supported] + +- CometNativeColumnarToRow + +- CometProject + +- CometBroadcastHashJoin + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- ReusedSubquery + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_page + +- CometBroadcastExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : +- ReusedSubquery + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.web_page + +Comet accelerated 288 out of 347 eligible operators (82%). Final plan contains 13 transitions between Spark and Comet. \ No newline at end of file From 3e5007fe6fb8a7f9014d6ebf9e25e5e4fb88a311 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 17 Jul 2026 11:22:31 -0600 Subject: [PATCH 12/14] fix: resolve Spark 4.2 test failures from the version bump Two Spark 4.2.0 changes surfaced by moving the profile off preview4: Iceberg scans job: Spark 4.2.0 turned `connector.catalog.View` from an interface into a class. No Iceberg spark-runtime is published for 4.2, so the build reuses the 4.0 runtime, whose `SparkView implements View` now throws IncompatibleClassChangeError at class-load and aborts the Iceberg suites. Report Iceberg as unavailable on Spark 4.2 in the shared probes so the suites skip. Guard the fuzz suite's beforeAll with an early return (cancelling from beforeAll aborts the suite) and add the missing `assume(icebergAvailable)` to one native-scan test. Expressions job: Spark 4.2 normalizes NaN / -0.0 for array_distinct and the array set operations by wrapping their inputs as `KnownFloatingPointNormalized(ArrayTransform(arr, x -> NormalizeNaNAndZero(x)))`. The serde only handled a scalar `NormalizeNaNAndZero` child and fell back. Since `KnownFloatingPointNormalized` is a runtime no-op tag, serialize any other child directly and let its serde (the ArrayTransform codegen dispatcher) carry the normalization, keeping these operations native. --- .../comet/serde/contraintExpressions.scala | 53 +++++++++---------- .../apache/comet/CometFuzzIcebergBase.scala | 25 ++++++--- .../comet/CometIcebergNativeSuite.scala | 1 + .../apache/comet/CometIcebergTestBase.scala | 18 ++++--- .../org/apache/comet/CometS3TestBase.scala | 17 ++++-- 5 files changed, 69 insertions(+), 45 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/contraintExpressions.scala b/spark/src/main/scala/org/apache/comet/serde/contraintExpressions.scala index 419a5be0c1..be8253be9b 100644 --- a/spark/src/main/scala/org/apache/comet/serde/contraintExpressions.scala +++ b/spark/src/main/scala/org/apache/comet/serde/contraintExpressions.scala @@ -28,40 +28,39 @@ import org.apache.comet.serde.QueryPlanSerde.{exprToProtoInternal, optExprWithFa object CometKnownFloatingPointNormalized extends CometExpressionSerde[KnownFloatingPointNormalized] { - override def getUnsupportedReasons(): Seq[String] = Seq( - "Only supports `NormalizeNaNAndZero` child expressions") - - override def getSupportLevel(expr: KnownFloatingPointNormalized): SupportLevel = { - expr.child match { - case _: NormalizeNaNAndZero => Compatible() - case _ => - Unsupported( - Some( - "KnownFloatingPointNormalized only supports NormalizeNaNAndZero child expressions")) - } - } - override def convert( expr: KnownFloatingPointNormalized, inputs: Seq[Attribute], binding: Boolean): Option[ExprOuterClass.Expr] = { - val wrapped = expr.child.asInstanceOf[NormalizeNaNAndZero].child + expr.child match { + case normalize: NormalizeNaNAndZero => + // Scalar float/double normalization: unwrap and emit the native NormalizeNaNAndZero node. + val wrapped = normalize.child + val dataType = serializeDataType(wrapped.dataType) + if (dataType.isEmpty) { + withFallbackReason(wrapped, s"Unsupported datatype ${wrapped.dataType}") + return None + } + val ex = exprToProtoInternal(wrapped, inputs, binding) + val optExpr = ex.map { child => + val builder = ExprOuterClass.NormalizeNaNAndZero + .newBuilder() + .setChild(child) + .setDatatype(dataType.get) + ExprOuterClass.Expr.newBuilder().setNormalizeNanAndZero(builder).build() + } + optExprWithFallbackReason(optExpr, expr, wrapped) - val dataType = serializeDataType(wrapped.dataType) - if (dataType.isEmpty) { - withFallbackReason(wrapped, s"Unsupported datatype ${wrapped.dataType}") - return None - } - val ex = exprToProtoInternal(wrapped, inputs, binding) - val optExpr = ex.map { child => - val builder = ExprOuterClass.NormalizeNaNAndZero - .newBuilder() - .setChild(child) - .setDatatype(dataType.get) - ExprOuterClass.Expr.newBuilder().setNormalizeNanAndZero(builder).build() + case child => + // Nested normalization (array / struct / map). Spark 4.2 normalizes the inputs to + // array_distinct and the array set operations by wrapping them as, e.g., + // `KnownFloatingPointNormalized(ArrayTransform(arr, x -> NormalizeNaNAndZero(x)))`. + // `KnownFloatingPointNormalized` is a runtime no-op tag, so serialize the child directly + // and let its serde (e.g. the ArrayTransform codegen dispatcher) carry the normalization. + val optExpr = exprToProtoInternal(child, inputs, binding) + optExprWithFallbackReason(optExpr, expr, child) } - optExprWithFallbackReason(optExpr, expr, wrapped) } } diff --git a/spark/src/test/scala/org/apache/comet/CometFuzzIcebergBase.scala b/spark/src/test/scala/org/apache/comet/CometFuzzIcebergBase.scala index 92f36f534b..fbace69140 100644 --- a/spark/src/test/scala/org/apache/comet/CometFuzzIcebergBase.scala +++ b/spark/src/test/scala/org/apache/comet/CometFuzzIcebergBase.scala @@ -35,6 +35,7 @@ import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.DecimalType +import org.apache.comet.CometSparkSessionExtensions.isSpark42Plus import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator, SchemaGenOptions} class CometFuzzIcebergBase extends CometTestBase with AdaptiveSparkPlanHelper { @@ -42,13 +43,18 @@ class CometFuzzIcebergBase extends CometTestBase with AdaptiveSparkPlanHelper { var warehouseDir: File = null val icebergTableName: String = "hadoop_catalog.db.fuzz_test" - // Skip these tests if Iceberg is not available in classpath + // Skip these tests if Iceberg is not available in classpath. Spark 4.2 has no published Iceberg + // spark-runtime yet; the build reuses the 4.0 runtime, whose `SparkView` is binary-incompatible + // with Spark 4.2's `connector.catalog.View` (now a class, not an interface), so treat Iceberg as + // unavailable on 4.2. private def icebergAvailable: Boolean = { - try { - Class.forName("org.apache.iceberg.catalog.Catalog") - true - } catch { - case _: ClassNotFoundException => false + !isSpark42Plus && { + try { + Class.forName("org.apache.iceberg.catalog.Catalog") + true + } catch { + case _: ClassNotFoundException => false + } } } @@ -88,7 +94,11 @@ class CometFuzzIcebergBase extends CometTestBase with AdaptiveSparkPlanHelper { override def beforeAll(): Unit = { super.beforeAll() - assume(icebergAvailable, "Iceberg not available in classpath") + // Skip Iceberg setup when Iceberg is unavailable (e.g. on Spark 4.2, which has no compatible + // runtime). `assume` cannot be used here: cancelling from beforeAll aborts the whole suite. + // Each test carries its own `assume(icebergAvailable)` (see the `test` override) so the tests + // report as cancelled instead. + if (!icebergAvailable) return warehouseDir = Files.createTempDirectory("comet-iceberg-fuzz-test").toFile val random = new Random(42) withSQLConf( @@ -146,6 +156,7 @@ class CometFuzzIcebergBase extends CometTestBase with AdaptiveSparkPlanHelper { override protected def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit pos: Position): Unit = { super.test(testName, testTags: _*) { + assume(icebergAvailable, "Iceberg not available in classpath") withSQLConf( "spark.sql.catalog.hadoop_catalog" -> "org.apache.iceberg.spark.SparkCatalog", "spark.sql.catalog.hadoop_catalog.type" -> "hadoop", diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala index 118d2e6143..7b9bd6b3d6 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala @@ -4311,6 +4311,7 @@ class CometIcebergNativeSuite } test("CometScanRule should report unsupported metadata columns") { + assume(icebergAvailable, "Iceberg not available in classpath") withTempIcebergDir { warehouseDir => withSQLConf( "spark.sql.catalog.test_cat" -> "org.apache.iceberg.spark.SparkCatalog", diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergTestBase.scala b/spark/src/test/scala/org/apache/comet/CometIcebergTestBase.scala index be7da6cbe8..33de4f3705 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergTestBase.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergTestBase.scala @@ -22,6 +22,7 @@ package org.apache.comet import java.io.File import java.nio.file.Files +import org.apache.comet.CometSparkSessionExtensions.isSpark42Plus import org.apache.comet.iceberg.IcebergReflection /** @@ -29,13 +30,18 @@ import org.apache.comet.iceberg.IcebergReflection */ trait CometIcebergTestBase { + // No Iceberg spark-runtime is published for Spark 4.2 yet, so the build reuses the 4.0 runtime. + // That jar is binary-incompatible with Spark 4.2, whose `connector.catalog.View` is a class + // rather than an interface, so loading `SparkView` throws IncompatibleClassChangeError. Report + // Iceberg as unavailable on 4.2 so these suites skip until a compatible runtime exists. protected def icebergAvailable: Boolean = - try { - IcebergReflection.loadClass("org.apache.iceberg.catalog.Catalog") - true - } catch { - case _: ClassNotFoundException => false - } + !isSpark42Plus && + (try { + IcebergReflection.loadClass("org.apache.iceberg.catalog.Catalog") + true + } catch { + case _: ClassNotFoundException => false + }) /** * Whether the Iceberg library on the classpath is at least the given (major, minor) version. diff --git a/spark/src/test/scala/org/apache/comet/CometS3TestBase.scala b/spark/src/test/scala/org/apache/comet/CometS3TestBase.scala index 467a360483..f8cc9f4e42 100644 --- a/spark/src/test/scala/org/apache/comet/CometS3TestBase.scala +++ b/spark/src/test/scala/org/apache/comet/CometS3TestBase.scala @@ -29,6 +29,8 @@ import org.testcontainers.utility.DockerImageName import org.apache.spark.SparkConf import org.apache.spark.sql.CometTestBase +import org.apache.comet.CometSparkSessionExtensions.isSpark42Plus + import software.amazon.awssdk.auth.credentials.{AwsBasicCredentials, StaticCredentialsProvider} import software.amazon.awssdk.services.s3.S3Client import software.amazon.awssdk.services.s3.model.{CreateBucketRequest, HeadBucketRequest} @@ -66,12 +68,17 @@ trait CometS3TestBase extends CometTestBase { conf.set("spark.hadoop.fs.s3a.path.style.access", "true") } + // Spark 4.2 has no published Iceberg spark-runtime yet; the build reuses the 4.0 runtime, whose + // `SparkView` is binary-incompatible with Spark 4.2's `connector.catalog.View` (now a class, not + // an interface), so treat Iceberg as unavailable on 4.2. protected def icebergAvailable: Boolean = { - try { - Class.forName("org.apache.iceberg.catalog.Catalog") - true - } catch { - case _: ClassNotFoundException => false + !isSpark42Plus && { + try { + Class.forName("org.apache.iceberg.catalog.Catalog") + true + } catch { + case _: ClassNotFoundException => false + } } } From 0936975c9f161ee09976abb831c0a0631871c18a Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 17 Jul 2026 14:12:42 -0600 Subject: [PATCH 13/14] fix: fall back to Spark when native scan schema has unsupported types CometNativeScan serializes the full data and partition schema, not just the required columns. A table with a GEOMETRY or GEOGRAPHY column (Spark 4.2) has no proto representation for those types, so schema2Proto threw NoSuchElementException during planning instead of falling back. Reject the native scan when any data or partition schema field type cannot be serialized. --- .../comet/serde/operator/CometNativeScan.scala | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometNativeScan.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometNativeScan.scala index 763602bb7d..5533082a4b 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometNativeScan.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometNativeScan.scala @@ -87,6 +87,19 @@ object CometNativeScan extends CometOperatorSerde[CometScanExec] with Logging { withFallbackReason(scanExec, "Full native scan disabled because ignoreMissingFiles enabled") } + // The native scan serializes the full data and partition schema (not just the required + // columns), so every field type must be serializable. Types with no proto representation + // (e.g. GEOMETRY / GEOGRAPHY) would otherwise crash schema serialization; fall back instead. + (scanExec.relation.dataSchema.fields ++ scanExec.relation.partitionSchema.fields).foreach { + field => + if (serializeDataType(field.dataType).isEmpty) { + withFallbackReason( + scanExec, + s"Native scan does not support data type ${field.dataType.simpleString} " + + s"in column ${field.name}") + } + } + // the scan is supported if no fallback reasons were added to the node !hasFallbackReason(scanExec) } From 5d70d8392c5da262ce7f4f28228bfcf94a9d7723 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 17 Jul 2026 14:12:42 -0600 Subject: [PATCH 14/14] test: resolve Spark 4.2 SQL test failures and file specific tracking issues Handle new-in-4.2 Spark SQL test suites that assert on Spark-internal execution mechanics Comet replaces wholesale, and repoint deferred assumes at specific issues instead of the closed generic tracker (#4142). - SLAM metric suites, segment-tree window suites, and the UnionExec whole-stage-codegen test are marked IgnoreComet / IgnoreCometSuite (#4963, #4964, #4965). - collect_set NaN/-0.0 normalization deferred (#4966). - Repoint assume(!isSpark42Plus) guards to specific issues: #4967 (ANSI arithmetic), #4968 (BloomFilter), #4969 (Iceberg REST catalog). - Geometry/geography scan tests now pass via native-scan fallback. --- dev/diffs/4.2.0.diff | 168 +++++++++++++++++- .../apache/comet/CometExpressionSuite.scala | 8 +- .../comet/CometIcebergNativeSuite.scala | 2 +- .../comet/exec/CometExec3_4PlusSuite.scala | 8 +- .../apache/comet/exec/CometExecSuite.scala | 2 +- .../comet/rules/CometExecRuleSuite.scala | 4 +- 6 files changed, 177 insertions(+), 15 deletions(-) diff --git a/dev/diffs/4.2.0.diff b/dev/diffs/4.2.0.diff index 8b9c654192..b1f51e1251 100644 --- a/dev/diffs/4.2.0.diff +++ b/dev/diffs/4.2.0.diff @@ -392,7 +392,7 @@ index 085dbcd8046..3090d321b6c 100644 withTempView("t0", "t1", "t2") { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala -index 5b8154d2900..632274c7fc8 100644 +index 5b8154d2900..f35da2951af 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala @@ -30,8 +30,9 @@ import org.apache.spark.sql.catalyst.util.AUTO_GENERATED_ALIAS @@ -406,7 +406,18 @@ index 5b8154d2900..632274c7fc8 100644 import org.apache.spark.sql.expressions.Window import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf -@@ -932,7 +933,9 @@ class DataFrameAggregateSuite extends SharedSparkSession +@@ -667,7 +668,9 @@ class DataFrameAggregateSuite extends SharedSparkSession + df.selectExpr("sort_array(collect_set(b) RESPECT NULLS)"), Seq(Row(Seq(null, 2)))) + } + +- test("SPARK-57298: collect_set normalizes NaN and -0.0 for floating-point types") { ++ test("SPARK-57298: collect_set normalizes NaN and -0.0 for floating-point types", ++ // Comet's native collect_set does not yet normalize NaN / -0.0. ++ IgnoreComet("https://github.com/apache/datafusion-comet/issues/4966")) { + checkAnswer( + sql("SELECT collect_set(v) FROM VALUES (double('NaN')), (double('NaN')) AS t(v)"), + Row(Seq(Double.NaN))) +@@ -932,7 +935,9 @@ class DataFrameAggregateSuite extends SharedSparkSession case _ => false }.isDefined) } else { @@ -417,7 +428,7 @@ index 5b8154d2900..632274c7fc8 100644 } // test case for ObjectHashAggregate and SortAggregate -@@ -996,7 +999,7 @@ class DataFrameAggregateSuite extends SharedSparkSession +@@ -996,7 +1001,7 @@ class DataFrameAggregateSuite extends SharedSparkSession assert(objHashAggPlans.nonEmpty) val exchangePlans = collect(aggPlan) { @@ -2044,6 +2055,31 @@ index b167dd13dcb..b2b6b88ce66 100644 try { spark.range(1).foreach { _ => columnarToRowExec.canonicalized +diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/UnionCodegenSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/UnionCodegenSuite.scala +index 835c55fe4c4..57cce5f6b03 100644 +--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/UnionCodegenSuite.scala ++++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/UnionCodegenSuite.scala +@@ -20,7 +20,7 @@ package org.apache.spark.sql.execution + import java.util.concurrent.{CountDownLatch, Executors, TimeUnit} + + import org.apache.spark.SparkConf +-import org.apache.spark.sql.{DataFrame, QueryTest, Row} ++import org.apache.spark.sql.{DataFrame, IgnoreComet, QueryTest, Row} + import org.apache.spark.sql.functions._ + import org.apache.spark.sql.internal.SQLConf + import org.apache.spark.sql.test.SharedSparkSession +@@ -605,7 +605,10 @@ class UnionCodegenSuite extends QueryTest with SharedSparkSession { + } + } + +- test("SPARK-56482: partitioning-aware union falls back to non-codegen") { ++ test("SPARK-56482: partitioning-aware union falls back to non-codegen", ++ // Comet replaces Spark's UnionExec with CometUnionExec, so the executed plan does not ++ // expose a UnionExec for this WSCG-internal assertion to inspect. ++ IgnoreComet("https://github.com/apache/datafusion-comet/issues/4965")) { + // After repartition, both children expose a `HashPartitioning` on the same key, + // so `UnionExec.outputPartitioning` is non-Unknown and the codegen path is denied. + // AQE is disabled here so the executedPlan exposes the UnionExec directly diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala index d70bd715879..074a9fa29d9 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala @@ -3276,6 +3312,80 @@ index b8f3ea3c6f3..bbd44221288 100644 withTempPath { workDir => val workDirPath = workDir.getAbsolutePath val input = spark.range(5).toDF("id") +diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/MetricsFailureInjectionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/MetricsFailureInjectionSuite.scala +index 2ae2ea6339f..30c8959a057 100644 +--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/MetricsFailureInjectionSuite.scala ++++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/MetricsFailureInjectionSuite.scala +@@ -19,7 +19,7 @@ package org.apache.spark.sql.execution.metric + import scala.util.Random + + import org.apache.spark.internal.config +-import org.apache.spark.sql.{Column, Dataset} ++import org.apache.spark.sql.{Column, Dataset, IgnoreCometSuite} + import org.apache.spark.sql.execution.adaptive.{AQETestHelper, DisableAdaptiveExecutionSuite} + import org.apache.spark.sql.functions.udf + import org.apache.spark.sql.internal.SQLConf +@@ -29,7 +29,10 @@ class MetricsFailureInjectionSuite + extends SharedSparkSession + with SQLMetricsTestUtils + // Need to control AQE per-test to ensure expected plan shapes. +- with DisableAdaptiveExecutionSuite { ++ with DisableAdaptiveExecutionSuite ++ // Comet replaces Spark operators with native equivalents, so the SQL Last Attempt Metric ++ // is not propagated through the plan. See https://github.com/apache/datafusion-comet/issues/4963 ++ with IgnoreCometSuite { + + import testImplicits._ + +diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLLastAttemptMetricIntegrationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLLastAttemptMetricIntegrationSuite.scala +index 2e7af075a3e..b8fa7d35783 100644 +--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLLastAttemptMetricIntegrationSuite.scala ++++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLLastAttemptMetricIntegrationSuite.scala +@@ -17,7 +17,7 @@ + package org.apache.spark.sql.execution.metric + + import org.apache.spark.internal.config +-import org.apache.spark.sql.Column ++import org.apache.spark.sql.{Column, IgnoreComet} + import org.apache.spark.sql.catalyst.expressions.Literal + import org.apache.spark.sql.catalyst.optimizer.BuildRight + import org.apache.spark.sql.catalyst.plans.RightOuter +@@ -255,7 +255,10 @@ class SQLLastAttemptMetricIntegrationSuite + logInfo(slam.logAccumulatorState) + } + +- test("dataset updates") { ++ test("dataset updates", ++ // Comet replaces Spark operators with native equivalents, so the SQL Last Attempt Metric ++ // is not propagated through the plan. ++ IgnoreComet("https://github.com/apache/datafusion-comet/issues/4963")) { + val slam1 = SQLLastAttemptMetrics.createMetric(spark.sparkContext, "SLAM1") + val df1 = spark.range(10).filter(Column(incrementMetric(slam1))) + +diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLLastAttemptMetricPlanShapesSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLLastAttemptMetricPlanShapesSuite.scala +index ea8d9568f7e..34b9c2eef01 100644 +--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLLastAttemptMetricPlanShapesSuite.scala ++++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLLastAttemptMetricPlanShapesSuite.scala +@@ -22,6 +22,7 @@ import scala.util.Random + import org.scalatest.Tag + + import org.apache.spark.internal.config ++import org.apache.spark.sql.IgnoreCometSuite + import org.apache.spark.sql.execution.{CollectLimitExec, RDDScanExec, SparkPlan} + import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, AQETestHelper, DisableAdaptiveExecutionSuite} + import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec +@@ -34,7 +35,10 @@ class SQLLastAttemptMetricPlanShapesSuite + extends SharedSparkSession + with SQLMetricsTestUtils + // Need to control AQE per-test to ensure expected plan shapes. +- with DisableAdaptiveExecutionSuite { ++ with DisableAdaptiveExecutionSuite ++ // Comet replaces Spark operators with native equivalents, so the SQL Last Attempt Metric ++ // is not propagated through the plan. See https://github.com/apache/datafusion-comet/issues/4963 ++ with IgnoreCometSuite { + + import testImplicits._ + diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala index 16cde3ef2c5..cfc26410733 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala @@ -3438,6 +3548,58 @@ index 65009a2c1c6..09c53621daf 100644 before { StateStore.stop() require(!StateStore.isMaintenanceRunning) +diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/window/SegmentTreeWindowMetricsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/window/SegmentTreeWindowMetricsSuite.scala +index 5c3edb7e06c..f6accc654cd 100644 +--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/window/SegmentTreeWindowMetricsSuite.scala ++++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/window/SegmentTreeWindowMetricsSuite.scala +@@ -17,7 +17,7 @@ + + package org.apache.spark.sql.execution.window + +-import org.apache.spark.sql.QueryTest ++import org.apache.spark.sql.{IgnoreCometSuite, QueryTest} + import org.apache.spark.sql.execution.SparkPlanInfo + import org.apache.spark.sql.execution.metric.SQLMetricsTestUtils + import org.apache.spark.sql.execution.ui.SparkPlanGraph +@@ -31,8 +31,11 @@ import org.apache.spark.sql.test.SharedSparkSession + * segtree path bumps `numSegmentTreeFrames`; fallback path bumps + * `numSegmentTreeFallbackFrames`; feature-flag off leaves both at 0. + */ ++// Comet replaces Spark's WindowExec with CometWindowExec, which has no segment-tree window ++// optimization and therefore does not expose the segment-tree frame metrics this suite reads. ++// See https://github.com/apache/datafusion-comet/issues/4964 + class SegmentTreeWindowMetricsSuite +- extends QueryTest with SharedSparkSession with SQLMetricsTestUtils { ++ extends QueryTest with SharedSparkSession with SQLMetricsTestUtils with IgnoreCometSuite { + + import testImplicits._ + +diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/window/WindowSegmentTreeAllowlistSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/window/WindowSegmentTreeAllowlistSuite.scala +index d3da8e40c85..505f48092ea 100644 +--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/window/WindowSegmentTreeAllowlistSuite.scala ++++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/window/WindowSegmentTreeAllowlistSuite.scala +@@ -17,7 +17,7 @@ + + package org.apache.spark.sql.execution.window + +-import org.apache.spark.sql.{DataFrame, QueryTest} ++import org.apache.spark.sql.{DataFrame, IgnoreCometSuite, QueryTest} + import org.apache.spark.sql.execution.SparkPlanInfo + import org.apache.spark.sql.execution.metric.SQLMetricsTestUtils + import org.apache.spark.sql.execution.ui.SparkPlanGraph +@@ -35,8 +35,11 @@ import org.apache.spark.sql.test.SharedSparkSession + * Eligibility gating only; exhaustive equivalence lives in + * [[SegmentTreeWindowFunctionSuite]]. + */ ++// Comet replaces Spark's WindowExec with CometWindowExec, which has no segment-tree window ++// optimization and therefore does not expose the segment-tree frame metrics this suite reads. ++// See https://github.com/apache/datafusion-comet/issues/4964 + class WindowSegmentTreeAllowlistSuite +- extends QueryTest with SharedSparkSession with SQLMetricsTestUtils { ++ extends QueryTest with SharedSparkSession with SQLMetricsTestUtils with IgnoreCometSuite { + + import testImplicits._ + diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala index 452b7b9dedb..b2a73a8b772 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index f3c442ad29..4cd1a4c6c2 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -2310,7 +2310,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } test("unary negative integer overflow test") { - assume(!isSpark42Plus, "https://github.com/apache/datafusion-comet/issues/4142") + assume(!isSpark42Plus, "https://github.com/apache/datafusion-comet/issues/4967") def withAnsiMode(enabled: Boolean)(f: => Unit): Unit = { withSQLConf( SQLConf.ANSI_ENABLED.key -> enabled.toString, @@ -2903,7 +2903,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } test("ANSI support for add") { - assume(!isSpark42Plus, "https://github.com/apache/datafusion-comet/issues/4142") + assume(!isSpark42Plus, "https://github.com/apache/datafusion-comet/issues/4967") val data = Seq((Integer.MAX_VALUE, 1), (Integer.MIN_VALUE, -1)) withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { withParquetTable(data, "tbl") { @@ -2924,7 +2924,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } test("ANSI support for subtract") { - assume(!isSpark42Plus, "https://github.com/apache/datafusion-comet/issues/4142") + assume(!isSpark42Plus, "https://github.com/apache/datafusion-comet/issues/4967") val data = Seq((Integer.MIN_VALUE, 1)) withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { withParquetTable(data, "tbl") { @@ -2944,7 +2944,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } test("ANSI support for multiply") { - assume(!isSpark42Plus, "https://github.com/apache/datafusion-comet/issues/4142") + assume(!isSpark42Plus, "https://github.com/apache/datafusion-comet/issues/4967") val data = Seq((Integer.MAX_VALUE, 10)) withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { withParquetTable(data, "tbl") { diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala index 7b9bd6b3d6..221f2ab756 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala @@ -2699,7 +2699,7 @@ class CometIcebergNativeSuite } test("REST catalog with native Iceberg scan") { - assume(!isSpark42Plus, "https://github.com/apache/datafusion-comet/issues/4142") + assume(!isSpark42Plus, "https://github.com/apache/datafusion-comet/issues/4969") assume(icebergAvailable, "Iceberg not available in classpath") withRESTCatalog { (restUri, _, warehouseDir) => diff --git a/spark/src/test/scala/org/apache/comet/exec/CometExec3_4PlusSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometExec3_4PlusSuite.scala index 3a406f9a72..edabb7e0af 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometExec3_4PlusSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometExec3_4PlusSuite.scala @@ -145,7 +145,7 @@ class CometExec3_4PlusSuite extends CometTestBase { } test("test BloomFilterMightContain can take a constant value input") { - assume(!isSpark42Plus, "https://github.com/apache/datafusion-comet/issues/4142") + assume(!isSpark42Plus, "https://github.com/apache/datafusion-comet/issues/4968") val table = "test" withTable(table) { @@ -159,7 +159,7 @@ class CometExec3_4PlusSuite extends CometTestBase { } test("test NULL inputs for BloomFilterMightContain") { - assume(!isSpark42Plus, "https://github.com/apache/datafusion-comet/issues/4142") + assume(!isSpark42Plus, "https://github.com/apache/datafusion-comet/issues/4968") val table = "test" withTable(table) { @@ -176,7 +176,7 @@ class CometExec3_4PlusSuite extends CometTestBase { } test("test BloomFilterMightContain from random input") { - assume(!isSpark42Plus, "https://github.com/apache/datafusion-comet/issues/4142") + assume(!isSpark42Plus, "https://github.com/apache/datafusion-comet/issues/4968") val (longs, bfBytes) = bloomFilterFromRandomInput(10000, 10000) val table = "test" @@ -199,7 +199,7 @@ class CometExec3_4PlusSuite extends CometTestBase { } test("bloom_filter_agg caps oversized numItems / numBits like Spark") { - assume(!isSpark42Plus, "https://github.com/apache/datafusion-comet/issues/4142") + assume(!isSpark42Plus, "https://github.com/apache/datafusion-comet/issues/4968") val table = "test" withTable(table) { sql(s"create table $table(col1 long) using parquet") diff --git a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala index ce6071bf1a..1762258b87 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala @@ -2860,7 +2860,7 @@ class CometExecSuite extends CometTestBase { } test("bloom_filter_agg") { - assume(!isSpark42Plus, "https://github.com/apache/datafusion-comet/issues/4142") + assume(!isSpark42Plus, "https://github.com/apache/datafusion-comet/issues/4968") val funcId_bloom_filter_agg = new FunctionIdentifier("bloom_filter_agg") spark.sessionState.functionRegistry.registerFunction( funcId_bloom_filter_agg, diff --git a/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala index 603e9033a4..d97626cb44 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala @@ -364,7 +364,7 @@ class CometExecRuleSuite extends CometTestBase { } test("CometExecRule should allow BloomFilter mixed Comet partial and Spark final") { - assume(!isSpark42Plus, "https://github.com/apache/datafusion-comet/issues/4142") + assume(!isSpark42Plus, "https://github.com/apache/datafusion-comet/issues/4968") val funcId = new FunctionIdentifier("bloom_filter_agg") spark.sessionState.functionRegistry.registerFunction( funcId, @@ -403,7 +403,7 @@ class CometExecRuleSuite extends CometTestBase { } test("CometExecRule should allow BloomFilter mixed Spark partial and Comet final") { - assume(!isSpark42Plus, "https://github.com/apache/datafusion-comet/issues/4142") + assume(!isSpark42Plus, "https://github.com/apache/datafusion-comet/issues/4968") val funcId = new FunctionIdentifier("bloom_filter_agg") spark.sessionState.functionRegistry.registerFunction( funcId,