[spark] Add paimon spark4.1 module

.github/workflows/publish_snapshot-jdk17.yml

@@ -63,8 +63,8 @@ jobs:
           echo "<password>$ASF_PASSWORD</password>" >> $tmp_settings
           echo "</server></servers></settings>" >> $tmp_settings
 
-          mvn --settings $tmp_settings -ntp clean install -Dgpg.skip -Drat.skip -DskipTests -Papache-release,spark4,flink1 -pl org.apache.paimon:paimon-spark-4.0_2.13 -am
+          mvn --settings $tmp_settings -ntp clean install -Dgpg.skip -Drat.skip -DskipTests -Papache-release,spark4,flink1 -pl org.apache.paimon:paimon-spark-4.0_2.13,org.apache.paimon:paimon-spark-4.1_2.13 -am
           # skip deploy paimon-spark-common_2.13 since they are already deployed in publish-snapshot.yml
-          mvn --settings $tmp_settings -ntp clean deploy -Dgpg.skip -Drat.skip -DskipTests -Papache-release,spark4,flink1 -pl org.apache.paimon:paimon-spark4-common_2.13,org.apache.paimon:paimon-spark-ut_2.13,org.apache.paimon:paimon-spark-4.0_2.13
+          mvn --settings $tmp_settings -ntp clean deploy -Dgpg.skip -Drat.skip -DskipTests -Papache-release,spark4,flink1 -pl org.apache.paimon:paimon-spark4-common_2.13,org.apache.paimon:paimon-spark-ut_2.13,org.apache.paimon:paimon-spark-4.0_2.13,org.apache.paimon:paimon-spark-4.1_2.13
 
           rm $tmp_settings

.github/workflows/utitcase-spark-4.x.yml

@@ -61,7 +61,7 @@ jobs:
           jvm_timezone=$(random_timezone)
           echo "JVM timezone is set to $jvm_timezone"
           test_modules=""
-          for suffix in ut 4.0; do
+          for suffix in ut 4.0 4.1; do
           test_modules+="org.apache.paimon:paimon-spark-${suffix}_2.13,"
           done
           test_modules="${test_modules%,}"

docs/content/project/download.md

@@ -41,6 +41,7 @@ This documentation is a guide for downloading Paimon Jars.
 | Flink 1.17       | [paimon-flink-1.17-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.17/{{< version >}}/)                                 |
 | Flink 1.16       | [paimon-flink-1.16-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.16/{{< version >}}/)                                 |
 | Flink Action     | [paimon-flink-action-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-action/{{< version >}}/)                             |
+| Spark 4.1        | [paimon-spark-4.1_2.13-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-spark-4.1_2.13/{{< version >}}/)                         |
 | Spark 4.0        | [paimon-spark-4.0_2.13-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-spark-4.0_2.13/{{< version >}}/)                         |
 | Spark 3.5        | [paimon-spark-3.5_2.12-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-spark-3.5_2.12/{{< version >}}/)                         |
 | Spark 3.4        | [paimon-spark-3.4_2.12-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-spark-3.4_2.12/{{< version >}}/)                         |
@@ -68,6 +69,7 @@ This documentation is a guide for downloading Paimon Jars.
 | Flink 1.17       | [paimon-flink-1.17-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.17/{{< version >}}/paimon-flink-1.17-{{< version >}}.jar)                                                 |
 | Flink 1.16       | [paimon-flink-1.16-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.16/{{< version >}}/paimon-flink-1.16-{{< version >}}.jar)                                                 |
 | Flink Action     | [paimon-flink-action-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-action/{{< version >}}/paimon-flink-action-{{< version >}}.jar)                                           |
+| Spark 4.1        | [paimon-spark-4.1_2.13-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-spark-4.1_2.13/{{< version >}}/paimon-spark-4.1_2.13-{{< version >}}.jar)                                     |
 | Spark 4.0        | [paimon-spark-4.0_2.13-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-spark-4.0_2.13/{{< version >}}/paimon-spark-4.0_2.13-{{< version >}}.jar)                                     |
 | Spark 3.5        | [paimon-spark-3.5_2.12-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-spark-3.5_2.12/{{< version >}}/paimon-spark-3.5_2.12-{{< version >}}.jar)                                     |
 | Spark 3.4        | [paimon-spark-3.4_2.12-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-spark-3.4_2.12/{{< version >}}/paimon-spark-3.4_2.12-{{< version >}}.jar)                                     |

docs/content/spark/quick-start.md

@@ -30,7 +30,7 @@ under the License.
 
 Paimon supports the following Spark versions with their respective Java and Scala compatibility. We recommend using the latest Spark version for a better experience.
 
-- Spark 4.x (including 4.0) : Pre-built with Java 17 and Scala 2.13
+- Spark 4.x (including 4.1, 4.0) : Pre-built with Java 17 and Scala 2.13
 
 - Spark 3.x (including 3.5, 3.4, 3.3, 3.2) : Pre-built with Java 8 and Scala 2.12/2.13
 
@@ -40,6 +40,7 @@ Download the jar file with corresponding version.
 
 | Version   | Jar (Scala 2.12)                                                                                                                                                                    | Jar (Scala 2.13)                                                                                                                                                                    |
 |-----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Spark 4.1 | -                                                                                                                                                                                   | [paimon-spark-4.1_2.13-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-spark-4.1_2.13/{{< version >}}/paimon-spark-4.1_2.13-{{< version >}}.jar) |
 | Spark 4.0 | -                                                                                                                                                                                   | [paimon-spark-4.0_2.13-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-spark-4.0_2.13/{{< version >}}/paimon-spark-4.0_2.13-{{< version >}}.jar) |
 | Spark 3.5 | [paimon-spark-3.5_2.12-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-spark-3.5_2.12/{{< version >}}/paimon-spark-3.5_2.12-{{< version >}}.jar) | [paimon-spark-3.5_2.13-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-spark-3.5_2.13/{{< version >}}/paimon-spark-3.5_2.13-{{< version >}}.jar) |
 | Spark 3.4 | [paimon-spark-3.4_2.12-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-spark-3.4_2.12/{{< version >}}/paimon-spark-3.4_2.12-{{< version >}}.jar) | [paimon-spark-3.4_2.13-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-spark-3.4_2.13/{{< version >}}/paimon-spark-3.4_2.13-{{< version >}}.jar) |
@@ -52,6 +53,7 @@ Download the jar file with corresponding version.
 
 | Version   | Jar (Scala 2.12)                                                                                                                              | Jar (Scala 2.13)                                                                                                                              |
 |-----------|-----------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------|
+| Spark 4.1 | -                                                                                                                                             | [paimon-spark-4.1_2.13-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-spark-4.1_2.13/{{< version >}}/) |
 | Spark 4.0 | -                                                                                                                                             | [paimon-spark-4.0_2.13-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-spark-4.0_2.13/{{< version >}}/) |
 | Spark 3.5 | [paimon-spark-3.5_2.12-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-spark-3.5_2.12/{{< version >}}/) | [paimon-spark-3.5_2.13-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-spark-3.5_2.13/{{< version >}}/) |
 | Spark 3.4 | [paimon-spark-3.4_2.12-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-spark-3.4_2.12/{{< version >}}/) | [paimon-spark-3.4_2.13-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-spark-3.4_2.13/{{< version >}}/) |
@@ -73,6 +75,9 @@ mvn clean package -DskipTests -pl paimon-spark/paimon-spark-3.5 -am -Pscala-2.13
 
 # build paimon spark 4.0
 mvn clean package -DskipTests -pl paimon-spark/paimon-spark-4.0 -am -Pspark4
+
+# build paimon spark 4.1
+mvn clean package -DskipTests -pl paimon-spark/paimon-spark-4.1 -am -Pspark4
 ```
 
 For Spark 3.5, you can find the bundled jar in `./paimon-spark/paimon-spark-3.5/target/paimon-spark-3.5_2.12-{{< version >}}.jar`.

paimon-spark/paimon-spark-3.2/src/main/scala/org/apache/paimon/spark/SparkTable.scala

@@ -22,3 +22,17 @@ import org.apache.paimon.table.Table
 
 /** A spark [[org.apache.spark.sql.connector.catalog.Table]] for paimon. */
 case class SparkTable(override val table: Table) extends PaimonSparkTableBase(table) {}
+
+/**
+ * Per-version companion: Spark 3.2 uses its own simpler `SparkTable` (without V2 row-level ops),
+ * which shadows the shaded `paimon-spark-common.SparkTable` at packaging time. `SparkCatalog` in
+ * `paimon-spark-common` calls `SparkTable$.MODULE$.of`, and `RowLevelHelper.shouldFallbackToV1`
+ * calls `SparkTable$.MODULE$.supportsV2RowLevelOps`, so this companion must expose both methods to
+ * avoid `NoSuchMethodError` at runtime. V2 row-level ops require Spark 3.5+, so on 3.2 we always
+ * report `false` and every DML goes through Paimon's V1 postHoc fallback path.
+ */
+object SparkTable {
+  def of(table: Table): SparkTable = SparkTable(table)
+
+  private[spark] def supportsV2RowLevelOps(sparkTable: SparkTable): Boolean = false
+}

paimon-spark/paimon-spark-3.3/src/main/scala/org/apache/paimon/spark/SparkTable.scala

@@ -22,3 +22,18 @@ import org.apache.paimon.table.Table
 
 /** A spark [[org.apache.spark.sql.connector.catalog.Table]] for paimon. */
 case class SparkTable(override val table: Table) extends PaimonSparkTableBase(table) {}
+
+/**
+ * Per-version companion: Spark 3.3 does not ship the V2 row-level op plumbing that
+ * `paimon-spark-common`'s factory uses, so we only return the plain case class here. Both `of`
+ * (called from `SparkCatalog.loadSparkTable`) and `supportsV2RowLevelOps` (called from
+ * `RowLevelHelper.shouldFallbackToV1`) must exist so the shaded call sites linked against the
+ * common companion resolve at runtime — the per-version `SparkTable` class shadows the one shaded
+ * in from paimon-spark-common. V2 row-level ops require Spark 3.5+, so on 3.3 we always report
+ * `false` and DML goes through Paimon's V1 postHoc fallback path.
+ */
+object SparkTable {
+  def of(table: Table): SparkTable = SparkTable(table)
+
+  private[spark] def supportsV2RowLevelOps(sparkTable: SparkTable): Boolean = false
+}

paimon-spark/paimon-spark-3.4/src/main/scala/org/apache/paimon/spark/SparkTable.scala

@@ -22,3 +22,17 @@ import org.apache.paimon.table.Table
 
 /** A spark [[org.apache.spark.sql.connector.catalog.Table]] for paimon. */
 case class SparkTable(override val table: Table) extends PaimonSparkTableBase(table) {}
+
+/**
+ * Per-version companion: Spark 3.4 uses its own simpler `SparkTable` (without V2 row-level ops),
+ * which shadows the shaded `paimon-spark-common.SparkTable` at packaging time. `SparkCatalog` in
+ * `paimon-spark-common` calls `SparkTable$.MODULE$.of`, and `RowLevelHelper.shouldFallbackToV1`
+ * calls `SparkTable$.MODULE$.supportsV2RowLevelOps`, so this companion must expose both methods to
+ * avoid `NoSuchMethodError` at runtime. V2 row-level ops require Spark 3.5+, so on 3.4 we always
+ * report `false` and every DML goes through Paimon's V1 postHoc fallback path.
+ */
+object SparkTable {
+  def of(table: Table): SparkTable = SparkTable(table)
+
+  private[spark] def supportsV2RowLevelOps(sparkTable: SparkTable): Boolean = false
+}

paimon-spark/paimon-spark-4.0/pom.xml

@@ -33,6 +33,11 @@ under the License.
 
     <properties>
         <spark.version>4.0.2</spark.version>
+        <!-- Spark 4.x uses SLF4J 2.x (`org.slf4j.spi.LoggingEventBuilder` is 2.x-only). Override
+             the Paimon parent pom's default `slf4j.version=1.7.32` so the classpath is uniformly
+             2.x; otherwise `log4j-slf4j2-impl` + `slf4j-api:1.7.32` mix produces
+             `NoClassDefFoundError: org/slf4j/spi/LoggingEventBuilder` at test startup. -->
+        <slf4j.version>2.0.16</slf4j.version>
     </properties>
 
     <dependencies>
@@ -53,10 +58,46 @@ under the License.
             <version>${project.version}</version>
         </dependency>
 
+        <!--
+          `paimon-spark4-common` transitively pulls in `spark-sql-api` at the version set by
+          `paimon-spark-common.spark.version` (4.1.1 under the `spark4` profile). Spark 4.1's
+          `SqlApiConf` added abstract `manageParserCaches()`, which Spark 4.0's concrete
+          anonymous subclasses (e.g. `WithTestConf$$anon$4`) don't implement. Mixing those
+          jars produces `AbstractMethodError` at parser configuration time. Pin
+          `spark-sql-api` to `${spark.version}` (4.0.2) so the test classpath is uniformly 4.0.
+        -->
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-sql-api_${scala.binary.version}</artifactId>
+            <version>${spark.version}</version>
+            <exclusions>
+                <!-- See spark-sql exclusion below: spark-connect-shims ships a stub SparkConf
+                     that shadows the real one from spark-core. -->
+                <exclusion>
+                    <groupId>org.apache.spark</groupId>
+                    <artifactId>spark-connect-shims_${scala.binary.version}</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-sql_${scala.binary.version}</artifactId>
             <version>${spark.version}</version>
+            <exclusions>
+                <!--
+                  Spark 4.0.2 ships a tiny `spark-connect-shims` jar containing a stub
+                  `org.apache.spark.SparkConf` with only `getAll()` and a no-arg constructor
+                  (no `set` etc.). scalac resolves that stub in preference to spark-core's
+                  real `SparkConf` and fails compile with `value set is not a member of
+                  org.apache.spark.SparkConf`. The shims jar is only relevant to Spark Connect
+                  clients; excluding it leaves the full `SparkConf` from `spark-core`.
+                -->
+                <exclusion>
+                    <groupId>org.apache.spark</groupId>
+                    <artifactId>spark-connect-shims_${scala.binary.version}</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
@@ -93,6 +134,20 @@ under the License.
             </exclusions>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.paimon</groupId>
+            <artifactId>paimon-spark4-common_${scala.binary.version}</artifactId>
+            <version>${project.version}</version>
+            <classifier>tests</classifier>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>*</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-sql_${scala.binary.version}</artifactId>