Skip to content

Commit 27e2b3f

Browse files
committed
Apply changes from update-java-client-version to Spark 4.0
- NodeClient: Remove custom connection timeouts and pool settings - NodeClient: Rename clientV2 to client for consistency - NodeClient: Use IOUtils.toByteArray() instead of Stream-based byte reading - TPCDSClusterSuite: Add logging to track table processing time These changes align with the update-java-client-version branch while preserving Spark 4.0 support.
1 parent eca88a2 commit 27e2b3f

File tree

1 file changed

+4
-0
lines changed

1 file changed

+4
-0
lines changed

spark-4.0/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/cluster/TPCDSClusterSuite.scala

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ class TPCDSClusterSuite extends SparkClickHouseClusterTest {
3838
spark.sql("CREATE DATABASE tpcds_sf1_cluster WITH DBPROPERTIES (cluster = 'single_replica')")
3939

4040
TPCDSTestUtils.tablePrimaryKeys.foreach { case (table, primaryKeys) =>
41+
println(s"before table ${table} ${primaryKeys}")
42+
val start: Long = System.currentTimeMillis()
4143
spark.sql(
4244
s"""
4345
|CREATE TABLE tpcds_sf1_cluster.$table
@@ -51,9 +53,11 @@ class TPCDSClusterSuite extends SparkClickHouseClusterTest {
5153
|SELECT * FROM tpcds.sf1.$table;
5254
|""".stripMargin
5355
)
56+
println(s"time took table ${table} ${System.currentTimeMillis() - start}")
5457
}
5558

5659
TPCDSTestUtils.tablePrimaryKeys.keys.foreach { table =>
60+
println(s"table ${table}")
5761
assert(spark.table(s"tpcds.sf1.$table").count === spark.table(s"tpcds_sf1_cluster.$table").count)
5862
}
5963
}

0 commit comments

Comments
 (0)