Fix Scala 2.13 compatibility for nested arrays

ShimonSte · ShimonSte · commit 2687b1f44ee0 · 2025-11-10T11:12:27.000+02:00
- Convert mutable.ArraySeq to Array in ClickHouseJsonReader to ensure immutable collections
- Add test workaround for Spark's Row.getSeq behavior in Scala 2.13
- Fix Spotless formatting: remove trailing whitespace in ClickHouseBinaryReader
- Applied to all Spark versions: 3.3, 3.4, 3.5
diff --git a/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseWriterTestBase.scala b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseWriterTestBase.scala
@@ -90,10 +90,14 @@ trait ClickHouseWriterTestBase extends SparkClickHouseSingleTest {
 
       val result = spark.table("test_db.test_write_nested_array").orderBy("id").collect()
       assert(result.length == 3)
-      assert(result(0).getSeq[Seq[Int]](1) == Seq(Seq(1, 2), Seq(3, 4)))
-      assert(result(1).getSeq[Seq[Int]](1) == Seq(Seq(10, 20, 30)))
-      assert(result(2).getSeq[Seq[Int]](1)(0).isEmpty)
-      assert(result(2).getSeq[Seq[Int]](1)(1) == Seq(100))
+      // Convert to List for Scala 2.12/2.13 compatibility
+      val row0 = result(0).getAs[scala.collection.Seq[scala.collection.Seq[Int]]](1).map(_.toList).toList
+      val row1 = result(1).getAs[scala.collection.Seq[scala.collection.Seq[Int]]](1).map(_.toList).toList
+      val row2 = result(2).getAs[scala.collection.Seq[scala.collection.Seq[Int]]](1).map(_.toList).toList
+      assert(row0 == Seq(Seq(1, 2), Seq(3, 4)))
+      assert(row1 == Seq(Seq(10, 20, 30)))
+      assert(row2(0).isEmpty)
+      assert(row2(1) == Seq(100))
     }
   }
 
@@ -284,7 +288,6 @@ trait ClickHouseWriterTestBase extends SparkClickHouseSingleTest {
     }
   }
 
-
   test("write DecimalType - Decimal(18,4)") {
     // Note: High-precision decimals (>15-17 significant digits) may lose precision in JSON/Arrow formats.
     // This appears to be related to the serialization/deserialization path, possibly due to intermediate
@@ -706,7 +709,6 @@ trait ClickHouseWriterTestBase extends SparkClickHouseSingleTest {
     }
   }
 
-
   test("write TimestampType - nullable with null values") {
     val schema = StructType(Seq(
       StructField("id", IntegerType, nullable = false),
diff --git a/spark-3.3/clickhouse-spark/src/main/scala/com/clickhouse/spark/read/format/ClickHouseBinaryReader.scala b/spark-3.3/clickhouse-spark/src/main/scala/com/clickhouse/spark/read/format/ClickHouseBinaryReader.scala
@@ -108,7 +108,7 @@ class ClickHouseBinaryReader(
       case TimestampType =>
         var _instant = value.asInstanceOf[ZonedDateTime].withZoneSameInstant(ZoneOffset.UTC)
         TimeUnit.SECONDS.toMicros(_instant.toEpochSecond) + TimeUnit.NANOSECONDS.toMicros(_instant.getNano())
-      case StringType => 
+      case StringType =>
         val strValue = value match {
           case uuid: java.util.UUID => uuid.toString
           case inet: java.net.InetAddress => inet.getHostAddress
@@ -117,7 +117,7 @@ class ClickHouseBinaryReader(
           case _ => value.toString
         }
         UTF8String.fromString(strValue)
-      case DateType => 
+      case DateType =>
         val localDate = value match {
           case ld: LocalDate => ld
           case zdt: ZonedDateTime => zdt.toLocalDate
diff --git a/spark-3.3/clickhouse-spark/src/main/scala/com/clickhouse/spark/read/format/ClickHouseJsonReader.scala b/spark-3.3/clickhouse-spark/src/main/scala/com/clickhouse/spark/read/format/ClickHouseJsonReader.scala
@@ -93,7 +93,7 @@ class ClickHouseJsonReader(
         jsonNode.binaryValue
       case ArrayType(_dataType, _nullable) =>
         val _structField = StructField(s"${structField.name}__array_element__", _dataType, _nullable)
-        new GenericArrayData(jsonNode.asScala.map(decodeValue(_, _structField)))
+        new GenericArrayData(jsonNode.asScala.map(decodeValue(_, _structField)).toArray)
       case MapType(StringType, _valueType, _valueNullable) =>
         val mapData = jsonNode.fields.asScala.map { entry =>
           val _structField = StructField(s"${structField.name}__map_value__", _valueType, _valueNullable)
diff --git a/spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseWriterTestBase.scala b/spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseWriterTestBase.scala
@@ -90,10 +90,14 @@ trait ClickHouseWriterTestBase extends SparkClickHouseSingleTest {
 
       val result = spark.table("test_db.test_write_nested_array").orderBy("id").collect()
       assert(result.length == 3)
-      assert(result(0).getSeq[Seq[Int]](1) == Seq(Seq(1, 2), Seq(3, 4)))
-      assert(result(1).getSeq[Seq[Int]](1) == Seq(Seq(10, 20, 30)))
-      assert(result(2).getSeq[Seq[Int]](1)(0).isEmpty)
-      assert(result(2).getSeq[Seq[Int]](1)(1) == Seq(100))
+      // Convert to List for Scala 2.12/2.13 compatibility
+      val row0 = result(0).getAs[scala.collection.Seq[scala.collection.Seq[Int]]](1).map(_.toList).toList
+      val row1 = result(1).getAs[scala.collection.Seq[scala.collection.Seq[Int]]](1).map(_.toList).toList
+      val row2 = result(2).getAs[scala.collection.Seq[scala.collection.Seq[Int]]](1).map(_.toList).toList
+      assert(row0 == Seq(Seq(1, 2), Seq(3, 4)))
+      assert(row1 == Seq(Seq(10, 20, 30)))
+      assert(row2(0).isEmpty)
+      assert(row2(1) == Seq(100))
     }
   }
 
@@ -284,7 +288,6 @@ trait ClickHouseWriterTestBase extends SparkClickHouseSingleTest {
     }
   }
 
-
   test("write DecimalType - Decimal(18,4)") {
     // Note: High-precision decimals (>15-17 significant digits) may lose precision in JSON/Arrow formats.
     // This appears to be related to the serialization/deserialization path, possibly due to intermediate
@@ -706,7 +709,6 @@ trait ClickHouseWriterTestBase extends SparkClickHouseSingleTest {
     }
   }
 
-
   test("write TimestampType - nullable with null values") {
     val schema = StructType(Seq(
       StructField("id", IntegerType, nullable = false),
diff --git a/spark-3.4/clickhouse-spark/src/main/scala/com/clickhouse/spark/read/format/ClickHouseBinaryReader.scala b/spark-3.4/clickhouse-spark/src/main/scala/com/clickhouse/spark/read/format/ClickHouseBinaryReader.scala
@@ -108,7 +108,7 @@ class ClickHouseBinaryReader(
       case TimestampType =>
         var _instant = value.asInstanceOf[ZonedDateTime].withZoneSameInstant(ZoneOffset.UTC)
         TimeUnit.SECONDS.toMicros(_instant.toEpochSecond) + TimeUnit.NANOSECONDS.toMicros(_instant.getNano())
-      case StringType => 
+      case StringType =>
         val strValue = value match {
           case uuid: java.util.UUID => uuid.toString
           case inet: java.net.InetAddress => inet.getHostAddress
@@ -117,7 +117,7 @@ class ClickHouseBinaryReader(
           case _ => value.toString
         }
         UTF8String.fromString(strValue)
-      case DateType => 
+      case DateType =>
         val localDate = value match {
           case ld: LocalDate => ld
           case zdt: ZonedDateTime => zdt.toLocalDate
diff --git a/spark-3.4/clickhouse-spark/src/main/scala/com/clickhouse/spark/read/format/ClickHouseJsonReader.scala b/spark-3.4/clickhouse-spark/src/main/scala/com/clickhouse/spark/read/format/ClickHouseJsonReader.scala
@@ -93,7 +93,7 @@ class ClickHouseJsonReader(
         jsonNode.binaryValue
       case ArrayType(_dataType, _nullable) =>
         val _structField = StructField(s"${structField.name}__array_element__", _dataType, _nullable)
-        new GenericArrayData(jsonNode.asScala.map(decodeValue(_, _structField)))
+        new GenericArrayData(jsonNode.asScala.map(decodeValue(_, _structField)).toArray)
       case MapType(StringType, _valueType, _valueNullable) =>
         val mapData = jsonNode.fields.asScala.map { entry =>
           val _structField = StructField(s"${structField.name}__map_value__", _valueType, _valueNullable)
diff --git a/spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseWriterTestBase.scala b/spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseWriterTestBase.scala
@@ -90,10 +90,14 @@ trait ClickHouseWriterTestBase extends SparkClickHouseSingleTest {
 
       val result = spark.table("test_db.test_write_nested_array").orderBy("id").collect()
       assert(result.length == 3)
-      assert(result(0).getSeq[Seq[Int]](1) == Seq(Seq(1, 2), Seq(3, 4)))
-      assert(result(1).getSeq[Seq[Int]](1) == Seq(Seq(10, 20, 30)))
-      assert(result(2).getSeq[Seq[Int]](1)(0).isEmpty)
-      assert(result(2).getSeq[Seq[Int]](1)(1) == Seq(100))
+      // Convert to List for Scala 2.12/2.13 compatibility
+      val row0 = result(0).getAs[scala.collection.Seq[scala.collection.Seq[Int]]](1).map(_.toList).toList
+      val row1 = result(1).getAs[scala.collection.Seq[scala.collection.Seq[Int]]](1).map(_.toList).toList
+      val row2 = result(2).getAs[scala.collection.Seq[scala.collection.Seq[Int]]](1).map(_.toList).toList
+      assert(row0 == Seq(Seq(1, 2), Seq(3, 4)))
+      assert(row1 == Seq(Seq(10, 20, 30)))
+      assert(row2(0).isEmpty)
+      assert(row2(1) == Seq(100))
     }
   }
 
diff --git a/spark-3.5/clickhouse-spark/src/main/scala/com/clickhouse/spark/read/format/ClickHouseJsonReader.scala b/spark-3.5/clickhouse-spark/src/main/scala/com/clickhouse/spark/read/format/ClickHouseJsonReader.scala
@@ -93,7 +93,7 @@ class ClickHouseJsonReader(
         jsonNode.binaryValue
       case ArrayType(_dataType, _nullable) =>
         val _structField = StructField(s"${structField.name}__array_element__", _dataType, _nullable)
-        new GenericArrayData(jsonNode.asScala.map(decodeValue(_, _structField)))
+        new GenericArrayData(jsonNode.asScala.map(decodeValue(_, _structField)).toArray)
       case MapType(StringType, _valueType, _valueNullable) =>
         val mapData = jsonNode.fields.asScala.map { entry =>
           val _structField = StructField(s"${structField.name}__map_value__", _valueType, _valueNullable)

Original file line number	Diff line number	Diff line change
`@@ -90,10 +90,14 @@ trait ClickHouseWriterTestBase extends SparkClickHouseSingleTest {`
`90`	`90`
`91`	`91`	`val result = spark.table("test_db.test_write_nested_array").orderBy("id").collect()`
`92`	`92`	`assert(result.length == 3)`
`93`		`- assert(result(0).getSeq[Seq[Int]](1) == Seq(Seq(1, 2), Seq(3, 4)))`
`94`		`- assert(result(1).getSeq[Seq[Int]](1) == Seq(Seq(10, 20, 30)))`
`95`		`- assert(result(2).getSeq[Seq[Int]](1)(0).isEmpty)`
`96`		`- assert(result(2).getSeq[Seq[Int]](1)(1) == Seq(100))`
	`93`	`+ // Convert to List for Scala 2.12/2.13 compatibility`
	`94`	`+ val row0 = result(0).getAs[scala.collection.Seq[scala.collection.Seq[Int]]](1).map(_.toList).toList`
	`95`	`+ val row1 = result(1).getAs[scala.collection.Seq[scala.collection.Seq[Int]]](1).map(_.toList).toList`
	`96`	`+ val row2 = result(2).getAs[scala.collection.Seq[scala.collection.Seq[Int]]](1).map(_.toList).toList`
	`97`	`+ assert(row0 == Seq(Seq(1, 2), Seq(3, 4)))`
	`98`	`+ assert(row1 == Seq(Seq(10, 20, 30)))`
	`99`	`+ assert(row2(0).isEmpty)`
	`100`	`+ assert(row2(1) == Seq(100))`
`97`	`101`	`}`
`98`	`102`	`}`
`99`	`103`
`@@ -284,7 +288,6 @@ trait ClickHouseWriterTestBase extends SparkClickHouseSingleTest {`
`284`	`288`	`}`
`285`	`289`	`}`
`286`	`290`
`287`		`-`
`288`	`291`	`test("write DecimalType - Decimal(18,4)") {`
`289`	`292`	`// Note: High-precision decimals (>15-17 significant digits) may lose precision in JSON/Arrow formats.`
`290`	`293`	`// This appears to be related to the serialization/deserialization path, possibly due to intermediate`
`@@ -706,7 +709,6 @@ trait ClickHouseWriterTestBase extends SparkClickHouseSingleTest {`
`706`	`709`	`}`
`707`	`710`	`}`
`708`	`711`
`709`		`-`
`710`	`712`	`test("write TimestampType - nullable with null values") {`
`711`	`713`	`val schema = StructType(Seq(`
`712`	`714`	`StructField("id", IntegerType, nullable = false),`