Skip to content

Commit 8a270a2

Browse files
committed
Spark 3.4 UDF: change pmod to mod because positiveModulo does not exists in early version of clickhouse
(cherry picked from commit ea0592d)
1 parent 936a18a commit 8a270a2

File tree

4 files changed

+7
-10
lines changed

4 files changed

+7
-10
lines changed

spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/cluster/ClusterShardByTransformSuite.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ class ClusterShardByTransformSuite extends SparkClickHouseClusterTest {
108108
("murmurHash3_64", Array("value")),
109109
("murmurHash3_32", Array("value")),
110110
("cityHash64", Array("value")),
111-
("positiveModulo", Array("toYYYYMM(create_date)", "10"))
111+
("modulo", Array("toYYYYMM(create_date)", "10"))
112112
).foreach {
113113
case (func_name: String, func_args: Array[String]) =>
114114
test(s"shard by $func_name(${func_args.mkString(",")})")(runTest(func_name, func_args))

spark-3.4/clickhouse-spark/src/main/scala/org/apache/spark/sql/clickhouse/ExprUtils.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -186,5 +186,5 @@ object ExprUtils extends SQLConfHelper with Serializable {
186186
}
187187

188188
def toSplitWithModulo(shardingKey: Expr, weight: Int): FuncExpr =
189-
FuncExpr("positiveModulo", List(shardingKey, StringLiteral(weight.toString)))
189+
FuncExpr("modulo", List(shardingKey, StringLiteral(weight.toString)))
190190
}

spark-3.4/clickhouse-spark/src/main/scala/xenon/clickhouse/func/FunctionRegistry.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ object StaticFunctionRegistry extends FunctionRegistry {
5959
"clickhouse_months" -> Months,
6060
"clickhouse_days" -> Days,
6161
"clickhouse_hours" -> Hours,
62-
"sharding_pmod" -> Pmod
62+
"sharding_mod" -> Mod
6363
)
6464

6565
override def list: Array[String] = functions.keys.toArray

spark-3.4/clickhouse-spark/src/main/scala/xenon/clickhouse/func/clickhouse/Pmod.scala renamed to spark-3.4/clickhouse-spark/src/main/scala/xenon/clickhouse/func/clickhouse/Mod.scala

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,15 +18,15 @@ import org.apache.spark.sql.connector.catalog.functions.{BoundFunction, ScalarFu
1818
import org.apache.spark.sql.types._
1919
import xenon.clickhouse.func.ClickhouseEquivFunction
2020

21-
object Pmod extends UnboundFunction with ScalarFunction[Long] with ClickhouseEquivFunction {
21+
object Mod extends UnboundFunction with ScalarFunction[Long] with ClickhouseEquivFunction {
2222

23-
override def name: String = "sharding_pmod"
23+
override def name: String = "sharding_mod"
2424

2525
override def canonicalName: String = s"clickhouse.$name"
2626

2727
override def toString: String = name
2828

29-
override val ckFuncNames: Array[String] = Array("positiveModulo", "positive_modulo", "pmod")
29+
override val ckFuncNames: Array[String] = Array("modulo", "remainder")
3030

3131
override def description: String = s"$name: (a: long, b: long) => mod: long"
3232

@@ -58,8 +58,5 @@ object Pmod extends UnboundFunction with ScalarFunction[Long] with ClickhouseEqu
5858

5959
override def isResultNullable: Boolean = false
6060

61-
def invoke(a: Long, b: Long): Long = {
62-
val mod = a % b
63-
if (mod < 0) mod + b else mod
64-
}
61+
def invoke(a: Long, b: Long): Long = a % b
6562
}

0 commit comments

Comments
 (0)