From 0d4481cdaea85e857503f529de26429e6fb6d4d1 Mon Sep 17 00:00:00 2001 From: Simon Kesenci Date: Thu, 16 Jul 2026 08:39:58 -0400 Subject: [PATCH 1/4] Convert Spark show and fix create_function in udf.register. Match Spark API show() signature to PySpark's. Convert (more or less) show() parameters to DuckDB show() parameters. - n -> max_rows - truncate -> max_col_width (not quite the same) - vertical -> render_mode='COLUMNS' (not quite the same) Call to create_function() under Spark API udf.register() fixes: - Pass None for third (function parameters) argument. - Assume udf.register() receives a PySpark return type. - Convert to DuckDB return type for create_function() call. --- duckdb/experimental/spark/sql/dataframe.py | 23 ++++++++++++++++++++-- duckdb/experimental/spark/sql/udf.py | 3 ++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/duckdb/experimental/spark/sql/dataframe.py b/duckdb/experimental/spark/sql/dataframe.py index 83b2dd09..03f14989 100644 --- a/duckdb/experimental/spark/sql/dataframe.py +++ b/duckdb/experimental/spark/sql/dataframe.py @@ -38,8 +38,27 @@ def __init__(self, relation: duckdb.DuckDBPyRelation, session: "SparkSession") - if self.relation is not None: self._schema = duckdb_to_spark_schema(self.relation.columns, self.relation.types) - def show(self, **kwargs) -> None: # noqa: D102 - self.relation.show() + def show( + self, + n: int = 20, + truncate: Union[bool, int] = True, + vertical: bool = False, + **kwargs, + ) -> None: # noqa: D102 + if isinstance(truncate, int): + max_col_width = truncate + elif truncate: + max_col_width = 20 + else: + max_col_width = 9999 + + render_mode = 'COLUMNS' if vertical else None + + self.relation.show( + max_rows=n, + max_col_width=max_col_width, + render_mode=render_mode + ) def toPandas(self) -> "PandasDataFrame": # noqa: D102 return self.relation.df() diff --git a/duckdb/experimental/spark/sql/udf.py b/duckdb/experimental/spark/sql/udf.py index c22f6be9..15c87efd 100644 --- a/duckdb/experimental/spark/sql/udf.py +++ b/duckdb/experimental/spark/sql/udf.py @@ -23,7 +23,8 @@ def register( # noqa: D102 f: "Callable[..., Any] | UserDefinedFunctionLike", returnType: Optional["DataTypeOrString"] = None, ) -> "UserDefinedFunctionLike": - self.sparkSession.conn.create_function(name, f, return_type=returnType) + self.sparkSession.conn.create_function(name, f, None, + return_type=returnType.duckdb_type) def registerJavaFunction( # noqa: D102 self, From 4ade1cddd6bd09bd51877e27ce1ea6a4f95bdbe2 Mon Sep 17 00:00:00 2001 From: Simon Kesenci Date: Thu, 16 Jul 2026 22:16:09 -0400 Subject: [PATCH 2/4] Handle null arguments to UDFs like PySpark. PySpark UDFs take nulls as valid input. Vis-a-vis other UDF options (exception handling and side effects) PySpark UDFs behave like the DuckDB defaults. --- duckdb/experimental/spark/sql/dataframe.py | 1 - duckdb/experimental/spark/sql/udf.py | 9 +++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/duckdb/experimental/spark/sql/dataframe.py b/duckdb/experimental/spark/sql/dataframe.py index 03f14989..1dae6662 100644 --- a/duckdb/experimental/spark/sql/dataframe.py +++ b/duckdb/experimental/spark/sql/dataframe.py @@ -43,7 +43,6 @@ def show( n: int = 20, truncate: Union[bool, int] = True, vertical: bool = False, - **kwargs, ) -> None: # noqa: D102 if isinstance(truncate, int): max_col_width = truncate diff --git a/duckdb/experimental/spark/sql/udf.py b/duckdb/experimental/spark/sql/udf.py index 15c87efd..47065236 100644 --- a/duckdb/experimental/spark/sql/udf.py +++ b/duckdb/experimental/spark/sql/udf.py @@ -23,8 +23,13 @@ def register( # noqa: D102 f: "Callable[..., Any] | UserDefinedFunctionLike", returnType: Optional["DataTypeOrString"] = None, ) -> "UserDefinedFunctionLike": - self.sparkSession.conn.create_function(name, f, None, - return_type=returnType.duckdb_type) + self.sparkSession.conn.create_function( + name, + f, + None, + return_type=returnType.duckdb_type, + null_handling='special', + ) def registerJavaFunction( # noqa: D102 self, From 1ac50db86cb40ae99bee80a30edce2d070aba647 Mon Sep 17 00:00:00 2001 From: Simon Kesenci Date: Fri, 17 Jul 2026 15:20:25 -0400 Subject: [PATCH 3/4] Superficial change to allow new commit. --- duckdb/experimental/spark/sql/udf.py | 2 +- external/duckdb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/duckdb/experimental/spark/sql/udf.py b/duckdb/experimental/spark/sql/udf.py index 47065236..fc944646 100644 --- a/duckdb/experimental/spark/sql/udf.py +++ b/duckdb/experimental/spark/sql/udf.py @@ -28,7 +28,7 @@ def register( # noqa: D102 f, None, return_type=returnType.duckdb_type, - null_handling='special', + null_handling='special', # PySpark UDFs handle nulls ) def registerJavaFunction( # noqa: D102 diff --git a/external/duckdb b/external/duckdb index 3a3c412a..a7378a20 160000 --- a/external/duckdb +++ b/external/duckdb @@ -1 +1 @@ -Subproject commit 3a3c412af5b55b6d4e33686732a8110b9bf03542 +Subproject commit a7378a20fd7505773ebe76c06bc2b77a2d7ef184 From 6e56c5eaeec3c8cd86f31bc7fa722f20688695ca Mon Sep 17 00:00:00 2001 From: Simon Kesenci Date: Fri, 17 Jul 2026 23:56:00 -0400 Subject: [PATCH 4/4] Set Spark UDF return type to None if passed None. --- duckdb/experimental/spark/sql/udf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/duckdb/experimental/spark/sql/udf.py b/duckdb/experimental/spark/sql/udf.py index fc944646..95f32500 100644 --- a/duckdb/experimental/spark/sql/udf.py +++ b/duckdb/experimental/spark/sql/udf.py @@ -27,7 +27,7 @@ def register( # noqa: D102 name, f, None, - return_type=returnType.duckdb_type, + return_type=returnType.duckdb_type if returnType else None, null_handling='special', # PySpark UDFs handle nulls )