diff --git a/duckdb/experimental/spark/sql/dataframe.py b/duckdb/experimental/spark/sql/dataframe.py index 83b2dd09..1dae6662 100644 --- a/duckdb/experimental/spark/sql/dataframe.py +++ b/duckdb/experimental/spark/sql/dataframe.py @@ -38,8 +38,26 @@ def __init__(self, relation: duckdb.DuckDBPyRelation, session: "SparkSession") - if self.relation is not None: self._schema = duckdb_to_spark_schema(self.relation.columns, self.relation.types) - def show(self, **kwargs) -> None: # noqa: D102 - self.relation.show() + def show( + self, + n: int = 20, + truncate: Union[bool, int] = True, + vertical: bool = False, + ) -> None: # noqa: D102 + if isinstance(truncate, int): + max_col_width = truncate + elif truncate: + max_col_width = 20 + else: + max_col_width = 9999 + + render_mode = 'COLUMNS' if vertical else None + + self.relation.show( + max_rows=n, + max_col_width=max_col_width, + render_mode=render_mode + ) def toPandas(self) -> "PandasDataFrame": # noqa: D102 return self.relation.df() diff --git a/duckdb/experimental/spark/sql/udf.py b/duckdb/experimental/spark/sql/udf.py index c22f6be9..95f32500 100644 --- a/duckdb/experimental/spark/sql/udf.py +++ b/duckdb/experimental/spark/sql/udf.py @@ -23,7 +23,13 @@ def register( # noqa: D102 f: "Callable[..., Any] | UserDefinedFunctionLike", returnType: Optional["DataTypeOrString"] = None, ) -> "UserDefinedFunctionLike": - self.sparkSession.conn.create_function(name, f, return_type=returnType) + self.sparkSession.conn.create_function( + name, + f, + None, + return_type=returnType.duckdb_type if returnType else None, + null_handling='special', # PySpark UDFs handle nulls + ) def registerJavaFunction( # noqa: D102 self, diff --git a/external/duckdb b/external/duckdb index 3a3c412a..a7378a20 160000 --- a/external/duckdb +++ b/external/duckdb @@ -1 +1 @@ -Subproject commit 3a3c412af5b55b6d4e33686732a8110b9bf03542 +Subproject commit a7378a20fd7505773ebe76c06bc2b77a2d7ef184