From 75d47a5f26be6aa731967bd162d5747fcbbd7f7a Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Tue, 4 Aug 2026 01:29:27 +0000 Subject: [PATCH 1/2] [SPARK-58533][PS] Use native Spark function for NumPy float_power --- python/pyspark/pandas/numpy_compat.py | 4 +--- .../pyspark/pandas/tests/test_numpy_compat.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/python/pyspark/pandas/numpy_compat.py b/python/pyspark/pandas/numpy_compat.py index 1653121d3124..ef45e5fe4e22 100644 --- a/python/pyspark/pandas/numpy_compat.py +++ b/python/pyspark/pandas/numpy_compat.py @@ -98,9 +98,7 @@ "copysign": pandas_udf( # type: ignore[call-overload] lambda s1, s2: np.copysign(s1, s2), DoubleType() ), - "float_power": pandas_udf( # type: ignore[call-overload] - lambda s1, s2: np.float_power(s1, s2), DoubleType() - ), + "float_power": lambda c1, c2: F.pow(c1.cast("double"), c2.cast("double")), "floor_divide": pandas_udf( # type: ignore[call-overload] lambda s1, s2: np.floor_divide(s1, s2), DoubleType() ), diff --git a/python/pyspark/pandas/tests/test_numpy_compat.py b/python/pyspark/pandas/tests/test_numpy_compat.py index 2530d6b04a80..3d9c63a9558e 100644 --- a/python/pyspark/pandas/tests/test_numpy_compat.py +++ b/python/pyspark/pandas/tests/test_numpy_compat.py @@ -149,6 +149,24 @@ def test_np_bitwise_shift_functions(self): np_func(psdf.value, psdf.bits), np_func(pdf.value, pdf.bits), almost=True ) + def test_np_float_power(self): + for pdf in ( + pd.DataFrame({"base": [-64, -2, -1, 0, 1, 2, 64], "exponent": [-2, -1, 0, 1, 2, 3, 2]}), + pd.DataFrame( + { + "base": [-np.inf, -64.0, -2.0, -0.0, 0.0, 2.0, 64.0, np.inf, np.nan], + "exponent": [2.0, 3.0, -2.0, -3.0, -3.0, 0.5, -2.0, 2.0, 2.0], + } + ), + ): + with self.subTest(base=pdf.base.tolist(), exponent=pdf.exponent.tolist()): + psdf = ps.from_pandas(pdf) + self.assert_eq( + np.float_power(psdf.base, psdf.exponent), + np.float_power(pdf.base, pdf.exponent), + almost=True, + ) + def test_np_spark_compat_series(self): from pyspark.pandas.numpy_compat import unary_np_spark_mappings, binary_np_spark_mappings From 7eb2c12e349f2ddef954d107a91c094a594771f7 Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Tue, 4 Aug 2026 07:02:17 +0000 Subject: [PATCH 2/2] [SPARK-58533][PS] Avoid subTest in NumPy float_power test --- python/pyspark/pandas/tests/test_numpy_compat.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/python/pyspark/pandas/tests/test_numpy_compat.py b/python/pyspark/pandas/tests/test_numpy_compat.py index 3d9c63a9558e..781ef1b0ac9c 100644 --- a/python/pyspark/pandas/tests/test_numpy_compat.py +++ b/python/pyspark/pandas/tests/test_numpy_compat.py @@ -159,13 +159,12 @@ def test_np_float_power(self): } ), ): - with self.subTest(base=pdf.base.tolist(), exponent=pdf.exponent.tolist()): - psdf = ps.from_pandas(pdf) - self.assert_eq( - np.float_power(psdf.base, psdf.exponent), - np.float_power(pdf.base, pdf.exponent), - almost=True, - ) + psdf = ps.from_pandas(pdf) + self.assert_eq( + np.float_power(psdf.base, psdf.exponent), + np.float_power(pdf.base, pdf.exponent), + almost=True, + ) def test_np_spark_compat_series(self): from pyspark.pandas.numpy_compat import unary_np_spark_mappings, binary_np_spark_mappings