Merge branch 'main' into api-date_range

Author: jbrockmendel

doc/source/whatsnew/v3.0.0.rst

@@ -233,6 +233,9 @@ Other enhancements
 - Support reading Stata 102-format (Stata 1) dta files (:issue:`58978`)
 - Support reading Stata 110-format (Stata 7) dta files (:issue:`47176`)
 - Switched wheel upload to **PyPI Trusted Publishing** (OIDC) for release-tag pushes in ``wheels.yml``. (:issue:`61718`)
+- Added a new :meth:`DataFrame.from_arrow` method to import any Arrow-compatible
+  tabular data object into a pandas :class:`DataFrame` through the
+  `Arrow PyCapsule Protocol <https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html>`__ (:issue:`59631`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_300.notable_bug_fixes:
@@ -748,6 +751,7 @@ Other API changes
 - :func:`date_range` and :func:`timedelta_range` no longer default to ``unit="ns"``, instead will infer a unit from the ``start``, ``end``, and ``freq`` parameters. Explicitly specify a desired ``unit`` to override these (:issue:`59031`)
 - :meth:`CategoricalIndex.append` no longer attempts to cast different-dtype indexes to the caller's dtype (:issue:`41626`)
 - :meth:`ExtensionDtype.construct_array_type` is now a regular method instead of a ``classmethod`` (:issue:`58663`)
+- Arithmetic operations between a :class:`Series`, :class:`Index`, or :class:`ExtensionArray` with a ``list`` now consistently wrap that list with an array equivalent to ``Series(my_list).array``. To do any other kind of type inference or casting, do so explicitly before operating (:issue:`62552`)
 - Comparison operations between :class:`Index` and :class:`Series` now consistently return :class:`Series` regardless of which object is on the left or right (:issue:`36759`)
 - Numpy functions like ``np.isinf`` that return a bool dtype when called on a :class:`Index` object now return a bool-dtype :class:`Index` instead of ``np.ndarray`` (:issue:`52676`)
 

pandas/_libs/internals.pyi

@@ -94,7 +94,3 @@ class BlockValuesRefs:
     def add_reference(self, blk: Block) -> None: ...
     def add_index_reference(self, index: Index) -> None: ...
     def has_reference(self) -> bool: ...
-
-class SetitemMixin:
-    def __setitem__(self, key, value) -> None: ...
-    def __delitem__(self, key) -> None: ...

pandas/_libs/internals.pyx

@@ -1,9 +1,6 @@
 from collections import defaultdict
-import sys
-import warnings
 
 cimport cython
-from cpython cimport PY_VERSION_HEX
 from cpython.object cimport PyObject
 from cpython.pyport cimport PY_SSIZE_T_MAX
 from cpython.slice cimport PySlice_GetIndicesEx
@@ -23,9 +20,6 @@ from numpy cimport (
 cnp.import_array()
 
 from pandas._libs.algos import ensure_int64
-from pandas.compat import CHAINED_WARNING_DISABLED
-from pandas.errors import ChainedAssignmentError
-from pandas.errors.cow import _chained_assignment_msg
 
 from pandas._libs.util cimport (
     is_array,
@@ -1002,47 +996,3 @@ cdef class BlockValuesRefs:
                 return self._has_reference_maybe_locked()
         ELSE:
             return self._has_reference_maybe_locked()
-
-
-cdef extern from "Python.h":
-    """
-    // python version < 3.14
-    #if PY_VERSION_HEX < 0x030E0000
-    // This function is unused and is declared to avoid a build warning
-    int __Pyx_PyUnstable_Object_IsUniqueReferencedTemporary(PyObject *ref) {
-        return Py_REFCNT(ref) == 1;
-    }
-    #else
-    #define __Pyx_PyUnstable_Object_IsUniqueReferencedTemporary \
-        PyUnstable_Object_IsUniqueReferencedTemporary
-    #endif
-    """
-    int PyUnstable_Object_IsUniqueReferencedTemporary\
-        "__Pyx_PyUnstable_Object_IsUniqueReferencedTemporary"(object o) except -1
-
-
-# Python version compatibility for PyUnstable_Object_IsUniqueReferencedTemporary
-cdef inline bint _is_unique_referenced_temporary(object obj) except -1:
-    if PY_VERSION_HEX >= 0x030E0000:
-        # Python 3.14+ has PyUnstable_Object_IsUniqueReferencedTemporary
-        return PyUnstable_Object_IsUniqueReferencedTemporary(obj)
-    else:
-        # Fallback for older Python versions using sys.getrefcount
-        return sys.getrefcount(obj) <= 1
-
-
-cdef class SetitemMixin:
-    # class used in DataFrame and Series for checking for chained assignment
-
-    def __setitem__(self, key, value) -> None:
-        cdef bint is_unique = 0
-        if not CHAINED_WARNING_DISABLED:
-            is_unique = _is_unique_referenced_temporary(self)
-            if is_unique:
-                warnings.warn(
-                    _chained_assignment_msg, ChainedAssignmentError, stacklevel=1
-                )
-        self._setitem(key, value)
-
-    def __delitem__(self, key) -> None:
-        self._delitem(key)

pandas/_libs/tslibs/timedeltas.pyx

@@ -83,6 +83,7 @@ from pandas._libs.tslibs.offsets import Day
 
 from pandas._libs.tslibs.util cimport (
     is_array,
+    is_bool_object,
     is_float_object,
     is_integer_object,
 )
@@ -2311,6 +2312,13 @@ class Timedelta(_Timedelta):
                 return self.__mul__(item)
             return other * self.to_timedelta64()
 
+        elif is_bool_object(other):
+            # GH#62316
+            raise TypeError(
+                "Cannot multiply Timedelta by bool. "
+                "Explicitly cast to integer instead."
+            )
+
         return NotImplemented
 
     __rmul__ = __mul__

pandas/_testing/contexts.py

@@ -12,10 +12,7 @@
 )
 import uuid
 
-from pandas.compat import (
-    CHAINED_WARNING_DISABLED,
-    CHAINED_WARNING_DISABLED_INPLACE_METHOD,
-)
+from pandas.compat import CHAINED_WARNING_DISABLED
 from pandas.errors import ChainedAssignmentError
 
 from pandas.io.common import get_handle
@@ -163,18 +160,10 @@ def with_csv_dialect(name: str, **kwargs) -> Generator[None]:
         csv.unregister_dialect(name)
 
 
-def raises_chained_assignment_error(
-    extra_warnings=(), extra_match=(), inplace_method=False
-):
+def raises_chained_assignment_error(extra_warnings=(), extra_match=()):
     from pandas._testing import assert_produces_warning
 
-    WARNING_DISABLED = (
-        CHAINED_WARNING_DISABLED_INPLACE_METHOD
-        if inplace_method
-        else CHAINED_WARNING_DISABLED
-    )
-
-    if WARNING_DISABLED:
+    if CHAINED_WARNING_DISABLED:
         if not extra_warnings:
             from contextlib import nullcontext
 

pandas/_typing.py

@@ -533,4 +533,44 @@ def closed(self) -> bool:
 
 SliceType: TypeAlias = Hashable | None
 
+
+# Arrow PyCapsule Interface
+# from https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html#protocol-typehints
+
+
+class ArrowArrayExportable(Protocol):
+    """
+    An object with an ``__arrow_c_array__`` method.
+
+    This method indicates the object is an Arrow-compatible object implementing
+    the `Arrow PyCapsule Protocol`_ (exposing the `Arrow C Data Interface`_ in
+    Python), enabling zero-copy Arrow data interchange across libraries.
+
+    .. _Arrow PyCapsule Protocol: https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html
+    .. _Arrow C Data Interface: https://arrow.apache.org/docs/format/CDataInterface.html
+
+    """
+
+    def __arrow_c_array__(
+        self, requested_schema: object | None = None
+    ) -> tuple[object, object]: ...
+
+
+class ArrowStreamExportable(Protocol):
+    """
+    An object with an ``__arrow_c_stream__`` method.
+
+    This method indicates the object is an Arrow-compatible object implementing
+    the `Arrow PyCapsule Protocol`_ (exposing the `Arrow C Data Interface`_
+    for streams in Python), enabling zero-copy Arrow data interchange across
+    libraries.
+
+    .. _Arrow PyCapsule Protocol: https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html
+    .. _Arrow C Stream Interface: https://arrow.apache.org/docs/format/CStreamInterface.html
+
+    """
+
+    def __arrow_c_stream__(self, requested_schema: object | None = None) -> object: ...
+
+
 __all__ = ["type_t"]

pandas/compat/__init__.py

@@ -17,7 +17,6 @@
 
 from pandas.compat._constants import (
     CHAINED_WARNING_DISABLED,
-    CHAINED_WARNING_DISABLED_INPLACE_METHOD,
     IS64,
     ISMUSL,
     PY312,
@@ -154,7 +153,6 @@ def is_ci_environment() -> bool:
 
 __all__ = [
     "CHAINED_WARNING_DISABLED",
-    "CHAINED_WARNING_DISABLED_INPLACE_METHOD",
     "HAS_PYARROW",
     "IS64",
     "ISMUSL",

pandas/compat/_constants.py

@@ -18,9 +18,11 @@
 PYPY = platform.python_implementation() == "PyPy"
 WASM = (sys.platform == "emscripten") or (platform.machine() in ["wasm32", "wasm64"])
 ISMUSL = "musl" in (sysconfig.get_config_var("HOST_GNU_TYPE") or "")
-REF_COUNT = 2
-CHAINED_WARNING_DISABLED = PYPY or (PY314 and not sys._is_gil_enabled())  # type: ignore[attr-defined]
-CHAINED_WARNING_DISABLED_INPLACE_METHOD = PYPY or PY314
+# the refcount for self in a chained __setitem__/.(i)loc indexing/method call
+REF_COUNT = 2 if PY314 else 3
+REF_COUNT_IDX = 2
+REF_COUNT_METHOD = 1 if PY314 else 2
+CHAINED_WARNING_DISABLED = PYPY
 
 
 __all__ = [

pandas/compat/pickle_compat.py

@@ -22,7 +22,6 @@
     PeriodArray,
     TimedeltaArray,
 )
-from pandas.core.generic import NDFrame
 from pandas.core.internals import BlockManager
 
 if TYPE_CHECKING:
@@ -91,10 +90,6 @@ def load_reduce(self) -> None:
                 cls = args[0]
                 stack[-1] = NDArrayBacked.__new__(*args)
                 return
-            elif args and issubclass(args[0], NDFrame):
-                cls = args[0]
-                stack[-1] = cls.__new__(cls)
-                return
             raise
 
     dispatch[pickle.REDUCE[0]] = load_reduce  # type: ignore[assignment]

pandas/core/arrays/interval.py

@@ -2100,18 +2100,9 @@ def isin(self, values: ArrayLike) -> npt.NDArray[np.bool_]:
                 return np.zeros(self.shape, dtype=bool)
 
             if self.dtype == values.dtype:
-                # GH#38353 instead of casting to object, operating on a
-                #  complex128 ndarray is much more performant.
-                left = self._combined.view("complex128")
-                right = values._combined.view("complex128")
-                # error: Argument 1 to "isin" has incompatible type
-                # "Union[ExtensionArray, ndarray[Any, Any],
-                # ndarray[Any, dtype[Any]]]"; expected
-                # "Union[_SupportsArray[dtype[Any]],
-                # _NestedSequence[_SupportsArray[dtype[Any]]], bool,
-                # int, float, complex, str, bytes, _NestedSequence[
-                # Union[bool, int, float, complex, str, bytes]]]"
-                return np.isin(left, right).ravel()  # type: ignore[arg-type]
+                left = self._combined
+                right = values._combined
+                return np.isin(left, right).ravel()
 
             elif needs_i8_conversion(self.left.dtype) ^ needs_i8_conversion(
                 values.left.dtype
@@ -2127,24 +2118,29 @@ def _combined(self) -> IntervalSide:
         # has no attribute "reshape"  [union-attr]
         left = self.left._values.reshape(-1, 1)  # type: ignore[union-attr]
         right = self.right._values.reshape(-1, 1)  # type: ignore[union-attr]
+        # GH#38353 instead of casting to object, operating on a
+        # complex128 ndarray is much more performant.
         if needs_i8_conversion(left.dtype):
             # error: Item "ndarray[Any, Any]" of "Any | ndarray[Any, Any]" has
             # no attribute "_concat_same_type"
             comb = left._concat_same_type(  # type: ignore[union-attr]
                 [left, right], axis=1
             )
+            comb = comb.view("complex128")[:, 0]
         else:
-            comb = np.concatenate([left, right], axis=1)
+            comb = (np.array(left.ravel(), dtype="complex128")) + (
+                1j * np.array(right.ravel(), dtype="complex128")
+            )
         return comb
 
     def _from_combined(self, combined: np.ndarray) -> IntervalArray:
         """
         Create a new IntervalArray with our dtype from a 1D complex128 ndarray.
         """
-        nc = combined.view("i8").reshape(-1, 2)
 
         dtype = self._left.dtype
         if needs_i8_conversion(dtype):
+            nc = combined.view("i8").reshape(-1, 2)
             assert isinstance(self._left, (DatetimeArray, TimedeltaArray))
             new_left: DatetimeArray | TimedeltaArray | np.ndarray = type(
                 self._left
@@ -2155,18 +2151,13 @@ def _from_combined(self, combined: np.ndarray) -> IntervalArray:
             )._from_sequence(nc[:, 1], dtype=dtype)
         else:
             assert isinstance(dtype, np.dtype)
-            new_left = nc[:, 0].view(dtype)
-            new_right = nc[:, 1].view(dtype)
+            new_left = np.real(combined).astype(dtype).ravel()
+            new_right = np.imag(combined).astype(dtype).ravel()
         return self._shallow_copy(left=new_left, right=new_right)
 
     def unique(self) -> IntervalArray:
-        # No overload variant of "__getitem__" of "ExtensionArray" matches argument
-        # type "Tuple[slice, int]"
-        nc = unique(
-            self._combined.view("complex128")[:, 0]  # type: ignore[call-overload]
-        )
-        nc = nc[:, None]
-        return self._from_combined(nc)
+        nc = unique(self._combined)
+        return self._from_combined(np.asarray(nc)[:, None])
 
 
 def _maybe_convert_platform_interval(values) -> ArrayLike: