Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions doc/whats-new.rst
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,12 @@ Bug Fixes
type error when applying reduction methods, due to the reduction methods being
dynamically generated (:issue:`8136`).
By `Andrew Scherer <https://github.com/andrew-s28>`_.
- :py:func:`concat` no longer raises ``TypeError: Cannot interpret
'<StringDtype(...)>' as a data type`` when the objects being joined carry
index coordinates with a mix of numpy and pandas extension dtypes, which
happens routinely under pandas 3 when one dimension coordinate came from a
:py:class:`pandas.Index` and another from a plain list (:issue:`11317`).
By `Dipak Chaudhari <https://github.com/dchaudhari7177>`_.

.. _`pandas-dev/pandas#64793`: https://github.com/pandas-dev/pandas/pull/64793

Expand Down
10 changes: 10 additions & 0 deletions xarray/core/indexes.py
Original file line number Diff line number Diff line change
Expand Up @@ -789,6 +789,16 @@ def concat(
indexes_coord_dtypes = {idx.coord_dtype for idx in indexes}
if len(indexes_coord_dtypes) == 1:
coord_dtype = next(iter(indexes_coord_dtypes))
elif any(
isinstance(dtype, pd.api.extensions.ExtensionDtype)
for dtype in indexes_coord_dtypes
):
# np.result_type cannot interpret a pandas extension dtype, and
# mixing one with a numpy dtype is ordinary now that pandas
# gives string indexes a StringDtype. _concat_indexes above has
# already had pandas promote these, so use the dtype it settled
# on rather than reimplementing pandas' rules here.
coord_dtype = new_pd_index.dtype
else:
coord_dtype = np.result_type(*indexes_coord_dtypes)

Expand Down
23 changes: 23 additions & 0 deletions xarray/tests/test_concat.py
Original file line number Diff line number Diff line change
Expand Up @@ -1491,6 +1491,29 @@ def test_concat_index_not_same_dim() -> None:
concat([ds1, ds2], dim="x")


def test_concat_mixed_numpy_and_extension_coord_dtype() -> None:
# GH11317. Passing a pd.Index as the concat dim gives that index a pandas
# extension dtype (StringDtype under pandas 3), while a coord built from a
# plain list keeps a numpy <U dtype. PandasIndex.concat promoted the two
# with np.result_type, which cannot interpret an extension dtype at all.
da = DataArray([0], dims=["dim_a"], coords={"dim_a": ["a"]})
db = concat([DataArray([0])], pd.Index(["b"], name="dim_a"))

actual = concat([da, db], dim="dim_a")

assert list(actual.coords["dim_a"].values) == ["a", "b"]
assert actual.sizes["dim_a"] == 2


def test_concat_same_numpy_coord_dtypes_still_promote_with_numpy() -> None:
# The extension-dtype branch must not swallow the ordinary case: two numpy
# string coords of different widths still promote to the wider numpy dtype.
da = DataArray([0], dims=["d"], coords={"d": ["aaa"]})
db = DataArray([1], dims=["d"], coords={"d": ["b"]})

assert concat([da, db], dim="d").coords["d"].dtype == np.dtype("<U3")


class TestNewDefaults:
def test_concat_second_empty_with_scalar_data_var_only_on_first(self) -> None:
ds1 = Dataset(data_vars={"a": ("y", [0.1]), "b": 0.1}, coords={"x": 0.1})
Expand Down
Loading