Merge pull request #11639 from jorisvandenbossche/cat-remove-unused

BUG: remove_unused_categories with NaN values (GH11599)

Author: jorisvandenbossche

doc/source/whatsnew/v0.17.1.txt

@@ -188,3 +188,4 @@ Bug Fixes
 - Bug in ``DataFrame.join()`` with ``how='right'`` producing a ``TypeError`` (:issue:`11519`)
 - Bug in ``Series.quantile`` with empty list results has ``Index`` with ``object`` dtype (:issue:`11588`)
 - Bug in ``pd.merge`` results in empty ``Int64Index`` rather than ``Index(dtype=object)`` when the merge result is empty (:issue:`11588`)
+- Bug in ``remove_unused_categories`` when having ``NaN`` values (:issue:`11599`).

pandas/core/categorical.py

@@ -817,6 +817,9 @@ def remove_unused_categories(self, inplace=False):
         """
         cat = self if inplace else self.copy()
         _used = sorted(np.unique(cat._codes))
+        if _used[0] == -1:
+            _used = _used[1:]
+
         new_categories = cat.categories.take(_ensure_platform_int(_used))
 
         from pandas.core.index import _ensure_index

pandas/tests/test_categorical.py

@@ -844,6 +844,11 @@ def test_remove_unused_categories(self):
         self.assert_numpy_array_equal(c.categories, exp_categories_dropped)
         self.assertIsNone(res)
 
+        # with NaN values (GH11599)
+        c = Categorical(["a","b","c",np.nan], categories=["a","b","c","d","e"])
+        res = c.remove_unused_categories()
+        self.assert_numpy_array_equal(res.categories, np.array(["a","b","c"]))
+        self.assert_numpy_array_equal(c.categories, exp_categories_all)
 
     def test_nan_handling(self):