ENH: Resotre original convert_objects and add _convert

Restores the v0.16 behavior of convert_objects and moves the new
version of _convert
Adds to_numeric for directly converting numeric data

closes #11116
closes #11133

Author: Kevin Sheppard

pandas/__init__.py

@@ -52,6 +52,7 @@
 from pandas.tools.pivot import pivot_table, crosstab
 from pandas.tools.plotting import scatter_matrix, plot_params
 from pandas.tools.tile import cut, qcut
+from pandas.tools.util import to_numeric
 from pandas.core.reshape import melt
 from pandas.util.print_versions import show_versions
 import pandas.util.testing

pandas/core/common.py

@@ -1857,23 +1857,84 @@ def _maybe_box_datetimelike(value):
 
 _values_from_object = lib.values_from_object
 
-
-def _possibly_convert_objects(values,
-                              datetime=True,
-                              numeric=True,
-                              timedelta=True,
-                              coerce=False,
+# TODO: Remove in 0.18 or 2017, which ever is sooner
+def _possibly_convert_objects(values, convert_dates=True,
+                              convert_numeric=True,
+                              convert_timedeltas=True,
                               copy=True):
     """ if we have an object dtype, try to coerce dates and/or numbers """
 
+    # if we have passed in a list or scalar
+    if isinstance(values, (list, tuple)):
+        values = np.array(values, dtype=np.object_)
+    if not hasattr(values, 'dtype'):
+        values = np.array([values], dtype=np.object_)
+
+    # convert dates
+    if convert_dates and values.dtype == np.object_:
+
+        # we take an aggressive stance and convert to datetime64[ns]
+        if convert_dates == 'coerce':
+            new_values = _possibly_cast_to_datetime(
+                values, 'M8[ns]', errors='coerce')
+
+            # if we are all nans then leave me alone
+            if not isnull(new_values).all():
+                values = new_values
+
+        else:
+            values = lib.maybe_convert_objects(
+                values, convert_datetime=convert_dates)
+
+    # convert timedeltas
+    if convert_timedeltas and values.dtype == np.object_:
+
+        if convert_timedeltas == 'coerce':
+            from pandas.tseries.timedeltas import to_timedelta
+            new_values = to_timedelta(values, coerce=True)
+
+            # if we are all nans then leave me alone
+            if not isnull(new_values).all():
+                values = new_values
+
+        else:
+            values = lib.maybe_convert_objects(
+                values, convert_timedelta=convert_timedeltas)
+
+    # convert to numeric
+    if values.dtype == np.object_:
+        if convert_numeric:
+            try:
+                new_values = lib.maybe_convert_numeric(
+                    values, set(), coerce_numeric=True)
+
+                # if we are all nans then leave me alone
+                if not isnull(new_values).all():
+                    values = new_values
+
+            except:
+                pass
+        else:
+            # soft-conversion
+            values = lib.maybe_convert_objects(values)
+
+    values = values.copy() if copy else values
+
+    return values
+
+
+def _soft_convert_objects(values, datetime=True, numeric=True, timedelta=True,
+                          coerce=False, copy=True):
+    """ if we have an object dtype, try to coerce dates and/or numbers """
+
     conversion_count = sum((datetime, numeric, timedelta))
     if conversion_count == 0:
-        import warnings
-        warnings.warn('Must explicitly pass type for conversion. Defaulting to '
-                      'pre-0.17 behavior where datetime=True, numeric=True, '
-                      'timedelta=True and coerce=False', DeprecationWarning)
-        datetime = numeric = timedelta = True
-        coerce = False
+        raise ValueError('At least one of datetime, numeric or timedelta must '
+                         'be True.')
+    elif conversion_count > 1 and coerce:
+            raise ValueError("Only one of 'datetime', 'numeric' or "
+                             "'timedelta' can be True when when coerce=True.")
+
 
     if isinstance(values, (list, tuple)):
         # List or scalar
@@ -1887,10 +1948,6 @@ def _possibly_convert_objects(values,
 
     # If 1 flag is coerce, ensure 2 others are False
     if coerce:
-        if conversion_count > 1:
-            raise ValueError("Only one of 'datetime', 'numeric' or "
-                             "'timedelta' can be True when when coerce=True.")
-
         # Immediate return if coerce
         if datetime:
             return pd.to_datetime(values, errors='coerce', box=False)

pandas/core/frame.py

@@ -3543,9 +3543,8 @@ def combine(self, other, func, fill_value=None, overwrite=True):
         # convert_objects just in case
         return self._constructor(result,
                                  index=new_index,
-                                 columns=new_columns).convert_objects(
-            datetime=True,
-            copy=False)
+                                 columns=new_columns)._convert(datetime=True,
+                                                               copy=False)
 
     def combine_first(self, other):
         """
@@ -4026,9 +4025,7 @@ def _apply_standard(self, func, axis, ignore_failures=False, reduce=True):
 
             if axis == 1:
                 result = result.T
-            result = result.convert_objects(datetime=True,
-                                            timedelta=True,
-                                            copy=False)
+            result = result._convert(datetime=True, timedelta=True, copy=False)
 
         else:
 
@@ -4158,7 +4155,7 @@ def append(self, other, ignore_index=False, verify_integrity=False):
             other = DataFrame(other.values.reshape((1, len(other))),
                               index=index,
                               columns=combined_columns)
-            other = other.convert_objects(datetime=True, timedelta=True)
+            other = other._convert(datetime=True, timedelta=True)
 
             if not self.columns.equals(combined_columns):
                 self = self.reindex(columns=combined_columns)

pandas/core/generic.py

@@ -2534,11 +2534,8 @@ def copy(self, deep=True):
         data = self._data.copy(deep=deep)
         return self._constructor(data).__finalize__(self)
 
-    @deprecate_kwarg(old_arg_name='convert_dates', new_arg_name='datetime')
-    @deprecate_kwarg(old_arg_name='convert_numeric', new_arg_name='numeric')
-    @deprecate_kwarg(old_arg_name='convert_timedeltas', new_arg_name='timedelta')
-    def convert_objects(self, datetime=False, numeric=False,
-                        timedelta=False, coerce=False, copy=True):
+    def _convert(self, datetime=False, numeric=False, timedelta=False,
+                 coerce=False, copy=True):
         """
         Attempt to infer better dtype for object columns
 
@@ -2563,31 +2560,48 @@ def convert_objects(self, datetime=False, numeric=False,
         -------
         converted : same as input object
         """
+        return self._constructor(
+            self._data.convert(datetime=datetime,
+                                numeric=numeric,
+                                timedelta=timedelta,
+                                coerce=coerce,
+                                copy=copy)).__finalize__(self)
+
+    # TODO: Remove in 0.18 or 2017, which ever is sooner
+    def convert_objects(self, convert_dates=True, convert_numeric=False,
+                        convert_timedeltas=True, copy=True):
+        """
+        Attempt to infer better dtype for object columns
+
+        Parameters
+        ----------
+        convert_dates : boolean, default True
+            If True, convert to date where possible. If 'coerce', force
+            conversion, with unconvertible values becoming NaT.
+        convert_numeric : boolean, default False
+            If True, attempt to coerce to numbers (including strings), with
+            unconvertible values becoming NaN.
+        convert_timedeltas : boolean, default True
+            If True, convert to timedelta where possible. If 'coerce', force
+            conversion, with unconvertible values becoming NaT.
+        copy : boolean, default True
+            If True, return a copy even if no copy is necessary (e.g. no
+            conversion was done). Note: This is meant for internal use, and
+            should not be confused with inplace.
 
-        # Deprecation code to handle usage change
-        issue_warning = False
-        if datetime == 'coerce':
-            datetime = coerce = True
-            numeric = timedelta = False
-            issue_warning = True
-        elif numeric == 'coerce':
-            numeric = coerce = True
-            datetime = timedelta = False
-            issue_warning = True
-        elif timedelta == 'coerce':
-            timedelta = coerce = True
-            datetime = numeric = False
-            issue_warning = True
-        if issue_warning:
-            warnings.warn("The use of 'coerce' as an input is deprecated. "
-                          "Instead set coerce=True.",
-                          FutureWarning)
+        Returns
+        -------
+        converted : same as input object
+        """
+        from warnings import warn
+        warn("convert_objects is deprecated.  Use the data-type specific "
+             "converters pd.to_datetime, pd.to_timestamp and pd.to_numeric.",
+             FutureWarning, stacklevel=2)
 
         return self._constructor(
-            self._data.convert(datetime=datetime,
-                               numeric=numeric,
-                               timedelta=timedelta,
-                               coerce=coerce,
+            self._data.convert(convert_dates=convert_dates,
+                               convert_numeric=convert_numeric,
+                               convert_timedeltas=convert_timedeltas,
                                copy=copy)).__finalize__(self)
 
     #----------------------------------------------------------------------

pandas/core/groupby.py

@@ -112,7 +112,7 @@ def f(self):
         except Exception:
             result = self.aggregate(lambda x: npfunc(x, axis=self.axis))
             if _convert:
-                result = result.convert_objects(datetime=True)
+                result = result._convert(datetime=True)
             return result
 
     f.__doc__ = "Compute %s of group values" % name
@@ -2882,7 +2882,7 @@ def aggregate(self, arg, *args, **kwargs):
             self._insert_inaxis_grouper_inplace(result)
             result.index = np.arange(len(result))
 
-        return result.convert_objects(datetime=True)
+        return result._convert(datetime=True)
 
     def _aggregate_multiple_funcs(self, arg):
         from pandas.tools.merge import concat
@@ -3123,22 +3123,22 @@ def _wrap_applied_output(self, keys, values, not_indexed_same=False):
                 # as we are stacking can easily have object dtypes here
                 if (self._selected_obj.ndim == 2 and
                         self._selected_obj.dtypes.isin(_DATELIKE_DTYPES).any()):
-                    result = result.convert_objects(numeric=True)
+                    result = result._convert(numeric=True)
                     date_cols = self._selected_obj.select_dtypes(
                         include=list(_DATELIKE_DTYPES)).columns
                     result[date_cols] = (result[date_cols]
-                                         .convert_objects(datetime=True,
+                                         ._convert(datetime=True,
                                                           coerce=True))
                 else:
-                    result = result.convert_objects(datetime=True)
+                    result = result._convert(datetime=True)
 
                 return self._reindex_output(result)
 
             else:
                 # only coerce dates if we find at least 1 datetime
                 coerce = True if any([ isinstance(v,Timestamp) for v in values ]) else False
                 return (Series(values, index=key_index)
-                        .convert_objects(datetime=True,
+                        ._convert(datetime=True,
                                          coerce=coerce))
 
         else:
@@ -3243,7 +3243,7 @@ def transform(self, func, *args, **kwargs):
             results = self._try_cast(results, obj[result.columns])
 
         return (DataFrame(results,columns=result.columns,index=obj.index)
-                .convert_objects(datetime=True))
+                ._convert(datetime=True))
 
     def _define_paths(self, func, *args, **kwargs):
         if isinstance(func, compat.string_types):
@@ -3436,7 +3436,7 @@ def _wrap_aggregated_output(self, output, names=None):
         if self.axis == 1:
             result = result.T
 
-        return self._reindex_output(result).convert_objects(datetime=True)
+        return self._reindex_output(result)._convert(datetime=True)
 
     def _wrap_agged_blocks(self, items, blocks):
         if not self.as_index:
@@ -3454,7 +3454,7 @@ def _wrap_agged_blocks(self, items, blocks):
         if self.axis == 1:
             result = result.T
 
-        return self._reindex_output(result).convert_objects(datetime=True)
+        return self._reindex_output(result)._convert(datetime=True)
 
     def _reindex_output(self, result):
         """

pandas/core/internals.py

@@ -1517,14 +1517,34 @@ def is_bool(self):
         """
         return lib.is_bool_array(self.values.ravel())
 
-    def convert(self, datetime=True, numeric=True, timedelta=True, coerce=False,
-                copy=True, by_item=True):
+    # TODO: Refactor when convert_objects is removed since there will be 1 path
+    def convert(self, *args, **kwargs):
         """ attempt to coerce any object types to better types
             return a copy of the block (if copy = True)
             by definition we ARE an ObjectBlock!!!!!
 
             can return multiple blocks!
             """
+        if args:
+            raise NotImplementedError
+        by_item = True if 'by_item' not in kwargs else kwargs['by_item']
+
+        new_inputs = ['coerce','datetime','numeric','timedelta']
+        new_style = False
+        for kw in new_inputs:
+            new_style |= kw in kwargs
+
+        if new_style:
+            fn = com._soft_convert_objects
+            fn_inputs = new_inputs + ['copy']
+        else:
+            fn = com._possibly_convert_objects
+            fn_inputs = ['convert_dates','convert_numeric','convert_timedeltas']
+
+        fn_kwargs = {}
+        for key in fn_inputs:
+            if key in kwargs:
+                fn_kwargs[key] = kwargs[key]
 
         # attempt to create new type blocks
         blocks = []
@@ -1533,30 +1553,14 @@ def convert(self, datetime=True, numeric=True, timedelta=True, coerce=False,
             for i, rl in enumerate(self.mgr_locs):
                 values = self.iget(i)
 
-                values = com._possibly_convert_objects(
-                    values.ravel(),
-                    datetime=datetime,
-                    numeric=numeric,
-                    timedelta=timedelta,
-                    coerce=coerce,
-                    copy=copy
-                ).reshape(values.shape)
+                values = fn(values.ravel(), **fn_kwargs).reshape(values.shape)
                 values = _block_shape(values, ndim=self.ndim)
-                newb = self.make_block(values,
-                                       placement=[rl])
+                newb = make_block(values, ndim=self.ndim, placement=[rl])
                 blocks.append(newb)
 
         else:
-
-            values = com._possibly_convert_objects(
-                self.values.ravel(),
-                datetime=datetime,
-                numeric=numeric,
-                timedelta=timedelta,
-                coerce=coerce,
-                copy=copy
-            ).reshape(self.values.shape)
-            blocks.append(self.make_block(values))
+            values = fn(self.values.ravel(), **fn_kwargs).reshape(self.values.shape)
+            blocks.append(make_block(values, ndim=self.ndim, placement=self.mgr_locs))
 
         return blocks
 
@@ -1597,8 +1601,7 @@ def _maybe_downcast(self, blocks, downcast=None):
         # split and convert the blocks
         result_blocks = []
         for blk in blocks:
-            result_blocks.extend(blk.convert(datetime=True,
-                                             numeric=False))
+            result_blocks.extend(blk.convert(datetime=True, numeric=False))
         return result_blocks
 
     def _can_hold_element(self, element):