Merge remote-tracking branch 'upstream/main' into python-db-dtypes-pandas-issue28

Author: tswast

.github/workflows/code-checks.yml

@@ -37,8 +37,8 @@ jobs:
     - name: Run pre-commit
       uses: pre-commit/action@v2.0.3
 
-  typing_and_docstring_validation:
-    name: Docstring and typing validation
+  docstring_typing_pylint:
+    name: Docstring validation, typing, and pylint
     runs-on: ubuntu-latest
     defaults:
       run:

.github/workflows/package-checks.yml

@@ -9,12 +9,14 @@ on:
     branches:
       - main
       - 1.5.x
+    types: [ labeled, opened, synchronize, reopened ]
 
 permissions:
   contents: read
 
 jobs:
   pip:
+    if: ${{ github.event.label.name == 'Build' || contains(github.event.pull_request.labels.*.name, 'Build') || github.event_name == 'push'}}
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -38,13 +40,6 @@ jobs:
         with:
           python-version: '3.8'
 
-      # Hacky patch to disable building cython extensions.
-      # This job should only check that the extras successfully install.
-      - name: Disable building ext_modules
-        run: |
-          sed -i '/ext_modules=/d' setup.py
-        shell: bash -el {0}
-
       - name: Install required dependencies
         run: |
           python -m pip install --upgrade pip setuptools wheel python-dateutil pytz numpy cython

.pre-commit-config.yaml

@@ -64,6 +64,25 @@ repos:
     hooks:
     -   id: pylint
         stages: [manual]
+-   repo: https://github.com/pycqa/pylint
+    rev: v2.15.5
+    hooks:
+    -   id: pylint
+        alias: redefined-outer-name
+        name: Redefining name from outer scope
+        files: ^pandas/
+        exclude: |
+            (?x)
+            ^pandas/tests  # keep excluded
+            |/_testing/  # keep excluded
+            |^pandas/util/_test_decorators\.py  # keep excluded
+            |^pandas/_version\.py  # keep excluded
+            |^pandas/conftest\.py  # keep excluded
+            |^pandas/core/tools/datetimes\.py
+            |^pandas/io/formats/format\.py
+            |^pandas/core/generic\.py
+        args: [--disable=all, --enable=redefined-outer-name]
+        stages: [manual]
 -   repo: https://github.com/PyCQA/isort
     rev: 5.10.1
     hooks:
@@ -201,7 +220,7 @@ repos:
         entry: python scripts/sync_flake8_versions.py
         files: ^(\.pre-commit-config\.yaml|environment\.yml)$
         pass_filenames: false
-        additional_dependencies: [pyyaml]
+        additional_dependencies: [pyyaml, toml]
     -   id: title-capitalization
         name: Validate correct capitalization among titles in documentation
         entry: python scripts/validate_rst_title_capitalization.py

doc/source/whatsnew/index.rst

@@ -24,6 +24,7 @@ Version 1.5
 .. toctree::
    :maxdepth: 2
 
+   v1.5.3
    v1.5.2
    v1.5.1
    v1.5.0

doc/source/whatsnew/v1.5.3.rst

@@ -0,0 +1,41 @@
+.. _whatsnew_153:
+
+What's new in 1.5.3 (December ??, 2022)
+---------------------------------------
+
+These are the changes in pandas 1.5.3. See :ref:`release` for a full changelog
+including other versions of pandas.
+
+{{ header }}
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_153.regressions:
+
+Fixed regressions
+~~~~~~~~~~~~~~~~~
+- Fixed performance regression in :meth:`Series.isin` when ``values`` is empty (:issue:`49839`)
+-
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_153.bug_fixes:
+
+Bug fixes
+~~~~~~~~~
+-
+-
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_153.other:
+
+Other
+~~~~~
+-
+-
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_153.contributors:
+
+Contributors
+~~~~~~~~~~~~
+
+.. contributors:: v1.5.2..v1.5.3|HEAD

doc/source/whatsnew/v2.0.0.rst

@@ -33,7 +33,7 @@ sql-other, html, xml, plot, output_formatting, clipboard, compression, test]`` (
 Configuration option, ``io.nullable_backend``, to return pyarrow-backed dtypes from IO functions
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-A new global configuration, ``io.nullable_backend`` can now be used in conjunction with the parameter ``use_nullable_dtypes=True`` in :func:`read_parquet` and :func:`read_csv` (with ``engine="pyarrow"``)
+A new global configuration, ``io.nullable_backend`` can now be used in conjunction with the parameter ``use_nullable_dtypes=True`` in :func:`read_parquet`, :func:`read_orc` and :func:`read_csv` (with ``engine="pyarrow"``)
 to return pyarrow-backed dtypes when set to ``"pyarrow"`` (:issue:`48957`).
 
 .. ipython:: python
@@ -45,7 +45,7 @@ to return pyarrow-backed dtypes when set to ``"pyarrow"`` (:issue:`48957`).
     """)
     with pd.option_context("io.nullable_backend", "pyarrow"):
         df = pd.read_csv(data, use_nullable_dtypes=True, engine="pyarrow")
-    df
+    df.dtypes
 
 .. _whatsnew_200.enhancements.other:
 
@@ -349,6 +349,8 @@ Other API changes
 - Changed behavior of :class:`Index` constructor with sequence containing at least one ``NaT`` and everything else either ``None`` or ``NaN`` to infer ``datetime64[ns]`` dtype instead of ``object``, matching :class:`Series` behavior (:issue:`49340`)
 - :func:`read_stata` with parameter ``index_col`` set to ``None`` (the default) will now set the index on the returned :class:`DataFrame` to a :class:`RangeIndex` instead of a :class:`Int64Index` (:issue:`49745`)
 - Changed behavior of :class:`Index` constructor with an object-dtype ``numpy.ndarray`` containing all-``bool`` values or all-complex values, this will now retain object dtype, consistent with the :class:`Series` behavior (:issue:`49594`)
+- Changed behavior of :meth:`DataFrame.shift` with ``axis=1``, an integer ``fill_value``, and homogeneous datetime-like dtype, this now fills new columns with integer dtypes instead of casting to datetimelike (:issue:`49842`)
+- :meth:`DataFrame.values`, :meth:`DataFrame.to_numpy`, :meth:`DataFrame.xs`, :meth:`DataFrame.reindex`, :meth:`DataFrame.fillna`, and :meth:`DataFrame.replace` no longer silently consolidate the underlying arrays; do ``df = df.copy()`` to ensure consolidation (:issue:`49356`)
 -
 
 .. ---------------------------------------------------------------------------
@@ -608,6 +610,7 @@ Performance improvements
 - Performance improvement in :class:`DataFrameGroupBy` and :class:`SeriesGroupBy` when ``by`` is a categorical type and ``observed=False`` (:issue:`49596`)
 - Performance improvement in :func:`read_stata` with parameter ``index_col`` set to ``None`` (the default). Now the index will be a :class:`RangeIndex` instead of :class:`Int64Index` (:issue:`49745`)
 - Performance improvement in :func:`merge` when not merging on the index - the new index will now be :class:`RangeIndex` instead of :class:`Int64Index` (:issue:`49478`)
+- Performance improvement in :meth:`DataFrame.to_dict` and :meth:`Series.to_dict` when using any non-object dtypes (:issue:`46470`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_200.bug_fixes:

pandas/_libs/lib.pyx

@@ -1438,6 +1438,8 @@ def infer_dtype(value: object, skipna: bool = True) -> str:
     else:
         if not isinstance(value, list):
             value = list(value)
+        if not value:
+            return "empty"
 
         from pandas.core.dtypes.cast import construct_1d_object_array_from_listlike
         values = construct_1d_object_array_from_listlike(value)

pandas/_libs/src/headers/portable.h

@@ -1,16 +1,10 @@
 #ifndef _PANDAS_PORTABLE_H_
 #define _PANDAS_PORTABLE_H_
 
-// To get `strdup` from strings.h
-#ifndef _XOPEN_SOURCE
-#define _XOPEN_SOURCE 600
-#endif
-
 #include <string.h>
 
 #if defined(_MSC_VER)
 #define strcasecmp( s1, s2 ) _stricmp( s1, s2 )
-#define strdup _strdup
 #endif
 
 // GH-23516 - works around locale perf issues

pandas/_libs/src/ujson/lib/ultrajsondec.c

@@ -1174,15 +1174,21 @@ JSOBJ JSON_DecodeObject(JSONObjectDecoder *dec, const char *buffer,
     ds.dec = dec;
 
     locale = setlocale(LC_NUMERIC, NULL);
+    if (!locale) {
+        return SetError(&ds, -1, "setlocale call failed");
+    }
+
     if (strcmp(locale, "C")) {
-        locale = strdup(locale);
-        if (!locale) {
-            return SetError(&ds, -1, "Could not reserve memory block");
+        size_t len = strlen(locale) + 1;
+        char *saved_locale = malloc(len);
+        if (saved_locale == NULL) {
+          return SetError(&ds, -1, "Could not reserve memory block");
         }
+        memcpy(saved_locale, locale, len);
         setlocale(LC_NUMERIC, "C");
         ret = decode_any(&ds);
-        setlocale(LC_NUMERIC, locale);
-        free(locale);
+        setlocale(LC_NUMERIC, saved_locale);
+        free(saved_locale);
     } else {
         ret = decode_any(&ds);
     }

pandas/_libs/src/ujson/lib/ultrajsonenc.c

@@ -1176,16 +1176,23 @@ char *JSON_EncodeObject(JSOBJ obj, JSONObjectEncoder *enc, char *_buffer,
     enc->offset = enc->start;
 
     locale = setlocale(LC_NUMERIC, NULL);
+    if (!locale) {
+        SetError(NULL, enc, "setlocale call failed");
+        return NULL;
+    }
+
     if (strcmp(locale, "C")) {
-        locale = strdup(locale);
-        if (!locale) {
-            SetError(NULL, enc, "Could not reserve memory block");
-            return NULL;
+        size_t len = strlen(locale) + 1;
+        char *saved_locale = malloc(len);
+        if (saved_locale == NULL) {
+          SetError(NULL, enc, "Could not reserve memory block");
+          return NULL;
         }
+        memcpy(saved_locale, locale, len);
         setlocale(LC_NUMERIC, "C");
         encode(obj, enc, NULL, 0);
-        setlocale(LC_NUMERIC, locale);
-        free(locale);
+        setlocale(LC_NUMERIC, saved_locale);
+        free(saved_locale);
     } else {
         encode(obj, enc, NULL, 0);
     }