Merge remote-tracking branch 'upstream/main' into python-db-dtypes-pandas-issue28

Author: tswast

.circleci/setup_env.sh

@@ -1,47 +1,16 @@
 #!/bin/bash -e
 
-# edit the locale file if needed
-if [[ "$(uname)" == "Linux" && -n "$LC_ALL" ]]; then
-    echo "Adding locale to the first line of pandas/__init__.py"
-    rm -f pandas/__init__.pyc
-    SEDC="3iimport locale\nlocale.setlocale(locale.LC_ALL, '$LC_ALL')\n"
-    sed -i "$SEDC" pandas/__init__.py
-
-    echo "[head -4 pandas/__init__.py]"
-    head -4 pandas/__init__.py
-    echo
-fi
+echo "Install Mambaforge"
+MAMBA_URL="https://github.com/conda-forge/miniforge/releases/download/4.14.0-0/Mambaforge-4.14.0-0-Linux-aarch64.sh"
+echo "Downloading $MAMBA_URL"
+wget -q $MAMBA_URL -O minimamba.sh
+chmod +x minimamba.sh
 
+MAMBA_DIR="$HOME/miniconda3"
+rm -rf $MAMBA_DIR
+./minimamba.sh -b -p $MAMBA_DIR
 
-MINICONDA_DIR=/usr/local/miniconda
-if [ -e $MINICONDA_DIR ] && [ "$BITS32" != yes ]; then
-    echo "Found Miniconda installation at $MINICONDA_DIR"
-else
-    echo "Install Miniconda"
-    DEFAULT_CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest"
-    if [[ "$(uname -m)" == 'aarch64' ]]; then
-        CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.10.1-4/Miniforge3-4.10.1-4-Linux-aarch64.sh"
-    elif [[ "$(uname)" == 'Linux' ]]; then
-        if [[ "$BITS32" == "yes" ]]; then
-            CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86.sh"
-        else
-            CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86_64.sh"
-        fi
-    elif [[ "$(uname)" == 'Darwin' ]]; then
-        CONDA_URL="$DEFAULT_CONDA_URL-MacOSX-x86_64.sh"
-    else
-      echo "OS $(uname) not supported"
-      exit 1
-    fi
-    echo "Downloading $CONDA_URL"
-    wget -q $CONDA_URL -O miniconda.sh
-    chmod +x miniconda.sh
-
-    MINICONDA_DIR="$HOME/miniconda3"
-    rm -rf $MINICONDA_DIR
-    ./miniconda.sh -b -p $MINICONDA_DIR
-fi
-export PATH=$MINICONDA_DIR/bin:$PATH
+export PATH=$MAMBA_DIR/bin:$PATH
 
 echo
 echo "which conda"
@@ -51,7 +20,7 @@ echo
 echo "update conda"
 conda config --set ssl_verify false
 conda config --set quiet true --set always_yes true --set changeps1 false
-conda install -y -c conda-forge -n base 'mamba>=0.21.2' pip setuptools
+mamba install -y -c conda-forge -n base pip setuptools
 
 echo "conda info -a"
 conda info -a
@@ -70,11 +39,6 @@ time mamba env update -n pandas-dev --file="${ENV_FILE}"
 echo "conda list -n pandas-dev"
 conda list -n pandas-dev
 
-if [[ "$BITS32" == "yes" ]]; then
-    # activate 32-bit compiler
-    export CONDA_BUILD=1
-fi
-
 echo "activate pandas-dev"
 source activate pandas-dev
 
@@ -90,15 +54,9 @@ if pip list | grep -q ^pandas; then
     pip uninstall -y pandas || true
 fi
 
-if [ "$(conda list -f qt --json)" != [] ]; then
-    echo
-    echo "remove qt"
-    echo "causes problems with the clipboard, we use xsel for that"
-    conda remove qt -y --force || true
-fi
-
 echo "Build extensions"
-python setup.py build_ext -q -j3
+# GH 47305: Parallel build can causes flaky ImportError from pandas/_libs/tslibs
+python setup.py build_ext -q -j1
 
 echo "Install pandas"
 python -m pip install --no-build-isolation --no-use-pep517 -e .

.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -68,5 +68,5 @@ body:
     attributes:
       label: Additional Context
       description: >
-        Please provide any relevant Github issues, code examples or references that help describe and support
+        Please provide any relevant GitHub issues, code examples or references that help describe and support
         the feature request.

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,4 +1,4 @@
-- [ ] closes #xxxx (Replace xxxx with the Github issue number)
+- [ ] closes #xxxx (Replace xxxx with the GitHub issue number)
 - [ ] [Tests added and passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#writing-tests) if fixing a bug or adding a new feature
 - [ ] All [code checks passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#pre-commit).
 - [ ] Added [type annotations](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#type-hints) to new arguments/methods/functions.

.github/workflows/32-bit-linux.yml

@@ -5,12 +5,10 @@ on:
     branches:
       - main
       - 1.5.x
-      - 1.4.x
   pull_request:
     branches:
       - main
       - 1.5.x
-      - 1.4.x
     paths-ignore:
       - "doc/**"
 

.github/workflows/code-checks.yml

@@ -5,12 +5,10 @@ on:
     branches:
       - main
       - 1.5.x
-      - 1.4.x
   pull_request:
     branches:
       - main
       - 1.5.x
-      - 1.4.x
 
 env:
   ENV_FILE: environment.yml
@@ -88,7 +86,7 @@ jobs:
         echo "PYTHONPATH=$PYTHONPATH" >> $GITHUB_ENV
       if: ${{ steps.build.outcome == 'success' && always() }}
 
-    - name: Typing
+    - name: Typing + pylint
       uses: pre-commit/action@v2.0.3
       with:
         extra_args: --hook-stage manual --all-files
@@ -153,6 +151,9 @@ jobs:
       - name: Build image
         run: docker build --pull --no-cache --tag pandas-dev-env .
 
+      - name: Show environment
+        run: docker run -w /home/pandas pandas-dev-env mamba run -n pandas-dev python -c "import pandas as pd; print(pd.show_versions())"
+
   requirements-dev-text-installable:
     name: Test install requirements-dev.txt
     runs-on: ubuntu-latest

.github/workflows/docbuild-and-upload.yml

@@ -5,14 +5,12 @@ on:
     branches:
       - main
       - 1.5.x
-      - 1.4.x
     tags:
       - '*'
   pull_request:
     branches:
       - main
       - 1.5.x
-      - 1.4.x
 
 env:
   ENV_FILE: environment.yml
@@ -66,22 +64,22 @@ jobs:
         mkdir -m 700 -p ~/.ssh
         echo "${{ secrets.server_ssh_key }}" > ~/.ssh/id_rsa
         chmod 600 ~/.ssh/id_rsa
-        echo "${{ secrets.server_ip }} ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE1Kkopomm7FHG5enATf7SgnpICZ4W2bw+Ho+afqin+w7sMcrsa0je7sbztFAV8YchDkiBKnWTG4cRT+KZgZCaY=" > ~/.ssh/known_hosts
-      if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        echo "${{ secrets.server_ip }} ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFjYkJBk7sos+r7yATODogQc3jUdW1aascGpyOD4bohj8dWjzwLJv/OJ/fyOQ5lmj81WKDk67tGtqNJYGL9acII=" > ~/.ssh/known_hosts
+      if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
 
     - name: Copy cheatsheets into site directory
       run: cp doc/cheatsheet/Pandas_Cheat_Sheet* web/build/
 
     - name: Upload web
-      run: rsync -az --delete --exclude='pandas-docs' --exclude='docs' web/build/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas
+      run: rsync -az --delete --exclude='pandas-docs' --exclude='docs' web/build/ web@${{ secrets.server_ip }}:/var/www/html
       if: github.event_name == 'push' && github.ref == 'refs/heads/main'
 
     - name: Upload dev docs
-      run: rsync -az --delete doc/build/html/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas/pandas-docs/dev
+      run: rsync -az --delete doc/build/html/ web@${{ secrets.server_ip }}:/var/www/html/pandas-docs/dev
       if: github.event_name == 'push' && github.ref == 'refs/heads/main'
 
     - name: Upload prod docs
-      run: rsync -az --delete doc/build/html/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas/pandas-docs/version/${GITHUB_REF_NAME}
+      run: rsync -az --delete doc/build/html/ web@${{ secrets.server_ip }}:/var/www/html/pandas-docs/version/${GITHUB_REF_NAME:1}
       if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
 
     - name: Move docs into site directory

.github/workflows/macos-windows.yml

@@ -1,16 +1,14 @@
-name: Windows-MacOS
+name: Windows-macOS
 
 on:
   push:
     branches:
       - main
       - 1.5.x
-      - 1.4.x
   pull_request:
     branches:
       - main
       - 1.5.x
-      - 1.4.x
     paths-ignore:
       - "doc/**"
 
@@ -28,7 +26,7 @@ jobs:
     defaults:
       run:
         shell: bash -el {0}
-    timeout-minutes: 120
+    timeout-minutes: 180
     strategy:
       matrix:
         os: [macos-latest, windows-latest]

.github/workflows/package-checks.yml

@@ -0,0 +1,56 @@
+name: Package Checks
+
+on:
+  push:
+    branches:
+      - main
+      - 1.5.x
+  pull_request:
+    branches:
+      - main
+      - 1.5.x
+
+permissions:
+  contents: read
+
+jobs:
+  pip:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        extra: ["test", "performance", "timezone", "computation", "fss", "aws", "gcp", "excel", "parquet", "feather", "hdf5", "spss", "postgresql", "mysql", "sql-other", "html", "xml", "plot", "output_formatting", "clipboard", "compression", "all"]
+      fail-fast: false
+    name: Install Extras - ${{ matrix.extra }}
+    concurrency:
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-pip-extras-${{ matrix.extra }}
+      cancel-in-progress: true
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Setup Python
+        id: setup_python
+        uses: actions/setup-python@v3
+        with:
+          python-version: '3.8'
+
+      # Hacky patch to disable building cython extensions.
+      # This job should only check that the extras successfully install.
+      - name: Disable building ext_modules
+        run: |
+          sed -i '/ext_modules=/d' setup.py
+        shell: bash -el {0}
+
+      - name: Install required dependencies
+        run: |
+          python -m pip install --upgrade pip setuptools wheel python-dateutil pytz numpy cython
+        shell: bash -el {0}
+
+      - name: Pip install with extra
+        run: |
+          python -m pip install -e .[${{ matrix.extra }}] --no-build-isolation
+        shell: bash -el {0}

.github/workflows/python-dev.yml

@@ -3,7 +3,7 @@
 #
 # In general, this file will remain frozen(present, but not running) until:
 #    - The next unreleased Python version has released beta 1
-#      - This version should be available on Github Actions.
+#      - This version should be available on GitHub Actions.
 #    - Our required build/runtime dependencies(numpy, pytz, Cython, python-dateutil)
 #      support that unreleased Python version.
 #    To unfreeze, comment out the ``if: false`` condition, and make sure you update
@@ -25,12 +25,10 @@ on:
     branches:
       - main
       - 1.5.x
-      - 1.4.x
   pull_request:
     branches:
       - main
       - 1.5.x
-      - 1.4.x
     paths-ignore:
       - "doc/**"
 
@@ -54,7 +52,7 @@ jobs:
         os: [ubuntu-latest, macOS-latest, windows-latest]
 
     name: actions-311-dev
-    timeout-minutes: 80
+    timeout-minutes: 120
 
     concurrency:
       #https://github.community/t/concurrecy-not-work-for-push/183068/7
@@ -75,15 +73,16 @@ jobs:
       run: |
         python --version
         python -m pip install --upgrade pip setuptools wheel
-        python -m pip install git+https://github.com/numpy/numpy.git
+        python -m pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
         python -m pip install git+https://github.com/nedbat/coveragepy.git
         python -m pip install python-dateutil pytz cython hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17
         python -m pip list
 
+    # GH 47305: Parallel build can cause flaky ImportError from pandas/_libs/tslibs
     - name: Build Pandas
       run: |
-        python setup.py build_ext -q -j2
-        python -m pip install -e . --no-build-isolation --no-use-pep517
+        python setup.py build_ext -q -j1
+        python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index
 
     - name: Build Version
       run: |

.github/workflows/scorecards.yml

@@ -0,0 +1,54 @@
+name: Scorecards supply-chain security
+on:
+  # Only the default branch is supported.
+  branch_protection_rule:
+  schedule:
+    - cron: '27 19 * * 4'
+  push:
+    branches: [ "main" ]
+
+# Declare default permissions as read only.
+permissions: read-all
+
+jobs:
+  analysis:
+    name: Scorecards analysis
+    runs-on: ubuntu-latest
+    permissions:
+      # Needed to upload the results to code-scanning dashboard.
+      security-events: write
+      # Used to receive a badge.
+      id-token: write
+
+    if: github.repository == 'pandas-dev/pandas'  # don't run on forks
+
+    steps:
+      - name: "Checkout code"
+        uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+
+      - name: "Run analysis"
+        uses: ossf/scorecard-action@v2.0.6
+        with:
+          results_file: results.sarif
+          results_format: sarif
+
+          # Publish the results for public repositories to enable scorecard badges. For more details, see
+          # https://github.com/ossf/scorecard-action#publishing-results.
+          publish_results: true
+
+      # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
+      # format to the repository Actions tab.
+      - name: "Upload artifact"
+        uses: actions/upload-artifact@v3
+        with:
+          name: SARIF file
+          path: results.sarif
+          retention-days: 5
+
+      # Upload the results to GitHub's code scanning dashboard.
+      - name: "Upload to code-scanning"
+        uses: github/codeql-action/upload-sarif@v1
+        with:
+          sarif_file: results.sarif