Merge branch 'main' into python-db-dtypes-pandas-issue28

Author: tswast

.github/workflows/code-checks.yml

@@ -79,6 +79,10 @@ jobs:
       run: ci/code_checks.sh docstrings
       if: ${{ steps.build.outcome == 'success' && always() }}
 
+    - name: Run check of documentation notebooks
+      run: ci/code_checks.sh notebooks
+      if: ${{ steps.build.outcome == 'success' && always() }}
+
     - name: Use existing environment for type checking
       run: |
         echo $PATH >> $GITHUB_PATH

.github/workflows/docbuild-and-upload.yml

@@ -54,11 +54,6 @@ jobs:
     - name: Build documentation zip
       run: doc/make.py zip_html
 
-    - name: Build the interactive terminal
-      run: |
-        cd web/interactive_terminal
-        jupyter lite build
-
     - name: Install ssh key
       run: |
         mkdir -m 700 -p ~/.ssh

.pre-commit-config.yaml

@@ -26,6 +26,7 @@ repos:
     hooks:
     -   id: codespell
         types_or: [python, rst, markdown]
+        additional_dependencies: [tomli]
 -   repo: https://github.com/MarcoGorelli/cython-lint
     rev: v0.2.1
     hooks:

asv_bench/benchmarks/frame_ctor.py

@@ -205,21 +205,4 @@ def time_frame_from_arrays_sparse(self):
         )
 
 
-class From3rdParty:
-    # GH#44616
-
-    def setup(self):
-        try:
-            import torch
-        except ImportError:
-            raise NotImplementedError
-
-        row = 700000
-        col = 64
-        self.val_tensor = torch.randn(row, col)
-
-    def time_from_torch(self):
-        DataFrame(self.val_tensor)
-
-
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/indexing.py

@@ -3,8 +3,6 @@
 lower-level methods directly on Index and subclasses, see index_object.py,
 indexing_engine.py, and index_cached.py
 """
-import itertools
-import string
 import warnings
 
 import numpy as np
@@ -353,15 +351,13 @@ def setup(self, index):
             "non_monotonic": CategoricalIndex(list("abc" * N)),
         }
         self.data = indices[index]
-        self.data_unique = CategoricalIndex(
-            ["".join(perm) for perm in itertools.permutations(string.printable, 3)]
-        )
+        self.data_unique = CategoricalIndex([str(i) for i in range(N * 3)])
 
         self.int_scalar = 10000
         self.int_list = list(range(10000))
 
         self.cat_scalar = "b"
-        self.cat_list = ["a", "c"]
+        self.cat_list = ["1", "3"]
 
     def time_getitem_scalar(self, index):
         self.data[self.int_scalar]

asv_bench/benchmarks/multiindex_object.py

@@ -379,4 +379,26 @@ def time_isin_large(self, dtype):
         self.midx.isin(self.values_large)
 
 
+class Putmask:
+    def setup(self):
+        N = 10**5
+        level1 = range(1_000)
+
+        level2 = date_range(start="1/1/2000", periods=N // 1000)
+        self.midx = MultiIndex.from_product([level1, level2])
+
+        level1 = range(1_000, 2_000)
+        self.midx_values = MultiIndex.from_product([level1, level2])
+
+        level2 = date_range(start="1/1/2010", periods=N // 1000)
+        self.midx_values_different = MultiIndex.from_product([level1, level2])
+        self.mask = np.array([True, False] * (N // 2))
+
+    def time_putmask(self):
+        self.midx.putmask(self.mask, self.midx_values)
+
+    def time_putmask_all_different(self):
+        self.midx.putmask(self.mask, self.midx_values_different)
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/reshape.py

@@ -210,7 +210,7 @@ def time_pivot_table_categorical_observed(self):
         )
 
     def time_pivot_table_margins_only_column(self):
-        self.df.pivot_table(columns=["key2", "key3"], margins=True)
+        self.df.pivot_table(columns=["key1", "key2", "key3"], margins=True)
 
 
 class Crosstab:

asv_bench/benchmarks/series_methods.py

@@ -348,4 +348,37 @@ def time_rank(self, dtype):
         self.s.rank()
 
 
+class Iter:
+
+    param_names = ["dtype"]
+    params = [
+        "bool",
+        "boolean",
+        "int64",
+        "Int64",
+        "float64",
+        "Float64",
+        "datetime64[ns]",
+    ]
+
+    def setup(self, dtype):
+        N = 10**5
+        if dtype in ["bool", "boolean"]:
+            data = np.repeat([True, False], N // 2)
+        elif dtype in ["int64", "Int64"]:
+            data = np.arange(N)
+        elif dtype in ["float64", "Float64"]:
+            data = np.random.randn(N)
+        elif dtype == "datetime64[ns]":
+            data = date_range("2000-01-01", freq="s", periods=N)
+        else:
+            raise NotImplementedError
+
+        self.s = Series(data, dtype=dtype)
+
+    def time_iter(self, dtype):
+        for v in self.s:
+            pass
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/sparse.py

@@ -219,12 +219,12 @@ def setup(self, fill_value):
         d = 1e-5
         arr = make_array(N, d, np.nan, np.float64)
         self.sp_arr = SparseArray(arr)
-        b_arr = np.full(shape=N, fill_value=fill_value, dtype=np.bool8)
+        b_arr = np.full(shape=N, fill_value=fill_value, dtype=np.bool_)
         fv_inds = np.unique(
             np.random.randint(low=0, high=N - 1, size=int(N * d), dtype=np.int32)
         )
         b_arr[fv_inds] = True if pd.isna(fill_value) else not fill_value
-        self.sp_b_arr = SparseArray(b_arr, dtype=np.bool8, fill_value=fill_value)
+        self.sp_b_arr = SparseArray(b_arr, dtype=np.bool_, fill_value=fill_value)
 
     def time_mask(self, fill_value):
         self.sp_arr[self.sp_b_arr]

ci/code_checks.sh

@@ -12,9 +12,10 @@
 #   $ ./ci/code_checks.sh doctests      # run doctests
 #   $ ./ci/code_checks.sh docstrings    # validate docstring errors
 #   $ ./ci/code_checks.sh single-docs   # check single-page docs build warning-free
+#   $ ./ci/code_checks.sh notebooks     # check execution of documentation notebooks
 
-[[ -z "$1" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "single-docs" ]] || \
-    { echo "Unknown command $1. Usage: $0 [code|doctests|docstrings]"; exit 9999; }
+[[ -z "$1" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "single-docs" || "$1" == "notebooks" ]] || \
+    { echo "Unknown command $1. Usage: $0 [code|doctests|docstrings|single-docs|notebooks]"; exit 9999; }
 
 BASE_DIR="$(dirname $0)/.."
 RET=0
@@ -84,6 +85,15 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
 
 fi
 
+### DOCUMENTATION NOTEBOOKS ###
+if [[ -z "$CHECK" || "$CHECK" == "notebooks" ]]; then
+
+    MSG='Notebooks' ; echo $MSG
+    jupyter nbconvert --execute $(find doc/source -name '*.ipynb') --to notebook
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
+fi
+
 ### SINGLE-PAGE DOCS ###
 if [[ -z "$CHECK" || "$CHECK" == "single-docs" ]]; then
     python doc/make.py --warnings-are-errors --single pandas.Series.value_counts