diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 64448fe..dac4ec7 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,7 +1,16 @@ {"image": "mcr.microsoft.com/devcontainers/python:3.13", "features": { - "ghcr.io/rocker-org/devcontainer-features/quarto-cli": - {"installChromium": true, "installTinyTex": true} + "ghcr.io/rocker-org/devcontainer-features/quarto-cli": + {"installChromium": false, "installTinyTex": false}, + "ghcr.io/devcontainers/features/node:2.1.0": {}, + "ghcr.io/anthropics/devcontainer-features/claude-code:1.0": {} }, -"postCreateCommand": "python -m pip install -r requirements.txt" +"remoteUser": "vscode", +"workspaceMount": "source=${localWorkspaceFolder},target=${localWorkspaceFolder},type=bind,consistency=cached", +"workspaceFolder": "${localWorkspaceFolder}", +"mounts": [ + "source=${localEnv:HOME}/.claude,target=/home/vscode/.claude,type=bind,consistency=cached", + "source=${localEnv:HOME}/.claude.json,target=/home/vscode/.claude.json,type=bind,consistency=cached" +], +"postCreateCommand": "bash .devcontainer/postCreate.sh" } diff --git a/.devcontainer/postCreate.sh b/.devcontainer/postCreate.sh new file mode 100755 index 0000000..285ca10 --- /dev/null +++ b/.devcontainer/postCreate.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -euo pipefail + +sudo apt-get update -qq + +# Librerías de sistema que requiere el Chrome headless de quarto (chrome-headless-shell) +# para poder arrancar. Sin ellas, quarto falla al rasterizar los diagramas mermaid con +# "error while loading shared libraries: libatk-1.0.so.0: cannot open shared object file". +sudo apt-get install -y --no-install-recommends \ + libatk1.0-0t64 \ + libatk-bridge2.0-0t64 \ + libatspi2.0-0t64 \ + libdbus-1-3 \ + libxcomposite1 \ + libxdamage1 \ + libxfixes3 \ + libxrandr2 \ + libgbm1 \ + libxkbcommon0 \ + libasound2t64 \ + fonts-liberation + +# Instalar GitHub CLI (gh) desde el repositorio oficial para tener la versión más reciente +(type -p wget >/dev/null || (sudo apt update && sudo apt install wget -y)) \ + && sudo mkdir -p -m 755 /etc/apt/keyrings \ + && out=$(mktemp) && wget -nv -O"$out" https://cli.github.com/packages/githubcli-archive-keyring.gpg \ + && cat "$out" | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ + && sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ + && sudo mkdir -p -m 755 /etc/apt/sources.list.d \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ + && sudo apt update \ + && sudo apt install gh -y + +# Instalar uv y usarlo para instalar los paquetes con el Python del sistema +# del contenedor (--system), sin crear un virtualenv: todo ya corre aislado +# dentro del propio docker, así que un venv sería una capa extra innecesaria. +curl -LsSf https://astral.sh/uv/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" + +# La imagen base deja site-packages y /usr/local/bin como propiedad de root, +# pero el devcontainer corre como el usuario "vscode": sin esto, uv falla +# con "Permission denied" al instalar en el Python del sistema. +sudo chown -R "$(id -u):$(id -g)" \ + "$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')" \ + /usr/local/bin /usr/local/share /usr/local/etc + +bash .devcontainer/python.sh diff --git a/.devcontainer/python.sh b/.devcontainer/python.sh new file mode 100644 index 0000000..2973b50 --- /dev/null +++ b/.devcontainer/python.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +uv pip install --system -e '.' +uv pip install --system -r requirements.txt diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..d4dcbca --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,140 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## What this is + +CompStats implements a bootstrap-based evaluation methodology for statistically comparing the +performance of multiple algorithms/systems in a competition-style setting (e.g., comparing several +classifiers' predictions against the same held-out gold labels). It follows the `sklearn.metrics` +API convention: score functions take `(y_true, y_pred)` but return a rich object (`Perf`) instead of +a bare float, giving access to bootstrap standard error, confidence intervals, and pairwise +significance testing between systems. + +## Language policy + +All project artifacts must be written in English: function/variable names, comments, parameters, +docstrings, and any other generated text. This is a research project, so code clarity matters — +follow the existing docstring/comment style shown throughout `CompStats/` (see `metrics.py`'s +`@metrics_docs` pattern, and the `:param:`/`:type:`/doctest-style docstrings in `interface.py`, +`bootstrap.py`, `measurements.py`) rather than introducing a different documentation style. + +Instructions from the user, in this conversation, may be given in either English or Spanish — +that does not change the above: respond and implement with English identifiers/comments/docs +regardless of the language the request was made in. + +## Workflow + +- Work is driven by GitHub issues: an issue is created describing the implementation to make, and + Claude is asked to read that issue and implement it. +- Never implement directly on the main branch. If not already on a branch other than main, create + a new branch first. +- The user typically works on a branch called `develop`. If already on `develop` (or another + non-main branch), it's fine to implement there directly — no need to create a new branch just + for that. + +## Commands + +Install the package and its dependencies (editable install, used by the devcontainer via +`.devcontainer/python.sh`): + +```bash +pip install -e . +pip install -r requirements.txt +``` + +Run the full test suite (this is what VS Code's test explorer and `.vscode/settings.json` are +configured to use — despite CI still invoking `nosetests`, day-to-day development here uses pytest): + +```bash +pytest CompStats +``` + +Run a single test file or test: + +```bash +pytest CompStats/tests/test_interface.py +pytest CompStats/tests/test_interface.py::test_Perf_name +``` + +Run with coverage (mirrors what CI collects, excluding the `tests` package per `.coveragerc`): + +```bash +coverage run -m pytest CompStats +coverage report +``` + +Build the Sphinx docs: + +```bash +cd docs && make html +``` + +Note: `.github/workflows/test.yaml` (CI) builds the environment with conda and runs `nosetests`, +not pytest — this is legacy and inconsistent with local/devcontainer tooling. Don't be surprised if +CI config and local dev commands diverge; prefer `pytest` locally. + +## Architecture + +The package is small and organized around one core data flow: raw predictions → bootstrap resampled +statistic → derived comparisons/plots. + +- **`bootstrap.py` — `StatisticSamples`**: the foundational primitive. Given a `statistic` callable + (e.g. `accuracy_score`), it draws `num_samples` bootstrap resamples (with replacement) of the + population and evaluates the statistic on each resample, optionally in parallel (`joblib`). + Results for a named system are cached in `self.calls[name]` (a dict of name → ndarray of bootstrap + samples). Bootstrap sample *indices* are cached per population size in `self._samples`, so multiple + algorithms evaluated against the same `y_true` reuse the same resampling (this is what makes + pairwise comparisons valid/paired). Supports `__sklearn_clone__` so `sklearn.base.clone` produces a + fresh instance carrying over params (used heavily to create `Difference` objects from a `Perf` + without recomputing bootstrap samples). + +- **`interface.py` — `Perf` and `Difference`**: the main user-facing entry point (re-exported at + package root). `Perf(y_true, *y_pred, name=..., score_func=..., error_func=..., **kwargs)` wraps + one or more systems' predictions against shared ground truth. Exactly one of `score_func` / + `error_func` must be set (asserted via XOR) — `score_func` implies bigger-is-better (`BiB=True`), + `error_func` implies smaller-is-better (`BiB=False`). Internally holds a `StatisticSamples` keyed + by system name; new predictions can be added later via `perf(y_pred, name=...)` (`__call__`). + `Perf.difference(wrt=...)` produces a `Difference` instance (comparing every system against the + best, or an explicit reference) whose `p_value()` is computed directly from the bootstrap + distribution of paired differences — no parametric test assumptions. `Perf.plot()` / + `Difference.plot()` render via seaborn `catplot`, with confidence intervals computed by + `measurements.CI` passed as the `errorbar` callback. + +- **`metrics.py`**: thin wrappers around `sklearn.metrics` functions (`accuracy_score`, + `balanced_accuracy_score`, `top_k_accuracy_score`, `f1_score`, etc.). Each wrapper closes over the + sklearn metric (plus its metric-specific kwargs like `average`, `normalize`) and constructs a + `Perf` with that as `score_func`/`error_func`. The `@metrics_docs` decorator (from `utils.py`) + injects the shared `Perf`-style docstring (params like `num_samples`, `n_jobs`, `use_tqdm`) into + each wrapper automatically — when adding a new metric wrapper, follow this same + `@metrics_docs(hy_name=..., attr_name=...)` + inner-function-closure pattern rather than duplicating + docstrings. + +- **`measurements.py`**: stateless helpers — `CI` (percentile bootstrap confidence interval), `SE` + (bootstrap standard error), `difference_p_value`. Each accepts either a raw ndarray of bootstrap + samples or a `StatisticSamples` instance (in which case it maps itself over `.calls`). + +- **`performance.py`**: an alternative, more functional (non-`Perf`) API operating directly on a + `pandas.DataFrame` (one gold column + one column per system) — `performance()`, + `difference()`/`all_differences()`, and the `plot_performance*`/`plot_difference*` family, plus + `*_multiple` variants for comparing several metrics at once (used for multi-metric competition + reports: coefficient of variation, PPI, distance-to-best per metric). This module is older/more + ad-hoc than `interface.py`'s `Perf`; new comparison features generally belong on `Perf`/`Difference` + unless they specifically need the DataFrame-of-multiple-metrics shape. + +- **`utils.py`**: `progress_bar` (tqdm wrapper, no-op if tqdm isn't installed or `use_tqdm=False`), + `metrics_docs` (docstring-injecting decorator described above), and `dataframe()` (melts a `Perf`'s + or `Difference`'s bootstrap samples into a long-format DataFrame for seaborn plotting). + +### Key invariants to preserve when modifying this code + +- Bootstrap resampling must stay *paired* across systems being compared — `StatisticSamples.samples` + caches resample indices by population size `N` precisely so every system's bootstrap replicate `i` + uses the same resampled indices. Don't introduce per-system independent resampling. +- `BiB` (Bigger is Better) must be threaded consistently: `score_func` → `BiB=True`, `error_func` → + `BiB=False`. Sorting, `best`, and p-value sign logic throughout `interface.py`/`performance.py` + depend on this flag rather than re-deriving it from the function. +- `sklearn.base.clone` / `__sklearn_clone__` is used to duplicate `Perf`/`StatisticSamples` instances + while reusing already-computed bootstrap samples (e.g. `Perf.difference()`, `performance.difference`). + Don't replace these with plain re-instantiation, as that silently redraws new bootstrap samples and + breaks paired comparisons. diff --git a/CompStats/__init__.py b/CompStats/__init__.py index c92f6cc..976141f 100644 --- a/CompStats/__init__.py +++ b/CompStats/__init__.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '0.1.15' +__version__ = '0.2.0' from CompStats.bootstrap import StatisticSamples from CompStats.measurements import CI, SE, difference_p_value from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference diff --git a/CompStats/bootstrap.py b/CompStats/bootstrap.py index 22b1994..791682a 100644 --- a/CompStats/bootstrap.py +++ b/CompStats/bootstrap.py @@ -27,6 +27,8 @@ class StatisticSamples: :type num_samples: int :param n_jobs: Number of jobs to run in parallel, default=1. :type n_jobs: int + :param BiB: Bigger is Better; a single bool for a scalar/vector statistic, or one bool per column when :py:attr:`statistic` returns the concatenation of several measures (see :py:class:`~CompStats.interface.Perf`'s multi-measure support). + :type BiB: bool or numpy.ndarray[bool] >>> from CompStats import StatisticSamples diff --git a/CompStats/interface.py b/CompStats/interface.py index b11b009..6e28e63 100644 --- a/CompStats/interface.py +++ b/CompStats/interface.py @@ -28,10 +28,12 @@ class Perf(object): :param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement. :type y_true: numpy.ndarray or pandas.DataFrame - :param score_func: Function to measure the performance, it is assumed that the best algorithm has the highest value. - :type score_func: Function where the first argument is :math:`y` and the second is :math:`\\hat{y}.` - :param error_func: Function to measure the performance where the best algorithm has the lowest value. - :type error_func: Function where the first argument is :math:`y` and the second is :math:`\\hat{y}.` + :param score_func: Function (or list of functions) to measure the performance, it is assumed that the best algorithm has the highest value. :py:attr:`score_func` and :py:attr:`error_func` can be given simultaneously to combine score-type and error-type measures into a single, multi-measure :py:class:`Perf.` + :type score_func: Function, or list of functions, where the first argument is :math:`y` and the second is :math:`\\hat{y}.` + :param error_func: Function (or list of functions) to measure the performance where the best algorithm has the lowest value. + :type error_func: Function, or list of functions, where the first argument is :math:`y` and the second is :math:`\\hat{y}.` + :param measure_names: Display name for each measure, only relevant when more than one measure is given; defaults to each function's ``__name__``. + :type measure_names: list :param y_pred: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`args.` :type y_pred: numpy.ndarray :param kwargs: Predictions, the algorithms will be identified using the keyword @@ -88,18 +90,32 @@ class Perf(object): 0.0222 (0.0237) <= alg-1 0.0222 (0.0215) <= forest + Two or more measures can be combined into a single :py:class:`Perf` instance + (e.g. macro-F1 together with macro-recall) by passing a list of functions + to :py:attr:`score_func`/:py:attr:`error_func` -- see :py:mod:`CompStats.metrics`'s + ``.measure`` factories (e.g. :py:func:`~CompStats.metrics.f1_score.measure`). Every + measure is evaluated on the same bootstrap resamples, so comparisons across + algorithms remain paired for each measure. + + >>> from CompStats.metrics import f1_score, recall_score + >>> perf = Perf(y_val, hy, forest=ens.predict(X_val), + ... score_func=[f1_score.measure(average='macro'), + ... recall_score.measure(average='macro')]) """ def __init__(self, y_true, *y_pred, name:str=None, score_func=balanced_accuracy_score, error_func=None, + measure_names:list=None, num_samples: int=500, n_jobs: int=-1, use_tqdm=True, **kwargs): - assert (score_func is None) ^ (error_func is None) - self.score_func = score_func - self.error_func = error_func + assert (len(self._as_list(score_func)) + + len(self._as_list(error_func))) >= 1 + self._score_func = score_func + self._error_func = error_func + self.measure_names = measure_names algs = {} if name is not None: if isinstance(name, str): @@ -117,10 +133,37 @@ def __init__(self, y_true, *y_pred, self.sorting_func = np.linalg.norm self._init() + @staticmethod + def _as_list(value): + """Normalize a score_func/error_func argument into a list of callables""" + if value is None: + return [] + if isinstance(value, (list, tuple)): + return list(value) + return [value] + + @property + def _measures(self): + """List of (callable, BiB) pairs, one per measure being evaluated + + Each callable's own :py:attr:`BiB` attribute (set by, e.g., a + :py:meth:`metrics.py ` wrapper's ``.measure`` factory) + takes precedence over the default direction implied by which + argument (:py:attr:`score_func` or :py:attr:`error_func`) it came from. + """ + def tagged(funcs, default_bib): + return [(f, bool(getattr(f, 'BiB', default_bib))) + for f in self._as_list(funcs)] + return tagged(self.score_func, True) + tagged(self.error_func, False) + def _init(self): """Compute the bootstrap statistic""" - bib = True if self.score_func is not None else False + measures = self._measures + if len(measures) == 1: + bib = measures[0][1] + else: + bib = np.array([b for _, b in measures]) if hasattr(self, '_statistic_samples'): _ = self.statistic_samples _.BiB = bib @@ -138,6 +181,7 @@ def get_params(self): return dict(y_true=self.y_true, score_func=self.score_func, error_func=self.error_func, + measure_names=self._measure_names, num_samples=self.num_samples, n_jobs=self.n_jobs) @@ -152,7 +196,12 @@ def __sklearn_clone__(self): def __repr__(self): """Prediction statistics with standard error in parenthesis""" - arg = 'score_func' if self.error_func is None else 'error_func' + if self.error_func is None: + arg = 'score_func' + elif self.score_func is None: + arg = 'error_func' + else: + arg = 'score_func/error_func' func_name = self.statistic_func.__name__ statistic = self.statistic if isinstance(statistic, dict): @@ -227,8 +276,9 @@ def difference(self, wrt: str=None): base = self.statistic_samples.calls[wrt] else: base = np.array([self.statistic_samples.calls[key][:, col] - for col, key in enumerate(wrt)]).T - sign = 1 if self.statistic_samples.BiB else -1 + for col, key in enumerate(wrt)]).T + BiB = self.statistic_samples.BiB + sign = np.where(BiB, 1, -1) if isinstance(BiB, np.ndarray) else (1 if BiB else -1) diff = dict() for k, v in self.statistic_samples.calls.items(): if base.ndim == 1 and k == wrt: @@ -254,20 +304,19 @@ def best(self): else: self._best = np.array([key] * value.shape[1]) return self._best - BiB = bool(self.statistic_samples.BiB) + BiB = self.statistic_samples.BiB keys = np.array(list(self.statistic.keys())) data = np.asanyarray([self.statistic[k] - for k in keys]) + for k in keys]) if isinstance(self.statistic[keys[0]], np.ndarray): - if BiB: - best = data.argmax(axis=0) + argmax_idx = data.argmax(axis=0) + argmin_idx = data.argmin(axis=0) + if isinstance(BiB, np.ndarray): + best = np.where(BiB, argmax_idx, argmin_idx) else: - best = data.argmin(axis=0) + best = argmax_idx if BiB else argmin_idx else: - if BiB: - best = data.argmax() - else: - best = data.argmin() + best = data.argmax() if bool(BiB) else data.argmin() self._best = keys[best] return self._best @@ -405,7 +454,9 @@ def plot(self, value_name:str=None, """ import seaborn as sns if value_name is None: - if self.score_func is not None: + if len(self._measures) > 1: + value_name = 'Value' + elif self.score_func is not None: value_name = 'Score' else: value_name = 'Error' @@ -469,9 +520,11 @@ def dataframe(self, comparison:bool=False, >>> df = perf.dataframe() """ if perf_names is None and isinstance(self.best, np.ndarray): - func_name = self.statistic_func.__name__ - perf_names = [f'{func_name}({i})' - for i, k in enumerate(self.best)] + perf_names = self.measure_names + if perf_names is None: + func_name = self.statistic_func.__name__ + perf_names = [f'{func_name}({i})' + for i, k in enumerate(self.best)] df = dataframe(self, value_name=value_name, var_name=var_name, alg_legend=alg_legend, @@ -516,10 +569,44 @@ def n_jobs(self, value): @property def statistic_func(self): - """Statistic function""" - if self.score_func is not None: - return self.score_func - return self.error_func + """Statistic function + + A single :py:attr:`score_func`/:py:attr:`error_func` callable is + returned as-is; when more than one measure is given (either as a + list, or by mixing :py:attr:`score_func` and :py:attr:`error_func`), + a composite callable is returned that concatenates every measure's + output into a single vector, evaluated on the same bootstrap samples. + """ + measures = self._measures + if len(measures) == 1: + return measures[0][0] + funcs = [f for f, _ in measures] + names = self.measure_names + + def composite(y, hy): + return np.concatenate([np.atleast_1d(f(y, hy)) for f in funcs]) + composite.__name__ = '+'.join(names) + return composite + + @property + def measure_names(self): + """Display name for each measure, used when combining more than one + + Defaults to each measure's function ``__name__`` (available because + every :py:mod:`CompStats.metrics` wrapper's inner function is + ``functools.wraps``-decorated with the corresponding sklearn metric). + """ + if self._measure_names is not None: + return self._measure_names + measures = self._measures + if len(measures) <= 1: + return None + return [getattr(f, '__name__', f'measure-{i}') + for i, (f, _) in enumerate(measures)] + + @measure_names.setter + def measure_names(self, value): + self._measure_names = value @property def statistic_samples(self): @@ -700,7 +787,8 @@ def p_value(self, right:bool=True): {'forest': np.float64(0.3)} """ values = [] - sign = 1 if self.statistic_samples.BiB else -1 + BiB = self.statistic_samples.BiB + sign = np.where(BiB, 1, -1) if isinstance(BiB, np.ndarray) else (1 if BiB else -1) delta_best = self._delta_best() for k, v in self.statistic_samples.calls.items(): delta = 2 * sign * (delta_best - self.statistic[k]) diff --git a/CompStats/metrics.py b/CompStats/metrics.py index 3c1de02..f220c9c 100644 --- a/CompStats/metrics.py +++ b/CompStats/metrics.py @@ -23,26 +23,50 @@ ######################################################## +def _accuracy_score_measure(normalize=True, sample_weight=None): + """Build the tagged (score-type, BiB=True) measure used by :py:func:`accuracy_score`""" + + @wraps(metrics.accuracy_score) + def inner(y, hy): + return metrics.accuracy_score(y, hy, + normalize=normalize, + sample_weight=sample_weight) + inner.BiB = True + return inner + + @metrics_docs(hy_name='y_pred', attr_name='score_func') def accuracy_score(y_true, *y_pred, normalize=True, sample_weight=None, num_samples: int=500, - n_jobs: int=-1, + n_jobs: int=-1, use_tqdm=True, **kwargs): """accuracy_score""" - @wraps(metrics.accuracy_score) - def inner(y, hy): - return metrics.accuracy_score(y, hy, - normalize=normalize, - sample_weight=sample_weight) - return Perf(y_true, *y_pred, score_func=inner, + return Perf(y_true, *y_pred, + score_func=_accuracy_score_measure(normalize=normalize, + sample_weight=sample_weight), num_samples=num_samples, n_jobs=n_jobs, use_tqdm=use_tqdm, **kwargs) +accuracy_score.measure = _accuracy_score_measure + + +def _balanced_accuracy_score_measure(sample_weight=None, adjusted=False): + """Build the tagged (score-type, BiB=True) measure used by :py:func:`balanced_accuracy_score`""" + + @wraps(metrics.balanced_accuracy_score) + def inner(y, hy): + return metrics.balanced_accuracy_score(y, hy, + adjusted=adjusted, + sample_weight=sample_weight) + inner.BiB = True + return inner + + @metrics_docs(hy_name='y_pred', attr_name='score_func') def balanced_accuracy_score(y_true, *y_pred, sample_weight=None, adjusted=False, @@ -52,17 +76,29 @@ def balanced_accuracy_score(y_true, *y_pred, **kwargs): """balanced_accuracy_score""" - @wraps(metrics.balanced_accuracy_score) - def inner(y, hy): - return metrics.balanced_accuracy_score(y, hy, - adjusted=adjusted, - sample_weight=sample_weight) - return Perf(y_true, *y_pred, score_func=inner, + return Perf(y_true, *y_pred, + score_func=_balanced_accuracy_score_measure(sample_weight=sample_weight, + adjusted=adjusted), num_samples=num_samples, n_jobs=n_jobs, use_tqdm=use_tqdm, **kwargs) +balanced_accuracy_score.measure = _balanced_accuracy_score_measure + + +def _top_k_accuracy_score_measure(k=2, normalize=True, sample_weight=None, labels=None): + """Build the tagged (score-type, BiB=True) measure used by :py:func:`top_k_accuracy_score`""" + + @wraps(metrics.top_k_accuracy_score) + def inner(y, hy): + return metrics.top_k_accuracy_score(y, hy, k=k, + normalize=normalize, sample_weight=sample_weight, + labels=labels) + inner.BiB = True + return inner + + @metrics_docs(hy_name='y_score', attr_name='score_func') def top_k_accuracy_score(y_true, *y_score, k=2, normalize=True, sample_weight=None, @@ -73,17 +109,30 @@ def top_k_accuracy_score(y_true, *y_score, k=2, **kwargs): """top_k_accuracy_score""" - @wraps(metrics.top_k_accuracy_score) - def inner(y, hy): - return metrics.top_k_accuracy_score(y, hy, k=k, - normalize=normalize, sample_weight=sample_weight, - labels=labels) - return Perf(y_true, *y_score, score_func=inner, + return Perf(y_true, *y_score, + score_func=_top_k_accuracy_score_measure(k=k, normalize=normalize, + sample_weight=sample_weight, + labels=labels), num_samples=num_samples, n_jobs=n_jobs, use_tqdm=use_tqdm, **kwargs) +top_k_accuracy_score.measure = _top_k_accuracy_score_measure + + +def _average_precision_score_measure(average='macro', sample_weight=None): + """Build the tagged (score-type, BiB=True) measure used by :py:func:`average_precision_score`""" + + @wraps(metrics.average_precision_score) + def inner(y, hy): + return metrics.average_precision_score(y, hy, + average=average, + sample_weight=sample_weight) + inner.BiB = True + return inner + + @metrics_docs(hy_name='y_score', attr_name='score_func') def average_precision_score(y_true, *y_score, average='macro', @@ -94,17 +143,29 @@ def average_precision_score(y_true, *y_score, **kwargs): """average_precision_score""" - @wraps(metrics.average_precision_score) - def inner(y, hy): - return metrics.average_precision_score(y, hy, - average=average, - sample_weight=sample_weight) - return Perf(y_true, *y_score, score_func=inner, + return Perf(y_true, *y_score, + score_func=_average_precision_score_measure(average=average, + sample_weight=sample_weight), num_samples=num_samples, n_jobs=n_jobs, use_tqdm=use_tqdm, **kwargs) +average_precision_score.measure = _average_precision_score_measure + + +def _brier_score_loss_measure(sample_weight=None, pos_label=None): + """Build the tagged (error-type, BiB=False) measure used by :py:func:`brier_score_loss`""" + + @wraps(metrics.brier_score_loss) + def inner(y, hy): + return metrics.brier_score_loss(y, hy, + sample_weight=sample_weight, + pos_label=pos_label) + inner.BiB = False + return inner + + @metrics_docs(hy_name='y_proba', attr_name='error_func') def brier_score_loss(y_true, *y_proba, sample_weight=None, @@ -112,28 +173,24 @@ def brier_score_loss(y_true, *y_proba, num_samples: int=500, n_jobs: int=-1, use_tqdm=True, - **kwargs + **kwargs ): """brier_score_loss""" - @wraps(metrics.brier_score_loss) - def inner(y, hy): - return metrics.brier_score_loss(y, hy, - sample_weight=sample_weight, - pos_label=pos_label) - return Perf(y_true, *y_proba, score_func=None, error_func=inner, + return Perf(y_true, *y_proba, score_func=None, + error_func=_brier_score_loss_measure(sample_weight=sample_weight, + pos_label=pos_label), num_samples=num_samples, n_jobs=n_jobs, use_tqdm=use_tqdm, **kwargs) -@metrics_docs(hy_name='y_pred', attr_name='score_func') -def f1_score(y_true, *y_pred, labels=None, pos_label=1, - average='binary', sample_weight=None, - zero_division='warn', num_samples: int=500, - n_jobs: int=-1, use_tqdm=True, - **kwargs): - """f1_score""" +brier_score_loss.measure = _brier_score_loss_measure + + +def _f1_score_measure(labels=None, pos_label=1, average='binary', + sample_weight=None, zero_division='warn'): + """Build the tagged (score-type, BiB=True) measure used by :py:func:`f1_score`""" @wraps(metrics.f1_score) def inner(y, hy): @@ -142,12 +199,43 @@ def inner(y, hy): average=average, sample_weight=sample_weight, zero_division=zero_division) - return Perf(y_true, *y_pred, score_func=inner, + inner.BiB = True + return inner + + +@metrics_docs(hy_name='y_pred', attr_name='score_func') +def f1_score(y_true, *y_pred, labels=None, pos_label=1, + average='binary', sample_weight=None, + zero_division='warn', num_samples: int=500, + n_jobs: int=-1, use_tqdm=True, + **kwargs): + """f1_score""" + + return Perf(y_true, *y_pred, + score_func=_f1_score_measure(labels=labels, pos_label=pos_label, + average=average, + sample_weight=sample_weight, + zero_division=zero_division), num_samples=num_samples, n_jobs=n_jobs, use_tqdm=use_tqdm, **kwargs) +f1_score.measure = _f1_score_measure + + +def _log_loss_measure(normalize=True, sample_weight=None, labels=None): + """Build the tagged (error-type, BiB=False) measure used by :py:func:`log_loss`""" + + @wraps(metrics.log_loss) + def inner(y, hy): + return metrics.log_loss(y, hy, normalize=normalize, + sample_weight=sample_weight, + labels=labels) + inner.BiB = False + return inner + + @metrics_docs(hy_name='y_pred', attr_name='error_func') def log_loss(y_true, *y_pred, normalize=True, @@ -159,17 +247,34 @@ def log_loss(y_true, *y_pred, **kwargs): """log_loss""" - @wraps(metrics.log_loss) - def inner(y, hy): - return metrics.log_loss(y, hy, normalize=normalize, - sample_weight=sample_weight, - labels=labels) - return Perf(y_true, *y_pred, error_func=inner, score_func=None, + return Perf(y_true, *y_pred, score_func=None, + error_func=_log_loss_measure(normalize=normalize, + sample_weight=sample_weight, + labels=labels), num_samples=num_samples, n_jobs=n_jobs, use_tqdm=use_tqdm, **kwargs) +log_loss.measure = _log_loss_measure + + +def _precision_score_measure(labels=None, pos_label=1, average='binary', + sample_weight=None, zero_division='warn'): + """Build the tagged (score-type, BiB=True) measure used by :py:func:`precision_score`""" + + @wraps(metrics.precision_score) + def inner(y, hy): + return metrics.precision_score(y, hy, + labels=labels, + pos_label=pos_label, + average=average, + sample_weight=sample_weight, + zero_division=zero_division) + inner.BiB = True + return inner + + @metrics_docs(hy_name='y_pred', attr_name='score_func') def precision_score(y_true, *y_pred, @@ -184,20 +289,35 @@ def precision_score(y_true, **kwargs): """precision_score""" - @wraps(metrics.precision_score) - def inner(y, hy): - return metrics.precision_score(y, hy, - labels=labels, - pos_label=pos_label, - average=average, - sample_weight=sample_weight, - zero_division=zero_division) - return Perf(y_true, *y_pred, score_func=inner, + return Perf(y_true, *y_pred, + score_func=_precision_score_measure(labels=labels, pos_label=pos_label, + average=average, + sample_weight=sample_weight, + zero_division=zero_division), num_samples=num_samples, n_jobs=n_jobs, use_tqdm=use_tqdm, **kwargs) +precision_score.measure = _precision_score_measure + + +def _recall_score_measure(labels=None, pos_label=1, average='binary', + sample_weight=None, zero_division='warn'): + """Build the tagged (score-type, BiB=True) measure used by :py:func:`recall_score`""" + + @wraps(metrics.recall_score) + def inner(y, hy): + return metrics.recall_score(y, hy, + labels=labels, + pos_label=pos_label, + average=average, + sample_weight=sample_weight, + zero_division=zero_division) + inner.BiB = True + return inner + + @metrics_docs(hy_name='y_pred', attr_name='score_func') def recall_score(y_true, *y_pred, @@ -212,20 +332,35 @@ def recall_score(y_true, **kwargs): """recall_score""" - @wraps(metrics.recall_score) - def inner(y, hy): - return metrics.recall_score(y, hy, - labels=labels, - pos_label=pos_label, - average=average, - sample_weight=sample_weight, - zero_division=zero_division) - return Perf(y_true, *y_pred, score_func=inner, + return Perf(y_true, *y_pred, + score_func=_recall_score_measure(labels=labels, pos_label=pos_label, + average=average, + sample_weight=sample_weight, + zero_division=zero_division), num_samples=num_samples, n_jobs=n_jobs, use_tqdm=use_tqdm, **kwargs) +recall_score.measure = _recall_score_measure + + +def _jaccard_score_measure(labels=None, pos_label=1, average='binary', + sample_weight=None, zero_division='warn'): + """Build the tagged (score-type, BiB=True) measure used by :py:func:`jaccard_score`""" + + @wraps(metrics.jaccard_score) + def inner(y, hy): + return metrics.jaccard_score(y, hy, + labels=labels, + pos_label=pos_label, + average=average, + sample_weight=sample_weight, + zero_division=zero_division) + inner.BiB = True + return inner + + @metrics_docs(hy_name='y_pred', attr_name='score_func') def jaccard_score(y_true, *y_pred, @@ -240,20 +375,35 @@ def jaccard_score(y_true, **kwargs): """jaccard_score""" - @wraps(metrics.jaccard_score) - def inner(y, hy): - return metrics.jaccard_score(y, hy, - labels=labels, - pos_label=pos_label, - average=average, - sample_weight=sample_weight, - zero_division=zero_division) - return Perf(y_true, *y_pred, score_func=inner, + return Perf(y_true, *y_pred, + score_func=_jaccard_score_measure(labels=labels, pos_label=pos_label, + average=average, + sample_weight=sample_weight, + zero_division=zero_division), num_samples=num_samples, n_jobs=n_jobs, use_tqdm=use_tqdm, **kwargs) +jaccard_score.measure = _jaccard_score_measure + + +def _roc_auc_score_measure(average='macro', sample_weight=None, max_fpr=None, + multi_class='raise', labels=None): + """Build the tagged (score-type, BiB=True) measure used by :py:func:`roc_auc_score`""" + + @wraps(metrics.roc_auc_score) + def inner(y, hy): + return metrics.roc_auc_score(y, hy, + average=average, + sample_weight=sample_weight, + max_fpr=max_fpr, + multi_class=multi_class, + labels=labels) + inner.BiB = True + return inner + + @metrics_docs(hy_name='y_score', attr_name='score_func') def roc_auc_score(y_true, *y_score, @@ -268,20 +418,32 @@ def roc_auc_score(y_true, **kwargs): """roc_auc_score""" - @wraps(metrics.roc_auc_score) - def inner(y, hy): - return metrics.roc_auc_score(y, hy, - average=average, - sample_weight=sample_weight, - max_fpr=max_fpr, - multi_class=multi_class, - labels=labels) - return Perf(y_true, *y_score, score_func=inner, + return Perf(y_true, *y_score, + score_func=_roc_auc_score_measure(average=average, + sample_weight=sample_weight, + max_fpr=max_fpr, + multi_class=multi_class, + labels=labels), num_samples=num_samples, n_jobs=n_jobs, use_tqdm=use_tqdm, **kwargs) +roc_auc_score.measure = _roc_auc_score_measure + + +def _d2_log_loss_score_measure(sample_weight=None, labels=None): + """Build the tagged (score-type, BiB=True) measure used by :py:func:`d2_log_loss_score`""" + + @wraps(metrics.d2_log_loss_score) + def inner(y, hy): + return metrics.d2_log_loss_score(y, hy, + sample_weight=sample_weight, + labels=labels) + inner.BiB = True + return inner + + @metrics_docs(hy_name='y_proba', attr_name='score_func') def d2_log_loss_score(y_true, *y_proba, sample_weight=None, @@ -292,35 +454,44 @@ def d2_log_loss_score(y_true, *y_proba, **kwargs): """d2_log_loss_score""" - @wraps(metrics.d2_log_loss_score) - def inner(y, hy): - return metrics.d2_log_loss_score(y, hy, - sample_weight=sample_weight, - labels=labels) - return Perf(y_true, *y_proba, score_func=inner, error_func=None, + return Perf(y_true, *y_proba, + score_func=_d2_log_loss_score_measure(sample_weight=sample_weight, + labels=labels), + error_func=None, num_samples=num_samples, n_jobs=n_jobs, use_tqdm=use_tqdm, **kwargs) +d2_log_loss_score.measure = _d2_log_loss_score_measure + + +def _macro_f1_measure(labels=None, sample_weight=None, zero_division='warn'): + """Build the tagged (score-type, BiB=True) measure used by :py:func:`macro_f1`""" + + return f1_score.measure(labels=labels, average='macro', + sample_weight=sample_weight, + zero_division=zero_division) + + def macro_f1(y_true, *y_pred, labels=None, sample_weight=None, zero_division='warn', num_samples: int=500, n_jobs: int=-1, use_tqdm=True, **kwargs): """:py:class:`~CompStats.interface.Perf` with :py:func:`~sklearn.metrics.f1_score` (as :py:attr:`score_func`) with the parameteres needed to compute the macro score. The parameters not described can be found in :py:func:`~sklearn.metrics.f1_score` - :param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement. - :type y_true: numpy.ndarray or pandas.DataFrame - :param y_pred: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`y_pred.` - :type y_pred: numpy.ndarray - :param kwargs: Predictions, the algorithms will be identified using the keyword - :type kwargs: numpy.ndarray - :param num_samples: Number of bootstrap samples, default=500. - :type num_samples: int - :param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads. - :type n_jobs: int - :param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True - :type use_tqdm: bool + :param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement. + :type y_true: numpy.ndarray or pandas.DataFrame + :param y_pred: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`y_pred.` + :type y_pred: numpy.ndarray + :param kwargs: Predictions, the algorithms will be identified using the keyword + :type kwargs: numpy.ndarray + :param num_samples: Number of bootstrap samples, default=500. + :type num_samples: int + :param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads. + :type n_jobs: int + :param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True + :type use_tqdm: bool """ return f1_score(y_true, *y_pred, labels=labels, average='macro', sample_weight=sample_weight, zero_division=zero_division, @@ -328,24 +499,35 @@ def macro_f1(y_true, *y_pred, labels=None, use_tqdm=use_tqdm, **kwargs) +macro_f1.measure = _macro_f1_measure + + +def _macro_recall_measure(labels=None, sample_weight=None, zero_division='warn'): + """Build the tagged (score-type, BiB=True) measure used by :py:func:`macro_recall`""" + + return recall_score.measure(labels=labels, average='macro', + sample_weight=sample_weight, + zero_division=zero_division) + + def macro_recall(y_true, *y_pred, labels=None, sample_weight=None, zero_division='warn', num_samples: int=500, n_jobs: int=-1, use_tqdm=True, **kwargs): """:py:class:`~CompStats.interface.Perf` with :py:func:`~sklearn.metrics.recall_score` (as :py:attr:`score_func`) with the parameteres needed to compute the macro score. The parameters not described can be found in :py:func:`~sklearn.metrics.recall_score` - :param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement. - :type y_true: numpy.ndarray or pandas.DataFrame - :param y_pred: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`y_pred.` - :type y_pred: numpy.ndarray - :param kwargs: Predictions, the algorithms will be identified using the keyword - :type kwargs: numpy.ndarray - :param num_samples: Number of bootstrap samples, default=500. - :type num_samples: int - :param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads. - :type n_jobs: int - :param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True - :type use_tqdm: bool + :param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement. + :type y_true: numpy.ndarray or pandas.DataFrame + :param y_pred: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`y_pred.` + :type y_pred: numpy.ndarray + :param kwargs: Predictions, the algorithms will be identified using the keyword + :type kwargs: numpy.ndarray + :param num_samples: Number of bootstrap samples, default=500. + :type num_samples: int + :param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads. + :type n_jobs: int + :param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True + :type use_tqdm: bool """ return recall_score(y_true, *y_pred, labels=labels, average='macro', sample_weight=sample_weight, zero_division=zero_division, @@ -353,24 +535,35 @@ def macro_recall(y_true, *y_pred, labels=None, use_tqdm=use_tqdm, **kwargs) +macro_recall.measure = _macro_recall_measure + + +def _macro_precision_measure(labels=None, sample_weight=None, zero_division='warn'): + """Build the tagged (score-type, BiB=True) measure used by :py:func:`macro_precision`""" + + return precision_score.measure(labels=labels, average='macro', + sample_weight=sample_weight, + zero_division=zero_division) + + def macro_precision(y_true, *y_pred, labels=None, sample_weight=None, zero_division='warn', num_samples: int=500, n_jobs: int=-1, use_tqdm=True, **kwargs): """:py:class:`~CompStats.interface.Perf` with :py:func:`~sklearn.metrics.precision_score` (as :py:attr:`score_func`) with the parameteres needed to compute the macro score. The parameters not described can be found in :py:func:`~sklearn.metrics.precision_score` - :param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement. - :type y_true: numpy.ndarray or pandas.DataFrame - :param y_pred: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`y_pred.` - :type y_pred: numpy.ndarray - :param kwargs: Predictions, the algorithms will be identified using the keyword - :type kwargs: numpy.ndarray - :param num_samples: Number of bootstrap samples, default=500. - :type num_samples: int - :param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads. - :type n_jobs: int - :param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True - :type use_tqdm: bool + :param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement. + :type y_true: numpy.ndarray or pandas.DataFrame + :param y_pred: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`y_pred.` + :type y_pred: numpy.ndarray + :param kwargs: Predictions, the algorithms will be identified using the keyword + :type kwargs: numpy.ndarray + :param num_samples: Number of bootstrap samples, default=500. + :type num_samples: int + :param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads. + :type n_jobs: int + :param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True + :type use_tqdm: bool """ return precision_score(y_true, *y_pred, labels=labels, average='macro', sample_weight=sample_weight, zero_division=zero_division, @@ -378,11 +571,28 @@ def macro_precision(y_true, *y_pred, labels=None, use_tqdm=use_tqdm, **kwargs) +macro_precision.measure = _macro_precision_measure + + ######################################################## #################### Regression ######################## ######################################################## +def _explained_variance_score_measure(sample_weight=None, multioutput='uniform_average', + force_finite=True): + """Build the tagged (score-type, BiB=True) measure used by :py:func:`explained_variance_score`""" + + @wraps(metrics.explained_variance_score) + def inner(y, hy): + return metrics.explained_variance_score(y, hy, + sample_weight=sample_weight, + multioutput=multioutput, + force_finite=force_finite) + inner.BiB = True + return inner + + @metrics_docs(hy_name='y_pred', attr_name='score_func') def explained_variance_score(y_true, *y_pred, @@ -395,35 +605,58 @@ def explained_variance_score(y_true, **kwargs): """explained_variance_score""" - @wraps(metrics.explained_variance_score) - def inner(y, hy): - return metrics.explained_variance_score(y, hy, - sample_weight=sample_weight, - multioutput=multioutput, - force_finite=force_finite) - return Perf(y_true, *y_pred, score_func=inner, + return Perf(y_true, *y_pred, + score_func=_explained_variance_score_measure(sample_weight=sample_weight, + multioutput=multioutput, + force_finite=force_finite), num_samples=num_samples, n_jobs=n_jobs, use_tqdm=use_tqdm, **kwargs) +explained_variance_score.measure = _explained_variance_score_measure + + +def _max_error_measure(): + """Build the tagged (error-type, BiB=False) measure used by :py:func:`max_error`""" + + @wraps(metrics.max_error) + def inner(y, hy): + return metrics.max_error(y, hy) + inner.BiB = False + return inner + + @metrics_docs(hy_name='y_pred', attr_name='error_func') -def max_error(y_true, *y_pred, +def max_error(y_true, *y_pred, num_samples: int=500, n_jobs: int=-1, use_tqdm=True, **kwargs): """max_error""" - @wraps(metrics.max_error) - def inner(y, hy): - return metrics.max_error(y, hy) - return Perf(y_true, *y_pred, score_func=None, error_func=inner, + return Perf(y_true, *y_pred, score_func=None, + error_func=_max_error_measure(), num_samples=num_samples, n_jobs=n_jobs, use_tqdm=use_tqdm, **kwargs) +max_error.measure = _max_error_measure + + +def _mean_absolute_error_measure(sample_weight=None, multioutput='uniform_average'): + """Build the tagged (error-type, BiB=False) measure used by :py:func:`mean_absolute_error`""" + + @wraps(metrics.mean_absolute_error) + def inner(y, hy): + return metrics.mean_absolute_error(y, hy, + sample_weight=sample_weight, + multioutput=multioutput) + inner.BiB = False + return inner + + @metrics_docs(hy_name='y_pred', attr_name='error_func') def mean_absolute_error(y_true, *y_pred, @@ -435,18 +668,29 @@ def mean_absolute_error(y_true, **kwargs): """mean_absolute_error""" - @wraps(metrics.mean_absolute_error) - def inner(y, hy): - return metrics.mean_absolute_error(y, hy, - sample_weight=sample_weight, - multioutput=multioutput) - - return Perf(y_true, *y_pred, score_func=None, error_func=inner, + return Perf(y_true, *y_pred, score_func=None, + error_func=_mean_absolute_error_measure(sample_weight=sample_weight, + multioutput=multioutput), num_samples=num_samples, n_jobs=n_jobs, use_tqdm=use_tqdm, **kwargs) +mean_absolute_error.measure = _mean_absolute_error_measure + + +def _mean_squared_error_measure(sample_weight=None, multioutput='uniform_average'): + """Build the tagged (error-type, BiB=False) measure used by :py:func:`mean_squared_error`""" + + @wraps(metrics.mean_squared_error) + def inner(y, hy): + return metrics.mean_squared_error(y, hy, + sample_weight=sample_weight, + multioutput=multioutput) + inner.BiB = False + return inner + + @metrics_docs(hy_name='y_pred', attr_name='error_func') def mean_squared_error(y_true, *y_pred, @@ -458,18 +702,29 @@ def mean_squared_error(y_true, **kwargs): """mean_squared_error""" - @wraps(metrics.mean_squared_error) - def inner(y, hy): - return metrics.mean_squared_error(y, hy, - sample_weight=sample_weight, - multioutput=multioutput) - - return Perf(y_true, *y_pred, score_func=None, error_func=inner, + return Perf(y_true, *y_pred, score_func=None, + error_func=_mean_squared_error_measure(sample_weight=sample_weight, + multioutput=multioutput), num_samples=num_samples, n_jobs=n_jobs, use_tqdm=use_tqdm, **kwargs) +mean_squared_error.measure = _mean_squared_error_measure + + +def _root_mean_squared_error_measure(sample_weight=None, multioutput='uniform_average'): + """Build the tagged (error-type, BiB=False) measure used by :py:func:`root_mean_squared_error`""" + + @wraps(metrics.root_mean_squared_error) + def inner(y, hy): + return metrics.root_mean_squared_error(y, hy, + sample_weight=sample_weight, + multioutput=multioutput) + inner.BiB = False + return inner + + @metrics_docs(hy_name='y_pred', attr_name='error_func') def root_mean_squared_error(y_true, *y_pred, @@ -481,18 +736,29 @@ def root_mean_squared_error(y_true, **kwargs): """root_mean_squared_error""" - @wraps(metrics.root_mean_squared_error) - def inner(y, hy): - return metrics.root_mean_squared_error(y, hy, - sample_weight=sample_weight, - multioutput=multioutput) - - return Perf(y_true, *y_pred, score_func=None, error_func=inner, + return Perf(y_true, *y_pred, score_func=None, + error_func=_root_mean_squared_error_measure(sample_weight=sample_weight, + multioutput=multioutput), num_samples=num_samples, n_jobs=n_jobs, use_tqdm=use_tqdm, **kwargs) +root_mean_squared_error.measure = _root_mean_squared_error_measure + + +def _mean_squared_log_error_measure(sample_weight=None, multioutput='uniform_average'): + """Build the tagged (error-type, BiB=False) measure used by :py:func:`mean_squared_log_error`""" + + @wraps(metrics.mean_squared_log_error) + def inner(y, hy): + return metrics.mean_squared_log_error(y, hy, + sample_weight=sample_weight, + multioutput=multioutput) + inner.BiB = False + return inner + + @metrics_docs(hy_name='y_pred', attr_name='error_func') def mean_squared_log_error(y_true, *y_pred, @@ -504,18 +770,29 @@ def mean_squared_log_error(y_true, **kwargs): """mean_squared_log_error""" - @wraps(metrics.mean_squared_log_error) - def inner(y, hy): - return metrics.mean_squared_log_error(y, hy, - sample_weight=sample_weight, - multioutput=multioutput) - - return Perf(y_true, *y_pred, score_func=None, error_func=inner, + return Perf(y_true, *y_pred, score_func=None, + error_func=_mean_squared_log_error_measure(sample_weight=sample_weight, + multioutput=multioutput), num_samples=num_samples, n_jobs=n_jobs, use_tqdm=use_tqdm, **kwargs) +mean_squared_log_error.measure = _mean_squared_log_error_measure + + +def _root_mean_squared_log_error_measure(sample_weight=None, multioutput='uniform_average'): + """Build the tagged (error-type, BiB=False) measure used by :py:func:`root_mean_squared_log_error`""" + + @wraps(metrics.root_mean_squared_log_error) + def inner(y, hy): + return metrics.root_mean_squared_log_error(y, hy, + sample_weight=sample_weight, + multioutput=multioutput) + inner.BiB = False + return inner + + @metrics_docs(hy_name='y_pred', attr_name='error_func') def root_mean_squared_log_error(y_true, *y_pred, @@ -527,18 +804,29 @@ def root_mean_squared_log_error(y_true, **kwargs): """root_mean_squared_log_error""" - @wraps(metrics.root_mean_squared_log_error) - def inner(y, hy): - return metrics.root_mean_squared_log_error(y, hy, - sample_weight=sample_weight, - multioutput=multioutput) - - return Perf(y_true, *y_pred, score_func=None, error_func=inner, + return Perf(y_true, *y_pred, score_func=None, + error_func=_root_mean_squared_log_error_measure(sample_weight=sample_weight, + multioutput=multioutput), num_samples=num_samples, n_jobs=n_jobs, use_tqdm=use_tqdm, **kwargs) +root_mean_squared_log_error.measure = _root_mean_squared_log_error_measure + + +def _median_absolute_error_measure(sample_weight=None, multioutput='uniform_average'): + """Build the tagged (error-type, BiB=False) measure used by :py:func:`median_absolute_error`""" + + @wraps(metrics.median_absolute_error) + def inner(y, hy): + return metrics.median_absolute_error(y, hy, + sample_weight=sample_weight, + multioutput=multioutput) + inner.BiB = False + return inner + + @metrics_docs(hy_name='y_pred', attr_name='error_func') def median_absolute_error(y_true, *y_pred, @@ -550,18 +838,30 @@ def median_absolute_error(y_true, **kwargs): """median_absolute_error""" - @wraps(metrics.median_absolute_error) - def inner(y, hy): - return metrics.median_absolute_error(y, hy, - sample_weight=sample_weight, - multioutput=multioutput) - - return Perf(y_true, *y_pred, score_func=None, error_func=inner, + return Perf(y_true, *y_pred, score_func=None, + error_func=_median_absolute_error_measure(sample_weight=sample_weight, + multioutput=multioutput), num_samples=num_samples, n_jobs=n_jobs, use_tqdm=use_tqdm, **kwargs) +median_absolute_error.measure = _median_absolute_error_measure + + +def _r2_score_measure(sample_weight=None, multioutput='uniform_average', force_finite=True): + """Build the tagged (score-type, BiB=True) measure used by :py:func:`r2_score`""" + + @wraps(metrics.r2_score) + def inner(y, hy): + return metrics.r2_score(y, hy, + sample_weight=sample_weight, + multioutput=multioutput, + force_finite=force_finite) + inner.BiB = True + return inner + + @metrics_docs(hy_name='y_pred', attr_name='score_func') def r2_score(y_true, *y_pred, @@ -574,19 +874,30 @@ def r2_score(y_true, **kwargs): """r2_score""" - @wraps(metrics.r2_score) - def inner(y, hy): - return metrics.r2_score(y, hy, - sample_weight=sample_weight, - multioutput=multioutput, - force_finite=force_finite) - - return Perf(y_true, *y_pred, score_func=inner, error_func=None, + return Perf(y_true, *y_pred, + score_func=_r2_score_measure(sample_weight=sample_weight, + multioutput=multioutput, + force_finite=force_finite), + error_func=None, num_samples=num_samples, n_jobs=n_jobs, use_tqdm=use_tqdm, **kwargs) +r2_score.measure = _r2_score_measure + + +def _mean_poisson_deviance_measure(sample_weight=None): + """Build the tagged (error-type, BiB=False) measure used by :py:func:`mean_poisson_deviance`""" + + @wraps(metrics.mean_poisson_deviance) + def inner(y, hy): + return metrics.mean_poisson_deviance(y, hy, + sample_weight=sample_weight) + inner.BiB = False + return inner + + @metrics_docs(hy_name='y_pred', attr_name='error_func') def mean_poisson_deviance(y_true, *y_pred, @@ -597,17 +908,27 @@ def mean_poisson_deviance(y_true, **kwargs): """mean_poisson_deviance""" - @wraps(metrics.mean_poisson_deviance) - def inner(y, hy): - return metrics.mean_poisson_deviance(y, hy, - sample_weight=sample_weight) - - return Perf(y_true, *y_pred, score_func=None, error_func=inner, + return Perf(y_true, *y_pred, score_func=None, + error_func=_mean_poisson_deviance_measure(sample_weight=sample_weight), num_samples=num_samples, n_jobs=n_jobs, use_tqdm=use_tqdm, **kwargs) +mean_poisson_deviance.measure = _mean_poisson_deviance_measure + + +def _mean_gamma_deviance_measure(sample_weight=None): + """Build the tagged (error-type, BiB=False) measure used by :py:func:`mean_gamma_deviance`""" + + @wraps(metrics.mean_gamma_deviance) + def inner(y, hy): + return metrics.mean_gamma_deviance(y, hy, + sample_weight=sample_weight) + inner.BiB = False + return inner + + @metrics_docs(hy_name='y_pred', attr_name='error_func') def mean_gamma_deviance(y_true, *y_pred, @@ -618,17 +939,28 @@ def mean_gamma_deviance(y_true, **kwargs): """mean_gamma_deviance""" - @wraps(metrics.mean_gamma_deviance) - def inner(y, hy): - return metrics.mean_gamma_deviance(y, hy, - sample_weight=sample_weight) - - return Perf(y_true, *y_pred, score_func=None, error_func=inner, + return Perf(y_true, *y_pred, score_func=None, + error_func=_mean_gamma_deviance_measure(sample_weight=sample_weight), num_samples=num_samples, n_jobs=n_jobs, use_tqdm=use_tqdm, **kwargs) +mean_gamma_deviance.measure = _mean_gamma_deviance_measure + + +def _mean_absolute_percentage_error_measure(sample_weight=None, multioutput='uniform_average'): + """Build the tagged (error-type, BiB=False) measure used by :py:func:`mean_absolute_percentage_error`""" + + @wraps(metrics.mean_absolute_percentage_error) + def inner(y, hy): + return metrics.mean_absolute_percentage_error(y, hy, + sample_weight=sample_weight, + multioutput=multioutput) + inner.BiB = False + return inner + + @metrics_docs(hy_name='y_pred', attr_name='error_func') def mean_absolute_percentage_error(y_true, *y_pred, @@ -640,18 +972,29 @@ def mean_absolute_percentage_error(y_true, **kwargs): """mean_absolute_percentage_error""" - @wraps(metrics.mean_absolute_percentage_error) - def inner(y, hy): - return metrics.mean_absolute_percentage_error(y, hy, - sample_weight=sample_weight, - multioutput=multioutput) - - return Perf(y_true, *y_pred, score_func=None, error_func=inner, + return Perf(y_true, *y_pred, score_func=None, + error_func=_mean_absolute_percentage_error_measure(sample_weight=sample_weight, + multioutput=multioutput), num_samples=num_samples, n_jobs=n_jobs, use_tqdm=use_tqdm, **kwargs) +mean_absolute_percentage_error.measure = _mean_absolute_percentage_error_measure + + +def _d2_absolute_error_score_measure(sample_weight=None, multioutput='uniform_average'): + """Build the tagged (score-type, BiB=True) measure used by :py:func:`d2_absolute_error_score`""" + + @wraps(metrics.d2_absolute_error_score) + def inner(y, hy): + return metrics.d2_absolute_error_score(y, hy, + sample_weight=sample_weight, + multioutput=multioutput) + inner.BiB = True + return inner + + def d2_absolute_error_score(y_true, *y_pred, sample_weight=None, @@ -662,18 +1005,30 @@ def d2_absolute_error_score(y_true, **kwargs): """d2_absolute_error_score""" - @wraps(metrics.d2_absolute_error_score) - def inner(y, hy): - return metrics.d2_absolute_error_score(y, hy, - sample_weight=sample_weight, - multioutput=multioutput) - - return Perf(y_true, *y_pred, score_func=inner, error_func=None, + return Perf(y_true, *y_pred, + score_func=_d2_absolute_error_score_measure(sample_weight=sample_weight, + multioutput=multioutput), + error_func=None, num_samples=num_samples, n_jobs=n_jobs, use_tqdm=use_tqdm, **kwargs) +d2_absolute_error_score.measure = _d2_absolute_error_score_measure + + +def _pearsonr_measure(alternative='two-sided', method=None): + """Build the tagged (score-type, BiB=True) measure used by :py:func:`pearsonr`""" + + @wraps(stats.pearsonr) + def inner(y, hy): + return stats.pearsonr(y, hy, + alternative=alternative, + method=method).statistic + inner.BiB = True + return inner + + def pearsonr(y_true, *y_pred, alternative='two-sided', method=None, num_samples: int=500, @@ -682,27 +1037,26 @@ def pearsonr(y_true, *y_pred, **kwargs): """:py:class:`~CompStats.interface.Perf` with :py:func:`~scipy.stats.pearsonr` as :py:attr:`score_func.` - :param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement. - :type y_true: numpy.ndarray or pandas.DataFrame - :param y_pred: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`y_pred.` - :type y_pred: numpy.ndarray - :param kwargs: Predictions, the algorithms will be identified using the keyword - :type kwargs: numpy.ndarray - :param num_samples: Number of bootstrap samples, default=500. - :type num_samples: int - :param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads. - :type n_jobs: int - :param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True - :type use_tqdm: bool + :param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement. + :type y_true: numpy.ndarray or pandas.DataFrame + :param y_pred: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`y_pred.` + :type y_pred: numpy.ndarray + :param kwargs: Predictions, the algorithms will be identified using the keyword + :type kwargs: numpy.ndarray + :param num_samples: Number of bootstrap samples, default=500. + :type num_samples: int + :param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads. + :type n_jobs: int + :param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True + :type use_tqdm: bool """ - @wraps(stats.pearsonr) - def inner(y, hy): - return stats.pearsonr(y, hy, - alternative=alternative, - method=method).statistic - - return Perf(y_true, *y_pred, score_func=inner, error_func=None, + return Perf(y_true, *y_pred, + score_func=_pearsonr_measure(alternative=alternative, method=method), + error_func=None, num_samples=num_samples, n_jobs=n_jobs, use_tqdm=use_tqdm, **kwargs) + + +pearsonr.measure = _pearsonr_measure diff --git a/CompStats/tests/test_interface.py b/CompStats/tests/test_interface.py index ed39699..5c20bf0 100644 --- a/CompStats/tests/test_interface.py +++ b/CompStats/tests/test_interface.py @@ -319,6 +319,98 @@ def test_Perf_input_dataframe(): assert 'INGEOTEC' in perf.statistic +def test_Perf_multi_measure_score_only(): + """Test Perf combining two score-type measures via metrics.py's .measure() factories""" + from CompStats.interface import Perf + from CompStats.metrics import f1_score, recall_score + + X, y = load_digits(return_X_y=True) + _ = train_test_split(X, y, test_size=0.3, random_state=0) + X_train, X_val, y_train, y_val = _ + ens = RandomForestClassifier(random_state=0).fit(X_train, y_train) + nb = GaussianNB().fit(X_train, y_train) + perf = Perf(y_val, ens.predict(X_val), nb=nb.predict(X_val), + score_func=[f1_score.measure(average='macro'), + recall_score.measure(average='macro')], + num_samples=20) + assert perf.measure_names == ['f1_score', 'recall_score'] + assert isinstance(perf.statistic['alg-1'], np.ndarray) + assert perf.statistic['alg-1'].shape == (2,) + assert np.all(perf.statistic_samples.BiB == np.array([True, True])) + df = perf.dataframe() + assert set(df['Performance']) == {'f1_score', 'recall_score'} + + +def test_Perf_multi_measure_mixed_bib(): + """Test Perf/Difference with mixed score-type and error-type measures + + Uses constant prediction arrays so the bootstrap statistic has zero + variance, making ``difference().p_value()`` exactly predictable; this + isolates the per-column BiB sign logic (interface.py's difference()/ + p_value()/best) from sampling noise. + """ + from CompStats.interface import Perf + + def score_stat(y, hy): + return hy.mean() + + def error_stat(y, hy): + return hy.mean() + + y_true = np.arange(10) + hyA = np.full(10, 5.0) + hyB = np.full(10, 2.0) + perf = Perf(y_true, A=hyA, B=hyB, + score_func=score_stat, error_func=error_stat, + num_samples=5) + assert perf.measure_names == ['score_stat', 'error_stat'] + assert np.all(perf.statistic_samples.BiB == np.array([True, False])) + # A has the higher value (wins the score-type column), + # B has the lower value (wins the error-type column) + assert list(perf.best) == ['A', 'B'] + diff = perf.difference() + p_values = diff.p_value() + assert np.allclose(p_values['A'], [1.0, 0.0]) + assert np.allclose(p_values['B'], [0.0, 1.0]) + + +def test_Perf_measure_tag_overrides_list_position(): + """A callable's own .BiB (set by a .measure() factory) wins over the + default direction implied by score_func/error_func placement""" + from CompStats.interface import Perf + from CompStats.metrics import f1_score + + y_true = np.array([0, 0, 0, 0, 1, 1, 1, 1, 0, 1]) + hy = np.array([0, 0, 0, 0, 1, 1, 1, 1, 0, 1]) + tagged = f1_score.measure(average='macro') + assert tagged.BiB is True + perf = Perf(y_true, alg=hy, score_func=None, + error_func=tagged, num_samples=5) + assert bool(perf.statistic_samples.BiB) is True + + +def test_Perf_multi_measure_clone(): + """Test that cloning a multi-measure Perf preserves measures and samples""" + from sklearn.base import clone + from CompStats.interface import Perf + from CompStats.metrics import f1_score, recall_score + + X, y = load_iris(return_X_y=True) + _ = train_test_split(X, y, test_size=0.3, random_state=0) + X_train, X_val, y_train, y_val = _ + ens = RandomForestClassifier(random_state=0).fit(X_train, y_train) + nb = GaussianNB().fit(X_train, y_train) + perf = Perf(y_val, forest=ens.predict(X_val), nb=nb.predict(X_val), + score_func=[f1_score.measure(average='macro'), + recall_score.measure(average='macro')], + num_samples=20) + samples = perf.statistic_samples._samples + perf2 = clone(perf) + assert perf2.measure_names == ['f1_score', 'recall_score'] + assert np.all(samples == perf2.statistic_samples._samples) + assert np.allclose(perf.statistic['forest'], perf2.statistic['forest']) + + def test_Perf_call(): """Test Perf call""" from CompStats.interface import Perf diff --git a/CompStats/tests/test_metrics.py b/CompStats/tests/test_metrics.py index 45b9268..2a3b5be 100644 --- a/CompStats/tests/test_metrics.py +++ b/CompStats/tests/test_metrics.py @@ -494,3 +494,56 @@ def test_pearsonr(): num_samples=50) _ = stats.pearsonr(y_val, hy) assert _.statistic == perf.statistic + + +def test_measure_factories_tag_bib(): + """Every wrapper's .measure() factory returns a callable tagged with the + same direction (BiB) implied by its wrapper's score_func/error_func""" + from CompStats import metrics as compstats_metrics + + score_type = ['accuracy_score', 'balanced_accuracy_score', + 'top_k_accuracy_score', 'average_precision_score', + 'f1_score', 'precision_score', 'recall_score', + 'jaccard_score', 'roc_auc_score', 'd2_log_loss_score', + 'macro_f1', 'macro_recall', 'macro_precision', + 'explained_variance_score', 'r2_score', + 'd2_absolute_error_score', 'pearsonr'] + error_type = ['brier_score_loss', 'log_loss', 'max_error', + 'mean_absolute_error', 'mean_squared_error', + 'root_mean_squared_error', 'mean_squared_log_error', + 'root_mean_squared_log_error', 'median_absolute_error', + 'mean_poisson_deviance', 'mean_gamma_deviance', + 'mean_absolute_percentage_error'] + for name in score_type: + func = getattr(compstats_metrics, name) + assert hasattr(func, 'measure'), f'{name} is missing .measure' + assert func.measure().BiB is True, f'{name}.measure().BiB should be True' + for name in error_type: + func = getattr(compstats_metrics, name) + assert hasattr(func, 'measure'), f'{name} is missing .measure' + assert func.measure().BiB is False, f'{name}.measure().BiB should be False' + + +def test_measure_compose_multi_metric_perf(): + """.measure() factories compose into a single, multi-measure Perf""" + from CompStats.interface import Perf + from CompStats.metrics import f1_score, recall_score, mean_absolute_error + + X, y = load_iris(return_X_y=True) + _ = train_test_split(X, y, test_size=0.3, random_state=0) + X_train, X_val, y_train, y_val = _ + ens = RandomForestClassifier(random_state=0).fit(X_train, y_train) + nb = GaussianNB().fit(X_train, y_train) + perf = Perf(y_val, ens.predict(X_val), nb=nb.predict(X_val), + score_func=[f1_score.measure(average='macro'), + recall_score.measure(average='macro')], + num_samples=20) + assert perf.measure_names == ['f1_score', 'recall_score'] + assert perf.statistic['alg-1'].shape == (2,) + + # error-type measure composed together with a score-type one + perf2 = Perf(y_val, ens.predict(X_val), nb=nb.predict(X_val), + score_func=f1_score.measure(average='macro'), + error_func=mean_absolute_error.measure(), + num_samples=20) + assert list(perf2.statistic_samples.BiB) == [True, False]