diff --git a/docs/api/calib/pyhealth.calib.predictionset.rst b/docs/api/calib/pyhealth.calib.predictionset.rst index 740b1c87c..e8dde1f6b 100644 --- a/docs/api/calib/pyhealth.calib.predictionset.rst +++ b/docs/api/calib/pyhealth.calib.predictionset.rst @@ -57,6 +57,17 @@ SCRIB (Set-classifier with Class-specific Risk Bounds) FavMac (Fast Value-Maximizing Prediction Sets) ----------------------------------------------- +.. note:: + + FavMac's threshold formulas are verified directly against the paper's + Eq. 18 (expected cost control) and Eq. 21 (violation control, + Appendix B.3 Algorithm 5 / Theorem 4.6). Like standard split-conformal + prediction, both guarantees assume exchangeability between the + calibration data and the point being predicted -- FavMac does not + correct for covariate shift; see + :class:`~pyhealth.calib.predictionset.CovariateLabel` for a method + that does. + .. autoclass:: pyhealth.calib.predictionset.FavMac :members: :undoc-members: diff --git a/examples/conformal_eeg/tuev_conventional_conformal.py b/examples/conformal_eeg/tuev_conventional_conformal.py index e5542c7d4..74d9ee785 100644 --- a/examples/conformal_eeg/tuev_conventional_conformal.py +++ b/examples/conformal_eeg/tuev_conventional_conformal.py @@ -17,6 +17,10 @@ python examples/conformal_eeg/tuev_conventional_conformal.py \\ --root downloads/tuev/v2.0.1/edf --model tfm --n-seeds 5 --seed 42 --alpha 0.1 \\ --log-file tuev_conventional_tfm_alpha0.1_5seeds.log + +Note: for multi-label problems where you want to control a cost (e.g. +false positives) while maximizing value, rather than marginal miscoverage, +see pyhealth.calib.predictionset.FavMac instead of LABEL. """ from __future__ import annotations diff --git a/pyhealth/calib/predictionset/favmac/__init__.py b/pyhealth/calib/predictionset/favmac/__init__.py index bd57b644c..b5baa53c8 100644 --- a/pyhealth/calib/predictionset/favmac/__init__.py +++ b/pyhealth/calib/predictionset/favmac/__init__.py @@ -128,6 +128,17 @@ class FavMac(SetPredictor): "Conformal prediction sets with limited false positives." ICML 2022. + Note: + Like standard split-conformal prediction, both guarantees above + (Theorem 4.6 of [1] for the violation-control case) assume + exchangeability between the calibration data and the point being + predicted. FavMac does not correct for covariate shift; expect the + target to be violated under real distribution shift (common in + clinical data) even with a fully correct implementation, since + that is an assumption violation, not an implementation bug. See + :class:`~pyhealth.calib.predictionset.CovariateLabel` for a + method that explicitly corrects for a known shift. + Args: model (BaseModel): A trained model. value_weights (Union[float, np.ndarray]): @@ -191,6 +202,10 @@ def __init__( self.device = model.device self.debug = debug + if target_cost <= 0: + raise ValueError(f"target_cost must be positive, got {target_cost!r}") + if delta is not None and not (0.0 < delta < 1.0): + raise ValueError(f"delta must be in (0, 1), got {delta!r}") self._cost_weights = cost_weights self._value_weights = value_weights @@ -211,6 +226,12 @@ def calibrate(self, cal_dataset): C_max = self._cost_weights.sum() else: C_max = _cal_data["logit"].shape[1] * self._cost_weights + if self.target_cost > C_max: + raise ValueError( + f"target_cost ({self.target_cost}) must be <= the maximum " + f"possible cost ({C_max}, from cost_weights); the paper's " + "guarantee (Eq. 18/21) requires target_cost in (0, C_max]." + ) self._favmac = FavMac_GreedyRatio( cost_fn=AdditiveSetFunction(self._cost_weights / C_max, mode='cost'), util_fn=AdditiveSetFunction(self._value_weights, mode='util'), diff --git a/pyhealth/calib/predictionset/favmac/core.py b/pyhealth/calib/predictionset/favmac/core.py index 7e9b7d354..3ad769268 100644 --- a/pyhealth/calib/predictionset/favmac/core.py +++ b/pyhealth/calib/predictionset/favmac/core.py @@ -11,6 +11,41 @@ class FavMac: + """Online value-maximizing prediction sets with conformal cost control. + + This is the internal calibration/inference engine backing the + user-facing :class:`pyhealth.calib.predictionset.FavMac`. It is + abstract: ``_greedy_sequence`` must be supplied by a subclass (see + :class:`FavMac_GreedyRatio`, the concrete class the public wrapper + actually uses). Costs, values, and their proxies must be normalized + so cost lies in ``[0, C_max]`` before being passed in here. + + Paper: + Lin, Zhen, Shubhendu Trivedi, Cao Xiao, and Jimeng Sun. "Fast + Online Value-Maximizing Prediction Sets with Conformal Cost + Control." ICML 2023. + + Examples: + >>> import numpy as np + >>> from pyhealth.calib.predictionset.favmac import AdditiveSetFunction + >>> from pyhealth.calib.predictionset.favmac.core import FavMac_GreedyRatio + >>> K = 3 + >>> C_max = float(K) + >>> cost_fn = AdditiveSetFunction(np.ones(K) / C_max, mode="cost") + >>> util_fn = AdditiveSetFunction(np.ones(K), mode="util") + >>> proxy_fn = AdditiveSetFunction(np.ones(K) / C_max, mode="proxy") + >>> fm = FavMac_GreedyRatio( + ... cost_fn, util_fn, proxy_fn, target_cost=1.0 / C_max, C_max=1.0) + >>> rng = np.random.default_rng(0) + >>> for _ in range(30): + ... logit = rng.normal(size=K) + ... y = (rng.uniform(size=K) < 0.4).astype(int) + ... _ = fm.update(logit, y) + >>> predset, _ = fm(np.array([1.0, -0.5, 0.2]), update=False) + >>> predset + array([1, 0, 0]) + """ + def __init__(self, cost_fn, util_fn, proxy_fn, target_cost, delta=None, C_max=1.) -> None: self.target_cost = target_cost self.delta = delta @@ -53,6 +88,13 @@ def _add_sample(self, predset, extra_info): def _query_threshold(self): n = len(self._queue) if self.delta is None: + # Matches the paper's Eq. 18 (expected cost control): + # T_c = sup{t : (C_max + sum_i C+(t, Z_i)) / (n+1) <= target_cost} + # which rearranges to sum_i C+(t, Z_i) <= target_cost*(n+1) - C_max, + # i.e. exactly this cutoff. _add_sample() decomposes each + # example's (proxy, cost) step function into weight increments + # at each proxy score, so the tree's cumulative weight at a + # given threshold t equals sum_i C+(t, Z_i) directly. cutoff = self.target_cost * (n+1) - self.C_max return self.quantiletree.query_cumu_weight(cutoff, prev=False) else: