Update FavMac

docs/api/calib/pyhealth.calib.predictionset.rst

@@ -57,6 +57,17 @@ SCRIB (Set-classifier with Class-specific Risk Bounds)
 FavMac (Fast Value-Maximizing Prediction Sets)
 -----------------------------------------------
 
+.. note::
+
+   FavMac's threshold formulas are verified directly against the paper's
+   Eq. 18 (expected cost control) and Eq. 21 (violation control,
+   Appendix B.3 Algorithm 5 / Theorem 4.6). Like standard split-conformal
+   prediction, both guarantees assume exchangeability between the
+   calibration data and the point being predicted -- FavMac does not
+   correct for covariate shift; see
+   :class:`~pyhealth.calib.predictionset.CovariateLabel` for a method
+   that does.
+
 .. autoclass:: pyhealth.calib.predictionset.FavMac
    :members:
    :undoc-members:

examples/conformal_eeg/tuev_conventional_conformal.py

@@ -17,6 +17,10 @@
   python examples/conformal_eeg/tuev_conventional_conformal.py \\
       --root downloads/tuev/v2.0.1/edf --model tfm --n-seeds 5 --seed 42 --alpha 0.1 \\
       --log-file tuev_conventional_tfm_alpha0.1_5seeds.log
+
+Note: for multi-label problems where you want to control a cost (e.g.
+false positives) while maximizing value, rather than marginal miscoverage,
+see pyhealth.calib.predictionset.FavMac instead of LABEL.
 """
 
 from __future__ import annotations

pyhealth/calib/predictionset/favmac/__init__.py

@@ -128,6 +128,17 @@ class FavMac(SetPredictor):
         "Conformal prediction sets with limited false positives."
         ICML 2022.
 
+    Note:
+        Like standard split-conformal prediction, both guarantees above
+        (Theorem 4.6 of [1] for the violation-control case) assume
+        exchangeability between the calibration data and the point being
+        predicted. FavMac does not correct for covariate shift; expect the
+        target to be violated under real distribution shift (common in
+        clinical data) even with a fully correct implementation, since
+        that is an assumption violation, not an implementation bug. See
+        :class:`~pyhealth.calib.predictionset.CovariateLabel` for a
+        method that explicitly corrects for a known shift.
+
     Args:
         model (BaseModel): A trained model.
         value_weights (Union[float, np.ndarray]):
@@ -191,6 +202,10 @@ def __init__(
         self.device = model.device
         self.debug = debug
 
+        if target_cost <= 0:
+            raise ValueError(f"target_cost must be positive, got {target_cost!r}")
+        if delta is not None and not (0.0 < delta < 1.0):
+            raise ValueError(f"delta must be in (0, 1), got {delta!r}")
 
         self._cost_weights = cost_weights
         self._value_weights = value_weights
@@ -211,6 +226,12 @@ def calibrate(self, cal_dataset):
             C_max = self._cost_weights.sum()
         else:
             C_max = _cal_data["logit"].shape[1] * self._cost_weights
+        if self.target_cost > C_max:
+            raise ValueError(
+                f"target_cost ({self.target_cost}) must be <= the maximum "
+                f"possible cost ({C_max}, from cost_weights); the paper's "
+                "guarantee (Eq. 18/21) requires target_cost in (0, C_max]."
+            )
         self._favmac = FavMac_GreedyRatio(
             cost_fn=AdditiveSetFunction(self._cost_weights / C_max, mode='cost'),
             util_fn=AdditiveSetFunction(self._value_weights, mode='util'),

pyhealth/calib/predictionset/favmac/core.py

@@ -11,6 +11,41 @@
 
 
 class FavMac:
+    """Online value-maximizing prediction sets with conformal cost control.
+
+    This is the internal calibration/inference engine backing the
+    user-facing :class:`pyhealth.calib.predictionset.FavMac`. It is
+    abstract: ``_greedy_sequence`` must be supplied by a subclass (see
+    :class:`FavMac_GreedyRatio`, the concrete class the public wrapper
+    actually uses). Costs, values, and their proxies must be normalized
+    so cost lies in ``[0, C_max]`` before being passed in here.
+
+    Paper:
+        Lin, Zhen, Shubhendu Trivedi, Cao Xiao, and Jimeng Sun. "Fast
+        Online Value-Maximizing Prediction Sets with Conformal Cost
+        Control." ICML 2023.
+
+    Examples:
+        >>> import numpy as np
+        >>> from pyhealth.calib.predictionset.favmac import AdditiveSetFunction
+        >>> from pyhealth.calib.predictionset.favmac.core import FavMac_GreedyRatio
+        >>> K = 3
+        >>> C_max = float(K)
+        >>> cost_fn = AdditiveSetFunction(np.ones(K) / C_max, mode="cost")
+        >>> util_fn = AdditiveSetFunction(np.ones(K), mode="util")
+        >>> proxy_fn = AdditiveSetFunction(np.ones(K) / C_max, mode="proxy")
+        >>> fm = FavMac_GreedyRatio(
+        ...     cost_fn, util_fn, proxy_fn, target_cost=1.0 / C_max, C_max=1.0)
+        >>> rng = np.random.default_rng(0)
+        >>> for _ in range(30):
+        ...     logit = rng.normal(size=K)
+        ...     y = (rng.uniform(size=K) < 0.4).astype(int)
+        ...     _ = fm.update(logit, y)
+        >>> predset, _ = fm(np.array([1.0, -0.5, 0.2]), update=False)
+        >>> predset
+        array([1, 0, 0])
+    """
+
     def __init__(self, cost_fn, util_fn, proxy_fn, target_cost, delta=None, C_max=1.) -> None:
         self.target_cost = target_cost
         self.delta = delta
@@ -53,6 +88,13 @@ def _add_sample(self, predset, extra_info):
     def _query_threshold(self):
         n = len(self._queue)
         if self.delta is None:
+            # Matches the paper's Eq. 18 (expected cost control):
+            #   T_c = sup{t : (C_max + sum_i C+(t, Z_i)) / (n+1) <= target_cost}
+            # which rearranges to sum_i C+(t, Z_i) <= target_cost*(n+1) - C_max,
+            # i.e. exactly this cutoff. _add_sample() decomposes each
+            # example's (proxy, cost) step function into weight increments
+            # at each proxy score, so the tree's cumulative weight at a
+            # given threshold t equals sum_i C+(t, Z_i) directly.
             cutoff = self.target_cost * (n+1) - self.C_max
             return self.quantiletree.query_cumu_weight(cutoff, prev=False)
         else: