diff --git a/docs/api/calib/pyhealth.calib.predictionset.rst b/docs/api/calib/pyhealth.calib.predictionset.rst index 740b1c87c..8e4d43e6f 100644 --- a/docs/api/calib/pyhealth.calib.predictionset.rst +++ b/docs/api/calib/pyhealth.calib.predictionset.rst @@ -49,6 +49,17 @@ LABEL (Least Ambiguous Set-valued Classifier) SCRIB (Set-classifier with Class-specific Risk Bounds) ------------------------------------------------------- +.. note:: + + SCRIB's Chance-Ambiguity and per-class risk are both defined by the + paper relative to singleton (``|H(X)|=1``) and multi-label + (``|H(X)|>1``) prediction sets; the paper does not specify how an + *empty* set (``|H(X)|=0``) should be scored. This implementation + treats any non-singleton set -- empty or multi-label alike -- as + "not sure," consistently in both the optimized loss and the + ``rejection_rate``/``error_ps`` metrics used to evaluate it. See the + class docstring's ``Note`` for why. + .. autoclass:: pyhealth.calib.predictionset.SCRIB :members: :undoc-members: diff --git a/examples/cxr/covid19cxr_conformal.py b/examples/cxr/covid19cxr_conformal.py index 47e2eebbc..783b80f49 100644 --- a/examples/cxr/covid19cxr_conformal.py +++ b/examples/cxr/covid19cxr_conformal.py @@ -6,6 +6,13 @@ 2. Conventional conformal prediction using LABEL 3. Covariate shift adaptive conformal prediction using CovariateLabel 4. Comparison of coverage and efficiency between the two methods + +Note: LABEL's (and SCRIB's) coverage guarantee assumes exchangeability +between the calibration and test distributions; it does not correct for +covariate shift. CovariateLabel is the method here designed to handle +that case explicitly -- under real distribution shift, expect LABEL to +under-cover even with a fully correct implementation, since that is an +assumption violation, not a bug. """ import numpy as np diff --git a/pyhealth/calib/predictionset/scrib/__init__.py b/pyhealth/calib/predictionset/scrib/__init__.py index 3406bef3b..e86a5f446 100644 --- a/pyhealth/calib/predictionset/scrib/__init__.py +++ b/pyhealth/calib/predictionset/scrib/__init__.py @@ -189,7 +189,24 @@ class SCRIB(SetPredictor): Lin, Zhen, Lucas Glass, M. Brandon Westover, Cao Xiao, and Jimeng Sun. "SCRIB: Set-classifier with Class-specific Risk Bounds for Blackbox Models." - AAAI 2022. + AAAI 2022. https://arxiv.org/abs/2103.03945 + + Note: + The paper defines Chance-Ambiguity as :math:`\\hat{P}\\{|H(X)|>1\\}` + and risk as :math:`\\hat{r}_k(H) := \\hat{P}_k\\{k \\notin H(X) \\mid + |H(X)|=1\\}` -- both conditioned specifically on singleton (exactly + one label) or multi-label sets, and the paper does not discuss how + an *empty* set (:math:`|H(X)|=0`) should be scored. This + implementation (and the ``rejection_rate``/``error_ps`` metrics + used to evaluate it) instead treats any non-singleton set -- + empty or multi-label alike -- as "not sure"/rejected, consistent + throughout both the optimized loss and the reported metrics. This + is a deliberate choice, not an oversight: under the paper's + literal definition, a degenerate classifier that always predicts + an empty set would show zero ambiguity loss (0 is not >1) and zero + risk loss (no singleton predictions to be wrong on), making it + appear loss-free despite being useless; scoring empty sets as + "not sure" closes that gap. Args: model (BaseModel): A trained model.