Fix sign toggle in removal-based interpretability metrics

docs/api/metrics/pyhealth.metrics.interpretability.rst

@@ -24,6 +24,11 @@ Functional API
 Removal-Based Metrics
 ---------------------
 
+For binary classifiers, a sample filter can mark class-0 predictions as
+``SampleClass.NEGATIVE``. Removal-based metrics then score probability changes
+from the class-0 perspective. Each percentage is evaluated independently, so a
+percentage's score does not depend on the other requested percentages.
+
 Base Class
 ^^^^^^^^^^
 

examples/interpretability/custom_sample_filter.py

@@ -4,7 +4,7 @@
 This example demonstrates:
 1. Loading a pre-trained StageNet model with processors and MIMIC-IV dataset
 2. Computing attributions with various interpretability methods
-3. Evaluating attribution faithfulness with Comprehensiveness & Sufficiency for each method
+3. Evaluating class-0 and class-1 predictions with a custom sample filter
 4. Presenting results in a summary table
 """
 

pyhealth/metrics/interpretability/base.py

@@ -24,6 +24,14 @@ class RemovalBasedMetric(ABC):
     This class provides common functionality for computing faithfulness metrics
     by removing or retaining features based on their importance scores.
 
+    Examples:
+        >>> from pyhealth.metrics.interpretability import (
+        ...     ComprehensivenessMetric,
+        ...     RemovalBasedMetric,
+        ... )
+        >>> issubclass(ComprehensivenessMetric, RemovalBasedMetric)
+        True
+
     Args:
         model: PyHealth BaseModel that accepts **kwargs and returns dict with
             'y_prob' or 'logit'.
@@ -373,8 +381,8 @@ def compute(
 
             If return_per_percentage=True:
                 Dict[float, torch.Tensor]: Maps percentage -> scores
-                (batch_size,). For binary classifiers, negative class
-                samples have value 0.
+                (batch_size,). For binary classifiers, negative-class samples
+                are scored from the class-0 perspective.
 
         Note:
             For binary classifiers, all samples are evaluated
@@ -453,10 +461,10 @@ def compute(
             )
 
             # Compute probability drop
-            original_class_probs = y_probs
+            original_class_probs = y_probs.clone()
             original_class_probs[neg_mask] = -original_class_probs[neg_mask]
 
-            ablated_class_probs = ablated_probs
+            ablated_class_probs = ablated_probs.clone()
             ablated_class_probs[neg_mask] = -ablated_class_probs[neg_mask]
 
             prob_drop = torch.zeros(batch_size, device=y_probs.device)
@@ -493,9 +501,9 @@ def compute(
 
                 # Check for unexpected negative values
                 evaluated_drops = prob_drop[val_mask]
-                neg_mask = evaluated_drops < 0
-                if neg_mask.any():
-                    neg_count = neg_mask.sum().item()
+                negative_drop_mask = evaluated_drops < 0
+                if negative_drop_mask.any():
+                    neg_count = negative_drop_mask.sum().item()
                     print(f"\n⚠ WARNING: {neg_count} negative detected!")
                     print("  Negative values mean ablation INCREASED " "confidence,")
                     print("  which suggests:")

tests/core/test_interp_metrics.py

@@ -16,6 +16,7 @@
 from pyhealth.metrics.interpretability import (
     ComprehensivenessMetric,
     Evaluator,
+    SampleClass,
     SufficiencyMetric,
     threshold_sample_filter,
 )
@@ -449,6 +450,64 @@ def test_percentage_sensitivity(self):
         self.assertTrue(torch.isfinite(torch.tensor(score_10)))
         self.assertTrue(torch.isfinite(torch.tensor(score_50)))
 
+    def test_negative_class_scores_independent_of_percentage_order(self):
+        """Test that a negative-class sample's score at a percentage is order-independent."""
+        attributions = self._create_attributions(self.batch)
+
+        def negative_filter(y_probs, classifier_type):
+            return torch.full(
+                (y_probs.shape[0],),
+                SampleClass.NEGATIVE,
+                dtype=torch.long,
+                device=y_probs.device,
+            )
+
+        def score_at_20(percentages):
+            comp = ComprehensivenessMetric(
+                self.model,
+                percentages=percentages,
+                ablation_strategy="zero",
+                sample_filter=negative_filter,
+            )
+            detailed = comp.compute(
+                self.batch, attributions, return_per_percentage=True
+            )
+            return detailed[20]
+
+        torch.testing.assert_close(score_at_20([20]), score_at_20([10, 20]))
+
+    def test_debug_output_does_not_change_negative_class_scores(self):
+        """Test that debug output does not change negative-class scores."""
+        attributions = self._create_attributions(self.batch)
+
+        def negative_filter(y_probs, classifier_type):
+            return torch.full(
+                (y_probs.shape[0],),
+                SampleClass.NEGATIVE,
+                dtype=torch.long,
+                device=y_probs.device,
+            )
+
+        def compute_scores(debug):
+            comp = ComprehensivenessMetric(
+                self.model,
+                percentages=[10, 20, 50],
+                ablation_strategy="zero",
+                sample_filter=negative_filter,
+            )
+            return comp.compute(
+                self.batch,
+                attributions,
+                return_per_percentage=True,
+                debug=debug,
+            )
+
+        scores = compute_scores(debug=False)
+        debug_scores = compute_scores(debug=True)
+
+        for percentage in [10, 20, 50]:
+            torch.testing.assert_close(scores[percentage], debug_scores[percentage])
+
     def test_attribution_shape_mismatch(self):
         """Test that mismatched attribution shapes are handled gracefully."""
         # Skip this test - shape mismatches may not always raise errors