Framework improvements to generalize use of external model harness

docs/configurations.md

@@ -159,13 +159,15 @@ KSWIN:
 kswin_alpha = 0.005
 kswin_window_size = 100
 kswin_stat_size = 30
+kswin_seed = 1337
 ```
 
 | Option | Type | Description |
 |----------|------|-------------|
 | `kswin_alpha` | float | KSWIN significance level. |
 | `kswin_window_size` | int | KSWIN reference window size. |
 | `kswin_stat_size` | int | KSWIN recent sample window size. |
+| `kswin_seed` | int | Seed for KSWIN's reference window sampling. Optional; leave unset for the historical non-reproducible behaviour. |
 
 Page-Hinkley:
 

src/apeiron/config/configuration.py

@@ -159,6 +159,7 @@ class DriftDetectionCfg:
     kswin_alpha: float = 0.005
     kswin_window_size: int = 100
     kswin_stat_size: int = 30
+    kswin_seed: int | None = None  # None = unseeded, as before
 
     # PageHinkley hyperparameters
     ph_min_instances: int = 30

src/apeiron/drift_detection/__init__.py

@@ -25,6 +25,7 @@
     KSWINDetector,
     PageHinkleyDetector,
 )
+
 from apeiron.drift_detection.detectors.model_performance_detector import (
     ModelPerformanceDetector,
     EnsembleDetector,

src/apeiron/drift_detection/detectors/model_performance_detector.py

@@ -10,8 +10,6 @@
 import numpy as np
 import pandas as pd
 from typing import Optional, List
-from evidently import Report
-from evidently.presets import DataDriftPreset
 from apeiron.drift_detection.detectors.base import (
     BaseDriftDetector,
     DriftSignal,
@@ -210,7 +208,12 @@ def update(
         if self.reference_targets is not None:
             reference_data["target"] = self.reference_targets
 
-        # Run drift detection
+        # Imported at the call site: this is the only place in the package that
+        # needs evidently, and at module scope `import apeiron` fails wherever it
+        # is absent, taking the statistical detectors down with it.
+        from evidently import Report
+        from evidently.presets import DataDriftPreset
+
         report = Report(metrics=[DataDriftPreset()])
         snapshot = report.run(reference_data=reference_data, current_data=current_data)
         result_dict = snapshot.dict()

src/apeiron/drift_detection/detectors/statistical_detectors.py

@@ -127,6 +127,7 @@ def __init__(
         minor_threshold: float = 0.3,
         moderate_threshold: float = 0.6,
         name: str = "KSWIN",
+        seed: int | None = None,
     ):
         """
         Initialize KSWIN detector.
@@ -138,15 +139,19 @@ def __init__(
             minor_threshold: Drift score threshold for continual learning
             moderate_threshold: Drift score threshold for fine-tuning
             name: Detector name
+            seed: Seed for KSWIN's reference-window sampling. KSWIN draws its
+                reference window at random, so runs are not reproducible unless
+                this is set.
         """
         super().__init__(name)
         self.alpha = alpha
         self.window_size = window_size
         self.stat_size = stat_size
         self.minor_threshold = minor_threshold
         self.moderate_threshold = moderate_threshold
+        self.seed = seed
         self.detector = river_drift.KSWIN(
-            alpha=alpha, window_size=window_size, stat_size=stat_size
+            alpha=alpha, window_size=window_size, stat_size=stat_size, seed=seed
         )
         self._drift_history: list[int] = []
         self._is_initialized = True
@@ -197,7 +202,10 @@ def update(self, value: float, **kwargs) -> DriftSignal:
     def reset(self) -> None:
         """Reset detector to initial state."""
         self.detector = river_drift.KSWIN(
-            alpha=self.alpha, window_size=self.window_size, stat_size=self.stat_size
+            alpha=self.alpha,
+            window_size=self.window_size,
+            stat_size=self.stat_size,
+            seed=self.seed,
         )
         self._drift_history = []
 

src/apeiron/drift_detection/load_drift_detector.py

@@ -32,6 +32,7 @@ def _build_detector(detector_name: str, cfg: Config) -> BaseDriftDetector:
             alpha=cfg.drift_detection.kswin_alpha,
             window_size=cfg.drift_detection.kswin_window_size,
             stat_size=cfg.drift_detection.kswin_stat_size,
+            seed=cfg.drift_detection.kswin_seed,
         )
     elif detector_name == "PageHinkleyDetector":
         from apeiron.drift_detection.detectors.statistical_detectors import (

src/apeiron/model/torch_model_harness.py

@@ -127,7 +127,7 @@ def eval(self) -> List[float]:
             # else:
             y_hat = self.model(x)
 
-            batch_size = y.size(0)
+            batch_size = y.shape[0]
             for i, m in enumerate(self.eval_metrics.values()):
                 metric_value = self._to_scalar(m(y_hat, y))
                 # For metrics that return percentages (like accuracy), we need to
@@ -171,7 +171,7 @@ def history_eval(self) -> Optional[List[float]]:
             # else:
             y_hat = self.model(x)
 
-            batch_size = y.size(0)
+            batch_size = y.shape[0]
             for i, m in enumerate(self.eval_metrics.values()):
                 metric_value = self._to_scalar(m(y_hat, y))
                 # For metrics that return percentages (like accuracy), we need to

src/apeiron/profilers/count_flops.py

@@ -221,9 +221,10 @@ def _estimate_flops_per_elem(self, prof: profile, debug: bool = False) -> int:
                 }
             )
 
-        # Create DataFrame
-        df = pd.DataFrame(data)
+        if not data:
+            return 0
 
+        df = pd.DataFrame(data)
         # Multiply FLOPs per element by the number of times each operation was called
         df["est_flops_per_param"] = df.apply(
             lambda row: ATEN_FLOPS_PER_ELEMENT.get(row.operation, 0) * row["count"],

src/apeiron/training/continuous_trainer.py

@@ -61,6 +61,28 @@ def _safe_next(
                 # If we cannot inspect batch size, just accept the batch
                 return current_iter, [b.to(self.cfg.device) for b in batch]
 
+    def _log_validation(
+        self,
+        tag: str,
+        cur: Optional[list],
+        hist: Optional[list],
+        drift_event_id: int,
+    ) -> None:
+        """Record every eval metric by name, for both domains.
+
+        ``eval()`` returns a positional list; ``eval_metrics`` holds the labels
+        in the same order.
+        """
+        logger = get_logger(__name__)
+        names = self.modelHarness.eval_metrics
+        payload: dict[str, float] = {"drift_event_id": drift_event_id}
+        for domain, values in (("cur", cur), ("hist", hist)):
+            for name, value in zip(names, values or ()):
+                payload[f"val_{tag}_{domain}_{name}"] = float(value)
+        logger.stage("eval")
+        # increment=False: annotate the CL round's step, do not advance it.
+        logger.log(payload, commit=False, increment=False)
+
     def outer_cl_training_loop(
         self,
         drift_event_id: int = 0,
@@ -76,10 +98,13 @@ def outer_cl_training_loop(
         else:
             hist_train_iter = None
 
-        # TODO: need to find away to explicitly match the metrics to their name/label
         cur_validation_metrics = self.modelHarness.eval()
         hist_validation_metrics = self.modelHarness.history_eval()
 
+        self._log_validation(
+            "pre", cur_validation_metrics, hist_validation_metrics, drift_event_id
+        )
+
         logger.info("==== Continual Learning ====")
         logger.info("\tInitial test acc: {}".format(cur_validation_metrics[0]), level=1)
         if hist_validation_metrics is not None:
@@ -126,6 +151,9 @@ def outer_cl_training_loop(
 
         cur_validation_metrics = self.modelHarness.eval()
         hist_validation_metrics = self.modelHarness.history_eval()
+        self._log_validation(
+            "post", cur_validation_metrics, hist_validation_metrics, drift_event_id
+        )
 
         logger.info(f"\tTest Accuracy: {cur_validation_metrics[0]:.1f}%", level=1)
         if hist_validation_metrics is not None:

src/main.py

@@ -10,16 +10,18 @@
 
 def main(argv: list[str] | None = None) -> int:
     cfg: Config = build_config(argv)
-    modelHarness = get_example(cfg=cfg)
 
-    # Configure logger
+    # Must precede get_example(): get_logger() ignores its arguments once an
+    # instance exists, so a harness that logs from __init__ would pin the config.
     backend = configure_backend(cfg)
     logger = get_logger(
         verbosity=cfg.verbosity,
         backend=backend,
         csv_path=cfg.visualization.input if cfg.visualization else None,
     )
 
+    modelHarness = get_example(cfg=cfg)
+
     # Determine project/experiment name
     project_name = "basesim-framework"
     if cfg.logging and cfg.logging.experiment_name:

tests/test_drift_detection.py

@@ -15,14 +15,27 @@
     KSWINDetector,
     PageHinkleyDetector,
 )
-from apeiron.drift_detection.detectors.model_performance_detector import (
-    EnsembleDetector,
-    ModelEvalDetector,
-    ModelPerformanceDetector,
-)
 from apeiron.config.configuration import DriftDetectionCfg
 from apeiron.drift_detection.load_drift_detector import load_drift_detector
 
+# Evidently has no wheel for every deployment target. Import them behind a
+# guard so the statistical-detector tests -- which need nothing beyond river --
+# still run there, instead of the whole module erroring out at collection.
+try:
+    from apeiron.drift_detection.detectors.model_performance_detector import (
+        EnsembleDetector,
+        ModelEvalDetector,
+        ModelPerformanceDetector,
+    )
+
+    HAS_EVIDENTLY = True
+except ModuleNotFoundError:  # pragma: no cover - environment dependent
+    HAS_EVIDENTLY = False
+
+requires_evidently = pytest.mark.skipif(
+    not HAS_EVIDENTLY, reason="evidently is not installed in this environment"
+)
+
 
 # ---------------------------------------------------------------------------
 # DriftSignal
@@ -127,6 +140,19 @@ def test_metadata_keys(self):
 # ---------------------------------------------------------------------------
 # KSWINDetector
 # ---------------------------------------------------------------------------
+def _shifting_stream() -> list[float]:
+    """Deterministic stream: 100 samples at mean 0, then 100 at mean 5."""
+    rng = np.random.default_rng(0)
+    return [
+        *rng.normal(0.0, 1.0, 100).tolist(),
+        *rng.normal(5.0, 1.0, 100).tolist(),
+    ]
+
+
+def _drift_steps(detector: KSWINDetector, stream: list[float]) -> list[int]:
+    return [i for i, v in enumerate(stream) if detector.update(v).drift_detected]
+
+
 class TestKSWINDetector:
     def test_init(self):
         d = KSWINDetector(alpha=0.01)
@@ -152,6 +178,24 @@ def test_confidence(self):
         signal = d.update(1.0)
         assert signal.confidence == pytest.approx(0.99)
 
+    def test_same_seed_gives_the_same_detections(self):
+        stream = _shifting_stream()
+        first = _drift_steps(
+            KSWINDetector(window_size=60, stat_size=20, seed=1337), stream
+        )
+        second = _drift_steps(
+            KSWINDetector(window_size=60, stat_size=20, seed=1337), stream
+        )
+        assert first == second
+        assert first, "expected at least one detection on a mean shift of 5 sigma"
+
+    def test_reset_restores_the_seeded_sequence(self):
+        stream = _shifting_stream()
+        d = KSWINDetector(window_size=60, stat_size=20, seed=1337)
+        first = _drift_steps(d, stream)
+        d.reset()
+        assert _drift_steps(d, stream) == first
+
 
 # ---------------------------------------------------------------------------
 # PageHinkleyDetector
@@ -191,6 +235,7 @@ def test_reset(self):
 # ---------------------------------------------------------------------------
 # ModelPerformanceDetector (simple value path)
 # ---------------------------------------------------------------------------
+@requires_evidently
 class TestModelPerformanceDetector:
     def test_not_initialized_raises(self):
         d = ModelPerformanceDetector()
@@ -223,6 +268,7 @@ def test_reset_clears_history(self):
 # ---------------------------------------------------------------------------
 # ModelEvalDetector
 # ---------------------------------------------------------------------------
+@requires_evidently
 class TestModelEvalDetector:
     def test_raises_without_harness(self):
         d = ModelEvalDetector()
@@ -378,6 +424,18 @@ def test_kswin(self, default_cfg):
         d = load_drift_detector(cfg)
         assert isinstance(d, KSWINDetector)
 
+    def test_kswin_seed_is_forwarded_from_config(self, default_cfg):
+        from dataclasses import replace
+
+        cfg = replace(
+            default_cfg,
+            drift_detection=DriftDetectionCfg(
+                detector_name="KSWINDetector", kswin_seed=1337
+            ),
+        )
+        d = load_drift_detector(cfg)
+        assert d.seed == 1337
+
     def test_page_hinkley(self, default_cfg):
         from dataclasses import replace
 
@@ -388,6 +446,7 @@ def test_page_hinkley(self, default_cfg):
         d = load_drift_detector(cfg)
         assert isinstance(d, PageHinkleyDetector)
 
+    @requires_evidently
     def test_model_performance(self, default_cfg):
         from dataclasses import replace
 
@@ -398,6 +457,7 @@ def test_model_performance(self, default_cfg):
         d = load_drift_detector(cfg)
         assert isinstance(d, ModelPerformanceDetector)
 
+    @requires_evidently
     def test_eval_detector(self, default_cfg):
         from dataclasses import replace
 

tests/test_profiler.py

@@ -121,6 +121,20 @@ def test_format_throughput_small(self, profiler):
         assert "FLOP/s" in profiler._format_throughput(500)
 
 
+class TestMeasureFlopsOptimizer:
+    def test_parameter_free_model_records_zero_flops(self):
+        class NoParamModel(nn.Module):
+            def forward(self, x: torch.Tensor) -> torch.Tensor:
+                return x.sum(dim=-1, keepdim=True)
+
+        p = FLOPSProfiler()
+        model = NoParamModel()
+        with p.measure_flops_optimizer(tag="optimizer", model=model, device="cpu"):
+            pass
+        assert p.profiles["optimizer"]["flop"] == [0]
+        assert p.profiles["optimizer"]["time"][0] >= 0
+
+
 class TestPrintPerformance:
     def test_no_data(self, capsys):
         p = FLOPSProfiler()