From 911f81e8620cada58a75d611ecf4448b61933998 Mon Sep 17 00:00:00 2001 From: Alvaro Sanchez-Villar Date: Tue, 11 Aug 2026 00:35:24 -0400 Subject: [PATCH 1/6] Use the target's shape for batch size so structured targets work in eval Co-authored-by: Ana Gainaru --- src/apeiron/model/torch_model_harness.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/apeiron/model/torch_model_harness.py b/src/apeiron/model/torch_model_harness.py index ac8c498..bb7ee9d 100644 --- a/src/apeiron/model/torch_model_harness.py +++ b/src/apeiron/model/torch_model_harness.py @@ -127,7 +127,7 @@ def eval(self) -> List[float]: # else: y_hat = self.model(x) - batch_size = y.size(0) + batch_size = y.shape[0] for i, m in enumerate(self.eval_metrics.values()): metric_value = self._to_scalar(m(y_hat, y)) # For metrics that return percentages (like accuracy), we need to @@ -171,7 +171,7 @@ def history_eval(self) -> Optional[List[float]]: # else: y_hat = self.model(x) - batch_size = y.size(0) + batch_size = y.shape[0] for i, m in enumerate(self.eval_metrics.values()): metric_value = self._to_scalar(m(y_hat, y)) # For metrics that return percentages (like accuracy), we need to From c27ffd3c23efcbd150aaa0a49f5ceb0c6e32b82d Mon Sep 17 00:00:00 2001 From: Alvaro Sanchez-Villar Date: Tue, 11 Aug 2026 00:35:24 -0400 Subject: [PATCH 2/6] Return zero FLOPs when the profiler captured no ATen events --- src/apeiron/profilers/count_flops.py | 5 +++-- tests/test_profiler.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/apeiron/profilers/count_flops.py b/src/apeiron/profilers/count_flops.py index 7573237..e4b891b 100644 --- a/src/apeiron/profilers/count_flops.py +++ b/src/apeiron/profilers/count_flops.py @@ -221,9 +221,10 @@ def _estimate_flops_per_elem(self, prof: profile, debug: bool = False) -> int: } ) - # Create DataFrame - df = pd.DataFrame(data) + if not data: + return 0 + df = pd.DataFrame(data) # Multiply FLOPs per element by the number of times each operation was called df["est_flops_per_param"] = df.apply( lambda row: ATEN_FLOPS_PER_ELEMENT.get(row.operation, 0) * row["count"], diff --git a/tests/test_profiler.py b/tests/test_profiler.py index dae2b40..52905e6 100644 --- a/tests/test_profiler.py +++ b/tests/test_profiler.py @@ -121,6 +121,20 @@ def test_format_throughput_small(self, profiler): assert "FLOP/s" in profiler._format_throughput(500) +class TestMeasureFlopsOptimizer: + def test_parameter_free_model_records_zero_flops(self): + class NoParamModel(nn.Module): + def forward(self, x: torch.Tensor) -> torch.Tensor: + return x.sum(dim=-1, keepdim=True) + + p = FLOPSProfiler() + model = NoParamModel() + with p.measure_flops_optimizer(tag="optimizer", model=model, device="cpu"): + pass + assert p.profiles["optimizer"]["flop"] == [0] + assert p.profiles["optimizer"]["time"][0] >= 0 + + class TestPrintPerformance: def test_no_data(self, capsys): p = FLOPSProfiler() From 8529331c2dc3d75a4ac08415fb5b2500d48e9df3 Mon Sep 17 00:00:00 2001 From: Alvaro Sanchez-Villar Date: Tue, 11 Aug 2026 00:35:24 -0400 Subject: [PATCH 3/6] Configure the logger before building the model harness Co-authored-by: Ana Gainaru --- src/main.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main.py b/src/main.py index 00cf1cf..7fd0253 100644 --- a/src/main.py +++ b/src/main.py @@ -10,9 +10,9 @@ def main(argv: list[str] | None = None) -> int: cfg: Config = build_config(argv) - modelHarness = get_example(cfg=cfg) - # Configure logger + # Must precede get_example(): get_logger() ignores its arguments once an + # instance exists, so a harness that logs from __init__ would pin the config. backend = configure_backend(cfg) logger = get_logger( verbosity=cfg.verbosity, @@ -20,6 +20,8 @@ def main(argv: list[str] | None = None) -> int: csv_path=cfg.visualization.input if cfg.visualization else None, ) + modelHarness = get_example(cfg=cfg) + # Determine project/experiment name project_name = "basesim-framework" if cfg.logging and cfg.logging.experiment_name: From 8946e87a643253fa44f1ea07f32181212cc92c6d Mon Sep 17 00:00:00 2001 From: Alvaro Sanchez-Villar Date: Tue, 11 Aug 2026 00:35:24 -0400 Subject: [PATCH 4/6] Report validation metrics by name and record them before adaptation --- src/apeiron/training/continuous_trainer.py | 30 +++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/src/apeiron/training/continuous_trainer.py b/src/apeiron/training/continuous_trainer.py index 160e6b0..8ac1f50 100644 --- a/src/apeiron/training/continuous_trainer.py +++ b/src/apeiron/training/continuous_trainer.py @@ -61,6 +61,28 @@ def _safe_next( # If we cannot inspect batch size, just accept the batch return current_iter, [b.to(self.cfg.device) for b in batch] + def _log_validation( + self, + tag: str, + cur: Optional[list], + hist: Optional[list], + drift_event_id: int, + ) -> None: + """Record every eval metric by name, for both domains. + + ``eval()`` returns a positional list; ``eval_metrics`` holds the labels + in the same order. + """ + logger = get_logger(__name__) + names = self.modelHarness.eval_metrics + payload: dict[str, float] = {"drift_event_id": drift_event_id} + for domain, values in (("cur", cur), ("hist", hist)): + for name, value in zip(names, values or ()): + payload[f"val_{tag}_{domain}_{name}"] = float(value) + logger.stage("eval") + # increment=False: annotate the CL round's step, do not advance it. + logger.log(payload, commit=False, increment=False) + def outer_cl_training_loop( self, drift_event_id: int = 0, @@ -76,10 +98,13 @@ def outer_cl_training_loop( else: hist_train_iter = None - # TODO: need to find away to explicitly match the metrics to their name/label cur_validation_metrics = self.modelHarness.eval() hist_validation_metrics = self.modelHarness.history_eval() + self._log_validation( + "pre", cur_validation_metrics, hist_validation_metrics, drift_event_id + ) + logger.info("==== Continual Learning ====") logger.info("\tInitial test acc: {}".format(cur_validation_metrics[0]), level=1) if hist_validation_metrics is not None: @@ -126,6 +151,9 @@ def outer_cl_training_loop( cur_validation_metrics = self.modelHarness.eval() hist_validation_metrics = self.modelHarness.history_eval() + self._log_validation( + "post", cur_validation_metrics, hist_validation_metrics, drift_event_id + ) logger.info(f"\tTest Accuracy: {cur_validation_metrics[0]:.1f}%", level=1) if hist_validation_metrics is not None: From 27dc6ae992c73acc554d154cefc2f7f7f4f3af19 Mon Sep 17 00:00:00 2001 From: Alvaro Sanchez-Villar Date: Tue, 11 Aug 2026 00:35:24 -0400 Subject: [PATCH 5/6] Add an optional seed for KSWIN's reference window Co-authored-by: Ana Gainaru --- docs/configurations.md | 2 + src/apeiron/config/configuration.py | 1 + .../detectors/statistical_detectors.py | 12 +++++- .../drift_detection/load_drift_detector.py | 1 + tests/test_drift_detection.py | 43 +++++++++++++++++++ 5 files changed, 57 insertions(+), 2 deletions(-) diff --git a/docs/configurations.md b/docs/configurations.md index a43bd91..7bacdc0 100644 --- a/docs/configurations.md +++ b/docs/configurations.md @@ -159,6 +159,7 @@ KSWIN: kswin_alpha = 0.005 kswin_window_size = 100 kswin_stat_size = 30 +kswin_seed = 1337 ``` | Option | Type | Description | @@ -166,6 +167,7 @@ kswin_stat_size = 30 | `kswin_alpha` | float | KSWIN significance level. | | `kswin_window_size` | int | KSWIN reference window size. | | `kswin_stat_size` | int | KSWIN recent sample window size. | +| `kswin_seed` | int | Seed for KSWIN's reference window sampling. Optional; leave unset for the historical non-reproducible behaviour. | Page-Hinkley: diff --git a/src/apeiron/config/configuration.py b/src/apeiron/config/configuration.py index 41561d5..7366056 100644 --- a/src/apeiron/config/configuration.py +++ b/src/apeiron/config/configuration.py @@ -159,6 +159,7 @@ class DriftDetectionCfg: kswin_alpha: float = 0.005 kswin_window_size: int = 100 kswin_stat_size: int = 30 + kswin_seed: int | None = None # None = unseeded, as before # PageHinkley hyperparameters ph_min_instances: int = 30 diff --git a/src/apeiron/drift_detection/detectors/statistical_detectors.py b/src/apeiron/drift_detection/detectors/statistical_detectors.py index 4e24bf9..d183a15 100644 --- a/src/apeiron/drift_detection/detectors/statistical_detectors.py +++ b/src/apeiron/drift_detection/detectors/statistical_detectors.py @@ -127,6 +127,7 @@ def __init__( minor_threshold: float = 0.3, moderate_threshold: float = 0.6, name: str = "KSWIN", + seed: int | None = None, ): """ Initialize KSWIN detector. @@ -138,6 +139,9 @@ def __init__( minor_threshold: Drift score threshold for continual learning moderate_threshold: Drift score threshold for fine-tuning name: Detector name + seed: Seed for KSWIN's reference-window sampling. KSWIN draws its + reference window at random, so runs are not reproducible unless + this is set. """ super().__init__(name) self.alpha = alpha @@ -145,8 +149,9 @@ def __init__( self.stat_size = stat_size self.minor_threshold = minor_threshold self.moderate_threshold = moderate_threshold + self.seed = seed self.detector = river_drift.KSWIN( - alpha=alpha, window_size=window_size, stat_size=stat_size + alpha=alpha, window_size=window_size, stat_size=stat_size, seed=seed ) self._drift_history: list[int] = [] self._is_initialized = True @@ -197,7 +202,10 @@ def update(self, value: float, **kwargs) -> DriftSignal: def reset(self) -> None: """Reset detector to initial state.""" self.detector = river_drift.KSWIN( - alpha=self.alpha, window_size=self.window_size, stat_size=self.stat_size + alpha=self.alpha, + window_size=self.window_size, + stat_size=self.stat_size, + seed=self.seed, ) self._drift_history = [] diff --git a/src/apeiron/drift_detection/load_drift_detector.py b/src/apeiron/drift_detection/load_drift_detector.py index 1273559..78cdd15 100644 --- a/src/apeiron/drift_detection/load_drift_detector.py +++ b/src/apeiron/drift_detection/load_drift_detector.py @@ -32,6 +32,7 @@ def _build_detector(detector_name: str, cfg: Config) -> BaseDriftDetector: alpha=cfg.drift_detection.kswin_alpha, window_size=cfg.drift_detection.kswin_window_size, stat_size=cfg.drift_detection.kswin_stat_size, + seed=cfg.drift_detection.kswin_seed, ) elif detector_name == "PageHinkleyDetector": from apeiron.drift_detection.detectors.statistical_detectors import ( diff --git a/tests/test_drift_detection.py b/tests/test_drift_detection.py index 78595eb..51fd058 100644 --- a/tests/test_drift_detection.py +++ b/tests/test_drift_detection.py @@ -127,6 +127,19 @@ def test_metadata_keys(self): # --------------------------------------------------------------------------- # KSWINDetector # --------------------------------------------------------------------------- +def _shifting_stream() -> list[float]: + """Deterministic stream: 100 samples at mean 0, then 100 at mean 5.""" + rng = np.random.default_rng(0) + return [ + *rng.normal(0.0, 1.0, 100).tolist(), + *rng.normal(5.0, 1.0, 100).tolist(), + ] + + +def _drift_steps(detector: KSWINDetector, stream: list[float]) -> list[int]: + return [i for i, v in enumerate(stream) if detector.update(v).drift_detected] + + class TestKSWINDetector: def test_init(self): d = KSWINDetector(alpha=0.01) @@ -152,6 +165,24 @@ def test_confidence(self): signal = d.update(1.0) assert signal.confidence == pytest.approx(0.99) + def test_same_seed_gives_the_same_detections(self): + stream = _shifting_stream() + first = _drift_steps( + KSWINDetector(window_size=60, stat_size=20, seed=1337), stream + ) + second = _drift_steps( + KSWINDetector(window_size=60, stat_size=20, seed=1337), stream + ) + assert first == second + assert first, "expected at least one detection on a mean shift of 5 sigma" + + def test_reset_restores_the_seeded_sequence(self): + stream = _shifting_stream() + d = KSWINDetector(window_size=60, stat_size=20, seed=1337) + first = _drift_steps(d, stream) + d.reset() + assert _drift_steps(d, stream) == first + # --------------------------------------------------------------------------- # PageHinkleyDetector @@ -378,6 +409,18 @@ def test_kswin(self, default_cfg): d = load_drift_detector(cfg) assert isinstance(d, KSWINDetector) + def test_kswin_seed_is_forwarded_from_config(self, default_cfg): + from dataclasses import replace + + cfg = replace( + default_cfg, + drift_detection=DriftDetectionCfg( + detector_name="KSWINDetector", kswin_seed=1337 + ), + ) + d = load_drift_detector(cfg) + assert d.seed == 1337 + def test_page_hinkley(self, default_cfg): from dataclasses import replace From 9443ac7bcdb2e93e464ede7c10cb691044dfbcf2 Mon Sep 17 00:00:00 2001 From: Alvaro Sanchez-Villar Date: Tue, 11 Aug 2026 00:35:24 -0400 Subject: [PATCH 6/6] Import evidently at its call site so apeiron loads without it Co-authored-by: Ana Gainaru --- src/apeiron/drift_detection/__init__.py | 1 + .../detectors/model_performance_detector.py | 9 ++++--- tests/test_drift_detection.py | 27 +++++++++++++++---- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/src/apeiron/drift_detection/__init__.py b/src/apeiron/drift_detection/__init__.py index 3cc95b7..f823257 100644 --- a/src/apeiron/drift_detection/__init__.py +++ b/src/apeiron/drift_detection/__init__.py @@ -25,6 +25,7 @@ KSWINDetector, PageHinkleyDetector, ) + from apeiron.drift_detection.detectors.model_performance_detector import ( ModelPerformanceDetector, EnsembleDetector, diff --git a/src/apeiron/drift_detection/detectors/model_performance_detector.py b/src/apeiron/drift_detection/detectors/model_performance_detector.py index a26eb32..efd62c6 100644 --- a/src/apeiron/drift_detection/detectors/model_performance_detector.py +++ b/src/apeiron/drift_detection/detectors/model_performance_detector.py @@ -10,8 +10,6 @@ import numpy as np import pandas as pd from typing import Optional, List -from evidently import Report -from evidently.presets import DataDriftPreset from apeiron.drift_detection.detectors.base import ( BaseDriftDetector, DriftSignal, @@ -210,7 +208,12 @@ def update( if self.reference_targets is not None: reference_data["target"] = self.reference_targets - # Run drift detection + # Imported at the call site: this is the only place in the package that + # needs evidently, and at module scope `import apeiron` fails wherever it + # is absent, taking the statistical detectors down with it. + from evidently import Report + from evidently.presets import DataDriftPreset + report = Report(metrics=[DataDriftPreset()]) snapshot = report.run(reference_data=reference_data, current_data=current_data) result_dict = snapshot.dict() diff --git a/tests/test_drift_detection.py b/tests/test_drift_detection.py index 51fd058..74d38ee 100644 --- a/tests/test_drift_detection.py +++ b/tests/test_drift_detection.py @@ -15,14 +15,27 @@ KSWINDetector, PageHinkleyDetector, ) -from apeiron.drift_detection.detectors.model_performance_detector import ( - EnsembleDetector, - ModelEvalDetector, - ModelPerformanceDetector, -) from apeiron.config.configuration import DriftDetectionCfg from apeiron.drift_detection.load_drift_detector import load_drift_detector +# Evidently has no wheel for every deployment target. Import them behind a +# guard so the statistical-detector tests -- which need nothing beyond river -- +# still run there, instead of the whole module erroring out at collection. +try: + from apeiron.drift_detection.detectors.model_performance_detector import ( + EnsembleDetector, + ModelEvalDetector, + ModelPerformanceDetector, + ) + + HAS_EVIDENTLY = True +except ModuleNotFoundError: # pragma: no cover - environment dependent + HAS_EVIDENTLY = False + +requires_evidently = pytest.mark.skipif( + not HAS_EVIDENTLY, reason="evidently is not installed in this environment" +) + # --------------------------------------------------------------------------- # DriftSignal @@ -222,6 +235,7 @@ def test_reset(self): # --------------------------------------------------------------------------- # ModelPerformanceDetector (simple value path) # --------------------------------------------------------------------------- +@requires_evidently class TestModelPerformanceDetector: def test_not_initialized_raises(self): d = ModelPerformanceDetector() @@ -254,6 +268,7 @@ def test_reset_clears_history(self): # --------------------------------------------------------------------------- # ModelEvalDetector # --------------------------------------------------------------------------- +@requires_evidently class TestModelEvalDetector: def test_raises_without_harness(self): d = ModelEvalDetector() @@ -431,6 +446,7 @@ def test_page_hinkley(self, default_cfg): d = load_drift_detector(cfg) assert isinstance(d, PageHinkleyDetector) + @requires_evidently def test_model_performance(self, default_cfg): from dataclasses import replace @@ -441,6 +457,7 @@ def test_model_performance(self, default_cfg): d = load_drift_detector(cfg) assert isinstance(d, ModelPerformanceDetector) + @requires_evidently def test_eval_detector(self, default_cfg): from dataclasses import replace