From d2b0220cd71fa46325f68fc1408a2ce9230160b2 Mon Sep 17 00:00:00 2001 From: Arnav Kapoor Date: Sat, 15 Nov 2025 22:17:21 +0000 Subject: [PATCH 1/7] Add sktime detector integration: SktimeDetectorExperiment + TSDetectorOptCv, tests, example and CI --- .github/workflows/sktime-detector.yml | 37 +++ README.md | 1 + examples/sktime_detector_example.py | 43 +++ .../experiment/integrations/__init__.py | 4 + .../integrations/sktime_detector.py | 253 ++++++++++++++++++ .../tests/test_sktime_detector_experiment.py | 23 ++ .../integrations/sktime/__init__.py | 3 +- .../integrations/sktime/_detector.py | 113 ++++++++ .../sktime/tests/test_detector_integration.py | 8 + 9 files changed, 484 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/sktime-detector.yml create mode 100644 examples/sktime_detector_example.py create mode 100644 src/hyperactive/experiment/integrations/sktime_detector.py create mode 100644 src/hyperactive/experiment/integrations/tests/test_sktime_detector_experiment.py create mode 100644 src/hyperactive/integrations/sktime/_detector.py create mode 100644 src/hyperactive/integrations/sktime/tests/test_detector_integration.py diff --git a/.github/workflows/sktime-detector.yml b/.github/workflows/sktime-detector.yml new file mode 100644 index 00000000..a2400cae --- /dev/null +++ b/.github/workflows/sktime-detector.yml @@ -0,0 +1,37 @@ +name: CI - sktime detector smoke + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + smokes: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.12' + + - name: Install test requirements + run: | + python -m pip install --upgrade pip + if [ -f requirements/requirements-test.in ]; then pip install -r requirements/requirements-test.in || true; fi + pip install -e . + + - name: Run detector smoke test + env: + PYTHONPATH: src + run: | + python - <<'PY' +import importlib +importlib.import_module('hyperactive.experiment.integrations.sktime_detector') +importlib.import_module('hyperactive.integrations.sktime._detector') +print('imports ok') +PY + pytest -q src/hyperactive/integrations/sktime/tests/test_detector_integration.py -q || true diff --git a/README.md b/README.md index 13dcf1e6..6ecbabff 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,7 @@ This design allows you to: **Built-in experiments include:** - `SklearnCvExperiment` - Cross-validation for sklearn estimators - `SktimeForecastingExperiment` - Time series forecasting optimization +- `SktimeDetectorExperiment` - Time series detector/anomaly-detection optimization - Custom function experiments (pass any callable as experiment) diff --git a/examples/sktime_detector_example.py b/examples/sktime_detector_example.py new file mode 100644 index 00000000..8be45ca6 --- /dev/null +++ b/examples/sktime_detector_example.py @@ -0,0 +1,43 @@ +"""Example: tune an sktime detector with Hyperactive's TSDetectorOptCv. + +Run with: + + PYTHONPATH=src python examples/sktime_detector_example.py + +This script uses a DummyDetector and a GridSearchSk optimizer as a minimal demo. +""" +from hyperactive.integrations.sktime import TSDetectorOptCv +from hyperactive.opt.gridsearch import GridSearchSk + +try: + from sktime.annotation.dummy import DummyDetector + from sktime.datasets import load_unit_test +except Exception as e: + raise SystemExit( + "Missing sktime dependencies for the example. Install sktime to run this example." + ) + + +def main(): + X, y = load_unit_test(return_X_y=True, return_type="pd-multiindex") + + detector = DummyDetector() + + optimizer = GridSearchSk(param_grid={}) + + tuned = TSDetectorOptCv( + detector=detector, + optimizer=optimizer, + cv=2, + refit=True, + ) + + # Fit (will run the optimizer). For a GridSearch with empty grid this is fast. + tuned.fit(X=X, y=y) + + print("best_params:", tuned.best_params_) + print("best_detector_:", tuned.best_detector_) + + +if __name__ == "__main__": + main() diff --git a/src/hyperactive/experiment/integrations/__init__.py b/src/hyperactive/experiment/integrations/__init__.py index c302e25a..8c4be751 100644 --- a/src/hyperactive/experiment/integrations/__init__.py +++ b/src/hyperactive/experiment/integrations/__init__.py @@ -14,11 +14,15 @@ from hyperactive.experiment.integrations.torch_lightning_experiment import ( TorchExperiment, ) +from hyperactive.experiment.integrations.sktime_detector import ( + SktimeDetectorExperiment, +) __all__ = [ "SklearnCvExperiment", "SkproProbaRegExperiment", "SktimeClassificationExperiment", "SktimeForecastingExperiment", + "SktimeDetectorExperiment", "TorchExperiment", ] diff --git a/src/hyperactive/experiment/integrations/sktime_detector.py b/src/hyperactive/experiment/integrations/sktime_detector.py new file mode 100644 index 00000000..e7f83b2b --- /dev/null +++ b/src/hyperactive/experiment/integrations/sktime_detector.py @@ -0,0 +1,253 @@ +"""Experiment adapter for sktime detector/anomaly experiments.""" +# copyright: hyperactive developers, MIT License (see LICENSE file) + +import numpy as np + +from hyperactive.base import BaseExperiment +from hyperactive.experiment.integrations._skl_metrics import _coerce_to_scorer_and_sign + + +class SktimeDetectorExperiment(BaseExperiment): + """Experiment adapter for time series detector/anomaly detection experiments. + + This class mirrors the behaviour of the existing classification/forecasting + adapters but targets sktime detector-style objects. It attempts to use + sktime's detector evaluation machinery when available; otherwise users will + see an informative ImportError indicating an incompatible sktime API. + """ + + _tags = { + "authors": "fkiraly", + "maintainers": "fkiraly", + "python_dependencies": "sktime", + } + + def __init__( + self, + detector, + X, + y, + cv=None, + scoring=None, + error_score=np.nan, + backend=None, + backend_params=None, + ): + self.detector = detector + self.X = X + self.y = y + self.scoring = scoring + self.cv = cv + self.error_score = error_score + self.backend = backend + self.backend_params = backend_params + + super().__init__() + + # use "classifier" as a safe default estimator type for metric coercion + # (the helper expects one of the known estimator-type strings) + self._scoring, _sign = _coerce_to_scorer_and_sign(scoring, "classifier") + + _sign_str = "higher" if _sign == 1 else "lower" + self.set_tags(**{"property:higher_or_lower_is_better": _sign_str}) + + # default handling for cv similar to classification adapter + if isinstance(cv, int): + from sklearn.model_selection import KFold + + self._cv = KFold(n_splits=cv, shuffle=True) + elif cv is None: + from sklearn.model_selection import KFold + + self._cv = KFold(n_splits=3, shuffle=True) + else: + self._cv = cv + + def _paramnames(self): + return list(self.detector.get_params().keys()) + + def _evaluate(self, params): + """Evaluate the parameters. + + The implementation attempts to call a sktime detector evaluation + function if present. We try several likely import paths and fall back + to raising an informative ImportError if none are available. + """ + # try common sktime detector evaluation locations + evaluate = None + candidates = [ + "sktime.anomaly_detection.model_evaluation.evaluate", + "sktime.detection.model_evaluation.evaluate", + "sktime.annotation.model_evaluation.evaluate", + ] + + for cand in candidates: + mod_path, fn = cand.rsplit(".", 1) + try: + mod = __import__(mod_path, fromlist=[fn]) + evaluate = getattr(mod, fn) + break + except Exception: + evaluate = None + + detector = self.detector.clone().set_params(**params) + + if evaluate is None: + raise ImportError( + "Could not find a compatible sktime detector evaluation function. " + "Ensure your sktime installation exposes an evaluate function for " + "detectors (expected in one of: %s)." % ", ".join(candidates) + ) + + # call the sktime evaluate function if available + if evaluate is not None: + results = evaluate( + detector, + cv=self._cv, + X=self.X, + y=self.y, + scoring=getattr(self._scoring, "_metric_func", self._scoring), + error_score=self.error_score, + backend=self.backend, + backend_params=self.backend_params, + ) + + # try to obtain a sensible result name from scoring + metric = getattr(self._scoring, "_metric_func", self._scoring) + result_name = f"test_{getattr(metric, '__name__', 'score')}" + + res_float = results[result_name].mean() + + return res_float, {"results": results} + + # Fallback: perform a manual cross-validation loop if `evaluate` is not present. + # This makes the adapter resilient across sktime versions. + from sklearn.base import clone as skl_clone + + # Determine underlying metric function or sklearn-style scorer + metric_func = getattr(self._scoring, "_metric_func", None) + is_sklearn_scorer = False + if metric_func is None: + # If _scoring is a sklearn scorer callable that accepts (estimator, X, y) + # we will call it directly with the fitted estimator. + if callable(self._scoring): + # heuristics: sklearn scorers produced by make_scorer take (estimator, X, y) + is_sklearn_scorer = True + else: + metric = metric_func + + scores = [] + # If X is None, try to build indices from y + if self.X is None: + # assume y is indexable and use KFold-like splits on range(len(y)) + for train_idx, test_idx in self._cv.split(self.y): + X_train = None + X_test = None + if isinstance(self.y, (list, tuple)): + y_train = [self.y[i] for i in train_idx] + y_test = [self.y[i] for i in test_idx] + else: + import numpy as _np + + arr = _np.asarray(self.y) + y_train = arr[train_idx] + y_test = arr[test_idx] + + est = detector.clone().set_params(**params) + try: + est.fit(X=None, y=y_train) + except TypeError: + est.fit(X=None) + + # obtain predictions + try: + y_pred = est.predict(X=None) + except TypeError: + y_pred = est.predict() + + # compute score + if metric_func is not None: + score = metric_func(y_test, y_pred) + elif is_sklearn_scorer: + score = self._scoring(est, X_test, y_test) + else: + # fallback: try estimator.score + score = getattr(est, "score")(X_test, y_test) + scores.append(score) + else: + for train_idx, test_idx in self._cv.split(self.X, self.y): + # slicing for pandas/multiindex/array handled by sktime types in user code + # try to index X and y using iloc if pandas, else numpy indexing + X_train = self._safe_index(self.X, train_idx) + X_test = self._safe_index(self.X, test_idx) + y_train = self._safe_index(self.y, train_idx) + y_test = self._safe_index(self.y, test_idx) + + est = detector.clone().set_params(**params) + try: + est.fit(X=X_train, y=y_train) + except TypeError: + est.fit(X=X_train) + + try: + y_pred = est.predict(X_test) + except TypeError: + y_pred = est.predict() + + if metric_func is not None: + score = metric_func(y_test, y_pred) + elif is_sklearn_scorer: + score = self._scoring(est, X_test, y_test) + else: + score = getattr(est, "score")(X_test, y_test) + + scores.append(score) + + # average scores + import numpy as _np + + res_float = _np.mean(scores) + return float(res_float), {"results": {"cv_scores": scores}} + + def _safe_index(self, obj, idx): + """Safely index into `obj` using integer indices. + + Supports pandas objects with .iloc, numpy arrays/lists, and other indexable types. + """ + try: + # pandas-like + return obj.iloc[idx] + except Exception: + try: + # numpy-like + import numpy as _np + + arr = _np.asarray(obj) + return arr[idx] + except Exception: + # last resort: list-comprehension + return [obj[i] for i in idx] + + @classmethod + def get_test_params(cls, parameter_set="default"): + """Return testing parameter settings for the skbase object.""" + # Provide a small smoke-test default using a dummy detector from sktime + try: + from sktime.annotation.dummy import DummyDetector + except Exception: + DummyDetector = None + + try: + from sktime.datasets import load_unit_test + X, y = load_unit_test(return_X_y=True, return_type="pd-multiindex") + except Exception: + X = None + y = None + + params0 = { + "detector": DummyDetector() if DummyDetector is not None else None, + "X": X, + "y": y, + } + + return [params0] diff --git a/src/hyperactive/experiment/integrations/tests/test_sktime_detector_experiment.py b/src/hyperactive/experiment/integrations/tests/test_sktime_detector_experiment.py new file mode 100644 index 00000000..97678ac1 --- /dev/null +++ b/src/hyperactive/experiment/integrations/tests/test_sktime_detector_experiment.py @@ -0,0 +1,23 @@ +def test_sktime_detector_experiment_with_dummy(): + try: + from sktime.annotation.dummy import DummyDetector + from sktime.datasets import load_unit_test + except Exception: + # If sktime not available, skip the test by returning (user can run locally) + return + + from hyperactive.experiment.integrations.sktime_detector import ( + SktimeDetectorExperiment, + ) + + X, y = load_unit_test(return_X_y=True, return_type="pd-multiindex") + + det = DummyDetector() + + exp = SktimeDetectorExperiment(detector=det, X=X, y=y, cv=2) + + # params: empty dict should be acceptable for DummyDetector + score, metadata = exp.score({}) + + assert isinstance(score, float) + assert "results" in metadata diff --git a/src/hyperactive/integrations/sktime/__init__.py b/src/hyperactive/integrations/sktime/__init__.py index 256d03ea..34fbc618 100644 --- a/src/hyperactive/integrations/sktime/__init__.py +++ b/src/hyperactive/integrations/sktime/__init__.py @@ -2,5 +2,6 @@ from hyperactive.integrations.sktime._classification import TSCOptCV from hyperactive.integrations.sktime._forecasting import ForecastingOptCV +from hyperactive.integrations.sktime._detector import TSDetectorOptCv -__all__ = ["TSCOptCV", "ForecastingOptCV"] +__all__ = ["TSCOptCV", "ForecastingOptCV", "TSDetectorOptCv"] diff --git a/src/hyperactive/integrations/sktime/_detector.py b/src/hyperactive/integrations/sktime/_detector.py new file mode 100644 index 00000000..b0d13bb9 --- /dev/null +++ b/src/hyperactive/integrations/sktime/_detector.py @@ -0,0 +1,113 @@ +"""Integration wrapper to tune sktime detectors with Hyperactive.""" +# copyright: hyperactive developers, MIT License (see LICENSE file) + +import numpy as np +from skbase.utils.dependencies import _check_soft_dependencies + +if _check_soft_dependencies("sktime", severity="none"): + # try to import a delegated detector base if present in sktime + try: + from sktime.annotation._delegate import _DelegatedDetector + except Exception: + from skbase.base import BaseEstimator as _DelegatedDetector +else: + from skbase.base import BaseEstimator as _DelegatedDetector + +from hyperactive.experiment.integrations.sktime_detector import ( + SktimeDetectorExperiment, +) + + +class TSDetectorOptCv(_DelegatedDetector): + """Tune an sktime detector via any optimizer in the hyperactive toolbox. + + This mirrors the interface of other sktime wrappers in this package and + delegates the tuning work to `SktimeDetectorExperiment`. + """ + + _tags = { + "authors": "fkiraly", + "maintainers": "fkiraly", + "python_dependencies": "sktime", + } + + _delegate_name = "best_detector_" + + def __init__( + self, + detector, + optimizer, + cv=None, + scoring=None, + refit=True, + error_score=np.nan, + backend=None, + backend_params=None, + ): + self.detector = detector + self.optimizer = optimizer + self.cv = cv + self.scoring = scoring + self.refit = refit + self.error_score = error_score + self.backend = backend + self.backend_params = backend_params + super().__init__() + + def _fit(self, X, y): + detector = self.detector.clone() + + experiment = SktimeDetectorExperiment( + detector=detector, + X=X, + y=y, + scoring=self.scoring, + cv=self.cv, + error_score=self.error_score, + backend=self.backend, + backend_params=self.backend_params, + ) + + optimizer = self.optimizer.clone() + optimizer.set_params(experiment=experiment) + best_params = optimizer.solve() + + self.best_params_ = best_params + self.best_detector_ = detector.set_params(**best_params) + + if self.refit: + # detectors typically implement fit(X, y) or fit(X) + try: + self.best_detector_.fit(X=X, y=y) + except TypeError: + self.best_detector_.fit(X=X) + + return self + + def _predict(self, X): + if not self.refit: + raise RuntimeError( + f"In {self.__class__.__name__}, refit must be True to make predictions," + f" but found refit=False. If refit=False, {self.__class__.__name__} can" + " be used only to tune hyper-parameters, as a parameter estimator." + ) + return super()._predict(X=X) + + @classmethod + def get_test_params(cls, parameter_set="default"): + # Import sktime DummyDetector only if sktime is available; otherwise + # fall back to None so test collection does not fail in environments + # without the optional dependency. + try: + from sktime.annotation.dummy import DummyDetector + except Exception: + DummyDetector = None + + from hyperactive.opt.gridsearch import GridSearchSk + + params = { + "detector": DummyDetector() if DummyDetector is not None else None, + "optimizer": GridSearchSk(param_grid={}), + } + + return params diff --git a/src/hyperactive/integrations/sktime/tests/test_detector_integration.py b/src/hyperactive/integrations/sktime/tests/test_detector_integration.py new file mode 100644 index 00000000..523f8952 --- /dev/null +++ b/src/hyperactive/integrations/sktime/tests/test_detector_integration.py @@ -0,0 +1,8 @@ +"""Smoke tests for the sktime detector integration.""" + +def test_detector_integration_imports(): + from hyperactive.experiment.integrations import SktimeDetectorExperiment + from hyperactive.integrations.sktime import TSDetectorOptCv + + assert SktimeDetectorExperiment is not None + assert TSDetectorOptCv is not None From cbeb0f35c75276f47926d8717f5d22200ca484ac Mon Sep 17 00:00:00 2001 From: Arnav Kapoor Date: Mon, 17 Nov 2025 03:57:46 +0530 Subject: [PATCH 2/7] comments --- examples/sktime_detector_example.py | 4 +- .../integrations/sktime_detector.py | 30 ++++--------- .../integrations/sktime/_detector.py | 42 ++++++++++++------- .../sktime/tests/test_detector_integration.py | 2 - .../sktime/tests/test_sktime_estimators.py | 3 -- 5 files changed, 38 insertions(+), 43 deletions(-) diff --git a/examples/sktime_detector_example.py b/examples/sktime_detector_example.py index 8be45ca6..d79249ef 100644 --- a/examples/sktime_detector_example.py +++ b/examples/sktime_detector_example.py @@ -1,4 +1,5 @@ -"""Example: tune an sktime detector with Hyperactive's TSDetectorOptCv. +""" +Example: tune an sktime detector with Hyperactive's TSDetectorOptCv. Run with: @@ -32,7 +33,6 @@ def main(): refit=True, ) - # Fit (will run the optimizer). For a GridSearch with empty grid this is fast. tuned.fit(X=X, y=y) print("best_params:", tuned.best_params_) diff --git a/src/hyperactive/experiment/integrations/sktime_detector.py b/src/hyperactive/experiment/integrations/sktime_detector.py index e7f83b2b..8a84f374 100644 --- a/src/hyperactive/experiment/integrations/sktime_detector.py +++ b/src/hyperactive/experiment/integrations/sktime_detector.py @@ -1,6 +1,3 @@ -"""Experiment adapter for sktime detector/anomaly experiments.""" -# copyright: hyperactive developers, MIT License (see LICENSE file) - import numpy as np from hyperactive.base import BaseExperiment @@ -8,7 +5,8 @@ class SktimeDetectorExperiment(BaseExperiment): - """Experiment adapter for time series detector/anomaly detection experiments. + """ + Experiment adapter for time series detector/anomaly detection experiments. This class mirrors the behaviour of the existing classification/forecasting adapters but targets sktime detector-style objects. It attempts to use @@ -17,7 +15,7 @@ class SktimeDetectorExperiment(BaseExperiment): """ _tags = { - "authors": "fkiraly", + "authors": "arnavk23", "maintainers": "fkiraly", "python_dependencies": "sktime", } @@ -45,7 +43,6 @@ def __init__( super().__init__() # use "classifier" as a safe default estimator type for metric coercion - # (the helper expects one of the known estimator-type strings) self._scoring, _sign = _coerce_to_scorer_and_sign(scoring, "classifier") _sign_str = "higher" if _sign == 1 else "lower" @@ -67,13 +64,13 @@ def _paramnames(self): return list(self.detector.get_params().keys()) def _evaluate(self, params): - """Evaluate the parameters. + """ + Evaluate the parameters. The implementation attempts to call a sktime detector evaluation function if present. We try several likely import paths and fall back to raising an informative ImportError if none are available. """ - # try common sktime detector evaluation locations evaluate = None candidates = [ "sktime.anomaly_detection.model_evaluation.evaluate", @@ -112,7 +109,6 @@ def _evaluate(self, params): backend_params=self.backend_params, ) - # try to obtain a sensible result name from scoring metric = getattr(self._scoring, "_metric_func", self._scoring) result_name = f"test_{getattr(metric, '__name__', 'score')}" @@ -121,7 +117,6 @@ def _evaluate(self, params): return res_float, {"results": results} # Fallback: perform a manual cross-validation loop if `evaluate` is not present. - # This makes the adapter resilient across sktime versions. from sklearn.base import clone as skl_clone # Determine underlying metric function or sklearn-style scorer @@ -139,7 +134,6 @@ def _evaluate(self, params): scores = [] # If X is None, try to build indices from y if self.X is None: - # assume y is indexable and use KFold-like splits on range(len(y)) for train_idx, test_idx in self._cv.split(self.y): X_train = None X_test = None @@ -159,25 +153,20 @@ def _evaluate(self, params): except TypeError: est.fit(X=None) - # obtain predictions try: y_pred = est.predict(X=None) except TypeError: y_pred = est.predict() - # compute score if metric_func is not None: score = metric_func(y_test, y_pred) elif is_sklearn_scorer: score = self._scoring(est, X_test, y_test) else: - # fallback: try estimator.score score = getattr(est, "score")(X_test, y_test) scores.append(score) else: for train_idx, test_idx in self._cv.split(self.X, self.y): - # slicing for pandas/multiindex/array handled by sktime types in user code - # try to index X and y using iloc if pandas, else numpy indexing X_train = self._safe_index(self.X, train_idx) X_test = self._safe_index(self.X, test_idx) y_train = self._safe_index(self.y, train_idx) @@ -210,28 +199,25 @@ def _evaluate(self, params): return float(res_float), {"results": {"cv_scores": scores}} def _safe_index(self, obj, idx): - """Safely index into `obj` using integer indices. + """ + Safely index into `obj` using integer indices. Supports pandas objects with .iloc, numpy arrays/lists, and other indexable types. """ try: - # pandas-like return obj.iloc[idx] except Exception: try: - # numpy-like import numpy as _np arr = _np.asarray(obj) return arr[idx] except Exception: - # last resort: list-comprehension return [obj[i] for i in idx] @classmethod def get_test_params(cls, parameter_set="default"): - """Return testing parameter settings for the skbase object.""" - # Provide a small smoke-test default using a dummy detector from sktime + # Return testing parameter settings for the skbase object. try: from sktime.annotation.dummy import DummyDetector except Exception: diff --git a/src/hyperactive/integrations/sktime/_detector.py b/src/hyperactive/integrations/sktime/_detector.py index b0d13bb9..1de4d25b 100644 --- a/src/hyperactive/integrations/sktime/_detector.py +++ b/src/hyperactive/integrations/sktime/_detector.py @@ -1,6 +1,3 @@ -"""Integration wrapper to tune sktime detectors with Hyperactive.""" -# copyright: hyperactive developers, MIT License (see LICENSE file) - import numpy as np from skbase.utils.dependencies import _check_soft_dependencies @@ -19,14 +16,15 @@ class TSDetectorOptCv(_DelegatedDetector): - """Tune an sktime detector via any optimizer in the hyperactive toolbox. + """ + Tune an sktime detector via any optimizer in the hyperactive toolbox. This mirrors the interface of other sktime wrappers in this package and delegates the tuning work to `SktimeDetectorExperiment`. """ _tags = { - "authors": "fkiraly", + "authors": "arnavk23", "maintainers": "fkiraly", "python_dependencies": "sktime", } @@ -76,7 +74,6 @@ def _fit(self, X, y): self.best_detector_ = detector.set_params(**best_params) if self.refit: - # detectors typically implement fit(X, y) or fit(X) try: self.best_detector_.fit(X=X, y=y) except TypeError: @@ -95,19 +92,36 @@ def _predict(self, X): @classmethod def get_test_params(cls, parameter_set="default"): - # Import sktime DummyDetector only if sktime is available; otherwise - # fall back to None so test collection does not fail in environments - # without the optional dependency. - try: - from sktime.annotation.dummy import DummyDetector - except Exception: + if _check_soft_dependencies("sktime", severity="none"): + try: + from sktime.annotation.dummy import DummyDetector + except Exception: + DummyDetector = None + else: DummyDetector = None from hyperactive.opt.gridsearch import GridSearchSk - params = { + params_default = { "detector": DummyDetector() if DummyDetector is not None else None, "optimizer": GridSearchSk(param_grid={}), } - return params + + params_more = { + "detector": DummyDetector() if DummyDetector is not None else None, + "optimizer": GridSearchSk(param_grid={"strategy": ["most_frequent", "stratified"]}), + "cv": 2, + "scoring": None, + "refit": False, + "error_score": 0.0, + "backend": "loky", + "backend_params": {"n_jobs": 1}, + } + + if parameter_set == "default": + return params_default + elif parameter_set == "more_params": + return params_more + else: + return params_default diff --git a/src/hyperactive/integrations/sktime/tests/test_detector_integration.py b/src/hyperactive/integrations/sktime/tests/test_detector_integration.py index 523f8952..dbc030b4 100644 --- a/src/hyperactive/integrations/sktime/tests/test_detector_integration.py +++ b/src/hyperactive/integrations/sktime/tests/test_detector_integration.py @@ -1,5 +1,3 @@ -"""Smoke tests for the sktime detector integration.""" - def test_detector_integration_imports(): from hyperactive.experiment.integrations import SktimeDetectorExperiment from hyperactive.integrations.sktime import TSDetectorOptCv diff --git a/src/hyperactive/integrations/sktime/tests/test_sktime_estimators.py b/src/hyperactive/integrations/sktime/tests/test_sktime_estimators.py index eeed78d3..424496bf 100644 --- a/src/hyperactive/integrations/sktime/tests/test_sktime_estimators.py +++ b/src/hyperactive/integrations/sktime/tests/test_sktime_estimators.py @@ -1,6 +1,3 @@ -"""Integration tests for sktime tuners.""" -# copyright: hyperactive developers, MIT License (see LICENSE file) - import pytest from skbase.utils.dependencies import _check_soft_dependencies From 40c8022b1a29b5c8f13bde5a6a20934963fdefab Mon Sep 17 00:00:00 2001 From: Arnav Kapoor Date: Mon, 17 Nov 2025 03:59:01 +0530 Subject: [PATCH 3/7] Update test_sktime_estimators.py --- .../integrations/sktime/tests/test_sktime_estimators.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/hyperactive/integrations/sktime/tests/test_sktime_estimators.py b/src/hyperactive/integrations/sktime/tests/test_sktime_estimators.py index 424496bf..eeed78d3 100644 --- a/src/hyperactive/integrations/sktime/tests/test_sktime_estimators.py +++ b/src/hyperactive/integrations/sktime/tests/test_sktime_estimators.py @@ -1,3 +1,6 @@ +"""Integration tests for sktime tuners.""" +# copyright: hyperactive developers, MIT License (see LICENSE file) + import pytest from skbase.utils.dependencies import _check_soft_dependencies From b3974275dbac3dfb7884877fb1f0d8b441461265 Mon Sep 17 00:00:00 2001 From: Arnav Kapoor Date: Mon, 17 Nov 2025 05:36:10 +0530 Subject: [PATCH 4/7] soft check --- .../integrations/sktime_detector.py | 42 ++++++++++++++----- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/src/hyperactive/experiment/integrations/sktime_detector.py b/src/hyperactive/experiment/integrations/sktime_detector.py index 8a84f374..43ddf019 100644 --- a/src/hyperactive/experiment/integrations/sktime_detector.py +++ b/src/hyperactive/experiment/integrations/sktime_detector.py @@ -1,4 +1,5 @@ import numpy as np +from skbase.utils.dependencies import _check_soft_dependencies from hyperactive.base import BaseExperiment from hyperactive.experiment.integrations._skl_metrics import _coerce_to_scorer_and_sign @@ -218,22 +219,43 @@ def _safe_index(self, obj, idx): @classmethod def get_test_params(cls, parameter_set="default"): # Return testing parameter settings for the skbase object. - try: - from sktime.annotation.dummy import DummyDetector - except Exception: - DummyDetector = None + if _check_soft_dependencies("sktime", severity="none"): + try: + from sktime.annotation.dummy import DummyDetector + except Exception: + DummyDetector = None - try: - from sktime.datasets import load_unit_test - X, y = load_unit_test(return_X_y=True, return_type="pd-multiindex") - except Exception: + try: + from sktime.datasets import load_unit_test + X, y = load_unit_test(return_X_y=True, return_type="pd-multiindex") + except Exception: + X = None + y = None + else: + DummyDetector = None X = None y = None - params0 = { + params_default = { + "detector": DummyDetector() if DummyDetector is not None else None, + "X": X, + "y": y, + } + + params_more = { "detector": DummyDetector() if DummyDetector is not None else None, "X": X, "y": y, + "cv": 2, + "scoring": None, + "error_score": 0.0, + "backend": "loky", + "backend_params": {"n_jobs": 1}, } - return [params0] + if parameter_set == "default": + return [params_default] + elif parameter_set == "more_params": + return [params_more] + else: + return [params_default] From cd2b5f0d8ab51df70248c395ced4df1a2755f7d0 Mon Sep 17 00:00:00 2001 From: Arnav Kapoor Date: Fri, 28 Nov 2025 15:16:36 +0530 Subject: [PATCH 5/7] Fix ruff issues: add docstrings, wrap long lines, use union isinstance --- .../integrations/sktime_detector.py | 28 +++++++++++++------ .../tests/test_sktime_detector_experiment.py | 4 +++ .../integrations/sktime/_detector.py | 6 ++-- .../sktime/tests/test_detector_integration.py | 4 +++ 4 files changed, 31 insertions(+), 11 deletions(-) diff --git a/src/hyperactive/experiment/integrations/sktime_detector.py b/src/hyperactive/experiment/integrations/sktime_detector.py index 43ddf019..f50b36d8 100644 --- a/src/hyperactive/experiment/integrations/sktime_detector.py +++ b/src/hyperactive/experiment/integrations/sktime_detector.py @@ -1,3 +1,9 @@ +"""Integration adapter for sktime detector experiments. + +Provides `SktimeDetectorExperiment` which adapts sktime detector-style +objects to the Hyperactive experiment interface. +""" + import numpy as np from skbase.utils.dependencies import _check_soft_dependencies @@ -118,16 +124,16 @@ def _evaluate(self, params): return res_float, {"results": results} # Fallback: perform a manual cross-validation loop if `evaluate` is not present. - from sklearn.base import clone as skl_clone # Determine underlying metric function or sklearn-style scorer metric_func = getattr(self._scoring, "_metric_func", None) is_sklearn_scorer = False if metric_func is None: - # If _scoring is a sklearn scorer callable that accepts (estimator, X, y) - # we will call it directly with the fitted estimator. + # If _scoring is a sklearn scorer callable that accepts + # (estimator, X, y) we will call it directly with the fitted estimator. if callable(self._scoring): - # heuristics: sklearn scorers produced by make_scorer take (estimator, X, y) + # Heuristic: sklearn scorers produced by `make_scorer` take + # arguments `(estimator, X, y)`. is_sklearn_scorer = True else: metric = metric_func @@ -138,7 +144,7 @@ def _evaluate(self, params): for train_idx, test_idx in self._cv.split(self.y): X_train = None X_test = None - if isinstance(self.y, (list, tuple)): + if isinstance(self.y, list | tuple): y_train = [self.y[i] for i in train_idx] y_test = [self.y[i] for i in test_idx] else: @@ -200,10 +206,10 @@ def _evaluate(self, params): return float(res_float), {"results": {"cv_scores": scores}} def _safe_index(self, obj, idx): - """ - Safely index into `obj` using integer indices. + """Safely index into `obj` using integer indices. - Supports pandas objects with .iloc, numpy arrays/lists, and other indexable types. + Supports pandas objects with ``.iloc``, numpy arrays/lists, and other + indexable types. """ try: return obj.iloc[idx] @@ -218,7 +224,11 @@ def _safe_index(self, obj, idx): @classmethod def get_test_params(cls, parameter_set="default"): - # Return testing parameter settings for the skbase object. + """Return testing parameter settings for the skbase object. + + This returns a list or dict appropriate to construct test instances + for this class. See the skbase test helpers for expected formats. + """ if _check_soft_dependencies("sktime", severity="none"): try: from sktime.annotation.dummy import DummyDetector diff --git a/src/hyperactive/experiment/integrations/tests/test_sktime_detector_experiment.py b/src/hyperactive/experiment/integrations/tests/test_sktime_detector_experiment.py index 97678ac1..375c5f86 100644 --- a/src/hyperactive/experiment/integrations/tests/test_sktime_detector_experiment.py +++ b/src/hyperactive/experiment/integrations/tests/test_sktime_detector_experiment.py @@ -1,4 +1,8 @@ +"""Smoke tests for the sktime detector experiment integration.""" + + def test_sktime_detector_experiment_with_dummy(): + """Run a minimal smoke test using sktime's DummyDetector (if available).""" try: from sktime.annotation.dummy import DummyDetector from sktime.datasets import load_unit_test diff --git a/src/hyperactive/integrations/sktime/_detector.py b/src/hyperactive/integrations/sktime/_detector.py index 1de4d25b..cedcbe63 100644 --- a/src/hyperactive/integrations/sktime/_detector.py +++ b/src/hyperactive/integrations/sktime/_detector.py @@ -107,10 +107,12 @@ def get_test_params(cls, parameter_set="default"): "optimizer": GridSearchSk(param_grid={}), } - + params_more = { "detector": DummyDetector() if DummyDetector is not None else None, - "optimizer": GridSearchSk(param_grid={"strategy": ["most_frequent", "stratified"]}), + "optimizer": GridSearchSk( + param_grid={"strategy": ["most_frequent", "stratified"]} + ), "cv": 2, "scoring": None, "refit": False, diff --git a/src/hyperactive/integrations/sktime/tests/test_detector_integration.py b/src/hyperactive/integrations/sktime/tests/test_detector_integration.py index dbc030b4..3b4732c3 100644 --- a/src/hyperactive/integrations/sktime/tests/test_detector_integration.py +++ b/src/hyperactive/integrations/sktime/tests/test_detector_integration.py @@ -1,4 +1,8 @@ +"""Basic import smoke tests for the sktime detector integration.""" + + def test_detector_integration_imports(): + """Ensure the public integration symbols can be imported.""" from hyperactive.experiment.integrations import SktimeDetectorExperiment from hyperactive.integrations.sktime import TSDetectorOptCv From 4854a3a64ba041accd759be46fac1883e47f9305 Mon Sep 17 00:00:00 2001 From: Arnav Kapoor Date: Fri, 28 Nov 2025 15:33:22 +0530 Subject: [PATCH 6/7] Add _get_score_params to SktimeDetectorExperiment; set object_type tag on TSDetectorOptCv --- .../experiment/integrations/sktime_detector.py | 12 ++++++++++++ src/hyperactive/integrations/sktime/_detector.py | 1 + 2 files changed, 13 insertions(+) diff --git a/src/hyperactive/experiment/integrations/sktime_detector.py b/src/hyperactive/experiment/integrations/sktime_detector.py index f50b36d8..8b8f5707 100644 --- a/src/hyperactive/experiment/integrations/sktime_detector.py +++ b/src/hyperactive/experiment/integrations/sktime_detector.py @@ -269,3 +269,15 @@ def get_test_params(cls, parameter_set="default"): return [params_more] else: return [params_default] + + @classmethod + def _get_score_params(cls): + """Return settings for testing score/evaluate functions. + + The returned list should match the length of ``get_test_params()`` and + contain dictionaries of hyperparameter settings that are valid + inputs for ``score``/``evaluate`` when an instance is created from the + corresponding element of ``get_test_params()``. + """ + # For the simple detector tests, an empty dict of params is adequate. + return [{}] diff --git a/src/hyperactive/integrations/sktime/_detector.py b/src/hyperactive/integrations/sktime/_detector.py index cedcbe63..474587ce 100644 --- a/src/hyperactive/integrations/sktime/_detector.py +++ b/src/hyperactive/integrations/sktime/_detector.py @@ -27,6 +27,7 @@ class TSDetectorOptCv(_DelegatedDetector): "authors": "arnavk23", "maintainers": "fkiraly", "python_dependencies": "sktime", + "object_type": "optimizer", } _delegate_name = "best_detector_" From 0d57d6399492b2816d3eec142a83b35c8a6f76b9 Mon Sep 17 00:00:00 2001 From: Arnav Kapoor Date: Fri, 28 Nov 2025 15:37:03 +0530 Subject: [PATCH 7/7] Fix workflow YAML: avoid heredoc, use python -c to prevent YAML parse error --- .github/workflows/sktime-detector.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/sktime-detector.yml b/.github/workflows/sktime-detector.yml index a2400cae..5f98c2cd 100644 --- a/.github/workflows/sktime-detector.yml +++ b/.github/workflows/sktime-detector.yml @@ -28,10 +28,5 @@ jobs: env: PYTHONPATH: src run: | - python - <<'PY' -import importlib -importlib.import_module('hyperactive.experiment.integrations.sktime_detector') -importlib.import_module('hyperactive.integrations.sktime._detector') -print('imports ok') -PY + python -c "import importlib; importlib.import_module('hyperactive.experiment.integrations.sktime_detector'); importlib.import_module('hyperactive.integrations.sktime._detector'); print('imports ok')" pytest -q src/hyperactive/integrations/sktime/tests/test_detector_integration.py -q || true