From 87cae5f331f150506ba5a157cd2510424af6d65d Mon Sep 17 00:00:00 2001 From: Yushu Li Date: Thu, 22 Oct 2026 17:25:48 -0700 Subject: [PATCH] feat(compat): add an explicit local AlphaEvolve experiment facade --- examples/alphaevolve_interop/README.md | 107 +++++ openevolve/compat/alpha_evolve/__init__.py | 7 + openevolve/compat/alpha_evolve/client.py | 59 +++ openevolve/compat/alpha_evolve/controller.py | 14 + openevolve/compat/alpha_evolve/experiment.py | 203 +++++++++ tests/test_alphaevolve_local_experiment.py | 450 +++++++++++++++++++ 6 files changed, 840 insertions(+) create mode 100644 openevolve/compat/alpha_evolve/__init__.py create mode 100644 openevolve/compat/alpha_evolve/client.py create mode 100644 openevolve/compat/alpha_evolve/controller.py create mode 100644 openevolve/compat/alpha_evolve/experiment.py create mode 100644 tests/test_alphaevolve_local_experiment.py diff --git a/examples/alphaevolve_interop/README.md b/examples/alphaevolve_interop/README.md index 4178d2de4..1b95c4e7b 100644 --- a/examples/alphaevolve_interop/README.md +++ b/examples/alphaevolve_interop/README.md @@ -11,6 +11,113 @@ configuration subset in the other direction. A minimal bundle exporter combines these reverse helpers with explicitly selected Python sources and a local seed check. These helpers do not replace the cloud SDK or implement all of #468. +## Use a local experiment facade + +The first SDK-entrypoint slice is explicitly namespaced under +`openevolve.compat.alpha_evolve`. It provides a small, synchronous lifecycle +backed by OpenEvolve's public `run_evolution` API, without installing Google's +SDK or reading Google credentials. It does **not** register a top-level +`alpha_evolve` alias or change imports in other code. + +This self-contained example runs only a seed by default: + +```python +import asyncio + +from openevolve.compat.alpha_evolve import ( + AlphaEvolveClient, + AlphaEvolveExperiment, + run_controller_loop, +) + + +def evaluate(candidate): + namespace = {} + exec(candidate["content"]["files"][0]["content"], namespace) + return {"scores": {"scores": [{"metric": "quality", "score": namespace["x"]}]}} + + +client = AlphaEvolveClient( + api_base="http://localhost:8000/v1", # Explicit OE endpoint, not Google base_url. + api_key="local-placeholder", # The provider client requires a non-empty key, even for a seed. + objective_metric="quality", + iterations=0, # Seed only. Use 1 to opt in to one generation attempt. +) +experiment = AlphaEvolveExperiment(client, evaluate, max_programs_evaluated=2) +experiment.create_experiment({ + "program_language": "python", + "problem_description": "Maximize x while preserving the interface.", + "run_settings": {"max_programs": 2}, + "generation_settings": {"models": [{"name": "my-local-model"}]}, +}) +experiment.create_initial_program({ + "content": {"files": [{ + "path": "program.py", + "content": "# EVOLVE-BLOCK-START\nx = 1\n# EVOLVE-BLOCK-END\n", + }]}, +}) +experiment.start_experiment() # Blocks until the local OE run completes. +asyncio.run(run_controller_loop(experiment)) # Completion check; no second run. +print(experiment.list_programs(params={"order_by": "quality desc"})) +``` + +Use model names actually served by the explicit endpoint. Set `api_key` on the +local client if required; omitting it retains OE's usual provider behavior, +including `OPENAI_API_KEY`. With `iterations=0`, seed evaluation executes but +no generation is requested. With positive iterations (or `None`, which uses +the imported budget), `start_experiment` invokes the configured provider and +may incur charges. Creation alone does not execute the evaluator, create files +or construct provider clients. Candidate/evaluator code is trusted, with no +additional sandbox. Call `start_experiment` on the main thread outside an +already-running asyncio loop; OpenEvolve installs signal handlers. + +The client accepts the six Google connection arguments (`project_id`, +`location`, `collection`, `engine`, `assistant`, `base_url`) only to ease import +migration. Supplied values produce one warning per construction and are ignored; +they are never used as the local endpoint or printed in that warning. The +existing configuration importer supplies model names/weights, problem text and +generation budget. Unsupported fields, including cloud generation concurrency, +still warn; they are not equated with OE evaluator concurrency. Only +`parallel_evaluation=False` is supported by this initial facade. + +As with the explicit `.env` importer, `max_programs_evaluated` must equal +`run_settings.max_programs` in this slice. **Neither is a strict evaluation cap:** +the resulting OE limit counts generation attempts, with seed evaluation and +retries additional. Distinct budgets are rejected. Explicit `iterations` may +only reduce the generation budget, including zero. No evaluation counts are +invented or presented as SDK statistics. + +The seed must be a mapping containing exactly one non-empty UTF-8 Python source +file; pass SDK objects as `model_dump()` dictionaries. Its path is metadata, +never a write target. Supplied seed scores are ignored with a warning: OE +re-evaluates the exact source. The state progresses from `NEW` to `CREATED`, +`READY`, then `RUNNING` and `COMPLETED` (or `FAILED`). A run cannot be restarted, +including after interruption; construct a new instance instead. Configuration +and candidate validation errors before execution can be corrected in place. +The facade's own copied seed is cleaned up on success and failure. Optional +`output_dir` preserves OE's normal run artifacts; without it, OE uses temporary +output. The completed `EvolutionResult` is available as `experiment.result`. + +`list_programs` returns **only the best result**, using the SDK's +`alphaEvolvePrograms` content/evaluation envelope. It is not an archive listing; +there is no pagination. `order_by=" asc|desc"` is accepted for +entrypoint migration but has no effect on a one-item result. Unknown metrics or +other query parameters raise errors. OE's internal `combined_score` is omitted +unless it was explicitly selected as the objective; numeric objective/feature +scores, including negatives, are retained. An error metric adds a generic +failure insight. `run_controller_loop` only checks successful completion and +accepts no cloud worker or timeout options. + +This is **not yet** enough to execute the original three upstream entrypoints +unchanged: SDK model types, visualization helpers, import aliases, complete +archive access and remote client/controller methods remain outside this slice. +The earlier runner below remains available for unchanged upstream evaluators. +No managed Google Cloud run is implied by a local facade run. + +```bash +python -m unittest discover -s tests -p 'test_alphaevolve_local_experiment.py' -v +``` + ## Evaluate an upstream seed locally From an installed OpenEvolve checkout: diff --git a/openevolve/compat/alpha_evolve/__init__.py b/openevolve/compat/alpha_evolve/__init__.py new file mode 100644 index 000000000..962c1a9f4 --- /dev/null +++ b/openevolve/compat/alpha_evolve/__init__.py @@ -0,0 +1,7 @@ +"""Explicit local experiment facade; does not install an ``alpha_evolve`` alias.""" + +from .client import AlphaEvolveClient +from .controller import run_controller_loop +from .experiment import AlphaEvolveExperiment + +__all__ = ["AlphaEvolveClient", "AlphaEvolveExperiment", "run_controller_loop"] diff --git a/openevolve/compat/alpha_evolve/client.py b/openevolve/compat/alpha_evolve/client.py new file mode 100644 index 000000000..1d70d3dbf --- /dev/null +++ b/openevolve/compat/alpha_evolve/client.py @@ -0,0 +1,59 @@ +"""Local execution options with a small AlphaEvolve-shaped constructor.""" + +import warnings +from pathlib import Path +from typing import Optional, Union + +from openevolve.compat.alphaevolve import adapt_evaluator + + +class AlphaEvolveClient: + """Configure local execution without a Google SDK, credentials or requests. + + ``api_base`` and ``objective_metric`` are explicit OpenEvolve settings. + Cloud arguments are accepted only for entrypoint migration and warn when + supplied; ``base_url`` never becomes the model endpoint. ``iterations`` may + reduce the imported generation budget, including zero for seed-only use. + An omitted key retains OpenEvolve's usual provider/environment behavior. + """ + + def __init__( + self, + project_id=None, + location=None, + collection=None, + engine=None, + assistant=None, + base_url=None, + *, + api_base: str, + objective_metric: str, + api_key: Optional[str] = None, + iterations: Optional[int] = None, + output_dir: Optional[Union[str, Path]] = None, + ): + if not isinstance(api_base, str) or not api_base.strip(): + raise ValueError("api_base must explicitly name the OpenEvolve model endpoint") + # Reuse the evaluator adapter's objective validation without executing it. + adapt_evaluator(lambda candidate: None, objective_metric=objective_metric) + if iterations is not None and ( + isinstance(iterations, bool) or not isinstance(iterations, int) or iterations < 0 + ): + raise ValueError("iterations must be a non-negative integer or None") + if output_dir is not None and not isinstance(output_dir, (str, Path)): + raise TypeError("output_dir must be a path or None") + if any( + value is not None + for value in (project_id, location, collection, engine, assistant, base_url) + ): + warnings.warn( + "Local AlphaEvolve facade ignores Google Cloud connection arguments; " + "api_base selects the OpenEvolve endpoint", + UserWarning, + stacklevel=2, + ) + self.api_base = api_base + self.objective_metric = objective_metric + self.api_key = api_key + self.iterations = iterations + self.output_dir = str(output_dir) if output_dir is not None else None diff --git a/openevolve/compat/alpha_evolve/controller.py b/openevolve/compat/alpha_evolve/controller.py new file mode 100644 index 000000000..afc305a74 --- /dev/null +++ b/openevolve/compat/alpha_evolve/controller.py @@ -0,0 +1,14 @@ +"""Async-shaped completion check for synchronous local experiments.""" + +from .experiment import AlphaEvolveExperiment + + +async def run_controller_loop(experiment: AlphaEvolveExperiment) -> None: + """Check completion without running again, polling queues or creating workers. + + The local ``start_experiment`` already ran OpenEvolve to completion. Cloud + sampler/evaluator/idle-timeout arguments are deliberately unsupported. + """ + if not isinstance(experiment, AlphaEvolveExperiment): + raise TypeError("experiment must be the local AlphaEvolveExperiment") + experiment._require_state("COMPLETED") diff --git a/openevolve/compat/alpha_evolve/experiment.py b/openevolve/compat/alpha_evolve/experiment.py new file mode 100644 index 000000000..f15c7cfaa --- /dev/null +++ b/openevolve/compat/alpha_evolve/experiment.py @@ -0,0 +1,203 @@ +"""One-shot synchronous local experiments using OpenEvolve's public API.""" + +import asyncio +import copy +import inspect +import math +import tempfile +import threading +import uuid +import warnings +from collections.abc import Mapping +from numbers import Real +from pathlib import Path +from typing import Any, Callable, Optional + +from openevolve.api import run_evolution +from openevolve.compat.alphaevolve import adapt_evaluator, config_from_alphaevolve + +from .client import AlphaEvolveClient + + +class AlphaEvolveExperiment: + """A limited SDK-shaped lifecycle, not a cloud client or full SDK replacement. + + Creation only validates inputs. ``start_experiment`` blocks until the local + run finishes and may invoke the explicitly configured model endpoint. + ``list_programs`` exposes only the best result, not the whole OE archive. + Instances are single-use, including after failure or interruption. + """ + + def __init__( + self, + ae_client: AlphaEvolveClient, + evaluator_function: Callable, + max_programs_evaluated: int, + parallel_evaluation: bool = False, + ): + if not isinstance(ae_client, AlphaEvolveClient): + raise TypeError("ae_client must be the local AlphaEvolveClient") + if ( + not callable(evaluator_function) + or inspect.iscoroutinefunction(evaluator_function) + or inspect.iscoroutinefunction(getattr(evaluator_function, "__call__", None)) + ): + raise TypeError("evaluator_function must be a synchronous callable") + if ( + isinstance(max_programs_evaluated, bool) + or not isinstance(max_programs_evaluated, int) + or max_programs_evaluated < 1 + ): + raise ValueError("max_programs_evaluated must be a positive integer") + if parallel_evaluation is not False: + raise ValueError("This local facade supports only parallel_evaluation=False") + self.client = ae_client + self.max_programs_evaluated = max_programs_evaluated + self.experiment_name = None + self.initial_program_name = None + self.result = None + self.state = "NEW" + self._evaluator = adapt_evaluator( + evaluator_function, objective_metric=ae_client.objective_metric + ) + self._config = None + self._source = None + self._source_path = None + + def _require_state(self, expected: str) -> None: + if self.state != expected: + raise RuntimeError(f"Expected experiment state {expected}, got {self.state}") + + def create_experiment(self, config: Mapping) -> None: + """Import the existing supported experiment-config subset; no execution.""" + self._require_state("NEW") + local = config_from_alphaevolve( + config, api_base=self.client.api_base, api_key=self.client.api_key + ) + if local.max_iterations != self.max_programs_evaluated: + raise ValueError( + "max_programs_evaluated must equal run_settings.max_programs in this facade; " + "distinct generation/evaluation budgets are unsupported" + ) + iterations = self.client.iterations + if iterations is not None: + if iterations > local.max_iterations: + raise ValueError("iterations cannot increase the imported generation budget") + local.max_iterations = iterations + warnings.warn( + "Local max_programs_evaluated is a generation-attempt ceiling, not an " + "evaluation/request/cost cap; seed evaluation and retries are additional", + UserWarning, + stacklevel=2, + ) + self._config = local + self.experiment_name = "local/experiments/" + uuid.uuid4().hex + self.state = "CREATED" + + def create_initial_program(self, initial_program: Mapping) -> None: + """Accept one UTF-8 Python source; supplied seed scores are not trusted.""" + self._require_state("CREATED") + if not isinstance(initial_program, Mapping): + raise ValueError("initial_program must be a candidate mapping") + content = initial_program.get("content") + files = content.get("files") if isinstance(content, Mapping) else None + if not isinstance(files, (list, tuple)) or len(files) != 1: + raise ValueError("Only single-file Python candidates are supported") + entry = files[0] + if not isinstance(entry, Mapping): + raise ValueError("Candidate file must be a mapping") + source, path = entry.get("content"), entry.get("path") + if not isinstance(source, str) or not source.strip(): + raise ValueError("Candidate content must be non-empty Python source") + if not isinstance(path, str) or not path.endswith(".py"): + raise ValueError("Candidate path must name a Python .py file") + if entry.get("program_language") not in (None, "python"): + raise ValueError("Only Python candidates are supported") + # Validate without executing; the supplied path is metadata, never a write target. + source.encode("utf-8") + compile(source, "", "exec") + if initial_program.get("evaluation") is not None: + warnings.warn( + "Supplied seed evaluation is ignored; OpenEvolve re-evaluates the seed", + UserWarning, + stacklevel=2, + ) + self._source = source + self._source_path = path + self.initial_program_name = self.experiment_name + "/programs/seed" + self.state = "READY" + + def start_experiment(self) -> None: + """Run once on the main thread, outside an already-running asyncio loop.""" + self._require_state("READY") + if threading.current_thread() is not threading.main_thread(): + raise RuntimeError("Start local experiments on the main thread for OE signal handling") + try: + asyncio.get_running_loop() + except RuntimeError: + pass + else: + raise RuntimeError("Call start_experiment outside a running asyncio loop") + self.state = "RUNNING" + try: + with tempfile.TemporaryDirectory(prefix="openevolve_ae_seed_") as directory: + seed = Path(directory) / "program.py" + seed.write_bytes(self._source.encode("utf-8")) + result = run_evolution( + seed, + self._evaluator, + config=copy.deepcopy(self._config), + output_dir=self.client.output_dir, + cleanup=True, + ) + if result.best_program is None: + raise RuntimeError("Local evolution returned no best program") + self.result = result + except BaseException: + self.state = "FAILED" + raise + self.state = "COMPLETED" + + def list_programs(self, params: Optional[Mapping] = None) -> dict[str, Any]: + """Return the best result in the SDK envelope; no archive or pagination.""" + self._require_state("COMPLETED") + if params is not None and not isinstance(params, Mapping): + raise ValueError("params must be a mapping or None") + params = params or {} + if set(params) - {"order_by"}: + raise ValueError("Only order_by is supported; archive queries are unavailable") + metrics = self.result.metrics + scores = [ + {"metric": metric, "score": float(value)} + for metric, value in metrics.items() + if isinstance(value, Real) + and not isinstance(value, bool) + and math.isfinite(value) + and metric != "error" + and (metric != "combined_score" or metric == self.client.objective_metric) + ] + if "order_by" in params: + order = params["order_by"] + parts = order.rsplit(" ", 1) if isinstance(order, str) else [] + if ( + len(parts) != 2 + or parts[1] not in ("asc", "desc") + or parts[0] not in {score["metric"] for score in scores} + ): + raise ValueError("order_by must name a returned metric and asc or desc") + evaluation = {"scores": {"scores": scores}} + if "error" in metrics: + evaluation["insights"] = { + "insights": [{"label": "OpenEvolve Evaluation Error", "text": "Evaluation failed"}] + } + return { + "alphaEvolvePrograms": [ + { + "name": self.experiment_name + "/programs/" + self.result.best_program.id, + "content": { + "files": [{"path": self._source_path, "content": self.result.best_code}] + }, + "evaluation": evaluation, + } + ] + } diff --git a/tests/test_alphaevolve_local_experiment.py b/tests/test_alphaevolve_local_experiment.py new file mode 100644 index 000000000..8f077c340 --- /dev/null +++ b/tests/test_alphaevolve_local_experiment.py @@ -0,0 +1,450 @@ +"""Local facade boundaries and real offline OpenEvolve execution.""" + +import asyncio +import contextlib +import copy +import importlib.util +import io +import os +import secrets +import signal +import sys +import tempfile +import threading +import unittest +import warnings +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from openevolve.compat.alpha_evolve import ( + AlphaEvolveClient, + AlphaEvolveExperiment, + run_controller_loop, +) +from openevolve.compat.alphaevolve import config_from_alphaevolve + +EXPERIMENT_MODULE = "openevolve.compat.alpha_evolve.experiment" +SEED = "# EVOLVE-BLOCK-START\nx = 1\n# EVOLVE-BLOCK-END\n" + + +def evaluate_candidate(candidate): + namespace = {} + exec(candidate["content"]["files"][0]["content"], namespace) + return { + "scores": { + "scores": [ + {"metric": "quality", "score": namespace["x"]}, + {"metric": "cost", "score": -2.0}, + ] + } + } + + +class OfflineModel: + def __init__(self, config): + self.model = config.name + + async def generate_with_context(self, system_message, messages, **kwargs): + return "<<<<<<< SEARCH\nx = 1\n=======\nx = 2\n>>>>>>> REPLACE" + + +def offline_model(config): + return OfflineModel(config) + + +class TestLocalAlphaEvolveExperiment(unittest.TestCase): + def setUp(self): + self.payload = { + "title": "Local test", + "program_language": "python", + "problem_description": "Maximize x while preserving the interface.", + "run_settings": {"max_programs": 2}, + "generation_settings": {"models": [{"name": "offline-model"}]}, + } + self.candidate = {"content": {"files": [{"path": "source.py", "content": SEED}]}} + + def experiment(self, **client_options): + client = AlphaEvolveClient( + api_base="http://localhost:8000/v1", + objective_metric="quality", + **client_options, + ) + return AlphaEvolveExperiment(client, evaluate_candidate, 2) + + def ready(self, **client_options): + experiment = self.experiment(**client_options) + with self.assertWarnsRegex(UserWarning, "generation-attempt ceiling"): + experiment.create_experiment(self.payload) + experiment.create_initial_program(self.candidate) + return experiment + + def fake_result(self, **overrides): + values = dict( + best_program=SimpleNamespace(id="best-id"), + best_code="x = 2\n", + metrics={"quality": 2, "cost": -2, "combined_score": 2}, + best_score=2, + ) + return SimpleNamespace(**(values | overrides)) + + def complete(self, experiment=None, result=None): + experiment = experiment or self.ready() + with patch(EXPERIMENT_MODULE + ".run_evolution", return_value=result or self.fake_result()): + experiment.start_experiment() + return experiment + + def test_setup_has_no_execution_io_or_google_namespace_side_effects(self): + candidate_before, config_before = copy.deepcopy(self.candidate), copy.deepcopy(self.payload) + sdk_before = { + name: value for name, value in sys.modules.items() if name.startswith("alpha_evolve") + } + with ( + patch("socket.socket.connect", side_effect=AssertionError("network")), + patch("tempfile.TemporaryDirectory", side_effect=AssertionError("filesystem")), + patch(EXPERIMENT_MODULE + ".run_evolution") as evolve, + ): + experiment = self.ready() + evolve.assert_not_called() + self.assertEqual(experiment.state, "READY") + self.assertEqual(self.payload, config_before) + self.assertEqual(self.candidate, candidate_before) + self.assertEqual( + {name: value for name, value in sys.modules.items() if name.startswith("alpha_evolve")}, + sdk_before, + ) + + def test_google_arguments_warn_without_leaking_or_becoming_endpoint(self): + private_value = secrets.token_urlsafe(20) + with self.assertWarns(UserWarning) as caught: + experiment = self.experiment( + project_id=private_value, engine=private_value, base_url=private_value + ) + self.assertNotIn(private_value, str(caught.warning)) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + experiment.create_experiment(self.payload) + self.assertEqual(experiment._config.llm.api_base, "http://localhost:8000/v1") + + def test_invalid_local_options_are_rejected(self): + for options in ( + {"api_base": ""}, + {"api_base": None}, + {"objective_metric": ""}, + {"objective_metric": "error"}, + {"iterations": -1}, + {"iterations": True}, + {"iterations": 1.5}, + {"output_dir": 123}, + ): + with self.subTest(options=options), self.assertRaises((ValueError, TypeError)): + AlphaEvolveClient( + **({"api_base": "local", "objective_metric": "quality"} | options) + ) + + def test_invalid_experiment_options_fail_before_creation(self): + client = self.experiment().client + + async def async_evaluator(candidate): + pass + + class AsyncEvaluator: + __call__ = staticmethod(async_evaluator) + + for evaluator in (None, async_evaluator, AsyncEvaluator()): + with self.subTest(evaluator=evaluator), self.assertRaises(TypeError): + AlphaEvolveExperiment(client, evaluator, 2) + for count in (0, -1, True, 1.5, None): + with self.subTest(count=count), self.assertRaises(ValueError): + AlphaEvolveExperiment(client, evaluate_candidate, count) + with self.assertRaises(TypeError): + AlphaEvolveExperiment(object(), evaluate_candidate, 2) + with self.assertRaisesRegex(ValueError, "parallel_evaluation"): + AlphaEvolveExperiment(client, evaluate_candidate, 2, parallel_evaluation=True) + + def test_config_validation_can_be_corrected_without_consuming_experiment(self): + experiment = self.experiment() + for payload in ({}, dict(self.payload, program_language="cpp")): + with self.subTest(payload=payload), self.assertRaises(ValueError): + experiment.create_experiment(payload) + self.assertEqual(experiment.state, "NEW") + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + experiment.create_experiment(self.payload) + self.assertEqual(experiment.state, "CREATED") + + def test_distinct_evaluation_budget_and_increased_override_are_rejected(self): + experiment = self.experiment() + self.payload["run_settings"]["max_programs"] = 3 + with self.assertRaisesRegex(ValueError, "distinct generation/evaluation"): + experiment.create_experiment(self.payload) + self.payload["run_settings"]["max_programs"] = 2 + with self.assertRaisesRegex(ValueError, "cannot increase"): + self.experiment(iterations=3).create_experiment(self.payload) + + def test_cloud_concurrency_is_warned_and_not_mapped_to_oe_workers(self): + self.payload["run_settings"]["concurrency"] = 100 + experiment = self.experiment(iterations=0) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + experiment.create_experiment(self.payload) + self.assertTrue(any("run_settings.concurrency" in str(item.message) for item in caught)) + self.assertEqual(experiment._config.evaluator.parallel_evaluations, 1) + self.assertEqual(experiment._config.max_iterations, 0) + + def test_supplied_seed_scores_are_ignored_and_sources_are_isolated(self): + experiment = self.experiment() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + experiment.create_experiment(self.payload) + self.candidate["evaluation"] = {"scores": {"scores": [{"metric": "quality", "score": 999}]}} + with self.assertWarnsRegex(UserWarning, "re-evaluates"): + experiment.create_initial_program(self.candidate) + self.candidate["content"]["files"][0]["content"] = "x = 999\n" + self.payload["generation_settings"]["models"][0]["name"] = "changed" + with patch(EXPERIMENT_MODULE + ".run_evolution") as evolve: + + def inspect_seed(path, evaluator, **kwargs): + self.assertEqual(path.read_bytes(), SEED.encode()) + self.assertEqual(evaluator(str(path)).metrics["quality"], 1) + self.assertEqual(kwargs["config"].llm.models[0].name, "offline-model") + return self.fake_result() + + evolve.side_effect = inspect_seed + experiment.start_experiment() + + def test_malformed_multifile_and_non_python_seeds_are_rejected(self): + invalid = ( + None, + {}, + {"content": {"files": []}}, + {"content": {"files": [None]}}, + {"content": {"files": [{}, {}]}}, + {"content": {"files": [{"path": "main.cpp", "content": "x = 1"}]}}, + {"content": {"files": [{"path": "main.py", "content": ""}]}}, + {"content": {"files": [{"path": "main.py", "content": "x ="}]}}, + { + "content": { + "files": [{"path": "main.py", "content": "x = 1", "program_language": "cpp"}] + } + }, + ) + experiment = self.experiment() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + experiment.create_experiment(self.payload) + for candidate in invalid: + with self.subTest(candidate=candidate), self.assertRaises((ValueError, SyntaxError)): + experiment.create_initial_program(candidate) + self.assertEqual(experiment.state, "CREATED") + + def test_seed_path_is_metadata_and_exact_bytes_are_preserved(self): + source = "# café\r\nx = 1\r\n" + self.candidate["content"]["files"][0].update(path="../../outside.py", content=source) + experiment = self.ready() + captured = [] + + def inspect_seed(path, evaluator, **kwargs): + captured.append(path) + self.assertEqual(path.name, "program.py") + self.assertEqual(path.read_bytes(), source.encode("utf-8")) + return self.fake_result() + + with patch(EXPERIMENT_MODULE + ".run_evolution", side_effect=inspect_seed): + experiment.start_experiment() + self.assertFalse(captured[0].exists()) + self.assertEqual( + experiment.list_programs()["alphaEvolvePrograms"][0]["content"]["files"][0]["path"], + "../../outside.py", + ) + + def test_lifecycle_guards_prevent_early_or_duplicate_execution(self): + experiment = self.experiment() + for action in ( + experiment.start_experiment, + experiment.list_programs, + lambda: experiment.create_initial_program(self.candidate), + lambda: asyncio.run(run_controller_loop(experiment)), + ): + with self.assertRaises(RuntimeError): + action() + experiment = self.complete() + for action in ( + experiment.start_experiment, + lambda: experiment.create_experiment(self.payload), + lambda: experiment.create_initial_program(self.candidate), + ): + with self.assertRaises(RuntimeError): + action() + with patch(EXPERIMENT_MODULE + ".run_evolution") as evolve: + asyncio.run(run_controller_loop(experiment)) + asyncio.run(run_controller_loop(experiment)) + evolve.assert_not_called() + + def test_async_and_worker_thread_start_fail_before_consuming_ready_state(self): + experiment = self.ready() + + async def start(): + with self.assertRaisesRegex(RuntimeError, "running asyncio"): + experiment.start_experiment() + + asyncio.run(start()) + errors = [] + + def start_in_thread(): + try: + experiment.start_experiment() + except RuntimeError as error: + errors.append(str(error)) + + thread = threading.Thread(target=start_in_thread) + thread.start() + thread.join(timeout=5) + self.assertEqual(len(errors), 1) + self.assertIn("main thread", errors[0]) + self.assertEqual(experiment.state, "READY") + self.complete(experiment) + + def test_failures_and_interruptions_are_terminal_and_clean_up_seed(self): + for failure in (ValueError("evaluator failed"), KeyboardInterrupt()): + experiment = self.ready() + paths = [] + + def fail(path, evaluator, **kwargs): + paths.append(path) + raise failure + + with ( + self.subTest(failure=type(failure)), + patch(EXPERIMENT_MODULE + ".run_evolution", side_effect=fail), + ): + with self.assertRaises(type(failure)): + experiment.start_experiment() + self.assertEqual(experiment.state, "FAILED") + self.assertFalse(paths[0].exists()) + self.assertIsNone(experiment.result) + with self.assertRaises(RuntimeError): + experiment.start_experiment() + with self.assertRaises(RuntimeError): + asyncio.run(run_controller_loop(experiment)) + + def test_missing_best_program_is_failure_not_completion(self): + experiment = self.ready() + with self.assertRaisesRegex(RuntimeError, "no best program"): + self.complete(experiment, self.fake_result(best_program=None)) + self.assertEqual(experiment.state, "FAILED") + + def test_best_only_envelope_preserves_negative_scores_and_does_not_alias_metrics(self): + experiment = self.complete( + result=self.fake_result(metrics={"quality": -3, "cost": -2, "combined_score": -3}) + ) + response = experiment.list_programs({"order_by": "quality desc"}) + self.assertEqual(len(response["alphaEvolvePrograms"]), 1) + program = response["alphaEvolvePrograms"][0] + self.assertEqual( + program["evaluation"]["scores"]["scores"], + [{"metric": "quality", "score": -3.0}, {"metric": "cost", "score": -2.0}], + ) + program["evaluation"]["scores"]["scores"][0]["score"] = 10 + self.assertEqual(experiment.result.metrics["quality"], -3) + for params in ( + 1, + {"page_size": 3}, + {"order_by": "unknown desc"}, + {"order_by": "quality"}, + {"order_by": None}, + {"order_by": "combined_score desc"}, + ): + with self.subTest(params=params), self.assertRaises(ValueError): + experiment.list_programs(params) + + def test_failed_evaluation_result_retains_failure_insight(self): + experiment = self.complete( + result=self.fake_result( + metrics={"quality": -1e12, "combined_score": -1e12, "error": 0.0} + ) + ) + evaluation = experiment.list_programs()["alphaEvolvePrograms"][0]["evaluation"] + self.assertEqual(evaluation["scores"]["scores"], [{"metric": "quality", "score": -1e12}]) + self.assertIn("Error", evaluation["insights"]["insights"][0]["label"]) + + def test_controller_rejects_other_sdk_and_cloud_worker_arguments(self): + with self.assertRaises(TypeError): + asyncio.run(run_controller_loop(object())) + with self.assertRaises(TypeError): + run_controller_loop(self.complete(), num_evaluators=4) + + def test_documented_seed_example_works_with_real_provider_client_and_no_ambient_key(self): + readme = Path(__file__).resolve().parents[1] / "examples/alphaevolve_interop/README.md" + source = readme.read_text().split("```python\n", 1)[1].split("```", 1)[0] + handlers = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)} + try: + with ( + patch.dict(os.environ, {}, clear=True), + patch("socket.socket.connect", side_effect=AssertionError("network")), + contextlib.redirect_stdout(io.StringIO()), + warnings.catch_warnings(), + ): + warnings.simplefilter("ignore", UserWarning) + namespace = {} + exec(compile(source, str(readme), "exec"), namespace) + self.assertEqual(namespace["experiment"].result.metrics["quality"], 1) + finally: + for sig, handler in handlers.items(): + signal.signal(sig, handler) + + def test_real_seed_only_and_worker_generation_with_file_loaded_evaluator(self): + # File-loaded evaluators must remain callable in OE's subprocess. + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + module_path = root / "local_candidate_evaluator.py" + module_path.write_text( + "def evaluate(candidate):\n" + " namespace = {}\n" + " exec(candidate['content']['files'][0]['content'], namespace)\n" + " return {'scores': {'scores': [{'metric': 'quality', 'score': namespace['x']}]}}\n" + ) + spec = importlib.util.spec_from_file_location("local_candidate_evaluator", module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + # Cloudpickle's normal by-reference mode requires an importable module + # on the worker path. Here, use its by-value mode as the example runner does. + old_handlers = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)} + self.addCleanup( + lambda: [signal.signal(sig, value) for sig, value in old_handlers.items()] + ) + + def offline_config(*args, **kwargs): + config = config_from_alphaevolve(*args, **kwargs) + config.database.num_islands = 1 + for model in config.llm.models + config.llm.evaluator_models: + model.init_client = offline_model + return config + + for iterations, expected in ((0, 1), (1, 2)): + client = AlphaEvolveClient( + api_base="http://localhost:8000/v1", + objective_metric="quality", + iterations=iterations, + output_dir=root / f"run-{iterations}", + ) + experiment = AlphaEvolveExperiment(client, module.evaluate, 2) + with ( + patch( + EXPERIMENT_MODULE + ".config_from_alphaevolve", side_effect=offline_config + ), + warnings.catch_warnings(), + ): + warnings.simplefilter("ignore", UserWarning) + experiment.create_experiment(self.payload) + experiment.create_initial_program(self.candidate) + with patch("socket.socket.connect", side_effect=AssertionError("network")): + experiment.start_experiment() + asyncio.run(run_controller_loop(experiment)) + self.assertEqual(experiment.result.metrics["quality"], expected) + self.assertEqual(experiment.state, "COMPLETED") + self.assertTrue((root / f"run-{iterations}" / "best" / "best_program.py").exists()) + + +if __name__ == "__main__": + unittest.main()