diff --git a/openevolve/cli.py b/openevolve/cli.py index 20c62120e..d1da28137 100644 --- a/openevolve/cli.py +++ b/openevolve/cli.py @@ -78,12 +78,11 @@ async def main_async() -> int: print(f"Error: Evaluation file '{args.evaluation_file}' not found") return 1 + # Load base config from file or defaults + config = load_config(args.config) + # Create config object with command-line overrides - config = None if args.api_base or args.primary_model or args.secondary_model: - # Load base config from file or defaults - config = load_config(args.config) - # Apply command-line overrides if args.api_base: config.llm.api_base = args.api_base @@ -110,7 +109,6 @@ async def main_async() -> int: initial_program_path=args.initial_program, evaluation_file=args.evaluation_file, config=config, - config_path=args.config if config is None else None, output_dir=args.output, ) diff --git a/openevolve/config.py b/openevolve/config.py index 7cc540b49..762356d1f 100644 --- a/openevolve/config.py +++ b/openevolve/config.py @@ -3,7 +3,7 @@ """ import os -from dataclasses import dataclass, field +from dataclasses import asdict, dataclass, field from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union @@ -41,7 +41,7 @@ class LLMModelConfig: # Reproducibility random_seed: Optional[int] = None - + # Reasoning parameters reasoning_effort: Optional[str] = None @@ -75,7 +75,7 @@ class LLMConfig(LLMModelConfig): primary_model_weight: float = None secondary_model: str = None secondary_model_weight: float = None - + # Reasoning parameters (inherited from LLMModelConfig but can be overridden) reasoning_effort: Optional[str] = None @@ -146,7 +146,7 @@ def rebuild_models(self) -> None: # Clear existing models lists self.models = [] self.evaluator_models = [] - + # Re-run model generation logic from __post_init__ if self.primary_model: # Create primary model @@ -205,6 +205,7 @@ class PromptConfig: template_variations: Dict[str, List[str]] = field(default_factory=dict) # Meta-prompting + # Note: meta-prompting features not implemented use_meta_prompting: bool = False meta_prompt_weight: float = 0.1 @@ -254,6 +255,7 @@ class DatabaseConfig: elite_selection_ratio: float = 0.1 exploration_ratio: float = 0.2 exploitation_ratio: float = 0.7 + # Note: diversity_metric fixed to "edit_distance" diversity_metric: str = "edit_distance" # Options: "edit_distance", "feature_based" # Feature map dimensions for MAP-Elites @@ -291,6 +293,7 @@ class DatabaseConfig: embedding_model: Optional[str] = None similarity_threshold: float = 0.99 + @dataclass class EvaluatorConfig: """Configuration for program evaluation""" @@ -300,6 +303,7 @@ class EvaluatorConfig: max_retries: int = 3 # Resource limits for evaluation + # Note: resource limits not implemented memory_limit_mb: Optional[int] = None cpu_limit: Optional[float] = None @@ -309,6 +313,7 @@ class EvaluatorConfig: # Parallel evaluation parallel_evaluations: int = 1 + # Note: distributed evaluation not implemented distributed: bool = False # LLM-based feedback @@ -323,7 +328,7 @@ class EvaluatorConfig: @dataclass class EvolutionTraceConfig: """Configuration for evolution trace logging""" - + enabled: bool = False format: str = "jsonl" # Options: "jsonl", "json", "hdf5" include_code: bool = False @@ -362,6 +367,9 @@ class Config: convergence_threshold: float = 0.001 early_stopping_metric: str = "combined_score" + # Parallel controller settings + max_tasks_per_child: Optional[int] = None + @classmethod def from_yaml(cls, path: Union[str, Path]) -> "Config": """Load configuration from a YAML file""" @@ -377,7 +385,9 @@ def from_dict(cls, config_dict: Dict[str, Any]) -> "Config": # Update top-level fields for key, value in config_dict.items(): - if key not in ["llm", "prompt", "database", "evaluator", "evolution_trace"] and hasattr(config, key): + if key not in ["llm", "prompt", "database", "evaluator", "evolution_trace"] and hasattr( + config, key + ): setattr(config, key, value) # Update nested configs @@ -406,87 +416,7 @@ def from_dict(cls, config_dict: Dict[str, Any]) -> "Config": return config def to_dict(self) -> Dict[str, Any]: - """Convert configuration to a dictionary""" - return { - # General settings - "max_iterations": self.max_iterations, - "checkpoint_interval": self.checkpoint_interval, - "log_level": self.log_level, - "log_dir": self.log_dir, - "random_seed": self.random_seed, - # Component configurations - "llm": { - "models": self.llm.models, - "evaluator_models": self.llm.evaluator_models, - "api_base": self.llm.api_base, - "temperature": self.llm.temperature, - "top_p": self.llm.top_p, - "max_tokens": self.llm.max_tokens, - "timeout": self.llm.timeout, - "retries": self.llm.retries, - "retry_delay": self.llm.retry_delay, - }, - "prompt": { - "template_dir": self.prompt.template_dir, - "system_message": self.prompt.system_message, - "evaluator_system_message": self.prompt.evaluator_system_message, - "num_top_programs": self.prompt.num_top_programs, - "num_diverse_programs": self.prompt.num_diverse_programs, - "use_template_stochasticity": self.prompt.use_template_stochasticity, - "template_variations": self.prompt.template_variations, - # Note: meta-prompting features not implemented - # "use_meta_prompting": self.prompt.use_meta_prompting, - # "meta_prompt_weight": self.prompt.meta_prompt_weight, - }, - "database": { - "db_path": self.database.db_path, - "in_memory": self.database.in_memory, - "population_size": self.database.population_size, - "archive_size": self.database.archive_size, - "num_islands": self.database.num_islands, - "elite_selection_ratio": self.database.elite_selection_ratio, - "exploration_ratio": self.database.exploration_ratio, - "exploitation_ratio": self.database.exploitation_ratio, - # Note: diversity_metric fixed to "edit_distance" - # "diversity_metric": self.database.diversity_metric, - "feature_dimensions": self.database.feature_dimensions, - "feature_bins": self.database.feature_bins, - "migration_interval": self.database.migration_interval, - "migration_rate": self.database.migration_rate, - "random_seed": self.database.random_seed, - "log_prompts": self.database.log_prompts, - }, - "evaluator": { - "timeout": self.evaluator.timeout, - "max_retries": self.evaluator.max_retries, - # Note: resource limits not implemented - # "memory_limit_mb": self.evaluator.memory_limit_mb, - # "cpu_limit": self.evaluator.cpu_limit, - "cascade_evaluation": self.evaluator.cascade_evaluation, - "cascade_thresholds": self.evaluator.cascade_thresholds, - "parallel_evaluations": self.evaluator.parallel_evaluations, - # Note: distributed evaluation not implemented - # "distributed": self.evaluator.distributed, - "use_llm_feedback": self.evaluator.use_llm_feedback, - "llm_feedback_weight": self.evaluator.llm_feedback_weight, - }, - "evolution_trace": { - "enabled": self.evolution_trace.enabled, - "format": self.evolution_trace.format, - "include_code": self.evolution_trace.include_code, - "include_prompts": self.evolution_trace.include_prompts, - "output_path": self.evolution_trace.output_path, - "buffer_size": self.evolution_trace.buffer_size, - "compress": self.evolution_trace.compress, - }, - # Evolution settings - "diff_based_evolution": self.diff_based_evolution, - "max_code_length": self.max_code_length, - # Early stopping settings - "early_stopping_patience": self.early_stopping_patience, - "convergence_threshold": self.convergence_threshold, - "early_stopping_metric": self.early_stopping_metric, - } + return asdict(self) def to_yaml(self, path: Union[str, Path]) -> None: """Save configuration to a YAML file""" diff --git a/openevolve/controller.py b/openevolve/controller.py index 4f27f0437..9bec7a090 100644 --- a/openevolve/controller.py +++ b/openevolve/controller.py @@ -16,15 +16,10 @@ from openevolve.evaluator import Evaluator from openevolve.evolution_trace import EvolutionTracer from openevolve.llm.ensemble import LLMEnsemble -from openevolve.prompt.sampler import PromptSampler from openevolve.process_parallel import ProcessParallelController -from openevolve.utils.code_utils import ( - extract_code_language, -) -from openevolve.utils.format_utils import ( - format_metrics_safe, - format_improvement_safe, -) +from openevolve.prompt.sampler import PromptSampler +from openevolve.utils.code_utils import extract_code_language +from openevolve.utils.format_utils import format_improvement_safe, format_metrics_safe logger = logging.getLogger(__name__) @@ -75,17 +70,11 @@ def __init__( self, initial_program_path: str, evaluation_file: str, - config_path: Optional[str] = None, - config: Optional[Config] = None, + config: Config, output_dir: Optional[str] = None, ): - # Load configuration - if config is not None: - # Use provided Config object directly - self.config = config - else: - # Load from file or use defaults - self.config = load_config(config_path) + # Load configuration (loaded in main_async) + self.config = config # Set up output directory self.output_dir = output_dir or os.path.join( @@ -98,9 +87,10 @@ def __init__( # Set random seed for reproducibility if specified if self.config.random_seed is not None: + import hashlib import random + import numpy as np - import hashlib # Set global random seeds random.seed(self.config.random_seed) @@ -139,7 +129,7 @@ def __init__( self.file_extension = f".{self.file_extension}" # Set the file_suffix in config (can be overridden in YAML) - if not hasattr(self.config, 'file_suffix') or self.config.file_suffix == ".py": + if not hasattr(self.config, "file_suffix") or self.config.file_suffix == ".py": self.config.file_suffix = self.file_extension # Initialize components @@ -175,10 +165,9 @@ def __init__( if not trace_output_path: # Default to output_dir/evolution_trace.{format} trace_output_path = os.path.join( - self.output_dir, - f"evolution_trace.{self.config.evolution_trace.format}" + self.output_dir, f"evolution_trace.{self.config.evolution_trace.format}" ) - + self.evolution_tracer = EvolutionTracer( output_path=trace_output_path, format=self.config.evolution_trace.format, @@ -186,7 +175,7 @@ def __init__( include_prompts=self.config.evolution_trace.include_prompts, enabled=True, buffer_size=self.config.evolution_trace.buffer_size, - compress=self.config.evolution_trace.compress + compress=self.config.evolution_trace.compress, ) logger.info(f"Evolution tracing enabled: {trace_output_path}") else: @@ -305,8 +294,11 @@ async def run( # Initialize improved parallel processing try: self.parallel_controller = ProcessParallelController( - self.config, self.evaluation_file, self.database, self.evolution_tracer, - file_suffix=self.config.file_suffix + self.config, + self.evaluation_file, + self.database, + self.evolution_tracer, + file_suffix=self.config.file_suffix, ) # Set up signal handlers for graceful shutdown @@ -349,7 +341,7 @@ def force_exit_handler(signum, frame): if self.parallel_controller: self.parallel_controller.stop() self.parallel_controller = None - + # Close evolution tracer if self.evolution_tracer: self.evolution_tracer.close() diff --git a/openevolve/process_parallel.py b/openevolve/process_parallel.py index 144449d91..1abc33bef 100644 --- a/openevolve/process_parallel.py +++ b/openevolve/process_parallel.py @@ -8,8 +8,9 @@ import pickle import signal import time -from concurrent.futures import ProcessPoolExecutor, Future, TimeoutError as FutureTimeoutError -from dataclasses import dataclass, asdict +from concurrent.futures import Future, ProcessPoolExecutor +from concurrent.futures import TimeoutError as FutureTimeoutError +from dataclasses import asdict, dataclass from pathlib import Path from typing import Any, Dict, List, Optional, Tuple @@ -37,11 +38,11 @@ class SerializableResult: def _worker_init(config_dict: dict, evaluation_file: str, parent_env: dict = None) -> None: """Initialize worker process with necessary components""" import os - + # Set environment from parent process if parent_env: os.environ.update(parent_env) - + global _worker_config global _worker_evaluation_file global _worker_evaluator @@ -55,8 +56,8 @@ def _worker_init(config_dict: dict, evaluation_file: str, parent_env: dict = Non DatabaseConfig, EvaluatorConfig, LLMConfig, - PromptConfig, LLMModelConfig, + PromptConfig, ) # Reconstruct model objects @@ -125,7 +126,7 @@ def _lazy_init_worker_components(): evaluator_llm, evaluator_prompt, database=None, # No shared database in worker - suffix=getattr(_worker_config, 'file_suffix', '.py'), + suffix=getattr(_worker_config, "file_suffix", ".py"), ) @@ -201,7 +202,7 @@ def _run_iteration_worker( # Parse response based on evolution mode if _worker_config.diff_based_evolution: - from openevolve.utils.code_utils import extract_diffs, apply_diff, format_diff_summary + from openevolve.utils.code_utils import apply_diff, extract_diffs, format_diff_summary diff_blocks = extract_diffs(llm_response) if not diff_blocks: @@ -275,7 +276,14 @@ def _run_iteration_worker( class ProcessParallelController: """Controller for process-based parallel evolution""" - def __init__(self, config: Config, evaluation_file: str, database: ProgramDatabase, evolution_tracer=None, file_suffix: str = ".py"): + def __init__( + self, + config: Config, + evaluation_file: str, + database: ProgramDatabase, + evolution_tracer=None, + file_suffix: str = ".py", + ): self.config = config self.evaluation_file = evaluation_file self.database = database @@ -298,7 +306,7 @@ def _serialize_config(self, config: Config) -> dict: # The asdict() call itself triggers the deepcopy which tries to serialize novelty_llm. Remove it first. config.database.novelty_llm = None - + return { "llm": { "models": [asdict(m) for m in config.llm.models], @@ -334,15 +342,27 @@ def start(self) -> None: # Pass current environment to worker processes import os + import sys + current_env = dict(os.environ) - - # Create process pool with initializer - self.executor = ProcessPoolExecutor( - max_workers=self.num_workers, - initializer=_worker_init, - initargs=(config_dict, self.evaluation_file, current_env), - ) + executor_kwargs = { + "max_workers": self.num_workers, + "initializer": _worker_init, + "initargs": (config_dict, self.evaluation_file, current_env), + } + if sys.version_info >= (3, 11): + logger.info(f"Set max {self.config.max_tasks_per_child} tasks per child") + executor_kwargs["max_tasks_per_child"] = self.config.max_tasks_per_child + elif self.config.max_tasks_per_child is not None: + logger.warn( + "max_tasks_per_child is only supported in Python 3.11+. " + "Ignoring max_tasks_per_child and using spawn start method." + ) + executor_kwargs["mp_context"] = mp.get_context("spawn") + + # Create process pool with initializer + self.executor = ProcessPoolExecutor(**executor_kwargs) logger.info(f"Started process pool with {self.num_workers} processes") def stop(self) -> None: @@ -426,7 +446,9 @@ async def run_evolution( completed_iterations = 0 # Island management - programs_per_island = self.config.database.programs_per_island or max(1, max_iterations // (self.config.database.num_islands * 10)) + programs_per_island = self.config.database.programs_per_island or max( + 1, max_iterations // (self.config.database.num_islands * 10) + ) current_island_counter = 0 # Early stopping tracking @@ -480,15 +502,19 @@ async def run_evolution( # Store artifacts if result.artifacts: self.database.store_artifacts(child_program.id, result.artifacts) - + # Log evolution trace if self.evolution_tracer: # Retrieve parent program for trace logging - parent_program = self.database.get(result.parent_id) if result.parent_id else None + parent_program = ( + self.database.get(result.parent_id) if result.parent_id else None + ) if parent_program: # Determine island ID - island_id = child_program.metadata.get("island", self.database.current_island) - + island_id = child_program.metadata.get( + "island", self.database.current_island + ) + self.evolution_tracer.log_trace( iteration=completed_iteration, parent_program=parent_program, @@ -500,7 +526,7 @@ async def run_evolution( metadata={ "iteration_time": result.iteration_time, "changes": child_program.metadata.get("changes", ""), - } + }, ) # Log prompts @@ -590,8 +616,10 @@ async def run_evolution( # Check target score if target_score is not None and child_program.metrics: - if ('combined_score' in child_program.metrics and - child_program.metrics['combined_score'] >= target_score): + if ( + "combined_score" in child_program.metrics + and child_program.metrics["combined_score"] >= target_score + ): logger.info( f"Target score {target_score} reached at iteration {completed_iteration}" ) @@ -701,8 +729,7 @@ def _submit_iteration( # Use thread-safe sampling that doesn't modify shared state # This fixes the race condition from GitHub issue #246 parent, inspirations = self.database.sample_from_island( - island_id=target_island, - num_inspirations=self.config.prompt.num_top_programs + island_id=target_island, num_inspirations=self.config.prompt.num_top_programs ) # Create database snapshot