diff --git a/docs/index.html b/docs/index.html index a025470..978566e 100644 --- a/docs/index.html +++ b/docs/index.html @@ -24,1237 +24,3831 @@

OPL – Optimisation problem library

Submit problems and corrections on GitHub with pull requests / issues, through the Google form, or by email: koen.van.der.blom@cwi.nl

- +
+
+
+ Toggle Visible Columns +
+ + +
+
+ +
+
+
+
+
- - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - + + + + - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - + - - - - - + - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + - - - - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - - - - - - - - - - - - - + - - - - - - - - - - - - - - + + + + + - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + - - - - - + + + + + - - - + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + + + - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - + + + + - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - + + + + - - - - - - - - - - - - + - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - + + + + - - - - - - - - - - - - + - - - - - - + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + - - - - - - + + + + - - - - - - - - - - - - + - - - - - - - - - - - - + + + - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - - - - - - - - - - - - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + -
nametextual descriptionsuite/generator/singleobjectivesdimensionalityvariable typeconstraintsdynamicnoisemulti-fidelitysource (real-world/artificial)referenceimplementationIDNameTypeVariable TypesTotal VariablesObjectivesPropertiesConstraint TypesTotal ConstraintsDynamicsNoisePartial EvaluationsIndependent ObjectivesFidelity LevelsFull NameDescriptionTagsReferencesImplementationsModalityExamplesSourceBinary VarsCategorical VarsContinuous VarsInteger VarsImplementation NamesImplementation LanguagesImplementation Evaluation TimesImplementation LinksImplementation DescriptionsImplementation RequirementsHard Box ConstraintsSoft Box ConstraintsHard Linear ConstraintsSoft Linear ConstraintsHard Function ConstraintsSoft Function Constraints
BBOBsuite1scalable
fn_atoATOProblem continuousnonononohttps://doi.org/10.1080/10556788.2020.1808977https://github.com/numbbo/coco
BBOB-biobjsuite10 22-40continuousnononono https://doi.org/10.48550/arXiv.1604.00359https://github.com/numbbo/coco
BBOB-noisy suite1scalablecontinuousnonoyesno https://hal.inria.fr/inria-00369466https://web.archive.org/web/20210416065610/https://coco.gforge.inria.fr/doku.php?id=downloads
BBOB-largescale suite120-640continuousnonono no https://doi.org/10.48550/arXiv.1903.06396https://github.com/numbbo/coco
BBOB-mixint suite15-160integer;continuous;mixednononono https://doi.org/10.1145/3321707.3321868https://github.com/numbbo/coco
BBOB-biobj-mixintParameters of the Modules of the Automatic Train Operation are optimized; two objectives: minimizing energy consumption and minimizing driving duration. suite25-160integer;continuous;mixednononono https://doi.org/10.1145/3321707.3321868https://github.com/numbbo/coco
BBOB-constrained suite12-40continuousyesnononounimodal http://numbbo.github.io/coco-doc/bbob-constrained/https://github.com/numbbo/coco
MOreporeal-world suite2?combinatorial???no 10 https://github.com/MCDMSociety/MOrepo
ZDT suite2scalablecontinuous;binarynononono https://doi.org/10.1162/106365600568202https://github.com/anyoptimization/pymoo
DTLZ suite2+scalablecontinuousnononono https://doi.org/10.1109/CEC.2002.1007032https://pymoo.org/problems/many/dtlz.html
WFG suite2+scalablecontinuousnononono https://doi.org/10.1109/TEVC.2005.861417https://pymoo.org/problems/many/wfg.html
CDMP suite2+scalablecontinuousyes??no https://doi.org/10.1145/3321707.3321878?
SDP suite2+scalablecontinuousnoyes?no https://doi.org/10.1109/TCYB.2019.2896021?
MaOP suite2+scalablecontinuousnono?no https://doi.org/10.1016/j.swevo.2019.02.003?
BP
fn_building_spatialBuilding spatial designProblemcontinuous | binary>=22 suite2+scalablecontinuousnono?nobox | unknown>=2 https://doi.org/10.1109/CEC.2019.8790277?
GPD generator2+scalablecontinuousoptionalnooptional no https://doi.org/10.1016/j.asoc.2020.106139?
ETMOF suite2-5025-10000continuousnoyesnono https://doi.org/10.48550/arXiv.2110.08033https://github.com/songbai-liu/etmo
MMOPPOptimise the spatial layout of a building to minimise energy consumption for climate control and minimise the strain on the structure. Many hard constraints; mixed-variable (continuous+binary); expensive evaluations. suite2-7??yesnononoBuilding spatial design, https://hdl.handle.net/1887/81789impl_bso_toolbox http://www5.zzu.edu.cn/system/_content/download.jsp?urltype=news.DownloadAttachUrl&owner=1327567121&wbfileid=4764412http://www5.zzu.edu.cn/ecilab/info/1036/1251.htm
CFDexpensive evaluations 30s-15msuite1-2scalable?yesnononoreal worldhttps://doi.org/10.1007/978-3-319-99259-4_24https://bitbucket.org/arahat/cfd-test-problem-suite
GBEAexpensive evaluations 5s-35s, RW-GAN-Mario and TopTrumps are part of GBEAsuite1-2scalablecontinuousnonoyesnoreal worldhttps://doi.org/10.1145/3321707.3321805https://github.com/ttusar/coco-gbea
Car structure54 constraintssuite2144-222discreteyesnononoreal worldhttps://doi.org/10.1145/3205651.3205702http://ladse.eng.isas.jaxa.jp/benchmark/
EMO2017 suite24-24continuousnonononoreal worldhttps://www.ini.rub.de/PEOPLE/glasmtbl/projects/bbcomp/https://www.ini.rub.de/PEOPLE/glasmtbl/projects/bbcomp/downloads/realworld-problems-bbcomp-EMO-2017.zip
JSEC2019expensive evaluations 3s; 22 constraintssingle1-532continuousyesnononoreal worldhttp://www.jpnsec.org/files/competition2019/EC-Symposium-2019-Competition-English.htmlhttp://www.jpnsec.org/files/competition2019/EC-Symposium-2019-Competition-English.html
REreal-world>=1 suite2-92-7continuous;integer;mixednonononoreal world likehttps://doi.org/10.1016/j.asoc.2020.106078https://github.com/ryojitanabe/reproblems
CRE>=1 suite2-53-7continuous;integer;mixedyesnononoreal world likehttps://doi.org/10.1016/j.asoc.2020.106078https://github.com/ryojitanabe/reproblems
Radar waveformBSO-toolboxC++~1s (smallest) to ~40s (larger)https://github.com/TUe-excellent-buildings/BSO-toolboxBuilding Spatial Design toolbox (TU/e) single94-12integeryesnononoreal worldhttps://doi.org/10.1007/978-3-540-70928-2_53http://code.evanhughes.org/
MF2>=1 suite11-ncontinuousnononoyes https://doi.org/10.21105/joss.02049https://github.com/sjvrijn/mf2
AMVOP suite1scalablemixed continuous+ordinal+categorical+bothnononono https://doi.org/10.1109/TEVC.2013.2281531?
RWMVOP suite1scalablecontinuous;mixed continuous+ordinal+categorical+bothyesnononoreal worldhttps://doi.org/10.1109/TEVC.2013.2281531?
SBOX-COSTproblems from BBOB but allows instances with the optimum close to the boundarysuite1scalable
fn_convex_dtlz2Convex DTLZ2Problem continuousnononono>=1[10, 2, 3, 4, 5, 6, 7, 8, 9]Variant of DTLZ2 with a convex Pareto front (instead of concave)Convex DTLZ2, https://doi.org/10.1109/TEVC.2013.2281535>=1 https://doi.org/10.48550/arXiv.2305.12221https://github.com/IOHprofiler/IOHexperimenter/
ρMNK-Landscapestunable variable and objective dimensions; tunable multimodality and correlation between objectivesgeneratorscalablescalablebinarynononono https://doi.org/10.1016/j.ejor.2012.12.019https://gitlab.com/aliefooghe/mocobench/
mUBQPtunable variable and objective dimensions; tunable density and correlation between objectivesgeneratorscalablescalablebinarynononono https://doi.org/10.1016/j.asoc.2013.11.008https://gitlab.com/aliefooghe/mocobench/
ρmTSPtunable variable and objective dimensions; tunable instance type (euclidian/random); tunable correlation between objectivesgeneratorscalablescalablepermutationsno (apart from being permutations)nonono https://doi.org/10.1007/978-3-319-45823-6_40https://gitlab.com/aliefooghe/mocobench/
CEC2015-DMOO suite2-3?continuous?yesnono Benchmark Functions for CEC 2015 Special Session and Competition on Dynamic Multi-objective Optimization
EalainReal-world-like, easily extensible to increase complexitygenerator1+scalablecontinuous,binary,integeroptionaloptionalnooptionalReal-world-likehttps://doi.org/10.1145/3638530.3654299https://github.com/qrenau/Ealain
MA-BBOBGenerator that creates affine combinations of BBOB functionsgenerator
fn_emdoElectric Motor Design OptimizationProblemcontinuous | integer26 1scalablecontinuousnonononoisyunknown | box>=14noisy noartificialhttps://doi.org/10.1145/3673908https://github.com/IOHprofiler/IOHexperimenter/blob/master/example/Competitions/MA-BBOB/Example_MABBOB.ipynbElectric Motor Design Optimization# Goal\nFind a design of a synchronous electric motor for power steering systems that minimizes costs and satisfies all constraints.\n\n# Motivation\nChallenging to find good solutions in a limited time.\n\n# Key Challenges\n* Time-consuming solution evaluation\n* Highly-constrained problem\n* Constraints are multimodal\n\nThis is not an available problem, but could be interesting to show to researchers which difficulties appear in real-world problems.A Multi-Step Evaluation Process in Electric Motor Design, Tea Tušar; Peter Korošec; Bogdan Filipič, https://dis.ijs.si/tea/Publications/Tusar23Multistep.pdfimpl_emdomultimodalreal-world1313Electric Motor Design OptimizationPython8 minutesNot publicly available>=1
MPM2nonlinear nonseparable nonsymmetric; scalable in terms of time to evaluate the objective functiongenerator
fn_fleetoptFleetOptProbleminteger{54, 13208} 1scalablecontinuousnononono https://ls11-www.cs.tu-dortmund.de/_media/techreports/tr15-01.pdfhttps://github.com/jakobbossek/smoof/blob/master/inst/mpm2.py
Convex DTLZ2Variant of DTLZ2 with a convex Pareto front (instead of concave)single2+scalablecontinuousnonononounknown>=1yesUK healthcare organisation fleet optimisation: reduce the fleet of non-emergency healthcare trip vehicles while still ensuring all trips can be covered. Bilevel: upper level 54 vars, lower level 13208 vars.FleetOpt, https://dl.acm.org/doi/abs/10.1145/3638530.3664137real-world{54, 13208} https://doi.org/10.1109/TEVC.2013.2281535?
Inverted DTLZ1Variant of DTLZ1 with an inverted Pareto frontsingle2+scalablecontinuousnononono https://doi.org/10.1109/TEVC.2013.2281534?
Minus DTLZVariant of DTLZ that minimises the inverse of the base DTLZ functionssuite2+scalablecontinuousnononono https://doi.org/10.1109/TEVC.2016.2587749?
Minus WFGVariant of WFG that minimises the inverse of the base WFG functionssuite2+scalablecontinuousnononono https://doi.org/10.1109/TEVC.2016.2587749?
L1-ZDTVariant of ZDT with linkages between variables within one of two groups but not between variables in a different group; Linear recombination operators can potentially take advantage of the problem structuresuite2scalablecontinuous;binarynononono https://doi.org/10.1145/1143997.1144179?
L2-ZDTVariant of ZDT with linkages between all variables; Linear recombination operators can potentially take advantage of the problem structuresuite
fn_gasolineGasoline direct injection engine designProbleminteger | continuous14 2scalablecontinuous;binarynonononomulti-fidelityunknown5[1, 2]Multi-objective optimization to minimize fuel consumption and NOx emissions over a two-minute dynamic duty cycle, subject to five constraints (turbine inlet temperature, knock occurrences, peak cylinder pressure, peak cylinder pressure rise, total work). Seven decision variables cover hardware choices and engine control parameters.Gasoline direct injection engine design, https://doi.org/10.1016/j.ejor.2022.08.032impl_gasolinereal-world77Gasoline direct injection engine designMatlab Simulink / Wave RThttps://doi.org/10.1016/j.ejor.2022.08.032Proprietary Matlab Simulink + Wave RT co-simulation https://doi.org/10.1145/1143997.1144179?
L3-ZDTVariant of L2-ZDT using a mapping to prevent linear recombination operators from potentially taking advantage of the problem structuresuite
fn_invdeceptive_deceptive_rotellInverseDeceptiveTrap+RotatedEllipsoid / DeceptiveTrap+RotatedEllipsoidProblemcontinuous | binary>=2 2scalablecontinuous;binarynononono https://doi.org/10.1145/1143997.1144179?
L2-DTLZVariant of DTLZ2 and DTLZ3 with linkages between all variables; Linear recombination operators can potentially take advantage of the problem structuresuite2+scalablecontinuousnononono https://doi.org/10.1145/1143997.1144179?
L3-DTLZVariant of L2-DTLZ using a mapping to prevent linear recombination operators from potentially taking advantage of the problem structuresuite2+scalablecontinuousnononono https://doi.org/10.1145/1143997.1144179?
CEC2018 DT - CEC2018 Competition on Dynamic Multiobjective Optimisation14 problems. Time-dependent: Pareto front/Pareto set geometry; irregular Pareto front shapes; variable-linkage; number of disconnected Pareto front segments; etc.suite2 or 3scalable??noyesnonoMixed-variable multi-objective test problems, https://doi.org/10.1145/3449726.3459521 artificialhttps://www.academia.edu/download/94499025/TR-CEC2018-DMOP-Competition.pdfhttps://pymoo.org/problems/dynamic/df.html
MODAct - multiobjective design of actuatorsRealistic Constrained Multi-Objective Optimization Benchmark Problems from Design. Need the https://github.com/epfl-lamd/modact package installed; evaluation times around 20mssuite2 3 4 or 520mixed; integer and continuousyesnononoreal-worldhttps://doi.org/10.1109/TEVC.2020.3020046https://pymoo.org/problems/constrained/modact.html
IOHClusteringSet of benchmark problems from clustering: optimization task is selecting cluster centers for a given set of data, with the number of clusters defining problem dimensionality. Includes both a suite and a generator. Based on ML clustering datasetssuite; generator1scalablecontinuousnonononoartificial, but based on real datahttps://arxiv.org/pdf/2505.09233https://github.com/IOHprofiler/IOHClustering>=1>=1
GNBG-IIGeneralized Numerical Benchmark Generator (version 2). Also in IOH https://github.com/IOHprofiler/IOHGNBGsuite; generator1scalable
fn_inverted_dtlz1Inverted DTLZ1Problem continuousnonononoartificialhttps://dl.acm.org/doi/pdf/10.1145/3712255.3734271https://github.com/rohitsalgotra/GNBG-II>=1[10, 2, 3, 4, 5, 6, 7, 8, 9]Variant of DTLZ1 with an inverted Pareto frontInverted DTLZ1, https://doi.org/10.1109/TEVC.2013.2281534>=1
fn_jsec2019JSEC2019Problemcontinuous32[1, 2, 3, 4, 5]unknown22expensive evaluations 3s; 22 constraintsJPNSEC EC-Symposium 2019 competition, http://www.jpnsec.org/files/competition2019/EC-Symposium-2019-Competition-English.htmlimpl_jsec2019real-world32JSEC 2019 competition3shttp://www.jpnsec.org/files/competition2019/EC-Symposium-2019-Competition-English.htmlJPNSEC EC-Symposium 2019 competition problem
fn_onemax_sphere_deceptive_rotellOnemax+Sphere / DeceptiveTrap+RotatedEllipsoidProblemcontinuous | binary>=22Mixed-variable multi-objective test problems, https://doi.org/10.1145/3449726.3459521artificial>=1>=1
fn_onemax_sphere_zeromax_sphereOnemax+Sphere / Zeromax+SphereProblemcontinuous | binary>=22Onemax+Sphere / Zeromax+Sphere, https://doi.org/10.1145/3449726.3459521artificial>=1>=1
fn_radar_waveformRadar waveformProbleminteger4-129unknown>=1Radar waveform design, https://doi.org/10.1007/978-3-540-70928-2_53impl_radar_waveformreal-world4-12Evan Hughes radar waveform codehttp://code.evanhughes.org/Radar waveform design reference implementation
gen_beaconBEACONGeneratorcontinuous>=12box0noContinuous Bi-objective Benchmark problems with Explicit Adjustable COrrelatioN controlGenerator for bi-objective benchmark problems with explicitly controlled correlations in continuous spaces. Multimodal with random structure.BEACON, https://dl.acm.org/doi/10.1145/3712255.3734303impl_beaconmultimodalartificial>=1BEACONPythonnegligiblehttps://github.com/Stebbet/BEACON/Continuous Bi-objective Benchmark with Explicit Adjustable COrrelatioN control0
gen_bono_benchBONO-BenchGeneratorcontinuous>=12box>=1noBi-objective Numerical Optimization BenchmarkBi-objective problem generator and suite with scalable continuous decision space. Features complex problem properties and Pareto front approximations with error guarantees for the hypervolume and exact R2 indicators.impl_bonobenchmultimodalartificial>=1BONO-BenchPythonhttps://github.com/schaepermeier/bonobenchBi-objective Numerical Optimization Benchmark (BONO-Bench)>=1
gen_ealainEalainGeneratorcontinuous | integer | binary>=3[1, 10, 2, 3, 4, 5, 6, 7, 8, 9]dynamic | multi-fidelityunknown>=1optional[1, 2]Real-world-like, easily extensible to increase complexityEalain, https://doi.org/10.1145/3638530.3654299impl_ealainreal-world-like>=1>=1>=1Ealainhttps://github.com/qrenau/EalainReal-world-like extensible benchmark problem generator
gen_gnbgGNBGGeneratorcontinuous>=11Generator counterpart of GNBG.GNBG, https://arxiv.org/abs/2312.07083impl_gnbgartificial>=1GNBG Generatorhttps://github.com/Danial-Yazdani/GNBG-GeneratorGeneralized Numerical Benchmark Generator
gen_gnbg_iiGNBG-IIGeneratorcontinuous>=11Generator counterpart of GNBG-II.GNBG-II, https://dl.acm.org/doi/pdf/10.1145/3712255.3734271["impl_gnbg_ii", "impl_iohgnbg"]artificial>=1IOHGNBG | GNBG-IIhttps://github.com/IOHprofiler/IOHGNBG | https://github.com/rohitsalgotra/GNBG-IIIOHprofiler version of GNBG | Generalized Numerical Benchmark Generator version 2
gen_gpdGPDGeneratorcontinuous>=1[10, 2, 3, 4, 5, 6, 7, 8, 9]noisyunknown>=1optionalGPD generator, https://doi.org/10.1016/j.asoc.2020.106139>=1
gen_iohclusteringIOHClusteringGeneratorcontinuous>=11Generator counterpart of the IOHClustering suite.IOHClustering, https://arxiv.org/pdf/2505.09233impl_iohclusteringmultimodalartificial-from-real-data>=1IOHClusteringhttps://github.com/IOHprofiler/IOHClusteringClustering-based optimization benchmark built on ML datasets
gen_ma_bbobMA-BBOBGeneratorcontinuous>=11Generator that creates affine combinations of BBOB functionsMA-BBOB, https://doi.org/10.1145/3673908["impl_iohexperimenter", "impl_ma_bbob"]multimodalartificial>=1MA-BBOB (IOHexperimenter) | IOHexperimenterC++/Pythonhttps://github.com/IOHprofiler/IOHexperimenter/blob/master/example/Competitions/MA-BBOB/Example_MABBOB.ipynb | https://github.com/IOHprofiler/IOHexperimenterExample notebook for MA-BBOB in IOHexperimenter | IOHprofiler experimenter framework
gen_mpm2MPM2Generatorcontinuous>=11nonlinear nonseparable nonsymmetric; scalable in terms of time to evaluate the objective functionMPM2 technical report TR15-01, https://ls11-www.cs.tu-dortmund.de/_media/techreports/tr15-01.pdfimpl_mpm2multimodal>=1MPM2 (smoof)Pythonhttps://github.com/jakobbossek/smoof/blob/master/inst/mpm2.pyPython implementation of MPM2 distributed with smoof
gen_mubqpmUBQPGeneratorbinary>=1[1, 10, 2, 3, 4, 5, 6, 7, 8, 9]tunable variable and objective dimensions; tunable density and correlation between objectivesmUBQP benchmark, https://doi.org/10.1016/j.asoc.2013.11.008impl_mocobench["multimodal", "quadratic"]>=1mocobenchC++https://gitlab.com/aliefooghe/mocobench/Multi-objective combinatorial optimization benchmark
gen_puboiPUBOiGeneratorbinary>=11noPolynomial Unconstrained Binary Optimization with tunable importanceA benchmark in which variable importance is tunable, based on the Walsh function.PUBOi, https://link.springer.com/chapter/10.1007/978-3-031-04148-8_12impl_puboiartificial>=1PUBO Importance BenchmarkPython / C++https://gitlab.com/verel/pubo-importance-benchmarkA benchmark in which variable importance is tunable, based on the Walsh function
gen_randoptgenRandOptGenGeneratorcontinuous | integer | binary>=3[1, 10, 2, 3, 4, 5, 6, 7, 8, 9]noRandOptGenA Unified Random Problem Generator for Single- and Multi-Objective Optimization Problems with Mixed-Variable Input Spaces.impl_randoptgenmultimodalartificial>=1>=1>=1RandOptGenPythonmillisecondshttps://github.com/MALEO-research-group/RandOptGen | https://doi.org/10.1145/3712256.3726478Unified Random Problem Generator for Single- and Multi-Objective Optimization with Mixed-Variable Input Spaces
gen_rho_mnk_landscapesρMNK-LandscapesGeneratorbinary>=1[1, 10, 2, 3, 4, 5, 6, 7, 8, 9]tunable variable and objective dimensions; tunable multimodality and correlation between objectivesOn the design of multi-objective evolutionary algorithms based on NK-landscapes, https://doi.org/10.1016/j.ejor.2012.12.019impl_mocobenchmultimodal>=1mocobenchC++https://gitlab.com/aliefooghe/mocobench/Multi-objective combinatorial optimization benchmark
gen_rho_mtspρmTSPGeneratorunknown>=1[1, 10, 2, 3, 4, 5, 6, 7, 8, 9]tunable variable and objective dimensions; tunable instance type (euclidean/random); tunable correlation between objectivesOn the impact of multi-objective scalability for the ρmTSP, https://doi.org/10.1007/978-3-319-45823-6_40impl_mocobench["multimodal", "quadratic"]mocobenchC++https://gitlab.com/aliefooghe/mocobench/Multi-objective combinatorial optimization benchmark
gen_wmodelW-modelGeneratorbinary>=11Tunable generator for binary optimization based on several difficulty featuresW-model, https://dl.acm.org/doi/abs/10.1145/3205651.3208240impl_wmodelartificial>=1BBDOB W-Modelhttps://github.com/thomasWeise/BBDOB_W_ModelTunable generator for binary optimization
suite_amvopAMVOPSuitecontinuous | integer | categorical>=31AMVOP, https://doi.org/10.1109/TEVC.2013.2281531multimodal>=1>=1>=1
suite_bbobBBOBSuitecontinuous>=11COCO: a platform for comparing continuous optimizers in a black-box setting, https://doi.org/10.1080/10556788.2020.1808977impl_cocomultimodal>=1COCO frameworkC/Pythonhttps://github.com/numbbo/cocoComparing Continuous Optimizers: black-box optimization benchmarking platform
suite_bbob_biobjBBOB-biobjSuitecontinuous2-402BBOB bi-objective test suite, https://doi.org/10.48550/arXiv.1604.00359impl_cocomultimodal2-40COCO frameworkC/Pythonhttps://github.com/numbbo/cocoComparing Continuous Optimizers: black-box optimization benchmarking platform
suite_bbob_biobj_mixintBBOB-biobj-mixintSuitecontinuous | integer10-3202BBOB bi-objective mixed-integer test suite, https://doi.org/10.1145/3321707.3321868impl_cocomultimodal5-1605-160COCO frameworkC/Pythonhttps://github.com/numbbo/cocoComparing Continuous Optimizers: black-box optimization benchmarking platform
suite_bbob_constrainedBBOB-constrainedSuitecontinuous2-401unknown>=1bbob-constrained documentation, http://numbbo.github.io/coco-doc/bbob-constrained/impl_cocomultimodal2-40COCO frameworkC/Pythonhttps://github.com/numbbo/cocoComparing Continuous Optimizers: black-box optimization benchmarking platform
suite_bbob_largescaleBBOB-largescaleSuitecontinuous20-6401BBOB large-scale test suite, https://doi.org/10.48550/arXiv.1903.06396impl_cocomultimodal20-640COCO frameworkC/Pythonhttps://github.com/numbbo/cocoComparing Continuous Optimizers: black-box optimization benchmarking platform
suite_bbob_mixintBBOB-mixintSuitecontinuous | integer10-3201BBOB mixed-integer test suite, https://doi.org/10.1145/3321707.3321868impl_cocomultimodal5-1605-160COCO frameworkC/Pythonhttps://github.com/numbbo/cocoComparing Continuous Optimizers: black-box optimization benchmarking platform
suite_bbob_noisyBBOB-noisySuitecontinuous>=11noisynoisyReal-parameter black-box optimization benchmarking: noisy functions definitions, https://hal.inria.fr/inria-00369466impl_coco_legacymultimodal>=1COCO legacy (bbob-noisy)C/Pythonhttps://web.archive.org/web/20210416065610/https://coco.gforge.inria.fr/doku.php?id=downloadsArchived COCO download page that hosted the bbob-noisy suite
suite_bpBPSuitecontinuous>=1[10, 2, 3, 4, 5, 6, 7, 8, 9]noisyunknownBP benchmark, https://doi.org/10.1109/CEC.2019.8790277>=1
suite_brachytherapyBrachytherapy treatment planningSuitecontinuous100-500[2, 3]multi-fidelityunknown>=1yes[1, 2]Brachytherapy treatment planningTreatment planning for internal radiation therapy. Multi-objective with aggregated objectives; no public source code.Brachytherapy treatment planning, https://www.sciencedirect.com/science/article/pii/S1538472123016781multimodalreal-world100-500
suite_car_structureCar structureSuiteinteger144-2222unknown5454 constraintsCar structure design benchmark, https://doi.org/10.1145/3205651.3205702impl_car_structurereal-world144-222Car-structure benchmarkhttp://ladse.eng.isas.jaxa.jp/benchmark/JAXA LADSE benchmark problems
suite_cdmpCDMPSuitecontinuous>=1[10, 2, 3, 4, 5, 6, 7, 8, 9]dynamic | noisyunknown>=1unknownunknownCDMP benchmark, https://doi.org/10.1145/3321707.3321878>=1
suite_cec2013CEC2013Suitecontinuous>=11suite used for cec2013 competition. Also in IOH.CEC2013 definitions, https://peerj.com/articles/cs-2671/CEC2013.pdf["impl_cec2013", "impl_iohexperimenter"]artificial>=1CEC2013 reference code | IOHexperimenterC++/Pythonhttps://github.com/P-N-Suganthan/CEC2013 | https://github.com/IOHprofiler/IOHexperimenterSuganthan's reference implementation | IOHprofiler experimenter framework
suite_cec2015_dmooCEC2015-DMOOSuitecontinuous0[2, 3]dynamicunknown>=1dynamicBenchmark Functions for CEC 2015 Special Session and Competition on Dynamic Multi-objective Optimization0
suite_cec2018_dtCEC2018 DTSuiteunknown>=1[2, 3]dynamicdynamicCEC2018 Competition on Dynamic Multiobjective Optimisation14 problems. Time-dependent: Pareto front/Pareto set geometry; irregular Pareto front shapes; variable-linkage; number of disconnected Pareto front segments; etc.CEC2018 DMOP Competition TR, https://www.academia.edu/download/94499025/TR-CEC2018-DMOP-Competition.pdfimpl_pymooartificialpymooPythonhttps://github.com/anyoptimization/pymooMulti-objective optimization in Python
suite_cec2022CEC2022Suitecontinuous>=11suite used for cec2022 competition. Also in IOH.CEC2022 TR, https://github.com/P-N-Suganthan/2022-SO-BO/blob/main/CEC2022%20TR.pdf["impl_cec2022", "impl_iohexperimenter"]artificial>=1IOHexperimenter | CEC2022 reference codeC++/Pythonhttps://github.com/IOHprofiler/IOHexperimenter | https://github.com/P-N-Suganthan/2022-SO-BOIOHprofiler experimenter framework | Suganthan's reference implementation
suite_cfdCFDSuiteunknown>=1[1, 2]unknown>=1expensive evaluations 30s-15mCFD test problem suite, https://doi.org/10.1007/978-3-319-99259-4_24impl_cfdreal-worldCFD test problem suite30s-15mhttps://bitbucket.org/arahat/cfd-test-problem-suiteExpensive real-world CFD-based test problems
suite_creCRESuitecontinuous | integer6-14[2, 3, 4, 5]unknown>=1Easy-to-evaluate real-world multi-objective optimization problems, Ryoji Tanabe; Hisao Ishibuchi, https://doi.org/10.1016/j.asoc.2020.106078impl_reproblemsreal-world-like3-73-7reproblemsPythonhttps://github.com/ryojitanabe/reproblemsReal-world inspired multi-objective optimization problem suite
suite_cuterCUTErSuitecontinuous | integer | binary>=31unknown>=1noA constrained and unconstrained testing environment.CUTEr, https://dl.acm.org/doi/10.1145/962437.962439artificial>=1>=1>=1
suite_cutestCUTEstSuitecontinuous | integer | binary>=31box | unknown>=2noConstrained and Unconstrained Testing Environment with safe threadsCUTEst for optimization softwareCUTEst, https://link.springer.com/article/10.1007/s10589-014-9687-3impl_pycutestmultimodalartificial>=1>=1>=1pycutestPython / C++ / Fortranhttps://github.com/jfowkes/pycutestPython interface to CUTEst>=1
suite_dtlzDTLZSuitecontinuous>=1[10, 2, 3, 4, 5, 6, 7, 8, 9]Scalable multi-objective optimization test problems, Kalyanmoy Deb; Lothar Thiele; Marco Laumanns; Eckart Zitzler, https://doi.org/10.1109/CEC.2002.1007032impl_pymoo>=1pymooPythonhttps://github.com/anyoptimization/pymooMulti-objective optimization in Python
suite_dynamicbinvalDynamicBinValSuitebinary>=11dynamicdynamicFour versions of the dynamic binary value problemDynamicBinVal, https://arxiv.org/pdf/2404.15837impl_iohexperimenterartificial>=1IOHexperimenterC++/Pythonhttps://github.com/IOHprofiler/IOHexperimenterIOHprofiler experimenter framework
suite_emo2017EMO2017Suitecontinuous4-242BBComp EMO 2017, https://www.ini.rub.de/PEOPLE/glasmtbl/projects/bbcomp/impl_emo2017real-world4-24EMO 2017 real-world problemshttps://www.ini.rub.de/PEOPLE/glasmtbl/projects/bbcomp/downloads/realworld-problems-bbcomp-EMO-2017.zipBBComp EMO-2017 real-world problem archive
suite_etmofETMOFSuitecontinuous25-10000[10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 2, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 3, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 4, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 5, 50, 6, 7, 8, 9]dynamicdynamicEvolutionary many-task optimization framework, https://doi.org/10.48550/arXiv.2110.08033impl_etmof25-10000ETMOFhttps://github.com/songbai-liu/etmoEvolutionary many-task optimization framework
suite_expobenchEXPObenchSuitecategorical | continuous | integer30-4051noisybox | unknown>=2["observational", "real-life"]noEXPensive Optimization benchmark libraryWind farm layout optimization, gas filter design, pipe shape optimization, hyperparameter tuning, and hospital simulationEXPObench, https://doi.org/10.1016/j.asoc.2023.110744impl_expobenchreal-world10-13510-13510-135EXPObenchPython2 to 80 secondshttps://github.com/AlgTUDelft/ExpensiveOptimBenchmarkEXPensive Optimization benchmark library (wind farm layout, gas filter design, pipe shape, hyperparameter tuning, hospital simulation)>=1
suite_gbeaGBEASuitecontinuous>=1[1, 2]noisynoisyexpensive evaluations 5s-35s, RW-GAN-Mario and TopTrumps are part of GBEAGame benchmark for evolutionary algorithms, https://doi.org/10.1145/3321707.3321805impl_gbeamultimodalreal-world>=1coco-gbea5s-35shttps://github.com/ttusar/coco-gbeaGame-Benchmark for Evolutionary Algorithms (COCO fork)
suite_gnbgGNBGSuitecontinuous>=11Generalized Numerical Benchmark GeneratorGNBG, https://arxiv.org/abs/2312.07083impl_gnbgartificial>=1GNBG Generatorhttps://github.com/Danial-Yazdani/GNBG-GeneratorGeneralized Numerical Benchmark Generator
suite_gnbg_iiGNBG-IISuitecontinuous>=11Generalized Numerical Benchmark Generator (version 2). Also available in IOH.GNBG-II, https://dl.acm.org/doi/pdf/10.1145/3712255.3734271["impl_gnbg_ii", "impl_iohgnbg"]artificial>=1IOHGNBG | GNBG-IIhttps://github.com/IOHprofiler/IOHGNBG | https://github.com/rohitsalgotra/GNBG-IIIOHprofiler version of GNBG | Generalized Numerical Benchmark Generator version 2
suite_iohclusteringIOHClusteringSuitecontinuous>=11Set of benchmark problems from clustering: optimization task is selecting cluster centers for a given set of data.IOHClustering, https://arxiv.org/pdf/2505.09233impl_iohclusteringmultimodalartificial-from-real-data>=1IOHClusteringhttps://github.com/IOHprofiler/IOHClusteringClustering-based optimization benchmark built on ML datasets
suite_kinematics_robotarmKinematicsRobotArmSuitecontinuous211Kinematics of a robot arm, https://doi.org/10.1023/A:1013258808932impl_transfer_rf_bbob_rwunimodalreal-world21Transfer Random Forests BBOB Real-worldhttps://github.com/ShuaiqunPan/Transfer_Random_forests_BBOB_Real_worldReal-world BBOB-like problem implementations (Porkchop, KinematicsRobotArm)
suite_l1_zdtL1-ZDTSuitecontinuous | binary>=22Variant of ZDT with linkages between variables within groupsLinkage ZDT/DTLZ variants, https://doi.org/10.1145/1143997.1144179>=1>=1
suite_l2_dtlzL2-DTLZSuitecontinuous>=1[10, 2, 3, 4, 5, 6, 7, 8, 9]Variant of DTLZ2/DTLZ3 with linkages between all variablesLinkage ZDT/DTLZ variants, https://doi.org/10.1145/1143997.1144179>=1
suite_l2_zdtL2-ZDTSuitecontinuous | binary>=22Variant of ZDT with linkages between all variablesLinkage ZDT/DTLZ variants, https://doi.org/10.1145/1143997.1144179>=1>=1
suite_l3_dtlzL3-DTLZSuitecontinuous>=1[10, 2, 3, 4, 5, 6, 7, 8, 9]Variant of L2-DTLZ with anti-linkage mappingLinkage ZDT/DTLZ variants, https://doi.org/10.1145/1143997.1144179>=1
suite_l3_zdtL3-ZDTSuitecontinuous | binary>=22Variant of L2-ZDT with anti-linkage mappingLinkage ZDT/DTLZ variants, https://doi.org/10.1145/1143997.1144179>=1>=1
suite_maopMaOPSuitecontinuous>=1[10, 2, 3, 4, 5, 6, 7, 8, 9]noisyunknownMaOP benchmark, https://doi.org/10.1016/j.swevo.2019.02.003>=1
suite_mechbenchMECHBenchSuitecontinuous>=11unknown{1, 2}noMECHBenchSet of problems inspired by Structural Mechanics Design Optimization. Embeds physical simulations (plasticity only, no fracture/damage). Unstructured/non-isotropic multimodality.MECHBench, https://arxiv.org/abs/2511.10821impl_mechbenchmultimodalreal-world>=1MECHBenchPython1-7 minuteshttps://github.com/BayesOptApp/MECHBenchStructural mechanics design optimization benchmark
suite_mf2MF2Suitecontinuous>=11multi-fidelity[1, 2]mf2: a collection of multi-fidelity benchmark functions in Python, https://doi.org/10.21105/joss.02049impl_mf2>=1mf2Pythonhttps://github.com/sjvrijn/mf2Multi-fidelity test function collection
suite_minus_dtlzMinus DTLZSuitecontinuous>=1[10, 2, 3, 4, 5, 6, 7, 8, 9]Variant of DTLZ that minimises the inverse of the base DTLZ functionsMinus DTLZ / Minus WFG, https://doi.org/10.1109/TEVC.2016.2587749>=1
suite_minus_wfgMinus WFGSuitecontinuous>=1[10, 2, 3, 4, 5, 6, 7, 8, 9]Variant of WFG that minimises the inverse of the base WFG functionsMinus DTLZ / Minus WFG, https://doi.org/10.1109/TEVC.2016.2587749>=1
suite_mmoppMMOPPSuiteunknown0[2, 3, 4, 5, 6, 7]unknown>=1MMOPP technical report, http://www5.zzu.edu.cn/system/_content/download.jsp?urltype=news.DownloadAttachUrl&owner=1327567121&wbfileid=4764412impl_mmoppmultimodalMMOPPhttp://www5.zzu.edu.cn/ecilab/info/1036/1251.htmECI lab distribution page for MMOPP
suite_modactMODActSuitecontinuous | integer40[2, 3, 4, 5]unknown>=1multiobjective design of actuatorsRealistic Constrained Multi-Objective Optimization Benchmark Problems from Design.MODAct, https://doi.org/10.1109/TEVC.2020.3020046["impl_modact", "impl_pymoo"]real-world2020modact | pymooPython20mshttps://github.com/epfl-lamd/modact | https://github.com/anyoptimization/pymooEPFL-LAMD modact package | Multi-objective optimization in Python
suite_morepoMOrepoSuiteunknown02dynamic | noisyunknown>=1unknownunknownimpl_morepoMOrepohttps://github.com/MCDMSociety/MOrepoMulti-objective optimisation problem repository
suite_pboPBOSuitebinary>=11Suite of 25 binary optimization problemsPBO benchmarks, https://dl.acm.org/doi/pdf/10.1145/3319619.3326810impl_iohexperimenterartificial>=1IOHexperimenterC++/Pythonhttps://github.com/IOHprofiler/IOHexperimenterIOHprofiler experimenter framework
suite_porkchopPorkchopPlotInterplanetaryTrajectorySuitecontinuous21Porkchop plot interplanetary trajectory benchmark, https://doi.org/10.1109/CEC65147.2025.11042973impl_transfer_rf_bbob_rwmultimodalreal-world2Transfer Random Forests BBOB Real-worldhttps://github.com/ShuaiqunPan/Transfer_Random_forests_BBOB_Real_worldReal-world BBOB-like problem implementations (Porkchop, KinematicsRobotArm)
suite_reRESuitecontinuous | integer4-14[2, 3, 4, 5, 6, 7, 8, 9]Easy-to-evaluate real-world multi-objective optimization problems, Ryoji Tanabe; Hisao Ishibuchi, https://doi.org/10.1016/j.asoc.2020.106078impl_reproblemsreal-world-like2-72-7reproblemsPythonhttps://github.com/ryojitanabe/reproblemsReal-world inspired multi-objective optimization problem suite
suite_rwmvopRWMVOPSuitecontinuous | integer | categorical>=31unknown>=1RWMVOP, https://doi.org/10.1109/TEVC.2013.2281531real-world>=1>=1>=1
GNBGGeneralized Numerical Benchmark Generatorsuite; generator1scalable
suite_sbox_costSBOX-COSTSuite continuousnonononoartificialhttps://arxiv.org/abs/2312.07083https://github.com/Danial-Yazdani/GNBG-Generator
DynamicBinValFour versions of the dynamic binary value problemsuite>=1 1scalablebinarynoyesnonoartificialhttps://arxiv.org/pdf/2404.15837problems from BBOB but allows instances with the optimum close to the boundarySBOX-COST, https://doi.org/10.48550/arXiv.2305.12221impl_iohexperimentermultimodal>=1IOHexperimenterC++/Python https://github.com/IOHprofiler/IOHexperimenterIOHprofiler experimenter framework
PBOSuite of 25 binary optimization problemssuite1scalablebinarynonononoartificialhttps://dl.acm.org/doi/pdf/10.1145/3319619.3326810https://github.com/IOHprofiler/IOHexperimenter
suite_sdpSDPSuitecontinuous>=1[10, 2, 3, 4, 5, 6, 7, 8, 9]dynamic | noisydynamicunknownSDP dynamic multi-objective benchmark, https://doi.org/10.1109/TCYB.2019.2896021>=1
W-modelTunable generator for binary optimization based on several difficulty featuresgenerator1scalable
suite_submodularSubmodular OptimizationSuite binarynonononoartificialhttps://dl.acm.org/doi/abs/10.1145/3205651.3208240?casa_token=S4U_Pi9f6MwAAAAA:U9ztNTPwmupT8K3GamWZfBL7-8fqjxPtr_kprv51vdwA-REsp0EyOFGa99BtbANb0XbqyrVg795hIwhttps://github.com/thomasWeise/BBDOB_W_Model
Submodular Optimitzationset of graph-based submodular optimization problems from 4 problem typessuite>=1 1scalablebinarynonononoset of graph-based submodular optimization problems from 4 problem typesSubmodular optimization benchmark, https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=10254181impl_iohexperimenter artificialhttps://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=10254181>=1IOHexperimenterC++/Python https://github.com/IOHprofiler/IOHexperimenterIOHprofiler experimenter framework
CEC2013suite used for cec2013 competition. Also in IOH https://github.com/IOHprofiler/IOHexperimentersuite1scalable
suite_tulipa_energyTulipaEnergySuite continuousnonononoartificialhttps://peerj.com/articles/cs-2671/CEC2013.pdfhttps://github.com/P-N-Suganthan/CEC2013
CEC2022suite used for cec2022 competition. Also in IOH https://github.com/IOHprofiler/IOHexperimentersuite>=1 1scalablecontinuousnonononoartificialhttps://github.com/P-N-Suganthan/2022-SO-BO/blob/main/CEC2022%20TR.pdfhttps://github.com/P-N-Suganthan/2022-SO-BO
Onemax+Sphere / Zeromax+Spherenoisy | multi-fidelityunknown>=2 single2scalablebinary and continuous;mixed;nonononoartificialhttps://doi.org/10.1145/3449726.3459521None
Onemax+Sphere / DeceptiveTrap+RotatedEllipsoidparameter single2scalablebinary and continuous;mixed;nonononoartificialhttps://doi.org/10.1145/3449726.3459521None
InverseDeceptiveTrap+RotatedEllipsoid / DeceptiveTrap+RotatedEllipsoid single2scalablebinary and continuous;mixed;nonononoartificialhttps://doi.org/10.1145/3449726.3459521None
PorkchopPlotInterplanetaryTrajectory[1, 2]TulipaEnergyModel.jlDetermine the optimal investment and operation decisions for different assets in the energy system (production, consumption, conversion, storage, transport) while minimizing loss of load. Modelled as a potentially very large linear program with multiple fidelity levels. suite12continuousnonononoreal-worldhttps://doi.org/10.1109/CEC65147.2025.11042973https://github.com/ShuaiqunPan/Transfer_Random_forests_BBOB_Real_world
KinematicsRobotArmTulipaEnergyModel.jl scientific references, https://tulipaenergy.github.io/TulipaEnergyModel.jl/stable/40-scientific-foundation/45-scientific-referencesimpl_tulipaunimodal suite121continuousnononono real-worldhttps://doi.org/10.1023/A:1013258808932https://github.com/ShuaiqunPan/Transfer_Random_forests_BBOB_Real_world
VehicleDynamics suite12continuousnonononoreal-worldhttps://www.scitepress.org/Papers/2023/121580/121580.pdfhttps://zenodo.org/records/8307853
MECHBenchThis is a set of problems with inspiration from Structural Mechanics Design Optimization. The suite comprises three physical models, from which the user may define different kind of problems which impact the final design output.Problem Suite1scalable'ContinuousyesnononoReal-World Applicationhttps://arxiv.org/abs/2511.10821https://github.com/BayesOptApp/MECHBench
EXPObenchWind farm layout optimization, gas filter design, pipe shape optimization, hyperparameter tuning, and hospital simulationProblem Suite110 to 135Continuous, Integer, Categorical, ConditionalyesnoyesnoReal-World Applicationhttps://doi.org/10.1016/j.asoc.2023.110744https://github.com/AlgTUDelft/ExpensiveOptimBenchmark
Gasoline direct injection engine designA multi-objective optimization problem seeking to minimize fuel consumption and NOx emissions over a two-minute dynamic duty cycle, subject to five constraints (turbine inlet temperature, number of knock occurrences, peak cylinder pressure, peak cylinder pressure rise, total work). Seven decision variables are defined: four define the hardware choices of cylinder compression ratio, turbo machinery and EGR cooler sizing; three relate to control variables that parameterise the engine control logic.Single Problem27Continuous, OrdinalyesnonoyesReal-World Application https://doi.org/10.1016/j.ejor.2022.08.032
BEACONGenerator for bi-objective benchmark problems with explicitly controlled correlations in continuous spaces.Generator2scalableContinuousnonononoArtificially Generatedhttps://dl.acm.org/doi/10.1145/3712255.3734303https://github.com/Stebbet/BEACON/
TulipaEnergyDetermine the optimal investment and operation decisions for different types of assets in the energy system (production, consumption, conversion, storage, and transport), while minimizing loss of load.Problem Suite1scalableContinuousyesnoyesyesReal-World ApplicationSee https://tulipaenergy.github.io/TulipaEnergyModel.jl/stable/40-scientific-foundation/45-scientific-referenceshttps://tulipaenergy.github.io/TulipaEnergyModel.jl/stable/
ATOParameters of the Modules of the Automatic Train Operation should be optimized. The parameters are continuous with different ranges. There are two objectives (minimizing energy consumption, minimizing driving duration.Single Problem210ContinuousnonononoReal-World Application>=1TulipaEnergyModel.jlJulia / JuMPminutes to hourshttps://tulipaenergy.github.io/TulipaEnergyModel.jl/stable/ | https://github.com/TulipaEnergy/Tulipa-OBZ-CaseStudyLarge linear program for optimal investment and operation of energy systems -
Brachytherapy treatment planningTreatment planning for internal radiation therapyProblem Suite2-3100-500ContinuousyesnonoyesReal-World Applicationhttps://www.sciencedirect.com/science/article/pii/S1538472123016781
FleetOptHealthcare organisation in the UK provided data about their current fleet of vehicles to conduct non-emergency heathcare trips in the Argyll and Bute region of Scotland, UK. They also provided historical data about the trips the vehicles took and about the bases which the vehicles return to. The aim is to reduce the existing fleet of vehicles while still ensuring all trips can be covered. Moving a vehicle from one base to another to help cover trips is OK as long as the original base can still cover its trips. Link to paper with more details: https://dl.acm.org/doi/abs/10.1145/3638530.3664137Single Problem1Upper level: 54; lower level: 13208IntegeryesnononoReal-World Applicationhttps://dl.acm.org/doi/abs/10.1145/3638530.3664137Not public: was done for real client with their private data
Building spatial designOptimise the spatial layout of a building to: minimise energy consumption for climate control, and minimise the strain on the structureSingle Problem
suite_vehicle_dynamicsVehicleDynamicsSuitecontinuous 2scalable depending on problem size (e.g. 90 for)Continuous, BooleanyesnononoReal-World Applicationhttps://hdl.handle.net/1887/81789https://github.com/TUe-excellent-buildings/BSO-toolbox
Electric Motor Design OptimizationThe goal is to find a design of a synchronous electric motor for power steering systems that minimizes costs and satisfies all constraints.Single Problem 113Continuous, IntegeryesnoyesnoReal-World Applicationhttps://dis.ijs.si/tea/Publications/Tusar23Multistep.pdf (paper in Slovene)Implementation not freely available
BONO-BenchBi-objective problem generator and suite with scalable continuous decision space. Features complex problem properties (different types of multimodality and challenges in decision and objective space) as well as Pareto front approximations with error guarantees for the hypervolume and exact R2 indicators.GeneratorVehicleDynamics benchmark, https://www.scitepress.org/Papers/2023/121580/121580.pdfimpl_vehicle_dynamicsmultimodalreal-world 2scalableContinuousnonononoArtificially Generated https://github.com/schaepermeier/bonobench
RandOptGenRandOptGen: A Unified Random Problem Generator for Single-and Multi-Objective Optimization Problems with Mixed-Variable Input SpacesGeneratorscalablescalableContinuous, Integer, BooleannonononoArtificially GeneratedVehicleDynamics (Zenodo)https://zenodo.org/records/8307853Zenodo archive for the vehicle dynamics benchmark https://github.com/MALEO-research-group/RandOptGen
CUTErA constrained and unconstrained testing environmentProblem Suite1scalableContinuous, Integer, BooleanyesnononoArtificially Generatedhttps://dl.acm.org/doi/10.1145/962437.962439Not Found
CUTEstThe Constrained and Unconstrained Testing Environment with safe threads (CUTEst) for optimization softwareProblem Suite1scalableContinuous, Integer, BooleanyesnononoArtificially Generatedhttps://link.springer.com/article/10.1007/s10589-014-9687-3https://github.com/jfowkes/pycutest
suite_wfgWFGSuitecontinuous>=1[10, 2, 3, 4, 5, 6, 7, 8, 9]A review of multiobjective test problems and a scalable test problem toolkit, Simon Huband; Philip Hingston; Luigi Barone; Lyndon While, https://doi.org/10.1109/TEVC.2005.861417impl_pymoo>=1pymooPythonhttps://github.com/anyoptimization/pymooMulti-objective optimization in Python
PUBOiA benchmark in which variable importance is tunable, based on the Walsh functionGenerator1scalableBooleannonononoArtificially Generatedhttps://link.springer.com/chapter/10.1007/978-3-031-04148-8_12https://gitlab.com/verel/pubo-importance-benchmark
suite_zdtZDTSuitecontinuous | binary>=22Comparison of multiobjective evolutionary algorithms: empirical results, Eckart Zitzler; Kalyanmoy Deb; Lothar Thiele, https://doi.org/10.1162/106365600568202impl_pymoo>=1>=1pymooPythonhttps://github.com/anyoptimization/pymooMulti-objective optimization in Python
name textual description suite/generator/single objectives dimensionality variable type constraints dynamic noise multi-fidelity source (real-world/artificial) reference implementation
+ID Name Type Variable Types Total Variables Objectives Properties Constraint Types Total Constraints Dynamics Noise Partial Evaluations Independent Objectives Fidelity Levels Full Name Description Tags References Implementations Modality Examples Source Binary Vars Categorical Vars Continuous Vars Integer Vars Implementation Names Implementation Languages Implementation Evaluation Times Implementation Links Implementation Descriptions Implementation Requirements Hard Box Constraints Soft Box Constraints Hard Linear Constraints Soft Linear Constraints Hard Function Constraints Soft Function Constraints + + + +
+

Problem details

+

Click a table row to inspect full details.

+
+
+

README.md

+

README content for this snippet folder will appear here.

+
+

Snippet: call_{problem_id}.py

+

Open snippet folder

+

Select a problem to check for an available code snippet.

+
+
+
+ diff --git a/docs/javascript.html b/docs/javascript.html index a194ca7..e3a771e 100644 --- a/docs/javascript.html +++ b/docs/javascript.html @@ -1,44 +1,511 @@ diff --git a/docs/problems.html b/docs/problems.html index 2324387..66031d4 100644 --- a/docs/problems.html +++ b/docs/problems.html @@ -1,1191 +1,3317 @@ - +
+
+
+ Toggle Visible Columns +
+ + +
+
+ +
+
+
+
+
- - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - + + + + - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - + - - - - - + - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + - - - - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - - - - - - - - - - - - - + - - - - - - - - - - - - - - + + + + + - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + - - - - - + + + + + - - - + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + + + - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - + + + + - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - + + + + - - - - + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + - - - - + + + + + + - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - + + + + - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + - - - - - - - + - - - - - - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + -
nametextual descriptionsuite/generator/singleobjectivesdimensionalityvariable typeconstraintsdynamicnoisemulti-fidelitysource (real-world/artificial)referenceimplementationIDNameTypeVariable TypesTotal VariablesObjectivesPropertiesConstraint TypesTotal ConstraintsDynamicsNoisePartial EvaluationsIndependent ObjectivesFidelity LevelsFull NameDescriptionTagsReferencesImplementationsModalityExamplesSourceBinary VarsCategorical VarsContinuous VarsInteger VarsImplementation NamesImplementation LanguagesImplementation Evaluation TimesImplementation LinksImplementation DescriptionsImplementation RequirementsHard Box ConstraintsSoft Box ConstraintsHard Linear ConstraintsSoft Linear ConstraintsHard Function ConstraintsSoft Function Constraints
BBOBsuite1scalable
fn_atoATOProblem continuousnonononohttps://doi.org/10.1080/10556788.2020.1808977https://github.com/numbbo/coco
BBOB-biobjsuite10 22-40continuousnononono https://doi.org/10.48550/arXiv.1604.00359https://github.com/numbbo/coco
BBOB-noisy suite1scalablecontinuousnonoyesno https://hal.inria.fr/inria-00369466https://web.archive.org/web/20210416065610/https://coco.gforge.inria.fr/doku.php?id=downloads
BBOB-largescale suite120-640continuousnonono no https://doi.org/10.48550/arXiv.1903.06396https://github.com/numbbo/coco
BBOB-mixint suite15-160integer;continuous;mixednononono https://doi.org/10.1145/3321707.3321868https://github.com/numbbo/coco
BBOB-biobj-mixintParameters of the Modules of the Automatic Train Operation are optimized; two objectives: minimizing energy consumption and minimizing driving duration. suite25-160integer;continuous;mixednononono https://doi.org/10.1145/3321707.3321868https://github.com/numbbo/coco
BBOB-constrained suite12-40continuousyesnononounimodal http://numbbo.github.io/coco-doc/bbob-constrained/https://github.com/numbbo/coco
MOreporeal-world suite2?combinatorial???no 10 https://github.com/MCDMSociety/MOrepo
ZDT suite2scalablecontinuous;binarynononono https://doi.org/10.1162/106365600568202https://github.com/anyoptimization/pymoo
DTLZ suite2+scalablecontinuousnononono https://doi.org/10.1109/CEC.2002.1007032https://pymoo.org/problems/many/dtlz.html
WFG suite2+scalablecontinuousnononono https://doi.org/10.1109/TEVC.2005.861417https://pymoo.org/problems/many/wfg.html
CDMP suite2+scalablecontinuousyes??no https://doi.org/10.1145/3321707.3321878?
SDP suite2+scalablecontinuousnoyes?no https://doi.org/10.1109/TCYB.2019.2896021?
MaOP suite2+scalablecontinuousnono?no https://doi.org/10.1016/j.swevo.2019.02.003?
BP
fn_building_spatialBuilding spatial designProblemcontinuous | binary>=22 suite2+scalablecontinuousnono?nobox | unknown>=2 https://doi.org/10.1109/CEC.2019.8790277?
GPD generator2+scalablecontinuousoptionalnooptional no https://doi.org/10.1016/j.asoc.2020.106139?
ETMOF suite2-5025-10000continuousnoyesnono https://doi.org/10.48550/arXiv.2110.08033https://github.com/songbai-liu/etmo
MMOPPOptimise the spatial layout of a building to minimise energy consumption for climate control and minimise the strain on the structure. Many hard constraints; mixed-variable (continuous+binary); expensive evaluations. suite2-7??yesnononoBuilding spatial design, https://hdl.handle.net/1887/81789impl_bso_toolbox http://www5.zzu.edu.cn/system/_content/download.jsp?urltype=news.DownloadAttachUrl&owner=1327567121&wbfileid=4764412http://www5.zzu.edu.cn/ecilab/info/1036/1251.htm
CFDexpensive evaluations 30s-15msuite1-2scalable?yesnononoreal worldhttps://doi.org/10.1007/978-3-319-99259-4_24https://bitbucket.org/arahat/cfd-test-problem-suite
GBEAexpensive evaluations 5s-35s, RW-GAN-Mario and TopTrumps are part of GBEAsuite1-2scalablecontinuousnonoyesnoreal worldhttps://doi.org/10.1145/3321707.3321805https://github.com/ttusar/coco-gbea
Car structure54 constraintssuite2144-222discreteyesnononoreal worldhttps://doi.org/10.1145/3205651.3205702http://ladse.eng.isas.jaxa.jp/benchmark/
EMO2017 suite24-24continuousnonononoreal worldhttps://www.ini.rub.de/PEOPLE/glasmtbl/projects/bbcomp/https://www.ini.rub.de/PEOPLE/glasmtbl/projects/bbcomp/downloads/realworld-problems-bbcomp-EMO-2017.zip
JSEC2019expensive evaluations 3s; 22 constraintssingle1-532continuousyesnononoreal worldhttp://www.jpnsec.org/files/competition2019/EC-Symposium-2019-Competition-English.htmlhttp://www.jpnsec.org/files/competition2019/EC-Symposium-2019-Competition-English.html
REreal-world>=1 suite2-92-7continuous;integer;mixednonononoreal world likehttps://doi.org/10.1016/j.asoc.2020.106078https://github.com/ryojitanabe/reproblems
CRE>=1 suite2-53-7continuous;integer;mixedyesnononoreal world likehttps://doi.org/10.1016/j.asoc.2020.106078https://github.com/ryojitanabe/reproblems
Radar waveformBSO-toolboxC++~1s (smallest) to ~40s (larger)https://github.com/TUe-excellent-buildings/BSO-toolboxBuilding Spatial Design toolbox (TU/e) single94-12integeryesnononoreal worldhttps://doi.org/10.1007/978-3-540-70928-2_53http://code.evanhughes.org/
MF2>=1 suite11-ncontinuousnononoyes https://doi.org/10.21105/joss.02049https://github.com/sjvrijn/mf2
AMVOP suite1scalablemixed continuous+ordinal+categorical+bothnononono https://doi.org/10.1109/TEVC.2013.2281531?
RWMVOP suite1scalablecontinuous;mixed continuous+ordinal+categorical+bothyesnononoreal worldhttps://doi.org/10.1109/TEVC.2013.2281531?
SBOX-COSTproblems from BBOB but allows instances with the optimum close to the boundarysuite1scalable
fn_convex_dtlz2Convex DTLZ2Problem continuousnononono>=1[10, 2, 3, 4, 5, 6, 7, 8, 9]Variant of DTLZ2 with a convex Pareto front (instead of concave)Convex DTLZ2, https://doi.org/10.1109/TEVC.2013.2281535>=1 https://doi.org/10.48550/arXiv.2305.12221https://github.com/IOHprofiler/IOHexperimenter/
ρMNK-Landscapestunable variable and objective dimensions; tunable multimodality and correlation between objectivesgeneratorscalablescalablebinarynononono https://doi.org/10.1016/j.ejor.2012.12.019https://gitlab.com/aliefooghe/mocobench/
mUBQPtunable variable and objective dimensions; tunable density and correlation between objectivesgeneratorscalablescalablebinarynononono https://doi.org/10.1016/j.asoc.2013.11.008https://gitlab.com/aliefooghe/mocobench/
ρmTSPtunable variable and objective dimensions; tunable instance type (euclidian/random); tunable correlation between objectivesgeneratorscalablescalablepermutationsno (apart from being permutations)nonono https://doi.org/10.1007/978-3-319-45823-6_40https://gitlab.com/aliefooghe/mocobench/
CEC2015-DMOO suite2-3?continuous?yesnono Benchmark Functions for CEC 2015 Special Session and Competition on Dynamic Multi-objective Optimization
EalainReal-world-like, easily extensible to increase complexitygenerator1+scalablecontinuous,binary,integeroptionaloptionalnooptionalReal-world-likehttps://doi.org/10.1145/3638530.3654299https://github.com/qrenau/Ealain
MA-BBOBGenerator that creates affine combinations of BBOB functionsgenerator
fn_emdoElectric Motor Design OptimizationProblemcontinuous | integer26 1scalablecontinuousnonononoisyunknown | box>=14noisy noartificialhttps://doi.org/10.1145/3673908https://github.com/IOHprofiler/IOHexperimenter/blob/master/example/Competitions/MA-BBOB/Example_MABBOB.ipynbElectric Motor Design Optimization# Goal\nFind a design of a synchronous electric motor for power steering systems that minimizes costs and satisfies all constraints.\n\n# Motivation\nChallenging to find good solutions in a limited time.\n\n# Key Challenges\n* Time-consuming solution evaluation\n* Highly-constrained problem\n* Constraints are multimodal\n\nThis is not an available problem, but could be interesting to show to researchers which difficulties appear in real-world problems.A Multi-Step Evaluation Process in Electric Motor Design, Tea Tušar; Peter Korošec; Bogdan Filipič, https://dis.ijs.si/tea/Publications/Tusar23Multistep.pdfimpl_emdomultimodalreal-world1313Electric Motor Design OptimizationPython8 minutesNot publicly available>=1
MPM2nonlinear nonseparable nonsymmetric; scalable in terms of time to evaluate the objective functiongenerator
fn_fleetoptFleetOptProbleminteger{54, 13208} 1scalablecontinuousnononono https://ls11-www.cs.tu-dortmund.de/_media/techreports/tr15-01.pdfhttps://github.com/jakobbossek/smoof/blob/master/inst/mpm2.py
Convex DTLZ2Variant of DTLZ2 with a convex Pareto front (instead of concave)single2+scalablecontinuousnonononounknown>=1yesUK healthcare organisation fleet optimisation: reduce the fleet of non-emergency healthcare trip vehicles while still ensuring all trips can be covered. Bilevel: upper level 54 vars, lower level 13208 vars.FleetOpt, https://dl.acm.org/doi/abs/10.1145/3638530.3664137real-world{54, 13208} https://doi.org/10.1109/TEVC.2013.2281535?
Inverted DTLZ1Variant of DTLZ1 with an inverted Pareto frontsingle2+scalablecontinuousnononono https://doi.org/10.1109/TEVC.2013.2281534?
Minus DTLZVariant of DTLZ that minimises the inverse of the base DTLZ functionssuite2+scalablecontinuousnononono https://doi.org/10.1109/TEVC.2016.2587749?
Minus WFGVariant of WFG that minimises the inverse of the base WFG functionssuite2+scalablecontinuousnononono https://doi.org/10.1109/TEVC.2016.2587749?
L1-ZDTVariant of ZDT with linkages between variables within one of two groups but not between variables in a different group; Linear recombination operators can potentially take advantage of the problem structuresuite2scalablecontinuous;binarynononono https://doi.org/10.1145/1143997.1144179?
L2-ZDTVariant of ZDT with linkages between all variables; Linear recombination operators can potentially take advantage of the problem structuresuite
fn_gasolineGasoline direct injection engine designProbleminteger | continuous14 2scalablecontinuous;binarynonononomulti-fidelityunknown5[1, 2]Multi-objective optimization to minimize fuel consumption and NOx emissions over a two-minute dynamic duty cycle, subject to five constraints (turbine inlet temperature, knock occurrences, peak cylinder pressure, peak cylinder pressure rise, total work). Seven decision variables cover hardware choices and engine control parameters.Gasoline direct injection engine design, https://doi.org/10.1016/j.ejor.2022.08.032impl_gasolinereal-world77Gasoline direct injection engine designMatlab Simulink / Wave RThttps://doi.org/10.1016/j.ejor.2022.08.032Proprietary Matlab Simulink + Wave RT co-simulation https://doi.org/10.1145/1143997.1144179?
L3-ZDTVariant of L2-ZDT using a mapping to prevent linear recombination operators from potentially taking advantage of the problem structuresuite
fn_invdeceptive_deceptive_rotellInverseDeceptiveTrap+RotatedEllipsoid / DeceptiveTrap+RotatedEllipsoidProblemcontinuous | binary>=2 2scalablecontinuous;binarynononono https://doi.org/10.1145/1143997.1144179?
L2-DTLZVariant of DTLZ2 and DTLZ3 with linkages between all variables; Linear recombination operators can potentially take advantage of the problem structuresuite2+scalablecontinuousnononono https://doi.org/10.1145/1143997.1144179?
L3-DTLZVariant of L2-DTLZ using a mapping to prevent linear recombination operators from potentially taking advantage of the problem structuresuite2+scalablecontinuousnononono https://doi.org/10.1145/1143997.1144179?
CEC2018 DT - CEC2018 Competition on Dynamic Multiobjective Optimisation14 problems. Time-dependent: Pareto front/Pareto set geometry; irregular Pareto front shapes; variable-linkage; number of disconnected Pareto front segments; etc.suite2 or 3scalable??noyesnonoMixed-variable multi-objective test problems, https://doi.org/10.1145/3449726.3459521 artificialhttps://www.academia.edu/download/94499025/TR-CEC2018-DMOP-Competition.pdfhttps://pymoo.org/problems/dynamic/df.html
MODAct - multiobjective design of actuatorsRealistic Constrained Multi-Objective Optimization Benchmark Problems from Design. Need the https://github.com/epfl-lamd/modact package installed; evaluation times around 20mssuite2 3 4 or 520mixed; integer and continuousyesnononoreal-worldhttps://doi.org/10.1109/TEVC.2020.3020046https://pymoo.org/problems/constrained/modact.html
IOHClusteringSet of benchmark problems from clustering: optimization task is selecting cluster centers for a given set of data, with the number of clusters defining problem dimensionality. Includes both a suite and a generator. Based on ML clustering datasetssuite; generator1scalablecontinuousnonononoartificial, but based on real datahttps://arxiv.org/pdf/2505.09233https://github.com/IOHprofiler/IOHClustering>=1>=1
GNBG-IIGeneralized Numerical Benchmark Generator (version 2). Also in IOH https://github.com/IOHprofiler/IOHGNBGsuite; generator1scalable
fn_inverted_dtlz1Inverted DTLZ1Problem continuousnonononoartificialhttps://dl.acm.org/doi/pdf/10.1145/3712255.3734271https://github.com/rohitsalgotra/GNBG-II>=1[10, 2, 3, 4, 5, 6, 7, 8, 9]Variant of DTLZ1 with an inverted Pareto frontInverted DTLZ1, https://doi.org/10.1109/TEVC.2013.2281534>=1
GNBGGeneralized Numerical Benchmark Generatorsuite; generator1scalable
fn_jsec2019JSEC2019Problem continuous32[1, 2, 3, 4, 5]unknown22expensive evaluations 3s; 22 constraintsJPNSEC EC-Symposium 2019 competition, http://www.jpnsec.org/files/competition2019/EC-Symposium-2019-Competition-English.htmlimpl_jsec2019real-world32JSEC 2019 competition3shttp://www.jpnsec.org/files/competition2019/EC-Symposium-2019-Competition-English.htmlJPNSEC EC-Symposium 2019 competition problem
fn_onemax_sphere_deceptive_rotellOnemax+Sphere / DeceptiveTrap+RotatedEllipsoidProblemcontinuous | binary>=22Mixed-variable multi-objective test problems, https://doi.org/10.1145/3449726.3459521artificial>=1>=1
fn_onemax_sphere_zeromax_sphereOnemax+Sphere / Zeromax+SphereProblemcontinuous | binary>=22Onemax+Sphere / Zeromax+Sphere, https://doi.org/10.1145/3449726.3459521artificial>=1>=1
fn_radar_waveformRadar waveformProbleminteger4-129unknown>=1Radar waveform design, https://doi.org/10.1007/978-3-540-70928-2_53impl_radar_waveformreal-world4-12Evan Hughes radar waveform codehttp://code.evanhughes.org/Radar waveform design reference implementation
gen_beaconBEACONGeneratorcontinuous>=12box0 noContinuous Bi-objective Benchmark problems with Explicit Adjustable COrrelatioN controlGenerator for bi-objective benchmark problems with explicitly controlled correlations in continuous spaces. Multimodal with random structure.BEACON, https://dl.acm.org/doi/10.1145/3712255.3734303impl_beaconmultimodalartificial>=1BEACONPythonnegligiblehttps://github.com/Stebbet/BEACON/Continuous Bi-objective Benchmark with Explicit Adjustable COrrelatioN control0
gen_bono_benchBONO-BenchGeneratorcontinuous>=12box>=1 nononoBi-objective Numerical Optimization BenchmarkBi-objective problem generator and suite with scalable continuous decision space. Features complex problem properties and Pareto front approximations with error guarantees for the hypervolume and exact R2 indicators.impl_bonobenchmultimodalartificial>=1BONO-BenchPythonhttps://github.com/schaepermeier/bonobenchBi-objective Numerical Optimization Benchmark (BONO-Bench)>=1
gen_ealainEalainGeneratorcontinuous | integer | binary>=3[1, 10, 2, 3, 4, 5, 6, 7, 8, 9]dynamic | multi-fidelityunknown>=1optional[1, 2]Real-world-like, easily extensible to increase complexityEalain, https://doi.org/10.1145/3638530.3654299impl_ealainreal-world-like>=1>=1>=1Ealainhttps://github.com/qrenau/EalainReal-world-like extensible benchmark problem generator
gen_gnbgGNBGGeneratorcontinuous>=11Generator counterpart of GNBG.GNBG, https://arxiv.org/abs/2312.07083impl_gnbg artificialhttps://arxiv.org/abs/2312.07083>=1GNBG Generator https://github.com/Danial-Yazdani/GNBG-GeneratorGeneralized Numerical Benchmark Generator
gen_gnbg_iiGNBG-IIGeneratorcontinuous>=11Generator counterpart of GNBG-II.GNBG-II, https://dl.acm.org/doi/pdf/10.1145/3712255.3734271["impl_gnbg_ii", "impl_iohgnbg"]artificial>=1IOHGNBG | GNBG-IIhttps://github.com/IOHprofiler/IOHGNBG | https://github.com/rohitsalgotra/GNBG-IIIOHprofiler version of GNBG | Generalized Numerical Benchmark Generator version 2
gen_gpdGPDGeneratorcontinuous>=1[10, 2, 3, 4, 5, 6, 7, 8, 9]noisyunknown>=1optionalGPD generator, https://doi.org/10.1016/j.asoc.2020.106139>=1
gen_iohclusteringIOHClusteringGeneratorcontinuous>=11Generator counterpart of the IOHClustering suite.IOHClustering, https://arxiv.org/pdf/2505.09233impl_iohclusteringmultimodalartificial-from-real-data>=1IOHClusteringhttps://github.com/IOHprofiler/IOHClusteringClustering-based optimization benchmark built on ML datasets
gen_ma_bbobMA-BBOBGeneratorcontinuous>=11Generator that creates affine combinations of BBOB functionsMA-BBOB, https://doi.org/10.1145/3673908["impl_iohexperimenter", "impl_ma_bbob"]multimodalartificial>=1MA-BBOB (IOHexperimenter) | IOHexperimenterC++/Pythonhttps://github.com/IOHprofiler/IOHexperimenter/blob/master/example/Competitions/MA-BBOB/Example_MABBOB.ipynb | https://github.com/IOHprofiler/IOHexperimenterExample notebook for MA-BBOB in IOHexperimenter | IOHprofiler experimenter framework
gen_mpm2MPM2Generatorcontinuous>=11nonlinear nonseparable nonsymmetric; scalable in terms of time to evaluate the objective functionMPM2 technical report TR15-01, https://ls11-www.cs.tu-dortmund.de/_media/techreports/tr15-01.pdfimpl_mpm2multimodal>=1MPM2 (smoof)Pythonhttps://github.com/jakobbossek/smoof/blob/master/inst/mpm2.pyPython implementation of MPM2 distributed with smoof
gen_mubqpmUBQPGeneratorbinary>=1[1, 10, 2, 3, 4, 5, 6, 7, 8, 9]tunable variable and objective dimensions; tunable density and correlation between objectivesmUBQP benchmark, https://doi.org/10.1016/j.asoc.2013.11.008impl_mocobench["multimodal", "quadratic"]>=1mocobenchC++https://gitlab.com/aliefooghe/mocobench/Multi-objective combinatorial optimization benchmark
gen_puboiPUBOiGeneratorbinary>=11noPolynomial Unconstrained Binary Optimization with tunable importanceA benchmark in which variable importance is tunable, based on the Walsh function.PUBOi, https://link.springer.com/chapter/10.1007/978-3-031-04148-8_12impl_puboiartificial>=1PUBO Importance BenchmarkPython / C++https://gitlab.com/verel/pubo-importance-benchmarkA benchmark in which variable importance is tunable, based on the Walsh function
gen_randoptgenRandOptGenGeneratorcontinuous | integer | binary>=3[1, 10, 2, 3, 4, 5, 6, 7, 8, 9]noRandOptGenA Unified Random Problem Generator for Single- and Multi-Objective Optimization Problems with Mixed-Variable Input Spaces.impl_randoptgenmultimodalartificial>=1>=1>=1RandOptGenPythonmillisecondshttps://github.com/MALEO-research-group/RandOptGen | https://doi.org/10.1145/3712256.3726478Unified Random Problem Generator for Single- and Multi-Objective Optimization with Mixed-Variable Input Spaces
gen_rho_mnk_landscapesρMNK-LandscapesGeneratorbinary>=1[1, 10, 2, 3, 4, 5, 6, 7, 8, 9]tunable variable and objective dimensions; tunable multimodality and correlation between objectivesOn the design of multi-objective evolutionary algorithms based on NK-landscapes, https://doi.org/10.1016/j.ejor.2012.12.019impl_mocobenchmultimodal>=1mocobenchC++https://gitlab.com/aliefooghe/mocobench/Multi-objective combinatorial optimization benchmark
gen_rho_mtspρmTSPGeneratorunknown>=1[1, 10, 2, 3, 4, 5, 6, 7, 8, 9]tunable variable and objective dimensions; tunable instance type (euclidean/random); tunable correlation between objectivesOn the impact of multi-objective scalability for the ρmTSP, https://doi.org/10.1007/978-3-319-45823-6_40impl_mocobench["multimodal", "quadratic"]mocobenchC++https://gitlab.com/aliefooghe/mocobench/Multi-objective combinatorial optimization benchmark
gen_wmodelW-modelGeneratorbinary>=11Tunable generator for binary optimization based on several difficulty featuresW-model, https://dl.acm.org/doi/abs/10.1145/3205651.3208240impl_wmodelartificial>=1BBDOB W-Modelhttps://github.com/thomasWeise/BBDOB_W_ModelTunable generator for binary optimization
suite_amvopAMVOPSuitecontinuous | integer | categorical>=31AMVOP, https://doi.org/10.1109/TEVC.2013.2281531multimodal>=1>=1>=1
suite_bbobBBOBSuitecontinuous>=11COCO: a platform for comparing continuous optimizers in a black-box setting, https://doi.org/10.1080/10556788.2020.1808977impl_cocomultimodal>=1COCO frameworkC/Pythonhttps://github.com/numbbo/cocoComparing Continuous Optimizers: black-box optimization benchmarking platform
suite_bbob_biobjBBOB-biobjSuitecontinuous2-402BBOB bi-objective test suite, https://doi.org/10.48550/arXiv.1604.00359impl_cocomultimodal2-40COCO frameworkC/Pythonhttps://github.com/numbbo/cocoComparing Continuous Optimizers: black-box optimization benchmarking platform
suite_bbob_biobj_mixintBBOB-biobj-mixintSuitecontinuous | integer10-3202BBOB bi-objective mixed-integer test suite, https://doi.org/10.1145/3321707.3321868impl_cocomultimodal5-1605-160COCO frameworkC/Pythonhttps://github.com/numbbo/cocoComparing Continuous Optimizers: black-box optimization benchmarking platform
suite_bbob_constrainedBBOB-constrainedSuitecontinuous2-401unknown>=1bbob-constrained documentation, http://numbbo.github.io/coco-doc/bbob-constrained/impl_cocomultimodal2-40COCO frameworkC/Pythonhttps://github.com/numbbo/cocoComparing Continuous Optimizers: black-box optimization benchmarking platform
suite_bbob_largescaleBBOB-largescaleSuitecontinuous20-6401BBOB large-scale test suite, https://doi.org/10.48550/arXiv.1903.06396impl_cocomultimodal20-640COCO frameworkC/Pythonhttps://github.com/numbbo/cocoComparing Continuous Optimizers: black-box optimization benchmarking platform
suite_bbob_mixintBBOB-mixintSuitecontinuous | integer10-3201BBOB mixed-integer test suite, https://doi.org/10.1145/3321707.3321868impl_cocomultimodal5-1605-160COCO frameworkC/Pythonhttps://github.com/numbbo/cocoComparing Continuous Optimizers: black-box optimization benchmarking platform
suite_bbob_noisyBBOB-noisySuitecontinuous>=11noisynoisyReal-parameter black-box optimization benchmarking: noisy functions definitions, https://hal.inria.fr/inria-00369466impl_coco_legacymultimodal>=1COCO legacy (bbob-noisy)C/Pythonhttps://web.archive.org/web/20210416065610/https://coco.gforge.inria.fr/doku.php?id=downloadsArchived COCO download page that hosted the bbob-noisy suite
suite_bpBPSuitecontinuous>=1[10, 2, 3, 4, 5, 6, 7, 8, 9]noisyunknownBP benchmark, https://doi.org/10.1109/CEC.2019.8790277>=1
suite_brachytherapyBrachytherapy treatment planningSuitecontinuous100-500[2, 3]multi-fidelityunknown>=1yes[1, 2]Brachytherapy treatment planningTreatment planning for internal radiation therapy. Multi-objective with aggregated objectives; no public source code.Brachytherapy treatment planning, https://www.sciencedirect.com/science/article/pii/S1538472123016781multimodalreal-world100-500
suite_car_structureCar structureSuiteinteger144-2222unknown5454 constraintsCar structure design benchmark, https://doi.org/10.1145/3205651.3205702impl_car_structurereal-world144-222Car-structure benchmarkhttp://ladse.eng.isas.jaxa.jp/benchmark/JAXA LADSE benchmark problems
suite_cdmpCDMPSuitecontinuous>=1[10, 2, 3, 4, 5, 6, 7, 8, 9]dynamic | noisyunknown>=1unknownunknownCDMP benchmark, https://doi.org/10.1145/3321707.3321878>=1
suite_cec2013CEC2013Suitecontinuous>=11suite used for cec2013 competition. Also in IOH.CEC2013 definitions, https://peerj.com/articles/cs-2671/CEC2013.pdf["impl_cec2013", "impl_iohexperimenter"]artificial>=1CEC2013 reference code | IOHexperimenterC++/Pythonhttps://github.com/P-N-Suganthan/CEC2013 | https://github.com/IOHprofiler/IOHexperimenterSuganthan's reference implementation | IOHprofiler experimenter framework
suite_cec2015_dmooCEC2015-DMOOSuitecontinuous0[2, 3]dynamicunknown>=1dynamicBenchmark Functions for CEC 2015 Special Session and Competition on Dynamic Multi-objective Optimization0
suite_cec2018_dtCEC2018 DTSuiteunknown>=1[2, 3]dynamicdynamicCEC2018 Competition on Dynamic Multiobjective Optimisation14 problems. Time-dependent: Pareto front/Pareto set geometry; irregular Pareto front shapes; variable-linkage; number of disconnected Pareto front segments; etc.CEC2018 DMOP Competition TR, https://www.academia.edu/download/94499025/TR-CEC2018-DMOP-Competition.pdfimpl_pymooartificialpymooPythonhttps://github.com/anyoptimization/pymooMulti-objective optimization in Python
suite_cec2022CEC2022Suitecontinuous>=11suite used for cec2022 competition. Also in IOH.CEC2022 TR, https://github.com/P-N-Suganthan/2022-SO-BO/blob/main/CEC2022%20TR.pdf["impl_cec2022", "impl_iohexperimenter"]artificial>=1IOHexperimenter | CEC2022 reference codeC++/Pythonhttps://github.com/IOHprofiler/IOHexperimenter | https://github.com/P-N-Suganthan/2022-SO-BOIOHprofiler experimenter framework | Suganthan's reference implementation
suite_cfdCFDSuiteunknown>=1[1, 2]unknown>=1expensive evaluations 30s-15mCFD test problem suite, https://doi.org/10.1007/978-3-319-99259-4_24impl_cfdreal-worldCFD test problem suite30s-15mhttps://bitbucket.org/arahat/cfd-test-problem-suiteExpensive real-world CFD-based test problems
suite_creCRESuitecontinuous | integer6-14[2, 3, 4, 5]unknown>=1Easy-to-evaluate real-world multi-objective optimization problems, Ryoji Tanabe; Hisao Ishibuchi, https://doi.org/10.1016/j.asoc.2020.106078impl_reproblemsreal-world-like3-73-7reproblemsPythonhttps://github.com/ryojitanabe/reproblemsReal-world inspired multi-objective optimization problem suite
suite_cuterCUTErSuitecontinuous | integer | binary>=31unknown>=1noA constrained and unconstrained testing environment.CUTEr, https://dl.acm.org/doi/10.1145/962437.962439artificial>=1>=1>=1
suite_cutestCUTEstSuitecontinuous | integer | binary>=31box | unknown>=2noConstrained and Unconstrained Testing Environment with safe threadsCUTEst for optimization softwareCUTEst, https://link.springer.com/article/10.1007/s10589-014-9687-3impl_pycutestmultimodalartificial>=1>=1>=1pycutestPython / C++ / Fortranhttps://github.com/jfowkes/pycutestPython interface to CUTEst>=1
suite_dtlzDTLZSuitecontinuous>=1[10, 2, 3, 4, 5, 6, 7, 8, 9]Scalable multi-objective optimization test problems, Kalyanmoy Deb; Lothar Thiele; Marco Laumanns; Eckart Zitzler, https://doi.org/10.1109/CEC.2002.1007032impl_pymoo>=1pymooPythonhttps://github.com/anyoptimization/pymooMulti-objective optimization in Python
suite_dynamicbinvalDynamicBinValSuitebinary>=11dynamicdynamicFour versions of the dynamic binary value problemDynamicBinVal, https://arxiv.org/pdf/2404.15837impl_iohexperimenterartificial>=1IOHexperimenterC++/Pythonhttps://github.com/IOHprofiler/IOHexperimenterIOHprofiler experimenter framework
suite_emo2017EMO2017Suitecontinuous4-242BBComp EMO 2017, https://www.ini.rub.de/PEOPLE/glasmtbl/projects/bbcomp/impl_emo2017real-world4-24EMO 2017 real-world problemshttps://www.ini.rub.de/PEOPLE/glasmtbl/projects/bbcomp/downloads/realworld-problems-bbcomp-EMO-2017.zipBBComp EMO-2017 real-world problem archive
suite_etmofETMOFSuitecontinuous25-10000[10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 2, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 3, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 4, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 5, 50, 6, 7, 8, 9]dynamicdynamicEvolutionary many-task optimization framework, https://doi.org/10.48550/arXiv.2110.08033impl_etmof25-10000ETMOFhttps://github.com/songbai-liu/etmoEvolutionary many-task optimization framework
suite_expobenchEXPObenchSuitecategorical | continuous | integer30-4051noisybox | unknown>=2["observational", "real-life"]noEXPensive Optimization benchmark libraryWind farm layout optimization, gas filter design, pipe shape optimization, hyperparameter tuning, and hospital simulationEXPObench, https://doi.org/10.1016/j.asoc.2023.110744impl_expobenchreal-world10-13510-13510-135EXPObenchPython2 to 80 secondshttps://github.com/AlgTUDelft/ExpensiveOptimBenchmarkEXPensive Optimization benchmark library (wind farm layout, gas filter design, pipe shape, hyperparameter tuning, hospital simulation)>=1
suite_gbeaGBEASuitecontinuous>=1[1, 2]noisynoisyexpensive evaluations 5s-35s, RW-GAN-Mario and TopTrumps are part of GBEAGame benchmark for evolutionary algorithms, https://doi.org/10.1145/3321707.3321805impl_gbeamultimodalreal-world>=1coco-gbea5s-35shttps://github.com/ttusar/coco-gbeaGame-Benchmark for Evolutionary Algorithms (COCO fork)
suite_gnbgGNBGSuitecontinuous>=11Generalized Numerical Benchmark GeneratorGNBG, https://arxiv.org/abs/2312.07083impl_gnbgartificial>=1GNBG Generatorhttps://github.com/Danial-Yazdani/GNBG-GeneratorGeneralized Numerical Benchmark Generator
suite_gnbg_iiGNBG-IISuitecontinuous>=11Generalized Numerical Benchmark Generator (version 2). Also available in IOH.GNBG-II, https://dl.acm.org/doi/pdf/10.1145/3712255.3734271["impl_gnbg_ii", "impl_iohgnbg"]artificial>=1IOHGNBG | GNBG-IIhttps://github.com/IOHprofiler/IOHGNBG | https://github.com/rohitsalgotra/GNBG-IIIOHprofiler version of GNBG | Generalized Numerical Benchmark Generator version 2
suite_iohclusteringIOHClusteringSuitecontinuous>=11Set of benchmark problems from clustering: optimization task is selecting cluster centers for a given set of data.IOHClustering, https://arxiv.org/pdf/2505.09233impl_iohclusteringmultimodalartificial-from-real-data>=1IOHClusteringhttps://github.com/IOHprofiler/IOHClusteringClustering-based optimization benchmark built on ML datasets
suite_kinematics_robotarmKinematicsRobotArmSuitecontinuous211Kinematics of a robot arm, https://doi.org/10.1023/A:1013258808932impl_transfer_rf_bbob_rwunimodalreal-world21Transfer Random Forests BBOB Real-worldhttps://github.com/ShuaiqunPan/Transfer_Random_forests_BBOB_Real_worldReal-world BBOB-like problem implementations (Porkchop, KinematicsRobotArm)
suite_l1_zdtL1-ZDTSuitecontinuous | binary>=22Variant of ZDT with linkages between variables within groupsLinkage ZDT/DTLZ variants, https://doi.org/10.1145/1143997.1144179>=1>=1
suite_l2_dtlzL2-DTLZSuitecontinuous>=1[10, 2, 3, 4, 5, 6, 7, 8, 9]Variant of DTLZ2/DTLZ3 with linkages between all variablesLinkage ZDT/DTLZ variants, https://doi.org/10.1145/1143997.1144179>=1
suite_l2_zdtL2-ZDTSuitecontinuous | binary>=22Variant of ZDT with linkages between all variablesLinkage ZDT/DTLZ variants, https://doi.org/10.1145/1143997.1144179>=1>=1
suite_l3_dtlzL3-DTLZSuitecontinuous>=1[10, 2, 3, 4, 5, 6, 7, 8, 9]Variant of L2-DTLZ with anti-linkage mappingLinkage ZDT/DTLZ variants, https://doi.org/10.1145/1143997.1144179>=1
suite_l3_zdtL3-ZDTSuitecontinuous | binary>=22Variant of L2-ZDT with anti-linkage mappingLinkage ZDT/DTLZ variants, https://doi.org/10.1145/1143997.1144179>=1>=1
suite_maopMaOPSuitecontinuous>=1[10, 2, 3, 4, 5, 6, 7, 8, 9]noisyunknownMaOP benchmark, https://doi.org/10.1016/j.swevo.2019.02.003>=1
suite_mechbenchMECHBenchSuitecontinuous>=11unknown{1, 2}noMECHBenchSet of problems inspired by Structural Mechanics Design Optimization. Embeds physical simulations (plasticity only, no fracture/damage). Unstructured/non-isotropic multimodality.MECHBench, https://arxiv.org/abs/2511.10821impl_mechbenchmultimodalreal-world>=1MECHBenchPython1-7 minuteshttps://github.com/BayesOptApp/MECHBenchStructural mechanics design optimization benchmark
suite_mf2MF2Suitecontinuous>=11multi-fidelity[1, 2]mf2: a collection of multi-fidelity benchmark functions in Python, https://doi.org/10.21105/joss.02049impl_mf2>=1mf2Pythonhttps://github.com/sjvrijn/mf2Multi-fidelity test function collection
suite_minus_dtlzMinus DTLZSuitecontinuous>=1[10, 2, 3, 4, 5, 6, 7, 8, 9]Variant of DTLZ that minimises the inverse of the base DTLZ functionsMinus DTLZ / Minus WFG, https://doi.org/10.1109/TEVC.2016.2587749>=1
suite_minus_wfgMinus WFGSuitecontinuous>=1[10, 2, 3, 4, 5, 6, 7, 8, 9]Variant of WFG that minimises the inverse of the base WFG functionsMinus DTLZ / Minus WFG, https://doi.org/10.1109/TEVC.2016.2587749>=1
suite_mmoppMMOPPSuiteunknown0[2, 3, 4, 5, 6, 7]unknown>=1MMOPP technical report, http://www5.zzu.edu.cn/system/_content/download.jsp?urltype=news.DownloadAttachUrl&owner=1327567121&wbfileid=4764412impl_mmoppmultimodalMMOPPhttp://www5.zzu.edu.cn/ecilab/info/1036/1251.htmECI lab distribution page for MMOPP
suite_modactMODActSuitecontinuous | integer40[2, 3, 4, 5]unknown>=1multiobjective design of actuatorsRealistic Constrained Multi-Objective Optimization Benchmark Problems from Design.MODAct, https://doi.org/10.1109/TEVC.2020.3020046["impl_modact", "impl_pymoo"]real-world2020modact | pymooPython20mshttps://github.com/epfl-lamd/modact | https://github.com/anyoptimization/pymooEPFL-LAMD modact package | Multi-objective optimization in Python
suite_morepoMOrepoSuiteunknown02dynamic | noisyunknown>=1unknownunknownimpl_morepoMOrepohttps://github.com/MCDMSociety/MOrepoMulti-objective optimisation problem repository
DynamicBinValFour versions of the dynamic binary value problemsuite1scalable
suite_pboPBOSuite binarynoyesnono>=11Suite of 25 binary optimization problemsPBO benchmarks, https://dl.acm.org/doi/pdf/10.1145/3319619.3326810impl_iohexperimenter artificialhttps://arxiv.org/pdf/2404.15837>=1IOHexperimenterC++/Python https://github.com/IOHprofiler/IOHexperimenterIOHprofiler experimenter framework
PBOSuite of 25 binary optimization problemssuite
suite_porkchopPorkchopPlotInterplanetaryTrajectorySuitecontinuous2 1scalablebinarynonononoartificialhttps://dl.acm.org/doi/pdf/10.1145/3319619.3326810https://github.com/IOHprofiler/IOHexperimenterPorkchop plot interplanetary trajectory benchmark, https://doi.org/10.1109/CEC65147.2025.11042973impl_transfer_rf_bbob_rwmultimodalreal-world2Transfer Random Forests BBOB Real-worldhttps://github.com/ShuaiqunPan/Transfer_Random_forests_BBOB_Real_worldReal-world BBOB-like problem implementations (Porkchop, KinematicsRobotArm)
W-modelTunable generator for binary optimization based on several difficulty featuresgenerator
suite_reRESuitecontinuous | integer4-14[2, 3, 4, 5, 6, 7, 8, 9]Easy-to-evaluate real-world multi-objective optimization problems, Ryoji Tanabe; Hisao Ishibuchi, https://doi.org/10.1016/j.asoc.2020.106078impl_reproblemsreal-world-like2-72-7reproblemsPythonhttps://github.com/ryojitanabe/reproblemsReal-world inspired multi-objective optimization problem suite
suite_rwmvopRWMVOPSuitecontinuous | integer | categorical>=3 1scalablebinarynonononoartificialhttps://dl.acm.org/doi/abs/10.1145/3205651.3208240?casa_token=S4U_Pi9f6MwAAAAA:U9ztNTPwmupT8K3GamWZfBL7-8fqjxPtr_kprv51vdwA-REsp0EyOFGa99BtbANb0XbqyrVg795hIwhttps://github.com/thomasWeise/BBDOB_W_Modelunknown>=1RWMVOP, https://doi.org/10.1109/TEVC.2013.2281531real-world>=1>=1>=1
Submodular Optimitzationset of graph-based submodular optimization problems from 4 problem typessuite
suite_sbox_costSBOX-COSTSuitecontinuous>=1 1scalableproblems from BBOB but allows instances with the optimum close to the boundarySBOX-COST, https://doi.org/10.48550/arXiv.2305.12221impl_iohexperimentermultimodal>=1IOHexperimenterC++/Pythonhttps://github.com/IOHprofiler/IOHexperimenterIOHprofiler experimenter framework
suite_sdpSDPSuitecontinuous>=1[10, 2, 3, 4, 5, 6, 7, 8, 9]dynamic | noisydynamicunknownSDP dynamic multi-objective benchmark, https://doi.org/10.1109/TCYB.2019.2896021>=1
suite_submodularSubmodular OptimizationSuite binarynononono>=11set of graph-based submodular optimization problems from 4 problem typesSubmodular optimization benchmark, https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=10254181impl_iohexperimenter artificialhttps://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=10254181>=1IOHexperimenterC++/Python https://github.com/IOHprofiler/IOHexperimenterIOHprofiler experimenter framework
suite_tulipa_energyTulipaEnergySuitecontinuous>=11noisy | multi-fidelityunknown>=2parameter[1, 2]TulipaEnergyModel.jlDetermine the optimal investment and operation decisions for different assets in the energy system (production, consumption, conversion, storage, transport) while minimizing loss of load. Modelled as a potentially very large linear program with multiple fidelity levels.TulipaEnergyModel.jl scientific references, https://tulipaenergy.github.io/TulipaEnergyModel.jl/stable/40-scientific-foundation/45-scientific-referencesimpl_tulipaunimodalreal-world>=1TulipaEnergyModel.jlJulia / JuMPminutes to hourshttps://tulipaenergy.github.io/TulipaEnergyModel.jl/stable/ | https://github.com/TulipaEnergy/Tulipa-OBZ-CaseStudyLarge linear program for optimal investment and operation of energy systems
CEC2013suite used for cec2013 competition. Also in IOH https://github.com/IOHprofiler/IOHexperimentersuite1scalable
suite_vehicle_dynamicsVehicleDynamicsSuite continuousnonononoartificialhttps://peerj.com/articles/cs-2671/CEC2013.pdfhttps://github.com/P-N-Suganthan/CEC2013
CEC2022suite used for cec2022 competition. Also in IOH https://github.com/IOHprofiler/IOHexperimentersuite2 1scalablecontinuousnonononoartificialhttps://github.com/P-N-Suganthan/2022-SO-BO/blob/main/CEC2022%20TR.pdfhttps://github.com/P-N-Suganthan/2022-SO-BO
Onemax+Sphere / Zeromax+Sphere single2scalablebinary and continuous;mixed;nonononoartificialhttps://doi.org/10.1145/3449726.3459521None
Onemax+Sphere / DeceptiveTrap+RotatedEllipsoid single2scalablebinary and continuous;mixed;nonononoartificialhttps://doi.org/10.1145/3449726.3459521None
InverseDeceptiveTrap+RotatedEllipsoid / DeceptiveTrap+RotatedEllipsoid single2scalablebinary and continuous;mixed;nonononoartificialhttps://doi.org/10.1145/3449726.3459521None
PorkchopPlotInterplanetaryTrajectory suite12continuousnonononoreal-worldhttps://doi.org/10.1109/CEC65147.2025.11042973https://github.com/ShuaiqunPan/Transfer_Random_forests_BBOB_Real_world
KinematicsRobotArm suite121continuousnonononoVehicleDynamics benchmark, https://www.scitepress.org/Papers/2023/121580/121580.pdfimpl_vehicle_dynamicsmultimodal real-worldhttps://doi.org/10.1023/A:1013258808932https://github.com/ShuaiqunPan/Transfer_Random_forests_BBOB_Real_world
VehicleDynamics suite1 2continuousnonononoreal-worldhttps://www.scitepress.org/Papers/2023/121580/121580.pdfVehicleDynamics (Zenodo) https://zenodo.org/records/8307853
MECHBenchThis is a set of problems with inspiration from Structural Mechanics Design Optimization. The suite comprises three physical models, from which the user may define different kind of problems which impact the final design output.Problem Suite1scalable'ContinuousyesnononoReal-World Applicationhttps://arxiv.org/abs/2511.10821https://github.com/BayesOptApp/MECHBench
EXPObenchWind farm layout optimization, gas filter design, pipe shape optimization, hyperparameter tuning, and hospital simulationProblem Suite110 to 135Continuous, Integer, Categorical, ConditionalyesnoyesnoReal-World Applicationhttps://doi.org/10.1016/j.asoc.2023.110744https://github.com/AlgTUDelft/ExpensiveOptimBenchmark
Gasoline direct injection engine designA multi-objective optimization problem seeking to minimize fuel consumption and NOx emissions over a two-minute dynamic duty cycle, subject to five constraints (turbine inlet temperature, number of knock occurrences, peak cylinder pressure, peak cylinder pressure rise, total work). Seven decision variables are defined: four define the hardware choices of cylinder compression ratio, turbo machinery and EGR cooler sizing; three relate to control variables that parameterise the engine control logic.Single Problem27Continuous, OrdinalyesnonoyesReal-World ApplicationZenodo archive for the vehicle dynamics benchmark https://doi.org/10.1016/j.ejor.2022.08.032
BEACONGenerator for bi-objective benchmark problems with explicitly controlled correlations in continuous spaces.Generator2scalableContinuousnonononoArtificially Generatedhttps://dl.acm.org/doi/10.1145/3712255.3734303https://github.com/Stebbet/BEACON/
TulipaEnergyDetermine the optimal investment and operation decisions for different types of assets in the energy system (production, consumption, conversion, storage, and transport), while minimizing loss of load.Problem Suite1scalableContinuousyesnoyesyesReal-World ApplicationSee https://tulipaenergy.github.io/TulipaEnergyModel.jl/stable/40-scientific-foundation/45-scientific-referenceshttps://tulipaenergy.github.io/TulipaEnergyModel.jl/stable/
ATOParameters of the Modules of the Automatic Train Operation should be optimized. The parameters are continuous with different ranges. There are two objectives (minimizing energy consumption, minimizing driving duration.Single Problem210ContinuousnonononoReal-World Application -
Brachytherapy treatment planningTreatment planning for internal radiation therapyProblem Suite2-3100-500ContinuousyesnonoyesReal-World Applicationhttps://www.sciencedirect.com/science/article/pii/S1538472123016781
FleetOptHealthcare organisation in the UK provided data about their current fleet of vehicles to conduct non-emergency heathcare trips in the Argyll and Bute region of Scotland, UK. They also provided historical data about the trips the vehicles took and about the bases which the vehicles return to. The aim is to reduce the existing fleet of vehicles while still ensuring all trips can be covered. Moving a vehicle from one base to another to help cover trips is OK as long as the original base can still cover its trips. Link to paper with more details: https://dl.acm.org/doi/abs/10.1145/3638530.3664137Single Problem1Upper level: 54; lower level: 13208IntegeryesnononoReal-World Applicationhttps://dl.acm.org/doi/abs/10.1145/3638530.3664137Not public: was done for real client with their private data
Building spatial designOptimise the spatial layout of a building to: minimise energy consumption for climate control, and minimise the strain on the structureSingle Problem2scalable depending on problem size (e.g. 90 for)Continuous, BooleanyesnononoReal-World Applicationhttps://hdl.handle.net/1887/81789https://github.com/TUe-excellent-buildings/BSO-toolbox
Electric Motor Design OptimizationThe goal is to find a design of a synchronous electric motor for power steering systems that minimizes costs and satisfies all constraints.Single Problem113Continuous, IntegeryesnoyesnoReal-World Applicationhttps://dis.ijs.si/tea/Publications/Tusar23Multistep.pdf (paper in Slovene)Implementation not freely available
suite_wfgWFGSuitecontinuous>=1[10, 2, 3, 4, 5, 6, 7, 8, 9]A review of multiobjective test problems and a scalable test problem toolkit, Simon Huband; Philip Hingston; Luigi Barone; Lyndon While, https://doi.org/10.1109/TEVC.2005.861417impl_pymoo>=1pymooPythonhttps://github.com/anyoptimization/pymooMulti-objective optimization in Python
BONO-BenchBi-objective problem generator and suite with scalable continuous decision space. Features complex problem properties (different types of multimodality and challenges in decision and objective space) as well as Pareto front approximations with error guarantees for the hypervolume and exact R2 indicators.Generator
suite_zdtZDTSuitecontinuous | binary>=2 2scalableContinuousnonononoArtificially Generated https://github.com/schaepermeier/bonobench
RandOptGenRandOptGen: A Unified Random Problem Generator for Single-and Multi-Objective Optimization Problems with Mixed-Variable Input SpacesGeneratorscalablescalableContinuous, Integer, BooleannonononoArtificially Generated https://github.com/MALEO-research-group/RandOptGen
CUTErA constrained and unconstrained testing environmentProblem Suite1scalableContinuous, Integer, BooleanyesnononoArtificially Generatedhttps://dl.acm.org/doi/10.1145/962437.962439Not Found
CUTEstThe Constrained and Unconstrained Testing Environment with safe threads (CUTEst) for optimization softwareProblem Suite1scalableContinuous, Integer, BooleanyesnononoArtificially Generatedhttps://link.springer.com/article/10.1007/s10589-014-9687-3https://github.com/jfowkes/pycutest
PUBOiA benchmark in which variable importance is tunable, based on the Walsh functionGenerator1scalableBooleannonononoArtificially Generatedhttps://link.springer.com/chapter/10.1007/978-3-031-04148-8_12https://gitlab.com/verel/pubo-importance-benchmarkComparison of multiobjective evolutionary algorithms: empirical results, Eckart Zitzler; Kalyanmoy Deb; Lothar Thiele, https://doi.org/10.1162/106365600568202impl_pymoo>=1>=1pymooPythonhttps://github.com/anyoptimization/pymooMulti-objective optimization in Python
name textual description suite/generator/single objectives dimensionality variable type constraints dynamic noise multi-fidelity source (real-world/artificial) reference implementation
\ No newline at end of file +ID Name Type Variable Types Total Variables Objectives Properties Constraint Types Total Constraints Dynamics Noise Partial Evaluations Independent Objectives Fidelity Levels Full Name Description Tags References Implementations Modality Examples Source Binary Vars Categorical Vars Continuous Vars Integer Vars Implementation Names Implementation Languages Implementation Evaluation Times Implementation Links Implementation Descriptions Implementation Requirements Hard Box Constraints Soft Box Constraints Hard Linear Constraints Soft Linear Constraints Hard Function Constraints Soft Function Constraints + + + +
+

Problem details

+

Click a table row to inspect full details.

+
+
+

README.md

+

README content for this snippet folder will appear here.

+
+

Snippet: call_{problem_id}.py

+

Open snippet folder

+

Select a problem to check for an available code snippet.

+
+
+
diff --git a/docs/table_styles.css b/docs/table_styles.css index 05418c8..d7cf374 100644 --- a/docs/table_styles.css +++ b/docs/table_styles.css @@ -1,35 +1,405 @@ -/* Apply to everything */ +:root { + --opl-bg: #f5f7fb; + --opl-surface: #ffffff; + --opl-text: #12263a; + --opl-muted: #607389; + --opl-border: #d7e1ec; + --opl-accent: #0f6c8b; + --opl-accent-soft: #eaf4fa; + --opl-heading: "Segoe UI", "Trebuchet MS", sans-serif; + --opl-body: "Tahoma", "Verdana", sans-serif; + --opl-table-font-size: 0.86rem; +} + * { - font-family: sans-serif; + box-sizing: border-box; } body { margin: 1vw; - background-color: WhiteSmoke; + font-family: var(--opl-body); + color: var(--opl-text); + background: + var(--opl-bg); +} + +h2 { + font-family: var(--opl-heading); + letter-spacing: 0.01em; +} + +.table-shell { + margin-top: 1rem; + border: 1px solid var(--opl-border); + border-radius: 16px; + background: var(--opl-surface); + box-shadow: 0 14px 30px rgba(17, 37, 61, 0.08); + overflow: hidden; +} + +.table-toolbar { + display: grid; + gap: 0.65rem; + padding: 0.9rem 1rem; + border-bottom: 1px solid var(--opl-border); + background: linear-gradient(90deg, #f7fafc, #edf6fb); +} + +.toolbar-title { + font-size: 0.78rem; + text-transform: uppercase; + letter-spacing: 0.08em; + font-weight: 700; + color: var(--opl-muted); +} + +.column-toolbar-disclosure { + display: grid; + gap: 0.65rem; +} + +.column-toolbar-disclosure > summary { + cursor: pointer; + list-style: none; +} + +.column-toolbar-disclosure > summary::-webkit-details-marker { + display: none; +} + +.column-toolbar-disclosure > summary::after { + content: "Show"; + margin-left: 0.5rem; + font-size: 0.72rem; + font-weight: 600; + letter-spacing: 0.04em; + text-transform: none; + color: #355670; +} + +.column-toolbar-disclosure[open] > summary::after { + content: "Hide"; +} + +.toolbar-actions { + display: flex; + flex-wrap: wrap; + gap: 0.45rem; +} + +.toolbar-btn { + border: 1px solid #b8c8df; + background: #fff; + color: #1b3249; + border-radius: 8px; + padding: 0.36rem 0.62rem; + font-size: 0.8rem; + font-weight: 600; + cursor: pointer; + transition: all 120ms ease; +} + +.toolbar-btn:hover { + background: #eff6fb; + border-color: #94adc9; +} + +.toolbar-btn:active { + transform: translateY(1px); +} + +.column-controls { + display: flex; + flex-wrap: wrap; + gap: 0.45rem; +} + +.column-chip { + display: inline-flex; + align-items: center; + gap: 0.4rem; + padding: 0.35rem 0.65rem; + border-radius: 999px; + border: 1px solid var(--opl-border); + background: #fff; + font-size: 0.82rem; + cursor: pointer; + user-select: none; + transition: all 120ms ease; +} + +.column-chip:hover { + border-color: #b8c8df; + background: #f7fbfd; +} + +.column-chip.is-off { + color: var(--opl-muted); + background: #f1f5f9; +} + +.column-chip input { + accent-color: var(--opl-accent); +} + +.table-wrap { + overflow-x: auto; + padding: 0.45rem; +} + +#problems { + width: 100%; + min-width: 980px; + border-collapse: separate; + border-spacing: 0; + font-size: var(--opl-table-font-size); +} + +#problems thead th { + background: #f9fcff; + color: #1d324a; + border-bottom: 1px solid var(--opl-border); + font-weight: 700; + white-space: nowrap; +} + +#problems td { + vertical-align: top; + line-height: 1.4; + font-size: inherit; +} + +#problems td * { + font-size: inherit; +} + +#problems th:first-child { + position: sticky; + left: 0; + z-index: 5; + background: #f9fcff; + box-shadow: 1px 0 0 0 var(--opl-border); +} + +#problems td:first-child { + position: sticky; + left: 0; + z-index: 2; + background: #fcfeff; + box-shadow: 1px 0 0 0 var(--opl-border); +} + +#problems tbody tr:nth-of-type(odd) td { + background: #fcfeff; +} + +#problems tbody tr:nth-of-type(even) td { + background: #f8fbfe; +} + +#problems tbody tr:hover td { + background: var(--opl-accent-soft); +} + +#problems tbody tr:hover td:first-child { + background: #deedf6; +} + +#problems tbody tr.is-active td { + background: #dceefa; +} + +#problems tbody tr.is-active td:first-child { + background: #cce5f5; +} + +#problems a { + color: #085f75; +} + +#problems thead input { + width: 100%; + padding: 0.35rem 0.45rem; + border-radius: 6px; + border: 1px solid var(--opl-border); + background: #fff; + color: var(--opl-text); + font-size: inherit; +} + +.details-shell { + margin-top: 0.9rem; + padding: 0.9rem 1rem; + border: 1px solid var(--opl-border); + border-radius: 14px; + background: var(--opl-surface); + box-shadow: 0 10px 24px rgba(17, 37, 61, 0.06); +} + +.details-title { + margin: 0 0 0.3rem; + font-family: var(--opl-heading); + font-size: 1.02rem; +} + +.details-hint { + margin: 0 0 0.75rem; + color: var(--opl-muted); + font-size: 0.88rem; +} + +.details-grid { + margin: 0; + display: grid; + grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); + gap: 0.6rem; +} + +.detail-item { + margin: 0; + border: 1px solid var(--opl-border); + border-radius: 10px; + padding: 0.55rem 0.65rem; + background: #fbfdff; +} + +.detail-item dt { + margin: 0; + color: #23415e; + font-size: 0.74rem; + text-transform: uppercase; + letter-spacing: 0.04em; + font-weight: 700; } -input[type=text] { - background-color: #3CBC8D; - color: white; +.detail-item dd { + margin: 0.35rem 0 0; + color: var(--opl-text); + font-size: 0.86rem; + line-height: 1.4; + overflow-wrap: anywhere; } -.styled-table { - font-size: 0.9em; - box-shadow: 0 0 20px rgba(0, 0, 0, 0.15); +.details-empty { + color: var(--opl-muted); + font-style: italic; } -.styled-table thead tr { - background-color: AntiqueWhite; +.snippet-shell { + margin-top: 0.85rem; + border: 1px solid var(--opl-border); + border-radius: 10px; + background: #f8fbfe; + padding: 0.7rem; } -.styled-table tbody tr:nth-of-type(odd) { - background-color: SeaShell; +.snippet-title { + margin: 0; + font-size: 0.84rem; + color: #23415e; } -.styled-table tbody tr:nth-of-type(even) { - background-color: Snow; +.snippet-hint { + margin: 0.4rem 0 0.55rem; + font-size: 0.82rem; + color: var(--opl-muted); } -.styled-table tbody tr:hover { - filter: brightness(.975); +.snippet-meta { + margin: 0.35rem 0 0.45rem; + font-size: 0.82rem; +} + +.snippet-meta a { + color: #0a5d74; + text-decoration: underline; +} + +.snippet-meta a[aria-disabled="true"] { + color: var(--opl-muted); + pointer-events: none; + text-decoration: none; +} + +.snippet-code { + margin: 0; + background: #11263b; + color: #e9f3ff; + border-radius: 8px; + padding: 0.7rem; + max-height: 320px; + overflow: auto; +} + +.snippet-code code { + font-family: Consolas, "Courier New", monospace; + font-size: 0.8rem; + line-height: 1.45; + white-space: pre; +} + +.snippet-readme { + margin-top: 0.35rem; + background: #ffffff; + border: 1px solid var(--opl-border); + border-radius: 8px; + padding: 0.75rem; + max-height: 380px; + overflow: auto; + font-size: 0.83rem; + line-height: 1.45; +} + +.snippet-readme h1, +.snippet-readme h2, +.snippet-readme h3, +.snippet-readme h4 { + margin: 0.7rem 0 0.45rem; + color: #18334e; +} + +.snippet-readme p, +.snippet-readme ul, +.snippet-readme ol { + margin: 0.45rem 0; +} + +.snippet-readme pre { + margin: 0.5rem 0; + background: #11263b; + color: #e9f3ff; + border-radius: 8px; + padding: 0.65rem; + overflow: auto; +} + +.snippet-readme code { + font-family: Consolas, "Courier New", monospace; + font-size: 0.8rem; +} + +.snippet-readme a { + color: #0a5d74; +} + +@media (max-width: 900px) { + body { + margin: 0.6rem; + } + + .table-toolbar { + padding: 0.8rem; + } + + .column-chip { + font-size: 0.76rem; + } + + .details-shell { + padding: 0.8rem; + } + + .details-grid { + grid-template-columns: 1fr; + } } diff --git a/docs/table_template.html b/docs/table_template.html new file mode 100644 index 0000000..ab18a45 --- /dev/null +++ b/docs/table_template.html @@ -0,0 +1,32 @@ +
+
+
+ Toggle Visible Columns +
+ + +
+
+ __COLUMN_TOGGLES__ +
+
+
+
+ __TABLE__ +
+
+ +
+

Problem details

+

Click a table row to inspect full details.

+
+
+

README.md

+

README content for this snippet folder will appear here.

+
+

Snippet: call_{problem_id}.py

+

Open snippet folder

+

Select a problem to check for an available code snippet.

+
+
+
diff --git a/examples/problems.py b/examples/problems.py new file mode 100644 index 0000000..0e387d7 --- /dev/null +++ b/examples/problems.py @@ -0,0 +1,2995 @@ +"""Conversion of problems.yaml into the opltools schema. + +Every entry from the original YAML is preserved as a `#!` comment block +directly above the Python object(s) it was converted into. Where the +original YAML carried information that does not fit into the new schema, +a `FIXME` comment is used to flag the loss. + +IDs use the prefixes: + fn_ - single Problem + suite_ - Suite + gen_ - Generator + impl_ - Implementation +""" + +from opltools import ( + Library, + Problem, + Suite, + Generator, + Implementation, + Reference, + Link, + Variable, + Constraint, + ValueRange, +) +from pydantic_yaml import to_yaml_str + +things = {} + + +# ===================================================================== +# Shared implementations (reused by multiple YAML entries). +# ===================================================================== + +things["impl_coco"] = Implementation( + name="COCO framework", + description="Comparing Continuous Optimizers: black-box optimization benchmarking platform", + language="C/Python", + links=[Link(type="repository", url="https://github.com/numbbo/coco")], +) + +things["impl_coco_legacy"] = Implementation( + name="COCO legacy (bbob-noisy)", + description="Archived COCO download page that hosted the bbob-noisy suite", + language="C/Python", + links=[ + Link( + type="archive", + url="https://web.archive.org/web/20210416065610/https://coco.gforge.inria.fr/doku.php?id=downloads", + ) + ], +) + +things["impl_iohexperimenter"] = Implementation( + name="IOHexperimenter", + description="IOHprofiler experimenter framework", + language="C++/Python", + links=[Link(type="repository", url="https://github.com/IOHprofiler/IOHexperimenter")], +) + +things["impl_pymoo"] = Implementation( + name="pymoo", + description="Multi-objective optimization in Python", + language="Python", + links=[Link(type="repository", url="https://github.com/anyoptimization/pymoo")], +) + +things["impl_mocobench"] = Implementation( + name="mocobench", + description="Multi-objective combinatorial optimization benchmark", + language="C++", + links=[Link(type="repository", url="https://gitlab.com/aliefooghe/mocobench/")], +) + +things["impl_reproblems"] = Implementation( + name="reproblems", + description="Real-world inspired multi-objective optimization problem suite", + language="Python", + links=[Link(type="repository", url="https://github.com/ryojitanabe/reproblems")], +) + + +# ===================================================================== +# Entries +# ===================================================================== + +#! - name: BBOB +#! suite/generator/single: suite +#! objectives: '1' +#! dimensionality: scalable +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: 'yes' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1080/10556788.2020.1808977 +#! implementation: https://github.com/numbbo/coco +#! source (real-world/artificial): '' +#! textual description: '' +things["suite_bbob"] = Suite( + name="BBOB", + objectives={1}, + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + modality={"multimodal"}, + references=[ + Reference( + title="COCO: a platform for comparing continuous optimizers in a black-box setting", + authors=[], + link=Link(url="https://doi.org/10.1080/10556788.2020.1808977"), + ) + ], + implementations={"impl_coco"}, +) + +#! - name: BBOB-biobj +#! suite/generator/single: suite +#! objectives: '2' +#! dimensionality: 2-40 +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: 'yes' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.48550/arXiv.1604.00359 +#! implementation: https://github.com/numbbo/coco +#! source (real-world/artificial): '' +#! textual description: '' +things["suite_bbob_biobj"] = Suite( + name="BBOB-biobj", + objectives={2}, + variables=[Variable(type="continuous", dim=ValueRange(min=2, max=40))], + modality={"multimodal"}, + references=[ + Reference( + title="BBOB bi-objective test suite", + authors=[], + link=Link(url="https://doi.org/10.48550/arXiv.1604.00359"), + ) + ], + implementations={"impl_coco"}, +) + +#! - name: BBOB-noisy +#! suite/generator/single: suite +#! objectives: '1' +#! dimensionality: scalable +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'yes' +#! multimodal: 'yes' +#! multi-fidelity: 'no' +#! reference: https://hal.inria.fr/inria-00369466 +#! implementation: https://web.archive.org/web/20210416065610/https://coco.gforge.inria.fr/doku.php?id=downloads +#! source (real-world/artificial): '' +#! textual description: '' +things["suite_bbob_noisy"] = Suite( + name="BBOB-noisy", + objectives={1}, + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + modality={"multimodal"}, + noise_type={"noisy"}, + references=[ + Reference( + title="Real-parameter black-box optimization benchmarking: noisy functions definitions", + authors=[], + link=Link(url="https://hal.inria.fr/inria-00369466"), + ) + ], + implementations={"impl_coco_legacy"}, +) + +#! - name: BBOB-largescale +#! suite/generator/single: suite +#! objectives: '1' +#! dimensionality: 20-640 +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: 'yes' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.48550/arXiv.1903.06396 +#! implementation: https://github.com/numbbo/coco +#! source (real-world/artificial): '' +#! textual description: '' +things["suite_bbob_largescale"] = Suite( + name="BBOB-largescale", + objectives={1}, + variables=[Variable(type="continuous", dim=ValueRange(min=20, max=640))], + modality={"multimodal"}, + references=[ + Reference( + title="BBOB large-scale test suite", + authors=[], + link=Link(url="https://doi.org/10.48550/arXiv.1903.06396"), + ) + ], + implementations={"impl_coco"}, +) + +#! - name: BBOB-mixint +#! suite/generator/single: suite +#! objectives: '1' +#! dimensionality: 5-160 +#! variable type: integer;continuous;mixed +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: 'yes' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1145/3321707.3321868 +#! implementation: https://github.com/numbbo/coco +#! source (real-world/artificial): '' +#! textual description: '' +things["suite_bbob_mixint"] = Suite( + name="BBOB-mixint", + objectives={1}, + variables=[ + Variable(type="continuous", dim=ValueRange(min=5, max=160)), + Variable(type="integer", dim=ValueRange(min=5, max=160)), + ], + modality={"multimodal"}, + references=[ + Reference( + title="BBOB mixed-integer test suite", + authors=[], + link=Link(url="https://doi.org/10.1145/3321707.3321868"), + ) + ], + implementations={"impl_coco"}, +) + +#! - name: BBOB-biobj-mixint +#! suite/generator/single: suite +#! objectives: '2' +#! dimensionality: 5-160 +#! variable type: integer;continuous;mixed +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: 'yes' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1145/3321707.3321868 +#! implementation: https://github.com/numbbo/coco +#! source (real-world/artificial): '' +#! textual description: '' +things["suite_bbob_biobj_mixint"] = Suite( + name="BBOB-biobj-mixint", + objectives={2}, + variables=[ + Variable(type="continuous", dim=ValueRange(min=5, max=160)), + Variable(type="integer", dim=ValueRange(min=5, max=160)), + ], + modality={"multimodal"}, + references=[ + Reference( + title="BBOB bi-objective mixed-integer test suite", + authors=[], + link=Link(url="https://doi.org/10.1145/3321707.3321868"), + ) + ], + implementations={"impl_coco"}, +) + +#! - name: BBOB-constrained +#! suite/generator/single: suite +#! objectives: '1' +#! dimensionality: 2-40 +#! variable type: continuous +#! constraints: 'yes' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: 'yes' +#! multi-fidelity: 'no' +#! reference: http://numbbo.github.io/coco-doc/bbob-constrained/ +#! implementation: https://github.com/numbbo/coco +#! source (real-world/artificial): '' +#! textual description: '' +things["suite_bbob_constrained"] = Suite( + name="BBOB-constrained", + objectives={1}, + variables=[Variable(type="continuous", dim=ValueRange(min=2, max=40))], + constraints=[Constraint(hard="yes")], + modality={"multimodal"}, + references=[ + Reference( + title="bbob-constrained documentation", + authors=[], + link=Link(url="http://numbbo.github.io/coco-doc/bbob-constrained/"), + ) + ], + implementations={"impl_coco"}, +) + +#! - name: MOrepo +#! suite/generator/single: suite +#! objectives: '2' +#! dimensionality: '?' +#! variable type: combinatorial +#! constraints: '?' +#! dynamic: '?' +#! noise: '?' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: '' +#! implementation: https://github.com/MCDMSociety/MOrepo +#! source (real-world/artificial): '' +#! textual description: '' +things["impl_morepo"] = Implementation( + name="MOrepo", + description="Multi-objective optimisation problem repository", + links=[Link(type="repository", url="https://github.com/MCDMSociety/MOrepo")], +) +# FIXME: "combinatorial" has no direct VariableType; dimensionality "?" unknown. +things["suite_morepo"] = Suite( + name="MOrepo", + objectives={2}, + variables=[Variable(type="unknown")], + constraints=[Constraint(hard="?")], + dynamic_type={"unknown"}, + noise_type={"unknown"}, + implementations={"impl_morepo"}, +) + +#! - name: ZDT +#! suite/generator/single: suite +#! objectives: '2' +#! dimensionality: scalable +#! variable type: continuous;binary +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1162/106365600568202 +#! implementation: https://github.com/anyoptimization/pymoo +#! source (real-world/artificial): '' +#! textual description: '' +things["suite_zdt"] = Suite( + name="ZDT", + objectives={2}, + variables=[ + Variable(type="continuous", dim=ValueRange(min=1)), + Variable(type="binary", dim=ValueRange(min=1)), + ], + references=[ + Reference( + title="Comparison of multiobjective evolutionary algorithms: empirical results", + authors=["Eckart Zitzler", "Kalyanmoy Deb", "Lothar Thiele"], + link=Link(url="https://doi.org/10.1162/106365600568202"), + ) + ], + implementations={"impl_pymoo"}, +) + +#! - name: DTLZ +#! suite/generator/single: suite +#! objectives: 2+ +#! dimensionality: scalable +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1109/CEC.2002.1007032 +#! implementation: https://pymoo.org/problems/many/dtlz.html +#! source (real-world/artificial): '' +#! textual description: '' +things["suite_dtlz"] = Suite( + name="DTLZ", + # FIXME: original "2+" - schema requires set[int]; truncated to 2..10. + objectives=set(range(2, 11)), + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + references=[ + Reference( + title="Scalable multi-objective optimization test problems", + authors=["Kalyanmoy Deb", "Lothar Thiele", "Marco Laumanns", "Eckart Zitzler"], + link=Link(url="https://doi.org/10.1109/CEC.2002.1007032"), + ) + ], + implementations={"impl_pymoo"}, +) + +#! - name: WFG +#! suite/generator/single: suite +#! objectives: 2+ +#! dimensionality: scalable +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1109/TEVC.2005.861417 +#! implementation: https://pymoo.org/problems/many/wfg.html +#! source (real-world/artificial): '' +#! textual description: '' +things["suite_wfg"] = Suite( + name="WFG", + # FIXME: original "2+" - truncated to 2..10. + objectives=set(range(2, 11)), + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + references=[ + Reference( + title="A review of multiobjective test problems and a scalable test problem toolkit", + authors=["Simon Huband", "Philip Hingston", "Luigi Barone", "Lyndon While"], + link=Link(url="https://doi.org/10.1109/TEVC.2005.861417"), + ) + ], + implementations={"impl_pymoo"}, +) + +#! - name: CDMP +#! suite/generator/single: suite +#! objectives: 2+ +#! dimensionality: scalable +#! variable type: continuous +#! constraints: 'yes' +#! dynamic: '?' +#! noise: '?' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1145/3321707.3321878 +#! implementation: '?' +#! source (real-world/artificial): '' +#! textual description: '' +# FIXME: implementation unknown. +things["suite_cdmp"] = Suite( + name="CDMP", + # FIXME: original "2+" - truncated to 2..10. + objectives=set(range(2, 11)), + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + constraints=[Constraint(hard="yes")], + dynamic_type={"unknown"}, + noise_type={"unknown"}, + references=[ + Reference( + title="CDMP benchmark", + authors=[], + link=Link(url="https://doi.org/10.1145/3321707.3321878"), + ) + ], +) + +#! - name: SDP +#! suite/generator/single: suite +#! objectives: 2+ +#! dimensionality: scalable +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'yes' +#! noise: '?' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1109/TCYB.2019.2896021 +#! implementation: '?' +#! source (real-world/artificial): '' +#! textual description: '' +# FIXME: implementation unknown. +things["suite_sdp"] = Suite( + name="SDP", + # FIXME: original "2+" - truncated to 2..10. + objectives=set(range(2, 11)), + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + dynamic_type={"dynamic"}, + noise_type={"unknown"}, + references=[ + Reference( + title="SDP dynamic multi-objective benchmark", + authors=[], + link=Link(url="https://doi.org/10.1109/TCYB.2019.2896021"), + ) + ], +) + +#! - name: MaOP +#! suite/generator/single: suite +#! objectives: 2+ +#! dimensionality: scalable +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'no' +#! noise: '?' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1016/j.swevo.2019.02.003 +#! implementation: '?' +#! source (real-world/artificial): '' +#! textual description: '' +# FIXME: implementation unknown. +things["suite_maop"] = Suite( + name="MaOP", + # FIXME: original "2+" - truncated to 2..10. + objectives=set(range(2, 11)), + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + noise_type={"unknown"}, + references=[ + Reference( + title="MaOP benchmark", + authors=[], + link=Link(url="https://doi.org/10.1016/j.swevo.2019.02.003"), + ) + ], +) + +#! - name: BP +#! suite/generator/single: suite +#! objectives: 2+ +#! dimensionality: scalable +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'no' +#! noise: '?' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1109/CEC.2019.8790277 +#! implementation: '?' +#! source (real-world/artificial): '' +#! textual description: '' +# FIXME: implementation unknown. +things["suite_bp"] = Suite( + name="BP", + # FIXME: original "2+" - truncated to 2..10. + objectives=set(range(2, 11)), + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + noise_type={"unknown"}, + references=[ + Reference( + title="BP benchmark", + authors=[], + link=Link(url="https://doi.org/10.1109/CEC.2019.8790277"), + ) + ], +) + +#! - name: GPD +#! suite/generator/single: generator +#! objectives: 2+ +#! dimensionality: scalable +#! variable type: continuous +#! constraints: optional +#! dynamic: 'no' +#! noise: optional +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1016/j.asoc.2020.106139 +#! implementation: '?' +#! source (real-world/artificial): '' +#! textual description: '' +# FIXME: implementation unknown. +things["gen_gpd"] = Generator( + name="GPD", + # FIXME: original "2+" - truncated to 2..10. + objectives=set(range(2, 11)), + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + constraints=[Constraint(hard="some")], + noise_type={"optional"}, + references=[ + Reference( + title="GPD generator", + authors=[], + link=Link(url="https://doi.org/10.1016/j.asoc.2020.106139"), + ) + ], +) + +#! - name: ETMOF +#! suite/generator/single: suite +#! objectives: 2-50 +#! dimensionality: 25-10000 +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'yes' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.48550/arXiv.2110.08033 +#! implementation: https://github.com/songbai-liu/etmo +#! source (real-world/artificial): '' +#! textual description: '' +things["impl_etmof"] = Implementation( + name="ETMOF", + description="Evolutionary many-task optimization framework", + links=[Link(type="repository", url="https://github.com/songbai-liu/etmo")], +) +things["suite_etmof"] = Suite( + name="ETMOF", + objectives=set(range(2, 51)), + variables=[Variable(type="continuous", dim=ValueRange(min=25, max=10000))], + dynamic_type={"dynamic"}, + references=[ + Reference( + title="Evolutionary many-task optimization framework", + authors=[], + link=Link(url="https://doi.org/10.48550/arXiv.2110.08033"), + ) + ], + implementations={"impl_etmof"}, +) + +#! - name: MMOPP +#! suite/generator/single: suite +#! objectives: 2-7 +#! dimensionality: '?' +#! variable type: '?' +#! constraints: 'yes' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: 'yes' +#! multi-fidelity: 'no' +#! reference: http://www5.zzu.edu.cn/system/_content/download.jsp?urltype=news.DownloadAttachUrl&owner=1327567121&wbfileid=4764412 +#! implementation: http://www5.zzu.edu.cn/ecilab/info/1036/1251.htm +#! source (real-world/artificial): '' +#! textual description: '' +things["impl_mmopp"] = Implementation( + name="MMOPP", + description="ECI lab distribution page for MMOPP", + links=[Link(type="website", url="http://www5.zzu.edu.cn/ecilab/info/1036/1251.htm")], +) +# FIXME: variable type and dimensionality unknown ("?"). +things["suite_mmopp"] = Suite( + name="MMOPP", + objectives=set(range(2, 8)), + variables=[Variable(type="unknown")], + constraints=[Constraint(hard="yes")], + modality={"multimodal"}, + references=[ + Reference( + title="MMOPP technical report", + authors=[], + link=Link( + url="http://www5.zzu.edu.cn/system/_content/download.jsp?urltype=news.DownloadAttachUrl&owner=1327567121&wbfileid=4764412" + ), + ) + ], + implementations={"impl_mmopp"}, +) + +#! - name: CFD +#! suite/generator/single: suite +#! objectives: 1-2 +#! dimensionality: scalable +#! variable type: '?' +#! constraints: 'yes' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1007/978-3-319-99259-4_24 +#! implementation: https://bitbucket.org/arahat/cfd-test-problem-suite +#! source (real-world/artificial): real world +#! textual description: expensive evaluations 30s-15m +things["impl_cfd"] = Implementation( + name="CFD test problem suite", + description="Expensive real-world CFD-based test problems", + evaluation_time="30s-15m", + links=[Link(type="repository", url="https://bitbucket.org/arahat/cfd-test-problem-suite")], +) +# FIXME: variable type unknown. +things["suite_cfd"] = Suite( + name="CFD", + description="expensive evaluations 30s-15m", + objectives={1, 2}, + variables=[Variable(type="unknown", dim=ValueRange(min=1))], + constraints=[Constraint(hard="yes")], + source={"real-world"}, + references=[ + Reference( + title="CFD test problem suite", + authors=[], + link=Link(url="https://doi.org/10.1007/978-3-319-99259-4_24"), + ) + ], + implementations={"impl_cfd"}, +) + +#! - name: GBEA +#! suite/generator/single: suite +#! objectives: 1-2 +#! dimensionality: scalable +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'yes' +#! multimodal: 'yes' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1145/3321707.3321805 +#! implementation: 'https://github.com/ttusar/coco-gbea' +#! source (real-world/artificial): real world +#! textual description: 'expensive evaluations 5s-35s, RW-GAN-Mario and TopTrumps are part of GBEA' +things["impl_gbea"] = Implementation( + name="coco-gbea", + description="Game-Benchmark for Evolutionary Algorithms (COCO fork)", + evaluation_time="5s-35s", + links=[Link(type="repository", url="https://github.com/ttusar/coco-gbea")], +) +things["suite_gbea"] = Suite( + name="GBEA", + description="expensive evaluations 5s-35s, RW-GAN-Mario and TopTrumps are part of GBEA", + objectives={1, 2}, + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + noise_type={"noisy"}, + modality={"multimodal"}, + source={"real-world"}, + references=[ + Reference( + title="Game benchmark for evolutionary algorithms", + authors=[], + link=Link(url="https://doi.org/10.1145/3321707.3321805"), + ) + ], + implementations={"impl_gbea"}, +) + +#! - name: Car structure +#! suite/generator/single: suite +#! objectives: '2' +#! dimensionality: 144-222 +#! variable type: discrete +#! constraints: 'yes' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1145/3205651.3205702 +#! implementation: http://ladse.eng.isas.jaxa.jp/benchmark/ +#! source (real-world/artificial): real world +#! textual description: 54 constraints +things["impl_car_structure"] = Implementation( + name="Car-structure benchmark", + description="JAXA LADSE benchmark problems", + links=[Link(type="website", url="http://ladse.eng.isas.jaxa.jp/benchmark/")], +) +# FIXME: "discrete" has no direct VariableType - using integer. +things["suite_car_structure"] = Suite( + name="Car structure", + description="54 constraints", + objectives={2}, + variables=[Variable(type="integer", dim=ValueRange(min=144, max=222))], + constraints=[Constraint(hard="yes", number=54)], + source={"real-world"}, + references=[ + Reference( + title="Car structure design benchmark", + authors=[], + link=Link(url="https://doi.org/10.1145/3205651.3205702"), + ) + ], + implementations={"impl_car_structure"}, +) + +#! - name: EMO2017 +#! suite/generator/single: suite +#! objectives: '2' +#! dimensionality: 4-24 +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://www.ini.rub.de/PEOPLE/glasmtbl/projects/bbcomp/ +#! implementation: https://www.ini.rub.de/PEOPLE/glasmtbl/projects/bbcomp/downloads/realworld-problems-bbcomp-EMO-2017.zip +#! source (real-world/artificial): real world +#! textual description: '' +things["impl_emo2017"] = Implementation( + name="EMO 2017 real-world problems", + description="BBComp EMO-2017 real-world problem archive", + links=[ + Link( + type="download", + url="https://www.ini.rub.de/PEOPLE/glasmtbl/projects/bbcomp/downloads/realworld-problems-bbcomp-EMO-2017.zip", + ) + ], +) +things["suite_emo2017"] = Suite( + name="EMO2017", + objectives={2}, + variables=[Variable(type="continuous", dim=ValueRange(min=4, max=24))], + source={"real-world"}, + references=[ + Reference( + title="BBComp EMO 2017", + authors=[], + link=Link(url="https://www.ini.rub.de/PEOPLE/glasmtbl/projects/bbcomp/"), + ) + ], + implementations={"impl_emo2017"}, +) + +#! - name: JSEC2019 +#! suite/generator/single: single +#! objectives: 1-5 +#! dimensionality: '32' +#! variable type: continuous +#! constraints: 'yes' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: http://www.jpnsec.org/files/competition2019/EC-Symposium-2019-Competition-English.html +#! implementation: http://www.jpnsec.org/files/competition2019/EC-Symposium-2019-Competition-English.html +#! source (real-world/artificial): real world +#! textual description: expensive evaluations 3s; 22 constraints +things["impl_jsec2019"] = Implementation( + name="JSEC 2019 competition", + description="JPNSEC EC-Symposium 2019 competition problem", + evaluation_time="3s", + links=[ + Link( + type="website", + url="http://www.jpnsec.org/files/competition2019/EC-Symposium-2019-Competition-English.html", + ) + ], +) +things["fn_jsec2019"] = Problem( + name="JSEC2019", + description="expensive evaluations 3s; 22 constraints", + objectives={1, 2, 3, 4, 5}, + variables=[Variable(type="continuous", dim=32)], + constraints=[Constraint(hard="yes", number=22)], + source={"real-world"}, + references=[ + Reference( + title="JPNSEC EC-Symposium 2019 competition", + authors=[], + link=Link( + url="http://www.jpnsec.org/files/competition2019/EC-Symposium-2019-Competition-English.html" + ), + ) + ], + implementations={"impl_jsec2019"}, +) + +#! - name: RE +#! suite/generator/single: suite +#! objectives: 2-9 +#! dimensionality: 2-7 +#! variable type: continuous;integer;mixed +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1016/j.asoc.2020.106078 +#! implementation: https://github.com/ryojitanabe/reproblems +#! source (real-world/artificial): real world like +#! textual description: '' +things["suite_re"] = Suite( + name="RE", + objectives=set(range(2, 10)), + variables=[ + Variable(type="continuous", dim=ValueRange(min=2, max=7)), + Variable(type="integer", dim=ValueRange(min=2, max=7)), + ], + source={"real-world-like"}, + references=[ + Reference( + title="Easy-to-evaluate real-world multi-objective optimization problems", + authors=["Ryoji Tanabe", "Hisao Ishibuchi"], + link=Link(url="https://doi.org/10.1016/j.asoc.2020.106078"), + ) + ], + implementations={"impl_reproblems"}, +) + +#! - name: CRE +#! suite/generator/single: suite +#! objectives: 2-5 +#! dimensionality: 3-7 +#! variable type: continuous;integer;mixed +#! constraints: 'yes' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1016/j.asoc.2020.106078 +#! implementation: https://github.com/ryojitanabe/reproblems +#! source (real-world/artificial): real world like +#! textual description: '' +things["suite_cre"] = Suite( + name="CRE", + objectives={2, 3, 4, 5}, + variables=[ + Variable(type="continuous", dim=ValueRange(min=3, max=7)), + Variable(type="integer", dim=ValueRange(min=3, max=7)), + ], + constraints=[Constraint(hard="yes")], + source={"real-world-like"}, + references=[ + Reference( + title="Easy-to-evaluate real-world multi-objective optimization problems", + authors=["Ryoji Tanabe", "Hisao Ishibuchi"], + link=Link(url="https://doi.org/10.1016/j.asoc.2020.106078"), + ) + ], + implementations={"impl_reproblems"}, +) + +#! - name: Radar waveform +#! suite/generator/single: single +#! objectives: '9' +#! dimensionality: 4-12 +#! variable type: integer +#! constraints: 'yes' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1007/978-3-540-70928-2_53 +#! implementation: http://code.evanhughes.org/ +#! source (real-world/artificial): real world +#! textual description: '' +things["impl_radar_waveform"] = Implementation( + name="Evan Hughes radar waveform code", + description="Radar waveform design reference implementation", + links=[Link(type="website", url="http://code.evanhughes.org/")], +) +things["fn_radar_waveform"] = Problem( + name="Radar waveform", + objectives={9}, + variables=[Variable(type="integer", dim=ValueRange(min=4, max=12))], + constraints=[Constraint(hard="yes")], + source={"real-world"}, + references=[ + Reference( + title="Radar waveform design", + authors=[], + link=Link(url="https://doi.org/10.1007/978-3-540-70928-2_53"), + ) + ], + implementations={"impl_radar_waveform"}, +) + +#! - name: MF2 +#! suite/generator/single: suite +#! objectives: '1' +#! dimensionality: 1-n +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'yes' +#! reference: https://doi.org/10.21105/joss.02049 +#! implementation: https://github.com/sjvrijn/mf2 +#! source (real-world/artificial): '' +#! textual description: '' +things["impl_mf2"] = Implementation( + name="mf2", + description="Multi-fidelity test function collection", + language="Python", + links=[Link(type="repository", url="https://github.com/sjvrijn/mf2")], +) +things["suite_mf2"] = Suite( + name="MF2", + objectives={1}, + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + fidelity_levels={1, 2}, + references=[ + Reference( + title="mf2: a collection of multi-fidelity benchmark functions in Python", + authors=[], + link=Link(url="https://doi.org/10.21105/joss.02049"), + ) + ], + implementations={"impl_mf2"}, +) + +#! - name: AMVOP +#! suite/generator/single: suite +#! objectives: '1' +#! dimensionality: scalable +#! variable type: mixed continuous+ordinal+categorical+both +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: 'yes' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1109/TEVC.2013.2281531 +#! implementation: '?' +#! source (real-world/artificial): '' +#! textual description: '' +# FIXME: implementation unknown. "ordinal" not representable, using integer+categorical+continuous. +things["suite_amvop"] = Suite( + name="AMVOP", + objectives={1}, + variables=[ + Variable(type="continuous", dim=ValueRange(min=1)), + Variable(type="integer", dim=ValueRange(min=1)), + Variable(type="categorical", dim=ValueRange(min=1)), + ], + modality={"multimodal"}, + references=[ + Reference( + title="AMVOP", + authors=[], + link=Link(url="https://doi.org/10.1109/TEVC.2013.2281531"), + ) + ], +) + +#! - name: RWMVOP +#! suite/generator/single: suite +#! objectives: '1' +#! dimensionality: scalable +#! variable type: continuous;mixed continuous+ordinal+categorical+both +#! constraints: 'yes' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1109/TEVC.2013.2281531 +#! implementation: '?' +#! source (real-world/artificial): real world +#! textual description: '' +# FIXME: implementation unknown. +things["suite_rwmvop"] = Suite( + name="RWMVOP", + objectives={1}, + variables=[ + Variable(type="continuous", dim=ValueRange(min=1)), + Variable(type="integer", dim=ValueRange(min=1)), + Variable(type="categorical", dim=ValueRange(min=1)), + ], + constraints=[Constraint(hard="yes")], + source={"real-world"}, + references=[ + Reference( + title="RWMVOP", + authors=[], + link=Link(url="https://doi.org/10.1109/TEVC.2013.2281531"), + ) + ], +) + +#! - name: SBOX-COST +#! suite/generator/single: suite +#! objectives: '1' +#! dimensionality: scalable +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: 'yes' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.48550/arXiv.2305.12221 +#! implementation: https://github.com/IOHprofiler/IOHexperimenter/ +#! source (real-world/artificial): '' +#! textual description: problems from BBOB but allows instances with the optimum close to the +#! boundary +things["suite_sbox_cost"] = Suite( + name="SBOX-COST", + description="problems from BBOB but allows instances with the optimum close to the boundary", + objectives={1}, + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + modality={"multimodal"}, + references=[ + Reference( + title="SBOX-COST", + authors=[], + link=Link(url="https://doi.org/10.48550/arXiv.2305.12221"), + ) + ], + implementations={"impl_iohexperimenter"}, +) + +#! - name: "\u03C1MNK-Landscapes" +#! suite/generator/single: generator +#! objectives: scalable +#! dimensionality: scalable +#! variable type: binary +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: 'yes' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1016/j.ejor.2012.12.019 +#! implementation: https://gitlab.com/aliefooghe/mocobench/ +#! source (real-world/artificial): '' +#! textual description: tunable variable and objective dimensions; tunable multimodality and +#! correlation between objectives +things["gen_rho_mnk_landscapes"] = Generator( + name="ρMNK-Landscapes", + description="tunable variable and objective dimensions; tunable multimodality and correlation between objectives", + # FIXME: original "scalable" - truncated to 1..10. + objectives=set(range(1, 11)), + variables=[Variable(type="binary", dim=ValueRange(min=1))], + modality={"multimodal"}, + references=[ + Reference( + title="On the design of multi-objective evolutionary algorithms based on NK-landscapes", + authors=[], + link=Link(url="https://doi.org/10.1016/j.ejor.2012.12.019"), + ) + ], + implementations={"impl_mocobench"}, +) + +#! - name: mUBQP +#! suite/generator/single: generator +#! objectives: scalable +#! dimensionality: scalable +#! variable type: binary +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: yes (quadratic) +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1016/j.asoc.2013.11.008 +#! implementation: https://gitlab.com/aliefooghe/mocobench/ +#! source (real-world/artificial): '' +#! textual description: tunable variable and objective dimensions; tunable density and correlation +#! between objectives +things["gen_mubqp"] = Generator( + name="mUBQP", + description="tunable variable and objective dimensions; tunable density and correlation between objectives", + # FIXME: original "scalable" - truncated to 1..10. + objectives=set(range(1, 11)), + variables=[Variable(type="binary", dim=ValueRange(min=1))], + modality={"multimodal", "quadratic"}, + references=[ + Reference( + title="mUBQP benchmark", + authors=[], + link=Link(url="https://doi.org/10.1016/j.asoc.2013.11.008"), + ) + ], + implementations={"impl_mocobench"}, +) + +#! - name: "\u03C1mTSP" +#! suite/generator/single: generator +#! objectives: scalable +#! dimensionality: scalable +#! variable type: permutations +#! constraints: no (apart from being permutations) +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: yes (quadratic) +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1007/978-3-319-45823-6_40 +#! implementation: https://gitlab.com/aliefooghe/mocobench/ +#! source (real-world/artificial): '' +#! textual description: tunable variable and objective dimensions; tunable instance type (euclidian/random); +#! tunable correlation between objectives +# FIXME: "permutations" has no direct VariableType; constraints are implicit permutations. +things["gen_rho_mtsp"] = Generator( + name="ρmTSP", + description="tunable variable and objective dimensions; tunable instance type (euclidean/random); tunable correlation between objectives", + # FIXME: original "scalable" - truncated to 1..10. + objectives=set(range(1, 11)), + variables=[Variable(type="unknown", dim=ValueRange(min=1))], + modality={"multimodal", "quadratic"}, + references=[ + Reference( + title="On the impact of multi-objective scalability for the ρmTSP", + authors=[], + link=Link(url="https://doi.org/10.1007/978-3-319-45823-6_40"), + ) + ], + implementations={"impl_mocobench"}, +) + +#! - name: CEC2015-DMOO +#! suite/generator/single: suite +#! objectives: 2-3 +#! dimensionality: '?' +#! variable type: continuous +#! constraints: '?' +#! dynamic: 'yes' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: Benchmark Functions for CEC 2015 Special Session and Competition on Dynamic +#! Multi-objective Optimization +#! implementation: '' +#! source (real-world/artificial): '' +#! textual description: '' +# FIXME: reference is a title-only string; implementation unavailable; dimensionality unknown. +things["suite_cec2015_dmoo"] = Suite( + name="CEC2015-DMOO", + objectives={2, 3}, + variables=[Variable(type="continuous")], + constraints=[Constraint(hard="?")], + dynamic_type={"dynamic"}, + references=[ + Reference( + title="Benchmark Functions for CEC 2015 Special Session and Competition on Dynamic Multi-objective Optimization", + authors=[], + ) + ], +) + +#! - name: Ealain +#! suite/generator/single: generator +#! objectives: 1+ +#! dimensionality: scalable +#! variable type: continuous,binary,integer +#! constraints: optional +#! dynamic: optional +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: optional +#! reference: https://doi.org/10.1145/3638530.3654299 +#! implementation: https://github.com/qrenau/Ealain +#! source (real-world/artificial): Real-world-like +#! textual description: Real-world-like, easily extensible to increase complexity +things["impl_ealain"] = Implementation( + name="Ealain", + description="Real-world-like extensible benchmark problem generator", + links=[Link(type="repository", url="https://github.com/qrenau/Ealain")], +) +things["gen_ealain"] = Generator( + name="Ealain", + description="Real-world-like, easily extensible to increase complexity", + # FIXME: original "1+" - truncated to 1..10. + objectives=set(range(1, 11)), + variables=[ + Variable(type="continuous", dim=ValueRange(min=1)), + Variable(type="binary", dim=ValueRange(min=1)), + Variable(type="integer", dim=ValueRange(min=1)), + ], + constraints=[Constraint(hard="some")], + dynamic_type={"optional"}, + fidelity_levels={1, 2}, + source={"real-world-like"}, + references=[ + Reference( + title="Ealain", + authors=[], + link=Link(url="https://doi.org/10.1145/3638530.3654299"), + ) + ], + implementations={"impl_ealain"}, +) + +#! - name: MA-BBOB +#! suite/generator/single: generator +#! objectives: '1' +#! dimensionality: scalable +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: 'yes' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1145/3673908 +#! implementation: https://github.com/IOHprofiler/IOHexperimenter/blob/master/example/Competitions/MA-BBOB/Example_MABBOB.ipynb +#! source (real-world/artificial): artificial +#! textual description: Generator that creates affine combinations of BBOB functions +things["impl_ma_bbob"] = Implementation( + name="MA-BBOB (IOHexperimenter)", + description="Example notebook for MA-BBOB in IOHexperimenter", + links=[ + Link( + type="example", + url="https://github.com/IOHprofiler/IOHexperimenter/blob/master/example/Competitions/MA-BBOB/Example_MABBOB.ipynb", + ) + ], +) +things["gen_ma_bbob"] = Generator( + name="MA-BBOB", + description="Generator that creates affine combinations of BBOB functions", + objectives={1}, + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + modality={"multimodal"}, + source={"artificial"}, + references=[ + Reference( + title="MA-BBOB", + authors=[], + link=Link(url="https://doi.org/10.1145/3673908"), + ) + ], + implementations={"impl_ma_bbob", "impl_iohexperimenter"}, +) + +#! - name: MPM2 +#! suite/generator/single: generator +#! objectives: '1' +#! dimensionality: scalable +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: 'yes' +#! multi-fidelity: 'no' +#! reference: https://ls11-www.cs.tu-dortmund.de/_media/techreports/tr15-01.pdf +#! implementation: https://github.com/jakobbossek/smoof/blob/master/inst/mpm2.py +#! source (real-world/artificial): '' +#! textual description: nonlinear nonseparable nonsymmetric; scalable in terms of time to evaluate +#! the objective function +things["impl_mpm2"] = Implementation( + name="MPM2 (smoof)", + description="Python implementation of MPM2 distributed with smoof", + language="Python", + links=[ + Link( + type="source", + url="https://github.com/jakobbossek/smoof/blob/master/inst/mpm2.py", + ) + ], +) +things["gen_mpm2"] = Generator( + name="MPM2", + description="nonlinear nonseparable nonsymmetric; scalable in terms of time to evaluate the objective function", + objectives={1}, + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + modality={"multimodal"}, + references=[ + Reference( + title="MPM2 technical report TR15-01", + authors=[], + link=Link(url="https://ls11-www.cs.tu-dortmund.de/_media/techreports/tr15-01.pdf"), + ) + ], + implementations={"impl_mpm2"}, +) + +#! - name: Convex DTLZ2 +#! suite/generator/single: single +#! objectives: 2+ +#! dimensionality: scalable +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1109/TEVC.2013.2281535 +#! implementation: '?' +#! source (real-world/artificial): '' +#! textual description: Variant of DTLZ2 with a convex Pareto front (instead of concave) +# FIXME: implementation unknown. +things["fn_convex_dtlz2"] = Problem( + name="Convex DTLZ2", + description="Variant of DTLZ2 with a convex Pareto front (instead of concave)", + # FIXME: original "2+" - truncated to 2..10. + objectives=set(range(2, 11)), + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + references=[ + Reference( + title="Convex DTLZ2", + authors=[], + link=Link(url="https://doi.org/10.1109/TEVC.2013.2281535"), + ) + ], +) + +#! - name: Inverted DTLZ1 +#! suite/generator/single: single +#! objectives: 2+ +#! dimensionality: scalable +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1109/TEVC.2013.2281534 +#! implementation: '?' +#! source (real-world/artificial): '' +#! textual description: Variant of DTLZ1 with an inverted Pareto front +# FIXME: implementation unknown. +things["fn_inverted_dtlz1"] = Problem( + name="Inverted DTLZ1", + description="Variant of DTLZ1 with an inverted Pareto front", + # FIXME: original "2+" - truncated to 2..10. + objectives=set(range(2, 11)), + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + references=[ + Reference( + title="Inverted DTLZ1", + authors=[], + link=Link(url="https://doi.org/10.1109/TEVC.2013.2281534"), + ) + ], +) + +#! - name: Minus DTLZ +#! suite/generator/single: suite +#! objectives: 2+ +#! dimensionality: scalable +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1109/TEVC.2016.2587749 +#! implementation: '?' +#! source (real-world/artificial): '' +#! textual description: Variant of DTLZ that minimises the inverse of the base DTLZ functions +# FIXME: implementation unknown. +things["suite_minus_dtlz"] = Suite( + name="Minus DTLZ", + description="Variant of DTLZ that minimises the inverse of the base DTLZ functions", + # FIXME: original "2+" - truncated to 2..10. + objectives=set(range(2, 11)), + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + references=[ + Reference( + title="Minus DTLZ / Minus WFG", + authors=[], + link=Link(url="https://doi.org/10.1109/TEVC.2016.2587749"), + ) + ], +) + +#! - name: Minus WFG +#! suite/generator/single: suite +#! objectives: 2+ +#! dimensionality: scalable +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1109/TEVC.2016.2587749 +#! implementation: '?' +#! source (real-world/artificial): '' +#! textual description: Variant of WFG that minimises the inverse of the base WFG functions +# FIXME: implementation unknown. +things["suite_minus_wfg"] = Suite( + name="Minus WFG", + description="Variant of WFG that minimises the inverse of the base WFG functions", + # FIXME: original "2+" - truncated to 2..10. + objectives=set(range(2, 11)), + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + references=[ + Reference( + title="Minus DTLZ / Minus WFG", + authors=[], + link=Link(url="https://doi.org/10.1109/TEVC.2016.2587749"), + ) + ], +) + +#! - name: L1-ZDT +#! suite/generator/single: suite +#! objectives: '2' +#! dimensionality: scalable +#! variable type: continuous;binary +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1145/1143997.1144179 +#! implementation: '?' +#! source (real-world/artificial): '' +#! textual description: Variant of ZDT with linkages between variables within one of two groups +#! but not between variables in a different group; Linear recombination operators +#! can potentially take advantage of the problem structure +# FIXME: implementation unknown. +things["suite_l1_zdt"] = Suite( + name="L1-ZDT", + description="Variant of ZDT with linkages between variables within groups", + objectives={2}, + variables=[ + Variable(type="continuous", dim=ValueRange(min=1)), + Variable(type="binary", dim=ValueRange(min=1)), + ], + references=[ + Reference( + title="Linkage ZDT/DTLZ variants", + authors=[], + link=Link(url="https://doi.org/10.1145/1143997.1144179"), + ) + ], +) + +#! - name: L2-ZDT +#! suite/generator/single: suite +#! objectives: '2' +#! dimensionality: scalable +#! variable type: continuous;binary +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1145/1143997.1144179 +#! implementation: '?' +#! source (real-world/artificial): '' +#! textual description: Variant of ZDT with linkages between all variables; Linear recombination +#! operators can potentially take advantage of the problem structure +# FIXME: implementation unknown. +things["suite_l2_zdt"] = Suite( + name="L2-ZDT", + description="Variant of ZDT with linkages between all variables", + objectives={2}, + variables=[ + Variable(type="continuous", dim=ValueRange(min=1)), + Variable(type="binary", dim=ValueRange(min=1)), + ], + references=[ + Reference( + title="Linkage ZDT/DTLZ variants", + authors=[], + link=Link(url="https://doi.org/10.1145/1143997.1144179"), + ) + ], +) + +#! - name: L3-ZDT +#! suite/generator/single: suite +#! objectives: '2' +#! dimensionality: scalable +#! variable type: continuous;binary +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1145/1143997.1144179 +#! implementation: '?' +#! source (real-world/artificial): '' +#! textual description: Variant of L2-ZDT using a mapping to prevent linear recombination operators +#! from potentially taking advantage of the problem structure +# FIXME: implementation unknown. +things["suite_l3_zdt"] = Suite( + name="L3-ZDT", + description="Variant of L2-ZDT with anti-linkage mapping", + objectives={2}, + variables=[ + Variable(type="continuous", dim=ValueRange(min=1)), + Variable(type="binary", dim=ValueRange(min=1)), + ], + references=[ + Reference( + title="Linkage ZDT/DTLZ variants", + authors=[], + link=Link(url="https://doi.org/10.1145/1143997.1144179"), + ) + ], +) + +#! - name: L2-DTLZ +#! suite/generator/single: suite +#! objectives: 2+ +#! dimensionality: scalable +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1145/1143997.1144179 +#! implementation: '?' +#! source (real-world/artificial): '' +#! textual description: Variant of DTLZ2 and DTLZ3 with linkages between all variables; Linear +#! recombination operators can potentially take advantage of the problem structure +# FIXME: implementation unknown. +things["suite_l2_dtlz"] = Suite( + name="L2-DTLZ", + description="Variant of DTLZ2/DTLZ3 with linkages between all variables", + # FIXME: original "2+" - truncated to 2..10. + objectives=set(range(2, 11)), + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + references=[ + Reference( + title="Linkage ZDT/DTLZ variants", + authors=[], + link=Link(url="https://doi.org/10.1145/1143997.1144179"), + ) + ], +) + +#! - name: L3-DTLZ +#! suite/generator/single: suite +#! objectives: 2+ +#! dimensionality: scalable +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1145/1143997.1144179 +#! implementation: '?' +#! source (real-world/artificial): '' +#! textual description: Variant of L2-DTLZ using a mapping to prevent linear recombination operators +#! from potentially taking advantage of the problem structure +# FIXME: implementation unknown. +things["suite_l3_dtlz"] = Suite( + name="L3-DTLZ", + description="Variant of L2-DTLZ with anti-linkage mapping", + # FIXME: original "2+" - truncated to 2..10. + objectives=set(range(2, 11)), + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + references=[ + Reference( + title="Linkage ZDT/DTLZ variants", + authors=[], + link=Link(url="https://doi.org/10.1145/1143997.1144179"), + ) + ], +) + +#! - name: CEC2018 DT - CEC2018 Competition on Dynamic Multiobjective Optimisation +#! suite/generator/single: suite +#! objectives: 2 or 3 +#! dimensionality: scalable? +#! variable type: '?' +#! constraints: 'no' +#! dynamic: 'yes' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://www.academia.edu/download/94499025/TR-CEC2018-DMOP-Competition.pdf +#! implementation: https://pymoo.org/problems/dynamic/df.html +#! source (real-world/artificial): artificial +#! textual description: '14 problems. Time-dependent: Pareto front/Pareto set geometry; +#! irregular Pareto front shapes; variable-linkage; number of disconnected Pareto +#! front segments; etc.' +# FIXME: variable type unknown. +things["suite_cec2018_dt"] = Suite( + name="CEC2018 DT", + long_name="CEC2018 Competition on Dynamic Multiobjective Optimisation", + description="14 problems. Time-dependent: Pareto front/Pareto set geometry; irregular Pareto front shapes; variable-linkage; number of disconnected Pareto front segments; etc.", + objectives={2, 3}, + variables=[Variable(type="unknown", dim=ValueRange(min=1))], + dynamic_type={"dynamic"}, + source={"artificial"}, + references=[ + Reference( + title="CEC2018 DMOP Competition TR", + authors=[], + link=Link(url="https://www.academia.edu/download/94499025/TR-CEC2018-DMOP-Competition.pdf"), + ) + ], + implementations={"impl_pymoo"}, +) + +#! - name: MODAct - multiobjective design of actuators +#! suite/generator/single: suite +#! objectives: 2 3 4 or 5 +#! dimensionality: '20' +#! variable type: mixed; integer and continuous +#! constraints: 'yes' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1109/TEVC.2020.3020046 +#! implementation: https://pymoo.org/problems/constrained/modact.html +#! source (real-world/artificial): real-world +#! textual description: Realistic Constrained Multi-Objective Optimization Benchmark +#! Problems from Design. Need the https://github.com/epfl-lamd/modact package installed; evaluation +#! times around 20ms +things["impl_modact"] = Implementation( + name="modact", + description="EPFL-LAMD modact package", + evaluation_time="20ms", + links=[Link(type="repository", url="https://github.com/epfl-lamd/modact")], +) +things["suite_modact"] = Suite( + name="MODAct", + long_name="multiobjective design of actuators", + description="Realistic Constrained Multi-Objective Optimization Benchmark Problems from Design.", + objectives={2, 3, 4, 5}, + variables=[ + Variable(type="continuous", dim=20), + Variable(type="integer", dim=20), + ], + constraints=[Constraint(hard="yes")], + source={"real-world"}, + references=[ + Reference( + title="MODAct", + authors=[], + link=Link(url="https://doi.org/10.1109/TEVC.2020.3020046"), + ) + ], + implementations={"impl_modact", "impl_pymoo"}, +) + +#! - name: IOHClustering +#! suite/generator/single: suite; generator +#! objectives: '1' +#! dimensionality: scalable +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: 'yes' +#! multi-fidelity: 'no ' +#! reference: https://arxiv.org/pdf/2505.09233 +#! implementation: https://github.com/IOHprofiler/IOHClustering +#! source (real-world/artificial): artificial, but based on real data +#! textual description: 'Set of benchmark problems from clustering: optimization task +#! is selecting cluster centers for a given set of data, with the number of clusters +#! defining problem dimensionality. Includes both a suite and a generator. Based on ML clustering datasets' +things["impl_iohclustering"] = Implementation( + name="IOHClustering", + description="Clustering-based optimization benchmark built on ML datasets", + links=[Link(type="repository", url="https://github.com/IOHprofiler/IOHClustering")], +) +things["suite_iohclustering"] = Suite( + name="IOHClustering", + description="Set of benchmark problems from clustering: optimization task is selecting cluster centers for a given set of data.", + objectives={1}, + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + modality={"multimodal"}, + source={"artificial-from-real-data"}, + references=[ + Reference( + title="IOHClustering", + authors=[], + link=Link(url="https://arxiv.org/pdf/2505.09233"), + ) + ], + implementations={"impl_iohclustering"}, +) +things["gen_iohclustering"] = Generator( + name="IOHClustering", + description="Generator counterpart of the IOHClustering suite.", + objectives={1}, + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + modality={"multimodal"}, + source={"artificial-from-real-data"}, + references=[ + Reference( + title="IOHClustering", + authors=[], + link=Link(url="https://arxiv.org/pdf/2505.09233"), + ) + ], + implementations={"impl_iohclustering"}, +) + +#! - name: GNBG-II +#! suite/generator/single: suite; generator +#! objectives: '1' +#! dimensionality: scalable +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://dl.acm.org/doi/pdf/10.1145/3712255.3734271 +#! implementation: https://github.com/rohitsalgotra/GNBG-II +#! source (real-world/artificial): artificial +#! textual description: Generalized Numerical Benchmark Generator (version 2). Also in IOH https://github.com/IOHprofiler/IOHGNBG +things["impl_gnbg_ii"] = Implementation( + name="GNBG-II", + description="Generalized Numerical Benchmark Generator version 2", + links=[Link(type="repository", url="https://github.com/rohitsalgotra/GNBG-II")], +) +things["impl_iohgnbg"] = Implementation( + name="IOHGNBG", + description="IOHprofiler version of GNBG", + links=[Link(type="repository", url="https://github.com/IOHprofiler/IOHGNBG")], +) +things["suite_gnbg_ii"] = Suite( + name="GNBG-II", + description="Generalized Numerical Benchmark Generator (version 2). Also available in IOH.", + objectives={1}, + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + source={"artificial"}, + references=[ + Reference( + title="GNBG-II", + authors=[], + link=Link(url="https://dl.acm.org/doi/pdf/10.1145/3712255.3734271"), + ) + ], + implementations={"impl_gnbg_ii", "impl_iohgnbg"}, +) +things["gen_gnbg_ii"] = Generator( + name="GNBG-II", + description="Generator counterpart of GNBG-II.", + objectives={1}, + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + source={"artificial"}, + references=[ + Reference( + title="GNBG-II", + authors=[], + link=Link(url="https://dl.acm.org/doi/pdf/10.1145/3712255.3734271"), + ) + ], + implementations={"impl_gnbg_ii", "impl_iohgnbg"}, +) + +#! - name: GNBG +#! suite/generator/single: suite; generator +#! objectives: '1' +#! dimensionality: scalable +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://arxiv.org/abs/2312.07083 +#! implementation: https://github.com/Danial-Yazdani/GNBG-Generator +#! source (real-world/artificial): artificial +#! textual description: Generalized Numerical Benchmark Generator +things["impl_gnbg"] = Implementation( + name="GNBG Generator", + description="Generalized Numerical Benchmark Generator", + links=[Link(type="repository", url="https://github.com/Danial-Yazdani/GNBG-Generator")], +) +things["suite_gnbg"] = Suite( + name="GNBG", + description="Generalized Numerical Benchmark Generator", + objectives={1}, + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + source={"artificial"}, + references=[ + Reference( + title="GNBG", + authors=[], + link=Link(url="https://arxiv.org/abs/2312.07083"), + ) + ], + implementations={"impl_gnbg"}, +) +things["gen_gnbg"] = Generator( + name="GNBG", + description="Generator counterpart of GNBG.", + objectives={1}, + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + source={"artificial"}, + references=[ + Reference( + title="GNBG", + authors=[], + link=Link(url="https://arxiv.org/abs/2312.07083"), + ) + ], + implementations={"impl_gnbg"}, +) + +#! - name: DynamicBinVal +#! suite/generator/single: suite +#! objectives: '1' +#! dimensionality: scalable +#! variable type: binary +#! constraints: 'no' +#! dynamic: 'yes' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://arxiv.org/pdf/2404.15837 +#! implementation: https://github.com/IOHprofiler/IOHexperimenter +#! source (real-world/artificial): artificial +#! textual description: Four versions of the dynamic binary value problem +things["suite_dynamicbinval"] = Suite( + name="DynamicBinVal", + description="Four versions of the dynamic binary value problem", + objectives={1}, + variables=[Variable(type="binary", dim=ValueRange(min=1))], + dynamic_type={"dynamic"}, + source={"artificial"}, + references=[ + Reference( + title="DynamicBinVal", + authors=[], + link=Link(url="https://arxiv.org/pdf/2404.15837"), + ) + ], + implementations={"impl_iohexperimenter"}, +) + +#! - name: PBO +#! suite/generator/single: suite +#! objectives: '1' +#! dimensionality: scalable +#! variable type: binary +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://dl.acm.org/doi/pdf/10.1145/3319619.3326810 +#! implementation: https://github.com/IOHprofiler/IOHexperimenter +#! source (real-world/artificial): artificial +#! textual description: Suite of 25 binary optimization problems +things["suite_pbo"] = Suite( + name="PBO", + description="Suite of 25 binary optimization problems", + objectives={1}, + variables=[Variable(type="binary", dim=ValueRange(min=1))], + source={"artificial"}, + references=[ + Reference( + title="PBO benchmarks", + authors=[], + link=Link(url="https://dl.acm.org/doi/pdf/10.1145/3319619.3326810"), + ) + ], + implementations={"impl_iohexperimenter"}, +) + +#! - name: W-model +#! suite/generator/single: generator +#! objectives: '1' +#! dimensionality: scalable +#! variable type: binary +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://dl.acm.org/doi/abs/10.1145/3205651.3208240?casa_token=S4U_Pi9f6MwAAAAA:U9ztNTPwmupT8K3GamWZfBL7-8fqjxPtr_kprv51vdwA-REsp0EyOFGa99BtbANb0XbqyrVg795hIw +#! implementation: https://github.com/thomasWeise/BBDOB_W_Model +#! source (real-world/artificial): artificial +#! textual description: Tunable generator for binary optimization based on several +#! difficulty features +things["impl_wmodel"] = Implementation( + name="BBDOB W-Model", + description="Tunable generator for binary optimization", + links=[Link(type="repository", url="https://github.com/thomasWeise/BBDOB_W_Model")], +) +things["gen_wmodel"] = Generator( + name="W-model", + description="Tunable generator for binary optimization based on several difficulty features", + objectives={1}, + variables=[Variable(type="binary", dim=ValueRange(min=1))], + source={"artificial"}, + references=[ + Reference( + title="W-model", + authors=[], + link=Link( + url="https://dl.acm.org/doi/abs/10.1145/3205651.3208240" + ), + ) + ], + implementations={"impl_wmodel"}, +) + +#! - name: Submodular Optimitzation +#! suite/generator/single: suite +#! objectives: '1' +#! dimensionality: scalable +#! variable type: binary +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=10254181 +#! implementation: https://github.com/IOHprofiler/IOHexperimenter +#! source (real-world/artificial): artificial +#! textual description: set of graph-based submodular optimization problems from 4 +#! problem types +things["suite_submodular"] = Suite( + name="Submodular Optimization", + description="set of graph-based submodular optimization problems from 4 problem types", + objectives={1}, + variables=[Variable(type="binary", dim=ValueRange(min=1))], + source={"artificial"}, + references=[ + Reference( + title="Submodular optimization benchmark", + authors=[], + link=Link(url="https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=10254181"), + ) + ], + implementations={"impl_iohexperimenter"}, +) + +#! - name: CEC2013 +#! suite/generator/single: suite +#! objectives: '1' +#! dimensionality: scalable +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://peerj.com/articles/cs-2671/CEC2013.pdf +#! implementation: https://github.com/P-N-Suganthan/CEC2013 +#! source (real-world/artificial): artificial +#! textual description: suite used for cec2013 competition. Also in IOH https://github.com/IOHprofiler/IOHexperimenter +things["impl_cec2013"] = Implementation( + name="CEC2013 reference code", + description="Suganthan's reference implementation", + links=[Link(type="repository", url="https://github.com/P-N-Suganthan/CEC2013")], +) +things["suite_cec2013"] = Suite( + name="CEC2013", + description="suite used for cec2013 competition. Also in IOH.", + objectives={1}, + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + source={"artificial"}, + references=[ + Reference( + title="CEC2013 definitions", + authors=[], + link=Link(url="https://peerj.com/articles/cs-2671/CEC2013.pdf"), + ) + ], + implementations={"impl_cec2013", "impl_iohexperimenter"}, +) + +#! - name: CEC2022 +#! suite/generator/single: suite +#! objectives: '1' +#! dimensionality: scalable +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '?' +#! multi-fidelity: 'no' +#! reference: https://github.com/P-N-Suganthan/2022-SO-BO/blob/main/CEC2022%20TR.pdf +#! implementation: https://github.com/P-N-Suganthan/2022-SO-BO +#! source (real-world/artificial): artificial +#! textual description: suite used for cec2022 competition. Also in IOH https://github.com/IOHprofiler/IOHexperimenter +things["impl_cec2022"] = Implementation( + name="CEC2022 reference code", + description="Suganthan's reference implementation", + links=[Link(type="repository", url="https://github.com/P-N-Suganthan/2022-SO-BO")], +) +things["suite_cec2022"] = Suite( + name="CEC2022", + description="suite used for cec2022 competition. Also in IOH.", + objectives={1}, + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + source={"artificial"}, + references=[ + Reference( + title="CEC2022 TR", + authors=[], + link=Link(url="https://github.com/P-N-Suganthan/2022-SO-BO/blob/main/CEC2022%20TR.pdf"), + ) + ], + implementations={"impl_cec2022", "impl_iohexperimenter"}, +) + +#! - name: Onemax+Sphere / Zeromax+Sphere +#! suite/generator/single: single +#! objectives: '2' +#! dimensionality: scalable +#! variable type: binary and continuous;mixed; +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1145/3449726.3459521 +#! implementation: +#! source (real-world/artificial): 'artificial' +#! textual description: '' +# FIXME: no implementation provided. +things["fn_onemax_sphere_zeromax_sphere"] = Problem( + name="Onemax+Sphere / Zeromax+Sphere", + objectives={2}, + variables=[ + Variable(type="binary", dim=ValueRange(min=1)), + Variable(type="continuous", dim=ValueRange(min=1)), + ], + source={"artificial"}, + references=[ + Reference( + title="Onemax+Sphere / Zeromax+Sphere", + authors=[], + link=Link(url="https://doi.org/10.1145/3449726.3459521"), + ) + ], +) + +#! - name: Onemax+Sphere / DeceptiveTrap+RotatedEllipsoid +#! suite/generator/single: single +#! objectives: '2' +#! dimensionality: scalable +#! variable type: binary and continuous;mixed; +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1145/3449726.3459521 +#! implementation: +#! source (real-world/artificial): 'artificial' +#! textual description: '' +# FIXME: no implementation provided. +things["fn_onemax_sphere_deceptive_rotell"] = Problem( + name="Onemax+Sphere / DeceptiveTrap+RotatedEllipsoid", + objectives={2}, + variables=[ + Variable(type="binary", dim=ValueRange(min=1)), + Variable(type="continuous", dim=ValueRange(min=1)), + ], + source={"artificial"}, + references=[ + Reference( + title="Mixed-variable multi-objective test problems", + authors=[], + link=Link(url="https://doi.org/10.1145/3449726.3459521"), + ) + ], +) + +#! - name: InverseDeceptiveTrap+RotatedEllipsoid / DeceptiveTrap+RotatedEllipsoid +#! suite/generator/single: single +#! objectives: '2' +#! dimensionality: scalable +#! variable type: binary and continuous;mixed; +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: '' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1145/3449726.3459521 +#! implementation: +#! source (real-world/artificial): 'artificial' +#! textual description: '' +# FIXME: no implementation provided. +things["fn_invdeceptive_deceptive_rotell"] = Problem( + name="InverseDeceptiveTrap+RotatedEllipsoid / DeceptiveTrap+RotatedEllipsoid", + objectives={2}, + variables=[ + Variable(type="binary", dim=ValueRange(min=1)), + Variable(type="continuous", dim=ValueRange(min=1)), + ], + source={"artificial"}, + references=[ + Reference( + title="Mixed-variable multi-objective test problems", + authors=[], + link=Link(url="https://doi.org/10.1145/3449726.3459521"), + ) + ], +) + +#! - name: PorkchopPlotInterplanetaryTrajectory +#! suite/generator/single: suite +#! objectives: '1' +#! dimensionality: 2 +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: 'yes' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1109/CEC65147.2025.11042973 +#! implementation: https://github.com/ShuaiqunPan/Transfer_Random_forests_BBOB_Real_world +#! source (real-world/artificial): 'real-world' +#! textual description: '' +things["impl_transfer_rf_bbob_rw"] = Implementation( + name="Transfer Random Forests BBOB Real-world", + description="Real-world BBOB-like problem implementations (Porkchop, KinematicsRobotArm)", + links=[ + Link( + type="repository", + url="https://github.com/ShuaiqunPan/Transfer_Random_forests_BBOB_Real_world", + ) + ], +) +things["suite_porkchop"] = Suite( + name="PorkchopPlotInterplanetaryTrajectory", + objectives={1}, + variables=[Variable(type="continuous", dim=2)], + modality={"multimodal"}, + source={"real-world"}, + references=[ + Reference( + title="Porkchop plot interplanetary trajectory benchmark", + authors=[], + link=Link(url="https://doi.org/10.1109/CEC65147.2025.11042973"), + ) + ], + implementations={"impl_transfer_rf_bbob_rw"}, +) + +#! - name: KinematicsRobotArm +#! suite/generator/single: suite +#! objectives: '1' +#! dimensionality: 21 +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: 'no' +#! multi-fidelity: 'no' +#! reference: https://doi.org/10.1023/A:1013258808932 +#! implementation: https://github.com/ShuaiqunPan/Transfer_Random_forests_BBOB_Real_world +#! source (real-world/artificial): 'real-world' +#! textual description: '' +things["suite_kinematics_robotarm"] = Suite( + name="KinematicsRobotArm", + objectives={1}, + variables=[Variable(type="continuous", dim=21)], + modality={"unimodal"}, + source={"real-world"}, + references=[ + Reference( + title="Kinematics of a robot arm", + authors=[], + link=Link(url="https://doi.org/10.1023/A:1013258808932"), + ) + ], + implementations={"impl_transfer_rf_bbob_rw"}, +) + +#! - name: VehicleDynamics +#! suite/generator/single: suite +#! objectives: '1' +#! dimensionality: 2 +#! variable type: continuous +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: 'yes' +#! multi-fidelity: 'no' +#! reference: https://www.scitepress.org/Papers/2023/121580/121580.pdf +#! implementation: https://zenodo.org/records/8307853 +#! source (real-world/artificial): 'real-world' +#! textual description: '' +things["impl_vehicle_dynamics"] = Implementation( + name="VehicleDynamics (Zenodo)", + description="Zenodo archive for the vehicle dynamics benchmark", + links=[Link(type="archive", url="https://zenodo.org/records/8307853")], +) +things["suite_vehicle_dynamics"] = Suite( + name="VehicleDynamics", + objectives={1}, + variables=[Variable(type="continuous", dim=2)], + modality={"multimodal"}, + source={"real-world"}, + references=[ + Reference( + title="VehicleDynamics benchmark", + authors=[], + link=Link(url="https://www.scitepress.org/Papers/2023/121580/121580.pdf"), + ) + ], + implementations={"impl_vehicle_dynamics"}, +) + +#! - name: MECHBench +#! suite/generator/single: Problem Suite +#! variable type: Continuous +#! dimensionality: scalable' +#! objectives: '1' +#! constraints: 'yes' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: 'yes' +#! multi-fidelity: 'no' +#! source (real-world/artificial): Real-World Application +#! implementation: https://github.com/BayesOptApp/MECHBench +#! textual description: This is a set of problems with inspiration from Structural +#! Mechanics Design Optimization. The suite comprises three physical models, from +#! which the user may define different kind of problems which impact the final design +#! output. +#! reference: https://arxiv.org/abs/2511.10821 +#! other info: +#! partial evaluations: 'no' +#! full name: MECHBench +#! constraint properties: Hard Constraints +#! number of constraints: 1 or 2 +#! description of multimodality: Unstructured or non isotropic multimodality +#! key challenges / characteristics: Embeds physical simulations and is flexible +#! and modular +#! scientific motivation: Bridge the black-box optimization techniques to a Mechanical +#! Design Problem which require these kinds of algorithms +#! limitations: The models do not include fracture or damage mechanics, just plasticity. +#! implementation languages: Python +#! approximate evaluation time: Times -> from 1 minute to 7 minutes +things["impl_mechbench"] = Implementation( + name="MECHBench", + description="Structural mechanics design optimization benchmark", + language="Python", + evaluation_time="1-7 minutes", + links=[Link(type="repository", url="https://github.com/BayesOptApp/MECHBench")], +) +things["suite_mechbench"] = Suite( + name="MECHBench", + long_name="MECHBench", + description="Set of problems inspired by Structural Mechanics Design Optimization. Embeds physical simulations (plasticity only, no fracture/damage). Unstructured/non-isotropic multimodality.", + objectives={1}, + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + constraints=[Constraint(hard="yes", number={1, 2})], + modality={"multimodal"}, + allows_partial_evaluation="no", + source={"real-world"}, + references=[ + Reference( + title="MECHBench", + authors=[], + link=Link(url="https://arxiv.org/abs/2511.10821"), + ) + ], + implementations={"impl_mechbench"}, +) + +#! - name: EXPObench +#! suite/generator/single: Problem Suite +#! variable type: Continuous, Integer, Categorical, Conditional +#! dimensionality: 10 to 135 +#! objectives: '1' +#! constraints: 'yes' +#! dynamic: 'no' +#! noise: 'yes' +#! multimodal: Unknown +#! multi-fidelity: 'no' +#! source (real-world/artificial): Real-World Application +#! implementation: https://github.com/AlgTUDelft/ExpensiveOptimBenchmark +#! textual description: Wind farm layout optimization, gas filter design, pipe shape +#! optimization, hyperparameter tuning, and hospital simulation +#! reference: https://doi.org/10.1016/j.asoc.2023.110744 +#! other info: +#! partial evaluations: 'no' +#! full name: EXPensive Optimization benchmark library +#! constraint properties: Hard Constraints, Soft Constraints, Box Constraints, only +#! box constraints implemented, others appear as penalty in objective +#! number of constraints: 2 per variable (box), other constraints unknown (simulator +#! fails) +#! form of noise model: real-life (unknown) +#! type of noise space: Observational +#! key challenges / characteristics: Expensive objectives +#! scientific motivation: Address the lack of real-life expensive benchmarks +#! limitations: single-objective only, constraints are handled naively (penalty in +#! objective), no parallelization +#! implementation languages: Python +#! approximate evaluation time: 2 to 80 seconds +# FIXME: "Conditional" variable type has no schema representation; box number expressed as 2 per variable cannot be encoded. +things["impl_expobench"] = Implementation( + name="EXPObench", + description="EXPensive Optimization benchmark library (wind farm layout, gas filter design, pipe shape, hyperparameter tuning, hospital simulation)", + language="Python", + evaluation_time="2 to 80 seconds", + links=[Link(type="repository", url="https://github.com/AlgTUDelft/ExpensiveOptimBenchmark")], +) +things["suite_expobench"] = Suite( + name="EXPObench", + long_name="EXPensive Optimization benchmark library", + description="Wind farm layout optimization, gas filter design, pipe shape optimization, hyperparameter tuning, and hospital simulation", + objectives={1}, + variables=[ + Variable(type="continuous", dim=ValueRange(min=10, max=135)), + Variable(type="integer", dim=ValueRange(min=10, max=135)), + Variable(type="categorical", dim=ValueRange(min=10, max=135)), + ], + constraints=[ + Constraint(type="box", hard="yes"), + Constraint(hard="some"), + ], + noise_type={"observational", "real-life"}, + allows_partial_evaluation="no", + source={"real-world"}, + references=[ + Reference( + title="EXPObench", + authors=[], + link=Link(url="https://doi.org/10.1016/j.asoc.2023.110744"), + ) + ], + implementations={"impl_expobench"}, +) + +#! - name: Gasoline direct injection engine design +#! suite/generator/single: Single Problem +#! variable type: Continuous, Ordinal +#! dimensionality: '7' +#! objectives: '2' +#! constraints: 'yes' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: Unknown +#! multi-fidelity: 'yes' +#! source (real-world/artificial): Real-World Application +#! implementation: https://doi.org/10.1016/j.ejor.2022.08.032 +#! textual description: ... +#! other info: +#! partial evaluations: Unknown +#! constraint properties: Hard Constraints, Soft Constraints +#! number of constraints: '5' +#! key challenges / characteristics: Expensive +#! limitations: Proprietary +#! implementation languages: Matlab Simulink and Wave RT co-simulation +# FIXME: "Ordinal" variable type not in schema; falling back to integer. +things["impl_gasoline"] = Implementation( + name="Gasoline direct injection engine design", + description="Proprietary Matlab Simulink + Wave RT co-simulation", + language="Matlab Simulink / Wave RT", + links=[Link(type="paper", url="https://doi.org/10.1016/j.ejor.2022.08.032")], +) +things["fn_gasoline"] = Problem( + name="Gasoline direct injection engine design", + description="Multi-objective optimization to minimize fuel consumption and NOx emissions over a two-minute dynamic duty cycle, subject to five constraints (turbine inlet temperature, knock occurrences, peak cylinder pressure, peak cylinder pressure rise, total work). Seven decision variables cover hardware choices and engine control parameters.", + objectives={2}, + variables=[ + Variable(type="continuous", dim=7), + Variable(type="integer", dim=7), + ], + constraints=[Constraint(hard="yes", number=5)], + fidelity_levels={1, 2}, + source={"real-world"}, + references=[ + Reference( + title="Gasoline direct injection engine design", + authors=[], + link=Link(url="https://doi.org/10.1016/j.ejor.2022.08.032"), + ) + ], + implementations={"impl_gasoline"}, +) + +#! - name: BEACON +#! suite/generator/single: Generator +#! variable type: Continuous +#! dimensionality: scalable +#! objectives: '2' +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: 'yes' +#! multi-fidelity: 'no' +#! source (real-world/artificial): Artificially Generated +#! implementation: https://github.com/Stebbet/BEACON/ +#! textual description: Generator for bi-objective benchmark problems with explicitly +#! controlled correlations in continuous spaces. +#! reference: https://dl.acm.org/doi/10.1145/3712255.3734303 +#! other info: +#! partial evaluations: 'no' +#! full name: Continuous Bi-objective Benchmark problems with Explicit Adjustable +#! COrrelatioN control +#! constraint properties: Box Constraints +#! number of constraints: '0' +#! description of multimodality: Random +#! key challenges / characteristics: Multimodal, different correlations among objectives +#! scientific motivation: Controlled correlation among objectives +#! limitations: No analytical Pareto front, only bi-objective +#! implementation languages: Python +#! approximate evaluation time: Negligible +things["impl_beacon"] = Implementation( + name="BEACON", + description="Continuous Bi-objective Benchmark with Explicit Adjustable COrrelatioN control", + language="Python", + evaluation_time="negligible", + links=[Link(type="repository", url="https://github.com/Stebbet/BEACON/")], +) +things["gen_beacon"] = Generator( + name="BEACON", + long_name="Continuous Bi-objective Benchmark problems with Explicit Adjustable COrrelatioN control", + description="Generator for bi-objective benchmark problems with explicitly controlled correlations in continuous spaces. Multimodal with random structure.", + objectives={2}, + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + constraints=[Constraint(type="box", hard="yes", number=0)], + modality={"multimodal"}, + allows_partial_evaluation="no", + source={"artificial"}, + references=[ + Reference( + title="BEACON", + authors=[], + link=Link(url="https://dl.acm.org/doi/10.1145/3712255.3734303"), + ) + ], + implementations={"impl_beacon"}, +) + +#! - name: TulipaEnergy +#! suite/generator/single: Problem Suite +#! variable type: Continuous +#! dimensionality: scalable +#! objectives: '1' +#! constraints: 'yes' +#! dynamic: 'no' +#! noise: 'yes' +#! multimodal: 'no' +#! multi-fidelity: 'yes' +#! source (real-world/artificial): Real-World Application +#! implementation: https://tulipaenergy.github.io/TulipaEnergyModel.jl/stable/ +#! textual description: Determine the optimal investment and operation decisions for +#! different types of assets in the energy system ... minimizing loss of load. +#! reference: See https://tulipaenergy.github.io/TulipaEnergyModel.jl/stable/40-scientific-foundation/45-scientific-references +#! other info: +#! partial evaluations: Unknown +#! full name: TulipaEnergyModel.jl +#! constraint properties: Hard Constraints, Soft Constraints +#! number of constraints: millions +#! type of dynamicism: none +#! form of noise model: depends on input — still working on stochastic inputs +#! type of noise space: Parameter +#! key challenges / characteristics: modeled as a potentially very large linear program +#! scientific motivation: new techniques for solving large whitebox linear optimization problems +#! limitations: not yet stochastic +#! implementation languages: Julia / JMP +#! approximate evaluation time: from minutes to hours +#! links to usage examples: https://github.com/TulipaEnergy/Tulipa-OBZ-CaseStudy +# FIXME: "number of constraints: millions" cannot be expressed precisely. +things["impl_tulipa"] = Implementation( + name="TulipaEnergyModel.jl", + description="Large linear program for optimal investment and operation of energy systems", + language="Julia / JuMP", + evaluation_time="minutes to hours", + links=[ + Link(type="website", url="https://tulipaenergy.github.io/TulipaEnergyModel.jl/stable/"), + Link(type="example", url="https://github.com/TulipaEnergy/Tulipa-OBZ-CaseStudy"), + ], +) +things["suite_tulipa_energy"] = Suite( + name="TulipaEnergy", + long_name="TulipaEnergyModel.jl", + description="Determine the optimal investment and operation decisions for different assets in the energy system (production, consumption, conversion, storage, transport) while minimizing loss of load. Modelled as a potentially very large linear program with multiple fidelity levels.", + objectives={1}, + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + constraints=[Constraint(hard="yes"), Constraint(hard="some")], + noise_type={"parameter"}, + modality={"unimodal"}, + fidelity_levels={1, 2}, + source={"real-world"}, + references=[ + Reference( + title="TulipaEnergyModel.jl scientific references", + authors=[], + link=Link( + url="https://tulipaenergy.github.io/TulipaEnergyModel.jl/stable/40-scientific-foundation/45-scientific-references" + ), + ) + ], + implementations={"impl_tulipa"}, +) + +#! - name: ATO +#! suite/generator/single: Single Problem +#! variable type: Continuous +#! dimensionality: '10' +#! objectives: '2' +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: 'no' +#! multi-fidelity: 'no' +#! source (real-world/artificial): Real-World Application +#! implementation: '-' +#! textual description: Parameters of the Modules of the Automatic Train Operation +#! should be optimized. The parameters are continuous with different ranges. There +#! are two objectives (minimizing energy consumption, minimizing driving duration. +#! other info: +#! partial evaluations: 'no' +# FIXME: no implementation available. +things["fn_ato"] = Problem( + name="ATO", + description="Parameters of the Modules of the Automatic Train Operation are optimized; two objectives: minimizing energy consumption and minimizing driving duration.", + objectives={2}, + variables=[Variable(type="continuous", dim=10)], + modality={"unimodal"}, + allows_partial_evaluation="no", + source={"real-world"}, +) + +#! - name: Brachytherapy treatment planning +#! suite/generator/single: Problem Suite +#! variable type: Continuous +#! dimensionality: 100-500 +#! objectives: 2-3 +#! constraints: 'yes' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: 'yes' +#! multi-fidelity: 'yes' +#! source (real-world/artificial): Real-World Application +#! textual description: Treatment planning for internal radiation therapy +#! reference: https://www.sciencedirect.com/science/article/pii/S1538472123016781 +#! other info: +#! partial evaluations: 'yes' +#! full name: Brachytherapy treatment planning +#! constraint properties: Hard Constraints +#! number of constraints: scalable +#! key challenges / characteristics: Multi-objective; aggregated objectives +#! limitations: No public source code +# FIXME: no public source code; no implementation URL. +things["suite_brachytherapy"] = Suite( + name="Brachytherapy treatment planning", + long_name="Brachytherapy treatment planning", + description="Treatment planning for internal radiation therapy. Multi-objective with aggregated objectives; no public source code.", + objectives={2, 3}, + variables=[Variable(type="continuous", dim=ValueRange(min=100, max=500))], + constraints=[Constraint(hard="yes", number=ValueRange(min=1))], + modality={"multimodal"}, + fidelity_levels={1, 2}, + allows_partial_evaluation="yes", + source={"real-world"}, + references=[ + Reference( + title="Brachytherapy treatment planning", + authors=[], + link=Link(url="https://www.sciencedirect.com/science/article/pii/S1538472123016781"), + ) + ], +) + +#! - name: FleetOpt +#! suite/generator/single: Single Problem +#! variable type: Integer +#! dimensionality: 'Upper level: 54; lower level: 13208' +#! objectives: '1' +#! constraints: 'yes' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: Unknown +#! multi-fidelity: 'no' +#! source (real-world/artificial): Real-World Application +#! implementation: 'Not public: was done for real client with their private data' +#! textual description: Healthcare organisation in the UK ... +#! reference: https://dl.acm.org/doi/abs/10.1145/3638530.3664137 +#! other info: +#! partial evaluations: 'yes' +# FIXME: bilevel dimensionality (upper 54 / lower 13208) expressed as {54, 13208}; impl not public. +things["fn_fleetopt"] = Problem( + name="FleetOpt", + description="UK healthcare organisation fleet optimisation: reduce the fleet of non-emergency healthcare trip vehicles while still ensuring all trips can be covered. Bilevel: upper level 54 vars, lower level 13208 vars.", + objectives={1}, + variables=[Variable(type="integer", dim={54, 13208})], + constraints=[Constraint(hard="yes")], + allows_partial_evaluation="yes", + source={"real-world"}, + references=[ + Reference( + title="FleetOpt", + authors=[], + link=Link(url="https://dl.acm.org/doi/abs/10.1145/3638530.3664137"), + ) + ], +) + +#! - name: Building spatial design +#! suite/generator/single: Single Problem +#! variable type: Continuous, Boolean +#! dimensionality: scalable depending on problem size (e.g. 90 for) +#! objectives: '2' +#! constraints: 'yes' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: Unknown +#! multi-fidelity: 'no' +#! source (real-world/artificial): Real-World Application +#! implementation: https://github.com/TUe-excellent-buildings/BSO-toolbox +#! textual description: 'Optimise the spatial layout of a building to: minimise energy +#! consumption for climate control, and minimise the strain on the structure' +#! reference: https://hdl.handle.net/1887/81789 +#! other info: +#! partial evaluations: 'no' +#! constraint properties: Hard Constraints, Box Constraints, Permutation Constraints +#! number of constraints: 2065 (as example, depends on problem size) +#! implementation languages: C++ +#! approximate evaluation time: Roughly 1 second per evaluation for the smallest +#! considered design, and roughly 40 seconds for the larger designs we considered. +# FIXME: Permutation Constraints not representable in ConstraintType; using multiple Constraint objects. +things["impl_bso_toolbox"] = Implementation( + name="BSO-toolbox", + description="Building Spatial Design toolbox (TU/e)", + language="C++", + evaluation_time="~1s (smallest) to ~40s (larger)", + links=[Link(type="repository", url="https://github.com/TUe-excellent-buildings/BSO-toolbox")], +) +things["fn_building_spatial"] = Problem( + name="Building spatial design", + description="Optimise the spatial layout of a building to minimise energy consumption for climate control and minimise the strain on the structure. Many hard constraints; mixed-variable (continuous+binary); expensive evaluations.", + objectives={2}, + variables=[ + Variable(type="continuous", dim=ValueRange(min=1)), + Variable(type="binary", dim=ValueRange(min=1)), + ], + constraints=[ + Constraint(hard="yes"), + Constraint(type="box", hard="yes"), + ], + allows_partial_evaluation="no", + source={"real-world"}, + references=[ + Reference( + title="Building spatial design", + authors=[], + link=Link(url="https://hdl.handle.net/1887/81789"), + ) + ], + implementations={"impl_bso_toolbox"}, +) + +#! - name: Electric Motor Design Optimization +#! suite/generator/single: Single Problem +#! variable type: Continuous, Integer +#! dimensionality: '13' +#! objectives: '1' +#! constraints: 'yes' +#! dynamic: 'no' +#! noise: 'yes' +#! multimodal: 'yes' +#! multi-fidelity: 'no' +#! source (real-world/artificial): Real-World Application +#! implementation: Implementation not freely available +#! textual description: The goal is to find a design of a synchronous electric motor +#! for power steering systems that minimizes costs and satisfies all constraints. +#! reference: https://dis.ijs.si/tea/Publications/Tusar23Multistep.pdf (paper in Slovene) +#! other info: +#! partial evaluations: 'no' +#! full name: Electric Motor Design Optimization +#! constraint properties: Hard Constraints, Soft Constraints, Box Constraints +#! number of constraints: '12' +#! description of multimodality: Constraints are multimodal +#! key challenges / characteristics: Time-consuming solution evaluation, highly-constrained +#! scientific motivation: Challenging to find good solutions in a limited time +#! limitations: 'Unavailability ...' +#! implementation languages: Python +#! approximate evaluation time: 8 minutes +#! general: This is not an available problem, but could be interesting to show to +#! researchers which difficulties appear in real-world problems +things["impl_emdo"] = Implementation( + name="Electric Motor Design Optimization", + description="Not publicly available", + language="Python", + evaluation_time="8 minutes", +) +things["fn_emdo"] = Problem( + name="Electric Motor Design Optimization", + long_name="Electric Motor Design Optimization", + description="""# Goal +Find a design of a synchronous electric motor for power steering systems that minimizes costs and satisfies all constraints. + +# Motivation +Challenging to find good solutions in a limited time. + +# Key Challenges +* Time-consuming solution evaluation +* Highly-constrained problem +* Constraints are multimodal + +This is not an available problem, but could be interesting to show to researchers which difficulties appear in real-world problems.""", + objectives={1}, + variables=[ + Variable(type="continuous", dim=13), + Variable(type="integer", dim=13), + ], + constraints=[ + Constraint(hard="yes", number=12), + Constraint(hard="some"), + Constraint(type="box", hard="yes"), + ], + noise_type={"noisy"}, + modality={"multimodal"}, + allows_partial_evaluation="no", + source={"real-world"}, + references=[ + Reference( + title="A Multi-Step Evaluation Process in Electric Motor Design", + authors=["Tea Tušar", "Peter Korošec", "Bogdan Filipič"], + link=Link(url="https://dis.ijs.si/tea/Publications/Tusar23Multistep.pdf"), + ) + ], + implementations={"impl_emdo"}, +) + +#! - name: BONO-Bench +#! suite/generator/single: Generator +#! variable type: Continuous +#! dimensionality: scalable +#! objectives: '2' +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: 'yes' +#! multi-fidelity: 'no' +#! source (real-world/artificial): Artificially Generated +#! implementation: https://github.com/schaepermeier/bonobench +#! textual description: Bi-objective problem generator and suite with scalable continuous +#! decision space. Features complex problem properties (different types of multimodality +#! and challenges in decision and objective space) as well as Pareto front approximations +#! with error guarantees for the hypervolume and exact R2 indicators. +#! other info: +#! partial evaluations: 'no' +#! full name: Bi-objective Numerical Optimization Benchmark (BONO-Bench) +#! constraint properties: Box Constraints +#! implementation languages: Python +things["impl_bonobench"] = Implementation( + name="BONO-Bench", + description="Bi-objective Numerical Optimization Benchmark (BONO-Bench)", + language="Python", + links=[Link(type="repository", url="https://github.com/schaepermeier/bonobench")], +) +things["gen_bono_bench"] = Generator( + name="BONO-Bench", + long_name="Bi-objective Numerical Optimization Benchmark", + description="Bi-objective problem generator and suite with scalable continuous decision space. Features complex problem properties and Pareto front approximations with error guarantees for the hypervolume and exact R2 indicators.", + objectives={2}, + variables=[Variable(type="continuous", dim=ValueRange(min=1))], + constraints=[Constraint(type="box", hard="yes")], + modality={"multimodal"}, + allows_partial_evaluation="no", + source={"artificial"}, + implementations={"impl_bonobench"}, +) + +#! - name: RandOptGen +#! suite/generator/single: Generator +#! variable type: Continuous, Integer, Boolean +#! dimensionality: scalable +#! objectives: scalable +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: 'yes' +#! multi-fidelity: 'no' +#! source (real-world/artificial): Artificially Generated +#! implementation: https://github.com/MALEO-research-group/RandOptGen +#! textual description: 'RandOptGen: A Unified Random Problem Generator for Single-and +#! Multi-Objective Optimization Problems with Mixed-Variable Input Spaces' +#! other info: +#! partial evaluations: 'no' +#! full name: RandOptGen +#! implementation languages: Python +#! approximate evaluation time: milliseconds +#! links to usage examples: https://doi.org/10.1145/3712256.3726478 +things["impl_randoptgen"] = Implementation( + name="RandOptGen", + description="Unified Random Problem Generator for Single- and Multi-Objective Optimization with Mixed-Variable Input Spaces", + language="Python", + evaluation_time="milliseconds", + links=[ + Link(type="repository", url="https://github.com/MALEO-research-group/RandOptGen"), + Link(type="example", url="https://doi.org/10.1145/3712256.3726478"), + ], +) +things["gen_randoptgen"] = Generator( + name="RandOptGen", + long_name="RandOptGen", + description="A Unified Random Problem Generator for Single- and Multi-Objective Optimization Problems with Mixed-Variable Input Spaces.", + # FIXME: original "scalable" - truncated to 1..10. + objectives=set(range(1, 11)), + variables=[ + Variable(type="continuous", dim=ValueRange(min=1)), + Variable(type="integer", dim=ValueRange(min=1)), + Variable(type="binary", dim=ValueRange(min=1)), + ], + modality={"multimodal"}, + allows_partial_evaluation="no", + source={"artificial"}, + implementations={"impl_randoptgen"}, +) + +#! - name: CUTEr +#! suite/generator/single: Problem Suite +#! variable type: Continuous, Integer, Boolean +#! dimensionality: scalable +#! objectives: '1' +#! constraints: 'yes' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: Unknown +#! multi-fidelity: 'no' +#! source (real-world/artificial): Artificially Generated +#! implementation: Not Found +#! textual description: A constrained and unconstrained testing environment +#! reference: https://dl.acm.org/doi/10.1145/962437.962439 +#! other info: +#! partial evaluations: 'no' +# FIXME: implementation not found. +things["suite_cuter"] = Suite( + name="CUTEr", + description="A constrained and unconstrained testing environment.", + objectives={1}, + variables=[ + Variable(type="continuous", dim=ValueRange(min=1)), + Variable(type="integer", dim=ValueRange(min=1)), + Variable(type="binary", dim=ValueRange(min=1)), + ], + constraints=[Constraint(hard="yes")], + allows_partial_evaluation="no", + source={"artificial"}, + references=[ + Reference( + title="CUTEr", + authors=[], + link=Link(url="https://dl.acm.org/doi/10.1145/962437.962439"), + ) + ], +) + +#! - name: CUTEst +#! suite/generator/single: Problem Suite +#! variable type: Continuous, Integer, Boolean +#! dimensionality: scalable +#! objectives: '1' +#! constraints: 'yes' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: 'yes' +#! multi-fidelity: 'no' +#! source (real-world/artificial): Artificially Generated +#! implementation: https://github.com/jfowkes/pycutest +#! textual description: The Constrained and Unconstrained Testing Environment with +#! safe threads (CUTEst) for optimization software +#! reference: https://link.springer.com/article/10.1007/s10589-014-9687-3 +#! other info: +#! partial evaluations: 'no' +#! full name: 'Constrained and Unconstrained Testing Environment with safe threads ' +#! constraint properties: Soft Constraints, Box Constraints +#! number of constraints: scalable +#! implementation languages: Python, C++, Fortran +#! general: 'Python implementation: https://github.com/jfowkes/pycutest' +things["impl_pycutest"] = Implementation( + name="pycutest", + description="Python interface to CUTEst", + language="Python / C++ / Fortran", + links=[Link(type="repository", url="https://github.com/jfowkes/pycutest")], +) +things["suite_cutest"] = Suite( + name="CUTEst", + long_name="Constrained and Unconstrained Testing Environment with safe threads", + description="CUTEst for optimization software", + objectives={1}, + variables=[ + Variable(type="continuous", dim=ValueRange(min=1)), + Variable(type="integer", dim=ValueRange(min=1)), + Variable(type="binary", dim=ValueRange(min=1)), + ], + constraints=[ + Constraint(hard="some", number=ValueRange(min=1)), + Constraint(type="box", hard="yes"), + ], + modality={"multimodal"}, + allows_partial_evaluation="no", + source={"artificial"}, + references=[ + Reference( + title="CUTEst", + authors=[], + link=Link(url="https://link.springer.com/article/10.1007/s10589-014-9687-3"), + ) + ], + implementations={"impl_pycutest"}, +) + +#! - name: PUBOi +#! suite/generator/single: Generator +#! variable type: Boolean +#! dimensionality: scalable +#! objectives: '1' +#! constraints: 'no' +#! dynamic: 'no' +#! noise: 'no' +#! multimodal: Unknown +#! multi-fidelity: 'no' +#! source (real-world/artificial): Artificially Generated +#! implementation: https://gitlab.com/verel/pubo-importance-benchmark +#! textual description: A benchmark in which variable importance is tunable, based +#! on the Walsh function +#! reference: https://link.springer.com/chapter/10.1007/978-3-031-04148-8_12 +#! other info: +#! partial evaluations: 'no' +#! full name: Polynomial Unconstrained Binary Optimization +#! key challenges / characteristics: Tunable variable importance +#! implementation languages: Python, C++ +things["impl_puboi"] = Implementation( + name="PUBO Importance Benchmark", + description="A benchmark in which variable importance is tunable, based on the Walsh function", + language="Python / C++", + links=[Link(type="repository", url="https://gitlab.com/verel/pubo-importance-benchmark")], +) + +things["gen_puboi"] = Generator( + name="PUBOi", + long_name="Polynomial Unconstrained Binary Optimization with tunable importance", + description="A benchmark in which variable importance is tunable, based on the Walsh function.", + objectives={1}, + variables=[Variable(type="binary", dim=ValueRange(min=1))], + allows_partial_evaluation="no", + source={"artificial"}, + references=[ + Reference( + title="PUBOi", + authors=[], + link=Link(url="https://link.springer.com/chapter/10.1007/978-3-031-04148-8_12"), + ) + ], + implementations={"impl_puboi"}, +) + + +library = Library(things) + +# Make sure model is really valid +Library.model_validate(library) + +if __name__ == "__main__": + print(to_yaml_str(library)) diff --git a/problems.yaml b/problems.yaml index 4328038..24caa52 100644 --- a/problems.yaml +++ b/problems.yaml @@ -1,1264 +1,3496 @@ -- name: BBOB - suite/generator/single: suite - objectives: '1' - dimensionality: scalable - variable type: continuous - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: 'yes' - multi-fidelity: 'no' - reference: https://doi.org/10.1080/10556788.2020.1808977 - implementation: https://github.com/numbbo/coco - source (real-world/artificial): '' - textual description: '' -- name: BBOB-biobj - suite/generator/single: suite - objectives: '2' - dimensionality: 2-40 - variable type: continuous - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: 'yes' - multi-fidelity: 'no' - reference: https://doi.org/10.48550/arXiv.1604.00359 - implementation: https://github.com/numbbo/coco - source (real-world/artificial): '' - textual description: '' -- name: BBOB-noisy - suite/generator/single: suite - objectives: '1' - dimensionality: scalable - variable type: continuous - constraints: 'no' - dynamic: 'no' - noise: 'yes' - multimodal: 'yes' - multi-fidelity: 'no' - reference: https://hal.inria.fr/inria-00369466 - implementation: https://web.archive.org/web/20210416065610/https://coco.gforge.inria.fr/doku.php?id=downloads - source (real-world/artificial): '' - textual description: '' -- name: BBOB-largescale - suite/generator/single: suite - objectives: '1' - dimensionality: 20-640 - variable type: continuous - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: 'yes' - multi-fidelity: 'no' - reference: https://doi.org/10.48550/arXiv.1903.06396 - implementation: https://github.com/numbbo/coco - source (real-world/artificial): '' - textual description: '' -- name: BBOB-mixint - suite/generator/single: suite - objectives: '1' - dimensionality: 5-160 - variable type: integer;continuous;mixed - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: 'yes' - multi-fidelity: 'no' - reference: https://doi.org/10.1145/3321707.3321868 - implementation: https://github.com/numbbo/coco - source (real-world/artificial): '' - textual description: '' -- name: BBOB-biobj-mixint - suite/generator/single: suite - objectives: '2' - dimensionality: 5-160 - variable type: integer;continuous;mixed - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: 'yes' - multi-fidelity: 'no' - reference: https://doi.org/10.1145/3321707.3321868 - implementation: https://github.com/numbbo/coco - source (real-world/artificial): '' - textual description: '' -- name: BBOB-constrained - suite/generator/single: suite - objectives: '1' - dimensionality: 2-40 - variable type: continuous - constraints: 'yes' - dynamic: 'no' - noise: 'no' - multimodal: 'yes' - multi-fidelity: 'no' - reference: http://numbbo.github.io/coco-doc/bbob-constrained/ - implementation: https://github.com/numbbo/coco - source (real-world/artificial): '' - textual description: '' -- name: MOrepo - suite/generator/single: suite - objectives: '2' - dimensionality: '?' - variable type: combinatorial - constraints: '?' - dynamic: '?' - noise: '?' - multimodal: '?' - multi-fidelity: 'no' - reference: '' - implementation: https://github.com/MCDMSociety/MOrepo - source (real-world/artificial): '' - textual description: '' -- name: ZDT - suite/generator/single: suite - objectives: '2' - dimensionality: scalable - variable type: continuous;binary - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://doi.org/10.1162/106365600568202 - implementation: https://github.com/anyoptimization/pymoo - source (real-world/artificial): '' - textual description: '' -- name: DTLZ - suite/generator/single: suite - objectives: 2+ - dimensionality: scalable - variable type: continuous - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://doi.org/10.1109/CEC.2002.1007032 - implementation: https://pymoo.org/problems/many/dtlz.html - source (real-world/artificial): '' - textual description: '' -- name: WFG - suite/generator/single: suite - objectives: 2+ - dimensionality: scalable - variable type: continuous - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://doi.org/10.1109/TEVC.2005.861417 - implementation: https://pymoo.org/problems/many/wfg.html - source (real-world/artificial): '' - textual description: '' -- name: CDMP - suite/generator/single: suite - objectives: 2+ - dimensionality: scalable - variable type: continuous - constraints: 'yes' - dynamic: '?' - noise: '?' - multimodal: '?' - multi-fidelity: 'no' - reference: https://doi.org/10.1145/3321707.3321878 - implementation: '?' - source (real-world/artificial): '' - textual description: '' -- name: SDP - suite/generator/single: suite - objectives: 2+ - dimensionality: scalable - variable type: continuous - constraints: 'no' - dynamic: 'yes' - noise: '?' - multimodal: '?' - multi-fidelity: 'no' - reference: https://doi.org/10.1109/TCYB.2019.2896021 - implementation: '?' - source (real-world/artificial): '' - textual description: '' -- name: MaOP - suite/generator/single: suite - objectives: 2+ - dimensionality: scalable - variable type: continuous - constraints: 'no' - dynamic: 'no' - noise: '?' - multimodal: '?' - multi-fidelity: 'no' - reference: https://doi.org/10.1016/j.swevo.2019.02.003 - implementation: '?' - source (real-world/artificial): '' - textual description: '' -- name: BP - suite/generator/single: suite - objectives: 2+ - dimensionality: scalable - variable type: continuous - constraints: 'no' - dynamic: 'no' - noise: '?' - multimodal: '?' - multi-fidelity: 'no' - reference: https://doi.org/10.1109/CEC.2019.8790277 - implementation: '?' - source (real-world/artificial): '' - textual description: '' -- name: GPD - suite/generator/single: generator - objectives: 2+ - dimensionality: scalable - variable type: continuous - constraints: optional - dynamic: 'no' - noise: optional - multimodal: '?' - multi-fidelity: 'no' - reference: https://doi.org/10.1016/j.asoc.2020.106139 - implementation: '?' - source (real-world/artificial): '' - textual description: '' -- name: ETMOF - suite/generator/single: suite - objectives: 2-50 - dimensionality: 25-10000 - variable type: continuous - constraints: 'no' - dynamic: 'yes' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://doi.org/10.48550/arXiv.2110.08033 - implementation: https://github.com/songbai-liu/etmo - source (real-world/artificial): '' - textual description: '' -- name: MMOPP - suite/generator/single: suite - objectives: 2-7 - dimensionality: '?' - variable type: '?' - constraints: 'yes' - dynamic: 'no' - noise: 'no' - multimodal: 'yes' - multi-fidelity: 'no' - reference: http://www5.zzu.edu.cn/system/_content/download.jsp?urltype=news.DownloadAttachUrl&owner=1327567121&wbfileid=4764412 - implementation: http://www5.zzu.edu.cn/ecilab/info/1036/1251.htm - source (real-world/artificial): '' - textual description: '' -- name: CFD - suite/generator/single: suite - objectives: 1-2 - dimensionality: scalable - variable type: '?' - constraints: 'yes' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://doi.org/10.1007/978-3-319-99259-4_24 - implementation: https://bitbucket.org/arahat/cfd-test-problem-suite - source (real-world/artificial): real world - textual description: expensive evaluations 30s-15m -- name: GBEA - suite/generator/single: suite - objectives: 1-2 - dimensionality: scalable - variable type: continuous - constraints: 'no' - dynamic: 'no' - noise: 'yes' - multimodal: 'yes' - multi-fidelity: 'no' - reference: https://doi.org/10.1145/3321707.3321805 - implementation: 'https://github.com/ttusar/coco-gbea' - source (real-world/artificial): real world - textual description: 'expensive evaluations 5s-35s, RW-GAN-Mario and TopTrumps are part of GBEA' -- name: Car structure - suite/generator/single: suite - objectives: '2' - dimensionality: 144-222 - variable type: discrete - constraints: 'yes' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://doi.org/10.1145/3205651.3205702 - implementation: http://ladse.eng.isas.jaxa.jp/benchmark/ - source (real-world/artificial): real world - textual description: 54 constraints -- name: EMO2017 - suite/generator/single: suite - objectives: '2' - dimensionality: 4-24 - variable type: continuous - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://www.ini.rub.de/PEOPLE/glasmtbl/projects/bbcomp/ - implementation: https://www.ini.rub.de/PEOPLE/glasmtbl/projects/bbcomp/downloads/realworld-problems-bbcomp-EMO-2017.zip - source (real-world/artificial): real world - textual description: '' -- name: JSEC2019 - suite/generator/single: single - objectives: 1-5 - dimensionality: '32' - variable type: continuous - constraints: 'yes' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: http://www.jpnsec.org/files/competition2019/EC-Symposium-2019-Competition-English.html - implementation: http://www.jpnsec.org/files/competition2019/EC-Symposium-2019-Competition-English.html - source (real-world/artificial): real world - textual description: expensive evaluations 3s; 22 constraints -- name: RE - suite/generator/single: suite - objectives: 2-9 - dimensionality: 2-7 - variable type: continuous;integer;mixed - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://doi.org/10.1016/j.asoc.2020.106078 - implementation: https://github.com/ryojitanabe/reproblems - source (real-world/artificial): real world like - textual description: '' -- name: CRE - suite/generator/single: suite - objectives: 2-5 - dimensionality: 3-7 - variable type: continuous;integer;mixed - constraints: 'yes' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://doi.org/10.1016/j.asoc.2020.106078 - implementation: https://github.com/ryojitanabe/reproblems - source (real-world/artificial): real world like - textual description: '' -- name: Radar waveform - suite/generator/single: single - objectives: '9' - dimensionality: 4-12 - variable type: integer - constraints: 'yes' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://doi.org/10.1007/978-3-540-70928-2_53 - implementation: http://code.evanhughes.org/ - source (real-world/artificial): real world - textual description: '' -- name: MF2 - suite/generator/single: suite - objectives: '1' - dimensionality: 1-n - variable type: continuous - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'yes' - reference: https://doi.org/10.21105/joss.02049 - implementation: https://github.com/sjvrijn/mf2 - source (real-world/artificial): '' - textual description: '' -- name: AMVOP - suite/generator/single: suite - objectives: '1' - dimensionality: scalable - variable type: mixed continuous+ordinal+categorical+both - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: 'yes' - multi-fidelity: 'no' - reference: https://doi.org/10.1109/TEVC.2013.2281531 - implementation: '?' - source (real-world/artificial): '' - textual description: '' -- name: RWMVOP - suite/generator/single: suite - objectives: '1' - dimensionality: scalable - variable type: continuous;mixed continuous+ordinal+categorical+both - constraints: 'yes' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://doi.org/10.1109/TEVC.2013.2281531 - implementation: '?' - source (real-world/artificial): real world - textual description: '' -- name: SBOX-COST - suite/generator/single: suite - objectives: '1' - dimensionality: scalable - variable type: continuous - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: 'yes' - multi-fidelity: 'no' - reference: https://doi.org/10.48550/arXiv.2305.12221 - implementation: https://github.com/IOHprofiler/IOHexperimenter/ - source (real-world/artificial): '' - textual description: problems from BBOB but allows instances with the optimum close to the - boundary -- name: "\u03C1MNK-Landscapes" - suite/generator/single: generator - objectives: scalable - dimensionality: scalable - variable type: binary - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: 'yes' - multi-fidelity: 'no' - reference: https://doi.org/10.1016/j.ejor.2012.12.019 - implementation: https://gitlab.com/aliefooghe/mocobench/ - source (real-world/artificial): '' - textual description: tunable variable and objective dimensions; tunable multimodality and +fn_ato: + allows_partial_evaluation: no + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: 'Parameters of the Modules of the Automatic Train Operation are optimized; + two objectives: minimizing energy consumption and minimizing driving duration.' + dynamic_type: null + fidelity_levels: null + implementations: null + instances: null + long_name: null + modality: + - unimodal + name: ATO + noise_type: null + objectives: + - 2 + references: null + source: + - real-world + tags: null + type: problem + variables: + - dim: 10 + type: continuous +fn_building_spatial: + allows_partial_evaluation: no + can_evaluate_objectives_independently: null + code_examples: null + constraints: + - equality: null + hard: yes + number: null + type: box + - equality: null + hard: yes + number: null + type: unknown + description: Optimise the spatial layout of a building to minimise energy + consumption for climate control and minimise the strain on the structure. + Many hard constraints; mixed-variable (continuous+binary); expensive + evaluations. + dynamic_type: null + fidelity_levels: null + implementations: + - impl_bso_toolbox + instances: null + long_name: null + modality: null + name: Building spatial design + noise_type: null + objectives: + - 2 + references: + - authors: [] + link: + type: null + url: https://hdl.handle.net/1887/81789 + title: Building spatial design + source: + - real-world + tags: null + type: problem + variables: + - dim: + max: null + min: 1 + type: binary + - dim: + max: null + min: 1 + type: continuous +fn_convex_dtlz2: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: Variant of DTLZ2 with a convex Pareto front (instead of concave) + dynamic_type: null + fidelity_levels: null + implementations: null + instances: null + long_name: null + modality: null + name: Convex DTLZ2 + noise_type: null + objectives: + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1109/TEVC.2013.2281535 + title: Convex DTLZ2 + source: null + tags: null + type: problem + variables: + - dim: + max: null + min: 1 + type: continuous +fn_emdo: + allows_partial_evaluation: no + can_evaluate_objectives_independently: null + code_examples: null + constraints: + - equality: null + hard: yes + number: 12 + type: unknown + - equality: null + hard: yes + number: null + type: box + - equality: null + hard: some + number: null + type: unknown + description: "# Goal\nFind a design of a synchronous electric motor for power steering + systems that minimizes costs and satisfies all constraints.\n\n# Motivation\n\ + Challenging to find good solutions in a limited time.\n\n# Key Challenges\n* Time-consuming + solution evaluation\n* Highly-constrained problem\n* Constraints are multimodal\n\ + \nThis is not an available problem, but could be interesting to show to researchers + which difficulties appear in real-world problems." + dynamic_type: null + fidelity_levels: null + implementations: + - impl_emdo + instances: null + long_name: Electric Motor Design Optimization + modality: + - multimodal + name: Electric Motor Design Optimization + noise_type: + - noisy + objectives: + - 1 + references: + - authors: + - Tea Tušar + - Peter Korošec + - Bogdan Filipič + link: + type: null + url: https://dis.ijs.si/tea/Publications/Tusar23Multistep.pdf + title: A Multi-Step Evaluation Process in Electric Motor Design + source: + - real-world + tags: null + type: problem + variables: + - dim: 13 + type: integer + - dim: 13 + type: continuous +fn_fleetopt: + allows_partial_evaluation: yes + can_evaluate_objectives_independently: null + code_examples: null + constraints: + - equality: null + hard: yes + number: null + type: unknown + description: 'UK healthcare organisation fleet optimisation: reduce the fleet of + non-emergency healthcare trip vehicles while still ensuring all trips can be covered. + Bilevel: upper level 54 vars, lower level 13208 vars.' + dynamic_type: null + fidelity_levels: null + implementations: null + instances: null + long_name: null + modality: null + name: FleetOpt + noise_type: null + objectives: + - 1 + references: + - authors: [] + link: + type: null + url: https://dl.acm.org/doi/abs/10.1145/3638530.3664137 + title: FleetOpt + source: + - real-world + tags: null + type: problem + variables: + - dim: + - 13208 + - 54 + type: integer +fn_gasoline: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: + - equality: null + hard: yes + number: 5 + type: unknown + description: Multi-objective optimization to minimize fuel consumption and NOx + emissions over a two-minute dynamic duty cycle, subject to five constraints + (turbine inlet temperature, knock occurrences, peak cylinder pressure, peak + cylinder pressure rise, total work). Seven decision variables cover hardware + choices and engine control parameters. + dynamic_type: null + fidelity_levels: + - 1 + - 2 + implementations: + - impl_gasoline + instances: null + long_name: null + modality: null + name: Gasoline direct injection engine design + noise_type: null + objectives: + - 2 + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1016/j.ejor.2022.08.032 + title: Gasoline direct injection engine design + source: + - real-world + tags: null + type: problem + variables: + - dim: 7 + type: integer + - dim: 7 + type: continuous +fn_invdeceptive_deceptive_rotell: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: null + dynamic_type: null + fidelity_levels: null + implementations: null + instances: null + long_name: null + modality: null + name: InverseDeceptiveTrap+RotatedEllipsoid / DeceptiveTrap+RotatedEllipsoid + noise_type: null + objectives: + - 2 + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1145/3449726.3459521 + title: Mixed-variable multi-objective test problems + source: + - artificial + tags: null + type: problem + variables: + - dim: + max: null + min: 1 + type: binary + - dim: + max: null + min: 1 + type: continuous +fn_inverted_dtlz1: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: Variant of DTLZ1 with an inverted Pareto front + dynamic_type: null + fidelity_levels: null + implementations: null + instances: null + long_name: null + modality: null + name: Inverted DTLZ1 + noise_type: null + objectives: + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1109/TEVC.2013.2281534 + title: Inverted DTLZ1 + source: null + tags: null + type: problem + variables: + - dim: + max: null + min: 1 + type: continuous +fn_jsec2019: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: + - equality: null + hard: yes + number: 22 + type: unknown + description: expensive evaluations 3s; 22 constraints + dynamic_type: null + fidelity_levels: null + implementations: + - impl_jsec2019 + instances: null + long_name: null + modality: null + name: JSEC2019 + noise_type: null + objectives: + - 1 + - 2 + - 3 + - 4 + - 5 + references: + - authors: [] + link: + type: null + url: + http://www.jpnsec.org/files/competition2019/EC-Symposium-2019-Competition-English.html + title: JPNSEC EC-Symposium 2019 competition + source: + - real-world + tags: null + type: problem + variables: + - dim: 32 + type: continuous +fn_onemax_sphere_deceptive_rotell: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: null + dynamic_type: null + fidelity_levels: null + implementations: null + instances: null + long_name: null + modality: null + name: Onemax+Sphere / DeceptiveTrap+RotatedEllipsoid + noise_type: null + objectives: + - 2 + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1145/3449726.3459521 + title: Mixed-variable multi-objective test problems + source: + - artificial + tags: null + type: problem + variables: + - dim: + max: null + min: 1 + type: binary + - dim: + max: null + min: 1 + type: continuous +fn_onemax_sphere_zeromax_sphere: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: null + dynamic_type: null + fidelity_levels: null + implementations: null + instances: null + long_name: null + modality: null + name: Onemax+Sphere / Zeromax+Sphere + noise_type: null + objectives: + - 2 + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1145/3449726.3459521 + title: Onemax+Sphere / Zeromax+Sphere + source: + - artificial + tags: null + type: problem + variables: + - dim: + max: null + min: 1 + type: binary + - dim: + max: null + min: 1 + type: continuous +fn_radar_waveform: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: + - equality: null + hard: yes + number: null + type: unknown + description: null + dynamic_type: null + fidelity_levels: null + implementations: + - impl_radar_waveform + instances: null + long_name: null + modality: null + name: Radar waveform + noise_type: null + objectives: + - 9 + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1007/978-3-540-70928-2_53 + title: Radar waveform design + source: + - real-world + tags: null + type: problem + variables: + - dim: + max: 12 + min: 4 + type: integer +gen_beacon: + allows_partial_evaluation: no + can_evaluate_objectives_independently: null + code_examples: null + constraints: + - equality: null + hard: yes + number: 0 + type: box + description: Generator for bi-objective benchmark problems with explicitly + controlled correlations in continuous spaces. Multimodal with random + structure. + dynamic_type: null + fidelity_levels: null + implementations: + - impl_beacon + long_name: Continuous Bi-objective Benchmark problems with Explicit Adjustable + COrrelatioN control + modality: + - multimodal + name: BEACON + noise_type: null + objectives: + - 2 + references: + - authors: [] + link: + type: null + url: https://dl.acm.org/doi/10.1145/3712255.3734303 + title: BEACON + source: + - artificial + tags: null + type: generator + variables: + - dim: + max: null + min: 1 + type: continuous +gen_bono_bench: + allows_partial_evaluation: no + can_evaluate_objectives_independently: null + code_examples: null + constraints: + - equality: null + hard: yes + number: null + type: box + description: Bi-objective problem generator and suite with scalable continuous + decision space. Features complex problem properties and Pareto front + approximations with error guarantees for the hypervolume and exact R2 + indicators. + dynamic_type: null + fidelity_levels: null + implementations: + - impl_bonobench + long_name: Bi-objective Numerical Optimization Benchmark + modality: + - multimodal + name: BONO-Bench + noise_type: null + objectives: + - 2 + references: null + source: + - artificial + tags: null + type: generator + variables: + - dim: + max: null + min: 1 + type: continuous +gen_ealain: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: + - equality: null + hard: some + number: null + type: unknown + description: Real-world-like, easily extensible to increase complexity + dynamic_type: + - optional + fidelity_levels: + - 1 + - 2 + implementations: + - impl_ealain + long_name: null + modality: null + name: Ealain + noise_type: null + objectives: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1145/3638530.3654299 + title: Ealain + source: + - real-world-like + tags: null + type: generator + variables: + - dim: + max: null + min: 1 + type: binary + - dim: + max: null + min: 1 + type: integer + - dim: + max: null + min: 1 + type: continuous +gen_gnbg: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: Generator counterpart of GNBG. + dynamic_type: null + fidelity_levels: null + implementations: + - impl_gnbg + long_name: null + modality: null + name: GNBG + noise_type: null + objectives: + - 1 + references: + - authors: [] + link: + type: null + url: https://arxiv.org/abs/2312.07083 + title: GNBG + source: + - artificial + tags: null + type: generator + variables: + - dim: + max: null + min: 1 + type: continuous +gen_gnbg_ii: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: Generator counterpart of GNBG-II. + dynamic_type: null + fidelity_levels: null + implementations: + - impl_iohgnbg + - impl_gnbg_ii + long_name: null + modality: null + name: GNBG-II + noise_type: null + objectives: + - 1 + references: + - authors: [] + link: + type: null + url: https://dl.acm.org/doi/pdf/10.1145/3712255.3734271 + title: GNBG-II + source: + - artificial + tags: null + type: generator + variables: + - dim: + max: null + min: 1 + type: continuous +gen_gpd: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: + - equality: null + hard: some + number: null + type: unknown + description: null + dynamic_type: null + fidelity_levels: null + implementations: null + long_name: null + modality: null + name: GPD + noise_type: + - optional + objectives: + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1016/j.asoc.2020.106139 + title: GPD generator + source: null + tags: null + type: generator + variables: + - dim: + max: null + min: 1 + type: continuous +gen_iohclustering: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: Generator counterpart of the IOHClustering suite. + dynamic_type: null + fidelity_levels: null + implementations: + - impl_iohclustering + long_name: null + modality: + - multimodal + name: IOHClustering + noise_type: null + objectives: + - 1 + references: + - authors: [] + link: + type: null + url: https://arxiv.org/pdf/2505.09233 + title: IOHClustering + source: + - artificial-from-real-data + tags: null + type: generator + variables: + - dim: + max: null + min: 1 + type: continuous +gen_ma_bbob: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: Generator that creates affine combinations of BBOB functions + dynamic_type: null + fidelity_levels: null + implementations: + - impl_iohexperimenter + - impl_ma_bbob + long_name: null + modality: + - multimodal + name: MA-BBOB + noise_type: null + objectives: + - 1 + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1145/3673908 + title: MA-BBOB + source: + - artificial + tags: null + type: generator + variables: + - dim: + max: null + min: 1 + type: continuous +gen_mpm2: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: nonlinear nonseparable nonsymmetric; scalable in terms of time to + evaluate the objective function + dynamic_type: null + fidelity_levels: null + implementations: + - impl_mpm2 + long_name: null + modality: + - multimodal + name: MPM2 + noise_type: null + objectives: + - 1 + references: + - authors: [] + link: + type: null + url: https://ls11-www.cs.tu-dortmund.de/_media/techreports/tr15-01.pdf + title: MPM2 technical report TR15-01 + source: null + tags: null + type: generator + variables: + - dim: + max: null + min: 1 + type: continuous +gen_mubqp: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: tunable variable and objective dimensions; tunable density and correlation between objectives -- name: mUBQP - suite/generator/single: generator - objectives: scalable - dimensionality: scalable - variable type: binary - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: yes (quadratic) - multi-fidelity: 'no' - reference: https://doi.org/10.1016/j.asoc.2013.11.008 - implementation: https://gitlab.com/aliefooghe/mocobench/ - source (real-world/artificial): '' - textual description: tunable variable and objective dimensions; tunable density and correlation - between objectives -- name: "\u03C1mTSP" - suite/generator/single: generator - objectives: scalable - dimensionality: scalable - variable type: permutations - constraints: no (apart from being permutations) - dynamic: 'no' - noise: 'no' - multimodal: yes (quadratic) - multi-fidelity: 'no' - reference: https://doi.org/10.1007/978-3-319-45823-6_40 - implementation: https://gitlab.com/aliefooghe/mocobench/ - source (real-world/artificial): '' - textual description: tunable variable and objective dimensions; tunable instance type (euclidian/random); - tunable correlation between objectives -- name: CEC2015-DMOO - suite/generator/single: suite - objectives: 2-3 - dimensionality: '?' - variable type: continuous - constraints: '?' - dynamic: 'yes' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: Benchmark Functions for CEC 2015 Special Session and Competition on Dynamic - Multi-objective Optimization - implementation: '' - source (real-world/artificial): '' - textual description: '' -- name: Ealain - suite/generator/single: generator - objectives: 1+ - dimensionality: scalable - variable type: continuous,binary,integer - constraints: optional - dynamic: optional - noise: 'no' - multimodal: '?' - multi-fidelity: optional - reference: https://doi.org/10.1145/3638530.3654299 - implementation: https://github.com/qrenau/Ealain - source (real-world/artificial): Real-world-like - textual description: Real-world-like, easily extensible to increase complexity -- name: MA-BBOB - suite/generator/single: generator - objectives: '1' - dimensionality: scalable - variable type: continuous - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: 'yes' - multi-fidelity: 'no' - reference: https://doi.org/10.1145/3673908 - implementation: https://github.com/IOHprofiler/IOHexperimenter/blob/master/example/Competitions/MA-BBOB/Example_MABBOB.ipynb - source (real-world/artificial): artificial - textual description: Generator that creates affine combinations of BBOB functions -- name: MPM2 - suite/generator/single: generator - objectives: '1' - dimensionality: scalable - variable type: continuous - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: 'yes' - multi-fidelity: 'no' - reference: https://ls11-www.cs.tu-dortmund.de/_media/techreports/tr15-01.pdf - implementation: https://github.com/jakobbossek/smoof/blob/master/inst/mpm2.py - source (real-world/artificial): '' - textual description: nonlinear nonseparable nonsymmetric; scalable in terms of time to evaluate - the objective function -- name: Convex DTLZ2 - suite/generator/single: single - objectives: 2+ - dimensionality: scalable - variable type: continuous - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://doi.org/10.1109/TEVC.2013.2281535 - implementation: '?' - source (real-world/artificial): '' - textual description: Variant of DTLZ2 with a convex Pareto front (instead of concave) -- name: Inverted DTLZ1 - suite/generator/single: single - objectives: 2+ - dimensionality: scalable - variable type: continuous - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://doi.org/10.1109/TEVC.2013.2281534 - implementation: '?' - source (real-world/artificial): '' - textual description: Variant of DTLZ1 with an inverted Pareto front -- name: Minus DTLZ - suite/generator/single: suite - objectives: 2+ - dimensionality: scalable - variable type: continuous - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://doi.org/10.1109/TEVC.2016.2587749 - implementation: '?' - source (real-world/artificial): '' - textual description: Variant of DTLZ that minimises the inverse of the base DTLZ functions -- name: Minus WFG - suite/generator/single: suite - objectives: 2+ - dimensionality: scalable - variable type: continuous - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://doi.org/10.1109/TEVC.2016.2587749 - implementation: '?' - source (real-world/artificial): '' - textual description: Variant of WFG that minimises the inverse of the base WFG functions -- name: L1-ZDT - suite/generator/single: suite - objectives: '2' - dimensionality: scalable - variable type: continuous;binary - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://doi.org/10.1145/1143997.1144179 - implementation: '?' - source (real-world/artificial): '' - textual description: Variant of ZDT with linkages between variables within one of two groups - but not between variables in a different group; Linear recombination operators - can potentially take advantage of the problem structure -- name: L2-ZDT - suite/generator/single: suite - objectives: '2' - dimensionality: scalable - variable type: continuous;binary - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://doi.org/10.1145/1143997.1144179 - implementation: '?' - source (real-world/artificial): '' - textual description: Variant of ZDT with linkages between all variables; Linear recombination - operators can potentially take advantage of the problem structure -- name: L3-ZDT - suite/generator/single: suite - objectives: '2' - dimensionality: scalable - variable type: continuous;binary - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://doi.org/10.1145/1143997.1144179 - implementation: '?' - source (real-world/artificial): '' - textual description: Variant of L2-ZDT using a mapping to prevent linear recombination operators - from potentially taking advantage of the problem structure -- name: L2-DTLZ - suite/generator/single: suite - objectives: 2+ - dimensionality: scalable - variable type: continuous - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://doi.org/10.1145/1143997.1144179 - implementation: '?' - source (real-world/artificial): '' - textual description: Variant of DTLZ2 and DTLZ3 with linkages between all variables; Linear - recombination operators can potentially take advantage of the problem structure -- name: L3-DTLZ - suite/generator/single: suite - objectives: 2+ - dimensionality: scalable - variable type: continuous - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://doi.org/10.1145/1143997.1144179 - implementation: '?' - source (real-world/artificial): '' - textual description: Variant of L2-DTLZ using a mapping to prevent linear recombination operators - from potentially taking advantage of the problem structure -- name: CEC2018 DT - CEC2018 Competition on Dynamic Multiobjective Optimisation - suite/generator/single: suite - objectives: 2 or 3 - dimensionality: scalable? - variable type: '?' - constraints: 'no' - dynamic: 'yes' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://www.academia.edu/download/94499025/TR-CEC2018-DMOP-Competition.pdf - implementation: https://pymoo.org/problems/dynamic/df.html - source (real-world/artificial): artificial - textual description: '14 problems. Time-dependent: Pareto front/Pareto set geometry; - irregular Pareto front shapes; variable-linkage; number of disconnected Pareto - front segments; etc.' -- name: MODAct - multiobjective design of actuators - suite/generator/single: suite - objectives: 2 3 4 or 5 - dimensionality: '20' - variable type: mixed; integer and continuous - constraints: 'yes' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://doi.org/10.1109/TEVC.2020.3020046 - implementation: https://pymoo.org/problems/constrained/modact.html - source (real-world/artificial): real-world - textual description: Realistic Constrained Multi-Objective Optimization Benchmark - Problems from Design. Need the https://github.com/epfl-lamd/modact package installed; evaluation - times around 20ms -- name: IOHClustering - suite/generator/single: suite; generator - objectives: '1' - dimensionality: scalable - variable type: continuous - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: 'yes' - multi-fidelity: 'no ' - reference: https://arxiv.org/pdf/2505.09233 - implementation: https://github.com/IOHprofiler/IOHClustering - source (real-world/artificial): artificial, but based on real data - textual description: 'Set of benchmark problems from clustering: optimization task - is selecting cluster centers for a given set of data, with the number of clusters - defining problem dimensionality. Includes both a suite and a generator. Based on ML clustering datasets' -- name: GNBG-II - suite/generator/single: suite; generator - objectives: '1' - dimensionality: scalable - variable type: continuous - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://dl.acm.org/doi/pdf/10.1145/3712255.3734271 - implementation: https://github.com/rohitsalgotra/GNBG-II - source (real-world/artificial): artificial - textual description: Generalized Numerical Benchmark Generator (version 2). Also in IOH https://github.com/IOHprofiler/IOHGNBG -- name: GNBG - suite/generator/single: suite; generator - objectives: '1' - dimensionality: scalable - variable type: continuous - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://arxiv.org/abs/2312.07083 - implementation: https://github.com/Danial-Yazdani/GNBG-Generator - source (real-world/artificial): artificial - textual description: Generalized Numerical Benchmark Generator -- name: DynamicBinVal - suite/generator/single: suite - objectives: '1' - dimensionality: scalable - variable type: binary - constraints: 'no' - dynamic: 'yes' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://arxiv.org/pdf/2404.15837 - implementation: https://github.com/IOHprofiler/IOHexperimenter - source (real-world/artificial): artificial - textual description: Four versions of the dynamic binary value problem -- name: PBO - suite/generator/single: suite - objectives: '1' - dimensionality: scalable - variable type: binary - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://dl.acm.org/doi/pdf/10.1145/3319619.3326810 - implementation: https://github.com/IOHprofiler/IOHexperimenter - source (real-world/artificial): artificial - textual description: Suite of 25 binary optimization problems -- name: W-model - suite/generator/single: generator - objectives: '1' - dimensionality: scalable - variable type: binary - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://dl.acm.org/doi/abs/10.1145/3205651.3208240?casa_token=S4U_Pi9f6MwAAAAA:U9ztNTPwmupT8K3GamWZfBL7-8fqjxPtr_kprv51vdwA-REsp0EyOFGa99BtbANb0XbqyrVg795hIw - implementation: https://github.com/thomasWeise/BBDOB_W_Model - source (real-world/artificial): artificial - textual description: Tunable generator for binary optimization based on several + dynamic_type: null + fidelity_levels: null + implementations: + - impl_mocobench + long_name: null + modality: + - multimodal + - quadratic + name: mUBQP + noise_type: null + objectives: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1016/j.asoc.2013.11.008 + title: mUBQP benchmark + source: null + tags: null + type: generator + variables: + - dim: + max: null + min: 1 + type: binary +gen_puboi: + allows_partial_evaluation: no + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: A benchmark in which variable importance is tunable, based on the + Walsh function. + dynamic_type: null + fidelity_levels: null + implementations: + - impl_puboi + long_name: Polynomial Unconstrained Binary Optimization with tunable + importance + modality: null + name: PUBOi + noise_type: null + objectives: + - 1 + references: + - authors: [] + link: + type: null + url: https://link.springer.com/chapter/10.1007/978-3-031-04148-8_12 + title: PUBOi + source: + - artificial + tags: null + type: generator + variables: + - dim: + max: null + min: 1 + type: binary +gen_randoptgen: + allows_partial_evaluation: no + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: A Unified Random Problem Generator for Single- and + Multi-Objective Optimization Problems with Mixed-Variable Input Spaces. + dynamic_type: null + fidelity_levels: null + implementations: + - impl_randoptgen + long_name: RandOptGen + modality: + - multimodal + name: RandOptGen + noise_type: null + objectives: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + references: null + source: + - artificial + tags: null + type: generator + variables: + - dim: + max: null + min: 1 + type: binary + - dim: + max: null + min: 1 + type: integer + - dim: + max: null + min: 1 + type: continuous +gen_rho_mnk_landscapes: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: tunable variable and objective dimensions; tunable multimodality + and correlation between objectives + dynamic_type: null + fidelity_levels: null + implementations: + - impl_mocobench + long_name: null + modality: + - multimodal + name: ρMNK-Landscapes + noise_type: null + objectives: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1016/j.ejor.2012.12.019 + title: On the design of multi-objective evolutionary algorithms based on + NK-landscapes + source: null + tags: null + type: generator + variables: + - dim: + max: null + min: 1 + type: binary +gen_rho_mtsp: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: tunable variable and objective dimensions; tunable instance type + (euclidean/random); tunable correlation between objectives + dynamic_type: null + fidelity_levels: null + implementations: + - impl_mocobench + long_name: null + modality: + - multimodal + - quadratic + name: ρmTSP + noise_type: null + objectives: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1007/978-3-319-45823-6_40 + title: On the impact of multi-objective scalability for the ρmTSP + source: null + tags: null + type: generator + variables: + - dim: + max: null + min: 1 + type: unknown +gen_wmodel: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: Tunable generator for binary optimization based on several difficulty features -- name: Submodular Optimitzation - suite/generator/single: suite - objectives: '1' - dimensionality: scalable - variable type: binary - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=10254181 - implementation: https://github.com/IOHprofiler/IOHexperimenter - source (real-world/artificial): artificial - textual description: set of graph-based submodular optimization problems from 4 - problem types -- name: CEC2013 - suite/generator/single: suite - objectives: '1' - dimensionality: scalable - variable type: continuous - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://peerj.com/articles/cs-2671/CEC2013.pdf - implementation: https://github.com/P-N-Suganthan/CEC2013 - source (real-world/artificial): artificial - textual description: suite used for cec2013 competition. Also in IOH https://github.com/IOHprofiler/IOHexperimenter -- name: CEC2022 - suite/generator/single: suite - objectives: '1' - dimensionality: scalable - variable type: continuous - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: '?' - multi-fidelity: 'no' - reference: https://github.com/P-N-Suganthan/2022-SO-BO/blob/main/CEC2022%20TR.pdf - implementation: https://github.com/P-N-Suganthan/2022-SO-BO - source (real-world/artificial): artificial - textual description: suite used for cec2022 competition. Also in IOH https://github.com/IOHprofiler/IOHexperimenter -- name: Onemax+Sphere / Zeromax+Sphere - suite/generator/single: single - objectives: '2' - dimensionality: scalable - variable type: binary and continuous;mixed; - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: '' - multi-fidelity: 'no' - reference: https://doi.org/10.1145/3449726.3459521 - implementation: - source (real-world/artificial): 'artificial' - textual description: '' -- name: Onemax+Sphere / DeceptiveTrap+RotatedEllipsoid - suite/generator/single: single - objectives: '2' - dimensionality: scalable - variable type: binary and continuous;mixed; - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: '' - multi-fidelity: 'no' - reference: https://doi.org/10.1145/3449726.3459521 - implementation: - source (real-world/artificial): 'artificial' - textual description: '' -- name: InverseDeceptiveTrap+RotatedEllipsoid / DeceptiveTrap+RotatedEllipsoid - suite/generator/single: single - objectives: '2' - dimensionality: scalable - variable type: binary and continuous;mixed; - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: '' - multi-fidelity: 'no' - reference: https://doi.org/10.1145/3449726.3459521 - implementation: - source (real-world/artificial): 'artificial' - textual description: '' -- name: PorkchopPlotInterplanetaryTrajectory - suite/generator/single: suite - objectives: '1' - dimensionality: 2 - variable type: continuous - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: 'yes' - multi-fidelity: 'no' - reference: https://doi.org/10.1109/CEC65147.2025.11042973 - implementation: https://github.com/ShuaiqunPan/Transfer_Random_forests_BBOB_Real_world - source (real-world/artificial): 'real-world' - textual description: '' -- name: KinematicsRobotArm - suite/generator/single: suite - objectives: '1' - dimensionality: 21 - variable type: continuous - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: 'no' - multi-fidelity: 'no' - reference: https://doi.org/10.1023/A:1013258808932 - implementation: https://github.com/ShuaiqunPan/Transfer_Random_forests_BBOB_Real_world - source (real-world/artificial): 'real-world' - textual description: '' -- name: VehicleDynamics - suite/generator/single: suite - objectives: '1' - dimensionality: 2 - variable type: continuous - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: 'yes' - multi-fidelity: 'no' - reference: https://www.scitepress.org/Papers/2023/121580/121580.pdf - implementation: https://zenodo.org/records/8307853 - source (real-world/artificial): 'real-world' - textual description: '' -- name: MECHBench - suite/generator/single: Problem Suite - variable type: Continuous - dimensionality: scalable' - objectives: '1' - constraints: 'yes' - dynamic: 'no' - noise: 'no' - multimodal: 'yes' - multi-fidelity: 'no' - source (real-world/artificial): Real-World Application - implementation: https://github.com/BayesOptApp/MECHBench - textual description: This is a set of problems with inspiration from Structural - Mechanics Design Optimization. The suite comprises three physical models, from - which the user may define different kind of problems which impact the final design - output. - reference: https://arxiv.org/abs/2511.10821 - other info: - partial evaluations: 'no' - full name: MECHBench - constraint properties: Hard Constraints - number of constraints: 1 or 2 - description of multimodality: Unstructured or non isotropic multimodality - key challenges / characteristics: Embeds physical simulations and is flexible - and modular - scientific motivation: Bridge the black-box optimization techniques to a Mechanical - Design Problem which require these kinds of algorithms - limitations: The models do not include fracture or damage mechanics, just plasticity. - implementation languages: Python - approximate evaluation time: Times -> from 1 minute to 7 minutes -- name: EXPObench - suite/generator/single: Problem Suite - variable type: Continuous, Integer, Categorical, Conditional - dimensionality: 10 to 135 - objectives: '1' - constraints: 'yes' - dynamic: 'no' - noise: 'yes' - multimodal: Unknown - multi-fidelity: 'no' - source (real-world/artificial): Real-World Application - implementation: https://github.com/AlgTUDelft/ExpensiveOptimBenchmark - textual description: Wind farm layout optimization, gas filter design, pipe shape + dynamic_type: null + fidelity_levels: null + implementations: + - impl_wmodel + long_name: null + modality: null + name: W-model + noise_type: null + objectives: + - 1 + references: + - authors: [] + link: + type: null + url: https://dl.acm.org/doi/abs/10.1145/3205651.3208240 + title: W-model + source: + - artificial + tags: null + type: generator + variables: + - dim: + max: null + min: 1 + type: binary +impl_beacon: + description: Continuous Bi-objective Benchmark with Explicit Adjustable + COrrelatioN control + evaluation_time: negligible + language: Python + links: + - type: repository + url: https://github.com/Stebbet/BEACON/ + name: BEACON + requirements: null + type: implementation +impl_bonobench: + description: Bi-objective Numerical Optimization Benchmark (BONO-Bench) + evaluation_time: null + language: Python + links: + - type: repository + url: https://github.com/schaepermeier/bonobench + name: BONO-Bench + requirements: null + type: implementation +impl_bso_toolbox: + description: Building Spatial Design toolbox (TU/e) + evaluation_time: ~1s (smallest) to ~40s (larger) + language: C++ + links: + - type: repository + url: https://github.com/TUe-excellent-buildings/BSO-toolbox + name: BSO-toolbox + requirements: null + type: implementation +impl_car_structure: + description: JAXA LADSE benchmark problems + evaluation_time: null + language: null + links: + - type: website + url: http://ladse.eng.isas.jaxa.jp/benchmark/ + name: Car-structure benchmark + requirements: null + type: implementation +impl_cec2013: + description: Suganthan's reference implementation + evaluation_time: null + language: null + links: + - type: repository + url: https://github.com/P-N-Suganthan/CEC2013 + name: CEC2013 reference code + requirements: null + type: implementation +impl_cec2022: + description: Suganthan's reference implementation + evaluation_time: null + language: null + links: + - type: repository + url: https://github.com/P-N-Suganthan/2022-SO-BO + name: CEC2022 reference code + requirements: null + type: implementation +impl_cfd: + description: Expensive real-world CFD-based test problems + evaluation_time: 30s-15m + language: null + links: + - type: repository + url: https://bitbucket.org/arahat/cfd-test-problem-suite + name: CFD test problem suite + requirements: null + type: implementation +impl_coco: + description: 'Comparing Continuous Optimizers: black-box optimization benchmarking + platform' + evaluation_time: null + language: C/Python + links: + - type: repository + url: https://github.com/numbbo/coco + name: COCO framework + requirements: null + type: implementation +impl_coco_legacy: + description: Archived COCO download page that hosted the bbob-noisy suite + evaluation_time: null + language: C/Python + links: + - type: archive + url: + https://web.archive.org/web/20210416065610/https://coco.gforge.inria.fr/doku.php?id=downloads + name: COCO legacy (bbob-noisy) + requirements: null + type: implementation +impl_ealain: + description: Real-world-like extensible benchmark problem generator + evaluation_time: null + language: null + links: + - type: repository + url: https://github.com/qrenau/Ealain + name: Ealain + requirements: null + type: implementation +impl_emdo: + description: Not publicly available + evaluation_time: 8 minutes + language: Python + links: null + name: Electric Motor Design Optimization + requirements: null + type: implementation +impl_emo2017: + description: BBComp EMO-2017 real-world problem archive + evaluation_time: null + language: null + links: + - type: download + url: + https://www.ini.rub.de/PEOPLE/glasmtbl/projects/bbcomp/downloads/realworld-problems-bbcomp-EMO-2017.zip + name: EMO 2017 real-world problems + requirements: null + type: implementation +impl_etmof: + description: Evolutionary many-task optimization framework + evaluation_time: null + language: null + links: + - type: repository + url: https://github.com/songbai-liu/etmo + name: ETMOF + requirements: null + type: implementation +impl_expobench: + description: EXPensive Optimization benchmark library (wind farm layout, gas + filter design, pipe shape, hyperparameter tuning, hospital simulation) + evaluation_time: 2 to 80 seconds + language: Python + links: + - type: repository + url: https://github.com/AlgTUDelft/ExpensiveOptimBenchmark + name: EXPObench + requirements: null + type: implementation +impl_gasoline: + description: Proprietary Matlab Simulink + Wave RT co-simulation + evaluation_time: null + language: Matlab Simulink / Wave RT + links: + - type: paper + url: https://doi.org/10.1016/j.ejor.2022.08.032 + name: Gasoline direct injection engine design + requirements: null + type: implementation +impl_gbea: + description: Game-Benchmark for Evolutionary Algorithms (COCO fork) + evaluation_time: 5s-35s + language: null + links: + - type: repository + url: https://github.com/ttusar/coco-gbea + name: coco-gbea + requirements: null + type: implementation +impl_gnbg: + description: Generalized Numerical Benchmark Generator + evaluation_time: null + language: null + links: + - type: repository + url: https://github.com/Danial-Yazdani/GNBG-Generator + name: GNBG Generator + requirements: null + type: implementation +impl_gnbg_ii: + description: Generalized Numerical Benchmark Generator version 2 + evaluation_time: null + language: null + links: + - type: repository + url: https://github.com/rohitsalgotra/GNBG-II + name: GNBG-II + requirements: null + type: implementation +impl_iohclustering: + description: Clustering-based optimization benchmark built on ML datasets + evaluation_time: null + language: null + links: + - type: repository + url: https://github.com/IOHprofiler/IOHClustering + name: IOHClustering + requirements: null + type: implementation +impl_iohexperimenter: + description: IOHprofiler experimenter framework + evaluation_time: null + language: C++/Python + links: + - type: repository + url: https://github.com/IOHprofiler/IOHexperimenter + name: IOHexperimenter + requirements: null + type: implementation +impl_iohgnbg: + description: IOHprofiler version of GNBG + evaluation_time: null + language: null + links: + - type: repository + url: https://github.com/IOHprofiler/IOHGNBG + name: IOHGNBG + requirements: null + type: implementation +impl_jsec2019: + description: JPNSEC EC-Symposium 2019 competition problem + evaluation_time: 3s + language: null + links: + - type: website + url: + http://www.jpnsec.org/files/competition2019/EC-Symposium-2019-Competition-English.html + name: JSEC 2019 competition + requirements: null + type: implementation +impl_ma_bbob: + description: Example notebook for MA-BBOB in IOHexperimenter + evaluation_time: null + language: null + links: + - type: example + url: + https://github.com/IOHprofiler/IOHexperimenter/blob/master/example/Competitions/MA-BBOB/Example_MABBOB.ipynb + name: MA-BBOB (IOHexperimenter) + requirements: null + type: implementation +impl_mechbench: + description: Structural mechanics design optimization benchmark + evaluation_time: 1-7 minutes + language: Python + links: + - type: repository + url: https://github.com/BayesOptApp/MECHBench + name: MECHBench + requirements: null + type: implementation +impl_mf2: + description: Multi-fidelity test function collection + evaluation_time: null + language: Python + links: + - type: repository + url: https://github.com/sjvrijn/mf2 + name: mf2 + requirements: null + type: implementation +impl_mmopp: + description: ECI lab distribution page for MMOPP + evaluation_time: null + language: null + links: + - type: website + url: http://www5.zzu.edu.cn/ecilab/info/1036/1251.htm + name: MMOPP + requirements: null + type: implementation +impl_mocobench: + description: Multi-objective combinatorial optimization benchmark + evaluation_time: null + language: C++ + links: + - type: repository + url: https://gitlab.com/aliefooghe/mocobench/ + name: mocobench + requirements: null + type: implementation +impl_modact: + description: EPFL-LAMD modact package + evaluation_time: 20ms + language: null + links: + - type: repository + url: https://github.com/epfl-lamd/modact + name: modact + requirements: null + type: implementation +impl_morepo: + description: Multi-objective optimisation problem repository + evaluation_time: null + language: null + links: + - type: repository + url: https://github.com/MCDMSociety/MOrepo + name: MOrepo + requirements: null + type: implementation +impl_mpm2: + description: Python implementation of MPM2 distributed with smoof + evaluation_time: null + language: Python + links: + - type: source + url: https://github.com/jakobbossek/smoof/blob/master/inst/mpm2.py + name: MPM2 (smoof) + requirements: null + type: implementation +impl_puboi: + description: A benchmark in which variable importance is tunable, based on the + Walsh function + evaluation_time: null + language: Python / C++ + links: + - type: repository + url: https://gitlab.com/verel/pubo-importance-benchmark + name: PUBO Importance Benchmark + requirements: null + type: implementation +impl_pycutest: + description: Python interface to CUTEst + evaluation_time: null + language: Python / C++ / Fortran + links: + - type: repository + url: https://github.com/jfowkes/pycutest + name: pycutest + requirements: null + type: implementation +impl_pymoo: + description: Multi-objective optimization in Python + evaluation_time: null + language: Python + links: + - type: repository + url: https://github.com/anyoptimization/pymoo + name: pymoo + requirements: null + type: implementation +impl_radar_waveform: + description: Radar waveform design reference implementation + evaluation_time: null + language: null + links: + - type: website + url: http://code.evanhughes.org/ + name: Evan Hughes radar waveform code + requirements: null + type: implementation +impl_randoptgen: + description: Unified Random Problem Generator for Single- and Multi-Objective + Optimization with Mixed-Variable Input Spaces + evaluation_time: milliseconds + language: Python + links: + - type: repository + url: https://github.com/MALEO-research-group/RandOptGen + - type: example + url: https://doi.org/10.1145/3712256.3726478 + name: RandOptGen + requirements: null + type: implementation +impl_reproblems: + description: Real-world inspired multi-objective optimization problem suite + evaluation_time: null + language: Python + links: + - type: repository + url: https://github.com/ryojitanabe/reproblems + name: reproblems + requirements: null + type: implementation +impl_transfer_rf_bbob_rw: + description: Real-world BBOB-like problem implementations (Porkchop, + KinematicsRobotArm) + evaluation_time: null + language: null + links: + - type: repository + url: https://github.com/ShuaiqunPan/Transfer_Random_forests_BBOB_Real_world + name: Transfer Random Forests BBOB Real-world + requirements: null + type: implementation +impl_tulipa: + description: Large linear program for optimal investment and operation of + energy systems + evaluation_time: minutes to hours + language: Julia / JuMP + links: + - type: website + url: https://tulipaenergy.github.io/TulipaEnergyModel.jl/stable/ + - type: example + url: https://github.com/TulipaEnergy/Tulipa-OBZ-CaseStudy + name: TulipaEnergyModel.jl + requirements: null + type: implementation +impl_vehicle_dynamics: + description: Zenodo archive for the vehicle dynamics benchmark + evaluation_time: null + language: null + links: + - type: archive + url: https://zenodo.org/records/8307853 + name: VehicleDynamics (Zenodo) + requirements: null + type: implementation +impl_wmodel: + description: Tunable generator for binary optimization + evaluation_time: null + language: null + links: + - type: repository + url: https://github.com/thomasWeise/BBDOB_W_Model + name: BBDOB W-Model + requirements: null + type: implementation +suite_amvop: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: null + dynamic_type: null + fidelity_levels: null + implementations: null + long_name: null + modality: + - multimodal + name: AMVOP + noise_type: null + objectives: + - 1 + problems: null + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1109/TEVC.2013.2281531 + title: AMVOP + source: null + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: categorical + - dim: + max: null + min: 1 + type: integer + - dim: + max: null + min: 1 + type: continuous +suite_bbob: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: null + dynamic_type: null + fidelity_levels: null + implementations: + - impl_coco + long_name: null + modality: + - multimodal + name: BBOB + noise_type: null + objectives: + - 1 + problems: null + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1080/10556788.2020.1808977 + title: 'COCO: a platform for comparing continuous optimizers in a black-box setting' + source: null + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: continuous +suite_bbob_biobj: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: null + dynamic_type: null + fidelity_levels: null + implementations: + - impl_coco + long_name: null + modality: + - multimodal + name: BBOB-biobj + noise_type: null + objectives: + - 2 + problems: null + references: + - authors: [] + link: + type: null + url: https://doi.org/10.48550/arXiv.1604.00359 + title: BBOB bi-objective test suite + source: null + tags: null + type: suite + variables: + - dim: + max: 40 + min: 2 + type: continuous +suite_bbob_biobj_mixint: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: null + dynamic_type: null + fidelity_levels: null + implementations: + - impl_coco + long_name: null + modality: + - multimodal + name: BBOB-biobj-mixint + noise_type: null + objectives: + - 2 + problems: null + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1145/3321707.3321868 + title: BBOB bi-objective mixed-integer test suite + source: null + tags: null + type: suite + variables: + - dim: + max: 160 + min: 5 + type: integer + - dim: + max: 160 + min: 5 + type: continuous +suite_bbob_constrained: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: + - equality: null + hard: yes + number: null + type: unknown + description: null + dynamic_type: null + fidelity_levels: null + implementations: + - impl_coco + long_name: null + modality: + - multimodal + name: BBOB-constrained + noise_type: null + objectives: + - 1 + problems: null + references: + - authors: [] + link: + type: null + url: http://numbbo.github.io/coco-doc/bbob-constrained/ + title: bbob-constrained documentation + source: null + tags: null + type: suite + variables: + - dim: + max: 40 + min: 2 + type: continuous +suite_bbob_largescale: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: null + dynamic_type: null + fidelity_levels: null + implementations: + - impl_coco + long_name: null + modality: + - multimodal + name: BBOB-largescale + noise_type: null + objectives: + - 1 + problems: null + references: + - authors: [] + link: + type: null + url: https://doi.org/10.48550/arXiv.1903.06396 + title: BBOB large-scale test suite + source: null + tags: null + type: suite + variables: + - dim: + max: 640 + min: 20 + type: continuous +suite_bbob_mixint: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: null + dynamic_type: null + fidelity_levels: null + implementations: + - impl_coco + long_name: null + modality: + - multimodal + name: BBOB-mixint + noise_type: null + objectives: + - 1 + problems: null + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1145/3321707.3321868 + title: BBOB mixed-integer test suite + source: null + tags: null + type: suite + variables: + - dim: + max: 160 + min: 5 + type: integer + - dim: + max: 160 + min: 5 + type: continuous +suite_bbob_noisy: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: null + dynamic_type: null + fidelity_levels: null + implementations: + - impl_coco_legacy + long_name: null + modality: + - multimodal + name: BBOB-noisy + noise_type: + - noisy + objectives: + - 1 + problems: null + references: + - authors: [] + link: + type: null + url: https://hal.inria.fr/inria-00369466 + title: 'Real-parameter black-box optimization benchmarking: noisy functions definitions' + source: null + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: continuous +suite_bp: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: null + dynamic_type: null + fidelity_levels: null + implementations: null + long_name: null + modality: null + name: BP + noise_type: + - unknown + objectives: + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + problems: null + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1109/CEC.2019.8790277 + title: BP benchmark + source: null + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: continuous +suite_brachytherapy: + allows_partial_evaluation: yes + can_evaluate_objectives_independently: null + code_examples: null + constraints: + - equality: null + hard: yes + number: + max: null + min: 1 + type: unknown + description: Treatment planning for internal radiation therapy. + Multi-objective with aggregated objectives; no public source code. + dynamic_type: null + fidelity_levels: + - 1 + - 2 + implementations: null + long_name: Brachytherapy treatment planning + modality: + - multimodal + name: Brachytherapy treatment planning + noise_type: null + objectives: + - 2 + - 3 + problems: null + references: + - authors: [] + link: + type: null + url: https://www.sciencedirect.com/science/article/pii/S1538472123016781 + title: Brachytherapy treatment planning + source: + - real-world + tags: null + type: suite + variables: + - dim: + max: 500 + min: 100 + type: continuous +suite_car_structure: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: + - equality: null + hard: yes + number: 54 + type: unknown + description: 54 constraints + dynamic_type: null + fidelity_levels: null + implementations: + - impl_car_structure + long_name: null + modality: null + name: Car structure + noise_type: null + objectives: + - 2 + problems: null + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1145/3205651.3205702 + title: Car structure design benchmark + source: + - real-world + tags: null + type: suite + variables: + - dim: + max: 222 + min: 144 + type: integer +suite_cdmp: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: + - equality: null + hard: yes + number: null + type: unknown + description: null + dynamic_type: + - unknown + fidelity_levels: null + implementations: null + long_name: null + modality: null + name: CDMP + noise_type: + - unknown + objectives: + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + problems: null + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1145/3321707.3321878 + title: CDMP benchmark + source: null + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: continuous +suite_cec2013: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: suite used for cec2013 competition. Also in IOH. + dynamic_type: null + fidelity_levels: null + implementations: + - impl_cec2013 + - impl_iohexperimenter + long_name: null + modality: null + name: CEC2013 + noise_type: null + objectives: + - 1 + problems: null + references: + - authors: [] + link: + type: null + url: https://peerj.com/articles/cs-2671/CEC2013.pdf + title: CEC2013 definitions + source: + - artificial + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: continuous +suite_cec2015_dmoo: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: + - equality: null + hard: '?' + number: null + type: unknown + description: null + dynamic_type: + - dynamic + fidelity_levels: null + implementations: null + long_name: null + modality: null + name: CEC2015-DMOO + noise_type: null + objectives: + - 2 + - 3 + problems: null + references: + - authors: [] + link: null + title: Benchmark Functions for CEC 2015 Special Session and Competition on + Dynamic Multi-objective Optimization + source: null + tags: null + type: suite + variables: + - dim: 0 + type: continuous +suite_cec2018_dt: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: '14 problems. Time-dependent: Pareto front/Pareto set geometry; irregular + Pareto front shapes; variable-linkage; number of disconnected Pareto front segments; + etc.' + dynamic_type: + - dynamic + fidelity_levels: null + implementations: + - impl_pymoo + long_name: CEC2018 Competition on Dynamic Multiobjective Optimisation + modality: null + name: CEC2018 DT + noise_type: null + objectives: + - 2 + - 3 + problems: null + references: + - authors: [] + link: + type: null + url: + https://www.academia.edu/download/94499025/TR-CEC2018-DMOP-Competition.pdf + title: CEC2018 DMOP Competition TR + source: + - artificial + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: unknown +suite_cec2022: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: suite used for cec2022 competition. Also in IOH. + dynamic_type: null + fidelity_levels: null + implementations: + - impl_iohexperimenter + - impl_cec2022 + long_name: null + modality: null + name: CEC2022 + noise_type: null + objectives: + - 1 + problems: null + references: + - authors: [] + link: + type: null + url: + https://github.com/P-N-Suganthan/2022-SO-BO/blob/main/CEC2022%20TR.pdf + title: CEC2022 TR + source: + - artificial + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: continuous +suite_cfd: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: + - equality: null + hard: yes + number: null + type: unknown + description: expensive evaluations 30s-15m + dynamic_type: null + fidelity_levels: null + implementations: + - impl_cfd + long_name: null + modality: null + name: CFD + noise_type: null + objectives: + - 1 + - 2 + problems: null + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1007/978-3-319-99259-4_24 + title: CFD test problem suite + source: + - real-world + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: unknown +suite_cre: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: + - equality: null + hard: yes + number: null + type: unknown + description: null + dynamic_type: null + fidelity_levels: null + implementations: + - impl_reproblems + long_name: null + modality: null + name: CRE + noise_type: null + objectives: + - 2 + - 3 + - 4 + - 5 + problems: null + references: + - authors: + - Ryoji Tanabe + - Hisao Ishibuchi + link: + type: null + url: https://doi.org/10.1016/j.asoc.2020.106078 + title: Easy-to-evaluate real-world multi-objective optimization problems + source: + - real-world-like + tags: null + type: suite + variables: + - dim: + max: 7 + min: 3 + type: integer + - dim: + max: 7 + min: 3 + type: continuous +suite_cuter: + allows_partial_evaluation: no + can_evaluate_objectives_independently: null + code_examples: null + constraints: + - equality: null + hard: yes + number: null + type: unknown + description: A constrained and unconstrained testing environment. + dynamic_type: null + fidelity_levels: null + implementations: null + long_name: null + modality: null + name: CUTEr + noise_type: null + objectives: + - 1 + problems: null + references: + - authors: [] + link: + type: null + url: https://dl.acm.org/doi/10.1145/962437.962439 + title: CUTEr + source: + - artificial + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: binary + - dim: + max: null + min: 1 + type: integer + - dim: + max: null + min: 1 + type: continuous +suite_cutest: + allows_partial_evaluation: no + can_evaluate_objectives_independently: null + code_examples: null + constraints: + - equality: null + hard: some + number: + max: null + min: 1 + type: unknown + - equality: null + hard: yes + number: null + type: box + description: CUTEst for optimization software + dynamic_type: null + fidelity_levels: null + implementations: + - impl_pycutest + long_name: Constrained and Unconstrained Testing Environment with safe threads + modality: + - multimodal + name: CUTEst + noise_type: null + objectives: + - 1 + problems: null + references: + - authors: [] + link: + type: null + url: https://link.springer.com/article/10.1007/s10589-014-9687-3 + title: CUTEst + source: + - artificial + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: binary + - dim: + max: null + min: 1 + type: integer + - dim: + max: null + min: 1 + type: continuous +suite_dtlz: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: null + dynamic_type: null + fidelity_levels: null + implementations: + - impl_pymoo + long_name: null + modality: null + name: DTLZ + noise_type: null + objectives: + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + problems: null + references: + - authors: + - Kalyanmoy Deb + - Lothar Thiele + - Marco Laumanns + - Eckart Zitzler + link: + type: null + url: https://doi.org/10.1109/CEC.2002.1007032 + title: Scalable multi-objective optimization test problems + source: null + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: continuous +suite_dynamicbinval: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: Four versions of the dynamic binary value problem + dynamic_type: + - dynamic + fidelity_levels: null + implementations: + - impl_iohexperimenter + long_name: null + modality: null + name: DynamicBinVal + noise_type: null + objectives: + - 1 + problems: null + references: + - authors: [] + link: + type: null + url: https://arxiv.org/pdf/2404.15837 + title: DynamicBinVal + source: + - artificial + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: binary +suite_emo2017: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: null + dynamic_type: null + fidelity_levels: null + implementations: + - impl_emo2017 + long_name: null + modality: null + name: EMO2017 + noise_type: null + objectives: + - 2 + problems: null + references: + - authors: [] + link: + type: null + url: https://www.ini.rub.de/PEOPLE/glasmtbl/projects/bbcomp/ + title: BBComp EMO 2017 + source: + - real-world + tags: null + type: suite + variables: + - dim: + max: 24 + min: 4 + type: continuous +suite_etmof: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: null + dynamic_type: + - dynamic + fidelity_levels: null + implementations: + - impl_etmof + long_name: null + modality: null + name: ETMOF + noise_type: null + objectives: + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + - 21 + - 22 + - 23 + - 24 + - 25 + - 26 + - 27 + - 28 + - 29 + - 30 + - 31 + - 32 + - 33 + - 34 + - 35 + - 36 + - 37 + - 38 + - 39 + - 40 + - 41 + - 42 + - 43 + - 44 + - 45 + - 46 + - 47 + - 48 + - 49 + - 50 + problems: null + references: + - authors: [] + link: + type: null + url: https://doi.org/10.48550/arXiv.2110.08033 + title: Evolutionary many-task optimization framework + source: null + tags: null + type: suite + variables: + - dim: + max: 10000 + min: 25 + type: continuous +suite_expobench: + allows_partial_evaluation: no + can_evaluate_objectives_independently: null + code_examples: null + constraints: + - equality: null + hard: yes + number: null + type: box + - equality: null + hard: some + number: null + type: unknown + description: Wind farm layout optimization, gas filter design, pipe shape optimization, hyperparameter tuning, and hospital simulation - reference: https://doi.org/10.1016/j.asoc.2023.110744 - other info: - partial evaluations: 'no' - full name: EXPensive Optimization benchmark library - constraint properties: Hard Constraints, Soft Constraints, Box Constraints, only - box constraints implemented, others appear as penalty in objective - number of constraints: 2 per variable (box), other constraints unknown (simulator - fails) - form of noise model: real-life (unknown) - type of noise space: Observational - key challenges / characteristics: Expensive objectives - scientific motivation: Address the lack of real-life expensive benchmarks - limitations: single-objective only, constraints are handled naively (penalty in - objective), no parallelization - implementation languages: Python - approximate evaluation time: 2 to 80 seconds -- name: Gasoline direct injection engine design - suite/generator/single: Single Problem - variable type: Continuous, Ordinal - dimensionality: '7' - objectives: '2' - constraints: 'yes' - dynamic: 'no' - noise: 'no' - multimodal: Unknown - multi-fidelity: 'yes' - source (real-world/artificial): Real-World Application - implementation: https://doi.org/10.1016/j.ejor.2022.08.032 - textual description: 'A multi-objective optimization problem seeking to minimize - fuel consumption and NOx emissions over a two-minute dynamic duty cycle, subject - to five constraints (turbine inlet temperature, number of knock occurrences, peak - cylinder pressure, peak cylinder pressure rise, total work). Seven decision variables - are defined: four define the hardware choices of cylinder compression ratio, turbo - machinery and EGR cooler sizing; three relate to control variables that parameterise - the engine control logic.' - other info: - partial evaluations: Unknown - constraint properties: Hard Constraints, Soft Constraints - number of constraints: '5' - key challenges / characteristics: Expensive - limitations: Proprietary - implementation languages: Matlab Simulink and Wave RT co-simulation -- name: BEACON - suite/generator/single: Generator - variable type: Continuous - dimensionality: scalable - objectives: '2' - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: 'yes' - multi-fidelity: 'no' - source (real-world/artificial): Artificially Generated - implementation: https://github.com/Stebbet/BEACON/ - textual description: Generator for bi-objective benchmark problems with explicitly - controlled correlations in continuous spaces. - reference: https://dl.acm.org/doi/10.1145/3712255.3734303 - other info: - partial evaluations: 'no' - full name: Continuous Bi-objective Benchmark problems with Explicit Adjustable - COrrelatioN control - constraint properties: Box Constraints - number of constraints: '0' - description of multimodality: Random - key challenges / characteristics: Multimodal, different correlations among objectives - scientific motivation: Controlled correlation among objectives - limitations: No analytical Pareto front, only bi-objective - implementation languages: Python - approximate evaluation time: Negligible -- name: TulipaEnergy - suite/generator/single: Problem Suite - variable type: Continuous - dimensionality: scalable - objectives: '1' - constraints: 'yes' - dynamic: 'no' - noise: 'yes' - multimodal: 'no' - multi-fidelity: 'yes' - source (real-world/artificial): Real-World Application - implementation: https://tulipaenergy.github.io/TulipaEnergyModel.jl/stable/ - textual description: Determine the optimal investment and operation decisions for - different types of assets in the energy system (production, consumption, conversion, - storage, and transport), while minimizing loss of load. - reference: See https://tulipaenergy.github.io/TulipaEnergyModel.jl/stable/40-scientific-foundation/45-scientific-references - other info: - partial evaluations: Unknown - full name: TulipaEnergyModel.jl - constraint properties: Hard Constraints, Soft Constraints - number of constraints: millions - type of dynamicism: none - form of noise model: "depends on input \u2014 still working on stochastic inputs" - type of noise space: Parameter - key challenges / characteristics: modeled as a potentially very large linear program, - different fidelities possible - scientific motivation: new techniques for solving large whitebox linear optimization - problems - limitations: not yet stochastic - implementation languages: Julia / JMP - approximate evaluation time: from minutes to hours - links to usage examples: https://github.com/TulipaEnergy/Tulipa-OBZ-CaseStudy -- name: ATO - suite/generator/single: Single Problem - variable type: Continuous - dimensionality: '10' - objectives: '2' - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: 'no' - multi-fidelity: 'no' - source (real-world/artificial): Real-World Application - implementation: '-' - textual description: Parameters of the Modules of the Automatic Train Operation - should be optimized. The parameters are continuous with different ranges. There - are two objectives (minimizing energy consumption, minimizing driving duration. - other info: - partial evaluations: 'no' -- name: Brachytherapy treatment planning - suite/generator/single: Problem Suite - variable type: Continuous - dimensionality: 100-500 - objectives: 2-3 - constraints: 'yes' - dynamic: 'no' - noise: 'no' - multimodal: 'yes' - multi-fidelity: 'yes' - source (real-world/artificial): Real-World Application - textual description: Treatment planning for internal radiation therapy - reference: https://www.sciencedirect.com/science/article/pii/S1538472123016781 - other info: - partial evaluations: 'yes' - full name: Brachytherapy treatment planning - constraint properties: Hard Constraints - number of constraints: scalable - key challenges / characteristics: Multi-objective; aggregated objectives - limitations: No public source code -- name: FleetOpt - suite/generator/single: Single Problem - variable type: Integer - dimensionality: 'Upper level: 54; lower level: 13208' - objectives: '1' - constraints: 'yes' - dynamic: 'no' - noise: 'no' - multimodal: Unknown - multi-fidelity: 'no' - source (real-world/artificial): Real-World Application - implementation: 'Not public: was done for real client with their private data' - textual description: 'Healthcare organisation in the UK provided data about their - current fleet of vehicles to conduct non-emergency heathcare trips in the Argyll - and Bute region of Scotland, UK. They also provided historical data about the - trips the vehicles took and about the bases which the vehicles return to. The - aim is to reduce the existing fleet of vehicles while still ensuring all trips - can be covered. Moving a vehicle from one base to another to help cover trips - is OK as long as the original base can still cover its trips. Link to paper with - more details: https://dl.acm.org/doi/abs/10.1145/3638530.3664137' - reference: https://dl.acm.org/doi/abs/10.1145/3638530.3664137 - other info: - partial evaluations: 'yes' -- name: Building spatial design - suite/generator/single: Single Problem - variable type: Continuous, Boolean - dimensionality: scalable depending on problem size (e.g. 90 for) - objectives: '2' - constraints: 'yes' - dynamic: 'no' - noise: 'no' - multimodal: Unknown - multi-fidelity: 'no' - source (real-world/artificial): Real-World Application - implementation: https://github.com/TUe-excellent-buildings/BSO-toolbox - textual description: 'Optimise the spatial layout of a building to: minimise energy - consumption for climate control, and minimise the strain on the structure' - reference: https://hdl.handle.net/1887/81789 - other info: - partial evaluations: 'no' - full name: Building spatial design - constraint properties: Hard Constraints, Box Constraints, Permutation Constraints - number of constraints: 2065 (as example, depends on problem size) - key challenges / characteristics: Many hard constraints (simulator cannot evaluate - the solution if these are violated); Mixed-variable search space (continuous - + binary); Multiple objectives; (Somewhat) expensive solution evaluations - implementation languages: C++ - approximate evaluation time: Roughly 1 second per evaluation for the smallest - considered design, and roughly 40 seconds for the larger designs we considered. - (Even the larger designs we considered are still relatively small for the considered - problem.) -- name: Electric Motor Design Optimization - suite/generator/single: Single Problem - variable type: Continuous, Integer - dimensionality: '13' - objectives: '1' - constraints: 'yes' - dynamic: 'no' - noise: 'yes' - multimodal: 'yes' - multi-fidelity: 'no' - source (real-world/artificial): Real-World Application - implementation: Implementation not freely available - textual description: The goal is to find a design of a synchronous electric motor - for power steering systems that minimizes costs and satisfies all constraints. - reference: https://dis.ijs.si/tea/Publications/Tusar23Multistep.pdf (paper in Slovene) - other info: - partial evaluations: 'no' - full name: Electric Motor Design Optimization - constraint properties: Hard Constraints, Soft Constraints, Box Constraints - number of constraints: '12' - description of multimodality: Constraints are multimodal - key challenges / characteristics: Time-consuming solution evaluation, highly-constrained - problem - scientific motivation: Challenging to find good solutions in a limited time - limitations: 'Unavailability, even if available, it wouldn''t be helpful to use - for benchmarking due taking a long time to evaluate a single solution ' - implementation languages: Python - approximate evaluation time: 8 minutes - general: This is not an available problem, but could be interesting to show to - researchers which difficulties appear in real-world problems -- name: BONO-Bench - suite/generator/single: Generator - variable type: Continuous - dimensionality: scalable - objectives: '2' - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: 'yes' - multi-fidelity: 'no' - source (real-world/artificial): Artificially Generated - implementation: https://github.com/schaepermeier/bonobench - textual description: Bi-objective problem generator and suite with scalable continuous - decision space. Features complex problem properties (different types of multimodality - and challenges in decision and objective space) as well as Pareto front approximations - with error guarantees for the hypervolume and exact R2 indicators. - other info: - partial evaluations: 'no' - full name: Bi-objective Numerical Optimization Benchmark (BONO-Bench) - constraint properties: Box Constraints - implementation languages: Python -- name: RandOptGen - suite/generator/single: Generator - variable type: Continuous, Integer, Boolean - dimensionality: scalable - objectives: scalable - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: 'yes' - multi-fidelity: 'no' - source (real-world/artificial): Artificially Generated - implementation: https://github.com/MALEO-research-group/RandOptGen - textual description: 'RandOptGen: A Unified Random Problem Generator for Single-and - Multi-Objective Optimization Problems with Mixed-Variable Input Spaces' - other info: - partial evaluations: 'no' - full name: RandOptGen - implementation languages: Python - approximate evaluation time: milliseconds - links to usage examples: https://doi.org/10.1145/3712256.3726478 -- name: CUTEr - suite/generator/single: Problem Suite - variable type: Continuous, Integer, Boolean - dimensionality: scalable - objectives: '1' - constraints: 'yes' - dynamic: 'no' - noise: 'no' - multimodal: Unknown - multi-fidelity: 'no' - source (real-world/artificial): Artificially Generated - implementation: Not Found - textual description: A constrained and unconstrained testing environment - reference: https://dl.acm.org/doi/10.1145/962437.962439 - other info: - partial evaluations: 'no' -- name: CUTEst - suite/generator/single: Problem Suite - variable type: Continuous, Integer, Boolean - dimensionality: scalable - objectives: '1' - constraints: 'yes' - dynamic: 'no' - noise: 'no' - multimodal: 'yes' - multi-fidelity: 'no' - source (real-world/artificial): Artificially Generated - implementation: https://github.com/jfowkes/pycutest - textual description: The Constrained and Unconstrained Testing Environment with - safe threads (CUTEst) for optimization software - reference: https://link.springer.com/article/10.1007/s10589-014-9687-3 - other info: - partial evaluations: 'no' - full name: 'Constrained and Unconstrained Testing Environment with safe threads ' - constraint properties: Soft Constraints, Box Constraints - number of constraints: scalable - implementation languages: Python, C++, Fortran - general: 'Python implementation: https://github.com/jfowkes/pycutest' -- name: PUBOi - suite/generator/single: Generator - variable type: Boolean - dimensionality: scalable - objectives: '1' - constraints: 'no' - dynamic: 'no' - noise: 'no' - multimodal: Unknown - multi-fidelity: 'no' - source (real-world/artificial): Artificially Generated - implementation: https://gitlab.com/verel/pubo-importance-benchmark - textual description: A benchmark in which variable importance is tunable, based - on the Walsh function - reference: https://link.springer.com/chapter/10.1007/978-3-031-04148-8_12 - other info: - partial evaluations: 'no' - full name: Polynomial Unconstrained Binary Optimization - key challenges / characteristics: Tunable variable importance - implementation languages: Python, C++ + dynamic_type: null + fidelity_levels: null + implementations: + - impl_expobench + long_name: EXPensive Optimization benchmark library + modality: null + name: EXPObench + noise_type: + - real-life + - observational + objectives: + - 1 + problems: null + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1016/j.asoc.2023.110744 + title: EXPObench + source: + - real-world + tags: null + type: suite + variables: + - dim: + max: 135 + min: 10 + type: integer + - dim: + max: 135 + min: 10 + type: categorical + - dim: + max: 135 + min: 10 + type: continuous +suite_gbea: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: expensive evaluations 5s-35s, RW-GAN-Mario and TopTrumps are part + of GBEA + dynamic_type: null + fidelity_levels: null + implementations: + - impl_gbea + long_name: null + modality: + - multimodal + name: GBEA + noise_type: + - noisy + objectives: + - 1 + - 2 + problems: null + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1145/3321707.3321805 + title: Game benchmark for evolutionary algorithms + source: + - real-world + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: continuous +suite_gnbg: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: Generalized Numerical Benchmark Generator + dynamic_type: null + fidelity_levels: null + implementations: + - impl_gnbg + long_name: null + modality: null + name: GNBG + noise_type: null + objectives: + - 1 + problems: null + references: + - authors: [] + link: + type: null + url: https://arxiv.org/abs/2312.07083 + title: GNBG + source: + - artificial + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: continuous +suite_gnbg_ii: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: Generalized Numerical Benchmark Generator (version 2). Also + available in IOH. + dynamic_type: null + fidelity_levels: null + implementations: + - impl_iohgnbg + - impl_gnbg_ii + long_name: null + modality: null + name: GNBG-II + noise_type: null + objectives: + - 1 + problems: null + references: + - authors: [] + link: + type: null + url: https://dl.acm.org/doi/pdf/10.1145/3712255.3734271 + title: GNBG-II + source: + - artificial + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: continuous +suite_iohclustering: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: 'Set of benchmark problems from clustering: optimization task is selecting + cluster centers for a given set of data.' + dynamic_type: null + fidelity_levels: null + implementations: + - impl_iohclustering + long_name: null + modality: + - multimodal + name: IOHClustering + noise_type: null + objectives: + - 1 + problems: null + references: + - authors: [] + link: + type: null + url: https://arxiv.org/pdf/2505.09233 + title: IOHClustering + source: + - artificial-from-real-data + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: continuous +suite_kinematics_robotarm: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: null + dynamic_type: null + fidelity_levels: null + implementations: + - impl_transfer_rf_bbob_rw + long_name: null + modality: + - unimodal + name: KinematicsRobotArm + noise_type: null + objectives: + - 1 + problems: null + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1023/A:1013258808932 + title: Kinematics of a robot arm + source: + - real-world + tags: null + type: suite + variables: + - dim: 21 + type: continuous +suite_l1_zdt: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: Variant of ZDT with linkages between variables within groups + dynamic_type: null + fidelity_levels: null + implementations: null + long_name: null + modality: null + name: L1-ZDT + noise_type: null + objectives: + - 2 + problems: null + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1145/1143997.1144179 + title: Linkage ZDT/DTLZ variants + source: null + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: binary + - dim: + max: null + min: 1 + type: continuous +suite_l2_dtlz: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: Variant of DTLZ2/DTLZ3 with linkages between all variables + dynamic_type: null + fidelity_levels: null + implementations: null + long_name: null + modality: null + name: L2-DTLZ + noise_type: null + objectives: + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + problems: null + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1145/1143997.1144179 + title: Linkage ZDT/DTLZ variants + source: null + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: continuous +suite_l2_zdt: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: Variant of ZDT with linkages between all variables + dynamic_type: null + fidelity_levels: null + implementations: null + long_name: null + modality: null + name: L2-ZDT + noise_type: null + objectives: + - 2 + problems: null + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1145/1143997.1144179 + title: Linkage ZDT/DTLZ variants + source: null + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: binary + - dim: + max: null + min: 1 + type: continuous +suite_l3_dtlz: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: Variant of L2-DTLZ with anti-linkage mapping + dynamic_type: null + fidelity_levels: null + implementations: null + long_name: null + modality: null + name: L3-DTLZ + noise_type: null + objectives: + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + problems: null + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1145/1143997.1144179 + title: Linkage ZDT/DTLZ variants + source: null + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: continuous +suite_l3_zdt: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: Variant of L2-ZDT with anti-linkage mapping + dynamic_type: null + fidelity_levels: null + implementations: null + long_name: null + modality: null + name: L3-ZDT + noise_type: null + objectives: + - 2 + problems: null + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1145/1143997.1144179 + title: Linkage ZDT/DTLZ variants + source: null + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: binary + - dim: + max: null + min: 1 + type: continuous +suite_maop: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: null + dynamic_type: null + fidelity_levels: null + implementations: null + long_name: null + modality: null + name: MaOP + noise_type: + - unknown + objectives: + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + problems: null + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1016/j.swevo.2019.02.003 + title: MaOP benchmark + source: null + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: continuous +suite_mechbench: + allows_partial_evaluation: no + can_evaluate_objectives_independently: null + code_examples: null + constraints: + - equality: null + hard: yes + number: + - 1 + - 2 + type: unknown + description: Set of problems inspired by Structural Mechanics Design + Optimization. Embeds physical simulations (plasticity only, no + fracture/damage). Unstructured/non-isotropic multimodality. + dynamic_type: null + fidelity_levels: null + implementations: + - impl_mechbench + long_name: MECHBench + modality: + - multimodal + name: MECHBench + noise_type: null + objectives: + - 1 + problems: null + references: + - authors: [] + link: + type: null + url: https://arxiv.org/abs/2511.10821 + title: MECHBench + source: + - real-world + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: continuous +suite_mf2: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: null + dynamic_type: null + fidelity_levels: + - 1 + - 2 + implementations: + - impl_mf2 + long_name: null + modality: null + name: MF2 + noise_type: null + objectives: + - 1 + problems: null + references: + - authors: [] + link: + type: null + url: https://doi.org/10.21105/joss.02049 + title: 'mf2: a collection of multi-fidelity benchmark functions in Python' + source: null + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: continuous +suite_minus_dtlz: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: Variant of DTLZ that minimises the inverse of the base DTLZ + functions + dynamic_type: null + fidelity_levels: null + implementations: null + long_name: null + modality: null + name: Minus DTLZ + noise_type: null + objectives: + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + problems: null + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1109/TEVC.2016.2587749 + title: Minus DTLZ / Minus WFG + source: null + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: continuous +suite_minus_wfg: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: Variant of WFG that minimises the inverse of the base WFG + functions + dynamic_type: null + fidelity_levels: null + implementations: null + long_name: null + modality: null + name: Minus WFG + noise_type: null + objectives: + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + problems: null + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1109/TEVC.2016.2587749 + title: Minus DTLZ / Minus WFG + source: null + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: continuous +suite_mmopp: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: + - equality: null + hard: yes + number: null + type: unknown + description: null + dynamic_type: null + fidelity_levels: null + implementations: + - impl_mmopp + long_name: null + modality: + - multimodal + name: MMOPP + noise_type: null + objectives: + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + problems: null + references: + - authors: [] + link: + type: null + url: + http://www5.zzu.edu.cn/system/_content/download.jsp?urltype=news.DownloadAttachUrl&owner=1327567121&wbfileid=4764412 + title: MMOPP technical report + source: null + tags: null + type: suite + variables: + - dim: 0 + type: unknown +suite_modact: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: + - equality: null + hard: yes + number: null + type: unknown + description: Realistic Constrained Multi-Objective Optimization Benchmark + Problems from Design. + dynamic_type: null + fidelity_levels: null + implementations: + - impl_pymoo + - impl_modact + long_name: multiobjective design of actuators + modality: null + name: MODAct + noise_type: null + objectives: + - 2 + - 3 + - 4 + - 5 + problems: null + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1109/TEVC.2020.3020046 + title: MODAct + source: + - real-world + tags: null + type: suite + variables: + - dim: 20 + type: continuous + - dim: 20 + type: integer +suite_morepo: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: + - equality: null + hard: '?' + number: null + type: unknown + description: null + dynamic_type: + - unknown + fidelity_levels: null + implementations: + - impl_morepo + long_name: null + modality: null + name: MOrepo + noise_type: + - unknown + objectives: + - 2 + problems: null + references: null + source: null + tags: null + type: suite + variables: + - dim: 0 + type: unknown +suite_pbo: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: Suite of 25 binary optimization problems + dynamic_type: null + fidelity_levels: null + implementations: + - impl_iohexperimenter + long_name: null + modality: null + name: PBO + noise_type: null + objectives: + - 1 + problems: null + references: + - authors: [] + link: + type: null + url: https://dl.acm.org/doi/pdf/10.1145/3319619.3326810 + title: PBO benchmarks + source: + - artificial + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: binary +suite_porkchop: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: null + dynamic_type: null + fidelity_levels: null + implementations: + - impl_transfer_rf_bbob_rw + long_name: null + modality: + - multimodal + name: PorkchopPlotInterplanetaryTrajectory + noise_type: null + objectives: + - 1 + problems: null + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1109/CEC65147.2025.11042973 + title: Porkchop plot interplanetary trajectory benchmark + source: + - real-world + tags: null + type: suite + variables: + - dim: 2 + type: continuous +suite_re: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: null + dynamic_type: null + fidelity_levels: null + implementations: + - impl_reproblems + long_name: null + modality: null + name: RE + noise_type: null + objectives: + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + problems: null + references: + - authors: + - Ryoji Tanabe + - Hisao Ishibuchi + link: + type: null + url: https://doi.org/10.1016/j.asoc.2020.106078 + title: Easy-to-evaluate real-world multi-objective optimization problems + source: + - real-world-like + tags: null + type: suite + variables: + - dim: + max: 7 + min: 2 + type: integer + - dim: + max: 7 + min: 2 + type: continuous +suite_rwmvop: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: + - equality: null + hard: yes + number: null + type: unknown + description: null + dynamic_type: null + fidelity_levels: null + implementations: null + long_name: null + modality: null + name: RWMVOP + noise_type: null + objectives: + - 1 + problems: null + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1109/TEVC.2013.2281531 + title: RWMVOP + source: + - real-world + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: categorical + - dim: + max: null + min: 1 + type: integer + - dim: + max: null + min: 1 + type: continuous +suite_sbox_cost: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: problems from BBOB but allows instances with the optimum close to + the boundary + dynamic_type: null + fidelity_levels: null + implementations: + - impl_iohexperimenter + long_name: null + modality: + - multimodal + name: SBOX-COST + noise_type: null + objectives: + - 1 + problems: null + references: + - authors: [] + link: + type: null + url: https://doi.org/10.48550/arXiv.2305.12221 + title: SBOX-COST + source: null + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: continuous +suite_sdp: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: null + dynamic_type: + - dynamic + fidelity_levels: null + implementations: null + long_name: null + modality: null + name: SDP + noise_type: + - unknown + objectives: + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + problems: null + references: + - authors: [] + link: + type: null + url: https://doi.org/10.1109/TCYB.2019.2896021 + title: SDP dynamic multi-objective benchmark + source: null + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: continuous +suite_submodular: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: set of graph-based submodular optimization problems from 4 + problem types + dynamic_type: null + fidelity_levels: null + implementations: + - impl_iohexperimenter + long_name: null + modality: null + name: Submodular Optimization + noise_type: null + objectives: + - 1 + problems: null + references: + - authors: [] + link: + type: null + url: https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=10254181 + title: Submodular optimization benchmark + source: + - artificial + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: binary +suite_tulipa_energy: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: + - equality: null + hard: some + number: null + type: unknown + - equality: null + hard: yes + number: null + type: unknown + description: Determine the optimal investment and operation decisions for + different assets in the energy system (production, consumption, conversion, + storage, transport) while minimizing loss of load. Modelled as a potentially + very large linear program with multiple fidelity levels. + dynamic_type: null + fidelity_levels: + - 1 + - 2 + implementations: + - impl_tulipa + long_name: TulipaEnergyModel.jl + modality: + - unimodal + name: TulipaEnergy + noise_type: + - parameter + objectives: + - 1 + problems: null + references: + - authors: [] + link: + type: null + url: + https://tulipaenergy.github.io/TulipaEnergyModel.jl/stable/40-scientific-foundation/45-scientific-references + title: TulipaEnergyModel.jl scientific references + source: + - real-world + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: continuous +suite_vehicle_dynamics: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: null + dynamic_type: null + fidelity_levels: null + implementations: + - impl_vehicle_dynamics + long_name: null + modality: + - multimodal + name: VehicleDynamics + noise_type: null + objectives: + - 1 + problems: null + references: + - authors: [] + link: + type: null + url: https://www.scitepress.org/Papers/2023/121580/121580.pdf + title: VehicleDynamics benchmark + source: + - real-world + tags: null + type: suite + variables: + - dim: 2 + type: continuous +suite_wfg: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: null + dynamic_type: null + fidelity_levels: null + implementations: + - impl_pymoo + long_name: null + modality: null + name: WFG + noise_type: null + objectives: + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + problems: null + references: + - authors: + - Simon Huband + - Philip Hingston + - Luigi Barone + - Lyndon While + link: + type: null + url: https://doi.org/10.1109/TEVC.2005.861417 + title: A review of multiobjective test problems and a scalable test problem + toolkit + source: null + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: continuous +suite_zdt: + allows_partial_evaluation: null + can_evaluate_objectives_independently: null + code_examples: null + constraints: null + description: null + dynamic_type: null + fidelity_levels: null + implementations: + - impl_pymoo + long_name: null + modality: null + name: ZDT + noise_type: null + objectives: + - 2 + problems: null + references: + - authors: + - Eckart Zitzler + - Kalyanmoy Deb + - Lothar Thiele + link: + type: null + url: https://doi.org/10.1162/106365600568202 + title: 'Comparison of multiobjective evolutionary algorithms: empirical results' + source: null + tags: null + type: suite + variables: + - dim: + max: null + min: 1 + type: binary + - dim: + max: null + min: 1 + type: continuous + diff --git a/pyproject.toml b/pyproject.toml index 9be5ec9..79e56ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ "pydantic>=2.13.3", "pydantic-yaml>=1.6.0", "pyyaml>=6.0.3", + "pandas", ] [project.scripts] diff --git a/utils/README.md b/utils/README.md index 2ad9e5e..263edda 100644 --- a/utils/README.md +++ b/utils/README.md @@ -1,19 +1,18 @@ # OPL YAML utils -This folder contains utility scripts for working with the YAML format to describe problems in context of OPL. They are mainly intended to be run automatically via GitHub Actions to make collaboration easier. +This folder contains utility scripts for working with the YAML format to describe problems in context of OPL. Some of them are mainly intended to be run automatically via GitHub Actions to make collaboration easier, others are utility functions for maintainers. The intended way of adding a new problem to the repository is thus as follows: -* Create a file in 'utils/new_problem.yaml' based on the template (see below). +* Create a new yaml file based on the template (see below). +* Run the [merge script](merge_yaml.py) locally to update the [problems.yaml](../problems.yaml) file and check that the formatting is correct. * Create a PR with the changes (for example with a fork). What happens in the background then is: -* On PR creation and commits to the PR, the [validate_yaml.py](validate_yaml.py) script is run to check that the YAML file is valid and consistent. It is expecting the changes to be in the [new_problem.yaml](new_problem.yaml) file. +* On PR creation and commits to the PR, the [validate_yaml.py](validate_yaml.py) script is run to check that the [problems.yaml](../problems.yaml) file is still valid and consistent. * Then the PR should be reviewed manually. -* When the PR is merged into the main branch, a second script runs (which doesn't exist yet), that adds the content of [new_problem.yaml](new_problem.yaml) to the [problems.yaml](../problems.yaml) file, and reverts the changes to the new_problem.yaml. - -:warning: Note that the GitHubActions do not exist yet either, this is a WIP. +* When the PR is merged into the main branch with changes to problems.yaml, the checks are run again. ## validate_yaml.py @@ -21,13 +20,24 @@ This script checks the new content for the following: * The YAML syntax is valid and is in expected format * The required fields are present. -* Specific fields are unique across the new set of problems (e.g. name) +* Specific fields are unique across the set of problems (e.g. name) + +:warning: Execute from root of the repository. Tested with python 3.12 + +```bash +pip install -r utils/requirements.txt +python utils/validate_yaml.py problems.yaml +``` + +## merge_yaml.py + +This script merges a new problem description in a separate yaml file into the main [problems.yaml](../problems.yaml) file. It runs the validation checks from the above script before merging and deletes the separate yaml file after merging. :warning: Execute from root of the repository. Tested with python 3.12 ```bash pip install -r utils/requirements.txt -python utils/validate_yaml.py utils/new_problem.yaml +python utils/merge_yaml.py new_problem.yaml problems.yaml ``` ## new problem example diff --git a/utils/merge_yaml.py b/utils/merge_yaml.py new file mode 100644 index 0000000..6705197 --- /dev/null +++ b/utils/merge_yaml.py @@ -0,0 +1,96 @@ +import yaml +import sys +from pathlib import Path +from typing import List, Dict + +# Add parent directory to sys.path +parent = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(parent)) + +from utils.validate_yaml import read_data, validate_data, validate_yaml + + +def write_data(filepath: str, data: List[Dict]) -> bool: + try: + with open(filepath, "w") as f: + yaml.safe_dump(data, f, sort_keys=False) + print(f"::notice::Wrote data to {filepath}.") + except FileNotFoundError: + print(f"::error::File not found: {filepath}") + return False + except OSError as e: + print(f"::error::Error writing file {filepath}: {e}") + return False + except yaml.YAMLError as e: + print(f"::error::YAML syntax error: {e}") + return False + return True + + +def update_existing_data( + existing_data: List[Dict], new_data: List[Dict], out_file: str +) -> bool: + existing_data.extend(new_data) + # validate merged data before writing + valid = validate_data(existing_data) + if not valid: + print(f"::error::Merged data is not valid, cannot write to {out_file}.") + return False + write_success = write_data(out_file, existing_data) + return write_success + + +def merge_new_problems(new_problems_yaml_path: str, big_yaml_path: str) -> bool: + # Read and validate new data + new_data_status, new_data = read_data(new_problems_yaml_path) + if new_data_status != 0 or new_data is None: + print( + f"::error::New problems data could not be read from {new_problems_yaml_path}." + ) + return False + valid = validate_data(new_data) + if not valid: + print(f"::error::New problems data in {new_problems_yaml_path} is not valid.") + return False + + # Read existing data + existing_data_status, existing_data = read_data(big_yaml_path) + if existing_data_status != 0 or existing_data is None: + print( + f"::error::Existing problems data could not be read from {big_yaml_path}." + ) + return False + + # All valid, we can now just merge the dicts + assert existing_data is not None + assert new_data is not None + updated = update_existing_data(existing_data, new_data, big_yaml_path) + if not updated: + print(f"::error::Failed to update existing problems data in {big_yaml_path}.") + return False + + # Validate resulting data + final_status, final_data = validate_yaml(big_yaml_path) + if final_status != 0 or final_data is None: + print( + f"::error::Merged data in {big_yaml_path} is not valid after merging new problems." + ) + return False + + print( + f"::notice::Merged {len(new_data)} new problems into {big_yaml_path}. {new_problems_yaml_path} can now be deleted." + ) + return True + + +if __name__ == "__main__": + if len(sys.argv) != 3: + print("Usage: python merge_yaml.py ") + sys.exit(1) + new_problems_yaml_path = sys.argv[1] + big_yaml_path = sys.argv[2] + status = merge_new_problems(new_problems_yaml_path, big_yaml_path) + if not status: + sys.exit(1) + else: + sys.exit(0) diff --git a/utils/validate_yaml.py b/utils/validate_yaml.py index 34f1899..79f4238 100644 --- a/utils/validate_yaml.py +++ b/utils/validate_yaml.py @@ -2,6 +2,7 @@ import sys from pathlib import Path +from typing import List, Dict, Tuple # Add parent directory to sys.path parent = Path(__file__).resolve().parent.parent @@ -16,7 +17,7 @@ UNIQUE_WARNING_FIELDS = ["reference", "implementation"] -def read_data(filepath): +def read_data(filepath: str) -> Tuple[int, List[Dict] | None]: try: with open(filepath, "r") as f: data = yaml.safe_load(f) @@ -29,8 +30,11 @@ def read_data(filepath): return 1, None -def check_format(data): +def check_format(data: List[Dict]) -> bool: num_problems = len(data) + if not isinstance(data, list): + print("::error::YAML file should contain a list of entries.") + return False if len(data) < 1: print("::error::YAML file should contain at least one top level entry.") return False @@ -42,14 +46,16 @@ def check_format(data): return False unique_fields.append({k: v for k, v in entry.items() if k in UNIQUE_FIELDS}) for k in UNIQUE_FIELDS: - values = [entry[k] for entry in unique_fields] + values = [ + entry[k] for entry in unique_fields if k in entry and entry[k] is not None + ] if len(values) != len(set(values)): print(f"::error::Field '{k}' must be unique across all entries.") return False return True -def check_fields(data): +def check_fields(data: Dict) -> bool: missing = [field for field in REQUIRED_FIELDS if field not in data] if missing: print(f"::error::Missing required fields: {', '.join(missing)}") @@ -79,7 +85,7 @@ def check_fields(data): return True -def check_novelty(data, checked_data): +def check_novelty(data: Dict, checked_data: List[Dict]) -> bool: for field in UNIQUE_FIELDS + UNIQUE_WARNING_FIELDS: # skip empty fields if not data.get(field): @@ -101,13 +107,10 @@ def check_novelty(data, checked_data): return True -def validate_yaml(filepath): - status, data = read_data(filepath) - if status != 0: - sys.exit(1) - if not check_format(data): - sys.exit(1) +def validate_data(data: List[Dict]) -> bool: assert data is not None + if not check_format(data): + return False checked_data = [] @@ -115,12 +118,24 @@ def validate_yaml(filepath): # Check required and unique fields if not check_fields(new_data) or not check_novelty(new_data, checked_data): print(f"::error::Validation failed for entry {i+1}.") - sys.exit(1) + return False checked_data.append(new_data) # Add to checked data for novelty checks # YAML is valid if we reach this point print("YAML syntax is valid.") - sys.exit(0) + + return True + + +def validate_yaml(filepath: str) -> None: + status, data = read_data(filepath) + if status != 0 or data is None: + sys.exit(1) + valid = validate_data(data) + if not valid: + sys.exit(1) + else: + sys.exit(0) if __name__ == "__main__": diff --git a/uv.lock b/uv.lock index 9850ab7..95c222e 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,15 @@ version = 1 revision = 3 requires-python = ">=3.10" +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.11'", +] [[package]] name = "annotated-types" @@ -143,7 +152,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -180,11 +189,165 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "numpy" +version = "2.2.6" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11'", +] +sdist = { url = "https://files.pythonhosted.org/packages/76/21/7d2a95e4bba9dc13d043ee156a356c0a8f0c6309dff6b21b4d71a073b8a8/numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd", size = 20276440, upload-time = "2025-05-17T22:38:04.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/3e/ed6db5be21ce87955c0cbd3009f2803f59fa08df21b5df06862e2d8e2bdd/numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb", size = 21165245, upload-time = "2025-05-17T21:27:58.555Z" }, + { url = "https://files.pythonhosted.org/packages/22/c2/4b9221495b2a132cc9d2eb862e21d42a009f5a60e45fc44b00118c174bff/numpy-2.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90", size = 14360048, upload-time = "2025-05-17T21:28:21.406Z" }, + { url = "https://files.pythonhosted.org/packages/fd/77/dc2fcfc66943c6410e2bf598062f5959372735ffda175b39906d54f02349/numpy-2.2.6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:37e990a01ae6ec7fe7fa1c26c55ecb672dd98b19c3d0e1d1f326fa13cb38d163", size = 5340542, upload-time = "2025-05-17T21:28:30.931Z" }, + { url = "https://files.pythonhosted.org/packages/7a/4f/1cb5fdc353a5f5cc7feb692db9b8ec2c3d6405453f982435efc52561df58/numpy-2.2.6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:5a6429d4be8ca66d889b7cf70f536a397dc45ba6faeb5f8c5427935d9592e9cf", size = 6878301, upload-time = "2025-05-17T21:28:41.613Z" }, + { url = "https://files.pythonhosted.org/packages/eb/17/96a3acd228cec142fcb8723bd3cc39c2a474f7dcf0a5d16731980bcafa95/numpy-2.2.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efd28d4e9cd7d7a8d39074a4d44c63eda73401580c5c76acda2ce969e0a38e83", size = 14297320, upload-time = "2025-05-17T21:29:02.78Z" }, + { url = "https://files.pythonhosted.org/packages/b4/63/3de6a34ad7ad6646ac7d2f55ebc6ad439dbbf9c4370017c50cf403fb19b5/numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc7b73d02efb0e18c000e9ad8b83480dfcd5dfd11065997ed4c6747470ae8915", size = 16801050, upload-time = "2025-05-17T21:29:27.675Z" }, + { url = "https://files.pythonhosted.org/packages/07/b6/89d837eddef52b3d0cec5c6ba0456c1bf1b9ef6a6672fc2b7873c3ec4e2e/numpy-2.2.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:74d4531beb257d2c3f4b261bfb0fc09e0f9ebb8842d82a7b4209415896adc680", size = 15807034, upload-time = "2025-05-17T21:29:51.102Z" }, + { url = "https://files.pythonhosted.org/packages/01/c8/dc6ae86e3c61cfec1f178e5c9f7858584049b6093f843bca541f94120920/numpy-2.2.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8fc377d995680230e83241d8a96def29f204b5782f371c532579b4f20607a289", size = 18614185, upload-time = "2025-05-17T21:30:18.703Z" }, + { url = "https://files.pythonhosted.org/packages/5b/c5/0064b1b7e7c89137b471ccec1fd2282fceaae0ab3a9550f2568782d80357/numpy-2.2.6-cp310-cp310-win32.whl", hash = "sha256:b093dd74e50a8cba3e873868d9e93a85b78e0daf2e98c6797566ad8044e8363d", size = 6527149, upload-time = "2025-05-17T21:30:29.788Z" }, + { url = "https://files.pythonhosted.org/packages/a3/dd/4b822569d6b96c39d1215dbae0582fd99954dcbcf0c1a13c61783feaca3f/numpy-2.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:f0fd6321b839904e15c46e0d257fdd101dd7f530fe03fd6359c1ea63738703f3", size = 12904620, upload-time = "2025-05-17T21:30:48.994Z" }, + { url = "https://files.pythonhosted.org/packages/da/a8/4f83e2aa666a9fbf56d6118faaaf5f1974d456b1823fda0a176eff722839/numpy-2.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f9f1adb22318e121c5c69a09142811a201ef17ab257a1e66ca3025065b7f53ae", size = 21176963, upload-time = "2025-05-17T21:31:19.36Z" }, + { url = "https://files.pythonhosted.org/packages/b3/2b/64e1affc7972decb74c9e29e5649fac940514910960ba25cd9af4488b66c/numpy-2.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c820a93b0255bc360f53eca31a0e676fd1101f673dda8da93454a12e23fc5f7a", size = 14406743, upload-time = "2025-05-17T21:31:41.087Z" }, + { url = "https://files.pythonhosted.org/packages/4a/9f/0121e375000b5e50ffdd8b25bf78d8e1a5aa4cca3f185d41265198c7b834/numpy-2.2.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3d70692235e759f260c3d837193090014aebdf026dfd167834bcba43e30c2a42", size = 5352616, upload-time = "2025-05-17T21:31:50.072Z" }, + { url = "https://files.pythonhosted.org/packages/31/0d/b48c405c91693635fbe2dcd7bc84a33a602add5f63286e024d3b6741411c/numpy-2.2.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:481b49095335f8eed42e39e8041327c05b0f6f4780488f61286ed3c01368d491", size = 6889579, upload-time = "2025-05-17T21:32:01.712Z" }, + { url = "https://files.pythonhosted.org/packages/52/b8/7f0554d49b565d0171eab6e99001846882000883998e7b7d9f0d98b1f934/numpy-2.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b64d8d4d17135e00c8e346e0a738deb17e754230d7e0810ac5012750bbd85a5a", size = 14312005, upload-time = "2025-05-17T21:32:23.332Z" }, + { url = "https://files.pythonhosted.org/packages/b3/dd/2238b898e51bd6d389b7389ffb20d7f4c10066d80351187ec8e303a5a475/numpy-2.2.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba10f8411898fc418a521833e014a77d3ca01c15b0c6cdcce6a0d2897e6dbbdf", size = 16821570, upload-time = "2025-05-17T21:32:47.991Z" }, + { url = "https://files.pythonhosted.org/packages/83/6c/44d0325722cf644f191042bf47eedad61c1e6df2432ed65cbe28509d404e/numpy-2.2.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd48227a919f1bafbdda0583705e547892342c26fb127219d60a5c36882609d1", size = 15818548, upload-time = "2025-05-17T21:33:11.728Z" }, + { url = "https://files.pythonhosted.org/packages/ae/9d/81e8216030ce66be25279098789b665d49ff19eef08bfa8cb96d4957f422/numpy-2.2.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9551a499bf125c1d4f9e250377c1ee2eddd02e01eac6644c080162c0c51778ab", size = 18620521, upload-time = "2025-05-17T21:33:39.139Z" }, + { url = "https://files.pythonhosted.org/packages/6a/fd/e19617b9530b031db51b0926eed5345ce8ddc669bb3bc0044b23e275ebe8/numpy-2.2.6-cp311-cp311-win32.whl", hash = "sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47", size = 6525866, upload-time = "2025-05-17T21:33:50.273Z" }, + { url = "https://files.pythonhosted.org/packages/31/0a/f354fb7176b81747d870f7991dc763e157a934c717b67b58456bc63da3df/numpy-2.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:e8213002e427c69c45a52bbd94163084025f533a55a59d6f9c5b820774ef3303", size = 12907455, upload-time = "2025-05-17T21:34:09.135Z" }, + { url = "https://files.pythonhosted.org/packages/82/5d/c00588b6cf18e1da539b45d3598d3557084990dcc4331960c15ee776ee41/numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff", size = 20875348, upload-time = "2025-05-17T21:34:39.648Z" }, + { url = "https://files.pythonhosted.org/packages/66/ee/560deadcdde6c2f90200450d5938f63a34b37e27ebff162810f716f6a230/numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c", size = 14119362, upload-time = "2025-05-17T21:35:01.241Z" }, + { url = "https://files.pythonhosted.org/packages/3c/65/4baa99f1c53b30adf0acd9a5519078871ddde8d2339dc5a7fde80d9d87da/numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3", size = 5084103, upload-time = "2025-05-17T21:35:10.622Z" }, + { url = "https://files.pythonhosted.org/packages/cc/89/e5a34c071a0570cc40c9a54eb472d113eea6d002e9ae12bb3a8407fb912e/numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282", size = 6625382, upload-time = "2025-05-17T21:35:21.414Z" }, + { url = "https://files.pythonhosted.org/packages/f8/35/8c80729f1ff76b3921d5c9487c7ac3de9b2a103b1cd05e905b3090513510/numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87", size = 14018462, upload-time = "2025-05-17T21:35:42.174Z" }, + { url = "https://files.pythonhosted.org/packages/8c/3d/1e1db36cfd41f895d266b103df00ca5b3cbe965184df824dec5c08c6b803/numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249", size = 16527618, upload-time = "2025-05-17T21:36:06.711Z" }, + { url = "https://files.pythonhosted.org/packages/61/c6/03ed30992602c85aa3cd95b9070a514f8b3c33e31124694438d88809ae36/numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49", size = 15505511, upload-time = "2025-05-17T21:36:29.965Z" }, + { url = "https://files.pythonhosted.org/packages/b7/25/5761d832a81df431e260719ec45de696414266613c9ee268394dd5ad8236/numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de", size = 18313783, upload-time = "2025-05-17T21:36:56.883Z" }, + { url = "https://files.pythonhosted.org/packages/57/0a/72d5a3527c5ebffcd47bde9162c39fae1f90138c961e5296491ce778e682/numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4", size = 6246506, upload-time = "2025-05-17T21:37:07.368Z" }, + { url = "https://files.pythonhosted.org/packages/36/fa/8c9210162ca1b88529ab76b41ba02d433fd54fecaf6feb70ef9f124683f1/numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2", size = 12614190, upload-time = "2025-05-17T21:37:26.213Z" }, + { url = "https://files.pythonhosted.org/packages/f9/5c/6657823f4f594f72b5471f1db1ab12e26e890bb2e41897522d134d2a3e81/numpy-2.2.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0811bb762109d9708cca4d0b13c4f67146e3c3b7cf8d34018c722adb2d957c84", size = 20867828, upload-time = "2025-05-17T21:37:56.699Z" }, + { url = "https://files.pythonhosted.org/packages/dc/9e/14520dc3dadf3c803473bd07e9b2bd1b69bc583cb2497b47000fed2fa92f/numpy-2.2.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:287cc3162b6f01463ccd86be154f284d0893d2b3ed7292439ea97eafa8170e0b", size = 14143006, upload-time = "2025-05-17T21:38:18.291Z" }, + { url = "https://files.pythonhosted.org/packages/4f/06/7e96c57d90bebdce9918412087fc22ca9851cceaf5567a45c1f404480e9e/numpy-2.2.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f1372f041402e37e5e633e586f62aa53de2eac8d98cbfb822806ce4bbefcb74d", size = 5076765, upload-time = "2025-05-17T21:38:27.319Z" }, + { url = "https://files.pythonhosted.org/packages/73/ed/63d920c23b4289fdac96ddbdd6132e9427790977d5457cd132f18e76eae0/numpy-2.2.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:55a4d33fa519660d69614a9fad433be87e5252f4b03850642f88993f7b2ca566", size = 6617736, upload-time = "2025-05-17T21:38:38.141Z" }, + { url = "https://files.pythonhosted.org/packages/85/c5/e19c8f99d83fd377ec8c7e0cf627a8049746da54afc24ef0a0cb73d5dfb5/numpy-2.2.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f92729c95468a2f4f15e9bb94c432a9229d0d50de67304399627a943201baa2f", size = 14010719, upload-time = "2025-05-17T21:38:58.433Z" }, + { url = "https://files.pythonhosted.org/packages/19/49/4df9123aafa7b539317bf6d342cb6d227e49f7a35b99c287a6109b13dd93/numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bc23a79bfabc5d056d106f9befb8d50c31ced2fbc70eedb8155aec74a45798f", size = 16526072, upload-time = "2025-05-17T21:39:22.638Z" }, + { url = "https://files.pythonhosted.org/packages/b2/6c/04b5f47f4f32f7c2b0e7260442a8cbcf8168b0e1a41ff1495da42f42a14f/numpy-2.2.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3143e4451880bed956e706a3220b4e5cf6172ef05fcc397f6f36a550b1dd868", size = 15503213, upload-time = "2025-05-17T21:39:45.865Z" }, + { url = "https://files.pythonhosted.org/packages/17/0a/5cd92e352c1307640d5b6fec1b2ffb06cd0dabe7d7b8227f97933d378422/numpy-2.2.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4f13750ce79751586ae2eb824ba7e1e8dba64784086c98cdbbcc6a42112ce0d", size = 18316632, upload-time = "2025-05-17T21:40:13.331Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3b/5cba2b1d88760ef86596ad0f3d484b1cbff7c115ae2429678465057c5155/numpy-2.2.6-cp313-cp313-win32.whl", hash = "sha256:5beb72339d9d4fa36522fc63802f469b13cdbe4fdab4a288f0c441b74272ebfd", size = 6244532, upload-time = "2025-05-17T21:43:46.099Z" }, + { url = "https://files.pythonhosted.org/packages/cb/3b/d58c12eafcb298d4e6d0d40216866ab15f59e55d148a5658bb3132311fcf/numpy-2.2.6-cp313-cp313-win_amd64.whl", hash = "sha256:b0544343a702fa80c95ad5d3d608ea3599dd54d4632df855e4c8d24eb6ecfa1c", size = 12610885, upload-time = "2025-05-17T21:44:05.145Z" }, + { url = "https://files.pythonhosted.org/packages/6b/9e/4bf918b818e516322db999ac25d00c75788ddfd2d2ade4fa66f1f38097e1/numpy-2.2.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0bca768cd85ae743b2affdc762d617eddf3bcf8724435498a1e80132d04879e6", size = 20963467, upload-time = "2025-05-17T21:40:44Z" }, + { url = "https://files.pythonhosted.org/packages/61/66/d2de6b291507517ff2e438e13ff7b1e2cdbdb7cb40b3ed475377aece69f9/numpy-2.2.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fc0c5673685c508a142ca65209b4e79ed6740a4ed6b2267dbba90f34b0b3cfda", size = 14225144, upload-time = "2025-05-17T21:41:05.695Z" }, + { url = "https://files.pythonhosted.org/packages/e4/25/480387655407ead912e28ba3a820bc69af9adf13bcbe40b299d454ec011f/numpy-2.2.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:5bd4fc3ac8926b3819797a7c0e2631eb889b4118a9898c84f585a54d475b7e40", size = 5200217, upload-time = "2025-05-17T21:41:15.903Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4a/6e313b5108f53dcbf3aca0c0f3e9c92f4c10ce57a0a721851f9785872895/numpy-2.2.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:fee4236c876c4e8369388054d02d0e9bb84821feb1a64dd59e137e6511a551f8", size = 6712014, upload-time = "2025-05-17T21:41:27.321Z" }, + { url = "https://files.pythonhosted.org/packages/b7/30/172c2d5c4be71fdf476e9de553443cf8e25feddbe185e0bd88b096915bcc/numpy-2.2.6-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1dda9c7e08dc141e0247a5b8f49cf05984955246a327d4c48bda16821947b2f", size = 14077935, upload-time = "2025-05-17T21:41:49.738Z" }, + { url = "https://files.pythonhosted.org/packages/12/fb/9e743f8d4e4d3c710902cf87af3512082ae3d43b945d5d16563f26ec251d/numpy-2.2.6-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f447e6acb680fd307f40d3da4852208af94afdfab89cf850986c3ca00562f4fa", size = 16600122, upload-time = "2025-05-17T21:42:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/12/75/ee20da0e58d3a66f204f38916757e01e33a9737d0b22373b3eb5a27358f9/numpy-2.2.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:389d771b1623ec92636b0786bc4ae56abafad4a4c513d36a55dce14bd9ce8571", size = 15586143, upload-time = "2025-05-17T21:42:37.464Z" }, + { url = "https://files.pythonhosted.org/packages/76/95/bef5b37f29fc5e739947e9ce5179ad402875633308504a52d188302319c8/numpy-2.2.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8e9ace4a37db23421249ed236fdcdd457d671e25146786dfc96835cd951aa7c1", size = 18385260, upload-time = "2025-05-17T21:43:05.189Z" }, + { url = "https://files.pythonhosted.org/packages/09/04/f2f83279d287407cf36a7a8053a5abe7be3622a4363337338f2585e4afda/numpy-2.2.6-cp313-cp313t-win32.whl", hash = "sha256:038613e9fb8c72b0a41f025a7e4c3f0b7a1b5d768ece4796b674c8f3fe13efff", size = 6377225, upload-time = "2025-05-17T21:43:16.254Z" }, + { url = "https://files.pythonhosted.org/packages/67/0e/35082d13c09c02c011cf21570543d202ad929d961c02a147493cb0c2bdf5/numpy-2.2.6-cp313-cp313t-win_amd64.whl", hash = "sha256:6031dd6dfecc0cf9f668681a37648373bddd6421fff6c66ec1624eed0180ee06", size = 12771374, upload-time = "2025-05-17T21:43:35.479Z" }, + { url = "https://files.pythonhosted.org/packages/9e/3b/d94a75f4dbf1ef5d321523ecac21ef23a3cd2ac8b78ae2aac40873590229/numpy-2.2.6-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0b605b275d7bd0c640cad4e5d30fa701a8d59302e127e5f79138ad62762c3e3d", size = 21040391, upload-time = "2025-05-17T21:44:35.948Z" }, + { url = "https://files.pythonhosted.org/packages/17/f4/09b2fa1b58f0fb4f7c7963a1649c64c4d315752240377ed74d9cd878f7b5/numpy-2.2.6-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:7befc596a7dc9da8a337f79802ee8adb30a552a94f792b9c9d18c840055907db", size = 6786754, upload-time = "2025-05-17T21:44:47.446Z" }, + { url = "https://files.pythonhosted.org/packages/af/30/feba75f143bdc868a1cc3f44ccfa6c4b9ec522b36458e738cd00f67b573f/numpy-2.2.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce47521a4754c8f4593837384bd3424880629f718d87c5d44f8ed763edd63543", size = 16643476, upload-time = "2025-05-17T21:45:11.871Z" }, + { url = "https://files.pythonhosted.org/packages/37/48/ac2a9584402fb6c0cd5b5d1a91dcf176b15760130dd386bbafdbfe3640bf/numpy-2.2.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d042d24c90c41b54fd506da306759e06e568864df8ec17ccc17e9e884634fd00", size = 12812666, upload-time = "2025-05-17T21:45:31.426Z" }, +] + +[[package]] +name = "numpy" +version = "2.4.4" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +sdist = { url = "https://files.pythonhosted.org/packages/d7/9f/b8cef5bffa569759033adda9481211426f12f53299629b410340795c2514/numpy-2.4.4.tar.gz", hash = "sha256:2d390634c5182175533585cc89f3608a4682ccb173cc9bb940b2881c8d6f8fa0", size = 20731587, upload-time = "2026-03-29T13:22:01.298Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/c6/4218570d8c8ecc9704b5157a3348e486e84ef4be0ed3e38218ab473c83d2/numpy-2.4.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f983334aea213c99992053ede6168500e5f086ce74fbc4acc3f2b00f5762e9db", size = 16976799, upload-time = "2026-03-29T13:18:15.438Z" }, + { url = "https://files.pythonhosted.org/packages/dd/92/b4d922c4a5f5dab9ed44e6153908a5c665b71acf183a83b93b690996e39b/numpy-2.4.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:72944b19f2324114e9dc86a159787333b77874143efcf89a5167ef83cfee8af0", size = 14971552, upload-time = "2026-03-29T13:18:18.606Z" }, + { url = "https://files.pythonhosted.org/packages/8a/dc/df98c095978fa6ee7b9a9387d1d58cbb3d232d0e69ad169a4ce784bde4fd/numpy-2.4.4-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:86b6f55f5a352b48d7fbfd2dbc3d5b780b2d79f4d3c121f33eb6efb22e9a2015", size = 5476566, upload-time = "2026-03-29T13:18:21.532Z" }, + { url = "https://files.pythonhosted.org/packages/28/34/b3fdcec6e725409223dd27356bdf5a3c2cc2282e428218ecc9cb7acc9763/numpy-2.4.4-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:ba1f4fc670ed79f876f70082eff4f9583c15fb9a4b89d6188412de4d18ae2f40", size = 6806482, upload-time = "2026-03-29T13:18:23.634Z" }, + { url = "https://files.pythonhosted.org/packages/68/62/63417c13aa35d57bee1337c67446761dc25ea6543130cf868eace6e8157b/numpy-2.4.4-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a87ec22c87be071b6bdbd27920b129b94f2fc964358ce38f3822635a3e2e03d", size = 15973376, upload-time = "2026-03-29T13:18:26.677Z" }, + { url = "https://files.pythonhosted.org/packages/cf/c5/9fcb7e0e69cef59cf10c746b84f7d58b08bc66a6b7d459783c5a4f6101a6/numpy-2.4.4-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df3775294accfdd75f32c74ae39fcba920c9a378a2fc18a12b6820aa8c1fb502", size = 16925137, upload-time = "2026-03-29T13:18:30.14Z" }, + { url = "https://files.pythonhosted.org/packages/7e/43/80020edacb3f84b9efdd1591120a4296462c23fd8db0dde1666f6ef66f13/numpy-2.4.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0d4e437e295f18ec29bc79daf55e8a47a9113df44d66f702f02a293d93a2d6dd", size = 17329414, upload-time = "2026-03-29T13:18:33.733Z" }, + { url = "https://files.pythonhosted.org/packages/fd/06/af0658593b18a5f73532d377188b964f239eb0894e664a6c12f484472f97/numpy-2.4.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6aa3236c78803afbcb255045fbef97a9e25a1f6c9888357d205ddc42f4d6eba5", size = 18658397, upload-time = "2026-03-29T13:18:37.511Z" }, + { url = "https://files.pythonhosted.org/packages/e6/ce/13a09ed65f5d0ce5c7dd0669250374c6e379910f97af2c08c57b0608eee4/numpy-2.4.4-cp311-cp311-win32.whl", hash = "sha256:30caa73029a225b2d40d9fae193e008e24b2026b7ee1a867b7ee8d96ca1a448e", size = 6239499, upload-time = "2026-03-29T13:18:40.372Z" }, + { url = "https://files.pythonhosted.org/packages/bd/63/05d193dbb4b5eec1eca73822d80da98b511f8328ad4ae3ca4caf0f4db91d/numpy-2.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:6bbe4eb67390b0a0265a2c25458f6b90a409d5d069f1041e6aff1e27e3d9a79e", size = 12614257, upload-time = "2026-03-29T13:18:42.95Z" }, + { url = "https://files.pythonhosted.org/packages/87/c5/8168052f080c26fa984c413305012be54741c9d0d74abd7fbeeccae3889f/numpy-2.4.4-cp311-cp311-win_arm64.whl", hash = "sha256:fcfe2045fd2e8f3cb0ce9d4ba6dba6333b8fa05bb8a4939c908cd43322d14c7e", size = 10486775, upload-time = "2026-03-29T13:18:45.835Z" }, + { url = "https://files.pythonhosted.org/packages/28/05/32396bec30fb2263770ee910142f49c1476d08e8ad41abf8403806b520ce/numpy-2.4.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15716cfef24d3a9762e3acdf87e27f58dc823d1348f765bbea6bef8c639bfa1b", size = 16689272, upload-time = "2026-03-29T13:18:49.223Z" }, + { url = "https://files.pythonhosted.org/packages/c5/f3/a983d28637bfcd763a9c7aafdb6d5c0ebf3d487d1e1459ffdb57e2f01117/numpy-2.4.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:23cbfd4c17357c81021f21540da84ee282b9c8fba38a03b7b9d09ba6b951421e", size = 14699573, upload-time = "2026-03-29T13:18:52.629Z" }, + { url = "https://files.pythonhosted.org/packages/9b/fd/e5ecca1e78c05106d98028114f5c00d3eddb41207686b2b7de3e477b0e22/numpy-2.4.4-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:8b3b60bb7cba2c8c81837661c488637eee696f59a877788a396d33150c35d842", size = 5204782, upload-time = "2026-03-29T13:18:55.579Z" }, + { url = "https://files.pythonhosted.org/packages/de/2f/702a4594413c1a8632092beae8aba00f1d67947389369b3777aed783fdca/numpy-2.4.4-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:e4a010c27ff6f210ff4c6ef34394cd61470d01014439b192ec22552ee867f2a8", size = 6552038, upload-time = "2026-03-29T13:18:57.769Z" }, + { url = "https://files.pythonhosted.org/packages/7f/37/eed308a8f56cba4d1fdf467a4fc67ef4ff4bf1c888f5fc980481890104b1/numpy-2.4.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f9e75681b59ddaa5e659898085ae0eaea229d054f2ac0c7e563a62205a700121", size = 15670666, upload-time = "2026-03-29T13:19:00.341Z" }, + { url = "https://files.pythonhosted.org/packages/0a/0d/0e3ecece05b7a7e87ab9fb587855548da437a061326fff64a223b6dcb78a/numpy-2.4.4-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:81f4a14bee47aec54f883e0cad2d73986640c1590eb9bfaaba7ad17394481e6e", size = 16645480, upload-time = "2026-03-29T13:19:03.63Z" }, + { url = "https://files.pythonhosted.org/packages/34/49/f2312c154b82a286758ee2f1743336d50651f8b5195db18cdb63675ff649/numpy-2.4.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:62d6b0f03b694173f9fcb1fb317f7222fd0b0b103e784c6549f5e53a27718c44", size = 17020036, upload-time = "2026-03-29T13:19:07.428Z" }, + { url = "https://files.pythonhosted.org/packages/7b/e9/736d17bd77f1b0ec4f9901aaec129c00d59f5d84d5e79bba540ef12c2330/numpy-2.4.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fbc356aae7adf9e6336d336b9c8111d390a05df88f1805573ebb0807bd06fd1d", size = 18368643, upload-time = "2026-03-29T13:19:10.775Z" }, + { url = "https://files.pythonhosted.org/packages/63/f6/d417977c5f519b17c8a5c3bc9e8304b0908b0e21136fe43bf628a1343914/numpy-2.4.4-cp312-cp312-win32.whl", hash = "sha256:0d35aea54ad1d420c812bfa0385c71cd7cc5bcf7c65fed95fc2cd02fe8c79827", size = 5961117, upload-time = "2026-03-29T13:19:13.464Z" }, + { url = "https://files.pythonhosted.org/packages/2d/5b/e1deebf88ff431b01b7406ca3583ab2bbb90972bbe1c568732e49c844f7e/numpy-2.4.4-cp312-cp312-win_amd64.whl", hash = "sha256:b5f0362dc928a6ecd9db58868fca5e48485205e3855957bdedea308f8672ea4a", size = 12320584, upload-time = "2026-03-29T13:19:16.155Z" }, + { url = "https://files.pythonhosted.org/packages/58/89/e4e856ac82a68c3ed64486a544977d0e7bdd18b8da75b78a577ca31c4395/numpy-2.4.4-cp312-cp312-win_arm64.whl", hash = "sha256:846300f379b5b12cc769334464656bc882e0735d27d9726568bc932fdc49d5ec", size = 10221450, upload-time = "2026-03-29T13:19:18.994Z" }, + { url = "https://files.pythonhosted.org/packages/14/1d/d0a583ce4fefcc3308806a749a536c201ed6b5ad6e1322e227ee4848979d/numpy-2.4.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:08f2e31ed5e6f04b118e49821397f12767934cfdd12a1ce86a058f91e004ee50", size = 16684933, upload-time = "2026-03-29T13:19:22.47Z" }, + { url = "https://files.pythonhosted.org/packages/c1/62/2b7a48fbb745d344742c0277f01286dead15f3f68e4f359fbfcf7b48f70f/numpy-2.4.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e823b8b6edc81e747526f70f71a9c0a07ac4e7ad13020aa736bb7c9d67196115", size = 14694532, upload-time = "2026-03-29T13:19:25.581Z" }, + { url = "https://files.pythonhosted.org/packages/e5/87/499737bfba066b4a3bebff24a8f1c5b2dee410b209bc6668c9be692580f0/numpy-2.4.4-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:4a19d9dba1a76618dd86b164d608566f393f8ec6ac7c44f0cc879011c45e65af", size = 5199661, upload-time = "2026-03-29T13:19:28.31Z" }, + { url = "https://files.pythonhosted.org/packages/cd/da/464d551604320d1491bc345efed99b4b7034143a85787aab78d5691d5a0e/numpy-2.4.4-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:d2a8490669bfe99a233298348acc2d824d496dee0e66e31b66a6022c2ad74a5c", size = 6547539, upload-time = "2026-03-29T13:19:30.97Z" }, + { url = "https://files.pythonhosted.org/packages/7d/90/8d23e3b0dafd024bf31bdec225b3bb5c2dbfa6912f8a53b8659f21216cbf/numpy-2.4.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:45dbed2ab436a9e826e302fcdcbe9133f9b0006e5af7168afb8963a6520da103", size = 15668806, upload-time = "2026-03-29T13:19:33.887Z" }, + { url = "https://files.pythonhosted.org/packages/d1/73/a9d864e42a01896bb5974475438f16086be9ba1f0d19d0bb7a07427c4a8b/numpy-2.4.4-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c901b15172510173f5cb310eae652908340f8dede90fff9e3bf6c0d8dfd92f83", size = 16632682, upload-time = "2026-03-29T13:19:37.336Z" }, + { url = "https://files.pythonhosted.org/packages/34/fb/14570d65c3bde4e202a031210475ae9cde9b7686a2e7dc97ee67d2833b35/numpy-2.4.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:99d838547ace2c4aace6c4f76e879ddfe02bb58a80c1549928477862b7a6d6ed", size = 17019810, upload-time = "2026-03-29T13:19:40.963Z" }, + { url = "https://files.pythonhosted.org/packages/8a/77/2ba9d87081fd41f6d640c83f26fb7351e536b7ce6dd9061b6af5904e8e46/numpy-2.4.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0aec54fd785890ecca25a6003fd9a5aed47ad607bbac5cd64f836ad8666f4959", size = 18357394, upload-time = "2026-03-29T13:19:44.859Z" }, + { url = "https://files.pythonhosted.org/packages/a2/23/52666c9a41708b0853fa3b1a12c90da38c507a3074883823126d4e9d5b30/numpy-2.4.4-cp313-cp313-win32.whl", hash = "sha256:07077278157d02f65c43b1b26a3886bce886f95d20aabd11f87932750dfb14ed", size = 5959556, upload-time = "2026-03-29T13:19:47.661Z" }, + { url = "https://files.pythonhosted.org/packages/57/fb/48649b4971cde70d817cf97a2a2fdc0b4d8308569f1dd2f2611959d2e0cf/numpy-2.4.4-cp313-cp313-win_amd64.whl", hash = "sha256:5c70f1cc1c4efbe316a572e2d8b9b9cc44e89b95f79ca3331553fbb63716e2bf", size = 12317311, upload-time = "2026-03-29T13:19:50.67Z" }, + { url = "https://files.pythonhosted.org/packages/ba/d8/11490cddd564eb4de97b4579ef6bfe6a736cc07e94c1598590ae25415e01/numpy-2.4.4-cp313-cp313-win_arm64.whl", hash = "sha256:ef4059d6e5152fa1a39f888e344c73fdc926e1b2dd58c771d67b0acfbf2aa67d", size = 10222060, upload-time = "2026-03-29T13:19:54.229Z" }, + { url = "https://files.pythonhosted.org/packages/99/5d/dab4339177a905aad3e2221c915b35202f1ec30d750dd2e5e9d9a72b804b/numpy-2.4.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4bbc7f303d125971f60ec0aaad5e12c62d0d2c925f0ab1273debd0e4ba37aba5", size = 14822302, upload-time = "2026-03-29T13:19:57.585Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e4/0564a65e7d3d97562ed6f9b0fd0fb0a6f559ee444092f105938b50043876/numpy-2.4.4-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:4d6d57903571f86180eb98f8f0c839fa9ebbfb031356d87f1361be91e433f5b7", size = 5327407, upload-time = "2026-03-29T13:20:00.601Z" }, + { url = "https://files.pythonhosted.org/packages/29/8d/35a3a6ce5ad371afa58b4700f1c820f8f279948cca32524e0a695b0ded83/numpy-2.4.4-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:4636de7fd195197b7535f231b5de9e4b36d2c440b6e566d2e4e4746e6af0ca93", size = 6647631, upload-time = "2026-03-29T13:20:02.855Z" }, + { url = "https://files.pythonhosted.org/packages/f4/da/477731acbd5a58a946c736edfdabb2ac5b34c3d08d1ba1a7b437fa0884df/numpy-2.4.4-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ad2e2ef14e0b04e544ea2fa0a36463f847f113d314aa02e5b402fdf910ef309e", size = 15727691, upload-time = "2026-03-29T13:20:06.004Z" }, + { url = "https://files.pythonhosted.org/packages/e6/db/338535d9b152beabeb511579598418ba0212ce77cf9718edd70262cc4370/numpy-2.4.4-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a285b3b96f951841799528cd1f4f01cd70e7e0204b4abebac9463eecfcf2a40", size = 16681241, upload-time = "2026-03-29T13:20:09.417Z" }, + { url = "https://files.pythonhosted.org/packages/e2/a9/ad248e8f58beb7a0219b413c9c7d8151c5d285f7f946c3e26695bdbbe2df/numpy-2.4.4-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:f8474c4241bc18b750be2abea9d7a9ec84f46ef861dbacf86a4f6e043401f79e", size = 17085767, upload-time = "2026-03-29T13:20:13.126Z" }, + { url = "https://files.pythonhosted.org/packages/b5/1a/3b88ccd3694681356f70da841630e4725a7264d6a885c8d442a697e1146b/numpy-2.4.4-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4e874c976154687c1f71715b034739b45c7711bec81db01914770373d125e392", size = 18403169, upload-time = "2026-03-29T13:20:17.096Z" }, + { url = "https://files.pythonhosted.org/packages/c2/c9/fcfd5d0639222c6eac7f304829b04892ef51c96a75d479214d77e3ce6e33/numpy-2.4.4-cp313-cp313t-win32.whl", hash = "sha256:9c585a1790d5436a5374bac930dad6ed244c046ed91b2b2a3634eb2971d21008", size = 6083477, upload-time = "2026-03-29T13:20:20.195Z" }, + { url = "https://files.pythonhosted.org/packages/d5/e3/3938a61d1c538aaec8ed6fd6323f57b0c2d2d2219512434c5c878db76553/numpy-2.4.4-cp313-cp313t-win_amd64.whl", hash = "sha256:93e15038125dc1e5345d9b5b68aa7f996ec33b98118d18c6ca0d0b7d6198b7e8", size = 12457487, upload-time = "2026-03-29T13:20:22.946Z" }, + { url = "https://files.pythonhosted.org/packages/97/6a/7e345032cc60501721ef94e0e30b60f6b0bd601f9174ebd36389a2b86d40/numpy-2.4.4-cp313-cp313t-win_arm64.whl", hash = "sha256:0dfd3f9d3adbe2920b68b5cd3d51444e13a10792ec7154cd0a2f6e74d4ab3233", size = 10292002, upload-time = "2026-03-29T13:20:25.909Z" }, + { url = "https://files.pythonhosted.org/packages/6e/06/c54062f85f673dd5c04cbe2f14c3acb8c8b95e3384869bb8cc9bff8cb9df/numpy-2.4.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:f169b9a863d34f5d11b8698ead99febeaa17a13ca044961aa8e2662a6c7766a0", size = 16684353, upload-time = "2026-03-29T13:20:29.504Z" }, + { url = "https://files.pythonhosted.org/packages/4c/39/8a320264a84404c74cc7e79715de85d6130fa07a0898f67fb5cd5bd79908/numpy-2.4.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:2483e4584a1cb3092da4470b38866634bafb223cbcd551ee047633fd2584599a", size = 14704914, upload-time = "2026-03-29T13:20:33.547Z" }, + { url = "https://files.pythonhosted.org/packages/91/fb/287076b2614e1d1044235f50f03748f31fa287e3dbe6abeb35cdfa351eca/numpy-2.4.4-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:2d19e6e2095506d1736b7d80595e0f252d76b89f5e715c35e06e937679ea7d7a", size = 5210005, upload-time = "2026-03-29T13:20:36.45Z" }, + { url = "https://files.pythonhosted.org/packages/63/eb/fcc338595309910de6ecabfcef2419a9ce24399680bfb149421fa2df1280/numpy-2.4.4-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:6a246d5914aa1c820c9443ddcee9c02bec3e203b0c080349533fae17727dfd1b", size = 6544974, upload-time = "2026-03-29T13:20:39.014Z" }, + { url = "https://files.pythonhosted.org/packages/44/5d/e7e9044032a716cdfaa3fba27a8e874bf1c5f1912a1ddd4ed071bf8a14a6/numpy-2.4.4-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:989824e9faf85f96ec9c7761cd8d29c531ad857bfa1daa930cba85baaecf1a9a", size = 15684591, upload-time = "2026-03-29T13:20:42.146Z" }, + { url = "https://files.pythonhosted.org/packages/98/7c/21252050676612625449b4807d6b695b9ce8a7c9e1c197ee6216c8a65c7c/numpy-2.4.4-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:27a8d92cd10f1382a67d7cf4db7ce18341b66438bdd9f691d7b0e48d104c2a9d", size = 16637700, upload-time = "2026-03-29T13:20:46.204Z" }, + { url = "https://files.pythonhosted.org/packages/b1/29/56d2bbef9465db24ef25393383d761a1af4f446a1df9b8cded4fe3a5a5d7/numpy-2.4.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e44319a2953c738205bf3354537979eaa3998ed673395b964c1176083dd46252", size = 17035781, upload-time = "2026-03-29T13:20:50.242Z" }, + { url = "https://files.pythonhosted.org/packages/e3/2b/a35a6d7589d21f44cea7d0a98de5ddcbb3d421b2622a5c96b1edf18707c3/numpy-2.4.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e892aff75639bbef0d2a2cfd55535510df26ff92f63c92cd84ef8d4ba5a5557f", size = 18362959, upload-time = "2026-03-29T13:20:54.019Z" }, + { url = "https://files.pythonhosted.org/packages/64/c9/d52ec581f2390e0f5f85cbfd80fb83d965fc15e9f0e1aec2195faa142cde/numpy-2.4.4-cp314-cp314-win32.whl", hash = "sha256:1378871da56ca8943c2ba674530924bb8ca40cd228358a3b5f302ad60cf875fc", size = 6008768, upload-time = "2026-03-29T13:20:56.912Z" }, + { url = "https://files.pythonhosted.org/packages/fa/22/4cc31a62a6c7b74a8730e31a4274c5dc80e005751e277a2ce38e675e4923/numpy-2.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:715d1c092715954784bc79e1174fc2a90093dc4dc84ea15eb14dad8abdcdeb74", size = 12449181, upload-time = "2026-03-29T13:20:59.548Z" }, + { url = "https://files.pythonhosted.org/packages/70/2e/14cda6f4d8e396c612d1bf97f22958e92148801d7e4f110cabebdc0eef4b/numpy-2.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:2c194dd721e54ecad9ad387c1d35e63dce5c4450c6dc7dd5611283dda239aabb", size = 10496035, upload-time = "2026-03-29T13:21:02.524Z" }, + { url = "https://files.pythonhosted.org/packages/b1/e8/8fed8c8d848d7ecea092dc3469643f9d10bc3a134a815a3b033da1d2039b/numpy-2.4.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2aa0613a5177c264ff5921051a5719d20095ea586ca88cc802c5c218d1c67d3e", size = 14824958, upload-time = "2026-03-29T13:21:05.671Z" }, + { url = "https://files.pythonhosted.org/packages/05/1a/d8007a5138c179c2bf33ef44503e83d70434d2642877ee8fbb230e7c0548/numpy-2.4.4-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:42c16925aa5a02362f986765f9ebabf20de75cdefdca827d14315c568dcab113", size = 5330020, upload-time = "2026-03-29T13:21:08.635Z" }, + { url = "https://files.pythonhosted.org/packages/99/64/ffb99ac6ae93faf117bcbd5c7ba48a7f45364a33e8e458545d3633615dda/numpy-2.4.4-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:874f200b2a981c647340f841730fc3a2b54c9d940566a3c4149099591e2c4c3d", size = 6650758, upload-time = "2026-03-29T13:21:10.949Z" }, + { url = "https://files.pythonhosted.org/packages/6e/6e/795cc078b78a384052e73b2f6281ff7a700e9bf53bcce2ee579d4f6dd879/numpy-2.4.4-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c9b39d38a9bd2ae1becd7eac1303d031c5c110ad31f2b319c6e7d98b135c934d", size = 15729948, upload-time = "2026-03-29T13:21:14.047Z" }, + { url = "https://files.pythonhosted.org/packages/5f/86/2acbda8cc2af5f3d7bfc791192863b9e3e19674da7b5e533fded124d1299/numpy-2.4.4-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b268594bccac7d7cf5844c7732e3f20c50921d94e36d7ec9b79e9857694b1b2f", size = 16679325, upload-time = "2026-03-29T13:21:17.561Z" }, + { url = "https://files.pythonhosted.org/packages/bc/59/cafd83018f4aa55e0ac6fa92aa066c0a1877b77a615ceff1711c260ffae8/numpy-2.4.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ac6b31e35612a26483e20750126d30d0941f949426974cace8e6b5c58a3657b0", size = 17084883, upload-time = "2026-03-29T13:21:21.106Z" }, + { url = "https://files.pythonhosted.org/packages/f0/85/a42548db84e65ece46ab2caea3d3f78b416a47af387fcbb47ec28e660dc2/numpy-2.4.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8e3ed142f2728df44263aaf5fb1f5b0b99f4070c553a0d7f033be65338329150", size = 18403474, upload-time = "2026-03-29T13:21:24.828Z" }, + { url = "https://files.pythonhosted.org/packages/ed/ad/483d9e262f4b831000062e5d8a45e342166ec8aaa1195264982bca267e62/numpy-2.4.4-cp314-cp314t-win32.whl", hash = "sha256:dddbbd259598d7240b18c9d87c56a9d2fb3b02fe266f49a7c101532e78c1d871", size = 6155500, upload-time = "2026-03-29T13:21:28.205Z" }, + { url = "https://files.pythonhosted.org/packages/c7/03/2fc4e14c7bd4ff2964b74ba90ecb8552540b6315f201df70f137faa5c589/numpy-2.4.4-cp314-cp314t-win_amd64.whl", hash = "sha256:a7164afb23be6e37ad90b2f10426149fd75aee07ca55653d2aa41e66c4ef697e", size = 12637755, upload-time = "2026-03-29T13:21:31.107Z" }, + { url = "https://files.pythonhosted.org/packages/58/78/548fb8e07b1a341746bfbecb32f2c268470f45fa028aacdbd10d9bc73aab/numpy-2.4.4-cp314-cp314t-win_arm64.whl", hash = "sha256:ba203255017337d39f89bdd58417f03c4426f12beed0440cfd933cb15f8669c7", size = 10566643, upload-time = "2026-03-29T13:21:34.339Z" }, + { url = "https://files.pythonhosted.org/packages/6b/33/8fae8f964a4f63ed528264ddf25d2b683d0b663e3cba26961eb838a7c1bd/numpy-2.4.4-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:58c8b5929fcb8287cbd6f0a3fae19c6e03a5c48402ae792962ac465224a629a4", size = 16854491, upload-time = "2026-03-29T13:21:38.03Z" }, + { url = "https://files.pythonhosted.org/packages/bc/d0/1aabee441380b981cf8cdda3ae7a46aa827d1b5a8cce84d14598bc94d6d9/numpy-2.4.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:eea7ac5d2dce4189771cedb559c738a71512768210dc4e4753b107a2048b3d0e", size = 14895830, upload-time = "2026-03-29T13:21:41.509Z" }, + { url = "https://files.pythonhosted.org/packages/a5/b8/aafb0d1065416894fccf4df6b49ef22b8db045187949545bced89c034b8e/numpy-2.4.4-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:51fc224f7ca4d92656d5a5eb315f12eb5fe2c97a66249aa7b5f562528a3be38c", size = 5400927, upload-time = "2026-03-29T13:21:44.747Z" }, + { url = "https://files.pythonhosted.org/packages/d6/77/063baa20b08b431038c7f9ff5435540c7b7265c78cf56012a483019ca72d/numpy-2.4.4-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:28a650663f7314afc3e6ec620f44f333c386aad9f6fc472030865dc0ebb26ee3", size = 6715557, upload-time = "2026-03-29T13:21:47.406Z" }, + { url = "https://files.pythonhosted.org/packages/c7/a8/379542d45a14f149444c5c4c4e7714707239ce9cc1de8c2803958889da14/numpy-2.4.4-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19710a9ca9992d7174e9c52f643d4272dcd1558c5f7af7f6f8190f633bd651a7", size = 15804253, upload-time = "2026-03-29T13:21:50.753Z" }, + { url = "https://files.pythonhosted.org/packages/a2/c8/f0a45426d6d21e7ea3310a15cf90c43a14d9232c31a837702dba437f3373/numpy-2.4.4-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9b2aec6af35c113b05695ebb5749a787acd63cafc83086a05771d1e1cd1e555f", size = 16753552, upload-time = "2026-03-29T13:21:54.344Z" }, + { url = "https://files.pythonhosted.org/packages/04/74/f4c001f4714c3ad9ce037e18cf2b9c64871a84951eaa0baf683a9ca9301c/numpy-2.4.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:f2cf083b324a467e1ab358c105f6cad5ea950f50524668a80c486ff1db24e119", size = 12509075, upload-time = "2026-03-29T13:21:57.644Z" }, +] + [[package]] name = "opltools" version = "0.1.0" source = { editable = "." } dependencies = [ + { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "pandas", version = "3.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "pydantic" }, { name = "pydantic-yaml" }, { name = "pyyaml" }, @@ -204,6 +367,7 @@ dev = [ [package.metadata] requires-dist = [ + { name = "pandas" }, { name = "pydantic", specifier = ">=2.13.3" }, { name = "pydantic-yaml", specifier = ">=1.6.0" }, { name = "pyyaml", specifier = ">=6.0.3" }, @@ -227,6 +391,138 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7a/c2/920ef838e2f0028c8262f16101ec09ebd5969864e5a64c4c05fad0617c56/packaging-26.1-py3-none-any.whl", hash = "sha256:5d9c0669c6285e491e0ced2eee587eaf67b670d94a19e94e3984a481aba6802f", size = 95831, upload-time = "2026-04-14T21:12:47.56Z" }, ] +[[package]] +name = "pandas" +version = "2.3.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11'", +] +dependencies = [ + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "python-dateutil", marker = "python_full_version < '3.11'" }, + { name = "pytz", marker = "python_full_version < '3.11'" }, + { name = "tzdata", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/f7/f425a00df4fcc22b292c6895c6831c0c8ae1d9fac1e024d16f98a9ce8749/pandas-2.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c", size = 11555763, upload-time = "2025-09-29T23:16:53.287Z" }, + { url = "https://files.pythonhosted.org/packages/13/4f/66d99628ff8ce7857aca52fed8f0066ce209f96be2fede6cef9f84e8d04f/pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a", size = 10801217, upload-time = "2025-09-29T23:17:04.522Z" }, + { url = "https://files.pythonhosted.org/packages/1d/03/3fc4a529a7710f890a239cc496fc6d50ad4a0995657dccc1d64695adb9f4/pandas-2.3.3-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5caf26f64126b6c7aec964f74266f435afef1c1b13da3b0636c7518a1fa3e2b1", size = 12148791, upload-time = "2025-09-29T23:17:18.444Z" }, + { url = "https://files.pythonhosted.org/packages/40/a8/4dac1f8f8235e5d25b9955d02ff6f29396191d4e665d71122c3722ca83c5/pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd7478f1463441ae4ca7308a70e90b33470fa593429f9d4c578dd00d1fa78838", size = 12769373, upload-time = "2025-09-29T23:17:35.846Z" }, + { url = "https://files.pythonhosted.org/packages/df/91/82cc5169b6b25440a7fc0ef3a694582418d875c8e3ebf796a6d6470aa578/pandas-2.3.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4793891684806ae50d1288c9bae9330293ab4e083ccd1c5e383c34549c6e4250", size = 13200444, upload-time = "2025-09-29T23:17:49.341Z" }, + { url = "https://files.pythonhosted.org/packages/10/ae/89b3283800ab58f7af2952704078555fa60c807fff764395bb57ea0b0dbd/pandas-2.3.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:28083c648d9a99a5dd035ec125d42439c6c1c525098c58af0fc38dd1a7a1b3d4", size = 13858459, upload-time = "2025-09-29T23:18:03.722Z" }, + { url = "https://files.pythonhosted.org/packages/85/72/530900610650f54a35a19476eca5104f38555afccda1aa11a92ee14cb21d/pandas-2.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:503cf027cf9940d2ceaa1a93cfb5f8c8c7e6e90720a2850378f0b3f3b1e06826", size = 11346086, upload-time = "2025-09-29T23:18:18.505Z" }, + { url = "https://files.pythonhosted.org/packages/c1/fa/7ac648108144a095b4fb6aa3de1954689f7af60a14cf25583f4960ecb878/pandas-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523", size = 11578790, upload-time = "2025-09-29T23:18:30.065Z" }, + { url = "https://files.pythonhosted.org/packages/9b/35/74442388c6cf008882d4d4bdfc4109be87e9b8b7ccd097ad1e7f006e2e95/pandas-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45", size = 10833831, upload-time = "2025-09-29T23:38:56.071Z" }, + { url = "https://files.pythonhosted.org/packages/fe/e4/de154cbfeee13383ad58d23017da99390b91d73f8c11856f2095e813201b/pandas-2.3.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66", size = 12199267, upload-time = "2025-09-29T23:18:41.627Z" }, + { url = "https://files.pythonhosted.org/packages/bf/c9/63f8d545568d9ab91476b1818b4741f521646cbdd151c6efebf40d6de6f7/pandas-2.3.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b", size = 12789281, upload-time = "2025-09-29T23:18:56.834Z" }, + { url = "https://files.pythonhosted.org/packages/f2/00/a5ac8c7a0e67fd1a6059e40aa08fa1c52cc00709077d2300e210c3ce0322/pandas-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791", size = 13240453, upload-time = "2025-09-29T23:19:09.247Z" }, + { url = "https://files.pythonhosted.org/packages/27/4d/5c23a5bc7bd209231618dd9e606ce076272c9bc4f12023a70e03a86b4067/pandas-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151", size = 13890361, upload-time = "2025-09-29T23:19:25.342Z" }, + { url = "https://files.pythonhosted.org/packages/8e/59/712db1d7040520de7a4965df15b774348980e6df45c129b8c64d0dbe74ef/pandas-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c", size = 11348702, upload-time = "2025-09-29T23:19:38.296Z" }, + { url = "https://files.pythonhosted.org/packages/9c/fb/231d89e8637c808b997d172b18e9d4a4bc7bf31296196c260526055d1ea0/pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53", size = 11597846, upload-time = "2025-09-29T23:19:48.856Z" }, + { url = "https://files.pythonhosted.org/packages/5c/bd/bf8064d9cfa214294356c2d6702b716d3cf3bb24be59287a6a21e24cae6b/pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35", size = 10729618, upload-time = "2025-09-29T23:39:08.659Z" }, + { url = "https://files.pythonhosted.org/packages/57/56/cf2dbe1a3f5271370669475ead12ce77c61726ffd19a35546e31aa8edf4e/pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908", size = 11737212, upload-time = "2025-09-29T23:19:59.765Z" }, + { url = "https://files.pythonhosted.org/packages/e5/63/cd7d615331b328e287d8233ba9fdf191a9c2d11b6af0c7a59cfcec23de68/pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89", size = 12362693, upload-time = "2025-09-29T23:20:14.098Z" }, + { url = "https://files.pythonhosted.org/packages/a6/de/8b1895b107277d52f2b42d3a6806e69cfef0d5cf1d0ba343470b9d8e0a04/pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98", size = 12771002, upload-time = "2025-09-29T23:20:26.76Z" }, + { url = "https://files.pythonhosted.org/packages/87/21/84072af3187a677c5893b170ba2c8fbe450a6ff911234916da889b698220/pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084", size = 13450971, upload-time = "2025-09-29T23:20:41.344Z" }, + { url = "https://files.pythonhosted.org/packages/86/41/585a168330ff063014880a80d744219dbf1dd7a1c706e75ab3425a987384/pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b", size = 10992722, upload-time = "2025-09-29T23:20:54.139Z" }, + { url = "https://files.pythonhosted.org/packages/cd/4b/18b035ee18f97c1040d94debd8f2e737000ad70ccc8f5513f4eefad75f4b/pandas-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713", size = 11544671, upload-time = "2025-09-29T23:21:05.024Z" }, + { url = "https://files.pythonhosted.org/packages/31/94/72fac03573102779920099bcac1c3b05975c2cb5f01eac609faf34bed1ca/pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8", size = 10680807, upload-time = "2025-09-29T23:21:15.979Z" }, + { url = "https://files.pythonhosted.org/packages/16/87/9472cf4a487d848476865321de18cc8c920b8cab98453ab79dbbc98db63a/pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d", size = 11709872, upload-time = "2025-09-29T23:21:27.165Z" }, + { url = "https://files.pythonhosted.org/packages/15/07/284f757f63f8a8d69ed4472bfd85122bd086e637bf4ed09de572d575a693/pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac", size = 12306371, upload-time = "2025-09-29T23:21:40.532Z" }, + { url = "https://files.pythonhosted.org/packages/33/81/a3afc88fca4aa925804a27d2676d22dcd2031c2ebe08aabd0ae55b9ff282/pandas-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c", size = 12765333, upload-time = "2025-09-29T23:21:55.77Z" }, + { url = "https://files.pythonhosted.org/packages/8d/0f/b4d4ae743a83742f1153464cf1a8ecfafc3ac59722a0b5c8602310cb7158/pandas-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493", size = 13418120, upload-time = "2025-09-29T23:22:10.109Z" }, + { url = "https://files.pythonhosted.org/packages/4f/c7/e54682c96a895d0c808453269e0b5928a07a127a15704fedb643e9b0a4c8/pandas-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f8bfc0e12dc78f777f323f55c58649591b2cd0c43534e8355c51d3fede5f4dee", size = 10993991, upload-time = "2025-09-29T23:25:04.889Z" }, + { url = "https://files.pythonhosted.org/packages/f9/ca/3f8d4f49740799189e1395812f3bf23b5e8fc7c190827d55a610da72ce55/pandas-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:75ea25f9529fdec2d2e93a42c523962261e567d250b0013b16210e1d40d7c2e5", size = 12048227, upload-time = "2025-09-29T23:22:24.343Z" }, + { url = "https://files.pythonhosted.org/packages/0e/5a/f43efec3e8c0cc92c4663ccad372dbdff72b60bdb56b2749f04aa1d07d7e/pandas-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74ecdf1d301e812db96a465a525952f4dde225fdb6d8e5a521d47e1f42041e21", size = 11411056, upload-time = "2025-09-29T23:22:37.762Z" }, + { url = "https://files.pythonhosted.org/packages/46/b1/85331edfc591208c9d1a63a06baa67b21d332e63b7a591a5ba42a10bb507/pandas-2.3.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78", size = 11645189, upload-time = "2025-09-29T23:22:51.688Z" }, + { url = "https://files.pythonhosted.org/packages/44/23/78d645adc35d94d1ac4f2a3c4112ab6f5b8999f4898b8cdf01252f8df4a9/pandas-2.3.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110", size = 12121912, upload-time = "2025-09-29T23:23:05.042Z" }, + { url = "https://files.pythonhosted.org/packages/53/da/d10013df5e6aaef6b425aa0c32e1fc1f3e431e4bcabd420517dceadce354/pandas-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86", size = 12712160, upload-time = "2025-09-29T23:23:28.57Z" }, + { url = "https://files.pythonhosted.org/packages/bd/17/e756653095a083d8a37cbd816cb87148debcfcd920129b25f99dd8d04271/pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc", size = 13199233, upload-time = "2025-09-29T23:24:24.876Z" }, + { url = "https://files.pythonhosted.org/packages/04/fd/74903979833db8390b73b3a8a7d30d146d710bd32703724dd9083950386f/pandas-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0", size = 11540635, upload-time = "2025-09-29T23:25:52.486Z" }, + { url = "https://files.pythonhosted.org/packages/21/00/266d6b357ad5e6d3ad55093a7e8efc7dd245f5a842b584db9f30b0f0a287/pandas-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593", size = 10759079, upload-time = "2025-09-29T23:26:33.204Z" }, + { url = "https://files.pythonhosted.org/packages/ca/05/d01ef80a7a3a12b2f8bbf16daba1e17c98a2f039cbc8e2f77a2c5a63d382/pandas-2.3.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c", size = 11814049, upload-time = "2025-09-29T23:27:15.384Z" }, + { url = "https://files.pythonhosted.org/packages/15/b2/0e62f78c0c5ba7e3d2c5945a82456f4fac76c480940f805e0b97fcbc2f65/pandas-2.3.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b", size = 12332638, upload-time = "2025-09-29T23:27:51.625Z" }, + { url = "https://files.pythonhosted.org/packages/c5/33/dd70400631b62b9b29c3c93d2feee1d0964dc2bae2e5ad7a6c73a7f25325/pandas-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6", size = 12886834, upload-time = "2025-09-29T23:28:21.289Z" }, + { url = "https://files.pythonhosted.org/packages/d3/18/b5d48f55821228d0d2692b34fd5034bb185e854bdb592e9c640f6290e012/pandas-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3", size = 13409925, upload-time = "2025-09-29T23:28:58.261Z" }, + { url = "https://files.pythonhosted.org/packages/a6/3d/124ac75fcd0ecc09b8fdccb0246ef65e35b012030defb0e0eba2cbbbe948/pandas-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5", size = 11109071, upload-time = "2025-09-29T23:32:27.484Z" }, + { url = "https://files.pythonhosted.org/packages/89/9c/0e21c895c38a157e0faa1fb64587a9226d6dd46452cac4532d80c3c4a244/pandas-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec", size = 12048504, upload-time = "2025-09-29T23:29:31.47Z" }, + { url = "https://files.pythonhosted.org/packages/d7/82/b69a1c95df796858777b68fbe6a81d37443a33319761d7c652ce77797475/pandas-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7", size = 11410702, upload-time = "2025-09-29T23:29:54.591Z" }, + { url = "https://files.pythonhosted.org/packages/f9/88/702bde3ba0a94b8c73a0181e05144b10f13f29ebfc2150c3a79062a8195d/pandas-2.3.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450", size = 11634535, upload-time = "2025-09-29T23:30:21.003Z" }, + { url = "https://files.pythonhosted.org/packages/a4/1e/1bac1a839d12e6a82ec6cb40cda2edde64a2013a66963293696bbf31fbbb/pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5", size = 12121582, upload-time = "2025-09-29T23:30:43.391Z" }, + { url = "https://files.pythonhosted.org/packages/44/91/483de934193e12a3b1d6ae7c8645d083ff88dec75f46e827562f1e4b4da6/pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788", size = 12699963, upload-time = "2025-09-29T23:31:10.009Z" }, + { url = "https://files.pythonhosted.org/packages/70/44/5191d2e4026f86a2a109053e194d3ba7a31a2d10a9c2348368c63ed4e85a/pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87", size = 13202175, upload-time = "2025-09-29T23:31:59.173Z" }, +] + +[[package]] +name = "pandas" +version = "3.0.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +dependencies = [ + { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "python-dateutil", marker = "python_full_version >= '3.11'" }, + { name = "tzdata", marker = "(python_full_version >= '3.11' and sys_platform == 'emscripten') or (python_full_version >= '3.11' and sys_platform == 'win32')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/da/99/b342345300f13440fe9fe385c3c481e2d9a595ee3bab4d3219247ac94e9a/pandas-3.0.2.tar.gz", hash = "sha256:f4753e73e34c8d83221ba58f232433fca2748be8b18dbca02d242ed153945043", size = 4645855, upload-time = "2026-03-31T06:48:30.816Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/97/35/6411db530c618e0e0005187e35aa02ce60ae4c4c4d206964a2f978217c27/pandas-3.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a727a73cbdba2f7458dc82449e2315899d5140b449015d822f515749a46cbbe0", size = 10326926, upload-time = "2026-03-31T06:46:08.29Z" }, + { url = "https://files.pythonhosted.org/packages/c4/d3/b7da1d5d7dbdc5ef52ed7debd2b484313b832982266905315dad5a0bf0b1/pandas-3.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dbbd4aa20ca51e63b53bbde6a0fa4254b1aaabb74d2f542df7a7959feb1d760c", size = 9926987, upload-time = "2026-03-31T06:46:11.724Z" }, + { url = "https://files.pythonhosted.org/packages/52/77/9b1c2d6070b5dbe239a7bc889e21bfa58720793fb902d1e070695d87c6d0/pandas-3.0.2-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:339dda302bd8369dedeae979cb750e484d549b563c3f54f3922cb8ff4978c5eb", size = 10757067, upload-time = "2026-03-31T06:46:14.903Z" }, + { url = "https://files.pythonhosted.org/packages/20/17/ec40d981705654853726e7ac9aea9ddbb4a5d9cf54d8472222f4f3de06c2/pandas-3.0.2-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:61c2fd96d72b983a9891b2598f286befd4ad262161a609c92dc1652544b46b76", size = 11258787, upload-time = "2026-03-31T06:46:17.683Z" }, + { url = "https://files.pythonhosted.org/packages/90/e3/3f1126d43d3702ca8773871a81c9f15122a1f412342cc56284ffda5b1f70/pandas-3.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c934008c733b8bbea273ea308b73b3156f0181e5b72960790b09c18a2794fe1e", size = 11771616, upload-time = "2026-03-31T06:46:20.532Z" }, + { url = "https://files.pythonhosted.org/packages/2e/cf/0f4e268e1f5062e44a6bda9f925806721cd4c95c2b808a4c82ebe914f96b/pandas-3.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:60a80bb4feacbef5e1447a3f82c33209c8b7e07f28d805cfd1fb951e5cb443aa", size = 12337623, upload-time = "2026-03-31T06:46:23.754Z" }, + { url = "https://files.pythonhosted.org/packages/44/a0/97a6339859d4acb2536efb24feb6708e82f7d33b2ed7e036f2983fcced82/pandas-3.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:ed72cb3f45190874eb579c64fa92d9df74e98fd63e2be7f62bce5ace0ade61df", size = 9897372, upload-time = "2026-03-31T06:46:26.703Z" }, + { url = "https://files.pythonhosted.org/packages/8f/eb/781516b808a99ddf288143cec46b342b3016c3414d137da1fdc3290d8860/pandas-3.0.2-cp311-cp311-win_arm64.whl", hash = "sha256:f12b1a9e332c01e09510586f8ca9b108fd631fd656af82e452d7315ef6df5f9f", size = 9154922, upload-time = "2026-03-31T06:46:30.284Z" }, + { url = "https://files.pythonhosted.org/packages/f3/b0/c20bd4d6d3f736e6bd6b55794e9cd0a617b858eaad27c8f410ea05d953b7/pandas-3.0.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:232a70ebb568c0c4d2db4584f338c1577d81e3af63292208d615907b698a0f18", size = 10347921, upload-time = "2026-03-31T06:46:33.36Z" }, + { url = "https://files.pythonhosted.org/packages/35/d0/4831af68ce30cc2d03c697bea8450e3225a835ef497d0d70f31b8cdde965/pandas-3.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:970762605cff1ca0d3f71ed4f3a769ea8f85fc8e6348f6e110b8fea7e6eb5a14", size = 9888127, upload-time = "2026-03-31T06:46:36.253Z" }, + { url = "https://files.pythonhosted.org/packages/61/a9/16ea9346e1fc4a96e2896242d9bc674764fb9049b0044c0132502f7a771e/pandas-3.0.2-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aff4e6f4d722e0652707d7bcb190c445fe58428500c6d16005b02401764b1b3d", size = 10399577, upload-time = "2026-03-31T06:46:39.224Z" }, + { url = "https://files.pythonhosted.org/packages/c4/a8/3a61a721472959ab0ce865ef05d10b0d6bfe27ce8801c99f33d4fa996e65/pandas-3.0.2-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef8b27695c3d3dc78403c9a7d5e59a62d5464a7e1123b4e0042763f7104dc74f", size = 10880030, upload-time = "2026-03-31T06:46:42.412Z" }, + { url = "https://files.pythonhosted.org/packages/da/65/7225c0ea4d6ce9cb2160a7fb7f39804871049f016e74782e5dade4d14109/pandas-3.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f8d68083e49e16b84734eb1a4dcae4259a75c90fb6e2251ab9a00b61120c06ab", size = 11409468, upload-time = "2026-03-31T06:46:45.2Z" }, + { url = "https://files.pythonhosted.org/packages/fa/5b/46e7c76032639f2132359b5cf4c785dd8cf9aea5ea64699eac752f02b9db/pandas-3.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:32cc41f310ebd4a296d93515fcac312216adfedb1894e879303987b8f1e2b97d", size = 11936381, upload-time = "2026-03-31T06:46:48.293Z" }, + { url = "https://files.pythonhosted.org/packages/7b/8b/721a9cff6fa6a91b162eb51019c6243b82b3226c71bb6c8ef4a9bd65cbc6/pandas-3.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:a4785e1d6547d8427c5208b748ae2efb64659a21bd82bf440d4262d02bfa02a4", size = 9744993, upload-time = "2026-03-31T06:46:51.488Z" }, + { url = "https://files.pythonhosted.org/packages/d5/18/7f0bd34ae27b28159aa80f2a6799f47fda34f7fb938a76e20c7b7fe3b200/pandas-3.0.2-cp312-cp312-win_arm64.whl", hash = "sha256:08504503f7101300107ecdc8df73658e4347586db5cfdadabc1592e9d7e7a0fd", size = 9056118, upload-time = "2026-03-31T06:46:54.548Z" }, + { url = "https://files.pythonhosted.org/packages/bf/ca/3e639a1ea6fcd0617ca4e8ca45f62a74de33a56ae6cd552735470b22c8d3/pandas-3.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b5918ba197c951dec132b0c5929a00c0bf05d5942f590d3c10a807f6e15a57d3", size = 10321105, upload-time = "2026-03-31T06:46:57.327Z" }, + { url = "https://files.pythonhosted.org/packages/0b/77/dbc82ff2fb0e63c6564356682bf201edff0ba16c98630d21a1fb312a8182/pandas-3.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d606a041c89c0a474a4702d532ab7e73a14fe35c8d427b972a625c8e46373668", size = 9864088, upload-time = "2026-03-31T06:46:59.935Z" }, + { url = "https://files.pythonhosted.org/packages/5c/2b/341f1b04bbca2e17e13cd3f08c215b70ef2c60c5356ef1e8c6857449edc7/pandas-3.0.2-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:710246ba0616e86891b58ab95f2495143bb2bc83ab6b06747c74216f583a6ac9", size = 10369066, upload-time = "2026-03-31T06:47:02.792Z" }, + { url = "https://files.pythonhosted.org/packages/12/c5/cbb1ffefb20a93d3f0e1fdcda699fb84976210d411b008f97f48bf6ce27e/pandas-3.0.2-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5d3cfe227c725b1f3dff4278b43d8c784656a42a9325b63af6b1492a8232209e", size = 10876780, upload-time = "2026-03-31T06:47:06.205Z" }, + { url = "https://files.pythonhosted.org/packages/98/fe/2249ae5e0a69bd0ddf17353d0a5d26611d70970111f5b3600cdc8be883e7/pandas-3.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c3b723df9087a9a9a840e263ebd9f88b64a12075d1bf2ea401a5a42f254f084d", size = 11375181, upload-time = "2026-03-31T06:47:09.383Z" }, + { url = "https://files.pythonhosted.org/packages/de/64/77a38b09e70b6464883b8d7584ab543e748e42c1b5d337a2ee088e0df741/pandas-3.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a3096110bf9eac0070b7208465f2740e2d8a670d5cb6530b5bb884eca495fd39", size = 11928899, upload-time = "2026-03-31T06:47:12.686Z" }, + { url = "https://files.pythonhosted.org/packages/5e/52/42855bf626868413f761addd574acc6195880ae247a5346477a4361c3acb/pandas-3.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:07a10f5c36512eead51bc578eb3354ad17578b22c013d89a796ab5eee90cd991", size = 9746574, upload-time = "2026-03-31T06:47:15.64Z" }, + { url = "https://files.pythonhosted.org/packages/88/39/21304ae06a25e8bf9fc820d69b29b2c495b2ae580d1e143146c309941760/pandas-3.0.2-cp313-cp313-win_arm64.whl", hash = "sha256:5fdbfa05931071aba28b408e59226186b01eb5e92bea2ab78b65863ca3228d84", size = 9047156, upload-time = "2026-03-31T06:47:18.595Z" }, + { url = "https://files.pythonhosted.org/packages/72/20/7defa8b27d4f330a903bb68eea33be07d839c5ea6bdda54174efcec0e1d2/pandas-3.0.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:dbc20dea3b9e27d0e66d74c42b2d0c1bed9c2ffe92adea33633e3bedeb5ac235", size = 10756238, upload-time = "2026-03-31T06:47:22.012Z" }, + { url = "https://files.pythonhosted.org/packages/e9/95/49433c14862c636afc0e9b2db83ff16b3ad92959364e52b2955e44c8e94c/pandas-3.0.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:b75c347eff42497452116ce05ef461822d97ce5b9ff8df6edacb8076092c855d", size = 10408520, upload-time = "2026-03-31T06:47:25.197Z" }, + { url = "https://files.pythonhosted.org/packages/3b/f8/462ad2b5881d6b8ec8e5f7ed2ea1893faa02290d13870a1600fe72ad8efc/pandas-3.0.2-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d1478075142e83a5571782ad007fb201ed074bdeac7ebcc8890c71442e96adf7", size = 10324154, upload-time = "2026-03-31T06:47:28.097Z" }, + { url = "https://files.pythonhosted.org/packages/0a/65/d1e69b649cbcddda23ad6e4c40ef935340f6f652a006e5cbc3555ac8adb3/pandas-3.0.2-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5880314e69e763d4c8b27937090de570f1fb8d027059a7ada3f7f8e98bdcb677", size = 10714449, upload-time = "2026-03-31T06:47:30.85Z" }, + { url = "https://files.pythonhosted.org/packages/47/a4/85b59bc65b8190ea3689882db6cdf32a5003c0ccd5a586c30fdcc3ffc4fc/pandas-3.0.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:b5329e26898896f06035241a626d7c335daa479b9bbc82be7c2742d048e41172", size = 11338475, upload-time = "2026-03-31T06:47:34.026Z" }, + { url = "https://files.pythonhosted.org/packages/1e/c4/bc6966c6e38e5d9478b935272d124d80a589511ed1612a5d21d36f664c68/pandas-3.0.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:81526c4afd31971f8b62671442a4b2b51e0aa9acc3819c9f0f12a28b6fcf85f1", size = 11786568, upload-time = "2026-03-31T06:47:36.941Z" }, + { url = "https://files.pythonhosted.org/packages/e8/74/09298ca9740beed1d3504e073d67e128aa07e5ca5ca2824b0c674c0b8676/pandas-3.0.2-cp313-cp313t-win_amd64.whl", hash = "sha256:7cadd7e9a44ec13b621aec60f9150e744cfc7a3dd32924a7e2f45edff31823b0", size = 10488652, upload-time = "2026-03-31T06:47:40.612Z" }, + { url = "https://files.pythonhosted.org/packages/bb/40/c6ea527147c73b24fc15c891c3fcffe9c019793119c5742b8784a062c7db/pandas-3.0.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:db0dbfd2a6cdf3770aa60464d50333d8f3d9165b2f2671bcc299b72de5a6677b", size = 10326084, upload-time = "2026-03-31T06:47:43.834Z" }, + { url = "https://files.pythonhosted.org/packages/95/25/bdb9326c3b5455f8d4d3549fce7abcf967259de146fe2cf7a82368141948/pandas-3.0.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0555c5882688a39317179ab4a0ed41d3ebc8812ab14c69364bbee8fb7a3f6288", size = 9914146, upload-time = "2026-03-31T06:47:46.67Z" }, + { url = "https://files.pythonhosted.org/packages/8d/77/3a227ff3337aa376c60d288e1d61c5d097131d0ac71f954d90a8f369e422/pandas-3.0.2-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:01f31a546acd5574ef77fe199bc90b55527c225c20ccda6601cf6b0fd5ed597c", size = 10444081, upload-time = "2026-03-31T06:47:49.681Z" }, + { url = "https://files.pythonhosted.org/packages/15/88/3cdd54fa279341afa10acf8d2b503556b1375245dccc9315659f795dd2e9/pandas-3.0.2-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:deeca1b5a931fdf0c2212c8a659ade6d3b1edc21f0914ce71ef24456ca7a6535", size = 10897535, upload-time = "2026-03-31T06:47:53.033Z" }, + { url = "https://files.pythonhosted.org/packages/06/9d/98cc7a7624f7932e40f434299260e2917b090a579d75937cb8a57b9d2de3/pandas-3.0.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0f48afd9bb13300ffb5a3316973324c787054ba6665cda0da3fbd67f451995db", size = 11446992, upload-time = "2026-03-31T06:47:56.193Z" }, + { url = "https://files.pythonhosted.org/packages/9a/cd/19ff605cc3760e80602e6826ddef2824d8e7050ed80f2e11c4b079741dc3/pandas-3.0.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6c4d8458b97a35717b62469a4ea0e85abd5ed8687277f5ccfc67f8a5126f8c53", size = 11968257, upload-time = "2026-03-31T06:47:59.137Z" }, + { url = "https://files.pythonhosted.org/packages/db/60/aba6a38de456e7341285102bede27514795c1eaa353bc0e7638b6b785356/pandas-3.0.2-cp314-cp314-win_amd64.whl", hash = "sha256:b35d14bb5d8285d9494fe93815a9e9307c0876e10f1e8e89ac5b88f728ec8dcf", size = 9865893, upload-time = "2026-03-31T06:48:02.038Z" }, + { url = "https://files.pythonhosted.org/packages/08/71/e5ec979dd2e8a093dacb8864598c0ff59a0cee0bbcdc0bfec16a51684d4f/pandas-3.0.2-cp314-cp314-win_arm64.whl", hash = "sha256:63d141b56ef686f7f0d714cfb8de4e320475b86bf4b620aa0b7da89af8cbdbbb", size = 9188644, upload-time = "2026-03-31T06:48:05.045Z" }, + { url = "https://files.pythonhosted.org/packages/f1/6c/7b45d85db19cae1eb524f2418ceaa9d85965dcf7b764ed151386b7c540f0/pandas-3.0.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:140f0cffb1fa2524e874dde5b477d9defe10780d8e9e220d259b2c0874c89d9d", size = 10776246, upload-time = "2026-03-31T06:48:07.789Z" }, + { url = "https://files.pythonhosted.org/packages/a8/3e/7b00648b086c106e81766f25322b48aa8dfa95b55e621dbdf2fdd413a117/pandas-3.0.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ae37e833ff4fed0ba352f6bdd8b73ba3ab3256a85e54edfd1ab51ae40cca0af8", size = 10424801, upload-time = "2026-03-31T06:48:10.897Z" }, + { url = "https://files.pythonhosted.org/packages/da/6e/558dd09a71b53b4008e7fc8a98ec6d447e9bfb63cdaeea10e5eb9b2dabe8/pandas-3.0.2-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d888a5c678a419a5bb41a2a93818e8ed9fd3172246555c0b37b7cc27027effd", size = 10345643, upload-time = "2026-03-31T06:48:13.7Z" }, + { url = "https://files.pythonhosted.org/packages/be/e3/921c93b4d9a280409451dc8d07b062b503bbec0531d2627e73a756e99a82/pandas-3.0.2-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b444dc64c079e84df91baa8bf613d58405645461cabca929d9178f2cd392398d", size = 10743641, upload-time = "2026-03-31T06:48:16.659Z" }, + { url = "https://files.pythonhosted.org/packages/56/ca/fd17286f24fa3b4d067965d8d5d7e14fe557dd4f979a0b068ac0deaf8228/pandas-3.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:4544c7a54920de8eeacaa1466a6b7268ecfbc9bc64ab4dbb89c6bbe94d5e0660", size = 11361993, upload-time = "2026-03-31T06:48:19.475Z" }, + { url = "https://files.pythonhosted.org/packages/e4/a5/2f6ed612056819de445a433ca1f2821ac3dab7f150d569a59e9cc105de1d/pandas-3.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:734be7551687c00fbd760dc0522ed974f82ad230d4a10f54bf51b80d44a08702", size = 11815274, upload-time = "2026-03-31T06:48:22.695Z" }, + { url = "https://files.pythonhosted.org/packages/00/2f/b622683e99ec3ce00b0854bac9e80868592c5b051733f2cf3a868e5fea26/pandas-3.0.2-cp314-cp314t-win_amd64.whl", hash = "sha256:57a07209bebcbcf768d2d13c9b78b852f9a15978dac41b9e6421a81ad4cdd276", size = 10888530, upload-time = "2026-03-31T06:48:25.806Z" }, + { url = "https://files.pythonhosted.org/packages/cb/2b/f8434233fab2bd66a02ec014febe4e5adced20e2693e0e90a07d118ed30e/pandas-3.0.2-cp314-cp314t-win_arm64.whl", hash = "sha256:5371b72c2d4d415d08765f32d689217a43227484e81b2305b52076e328f6f482", size = 9455341, upload-time = "2026-03-31T06:48:28.418Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -422,6 +718,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, ] +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "pytz" +version = "2026.1.post1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/56/db/b8721d71d945e6a8ac63c0fc900b2067181dbb50805958d4d4661cf7d277/pytz-2026.1.post1.tar.gz", hash = "sha256:3378dde6a0c3d26719182142c56e60c7f9af7e968076f31aae569d72a0358ee1", size = 321088, upload-time = "2026-03-03T07:47:50.683Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/99/781fe0c827be2742bcc775efefccb3b048a3a9c6ce9aec0cbf4a101677e5/pytz-2026.1.post1-py2.py3-none-any.whl", hash = "sha256:f2fd16142fda348286a75e1a524be810bb05d444e5a081f37f7affc635035f7a", size = 510489, upload-time = "2026-03-03T07:47:49.167Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -594,6 +911,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/63/b6/aeadee5443e49baa2facd51131159fd6301cc4ccfc1541e4df7b021c37dd/ruff-0.15.11-py3-none-win_arm64.whl", hash = "sha256:063fed18cc1bbe0ee7393957284a6fe8b588c6a406a285af3ee3f46da2391ee4", size = 11032614, upload-time = "2026-04-16T18:46:34.487Z" }, ] +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + [[package]] name = "tomli" version = "2.4.1" @@ -668,3 +994,12 @@ sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac wheels = [ { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, ] + +[[package]] +name = "tzdata" +version = "2026.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/19/f5/cd531b2d15a671a40c0f66cf06bc3570a12cd56eef98960068ebbad1bf5a/tzdata-2026.1.tar.gz", hash = "sha256:67658a1903c75917309e753fdc349ac0efd8c27db7a0cb406a25be4840f87f98", size = 197639, upload-time = "2026-04-03T11:25:22.002Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/70/d460bd685a170790ec89317e9bd33047988e4bce507b831f5db771e142de/tzdata-2026.1-py2.py3-none-any.whl", hash = "sha256:4b1d2be7ac37ceafd7327b961aa3a54e467efbdb563a23655fbfe0d39cfc42a9", size = 348952, upload-time = "2026-04-03T11:25:20.313Z" }, +] diff --git a/yaml_to_html.py b/yaml_to_html.py index a9a219f..7ee5a6e 100755 --- a/yaml_to_html.py +++ b/yaml_to_html.py @@ -1,79 +1,783 @@ #!/usr/bin/env python3 -import pandas as pd -import yaml +import json +import re import shutil +from html import escape -import re +import pandas as pd +from pydantic_yaml import parse_yaml_raw_as + +from opltools.schema import ConstraintType, Implementation, Library, ProblemLike URL_RE = re.compile(r'\b((?:https?://|www\.)[^\s<>"\']+)', re.IGNORECASE) +VISIBLE_VARIABLE_TYPES = ["binary", "categorical", "continuous", "integer"] +VARIABLE_COLUMN_NAME_BY_TYPE = { + "binary": "binary Variables", + "categorical": "categorical Variables", + "continuous": "continuous Variables", + "integer": "integer Variables", +} +COLUMN_DISPLAY_NAMES = { + "problem_id": "ID", + "long_name": "Full Name", + "description": "Description", + "tags": "Tags", + "references": "References", + "implementations": "Implementations", + "dynamic_type": "Dynamics", + "noise_type": "Noise", + "allows_partial_evaluation": "Partial Evaluations", + "can_evaluate_objectives_independently": "Independent Objectives", + "modality": "Modality", + "fidelity_levels": "Fidelity Levels", + "code_examples": "Examples", + "source": "Source", + "binary Variables": "Binary Vars", + "categorical Variables": "Categorical Vars", + "continuous Variables": "Continuous Vars", + "integer Variables": "Integer Vars", + "Total Variables": "Total Variables", + "Hard box Constraints": "Hard Box Constraints", + "Soft box Constraints": "Soft Box Constraints", + "Hard linear Constraints": "Hard Linear Constraints", + "Soft linear Constraints": "Soft Linear Constraints", + "Hard function Constraints": "Hard Function Constraints", + "Soft function Constraints": "Soft Function Constraints", + "Total Constraints": "Total Constraints", + "Variable Types": "Variable Types", + "Constraint Types": "Constraint Types", + "Properties": "Properties", + "Implementation Names": "Implementation Names", + "Implementation Languages": "Implementation Languages", + "Implementation Evaluation Times": "Implementation Evaluation Times", + "Implementation Links": "Implementation Links", + "Implementation Descriptions": "Implementation Descriptions", + "Implementation Requirements": "Implementation Requirements", +} +DEFAULT_VISIBLE_COLUMNS = { + "Name", + "Type", + "Objectives", + "Dynamics", + "Noise", + "Partial Evaluations", + "Independent Objectives", + "Fidelity Levels", + "Variable Types", + "Constraint Types", + "Properties", + "Total Variables", + "Total Constraints", +} def linkify_cell(value): if not isinstance(value, str): return value - def repl(m): - url = m.group(1) + def repl(match): + url = match.group(1) href = url if url.lower().startswith(("http://", "https://")) else f"https://{url}" return f'{url}' return URL_RE.sub(repl, value) -yaml_file = "problems.yaml" - -html_dir = "docs/" -html_table = f"{html_dir}problems.html" -html_header = f"{html_dir}header.html" -html_scripts = f"{html_dir}javascript.html" -html_footer = f"{html_dir}footer.html" -html_index = f"{html_dir}index.html" - -# Load data -with open(yaml_file) as in_file: - data = pd.json_normalize(yaml.safe_load(in_file)) - -# Choose desired columns -all_columns = False -default_columns = ["name", - "textual description", - "suite/generator/single", - "objectives", - "dimensionality", - "variable type", - "constraints", - "dynamic", - "noise", - "multi-fidelity", - "source (real-world/artificial)", - "reference", - "implementation"] - -if all_columns is False: - columns = default_columns - data = data[columns] - -data = data.map(linkify_cell) - -# Generate plain table -table = data.to_html(render_links=False, - escape=False, # Don't escape HTML in cells (to allow links) - index=False, - table_id="problems", - classes=["display compact", "display", "styled-table"], # Set display style - border=0, - na_rep="") # Leave NaN cells empty - -# Add footer to facilitate individual column search -idx = table.index('') -final_table = table[:idx] + "" + " ".join([""+ i +"" for i in data.columns])+" " + table[idx:] - -# Write table to file -with open(html_table, "w") as table_file: - table_file.write(final_table) - -# Merge table and scripts into HTML page -with open(html_index, "wb") as output_file: - for in_file in [html_header, html_table, html_scripts, html_footer]: - with open(in_file, "rb") as in_file: - shutil.copyfileobj(in_file, output_file) + +def to_problem_id(value): + if not isinstance(value, str): + return "" + + normalized = value.strip().lower().replace("-", "_").replace(" ", "_") + normalized = re.sub(r"[^a-z0-9_]", "", normalized) + normalized = re.sub(r"_+", "_", normalized).strip("_") + return normalized + + +def display_column_name(column): + mapped = COLUMN_DISPLAY_NAMES.get(column) + if mapped: + return mapped + + text = str(column or "").replace("_", " ").strip() + if not text: + return str(column) + + words = [] + for word in text.split(): + words.append("ID" if word.lower() == "id" else word.capitalize()) + return " ".join(words) + + +def add_problem_id_row_attributes(table_html, problem_ids): + marker = "" + if marker not in table_html: + return table_html + + body_start = table_html.index(marker) + len(marker) + prefix = table_html[:body_start] + body = table_html[body_start:] + + for problem_id in problem_ids: + safe_id = escape(problem_id, quote=True) + body = body.replace("", f'', 1) + + return prefix + body + + +def format_type(value): + text = str(value or "").strip().lower() + if text == "opltype.problem": + return "Problem" + if text == "opltype.generator": + return "Generator" + if text == "opltype.suite": + return "Suite" + return str(value or "") + + +def format_references(refs): + if not refs: + return "" + + parts = [] + for ref in refs: + ref_dict = ref.model_dump() if hasattr(ref, "model_dump") else ref + if not isinstance(ref_dict, dict): + parts.append(str(ref_dict)) + continue + + title = str(ref_dict.get("title") or "").strip() + authors = ref_dict.get("authors") or [] + if isinstance(authors, list): + authors_txt = "; ".join(str(author) for author in authors if author) + else: + authors_txt = str(authors).strip() + + link = ref_dict.get("link") or {} + if hasattr(link, "model_dump"): + link = link.model_dump() + if isinstance(link, dict): + url = str(link.get("url") or "").strip() + else: + url = str(link).strip() + + triplet = ", ".join(part for part in [title, authors_txt, url] if part) + if triplet: + parts.append(triplet) + + return " | ".join(parts) + + +def normalize_variable_type_name(raw_type): + text = str(raw_type or "").strip().lower().split(".")[-1] + if text in VISIBLE_VARIABLE_TYPES: + return text + return "unknown" + + +def format_dim(dim): + if dim is None: + return "" + if isinstance(dim, (int, float)): + return str(dim) + if hasattr(dim, "model_dump"): + dim = dim.model_dump() + if isinstance(dim, set): + dim = sorted(dim) + if isinstance(dim, list): + return "{" + ", ".join(str(item) for item in dim) + "}" + if isinstance(dim, dict): + dmin = dim.get("min") + dmax = dim.get("max") + if dmin is not None and dmax is not None: + return f"{dmin}-{dmax}" + if dmin is not None: + return f">={dmin}" + if dmax is not None: + return f"<={dmax}" + return "" + return str(dim) + + +def format_total_bounds(total_min, total_max): + if total_min is None and total_max is None: + return "" + if total_min is not None and total_max is not None and total_min == total_max: + return str(total_min) + if total_min is not None and total_max is not None: + return f"{total_min}-{total_max}" + if total_min is not None: + return f">={total_min}" + if total_max is not None: + return f"<={total_max}" + return "" + + +def unique_preserve_order(values): + seen = set() + result = [] + for value in values: + if value in seen: + continue + seen.add(value) + result.append(value) + return result + + +def dim_domain(dim): + if dim is None: + return None + + if hasattr(dim, "model_dump"): + dim = dim.model_dump() + + if isinstance(dim, (int, float)): + return {"kind": "set", "values": [int(dim)]} + + if isinstance(dim, set): + dim = sorted(dim) + + if isinstance(dim, list): + numeric = [int(item) for item in dim if isinstance(item, (int, float))] + if not numeric: + return None + return {"kind": "set", "values": unique_preserve_order(numeric)} + + if isinstance(dim, dict): + dmin = dim.get("min") + dmax = dim.get("max") + dmin = int(dmin) if isinstance(dmin, (int, float)) else None + dmax = int(dmax) if isinstance(dmax, (int, float)) else None + if dmin is None and dmax is None: + return None + return {"kind": "range", "min": dmin, "max": dmax} + + return None + + +def combine_domains_for_total(domains): + if not domains: + return "" + + if any(domain["kind"] == "range" for domain in domains): + total_min = 0 + total_max = 0 + open_upper = False + for domain in domains: + if domain["kind"] == "set": + values = domain["values"] + if not values: + continue + total_min += min(values) + total_max += max(values) + continue + + dmin = domain.get("min") + dmax = domain.get("max") + total_min += dmin or 0 + if dmax is None: + open_upper = True + else: + total_max += dmax + + return format_total_bounds(total_min, None if open_upper else total_max) + + totals = [0] + for domain in domains: + new_totals = [] + for base in totals: + for value in domain["values"]: + new_totals.append(base + value) + totals = unique_preserve_order(new_totals) + + if not totals: + return "" + if len(totals) == 1: + return str(totals[0]) + return "{" + ", ".join(str(value) for value in totals) + "}" + + +def format_variables_by_type(variables): + values = {variable_type: [] for variable_type in VISIBLE_VARIABLE_TYPES} + domains = [] + + if not variables: + return values, "" + + for variable in variables: + variable_dict = variable.model_dump() if hasattr(variable, "model_dump") else variable + if not isinstance(variable_dict, dict): + continue + + variable_type = normalize_variable_type_name(variable_dict.get("type")) + dim = variable_dict.get("dim") + dim_text = format_dim(dim) + if variable_type in values and dim_text: + values[variable_type].append(dim_text) + + domain = dim_domain(dim) + if domain is not None: + domains.append(domain) + + for variable_type in values: + values[variable_type] = " | ".join(values[variable_type]) + + return values, combine_domains_for_total(domains) + + +def normalize_constraint_type(raw_type): + text = str(raw_type or "").strip().lower().split(".")[-1] + return text if text else "unknown" + + +def normalize_constraint_hard(raw_hard): + text = str(raw_hard or "").strip().lower().split(".")[-1] + if text in {"yes", "hard", "true"}: + return "hard" + if text in {"no", "soft", "false"}: + return "soft" + if text in {"some", "mixed", "both"}: + return "mixed" + return "unknown" + + +def list_variable_types(variables): + if not variables: + return "" + + types = [] + for variable in variables: + variable_dict = variable.model_dump() if hasattr(variable, "model_dump") else variable + if not isinstance(variable_dict, dict): + continue + types.append(normalize_variable_type_name(variable_dict.get("type"))) + + types = [v for v in unique_preserve_order(types) if v] + if not types: + return "" + return " | ".join(types) + + +def list_constraint_types(constraints): + if not constraints: + return "" + + types = [] + for constraint in constraints: + constraint_dict = constraint.model_dump() if hasattr(constraint, "model_dump") else constraint + if not isinstance(constraint_dict, dict): + continue + types.append(normalize_constraint_type(constraint_dict.get("type"))) + + types = [c for c in unique_preserve_order(types) if c] + if not types: + return "" + return " | ".join(types) + + +def has_nonzero_info(value, yes_only=False): + if value is None: + return False + + if isinstance(value, (set, list, tuple, dict)): + return len(value) > 0 + + text = str(value).strip().lower() + if not text: + return False + + if yes_only: + return text in {"yes", "some", "true", "1"} + + return text not in {"no", "none", "null", "unknown", "?", "0", "false", "[]", "{}"} + + +def build_properties(item): + properties = [] + if has_nonzero_info(getattr(item, "dynamic_type", None)): + properties.append("dynamic") + if has_nonzero_info(getattr(item, "noise_type", None)): + properties.append("noisy") + if has_nonzero_info(getattr(item, "allows_partial_evaluation", None), yes_only=True): + properties.append("partial evaluations allowed") + if has_nonzero_info(getattr(item, "can_evaluate_objectives_independently", None), yes_only=True): + properties.append("independent objective evaluations") + if has_nonzero_info(getattr(item, "fidelity_levels", None)): + properties.append("multi-fidelity") + if not properties: + return "" + return " | ".join(properties) + + +def format_implementation_links(links): + if not links: + return "" + + urls = [] + for link in links: + link_dict = link.model_dump() if hasattr(link, "model_dump") else link + if isinstance(link_dict, dict): + url = str(link_dict.get("url") or "").strip() + if url: + urls.append(url) + elif link_dict: + urls.append(str(link_dict)) + + if not urls: + return "" + return " | ".join(unique_preserve_order(urls)) + + +def normalize_requirements(requirements): + if requirements is None: + return "" + if isinstance(requirements, list): + values = [str(item).strip() for item in requirements if str(item).strip()] + if not values: + return "" + return " | ".join(unique_preserve_order(values)) + return str(requirements).strip() + + +def extract_implementation_fields(implementation_ids, library_items): + result = { + "Implementation Names": "", + "Implementation Languages": "", + "Implementation Evaluation Times": "", + "Implementation Links": "", + "Implementation Descriptions": "", + "Implementation Requirements": "", + } + + if not implementation_ids: + return result + + names = [] + languages = [] + evaluation_times = [] + links = [] + descriptions = [] + requirements = [] + + for implementation_id in implementation_ids: + implementation = library_items.get(implementation_id) + if not isinstance(implementation, Implementation): + continue + + if implementation.name: + names.append(str(implementation.name).strip()) + if implementation.language: + languages.append(str(implementation.language).strip()) + if implementation.evaluation_time: + evaluation_times.append(str(implementation.evaluation_time).strip()) + if implementation.description: + descriptions.append(str(implementation.description).strip()) + + req = normalize_requirements(implementation.requirements) + if req: + requirements.append(req) + + impl_links = format_implementation_links(implementation.links) + if impl_links: + links.extend(impl_links.split(" | ")) + + result["Implementation Names"] = " | ".join(unique_preserve_order([v for v in names if v])) + result["Implementation Languages"] = " | ".join(unique_preserve_order([v for v in languages if v])) + result["Implementation Evaluation Times"] = " | ".join(unique_preserve_order([v for v in evaluation_times if v])) + result["Implementation Links"] = " | ".join(unique_preserve_order([v for v in links if v])) + result["Implementation Descriptions"] = " | ".join(unique_preserve_order([v for v in descriptions if v])) + result["Implementation Requirements"] = " | ".join(unique_preserve_order([v for v in requirements if v])) + + return result + + +def format_constraint_count(number_value): + if number_value is None: + return ">=1" + return format_dim(number_value) + + +def constraint_count_domain(number_value): + if number_value is None: + return {"kind": "range", "min": 1, "max": None} + return dim_domain(number_value) + + +def format_constraints_by_type(constraints, constraint_types): + hard_values = {constraint_type: [] for constraint_type in constraint_types} + soft_values = {constraint_type: [] for constraint_type in constraint_types} + hard_domains = [] + soft_domains = [] + total_domains = [] + + if not constraints: + return ( + {constraint_type: "" for constraint_type in constraint_types}, + {constraint_type: "" for constraint_type in constraint_types}, + "", + "", + "", + ) + + for constraint in constraints: + constraint_dict = constraint.model_dump() if hasattr(constraint, "model_dump") else constraint + if not isinstance(constraint_dict, dict): + continue + + constraint_type = normalize_constraint_type(constraint_dict.get("type")) + count_text = format_constraint_count(constraint_dict.get("number")) + count_domain = constraint_count_domain(constraint_dict.get("number")) + hardness = normalize_constraint_hard(constraint_dict.get("hard")) + if count_domain is not None: + total_domains.append(count_domain) + + if hardness == "hard": + if constraint_type in hard_values: + hard_values[constraint_type].append(count_text) + if count_domain is not None: + hard_domains.append(count_domain) + elif hardness == "soft": + if constraint_type in soft_values: + soft_values[constraint_type].append(count_text) + if count_domain is not None: + soft_domains.append(count_domain) + else: + if constraint_type in hard_values: + hard_values[constraint_type].append(count_text) + if constraint_type in soft_values: + soft_values[constraint_type].append(count_text) + if count_domain is not None: + hard_domains.append(count_domain) + soft_domains.append(count_domain) + + for constraint_type in constraint_types: + hard_values[constraint_type] = " | ".join(hard_values[constraint_type]) + soft_values[constraint_type] = " | ".join(soft_values[constraint_type]) + + hard_total = combine_domains_for_total(hard_domains) + soft_total = combine_domains_for_total(soft_domains) + total_constraints = combine_domains_for_total(total_domains) + return hard_values, soft_values, hard_total, soft_total, total_constraints + + +def normalize_scalar(value): + if value is None: + return "" + if hasattr(value, "value") and not isinstance(value, (str, int, float, bool)): + return normalize_scalar(value.value) + if isinstance(value, (str, int, float, bool)): + return value + return value + + +def normalize_recursive(value): + value = normalize_scalar(value) + if value in (None, ""): + return "" + + if hasattr(value, "model_dump"): + value = value.model_dump() + + if isinstance(value, dict): + normalized = {key: normalize_recursive(item) for key, item in value.items()} + return {key: item for key, item in normalized.items() if item not in ["", [], {}, set()]} + + if isinstance(value, set): + normalized_items = [normalize_recursive(item) for item in value] + normalized_items = [item for item in normalized_items if item not in ["", [], {}, set()]] + normalized_items = sorted(normalized_items, key=str) + if len(normalized_items) == 1: + return normalized_items[0] + return normalized_items + + if isinstance(value, list): + normalized_items = [normalize_recursive(item) for item in value] + normalized_items = [item for item in normalized_items if item not in ["", [], {}, set()]] + if len(normalized_items) == 1: + return normalized_items[0] + return normalized_items + + return value + + +def to_cell(value): + normalized = normalize_recursive(value) + if normalized in ("", None): + return "" + if isinstance(normalized, (str, int, float, bool)): + return str(normalized) + return json.dumps(normalized, ensure_ascii=False) + + +def load_library(path): + with open(path, encoding="utf-8") as yaml_input: + raw = yaml_input.read() + return parse_yaml_raw_as(Library, raw) + + +def build_problemlike_dataframe(library): + library_items = library.root if hasattr(library, "root") else {} + problemlike_items = { + item_key: item + for item_key, item in library_items.items() + if isinstance(item, ProblemLike) + } + problemlike_fields = list(ProblemLike.model_fields.keys()) + constraint_types = [constraint_type.value for constraint_type in ConstraintType if constraint_type.value != "unknown"] + + rows = [] + for item_key, item in problemlike_items.items(): + row = {"problem_id": item_key or to_problem_id(getattr(item, "name", ""))} + raw_variables = getattr(item, "variables", None) + raw_constraints = getattr(item, "constraints", None) + raw_implementations = getattr(item, "implementations", None) + variable_values, variable_total = format_variables_by_type(raw_variables) + hard_constraints, soft_constraints, hard_total, soft_total, total_constraints = format_constraints_by_type( + raw_constraints, + constraint_types, + ) + + for field in problemlike_fields: + value = getattr(item, field, None) + if field == "references": + row[field] = format_references(value) + elif field == "type": + row[field] = format_type(value) + elif field in {"variables", "constraints"}: + continue + else: + row[field] = to_cell(value) + + for variable_type in VISIBLE_VARIABLE_TYPES: + row[VARIABLE_COLUMN_NAME_BY_TYPE[variable_type]] = variable_values.get(variable_type, "") + row["Total Variables"] = variable_total + row["Variable Types"] = list_variable_types(raw_variables) + row["Constraint Types"] = list_constraint_types(raw_constraints) + row["Properties"] = build_properties(item) + row.update(extract_implementation_fields(raw_implementations, library_items)) + + for constraint_type in constraint_types: + row[f"Hard {constraint_type} Constraints"] = hard_constraints.get(constraint_type, "") + row[f"Soft {constraint_type} Constraints"] = soft_constraints.get(constraint_type, "") + row["Total Constraints"] = total_constraints + rows.append(row) + + base_columns = ["problem_id"] + [field for field in problemlike_fields if field not in {"variables", "constraints"}] + split_variable_columns = [ + "Variable Types", + "binary Variables", + "categorical Variables", + "continuous Variables", + "integer Variables", + "Total Variables", + ] + split_implementation_columns = [ + "Implementation Names", + "Implementation Languages", + "Implementation Evaluation Times", + "Implementation Links", + "Implementation Descriptions", + "Implementation Requirements", + ] + split_constraint_columns = [] + for constraint_type in constraint_types: + split_constraint_columns.append(f"Hard {constraint_type} Constraints") + split_constraint_columns.append(f"Soft {constraint_type} Constraints") + split_constraint_columns += ["Constraint Types", "Total Constraints", "Properties"] + + all_columns = base_columns + split_variable_columns + split_implementation_columns + split_constraint_columns + + # Keep id/type before name, then enforce the requested high-priority reading order. + preferred_order = [ + "problem_id", + "name", + "type", + "Variable Types", + "Total Variables", + "objectives", + "Properties", + "Constraint Types", + "Total Constraints", + "dynamic_type", + "noise_type", + "allows_partial_evaluation", + "can_evaluate_objectives_independently", + "fidelity_levels", + ] + table_columns = [column for column in preferred_order if column in all_columns] + table_columns += [column for column in all_columns if column not in table_columns] + dataframe = pd.DataFrame(rows, columns=table_columns) + dataframe = dataframe.fillna("") + return dataframe.rename(columns=display_column_name) + + +def render_table(dataframe): + linked_data = dataframe.map(linkify_cell) + table_html = linked_data.to_html( + render_links=False, + escape=False, + index=False, + table_id="problems", + classes=["display compact", "display", "styled-table"], + border=0, + na_rep="", + ) + table_html = add_problem_id_row_attributes(table_html, dataframe["ID"].astype(str).tolist()) + footer = "" + " ".join(f"{escape(column)}" for column in dataframe.columns) + "" + idx = table_html.index("") + return table_html[:idx] + footer + table_html[idx:] + + +def render_column_toggles(columns): + return "".join( + ( + f'' + ) + for index, column in enumerate(columns) + ) + + +def build_html_page(table_markup, docs_dir): + html_table = f"{docs_dir}problems.html" + html_header = f"{docs_dir}header.html" + html_scripts = f"{docs_dir}javascript.html" + html_footer = f"{docs_dir}footer.html" + html_index = f"{docs_dir}index.html" + + with open(html_table, "w", encoding="utf-8") as table_file: + table_file.write(table_markup) + + with open(html_index, "wb") as output_file: + for part_path in [html_header, html_table, html_scripts, html_footer]: + with open(part_path, "rb") as part_file: + shutil.copyfileobj(part_file, output_file) + + +if __name__ == "__main__": + yaml_file = "problems.yaml" + html_dir = "docs/" + html_table_template = f"{html_dir}table_template.html" + + try: + library = load_library(yaml_file) + except Exception as exc: + raise SystemExit(f"Error parsing YAML file '{yaml_file}': {exc}") from exc + + data = build_problemlike_dataframe(library) + final_table = render_table(data) + column_toggles = render_column_toggles(data.columns) + + with open(html_table_template, encoding="utf-8") as template_file: + table_template = template_file.read() + + table_markup = ( + table_template + .replace("__COLUMN_TOGGLES__", column_toggles) + .replace("__TABLE__", final_table) + ) + build_html_page(table_markup, html_dir)