Add reproducible scheduling benchmark and experimental comparison

.github/workflows/checks.yml

@@ -13,7 +13,7 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-      - run: python -m pip install -e '.[dev]'
+      - run: python -m pip install -e '.[dev,benchmark]'
       - run: python -m pytest
       - run: ruff check .
       - run: ruff format --check .

.gitignore

@@ -218,3 +218,6 @@ __marimo__/
 
 # Streamlit
 .streamlit/secrets.toml
+
+# Default local benchmark output; reference results live in benchmarks/default.
+benchmark-results/

README.md

@@ -4,12 +4,13 @@ A lightweight Python project for experimenting with adaptive background job
 scheduling. Includes a discrete-time, single-worker simulator, FIFO, SPT, EDF,
 Priority, and seeded Random baselines, and a tabular Q-Learning meta-scheduler.
 
-Requires Python 3.10 or newer. There are no runtime dependencies.
+Requires Python 3.10 or newer. Simulation and learning have no runtime
+dependencies; benchmark figures use the optional Matplotlib extra.
 
 ```sh
 python -m venv .venv
 source .venv/bin/activate
-python -m pip install -e '.[dev]'
+python -m pip install -e '.[dev,benchmark]'
 ```
 
 Run an episode with 60 jobs:
@@ -35,6 +36,17 @@ Evaluation defaults to seeds following the saved training range, with exploratio
 disabled. Use `--seed` to choose a different disjoint range. Both commands accept
 the workload options below; `train --help` lists learning and exploration settings.
 
+Run the fixed paired benchmark (three independently trained models and all five
+baselines, across three workloads and 30 evaluation seeds):
+
+```sh
+adaptivequeue benchmark --output-dir benchmark-results
+```
+
+Models are cached in the output directory; `--retrain` regenerates them.
+See [experimental results and reproduction details](docs/experiments.md) for
+the observed comparisons and CSV definitions.
+
 ```sh
 python -m adaptivequeue --workload bursty --policy spt --seed 7 \
   --jobs 60 --arrival-rate 0.3 --deadline-factor 3
@@ -46,5 +58,5 @@ mypy
 
 See [simulation rules and parameters](docs/simulation.md) or run
 `adaptivequeue --help`. The Python API also exposes per-job execution records.
-The current scope is simulation, classical baselines, and CPU-only tabular
-Q-Learning; no additional runtime dependencies are needed.
+The current scope is simulation, classical baselines, CPU-only tabular
+Q-Learning, and reproducible benchmarking.

benchmarks/default/action_distribution.csv

@@ -0,0 +1,37 @@
+workload,training_seed,action,policy,selections,fraction,contended_selections,contended_fraction
+stable,42,0,fifo,191,0.10611111111111111,73,0.05615384615384615
+stable,42,1,spt,677,0.3761111111111111,590,0.45384615384615384
+stable,42,2,edf,263,0.1461111111111111,222,0.17076923076923076
+stable,42,3,priority,669,0.37166666666666665,415,0.3192307692307692
+stable,2042,0,fifo,202,0.11222222222222222,141,0.10854503464203233
+stable,2042,1,spt,377,0.20944444444444443,358,0.27559661277906083
+stable,2042,2,edf,818,0.45444444444444443,481,0.37028483448806776
+stable,2042,3,priority,403,0.2238888888888889,319,0.2455735180908391
+stable,4042,0,fifo,197,0.10944444444444444,184,0.1399239543726236
+stable,4042,1,spt,545,0.30277777777777776,522,0.39695817490494295
+stable,4042,2,edf,533,0.2961111111111111,266,0.20228136882129277
+stable,4042,3,priority,525,0.2916666666666667,343,0.26083650190114066
+bursty,42,0,fifo,36,0.02,30,0.018115942028985508
+bursty,42,1,spt,969,0.5383333333333333,949,0.5730676328502415
+bursty,42,2,edf,316,0.17555555555555555,231,0.13949275362318841
+bursty,42,3,priority,479,0.26611111111111113,446,0.2693236714975845
+bursty,2042,0,fifo,329,0.1827777777777778,256,0.15440289505428226
+bursty,2042,1,spt,741,0.4116666666666667,730,0.44028950542822676
+bursty,2042,2,edf,428,0.23777777777777778,395,0.2382388419782871
+bursty,2042,3,priority,302,0.16777777777777778,277,0.16706875753920386
+bursty,4042,0,fifo,148,0.08222222222222222,142,0.08559373116335142
+bursty,4042,1,spt,826,0.4588888888888889,768,0.4629294755877034
+bursty,4042,2,edf,338,0.18777777777777777,289,0.17420132610006028
+bursty,4042,3,priority,488,0.27111111111111114,460,0.27727546714888485
+mixed,42,0,fifo,190,0.10555555555555556,130,0.09537784299339692
+mixed,42,1,spt,642,0.3566666666666667,549,0.4027879677182685
+mixed,42,2,edf,209,0.11611111111111111,164,0.12032281731474688
+mixed,42,3,priority,759,0.4216666666666667,520,0.3815113719735877
+mixed,2042,0,fifo,261,0.145,155,0.1151560178306092
+mixed,2042,1,spt,517,0.2872222222222222,478,0.3551263001485884
+mixed,2042,2,edf,640,0.35555555555555557,401,0.29791976225854383
+mixed,2042,3,priority,382,0.21222222222222223,312,0.23179791976225855
+mixed,4042,0,fifo,240,0.13333333333333333,237,0.17400881057268722
+mixed,4042,1,spt,452,0.2511111111111111,441,0.3237885462555066
+mixed,4042,2,edf,683,0.3794444444444444,382,0.28046989720998533
+mixed,4042,3,priority,425,0.2361111111111111,302,0.22173274596182085