Expand Scientific Regression Coverage for SIST 1.0.0

tests/conftest.py

@@ -11,11 +11,19 @@
 
 REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
 
+SIST_TRANSITIONS = (
+    pytest.param(("melting", "M"), id="melting"),
+    pytest.param(("z-dna", "Z"), id="z-dna"),
+    pytest.param(("cruciform", "C"), id="cruciform"),
+)
+
 
 @dataclass(frozen=True)
-class CompetitionRun:
-    """Result of running the SIST competition example."""
+class SistRun:
+    """Result of running a SIST calculation."""
 
+    name: str
+    algorithm: str
     process: subprocess.CompletedProcess[str]
     output_path: Path
 
@@ -64,7 +72,7 @@ def built_sist(tmp_path_factory: pytest.TempPathFactory) -> Path:
     Copy and build SIST in a temporary directory.
 
     This ensures the tests use executables built from the maintained source
-    without overwriting the inherited binaries in the repository.
+    without modifying the repository working tree.
     """
 
     build_root = tmp_path_factory.mktemp("sist-build")
@@ -113,19 +121,23 @@ def built_sist(tmp_path_factory: pytest.TempPathFactory) -> Path:
     return working_copy
 
 
-@pytest.fixture(scope="session")
-def competition_run(built_sist: Path) -> CompetitionRun:
-    """Run the documented competition example once."""
+def run_sist_calculation(
+    built_sist: Path,
+    *,
+    name: str,
+    algorithm: str,
+) -> SistRun:
+    """Run one SIST calculation using the regression test sequence."""
 
     source_input = built_sist / "tests" / "data" / "pbr322.toy.fa"
     runtime_input = built_sist / "pbr322.toy.fa"
 
     shutil.copy2(source_input, runtime_input)
 
     output_directory = built_sist / "test-results"
-    output_directory.mkdir()
+    output_directory.mkdir(exist_ok=True)
 
-    output_path = output_directory / "competition.txt"
+    output_path = output_directory / f"{name}.txt"
 
     result = run_command(
         [
@@ -134,7 +146,7 @@ def competition_run(built_sist: Path) -> CompetitionRun:
             "-f",
             runtime_input.name,
             "-a",
-            "A",
+            algorithm,
             "-o",
             str(output_path.relative_to(built_sist)),
             "-b",
@@ -144,7 +156,39 @@ def competition_run(built_sist: Path) -> CompetitionRun:
         cwd=built_sist,
     )
 
-    return CompetitionRun(
+    return SistRun(
+        name=name,
+        algorithm=algorithm,
         process=result,
         output_path=output_path,
     )
+
+
+@pytest.fixture(scope="session")
+def competition_run(built_sist: Path) -> SistRun:
+    """Run the SIST competition calculation once."""
+
+    return run_sist_calculation(
+        built_sist,
+        name="competition",
+        algorithm="A",
+    )
+
+
+@pytest.fixture(
+    scope="session",
+    params=SIST_TRANSITIONS,
+)
+def transition_run(
+    request: pytest.FixtureRequest,
+    built_sist: Path,
+) -> SistRun:
+    """Run each supported individual SIST transition calculation once."""
+
+    name, algorithm = request.param
+
+    return run_sist_calculation(
+        built_sist,
+        name=name,
+        algorithm=algorithm,
+    )

tests/reference/v1.0.0/README.md

@@ -1,34 +1,106 @@
-# SIST 1.0.0 competition reference
+# SIST 1.0.0 scientific regression references
 
-This directory contains reference output for the SIST competition calculation.
+This directory contains the reference outputs used to protect the established scientific behaviour of SIST for the 1.0.0 release.
+
+The regression suite covers the individual melting, Z-DNA, and cruciform calculations as well as the competition calculation.
 
 ## Baseline
 
-* Commit: `dab84fd57f81c3bbb1bb26bd7d4a2fa9674d8a2d`
-* Input: `tests/data/pbr322.toy.fa`
-* Algorithm: competition (`A`)
+All maintained reference calculations use:
+
+- Input: `tests/data/pbr322.toy.fa`
+- Melting algorithm: M
+- Z-DNA algorithm: Z
+- Cruciform algorithm: C
+- Competition algorithm: A
+
+The reference outputs were generated from executables rebuilt from the maintained SIST source.
+
+Repeated calculations produced identical deterministic output. The Run time value is excluded from regression comparison because it varies between executions.
+
+## Commands
+
+### Melting
+
+    perl master.pl \
+        -f pbr322.toy.fa \
+        -a M \
+        -o melting.txt \
+        -b \
+        -p \
+        -r
+
+### Z-DNA
+
+    perl master.pl \
+        -f pbr322.toy.fa \
+        -a Z \
+        -o z-dna.txt \
+        -b \
+        -p \
+        -r
+
+### Cruciform
+
+    perl master.pl \
+        -f pbr322.toy.fa \
+        -a C \
+        -o cruciform.txt \
+        -b \
+        -p \
+        -r
 
-## Command
+### Competition
 
-```bash
-perl master.pl \
-    -f pbr322.toy.fa \
-    -a A \
-    -o pbr322.toy.compete.txt \
-    -b \
-    -p \
-    -r
-```
+    perl master.pl \
+        -f pbr322.toy.fa \
+        -a A \
+        -o competition.txt \
+        -b \
+        -p \
+        -r
 
 ## Files
 
-* `competition.rebuilt.txt` - output produced from executables rebuilt from the maintained source. This is the regression-test reference.
-* `competition.inherited.txt` - output produced by the executable originally included in the repository. This is retained for historical comparison.
+- melting.txt - maintained reference output for the melting calculation.
+- z-dna.txt - maintained reference output for the Z-DNA calculation.
+- cruciform.txt - maintained reference output for the cruciform calculation.
+- competition.rebuilt.txt - maintained reference output for the competition calculation.
+- competition.inherited.txt - output produced by the competition executable originally included in the repository. This is retained for historical provenance and is not the maintained regression reference.
+
+## Scientific regression comparison
+
+The regression suite compares deterministic scientific output against these references with zero relative and absolute tolerance.
+
+For individual transition calculations, the comparison includes:
+
+- Reported deterministic scientific metrics.
+- Every sequence position.
+- The base reported at every position.
+- P(x) at every position.
+- G(x) where it is produced by the calculation.
+
+For the competition calculation, the comparison includes:
+
+- Reported deterministic scientific metrics.
+- Every sequence position.
+- The base reported at every position.
+- P_melt, P_Z, and P_cruciform at every position.
+
+The structure of the scientific profile output is also checked so that unexpected changes to the reported columns cause the regression tests to fail.
+
+The variable Run time line is deliberately ignored because execution time is not scientific output.
+
+## Historical competition output
+
+The maintained competition output and the inherited executable output matched exactly for all values common to both, excluding the variable Run time line.
+
+The inherited executable additionally prints 24 derived probability fields that are not printed by the rebuilt maintained executable. These historical fields are retained for provenance but are not part of the SIST 1.0.0 regression contract.
 
-All values common to both executables matched exactly, excluding the variable `Run time` line.
+## Updating references
 
-The inherited executable also prints 24 additional derived probability fields that are not printed by the rebuilt executable. These are being investigated separately and are not part of the initial regression comparison.
+Reference outputs must not be updated solely to make a failing regression test pass.
 
-## Regression comparison
+Any numerical difference from these baselines should be investigated and understood before a reference is changed.
 
-The test compares the deterministic calculation results and ignores the `Run time` line.
+Intentional scientific changes should document why the established SIST 1.0.0 behaviour has changed.