Generate a suite with one writer per use case, and persona values the platform understands

harness-ui/server.py

@@ -460,6 +460,20 @@ async def world():
         db.close()
 
 
+@app.get("/api/generation")
+async def generation(session: str = ""):
+    """What the suite generation is doing right now, so the page can draw it while it runs.
+
+    Polled rather than streamed: the page may be opened halfway through a suite, refreshed, or
+    opened somewhere else entirely, and each of those has to show the same thing. Empty when
+    nothing has been generated here, which the page reads as "no fan-out to show".
+    """
+    from fi.alk.harness import progress
+
+    out = _folder(session)
+    return progress.read(out) if out else {}
+
+
 @app.get("/api/scenarios")
 async def scenarios():
     """Every scenario, with its files and its three gates re-run.

src/fi/alk/harness/persona_guides.py

@@ -0,0 +1,199 @@
+"""The behaviour guidance the platform already uses for a simulated caller.
+
+A persona profile names what somebody is like: impatient and direct, cautious and skeptical. It
+does not say how that should sound turn by turn, and a model handed only the label improvises
+one, which is how "in a hurry" became a caller who says it every turn instead of a caller who
+cuts in once and accepts the first workable answer.
+
+The platform solved that with lookup tables mapping each value to a sentence of guidance, and
+voice simulation has run on them for months. They are read from there rather than restated here,
+because two copies of the same wording drift and then a caller behaves one way on the platform
+and another way through the harness, for reasons nobody can see.
+
+Read, not imported: the tables live inside a Django app this package cannot import, but they are
+plain literals, so they are parsed out of the file. Absent, every lookup answers with nothing and
+a persona still renders — one without guidance, never a crash.
+"""
+
+from __future__ import annotations
+
+import ast
+import os
+from functools import lru_cache
+from pathlib import Path
+
+# Where the platform's tables are mounted. Colon-separated so voice and chat guides can both be
+# offered; the first file defining a table wins, so voice takes precedence when both are present.
+GUIDES_ENV = "HARNESS_PERSONA_GUIDES"
+
+WANTED = (
+    "VOICE_PERSONALITY_GUIDES",
+    "VOICE_COMMUNICATION_STYLE_GUIDES",
+    "CHAT_PERSONALITY_GUIDES",
+    "CHAT_COMMUNICATION_STYLE_GUIDES",
+    "CHAT_TONE_GUIDES",
+    "CHAT_VERBOSITY_GUIDES",
+)
+
+
+def _tables_in(path: Path) -> dict[str, dict[str, str]]:
+    """Every guidance table defined in one file, by name.
+
+    Parsed rather than executed. The file sits in an app with imports this process cannot
+    satisfy, and running it to read a dictionary would fail for reasons that have nothing to do
+    with the dictionary.
+    """
+    found: dict[str, dict[str, str]] = {}
+    try:
+        tree = ast.parse(path.read_text(encoding="utf-8"))
+    except (OSError, SyntaxError):
+        return found
+    for node in tree.body:
+        targets = (
+            [node.target] if isinstance(node, ast.AnnAssign) else getattr(node, "targets", [])
+        )
+        for target in targets:
+            name = getattr(target, "id", "")
+            if name not in WANTED or node.value is None:
+                continue
+            try:
+                value = ast.literal_eval(node.value)
+            except ValueError:
+                continue
+            if isinstance(value, dict) and value:
+                found[name] = {str(k).lower(): str(v) for k, v in value.items()}
+    return found
+
+
+@lru_cache(maxsize=1)
+def guides() -> dict[str, dict[str, str]]:
+    """Every table the platform offers this harness, merged."""
+    merged: dict[str, dict[str, str]] = {}
+    for raw in (os.environ.get(GUIDES_ENV) or "").split(":"):
+        if not raw.strip():
+            continue
+        for name, table in _tables_in(Path(raw.strip())).items():
+            merged.setdefault(name, table)
+    return merged
+
+
+def guidance_for(kind: str, value: str, *, voice: bool = True) -> str:
+    """The platform's sentence for one persona value, or nothing.
+
+    ``kind`` is ``personality``, ``communication_style``, ``tone`` or ``verbosity``. Voice tables
+    are preferred for a spoken call and the chat table is the fallback, because the two describe
+    the same disposition and only one of them is written for speech.
+    """
+    if not value.strip():
+        return ""
+    tables = guides()
+    order = ("VOICE", "CHAT") if voice else ("CHAT", "VOICE")
+    for prefix in order:
+        table = tables.get(f"{prefix}_{kind.upper()}_GUIDES") or {}
+        found = table.get(value.strip().lower())
+        if found:
+            return found
+    return ""
+
+
+def available() -> bool:
+    """Whether any guidance was found, so a build can say so rather than silently omitting it."""
+    return bool(guides())
+
+
+# Where the platform's persona model is mounted, for the values it accepts.
+VOCABULARY_ENV = "HARNESS_PERSONA_VOCABULARY"
+
+# The persona fields worth constraining, and the choice class each is drawn from. Only the ones
+# that change behaviour or routing: a free-text occupation harms nothing, an accent nobody
+# recognises silently loses the voice it was supposed to select.
+FIELDS = {
+    "gender": "GenderChoices",
+    "age_group": "AgeGroupChoices",
+    "occupation": "ProfessionChoices",
+    "location": "LocationChoices",
+    "personality": "PersonalityChoices",
+    "communication_style": "CommunicationStyleChoices",
+    "accent": "AccentChoices",
+    "languages": "LanguageChoices",
+}
+
+# Constrained because something downstream reads them. The rest are offered as vocabulary but a
+# writer who needs a value outside them is not stopped: an unknown occupation costs nothing,
+# an unknown accent costs the voice.
+ENFORCED = ("personality", "communication_style", "accent", "languages")
+
+
+@lru_cache(maxsize=1)
+def vocabulary() -> dict[str, list[str]]:
+    """What the platform accepts for each persona field.
+
+    Parsed out of the model's ``TextChoices`` classes for the same reason the guidance is read
+    rather than restated: the platform is the one that has to understand these values, so it is
+    the one that decides what they are. A persona written in words of its own renders fine, gets
+    no behaviour guidance, and cannot be grouped with anything on the platform afterwards.
+    """
+    path = os.environ.get(VOCABULARY_ENV) or ""
+    if not path or not Path(path).exists():
+        return {}
+    try:
+        tree = ast.parse(Path(path).read_text(encoding="utf-8"))
+    except (OSError, SyntaxError):
+        return {}
+
+    by_class: dict[str, list[str]] = {}
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.ClassDef):
+            continue
+        values: list[str] = []
+        for item in node.body:
+            if not isinstance(item, ast.Assign):
+                continue
+            try:
+                held = ast.literal_eval(item.value)
+            except ValueError:
+                continue
+            # ``NAME = "value", "Label"`` is the choices shape; a bare string is also accepted.
+            if isinstance(held, tuple) and held and isinstance(held[0], str):
+                values.append(held[0])
+            elif isinstance(held, str):
+                values.append(held)
+        if values:
+            by_class[node.name] = values
+
+    return {
+        field: by_class[cls] for field, cls in FIELDS.items() if by_class.get(cls)
+    }
+
+
+def offered(field: str) -> list[str]:
+    """The values this field accepts, or nothing if the platform's model was not readable."""
+    return list(vocabulary().get(field, []))
+
+
+def unrecognised(persona: dict[str, object]) -> list[str]:
+    """Persona values the platform would not recognise, as sentences saying what to use instead.
+
+    Only the fields something downstream actually reads, and only when the vocabulary was found:
+    a harness that cannot see the platform's model must not start refusing personas over it.
+    """
+    known = vocabulary()
+    if not known:
+        return []
+    problems: list[str] = []
+    for field in ENFORCED:
+        allowed = known.get(field) or []
+        if not allowed:
+            continue
+        held = persona.get(field)
+        values = held if isinstance(held, list) else ([held] if held else [])
+        lowered = {str(one).strip().lower() for one in allowed}
+        for one in values:
+            text = str(one).strip()
+            if text and text.lower() not in lowered:
+                problems.append(
+                    f"persona {field} {text!r} is not one the platform knows, so it will not "
+                    f"reach the call. Use one of: {', '.join(allowed)}. Anything else this "
+                    "person is like belongs in persona.metadata."
+                )
+    return problems

src/fi/alk/harness/progress.py

@@ -0,0 +1,113 @@
+"""What the fan-out is doing right now, written where a UI can read it.
+
+Generating a suite in parallel is the one thing this harness does where nothing appears for
+several minutes and then everything appears at once. Told nothing, a person cannot tell a
+working run from a hung one, and the honest answer to "is it stuck" is the only thing they want.
+
+So the fan-out writes its own state as it goes: which use cases it split the work into, which
+are running, how many scenarios each has proved, and which have finished. A file rather than a
+stream, because the reader is a page that may be opened halfway through, refreshed, or opened on
+another machine, and each of those has to show the same thing.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+from pathlib import Path
+from typing import Any
+
+PROGRESS = "generation.json"
+
+WAITING = "waiting"
+RUNNING = "running"
+DONE = "done"
+FAILED = "failed"
+
+
+def _path(destination: Path) -> Path:
+    return Path(destination) / PROGRESS
+
+
+def _write(destination: Path, state: dict[str, Any]) -> None:
+    """Replace the file atomically.
+
+    A reader polling this will otherwise catch a half-written file and show nothing, which looks
+    exactly like the failure it is meant to rule out.
+    """
+    path = _path(destination)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    handle, temporary = tempfile.mkstemp(dir=str(path.parent), suffix=".tmp")
+    try:
+        with os.fdopen(handle, "w", encoding="utf-8") as writing:
+            json.dump(state, writing, indent=2)
+        os.replace(temporary, path)
+    except BaseException:
+        Path(temporary).unlink(missing_ok=True)
+        raise
+
+
+def read(destination: Path) -> dict[str, Any]:
+    """The current state, or nothing if no suite has been generated here."""
+    path = _path(destination)
+    if not path.exists():
+        return {}
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return {}
+
+
+def planned(
+    destination: Path, allocation: list[tuple[str, int]], *, at_once: int, asked: int
+) -> None:
+    """The split, before any of it starts. Written first so the tree appears immediately."""
+    _write(
+        destination,
+        {
+            "state": RUNNING,
+            "asked": asked,
+            "at_once": at_once,
+            "kept": 0,
+            "slices": [
+                {"use_case": case, "wanted": count, "kept": 0, "state": WAITING}
+                for case, count in allocation
+            ],
+        },
+    )
+
+
+def _change(destination: Path, use_case: str, **fields: Any) -> None:
+    state = read(destination)
+    for slice_ in state.get("slices", []):
+        if slice_.get("use_case") == use_case:
+            slice_.update(fields)
+            break
+    state["kept"] = sum(one.get("kept", 0) for one in state.get("slices", []))
+    _write(destination, state)
+
+
+def started(destination: Path, use_case: str) -> None:
+    _change(destination, use_case, state=RUNNING)
+
+
+def kept(destination: Path, use_case: str, count: int) -> None:
+    """How many this slice has proved so far. Called as they land, not at the end."""
+    _change(destination, use_case, kept=count)
+
+
+def finished(destination: Path, use_case: str, count: int) -> None:
+    _change(destination, use_case, state=DONE, kept=count)
+
+
+def failed(destination: Path, use_case: str, why: str) -> None:
+    _change(destination, use_case, state=FAILED, why=why[:300])
+
+
+def settled(destination: Path, *, kept_total: int) -> None:
+    """The whole fan-out is over and the suite is written."""
+    state = read(destination)
+    state["state"] = DONE
+    state["kept"] = kept_total
+    _write(destination, state)

src/fi/alk/harness/scenario.py

@@ -154,6 +154,11 @@ class Scenario(BaseModel):
 
     name: str
     use_case: str = ""
+    # Which branch of that use case this is: the condition that makes this row different from
+    # its siblings. A use case fans out into several — the ordinary path, the one that cannot be
+    # completed, the rule under pressure — and each is its own test. Coverage is counted on the
+    # pair, so a use case can carry many scenarios without any of them reading as a duplicate.
+    branch: str = ""
     tests: str = ""
 
     # What this scenario changes about the world after it is reset, as code: a file defining
@@ -232,6 +237,13 @@ def validate_scenario(
         missing := scenario.persona.missing_profile_fields()
     ):
         problems.append("persona is incomplete: " + ", ".join(missing))
+    elif scenario.persona is not None:
+        # A persona written in words of its own renders fine and then does nothing: no behaviour
+        # guidance attaches to it, and the accent it names selects no voice. Caught here, where
+        # the writer is still holding the scenario and can fix it in one turn.
+        from .persona_guides import unrecognised
+
+        problems.extend(unrecognised(scenario.persona.model_dump()))
     if not scenario.sub_goals:
         problems.append(
             "no sub_goals: nothing would be graded. Name the entries of the catalogue this "