diff --git a/eval/PREREG-desk-11.md b/eval/PREREG-desk-11.md new file mode 100644 index 00000000..c445b762 --- /dev/null +++ b/eval/PREREG-desk-11.md @@ -0,0 +1,81 @@ +# Round 11: round 10 again, on a Solid with every eval bug fixed and the current guide + +Pre-registration, committed before any round 11 agent runs. Status: FROZEN (Ryan, 2026-11-10: "freeze"). + +## Why + +Round 10 (`results/desk-2026-11-02`) confirmed H10a: small .rx desks had 6.7 defects, small React desks 14.7. It +rejected H10c: frontier React desks had 3.0. Most of the .rx frontier defects traced to Solid bugs, and those have +all been fixed since (#4627 #4629 #4631 #4638 #4661 #4666 #4689). Regrading round 10's own apps on the fixed Solid +moved frontier .rx from 7.3 to 4.7 raw defects. Since round 10 the guide has gained `guide/reference.md`, the rules on +one page (PR #29), and the instant model (PR #34). A 3-desk check with `reference.md` (`results/cost-2026-11-03`, not +counted) gave frontier .rx 2.0 defects. Round 11 asks whether that holds with fresh agents, all four arms, both tiers, +and the same design as round 10, so the two rounds compare directly. + +Round 10's plan named maintenance as round 11. It moves to round 12 and runs on round 11's apps, so the apps it extends +were written with the current guide on a fixed Solid. + +## Task and arms (as round 10 unless stated) + +- Task: `tasks/desk9` with `brief-plain.md`. **One change from round 10**: the detail's `retry` button is now always + present while an issue is selected, not only after a failure. In round 10, 25 of 36 desks across all arms failed + "a detail that loads while a rename is out keeps the new title" because the brief left this unclear. Graded by + `hidden.mjs` (77) and `hidden-overlap.mjs` (38), unchanged. +- Arms: `rx` with `--inline-docs` (now `language.md`, `async.md` and `reference.md`), `foldkit` with `--inline-docs`, + `react`, `vanilla`. `rxc` is built from the commit that holds this file once it is frozen. +- Solid: `solid-next` at `next` 6751472f (2026-11-10; includes the fixes for #4689, #4724, #4737, #4738, #4739, #4748 + and #4751, none yet in an rc), built from source. The same build is used for every .rx run and grade. rc.22 isn't out; if it ships before the round starts and + holds every fix, it replaces this pin. +- Harness: subagents, as in round 10. Numbers are comparable only within one harness: on the Foldkit side check + (2026-11-08), the CLI harness alone moved Foldkit from 4.7 to 2.7 defects. + +## Runs, grading + +As round 10: 3 frontier and 6 small runs per arm (36), one interleaved batch, at most 15 `./check`/`./test` calls, +graded by `harness/grade.sh`. The definitions of correct and cost are round 10's. **Only defects are counted**: no +legal hold and no failure the brief caused is in any count or table. + +- A legal hold is a consistent hold that settles (2026-10-30 ruling). The one hidden test that asserted one (overlap, + "two issues assigned at once": it required the declined row to revert while the other assignment was still out, + which a list held whole until both settle does not do) now checks only what is a defect, that the decline does not + take the other row with it and that both rows are right once everything has answered. Committed with this file. +- A fuzz hole counts unless it is traced to a legal hold or to behaviour Solid's rules make legal; each exclusion is + listed with its trace. +- Round 10's numbers, wherever they are compared with round 11's, are regraded with the same tests, and without "a + detail that loads while a rename is out keeps the new title", which round 10's brief caused (above). + +## Hypotheses and tests + +- **H11a** (primary, replication of H10a): small .rx runs have fewer defects than small React runs. One-sided + permutation test on mean defects, 100,000 relabellings, seed 11. +- **H11b**: small .rx has fewer defects than small plain, and fewer than small Foldkit. Same test, Holm-corrected with + H11a. +- **H11c**: frontier .rx runs have no more defects than frontier React runs. Difference in means with a 90% bootstrap + interval (10,000 resamples, seed 11); the claim holds if the upper end is under 1 defect. (Round 10's H10c compared + small .rx with frontier React. That comparison is reported again, as H10c was.) +- **Cost per correct app** (descriptive), by arm and tier. In round 10 no app was correct, so this was undefined. +- Reported alongside: the change from round 10 for each arm and tier, defects by kind, turns, cost per run, and lines. + +The direction is the claim being tested. A null or reversed result is reported as plainly. + +## Gate before freezing + +The 12 existing .rx desks (round 10 1-9, cost check 1-3) are regraded on the pinned Solid. A regression against +rc.21 is filed upstream and fixed or recorded before the round starts. + +Result (2026-11-08): on 2789ca9c one desk had a new deterministic hole (round 10 rx-8, fuzz seed 3: a close guess +during another action's refresh, then the refresh fails). It came from the lowering, not from the app: a whole guess +(`optimistic rows := rs => rs.map(...)`) was written into the draft row by row, which replaces each matched row. It now +lowers to the setter's return, which Solid diffs shallowly against the view as guesses (its default). With that, the +other 11 desks grade as before the change (52 defects over the 12; round 10 rx-9 fuzz seed 8, a known hole, is +deterministic on both lowerings and was missed once by timing), and seed 3 is gone. The plain Solid case is solidjs/solid#4724. + +Result (2026-11-10), repinned to 6751472f: regrading on the intermediate next builds (c9816808, 79b6541d) turned up +five more Solid bugs, each reduced to a plain Solid repro and fixed upstream (#4724 by #4729, #4737/#4738/#4739 by +#4740, #4748 by #4749, #4751 by #4753). On 6751472f the 12 desks total 51 defects, against 52 on 2789ca9c: no desk +gained a hole, and round 10 rx-6 fuzz seed 12 is gone. The gate passes. + +## Cost + +About $30 at round 10's prices: frontier $2.90 (.rx), $1.35 (React), $1.24 (plain), $3.56 (Foldkit) per run; +small under $0.16. diff --git a/eval/results/desk-2026-11-11/README.md b/eval/results/desk-2026-11-11/README.md new file mode 100644 index 00000000..f366616f --- /dev/null +++ b/eval/results/desk-2026-11-11/README.md @@ -0,0 +1,110 @@ +# Round 11 results: defects and cost by model tier, four approaches (2026-11-11) + +**Pre-registration:** `eval/PREREG-desk-11.md`, frozen at b8b9e0b. + +**Runs:** 36, which is 4 arms × (3 frontier + 6 small). Frontier runs used the current Opus model, small runs the current Haiku model; the model is read from each transcript. + +**Toolchain:** Solid `next` 6751472f, built from source; rxc from b8b9e0b. + +**Grading:** `harness/grade.sh` (hidden 77, overlap 38, 20 fuzz seeds). Defects are failed hidden tests, plus failed overlap tests, plus fuzz seeds that found a hole. Only defects are counted. No fuzz hole was traced to a legal hold, so nothing is excluded. + +## Defects per app + +| arm | frontier mean | small mean | small per desk | +|---|---|---|---| +| .rx (inline docs) | 2.0 | **3.8** | 3, 3, 4, 2, 9, 2 | +| React | 2.0 | 9.7 | 13, 2, 17, 14, 1, 11 | +| plain TypeScript | 3.0 | 14.5 | 12, 16, 11, 9, 20, 19 | +| Foldkit (inline docs) | 2.7 | 13.2 | 17, 13, 13, 12, 10, 14 | + +Frontier per desk: .rx 0, 5, 1; React 0, 2, 4; plain 3, 3, 3; Foldkit 4, 2, 2. + +## Tests + +All tests are on the small tier unless named otherwise; full output is in `stats.txt`. + +- **H11a** (.rx < React): permutation p = 0.047, Holm 0.047. **Supported.** +- **H11b** (.rx < plain): p = 0.0021, Holm 0.0041. **Supported.** +- **H11b** (.rx < Foldkit): p = 0.0010, Holm 0.0031. **Supported.** +- **H11c** (frontier .rx no worse than frontier React): difference 0.0, 90% bootstrap interval [−2.3, 2.3]. The upper end is not under 1, so it is **not shown**. Both arms average 2.0, but three runs each are too few. +- **H10c again** (small .rx no worse than frontier React): +1.8, interval [−0.3, 4.2]. **Not shown.** + +## Correct apps and cost + +Two apps have no defect, the first in any round: frontier `.rx-1` and frontier `React-1`. + +**Cost per correct app:** frontier .rx $6.14 and frontier React $4.15 (each arm's three runs over its one correct app). Every other cell has no correct app, so its cost per correct app is undefined. + +Cost per run is billed tokens (`cost.json`) at round 10's rates (`usd.json`): + +| arm | frontier $/run (round 10) | small $/run (round 10) | frontier calls | small calls | +|---|---|---|---|---| +| .rx | **2.05** (2.90) | 0.092 (0.085) | 21.3 | 15.5 | +| React | 1.38 (1.35) | 0.069 (0.066) | 12.7 | 11.8 | +| plain | 1.27 (1.24) | 0.077 (0.082) | 12.3 | 10.0 | +| Foldkit | 3.63 (3.56) | 0.175 (0.155) | 37.3 | 38.5 | + +Frontier .rx cost about 30% less than in round 10. The other arms cost about the same. + +## Change from round 10, same tests + +Round 10's desks were regraded on this round's toolchain and tests (`r10-regraded/`). As pre-registered, the rename-reload test that round 10's brief caused is dropped from those numbers. + +| arm, tier | round 10 regraded | round 11 | +|---|---|---| +| .rx frontier | 2.7 | 2.0 | +| .rx small | 4.7 | 3.8 | +| React frontier | 3.0 | 2.0 | +| React small | 13.7 | 9.7 | +| plain frontier | 2.7 | 3.0 | +| plain small | 8.8 | 14.5 | +| Foldkit frontier | 4.0 | 2.7 | +| Foldkit small | 13.5 | 13.2 | + +Round 10 as first published had .rx at 6.7 in both tiers. The Solid fixes since then account for most of the drop to 2.7 and 4.7. + +## Defects by kind + +- **.rx:** no failure repeats in more than two desks. The overlap tests that fail in most desks of the other arms ("labels / comments that load while an add is out keep it") fail in only one .rx desk (rx-2). +- **React, plain, Foldkit (small):** most defects are overlap. Each fails 7 to 11 of 38 overlap tests on average: loads that land while a change is out drop the change, and late saves overwrite newer ones. + +### .rx fuzz holes, traced + +- **rx-2, seeds 1, 3, 18: a Solid bug.** A keyed projection whose function returns another keyed projection loses a row when the question switches while a guess is out. In a browser this shows as a duplicated row. The plain Solid repro is filed as solidjs/solid#4762. It counts against .rx, as pre-registered. +- **rx-5, rx-6, rx-7, the "row assignee" holes: agent mistakes.** The assign guess is written only when the row is in the list on screen (`if (i >= 0)`), so a row hidden by a close guess never gets it. +- **rx-5 and rx-8, the "detail title" holes: agent mistakes.** rx-5 seed 15 was replayed; rx-8 has the same code shape and was not replayed. The rename guess is written to a source derived from the detail before the detail has loaded. Solid drops a guess that was never shown (D-34), so the old title stays on screen while the rename is out. +- **rx-4 seed 15 (detail title): agent mistake.** Rename guesses are kept in their own list, and the heading reads it with `find`, which returns the oldest guess. With two renames out, the heading shows the first one, while the row shows the second. + +## Lines of source per app + +Non-blank lines, without the given `api` and `flights` files. + +| arm | frontier | small | +|---|---|---| +| .rx | 414 | 373 | +| React | 784 | 744 | +| plain | 873 | 967 | +| Foldkit | 2276 | 2340 | + +## Deviations + +- foldkit-6 and foldkit-7 ended without a final report and were graded as they stood. +- rx-5 used 16 check/test runs, one over the cap of 15. +- react-5 listed the file names in `eval/` (it read no contents) and said so. +- Three agents (rx-1, rx-2, react-4) added temporary visible tests and then restored the file. +- None of these affects grading. + +## Reading + +- **Small tier:** the round 10 result replicates. Small .rx writes fewer than half of React's defects and about a quarter of plain's and Foldkit's. +- **Frontier tier:** all four arms are now close, between 2 and 3 defects. .rx is level with React, but three runs per arm cannot show "no worse". +- **Cost:** frontier .rx is still dearer than React and plain ($2.05 against about $1.30) but cheaper than Foldkit. With 3 of its 6 frontier defects from one Solid bug, the frontier .rx count is partly Solid's still. + +## Files + +- `summary.txt`: per desk. +- `graded/`: `grade.sh` output. +- `r10-regraded/`: round 10's desks on this toolchain. +- `stats.py`, `stats.txt`. +- `cost.json`, `usd.json`. +- `agents-11.txt`: desk to transcript. diff --git a/eval/results/desk-2026-11-11/agents-11.txt b/eval/results/desk-2026-11-11/agents-11.txt new file mode 100644 index 00000000..f6ba6e2c --- /dev/null +++ b/eval/results/desk-2026-11-11/agents-11.txt @@ -0,0 +1,36 @@ +desk11-rx-1 a84e1656cb0ba6df6 +desk11-react-1 a2f24068924bc4fa4 +desk11-vanilla-1 afe87582e2a06182e +desk11-foldkit-1 adefd18cee0357245 +desk11-rx-4 aa00b48b4d63e1f4f +desk11-react-4 a9ad76d0fefab4fb9 +desk11-vanilla-4 a44863179ca8f66d2 +desk11-foldkit-4 ae41cd1296a281f94 +desk11-rx-2 a22838ba99d99ae05 +desk11-react-2 ae4cfba0be84e5fde +desk11-vanilla-2 abbd5e1185c32e849 +desk11-foldkit-2 a86728ef1f49707e3 +desk11-rx-5 aca8962526f6412eb +desk11-react-5 accf3998eda68b899 +desk11-vanilla-5 acdf73c2cc217936e +desk11-foldkit-5 a267361eee853d3d5 +desk11-rx-6 aacc6363979cdbc04 +desk11-react-6 a7113a69ad5ba8dc5 +desk11-vanilla-6 a3aa74d9de8c92134 +desk11-foldkit-6 a6f3d7ae6bddace40 +desk11-rx-3 a981489adbd708190 +desk11-react-3 afd3f0953f7a4f889 +desk11-vanilla-3 a3a9a9c11a9591692 +desk11-foldkit-3 abf32ed818e83da60 +desk11-rx-7 ac6e74d314c0c60b7 +desk11-react-7 a354f1917af348048 +desk11-vanilla-7 ad6fbfda274aef3d6 +desk11-foldkit-7 afe411d8be72215e2 +desk11-rx-8 ad97b6d0dc3a855a3 +desk11-react-8 adef2bade38df0577 +desk11-vanilla-8 ae61d4a5ff5667c62 +desk11-foldkit-8 a68e6145c8637d77a +desk11-rx-9 aa3711f9a5483d068 +desk11-react-9 adf12270373046254 +desk11-vanilla-9 ac7ec2aeac924ce56 +desk11-foldkit-9 aafaabb4bcd01167f diff --git a/eval/results/desk-2026-11-11/cost.json b/eval/results/desk-2026-11-11/cost.json new file mode 100644 index 00000000..8ddc8cc5 --- /dev/null +++ b/eval/results/desk-2026-11-11/cost.json @@ -0,0 +1,445 @@ +{ + "weights": { + "input": 1, + "cacheWrite": 1.25, + "cacheRead": 0.1, + "output": 5 + }, + "runs": [ + { + "run": "desk11-rx-1", + "task": "desk11", + "arm": "rx", + "n": 1, + "input": 48, + "cacheWrite": 114816, + "cacheRead": 2054728, + "output": 37627, + "calls": 24, + "billed": 537175.8 + }, + { + "run": "desk11-react-1", + "task": "desk11", + "arm": "react", + "n": 1, + "input": 20, + "cacheWrite": 88727, + "cacheRead": 586612, + "output": 36111, + "calls": 10, + "billed": 350144.95 + }, + { + "run": "desk11-vanilla-1", + "task": "desk11", + "arm": "vanilla", + "n": 1, + "input": 18, + "cacheWrite": 83028, + "cacheRead": 482086, + "output": 31010, + "calls": 9, + "billed": 307061.6 + }, + { + "run": "desk11-foldkit-1", + "task": "desk11", + "arm": "foldkit", + "n": 1, + "input": 78, + "cacheWrite": 157838, + "cacheRead": 4085803, + "output": 57285, + "calls": 39, + "billed": 892380.8 + }, + { + "run": "desk11-rx-4", + "task": "desk11", + "arm": "rx", + "n": 4, + "input": 42, + "cacheWrite": 242602, + "cacheRead": 2742416, + "output": 131811, + "calls": 21, + "billed": 1236591.1 + }, + { + "run": "desk11-react-4", + "task": "desk11", + "arm": "react", + "n": 4, + "input": 30, + "cacheWrite": 181839, + "cacheRead": 1622394, + "output": 99373, + "calls": 15, + "billed": 886433.15 + }, + { + "run": "desk11-vanilla-4", + "task": "desk11", + "arm": "vanilla", + "n": 4, + "input": 16, + "cacheWrite": 168758, + "cacheRead": 564851, + "output": 93255, + "calls": 8, + "billed": 733723.6 + }, + { + "run": "desk11-foldkit-4", + "task": "desk11", + "arm": "foldkit", + "n": 4, + "input": 66, + "cacheWrite": 279117, + "cacheRead": 5373929, + "output": 188260, + "calls": 33, + "billed": 1827655.15 + }, + { + "run": "desk11-rx-2", + "task": "desk11", + "arm": "rx", + "n": 2, + "input": 52, + "cacheWrite": 127955, + "cacheRead": 2452851, + "output": 46742, + "calls": 26, + "billed": 638990.85 + }, + { + "run": "desk11-react-2", + "task": "desk11", + "arm": "react", + "n": 2, + "input": 24, + "cacheWrite": 80906, + "cacheRead": 689435, + "output": 28687, + "calls": 12, + "billed": 313535 + }, + { + "run": "desk11-vanilla-2", + "task": "desk11", + "arm": "vanilla", + "n": 2, + "input": 32, + "cacheWrite": 85314, + "cacheRead": 1051601, + "output": 31156, + "calls": 16, + "billed": 367614.6 + }, + { + "run": "desk11-foldkit-2", + "task": "desk11", + "arm": "foldkit", + "n": 2, + "input": 78, + "cacheWrite": 173428, + "cacheRead": 4429660, + "output": 68564, + "calls": 39, + "billed": 1002649 + }, + { + "run": "desk11-rx-5", + "task": "desk11", + "arm": "rx", + "n": 5, + "input": 30, + "cacheWrite": 172922, + "cacheRead": 1624483, + "output": 107574, + "calls": 15, + "billed": 916500.8 + }, + { + "run": "desk11-react-5", + "task": "desk11", + "arm": "react", + "n": 5, + "input": 20, + "cacheWrite": 110131, + "cacheRead": 658688, + "output": 58065, + "calls": 10, + "billed": 493877.55 + }, + { + "run": "desk11-vanilla-5", + "task": "desk11", + "arm": "vanilla", + "n": 5, + "input": 18, + "cacheWrite": 137313, + "cacheRead": 705916, + "output": 85399, + "calls": 9, + "billed": 669245.85 + }, + { + "run": "desk11-foldkit-5", + "task": "desk11", + "arm": "foldkit", + "n": 5, + "input": 64, + "cacheWrite": 255010, + "cacheRead": 4969325, + "output": 168981, + "calls": 32, + "billed": 1660664 + }, + { + "run": "desk11-rx-6", + "task": "desk11", + "arm": "rx", + "n": 6, + "input": 28, + "cacheWrite": 154278, + "cacheRead": 1341121, + "output": 88846, + "calls": 14, + "billed": 771217.6 + }, + { + "run": "desk11-react-6", + "task": "desk11", + "arm": "react", + "n": 6, + "input": 26, + "cacheWrite": 137985, + "cacheRead": 1077077, + "output": 81181, + "calls": 13, + "billed": 686119.95 + }, + { + "run": "desk11-vanilla-6", + "task": "desk11", + "arm": "vanilla", + "n": 6, + "input": 26, + "cacheWrite": 192119, + "cacheRead": 1406897, + "output": 112557, + "calls": 13, + "billed": 943649.45 + }, + { + "run": "desk11-foldkit-6", + "task": "desk11", + "arm": "foldkit", + "n": 6, + "input": 58, + "cacheWrite": 231135, + "cacheRead": 3634198, + "output": 142901, + "calls": 29, + "billed": 1366901.55 + }, + { + "run": "desk11-rx-3", + "task": "desk11", + "arm": "rx", + "n": 3, + "input": 28, + "cacheWrite": 70566, + "cacheRead": 1083936, + "output": 32630, + "calls": 14, + "billed": 359779.1 + }, + { + "run": "desk11-react-3", + "task": "desk11", + "arm": "react", + "n": 3, + "input": 32, + "cacheWrite": 62105, + "cacheRead": 1117911, + "output": 36921, + "calls": 16, + "billed": 374059.35 + }, + { + "run": "desk11-vanilla-3", + "task": "desk11", + "arm": "vanilla", + "n": 3, + "input": 24, + "cacheWrite": 52234, + "cacheRead": 727341, + "output": 27242, + "calls": 12, + "billed": 274260.6 + }, + { + "run": "desk11-foldkit-3", + "task": "desk11", + "arm": "foldkit", + "n": 3, + "input": 68, + "cacheWrite": 130992, + "cacheRead": 3590620, + "output": 61354, + "calls": 34, + "billed": 829640 + }, + { + "run": "desk11-rx-7", + "task": "desk11", + "arm": "rx", + "n": 7, + "input": 24, + "cacheWrite": 169319, + "cacheRead": 1150004, + "output": 94694, + "calls": 12, + "billed": 800143.15 + }, + { + "run": "desk11-react-7", + "task": "desk11", + "arm": "react", + "n": 7, + "input": 16, + "cacheWrite": 99773, + "cacheRead": 501227, + "output": 78701, + "calls": 8, + "billed": 568359.95 + }, + { + "run": "desk11-vanilla-7", + "task": "desk11", + "arm": "vanilla", + "n": 7, + "input": 14, + "cacheWrite": 133174, + "cacheRead": 450749, + "output": 82397, + "calls": 7, + "billed": 623541.4 + }, + { + "run": "desk11-foldkit-7", + "task": "desk11", + "arm": "foldkit", + "n": 7, + "input": 82, + "cacheWrite": 212489, + "cacheRead": 6393572, + "output": 160068, + "calls": 41, + "billed": 1705390.4500000002 + }, + { + "run": "desk11-rx-8", + "task": "desk11", + "arm": "rx", + "n": 8, + "input": 22, + "cacheWrite": 118421, + "cacheRead": 913795, + "output": 80266, + "calls": 11, + "billed": 640757.75 + }, + { + "run": "desk11-react-8", + "task": "desk11", + "arm": "react", + "n": 8, + "input": 32, + "cacheWrite": 171479, + "cacheRead": 1614783, + "output": 117355, + "calls": 16, + "billed": 962634.05 + }, + { + "run": "desk11-vanilla-8", + "task": "desk11", + "arm": "vanilla", + "n": 8, + "input": 32, + "cacheWrite": 171439, + "cacheRead": 1876839, + "output": 118254, + "calls": 16, + "billed": 993284.65 + }, + { + "run": "desk11-foldkit-8", + "task": "desk11", + "arm": "foldkit", + "n": 8, + "input": 118, + "cacheWrite": 277940, + "cacheRead": 10575743, + "output": 193296, + "calls": 59, + "billed": 2371597.3 + }, + { + "run": "desk11-rx-9", + "task": "desk11", + "arm": "rx", + "n": 9, + "input": 40, + "cacheWrite": 209701, + "cacheRead": 2508495, + "output": 131593, + "calls": 20, + "billed": 1170980.75 + }, + { + "run": "desk11-react-9", + "task": "desk11", + "arm": "react", + "n": 9, + "input": 18, + "cacheWrite": 98703, + "cacheRead": 537861, + "output": 78219, + "calls": 9, + "billed": 568277.85 + }, + { + "run": "desk11-vanilla-9", + "task": "desk11", + "arm": "vanilla", + "n": 9, + "input": 14, + "cacheWrite": 142091, + "cacheRead": 468383, + "output": 92217, + "calls": 7, + "billed": 685551.05 + }, + { + "run": "desk11-foldkit-9", + "task": "desk11", + "arm": "foldkit", + "n": 9, + "input": 74, + "cacheWrite": 272538, + "cacheRead": 4876740, + "output": 145200, + "calls": 37, + "billed": 1554420.5 + } + ], + "diff": -207632.78888888902, + "ratio": 1.3591267418422939, + "p": 0.9435805641943581 +} diff --git a/eval/results/desk-2026-11-11/graded/foldkit-1.txt b/eval/results/desk-2026-11-11/graded/foldkit-1.txt new file mode 100644 index 00000000..a6ca8b5a --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/foldkit-1.txt @@ -0,0 +1,12 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk11-foldkit-1: 2/20 seeds found a hole + seed 3 step 39: at rest: labels: #4 shows [docs], server [docs,ui] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; add-label bug ; fail addLabel:4,bug ; land fetchIssue:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; add-label p1 ; assign ; land closeIssue:4 ; fail removeL + seed 12 step 36: at rest: comments: #4 shows [guest: me too | guest: me too], server [guest: me too] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land assign:4,guest ; land fetchComments:1 ; land fetchLabels:1 ; post me too ; close ; fail renameIssue:4,Alpha ; assign ; land addComment:4,guest,me too ; close ; diff --git a/eval/results/desk-2026-11-11/graded/foldkit-2.txt b/eval/results/desk-2026-11-11/graded/foldkit-2.txt new file mode 100644 index 00000000..2af4cb66 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/foldkit-2.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk11-foldkit-2: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/graded/foldkit-3.txt b/eval/results/desk-2026-11-11/graded/foldkit-3.txt new file mode 100644 index 00000000..f357c830 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/foldkit-3.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk11-foldkit-3: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/graded/foldkit-4.txt b/eval/results/desk-2026-11-11/graded/foldkit-4.txt new file mode 100644 index 00000000..b0d4db20 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/foldkit-4.txt @@ -0,0 +1,27 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +25/38 passed +== fuzz +desk11-foldkit-4: 4/20 seeds found a hole + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 10 step 25: row assignee: #2 shows bob, server guest + focus ; focus ; filter closed ; fail fetchIssues:open ; focus ; land fetchIssues:closed ; row-assign 0 ; land assign:2,guest ; select #2 ; close ; rename Bravo ; fail fetchLabels:2 ; select #2 ; fail fetchIssue:2 ; land closeIssue:2 ; row-assign 0 ; land fetchComments:2 ; land assign:2,guest ; assign ; user bob ; land assign:2,guest ; row-assign 0 ; rename Alpha ; assign ; fail assign:2,bob + seed 15 step 36: row title: #3 shows "Crash on save", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land closeIssue:4 ; land fetchIssue:3 ; assign ; land search:Crash on save ; rename Delta ; + seed 18 step 35: row assignee: #4 shows bob, server dana + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail assign:4,guest ; fail closeIssue:4 ; land fetchComments:4 ; land closeIssue:3 ; user bob ; row-assign 0 ; add-label ui ; rename Delta ; assign ; row-close 0 ; fail closeIssue:1 ; rename Alpha ; close ; fail closeIssue:4 ; focus ; close ; focus ; row-assign 0 diff --git a/eval/results/desk-2026-11-11/graded/foldkit-5.txt b/eval/results/desk-2026-11-11/graded/foldkit-5.txt new file mode 100644 index 00000000..06c2b480 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/foldkit-5.txt @@ -0,0 +1,21 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +27/38 passed +== fuzz +desk11-foldkit-5: 2/20 seeds found a hole + seed 4 step 24: count: tab Issues (3), 2 open rows shown + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 18 step 38: row assignee: #4 shows bob, server dana + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; (move failed: locator.click: Timeout 2000ms exceeded.) ; fail assign:4,guest ; fail closeIssue:4 ; land fetchComments:4 ; land closeIssue:3 ; user bob ; row-assign 0 ; add-label ui ; rename Delta ; (move failed: locator.click: Timeout 2000ms exceeded.) ; assign ; diff --git a/eval/results/desk-2026-11-11/graded/foldkit-6.txt b/eval/results/desk-2026-11-11/graded/foldkit-6.txt new file mode 100644 index 00000000..bfa425f1 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/foldkit-6.txt @@ -0,0 +1,20 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ base: settings: an earlier save failing after a later one succeeded leaves the later one +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: open issues that load while a row close is out keep the row out +26/38 passed +== fuzz +desk11-foldkit-6: 1/20 seeds found a hole + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 diff --git a/eval/results/desk-2026-11-11/graded/foldkit-7.txt b/eval/results/desk-2026-11-11/graded/foldkit-7.txt new file mode 100644 index 00000000..a0c4a90c --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/foldkit-7.txt @@ -0,0 +1,19 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +27/38 passed +== fuzz +desk11-foldkit-7: 1/20 seeds found a hole + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 diff --git a/eval/results/desk-2026-11-11/graded/foldkit-8.txt b/eval/results/desk-2026-11-11/graded/foldkit-8.txt new file mode 100644 index 00000000..cc802ae3 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/foldkit-8.txt @@ -0,0 +1,17 @@ +== hidden +✗ base: a declined close brings the row back in place and shows the message +76/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: open issues that load while a row close is out keep the row out +30/38 passed +== fuzz +desk11-foldkit-8: 1/20 seeds found a hole + seed 3 step 40: at rest: labels: #4 shows [bug,ui], server [ui,bug] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land fetchLabels:4 ; add-label bug ; fail assign:4,bob ; land fetchComments:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; add-label p1 ; assign ; land closeIssue:4 ; fail remov diff --git a/eval/results/desk-2026-11-11/graded/foldkit-9.txt b/eval/results/desk-2026-11-11/graded/foldkit-9.txt new file mode 100644 index 00000000..7f3bbea7 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/foldkit-9.txt @@ -0,0 +1,24 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +28/38 passed +== fuzz +desk11-foldkit-9: 4/20 seeds found a hole + seed 3 step 40: at rest: labels: #4 shows [bug,ui], server [ui,bug] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land fetchLabels:4 ; add-label bug ; fail assign:4,bob ; land fetchComments:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; add-label p1 ; assign ; land closeIssue:4 ; fail remov + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 15 step 37: row assignee: #3 shows guest, server null + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land closeIssue:4 ; land fetchIssue:3 ; assign ; land search:Crash on save ; rename Delta ; + seed 18 step 35: row assignee: #4 shows bob, server dana + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail assign:4,guest ; fail closeIssue:4 ; land fetchComments:4 ; land closeIssue:3 ; user bob ; row-assign 0 ; add-label ui ; rename Delta ; assign ; row-close 0 ; fail closeIssue:1 ; rename Alpha ; close ; fail closeIssue:4 ; focus ; close ; focus ; row-assign 0 diff --git a/eval/results/desk-2026-11-11/graded/react-1.txt b/eval/results/desk-2026-11-11/graded/react-1.txt new file mode 100644 index 00000000..dbd2d14a --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/react-1.txt @@ -0,0 +1,6 @@ +== hidden +77/77 passed +== overlap +38/38 passed +== fuzz +desk11-react-1: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/graded/react-2.txt b/eval/results/desk-2026-11-11/graded/react-2.txt new file mode 100644 index 00000000..839e7b30 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/react-2.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk11-react-2: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/graded/react-3.txt b/eval/results/desk-2026-11-11/graded/react-3.txt new file mode 100644 index 00000000..51d4a922 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/react-3.txt @@ -0,0 +1,12 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk11-react-3: 2/20 seeds found a hole + seed 4 step 36: at rest: comments: #3 shows [dana: seen it too | guest: me too | guest: me too], server [dana: seen it too | guest: me too] + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land closeIssue:1 ; select #3 ; row-close 1 ; rena + seed 12 step 36: at rest: comments: #4 shows [guest: me too | guest: me too], server [guest: me too] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land assign:4,guest ; land fetchComments:1 ; land fetchLabels:1 ; post me too ; close ; fail renameIssue:4,Alpha ; assign ; land addComment:4,guest,me too ; close ; diff --git a/eval/results/desk-2026-11-11/graded/react-4.txt b/eval/results/desk-2026-11-11/graded/react-4.txt new file mode 100644 index 00000000..b574fd01 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/react-4.txt @@ -0,0 +1,21 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: open issues that load while a row close is out keep the row out +27/38 passed +== fuzz +desk11-react-4: 2/20 seeds found a hole + seed 15 step 36: row title: #3 shows "Crash on save", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan + seed 16 step 39: at rest: comments: #2 shows [guest: fixed? | guest: me too | guest: me too], server [guest: fixed? | guest: me too] + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchComments:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; close ; post fixed? ; post ping ; assign ; land addComment:2,guest,fixed? ; land fetchLabels:2 ; fail assign:2,guest ; select #2 ; rename Bravo ; as diff --git a/eval/results/desk-2026-11-11/graded/react-5.txt b/eval/results/desk-2026-11-11/graded/react-5.txt new file mode 100644 index 00000000..28635c23 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/react-5.txt @@ -0,0 +1,8 @@ +== hidden +✗ base: an accepted post stays after switching issues and back +76/77 passed +== overlap +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +37/38 passed +== fuzz +desk11-react-5: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/graded/react-6.txt b/eval/results/desk-2026-11-11/graded/react-6.txt new file mode 100644 index 00000000..a39d1eaa --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/react-6.txt @@ -0,0 +1,28 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +26/38 passed +== fuzz +desk11-react-6: 5/20 seeds found a hole + seed 2 step 15: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchComments:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail fetchIssue:2 ; land closeIssue:2 ; fail assign:2,guest + seed 3 step 39: at rest: labels: #4 shows [docs,p1], server [docs] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchIssue:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; row-close 0 ; fail search:Typo in footer ; land fetchLabels:4 ; row-close 1 ; row-close 0 ; land closeIssue:4 ; close ; add-label p1 ; remove-label 0 ; remove-label 0 ; filter closed ; add-label p1 ; assign ; land addLabel:4,p1 ; fai + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 16 step 36: at rest: comments: #2 shows [guest: ping | guest: ping], server [guest: ping] + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchLabels:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; row-assign 0 ; close ; post ping ; assign ; land addComment:2,guest,ping ; land fetchIssue:2 ; fail assign:2,guest ; select #2 ; rename Bravo ; assign + seed 18 step 13: row title: #4 shows "Typo in footer", server "Delta" + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchIssue:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail assign:4,guest ; fail closeIssue:4 ; land closeIssue:1 ; land renameIssue:4,Delta diff --git a/eval/results/desk-2026-11-11/graded/react-7.txt b/eval/results/desk-2026-11-11/graded/react-7.txt new file mode 100644 index 00000000..2f357b3c --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/react-7.txt @@ -0,0 +1,24 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +28/38 passed +== fuzz +desk11-react-7: 4/20 seeds found a hole + seed 2 step 37: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land closeIssue:2 ; rename Alpha ; rename B + seed 4 step 24: count: tab Issues (3), 2 open rows shown + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 15 step 33: detail title: "#3 Crash on savestack trace attachedretryassign to guestrena", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan + seed 16 step 39: at rest: comments: #2 shows [guest: fixed? | guest: me too | guest: me too], server [guest: fixed? | guest: me too] + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchComments:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; close ; post fixed? ; post ping ; assign ; land addComment:2,guest,fixed? ; land fetchLabels:2 ; fail assign:2,guest ; select #2 ; rename Bravo ; as diff --git a/eval/results/desk-2026-11-11/graded/react-8.txt b/eval/results/desk-2026-11-11/graded/react-8.txt new file mode 100644 index 00000000..3f4b8692 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/react-8.txt @@ -0,0 +1,7 @@ +== hidden +77/77 passed +== overlap +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +37/38 passed +== fuzz +desk11-react-8: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/graded/react-9.txt b/eval/results/desk-2026-11-11/graded/react-9.txt new file mode 100644 index 00000000..5b99cbc7 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/react-9.txt @@ -0,0 +1,20 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +30/38 passed +== fuzz +desk11-react-9: 3/20 seeds found a hole + seed 2 step 37: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land closeIssue:2 ; rename Alpha ; rename B + seed 15 step 36: row title: #3 shows "Crash on save", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan + seed 16 step 39: at rest: labels: #2 shows [old], server [old,bug] + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchComments:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; close ; post fixed? ; post ping ; assign ; land addComment:2,guest,fixed? ; land fetchLabels:2 ; fail assign:2,guest ; select #2 ; rename Bravo ; as diff --git a/eval/results/desk-2026-11-11/graded/rx-1.txt b/eval/results/desk-2026-11-11/graded/rx-1.txt new file mode 100644 index 00000000..21eab8c1 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/rx-1.txt @@ -0,0 +1,6 @@ +== hidden +77/77 passed +== overlap +38/38 passed +== fuzz +desk11-rx-1: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/graded/rx-2.txt b/eval/results/desk-2026-11-11/graded/rx-2.txt new file mode 100644 index 00000000..552929e5 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/rx-2.txt @@ -0,0 +1,14 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk11-rx-2: 3/20 seeds found a hole + seed 1 step 24: open row in closed list: #4 + user bob ; focus ; land fetchIssues:open ; row-assign 2 ; row-close 0 ; row-assign 0 ; select #3 ; rename Alpha ; land assign:3,bob ; rename Alpha ; row-assign 1 ; land fetchLabels:3 ; focus ; land fetchComments:3 ; fail closeIssue:1 ; land renameIssue:3,Alpha ; fail assign:4,bob ; post me too ; add-label ui ; land addComment:3,bob,me too ; row-close 2 ; filter closed ; land fetchIssue:3 ; lan + seed 3 step 37: open row in closed list: #1 + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; add-label bug ; fail addLabel:4,bug ; land fetchIssue:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; add-label p1 ; assign ; land closeIssue:4 ; fail removeL + seed 18 step 16: open row in closed list: #3 + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail closeIssue:3 ; fail closeIssue:4 ; land fetchComments:4 ; land renameIssue:4,Delta ; filter closed ; land fetchIssues:closed ; land closeIssue:1 diff --git a/eval/results/desk-2026-11-11/graded/rx-3.txt b/eval/results/desk-2026-11-11/graded/rx-3.txt new file mode 100644 index 00000000..9be7d5bc --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/rx-3.txt @@ -0,0 +1,7 @@ +== hidden +77/77 passed +== overlap +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +37/38 passed +== fuzz +desk11-rx-3: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/graded/rx-4.txt b/eval/results/desk-2026-11-11/graded/rx-4.txt new file mode 100644 index 00000000..abb997d1 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/rx-4.txt @@ -0,0 +1,10 @@ +== hidden +✗ base: retry after not found shows the issue +76/77 passed +== overlap +✗ base: an earlier rename declined after a later one was accepted keeps the later title +37/38 passed +== fuzz +desk11-rx-4: 1/20 seeds found a hole + seed 15 step 38: detail title: "retry#2 Bravolong goneassign to guestcloserenamesaving files", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; fail fetchIssues:open ; tab help ; tab issues ; land fetchIssues:open ; focus ; filter closed ; land fetchIssues:closed ; select #2 ; land fetchLabels:2 ; rename Bravo ; select #2 ; assign ; land fetchIssue:2 ; row-assign 0 ; select #2 ; land fetchComments:2 ; assign ; remove-label 0 ; land search:Old bug ; land renameIssue:2 diff --git a/eval/results/desk-2026-11-11/graded/rx-5.txt b/eval/results/desk-2026-11-11/graded/rx-5.txt new file mode 100644 index 00000000..c06c929a --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/rx-5.txt @@ -0,0 +1,11 @@ +== hidden +77/77 passed +== overlap +✗ a label added on one issue does not show on another +37/38 passed +== fuzz +desk11-rx-5: 2/20 seeds found a hole + seed 15 step 24: detail title: "retry#2 Old buglong gonesaving filescrash reportsassign to g", server "Bravo" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; fail fetchIssues:open ; tab help ; tab issues ; land fetchIssues:open ; focus ; filter closed ; land fetchIssues:closed ; select #2 ; land fetchLabels:2 ; rename Bravo ; select #2 ; assign ; land fetchIssue:2 ; row-assign 0 ; select #2 ; land fetchComments:2 ; assign ; remove-label 0 ; land search:Old bug ; land renameIssue:2 + seed 18 step 32: row assignee: #4 shows dana, server guest + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail closeIssue:3 ; fail closeIssue:4 ; land fetchComments:4 ; land renameIssue:4,Delta ; filter closed ; land fetchIssues:closed ; land closeIssue:1 ; close ; land closeIssue:4 ; land fetchIssues:closed ; rename Alpha ; land fetchIssues:open ; land renameIssue:4, diff --git a/eval/results/desk-2026-11-11/graded/rx-6.txt b/eval/results/desk-2026-11-11/graded/rx-6.txt new file mode 100644 index 00000000..92c0d054 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/rx-6.txt @@ -0,0 +1,12 @@ +== hidden +✗ base: selecting after not found shows loading, not the old message +✗ related help landing out of order shows the selected issue's +75/77 passed +== overlap +38/38 passed +== fuzz +desk11-rx-6: 2/20 seeds found a hole + seed 11 step 34: row assignee: #4 shows dana, server ann + tab help ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; focus ; tab help ; tab issues ; land fetchIssues:open ; row-assign 1 ; land assign:3,guest ; filter closed ; user ann ; row-assign 0 ; focus ; land fetchIssues:open ; land assign:1,ann ; select #4 ; close ; land fetchIssues:closed ; close ; land fetchLabels:4 ; fail fetchComments:4 ; row-assign 0 ; close ; land fetchIssue: + seed 18 step 32: row assignee: #4 shows dana, server guest + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail closeIssue:3 ; fail closeIssue:4 ; land fetchComments:4 ; land renameIssue:4,Delta ; filter closed ; land fetchIssues:closed ; land closeIssue:1 ; close ; land closeIssue:4 ; land fetchIssues:closed ; rename Alpha ; land fetchIssues:open ; land renameIssue:4, diff --git a/eval/results/desk-2026-11-11/graded/rx-7.txt b/eval/results/desk-2026-11-11/graded/rx-7.txt new file mode 100644 index 00000000..914efbb8 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/rx-7.txt @@ -0,0 +1,9 @@ +== hidden +✗ base: retry after not found shows the issue +76/77 passed +== overlap +38/38 passed +== fuzz +desk11-rx-7: 1/20 seeds found a hole + seed 11 step 34: row assignee: #4 shows dana, server ann + tab help ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; focus ; tab help ; tab issues ; land fetchIssues:open ; row-assign 1 ; land assign:3,guest ; filter closed ; user ann ; row-assign 0 ; focus ; land fetchIssues:open ; land assign:1,ann ; select #4 ; close ; land fetchIssues:closed ; close ; land fetchLabels:4 ; fail fetchComments:4 ; row-assign 0 ; close ; land fetchIssue: diff --git a/eval/results/desk-2026-11-11/graded/rx-8.txt b/eval/results/desk-2026-11-11/graded/rx-8.txt new file mode 100644 index 00000000..23b52ade --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/rx-8.txt @@ -0,0 +1,18 @@ +== hidden +✗ base: selecting after not found shows loading, not the old message +✗ base: help: selecting after a failure shows loading…, not the old failure +75/77 passed +== overlap +✗ base: a declined assignment keeps a rename in flight on the same issue +✗ base: a declined rename keeps an assignment in flight on the same issue +✗ a label added on one issue does not show on another +✗ rows: a declined row close leaves a rename out on another issue +34/38 passed +== fuzz +desk11-rx-8: 3/20 seeds found a hole + seed 8 step 46: detail title: "assign to guest close retryrename#3 Crash on savestack trace", server "Alpha" + land fetchIssues:open ; row-close 1 ; filter closed ; fail fetchIssues:closed ; select #4 ; land closeIssue:3 ; select #1 ; land fetchIssues:closed ; post me too ; rename Bravo ; land fetchComments:1 ; rename Delta ; close ; row-assign 1 ; land fetchLabels:4 ; land assign:3,guest ; select #2 ; focus ; fail fetchComments:2 ; close ; fail addComment:1,guest,me too ; land closeIssue:2 ; land fetc + seed 11 step 44: detail title: "assign to ann close retryrename#3 Crash on savestack trace a", server "Delta" + tab help ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; focus ; tab help ; tab issues ; land fetchIssues:open ; row-assign 1 ; land assign:3,guest ; filter closed ; user ann ; row-assign 0 ; focus ; land fetchIssues:open ; land assign:1,ann ; select #4 ; close ; land fetchIssues:closed ; row-assign 0 ; row-close 0 ; select #3 ; focus ; close ; land fetchIssue:4 ; post me too ; + seed 15 step 24: detail title: "assign to guest close retryrename#2 Old buglong gonesaving f", server "Bravo" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; fail fetchIssues:open ; tab help ; tab issues ; land fetchIssues:open ; focus ; filter closed ; land fetchIssues:closed ; select #2 ; land fetchLabels:2 ; rename Bravo ; select #2 ; assign ; land fetchIssue:2 ; row-assign 0 ; select #2 ; land fetchComments:2 ; assign ; remove-label 0 ; land search:Old bug ; land renameIssue:2 diff --git a/eval/results/desk-2026-11-11/graded/rx-9.txt b/eval/results/desk-2026-11-11/graded/rx-9.txt new file mode 100644 index 00000000..69806e36 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/rx-9.txt @@ -0,0 +1,8 @@ +== hidden +✗ base: help: selecting after a failure shows loading…, not the old failure +✗ related help landing out of order shows the selected issue's +75/77 passed +== overlap +38/38 passed +== fuzz +desk11-rx-9: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/graded/vanilla-1.txt b/eval/results/desk-2026-11-11/graded/vanilla-1.txt new file mode 100644 index 00000000..4228f3ca --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/vanilla-1.txt @@ -0,0 +1,10 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk11-vanilla-1: 1/20 seeds found a hole + seed 12 step 36: at rest: comments: #4 shows [guest: me too | guest: me too], server [guest: me too] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land assign:4,guest ; land fetchComments:1 ; land fetchLabels:1 ; post me too ; close ; fail renameIssue:4,Alpha ; assign ; land addComment:4,guest,me too ; close ; diff --git a/eval/results/desk-2026-11-11/graded/vanilla-2.txt b/eval/results/desk-2026-11-11/graded/vanilla-2.txt new file mode 100644 index 00000000..24331212 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/vanilla-2.txt @@ -0,0 +1,10 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk11-vanilla-2: 1/20 seeds found a hole + seed 12 step 36: at rest: comments: #4 shows [guest: me too | guest: me too], server [guest: me too] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land assign:4,guest ; land fetchComments:1 ; land fetchLabels:1 ; post me too ; close ; fail renameIssue:4,Alpha ; assign ; land addComment:4,guest,me too ; close ; diff --git a/eval/results/desk-2026-11-11/graded/vanilla-3.txt b/eval/results/desk-2026-11-11/graded/vanilla-3.txt new file mode 100644 index 00000000..9460fce0 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/vanilla-3.txt @@ -0,0 +1,10 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk11-vanilla-3: 1/20 seeds found a hole + seed 12 step 36: at rest: comments: #4 shows [guest: me too | guest: me too], server [guest: me too] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land assign:4,guest ; land fetchComments:1 ; land fetchLabels:1 ; post me too ; close ; fail renameIssue:4,Alpha ; assign ; land addComment:4,guest,me too ; close ; diff --git a/eval/results/desk-2026-11-11/graded/vanilla-4.txt b/eval/results/desk-2026-11-11/graded/vanilla-4.txt new file mode 100644 index 00000000..20214ecd --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/vanilla-4.txt @@ -0,0 +1,21 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +29/38 passed +== fuzz +desk11-vanilla-4: 3/20 seeds found a hole + seed 3 step 39: at rest: labels: #4 shows [docs], server [docs,ui] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; add-label bug ; fail addLabel:4,bug ; land fetchIssue:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; add-label p1 ; assign ; land closeIssue:4 ; fail removeL + seed 15 step 36: row title: #3 shows "Crash on save", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan + seed 18 step 13: row title: #4 shows "Typo in footer", server "Delta" + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail closeIssue:3 ; fail closeIssue:4 ; land fetchComments:4 ; land renameIssue:4,Delta diff --git a/eval/results/desk-2026-11-11/graded/vanilla-5.txt b/eval/results/desk-2026-11-11/graded/vanilla-5.txt new file mode 100644 index 00000000..76057d93 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/vanilla-5.txt @@ -0,0 +1,26 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +26/38 passed +== fuzz +desk11-vanilla-5: 4/20 seeds found a hole + seed 3 step 39: at rest: labels: #4 shows [docs], server [docs,ui] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; add-label bug ; fail addLabel:4,bug ; land fetchIssue:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; add-label p1 ; assign ; land closeIssue:4 ; fail removeL + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 15 step 36: row title: #3 shows "Crash on save", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan + seed 18 step 13: row title: #4 shows "Typo in footer", server "Delta" + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail closeIssue:3 ; fail closeIssue:4 ; land fetchComments:4 ; land renameIssue:4,Delta diff --git a/eval/results/desk-2026-11-11/graded/vanilla-6.txt b/eval/results/desk-2026-11-11/graded/vanilla-6.txt new file mode 100644 index 00000000..f79ff1f4 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/vanilla-6.txt @@ -0,0 +1,19 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +29/38 passed +== fuzz +desk11-vanilla-6: 2/20 seeds found a hole + seed 2 step 33: detail title: "#2 Old buglong goneretryassign to guestrenamecloseassign err", server "Bravo" + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land fetchLabels:2 ; rename Alpha ; rename + seed 15 step 36: two views: row "#3 Crash on save", detail "#3 Deltastack trace attachedretryassign " + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land closeIssue:4 ; land fetchIssue:3 ; assign ; land search:Crash on save ; rename Delta ; diff --git a/eval/results/desk-2026-11-11/graded/vanilla-7.txt b/eval/results/desk-2026-11-11/graded/vanilla-7.txt new file mode 100644 index 00000000..2f8ab878 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/vanilla-7.txt @@ -0,0 +1,15 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +29/38 passed +== fuzz +desk11-vanilla-7: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/graded/vanilla-8.txt b/eval/results/desk-2026-11-11/graded/vanilla-8.txt new file mode 100644 index 00000000..37cf050b --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/vanilla-8.txt @@ -0,0 +1,33 @@ +== hidden +✗ the results for a related hit may land after the help page opens +✗ a related item that fails can be retried on the help page +75/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ two closes at once: a declined one leaves the other off the count +✗ open issues that load while a close is out keep it off the count +✗ rows: open issues that load while a row close is out keep the row out +27/38 passed +== fuzz +desk11-vanilla-8: 7/20 seeds found a hole + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 7 step 6: count: tab Issues (1), 2 open rows shown + land fetchIssues:open ; row-close 1 ; row-assign 0 ; row-close 0 ; land closeIssue:1 ; fail closeIssue:3 + seed 9 step 40: two views: row "#4 Bravo", detail "#4 Deltafooter says 2019retryassign to g" + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt + seed 12 step 34: row assignee: #4 shows dana, server guest + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; land search:Login fails ; land renameIssue:4,Alpha ; land fetchComments:1 ; land fetchComments:4 ; filter closed ; land closeIssue:1 ; fail assign:4,guest ; assign ; land search:Typo i + seed 14 step 20: count: tab Issues (0), 1 open rows shown + filter closed ; filter open ; land fetchIssues:open ; land fetchIssues:closed ; row-assign 0 ; select #1 ; land fetchIssue:1 ; land fetchLabels:1 ; close ; row-close 1 ; fail fetchComments:1 ; rename Alpha ; rename Alpha ; row-assign 0 ; row-close 0 ; close ; land closeIssue:1 ; rename Bravo ; fail assign:3,guest ; fail closeIssue:4 + seed 15 step 36: count: tab Issues (0), 1 open rows shown + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan + seed 18 step 10: count: tab Issues (0), 1 open rows shown + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail closeIssue:3 diff --git a/eval/results/desk-2026-11-11/graded/vanilla-9.txt b/eval/results/desk-2026-11-11/graded/vanilla-9.txt new file mode 100644 index 00000000..29d11bad --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/vanilla-9.txt @@ -0,0 +1,29 @@ +== hidden +✗ the results for a related hit may land after the help page opens +✗ a related item that fails can be retried on the help page +75/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +25/38 passed +== fuzz +desk11-vanilla-9: 4/20 seeds found a hole + seed 3 step 39: count: tab Issues (3), 2 open rows shown + assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchIssues:open ; select #4 ; add-label p1 ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchLabels:4 ; row- + seed 8 step 16: row title: #4 shows "Typo in footer", server "Bravo" + land fetchIssues:open ; row-close 1 ; post me too ; (move failed: locator.fill: Timeout 2000ms exceeded.) ; select #4 ; land fetchComments:4 ; land fetchLabels:4 ; fail closeIssue:3 ; close ; remove-label 0 ; close ; rename Bravo ; land renameIssue:4,Bravo ; user bob ; add-label ui ; fail closeIssue:4 + seed 11 step 46: detail title: "#3 Crash on savestack trace attachedretryassign to guest ren", server "Bravo" + close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchIssues:open ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; post fixed? ; (move failed: locator.fill: Timeo + seed 17 step 19: dropped guess: #3 title: showed "Alpha" while it was in flight, now "Crash on save" + assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchIssues:open ; row-assign 0 ; add-label bug ; (move failed: locator.fill: Timeout 2000ms exceeded.) ; land assign:1,guest ; row-close 0 ; row-close 1 ; row-assign 0 ; select #3 ; rename Bravo ; row-assign 0 ; land fetchLabels:3 ; rename Alpha ; assign ; f diff --git a/eval/results/desk-2026-11-11/r10-regraded/foldkit-1.txt b/eval/results/desk-2026-11-11/r10-regraded/foldkit-1.txt new file mode 100644 index 00000000..8c8e4b52 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/foldkit-1.txt @@ -0,0 +1,11 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +33/38 passed +== fuzz +desk10-foldkit-1: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/r10-regraded/foldkit-2.txt b/eval/results/desk-2026-11-11/r10-regraded/foldkit-2.txt new file mode 100644 index 00000000..e123e888 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/foldkit-2.txt @@ -0,0 +1,13 @@ +== hidden +77/77 passed +== overlap +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +35/38 passed +== fuzz +desk10-foldkit-2: 2/20 seeds found a hole + seed 2 step 37: at rest: comments: #2 shows [], server [guest: fixed? | guest: fixed?] + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land closeIssue:2 ; rename Alpha ; rename B + seed 12 step 36: at rest: comments: #4 shows [guest: me too | guest: me too], server [guest: me too] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land assign:4,guest ; land fetchComments:1 ; land fetchLabels:1 ; post me too ; close ; fail renameIssue:4,Alpha ; assign ; land addComment:4,guest,me too ; close ; diff --git a/eval/results/desk-2026-11-11/r10-regraded/foldkit-3.txt b/eval/results/desk-2026-11-11/r10-regraded/foldkit-3.txt new file mode 100644 index 00000000..775eae3b --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/foldkit-3.txt @@ -0,0 +1,12 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk10-foldkit-3: 2/20 seeds found a hole + seed 3 step 39: at rest: labels: #4 shows [docs], server [docs,ui] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; add-label bug ; fail addLabel:4,bug ; land fetchIssue:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; add-label p1 ; assign ; land closeIssue:4 ; fail removeL + seed 12 step 36: at rest: comments: #4 shows [guest: me too | guest: me too], server [guest: me too] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land assign:4,guest ; land fetchComments:1 ; land fetchLabels:1 ; post me too ; close ; fail renameIssue:4,Alpha ; assign ; land addComment:4,guest,me too ; close ; diff --git a/eval/results/desk-2026-11-11/r10-regraded/foldkit-4.txt b/eval/results/desk-2026-11-11/r10-regraded/foldkit-4.txt new file mode 100644 index 00000000..9bd2825f --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/foldkit-4.txt @@ -0,0 +1,24 @@ +== hidden +✗ base: retry after not found shows the issue +76/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +25/38 passed +== fuzz +desk10-foldkit-4: 2/20 seeds found a hole + seed 2 step 36: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land closeIssue:2 ; rename Alpha ; rename B + seed 15 step 32: detail title: "#3 Crash on savestack trace attachedassign to guestcloserena", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land closeIssue:4 ; land fetchIssue:3 ; assign ; land search:Crash on save ; rename Delta ; diff --git a/eval/results/desk-2026-11-11/r10-regraded/foldkit-5.txt b/eval/results/desk-2026-11-11/r10-regraded/foldkit-5.txt new file mode 100644 index 00000000..6bff2dcb --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/foldkit-5.txt @@ -0,0 +1,23 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +✗ labels that load while an add is out keep the add +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +29/38 passed +== fuzz +desk10-foldkit-5: 4/20 seeds found a hole + seed 2 step 35: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land closeIssue:2 ; rename Alpha ; rename B + seed 12 step 36: two views: row "#4 Alpha", detail "#4 Bravofooter says 2019assign to guestr" + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; land search:Typo in footer ; land closeIssue:4 ; land fetchComments:1 ; land fetchComments:4 ; filter closed ; land renameIssue:4,Alpha ; fail assign:4,guest ; assign ; land closeIssue + seed 15 step 37: row assignee: #3 shows guest, server null + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land closeIssue:4 ; land fetchIssue:3 ; assign ; land search:Crash on save ; rename Delta ; + seed 18 step 35: row assignee: #4 shows bob, server dana + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail assign:4,guest ; fail closeIssue:4 ; land fetchComments:4 ; land closeIssue:3 ; user bob ; row-assign 0 ; add-label ui ; rename Delta ; assign ; row-close 0 ; fail closeIssue:1 ; rename Alpha ; close ; fail closeIssue:4 ; focus ; close ; focus ; row-assign 0 diff --git a/eval/results/desk-2026-11-11/r10-regraded/foldkit-6.txt b/eval/results/desk-2026-11-11/r10-regraded/foldkit-6.txt new file mode 100644 index 00000000..a1778a64 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/foldkit-6.txt @@ -0,0 +1,25 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +27/38 passed +== fuzz +desk10-foldkit-6: 4/20 seeds found a hole + seed 2 step 37: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land closeIssue:2 ; rename Alpha ; rename B + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 9 step 40: two views: row "#4 Bravo", detail "#4 Deltafooter says 2019assign to guestc" + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt + seed 15 step 33: detail title: "#3 Crash on savestack trace attachedassign to guestcloserena", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan diff --git a/eval/results/desk-2026-11-11/r10-regraded/foldkit-7.txt b/eval/results/desk-2026-11-11/r10-regraded/foldkit-7.txt new file mode 100644 index 00000000..baa65a90 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/foldkit-7.txt @@ -0,0 +1,21 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +25/38 passed +== fuzz +desk10-foldkit-7: 1/20 seeds found a hole + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 diff --git a/eval/results/desk-2026-11-11/r10-regraded/foldkit-8.txt b/eval/results/desk-2026-11-11/r10-regraded/foldkit-8.txt new file mode 100644 index 00000000..4f5fd511 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/foldkit-8.txt @@ -0,0 +1,23 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +25/38 passed +== fuzz +desk10-foldkit-8: 2/20 seeds found a hole + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 18 step 35: row assignee: #4 shows bob, server dana + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail assign:4,guest ; fail closeIssue:4 ; land fetchComments:4 ; land closeIssue:3 ; user bob ; row-assign 0 ; add-label ui ; rename Delta ; assign ; row-close 0 ; fail closeIssue:1 ; rename Alpha ; close ; fail closeIssue:4 ; focus ; close ; focus ; row-assign 0 diff --git a/eval/results/desk-2026-11-11/r10-regraded/foldkit-9.txt b/eval/results/desk-2026-11-11/r10-regraded/foldkit-9.txt new file mode 100644 index 00000000..7f62c409 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/foldkit-9.txt @@ -0,0 +1,22 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +26/38 passed +== fuzz +desk10-foldkit-9: 2/20 seeds found a hole + seed 2 step 37: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land closeIssue:2 ; rename Alpha ; rename B + seed 9 step 40: row assignee: #4 shows null, server dana + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt diff --git a/eval/results/desk-2026-11-11/r10-regraded/react-1.txt b/eval/results/desk-2026-11-11/r10-regraded/react-1.txt new file mode 100644 index 00000000..d66e2727 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/react-1.txt @@ -0,0 +1,10 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk10-react-1: 1/20 seeds found a hole + seed 16 step 36: at rest: comments: #2 shows [guest: ping | guest: ping], server [guest: ping] + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchLabels:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; row-assign 0 ; close ; post ping ; assign ; land addComment:2,guest,ping ; land fetchIssue:2 ; fail assign:2,guest ; select #2 ; rename Bravo ; assign diff --git a/eval/results/desk-2026-11-11/r10-regraded/react-2.txt b/eval/results/desk-2026-11-11/r10-regraded/react-2.txt new file mode 100644 index 00000000..846c1800 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/react-2.txt @@ -0,0 +1,12 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk10-react-2: 2/20 seeds found a hole + seed 3 step 39: at rest: labels: #4 shows [docs], server [docs,ui] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; add-label bug ; fail addLabel:4,bug ; land fetchIssue:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; add-label p1 ; assign ; land closeIssue:4 ; fail removeL + seed 12 step 36: at rest: comments: #4 shows [guest: me too | guest: me too], server [guest: me too] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land assign:4,guest ; land fetchComments:1 ; land fetchLabels:1 ; post me too ; close ; fail renameIssue:4,Alpha ; assign ; land addComment:4,guest,me too ; close ; diff --git a/eval/results/desk-2026-11-11/r10-regraded/react-3.txt b/eval/results/desk-2026-11-11/r10-regraded/react-3.txt new file mode 100644 index 00000000..ba0de1c2 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/react-3.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk10-react-3: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/r10-regraded/react-4.txt b/eval/results/desk-2026-11-11/r10-regraded/react-4.txt new file mode 100644 index 00000000..8158bbf6 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/react-4.txt @@ -0,0 +1,23 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +27/38 passed +== fuzz +desk10-react-4: 3/20 seeds found a hole + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 12 step 39: row assignee: #1 shows guest, server null + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchComments:1 ; land fetchComments:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land fetchIssue:4 ; land fetchLabels:1 ; land fetchIssue:1 ; post me too ; close ; fail closeIssue:1 ; select #1 ; row-assign 0 ; assign ; land closeIssue:4 ; + seed 16 step 36: at rest: comments: #2 shows [guest: ping | guest: ping], server [guest: ping] + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchLabels:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; row-assign 0 ; close ; post ping ; assign ; land addComment:2,guest,ping ; land fetchIssue:2 ; fail assign:2,guest ; select #2 ; rename Bravo ; assign diff --git a/eval/results/desk-2026-11-11/r10-regraded/react-5.txt b/eval/results/desk-2026-11-11/r10-regraded/react-5.txt new file mode 100644 index 00000000..12bed383 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/react-5.txt @@ -0,0 +1,13 @@ +== hidden +77/77 passed +== overlap +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ an add and a remove at once: a declined add does not undo the remove +✗ two adds at once: a declined one does not take the other with it +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +31/38 passed +== fuzz +desk10-react-5: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/r10-regraded/react-6.txt b/eval/results/desk-2026-11-11/r10-regraded/react-6.txt new file mode 100644 index 00000000..f2d25d13 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/react-6.txt @@ -0,0 +1,32 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: open issues that load while a row close is out keep the row out +28/38 passed +== fuzz +desk10-react-6: 8/20 seeds found a hole + seed 2 step 36: two views: row "#2 Bravo", detail "#2 Alphalong goneNew titlerenameassign t" + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchLabels:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchComments:2 ; fail fetchIssue:2 ; land closeIssue:2 ; fail assign:2,guest ; focus ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land closeIssue:2 ; rename Alpha ; + seed 3 step 39: at rest: comments: #4 shows [bob: me too | bob: me too], server [bob: me too] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchIssue:4 ; row-assign 2 ; select #4 ; land fetchIssue:4 ; row-close 0 ; fail assign:4,bob ; land addLabel:4,p1 ; row-close 1 ; row-assign 0 ; land closeIssue:1 ; close ; row-assign 0 ; row-close 0 ; add-label ui ; filter closed ; add-label p1 ; assign ; land closeIssue:3 ; fail assign:3,bob + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 10 step 17: detail title: "#2 Old buglong goneNew titlerenameassign to guestclosefindin", server "Bravo" + focus ; focus ; filter closed ; fail fetchIssues:open ; focus ; land fetchIssues:closed ; row-assign 0 ; land assign:2,guest ; select #2 ; close ; rename Bravo ; fail fetchIssue:2 ; select #2 ; fail fetchLabels:2 ; land fetchIssue:2 ; row-assign 0 ; land renameIssue:2,Bravo + seed 15 step 39: at rest: labels: #4 shows [docs], server [docs,ui] + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchIssue:3 ; close ; row-assign 0 ; row-close 0 ; land assign:1,guest ; row-assign 0 ; select #4 ; land fetchLabels:3 ; assign ; add-label ui ; land fetchLabels:4 ; land fetchComments:3 ; assign ; land addLabel:4,ui ; assign ; rename + seed 16 step 38: detail title: "#2 Old buglong goneNew titlerenameassign to guestassign erro", server "Bravo" + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchComments:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; close ; post fixed? ; post ping ; assign ; land addComment:2,guest,fixed? ; land fetchIssue:2 ; fail addComment:2,guest,ping ; select #2 ; rename Br + seed 18 step 13: row title: #4 shows "Typo in footer", server "Delta" + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchIssue:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail assign:4,guest ; fail closeIssue:4 ; land closeIssue:1 ; land renameIssue:4,Delta + seed 19 step 17: detail title: "#3 Crash on savestack trace attachedNew titlerenameassign to", server "Alpha" + land fetchIssues:open ; user guest ; row-assign 1 ; row-close 2 ; land assign:3,guest ; fail closeIssue:4 ; select #3 ; post fixed? ; land fetchLabels:3 ; rename Alpha ; rename Delta ; row-assign 0 ; land assign:1,guest ; land fetchIssue:3 ; row-assign 1 ; land addComment:3,guest,fixed? ; land renameIssue:3,Alpha diff --git a/eval/results/desk-2026-11-11/r10-regraded/react-7.txt b/eval/results/desk-2026-11-11/r10-regraded/react-7.txt new file mode 100644 index 00000000..b724b7a1 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/react-7.txt @@ -0,0 +1,28 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +26/38 passed +== fuzz +desk10-react-7: 5/20 seeds found a hole + seed 3 step 40: at rest: labels: #4 shows [docs], server [docs,ui] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; add-label bug ; fail addLabel:4,bug ; land fetchIssue:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 12 step 34: row assignee: #4 shows dana, server guest + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; land search:Login fails ; land renameIssue:4,Alpha ; land fetchComments:1 ; land fetchComments:4 ; filter closed ; land closeIssue:1 ; fail assign:4,guest ; assign ; land search:Typo i + seed 15 step 39: two views: row "#3 Crash on save", detail "#3 Deltastack trace attachedassign to gu" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land closeIssue:4 ; land fetchIssu + seed 18 step 13: row title: #4 shows "Typo in footer", server "Delta" + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail closeIssue:3 ; fail closeIssue:4 ; land fetchComments:4 ; land renameIssue:4,Delta diff --git a/eval/results/desk-2026-11-11/r10-regraded/react-8.txt b/eval/results/desk-2026-11-11/r10-regraded/react-8.txt new file mode 100644 index 00000000..e376a7f1 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/react-8.txt @@ -0,0 +1,26 @@ +== hidden +✗ rows: row errors and detail errors leave each other alone +76/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +27/38 passed +== fuzz +desk10-react-8: 4/20 seeds found a hole + seed 2 step 15: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchComments:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail fetchIssue:2 ; land closeIssue:2 ; fail assign:2,guest + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 10 step 17: detail title: "#2 Old buglong goneassign to guestNew title RenamecloseComme", server "Bravo" + focus ; focus ; filter closed ; fail fetchIssues:open ; focus ; land fetchIssues:closed ; row-assign 0 ; land assign:2,guest ; select #2 ; close ; rename Bravo ; fail fetchIssue:2 ; select #2 ; fail fetchComments:2 ; land fetchIssue:2 ; row-assign 0 ; land renameIssue:2,Bravo + seed 15 step 39: at rest: comments: #4 shows [], server [guest: fixed?] + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchIssue:3 ; close ; row-assign 0 ; row-close 0 ; land assign:1,guest ; row-assign 0 ; select #4 ; land fetchComments:3 ; assign ; add-label ui ; land assign:4,guest ; land fetchLabels:3 ; assign ; land assign:4,guest ; assign ; rena diff --git a/eval/results/desk-2026-11-11/r10-regraded/react-9.txt b/eval/results/desk-2026-11-11/r10-regraded/react-9.txt new file mode 100644 index 00000000..0c5e52e0 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/react-9.txt @@ -0,0 +1,27 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +27/38 passed +== fuzz +desk10-react-9: 5/20 seeds found a hole + seed 2 step 37: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land closeIssue:2 ; rename Alpha ; rename B + seed 4 step 24: count: tab Issues (3), 2 open rows shown + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 9 step 40: two views: row "#4 Bravo", detail "#4 Deltafooter says 2019assign to guesta" + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt + seed 12 step 35: two views: row "#4 Alpha", detail "#4 Bravofooter says 2019assign to guestr" + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; land search:Login fails ; land renameIssue:4,Alpha ; land fetchComments:1 ; land fetchComments:4 ; filter closed ; land closeIssue:1 ; fail assign:4,guest ; assign ; land search:Typo i + seed 15 step 36: row title: #3 shows "Crash on save", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan diff --git a/eval/results/desk-2026-11-11/r10-regraded/rx-1.txt b/eval/results/desk-2026-11-11/r10-regraded/rx-1.txt new file mode 100644 index 00000000..926ea8de --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/rx-1.txt @@ -0,0 +1,7 @@ +== hidden +77/77 passed +== overlap +✗ base: a detail that loads while a rename is out keeps the new title +37/38 passed +== fuzz +desk10-rx-1: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/r10-regraded/rx-2.txt b/eval/results/desk-2026-11-11/r10-regraded/rx-2.txt new file mode 100644 index 00000000..19872761 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/rx-2.txt @@ -0,0 +1,15 @@ +== hidden +✗ base: a failing list leaves the detail alone +76/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk10-rx-2: 3/20 seeds found a hole + seed 11 step 33: detail title: "#4 Typo in footerfooter says 2019retryfinding related helpas", server "Delta" + tab help ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; focus ; tab help ; tab issues ; land fetchIssues:open ; row-assign 1 ; land assign:3,guest ; filter closed ; user ann ; row-assign 0 ; focus ; land fetchIssues:open ; land assign:1,ann ; select #4 ; close ; land fetchIssues:closed ; close ; land fetchIssue:4 ; fail closeIssue:4 ; row-assign 0 ; close ; land fetchComments:4 + seed 15 step 40: two views: row "#2 Bravo", detail "#2 Deltalong goneretrysaving filescrash " + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; fail fetchIssues:open ; tab help ; tab issues ; land fetchIssues:open ; focus ; filter closed ; land fetchIssues:closed ; select #2 ; land fetchIssue:2 ; rename Bravo ; select #2 ; assign ; land fetchComments:2 ; row-assign 0 ; select #2 ; land fetchLabels:2 ; assign ; remove-label 0 ; land assign:2,guest ; land search:Old bu + seed 18 step 36: detail title: "#4 Typo in footerfooter says 2019retrysaving filescrash repo", server "Delta" + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchIssue:4 ; assign ; row-assign 0 ; row-close 1 ; assign ; rename Delta ; fail assign:4,guest ; fail closeIssue:4 ; land closeIssue:1 ; land renameIssue:4,Delta ; user bob ; row-assign 0 ; row-close 1 ; rename Delta ; assign ; row-assign 0 ; land fetchLabels:4 ; rename Alpha ; close ; fail renameIssue:4,Alpha ; focus ; close ; focus ; r diff --git a/eval/results/desk-2026-11-11/r10-regraded/rx-3.txt b/eval/results/desk-2026-11-11/r10-regraded/rx-3.txt new file mode 100644 index 00000000..272a3dab --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/rx-3.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk10-rx-3: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/r10-regraded/rx-4.txt b/eval/results/desk-2026-11-11/r10-regraded/rx-4.txt new file mode 100644 index 00000000..79a74023 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/rx-4.txt @@ -0,0 +1,9 @@ +== hidden +✗ base: help: selecting after a failure shows loading…, not the old failure +76/77 passed +== overlap +✗ base: a detail that loads while a rename is out keeps the new title +✗ a label added on one issue does not show on another +36/38 passed +== fuzz +desk10-rx-4: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/r10-regraded/rx-5.txt b/eval/results/desk-2026-11-11/r10-regraded/rx-5.txt new file mode 100644 index 00000000..d6cbdf46 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/rx-5.txt @@ -0,0 +1,9 @@ +== hidden +77/77 passed +== overlap +✗ base: a detail that loads while a rename is out keeps the new title +37/38 passed +== fuzz +desk10-rx-5: 1/20 seeds found a hole + seed 8 step 34: open row in closed list: #4 + land fetchIssues:open ; row-close 1 ; filter closed ; fail fetchIssues:closed ; user bob ; land closeIssue:3 ; tab help ; tab issues ; land fetchIssues:open ; row-assign 1 ; select #1 ; land fetchComments:1 ; select #4 ; land fetchComments:4 ; row-close 1 ; row-assign 0 ; land fetchLabels:1 ; row-assign 0 ; user guest ; fail assign:1,guest ; row-close 0 ; user ann ; focus ; land assign:4,guest diff --git a/eval/results/desk-2026-11-11/r10-regraded/rx-6.txt b/eval/results/desk-2026-11-11/r10-regraded/rx-6.txt new file mode 100644 index 00000000..8d3ca347 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/rx-6.txt @@ -0,0 +1,11 @@ +== hidden +77/77 passed +== overlap +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: a post on one issue does not show on another +✗ a label added on one issue does not show on another +35/38 passed +== fuzz +desk10-rx-6: 1/20 seeds found a hole + seed 19 step 39: detail title: "#1 Alphaon Safari onlysaving filescrash reportsassign to gue", server "Login fails" + land fetchIssues:open ; user guest ; row-assign 1 ; row-close 2 ; land assign:3,guest ; fail closeIssue:4 ; select #3 ; post fixed? ; land fetchIssue:3 ; rename Alpha ; rename Delta ; close ; land fetchLabels:3 ; select #1 ; filter closed ; close ; land search:Crash on save ; land renameIssue:3,Delta ; filter open ; rename Alpha ; row-close 1 ; select #1 ; assign ; fail renameIssue:3,Delta ; s diff --git a/eval/results/desk-2026-11-11/r10-regraded/rx-7.txt b/eval/results/desk-2026-11-11/r10-regraded/rx-7.txt new file mode 100644 index 00000000..aa905117 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/rx-7.txt @@ -0,0 +1,23 @@ +== hidden +✗ base: an accepted post stays after switching issues and back +76/77 passed +== overlap +✗ base: two issues assigned at once: declining one reverts only that row +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: a declined close keeps an assignment in flight on another issue +✗ base: a declined assignment keeps a close in flight on another issue +✗ base: two closes at once: a declined one does not undo the other +✗ base: two posts at once: a declined one does not take the other with it +✗ two adds at once: a declined one does not take the other with it +✗ two closes at once: a declined one leaves the other off the count +30/38 passed +== fuzz +desk10-rx-7: 4/20 seeds found a hole + seed 2 step 36: at rest: comments: #2 shows [guest: fixed? | guest: fixed?], server [guest: fixed?] + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail search:Delta ; fail assign:2,guest ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail renameIssue:2,Delta ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; row-assign 0 ; filter open ; rename Alpha ; rename Bravo + seed 8 step 34: open row in closed list: #4 + land fetchIssues:open ; row-close 1 ; filter closed ; fail fetchIssues:closed ; user bob ; land closeIssue:3 ; tab help ; tab issues ; land fetchIssues:open ; row-assign 1 ; select #1 ; land fetchComments:1 ; select #4 ; land fetchComments:4 ; row-close 1 ; row-assign 0 ; land fetchLabels:1 ; row-assign 0 ; user guest ; fail assign:1,guest ; row-close 0 ; user ann ; focus ; land assign:4,guest + seed 13 step 32: detail title: "#1 Login failson Safari onlyassign to guestrenamecloseguest:", server "Bravo" + filter closed ; fail fetchIssues:open ; land fetchIssues:closed ; row-assign 0 ; filter open ; land assign:2,guest ; land fetchIssues:open ; select #3 ; row-close 1 ; land fetchLabels:3 ; fail fetchIssue:3 ; land fetchComments:3 ; fail closeIssue:3 ; row-assign 0 ; select #3 ; land assign:1,guest ; select #1 ; land fetchLabels:1 ; land fetchComments:1 ; land fetchIssue:1 ; add-label bug ; land + seed 15 step 27: closed row shown: #3 is closed on the server + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; row-assign 1 ; select #4 ; select #1 ; land fetchComments:1 ; row-close 2 ; land fetchLabels:1 ; land fetchLabels:4 ; land fetchIssue:1 ; land fetchComments:3 ; rename Alpha ; row-assign 0 ; land search:Alpha ; assign ; diff --git a/eval/results/desk-2026-11-11/r10-regraded/rx-8.txt b/eval/results/desk-2026-11-11/r10-regraded/rx-8.txt new file mode 100644 index 00000000..4cd277b9 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/rx-8.txt @@ -0,0 +1,9 @@ +== hidden +77/77 passed +== overlap +✗ base: a detail that loads while a rename is out keeps the new title +37/38 passed +== fuzz +desk10-rx-8: 1/20 seeds found a hole + seed 13 step 31: page error: declined + filter closed ; fail fetchIssues:open ; land fetchIssues:closed ; row-assign 0 ; filter open ; land assign:2,guest ; land fetchIssues:open ; select #3 ; row-close 1 ; land fetchIssue:3 ; fail fetchLabels:3 ; land fetchComments:3 ; land search:Crash on save ; land closeIssue:3 ; fail fetchIssues:open ; assign ; assign ; land assign:3,guest ; land fetchIssues:open ; assign ; land assign:3,guest diff --git a/eval/results/desk-2026-11-11/r10-regraded/rx-9.txt b/eval/results/desk-2026-11-11/r10-regraded/rx-9.txt new file mode 100644 index 00000000..767e41fe --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/rx-9.txt @@ -0,0 +1,18 @@ +== hidden +✗ base: selecting after not found shows loading, not the old message +✗ base: help: selecting after a failure shows loading…, not the old failure +✗ related help landing out of order shows the selected issue's +74/77 passed +== overlap +✗ base: a declined assignment keeps a rename in flight on the same issue +✗ base: a declined rename keeps an assignment in flight on the same issue +✗ base: a detail that loads while a rename is out keeps the new title +✗ a label added on one issue does not show on another +✗ rows: a declined row close leaves a rename out on another issue +33/38 passed +== fuzz +desk10-rx-9: 2/20 seeds found a hole + seed 8 step 12: open row in closed list: #1 + land fetchIssues:open ; row-close 1 ; filter closed ; fail fetchIssues:closed ; select #4 ; land closeIssue:3 ; user ann ; land fetchComments:4 ; focus ; land fetchIssue:4 ; rename Bravo ; land renameIssue:4,Bravo + seed 12 step 38: at rest: labels: #1 shows [docs,bug], server [] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-close 1 ; row-assign 0 ; row-assign 0 ; land fetchComments:1 ; land fetchLabels:1 ; land fetchIssue:4 ; rename Bravo ; land fetchComments:4 ; assign ; land search:Typo in footer ; land fetchIssue:1 ; land fetchLabels:4 ; post me too ; close ; fail renameIssue:4,Bravo ; select #1 ; row-assign 0 ; assig diff --git a/eval/results/desk-2026-11-11/r10-regraded/vanilla-1.txt b/eval/results/desk-2026-11-11/r10-regraded/vanilla-1.txt new file mode 100644 index 00000000..03fdec29 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/vanilla-1.txt @@ -0,0 +1,9 @@ +== hidden +✗ base: settings: saving an invalid name asks nothing +76/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk10-vanilla-1: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/r10-regraded/vanilla-2.txt b/eval/results/desk-2026-11-11/r10-regraded/vanilla-2.txt new file mode 100644 index 00000000..fc3bc1c6 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/vanilla-2.txt @@ -0,0 +1,10 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk10-vanilla-2: 1/20 seeds found a hole + seed 12 step 36: at rest: comments: #4 shows [guest: me too | guest: me too], server [guest: me too] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land assign:4,guest ; land fetchComments:1 ; land fetchLabels:1 ; post me too ; close ; fail renameIssue:4,Alpha ; assign ; land addComment:4,guest,me too ; close ; diff --git a/eval/results/desk-2026-11-11/r10-regraded/vanilla-3.txt b/eval/results/desk-2026-11-11/r10-regraded/vanilla-3.txt new file mode 100644 index 00000000..54f0629d --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/vanilla-3.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk10-vanilla-3: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/r10-regraded/vanilla-4.txt b/eval/results/desk-2026-11-11/r10-regraded/vanilla-4.txt new file mode 100644 index 00000000..a6874019 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/vanilla-4.txt @@ -0,0 +1,22 @@ +== hidden +✗ the results for a related hit may land after the help page opens +✗ a related item that fails can be retried on the help page +75/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +30/38 passed +== fuzz +desk10-vanilla-4: 3/20 seeds found a hole + seed 2 step 37: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land closeIssue:2 ; rename Alpha ; rename B + seed 12 step 34: row assignee: #4 shows dana, server guest + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; land search:Login fails ; land renameIssue:4,Alpha ; land fetchComments:1 ; land fetchComments:4 ; filter closed ; land closeIssue:1 ; fail assign:4,guest ; assign ; land search:Typo i + seed 15 step 31: detail title: "retry#3 stack trace attachedrenameassign to guestcloserename", server "Crash on save" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan diff --git a/eval/results/desk-2026-11-11/r10-regraded/vanilla-5.txt b/eval/results/desk-2026-11-11/r10-regraded/vanilla-5.txt new file mode 100644 index 00000000..6b13ef2d --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/vanilla-5.txt @@ -0,0 +1,16 @@ +== hidden +✗ the results for a related hit may land after the help page opens +✗ a related item that fails can be retried on the help page +75/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +32/38 passed +== fuzz +desk10-vanilla-5: 1/20 seeds found a hole + seed 10 step 37: row assignee: #2 shows guest, server eve + tab help ; tab issues ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; fail fetchIssues:open ; focus ; tab help ; tab issues ; land fetchIssues:open ; filter closed ; land fetchIssues:closed ; assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; rename Delta ; (move failed: locator.fill: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 200 diff --git a/eval/results/desk-2026-11-11/r10-regraded/vanilla-6.txt b/eval/results/desk-2026-11-11/r10-regraded/vanilla-6.txt new file mode 100644 index 00000000..d8a48df3 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/vanilla-6.txt @@ -0,0 +1,15 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: open issues that load while a row close is out keep the row out +29/38 passed +== fuzz +desk10-vanilla-6: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/r10-regraded/vanilla-7.txt b/eval/results/desk-2026-11-11/r10-regraded/vanilla-7.txt new file mode 100644 index 00000000..eba1ff32 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/vanilla-7.txt @@ -0,0 +1,17 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +29/38 passed +== fuzz +desk10-vanilla-7: 1/20 seeds found a hole + seed 9 step 40: row assignee: #4 shows guest, server dana + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt diff --git a/eval/results/desk-2026-11-11/r10-regraded/vanilla-8.txt b/eval/results/desk-2026-11-11/r10-regraded/vanilla-8.txt new file mode 100644 index 00000000..550ad65c --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/vanilla-8.txt @@ -0,0 +1,18 @@ +== hidden +✗ base: retry after not found shows the issue +76/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +33/38 passed +== fuzz +desk10-vanilla-8: 3/20 seeds found a hole + seed 3 step 38: count: tab Issues (3), 2 open rows shown + assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchIssues:open ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land fetchLabels:4 ; add-label bug + seed 8 step 36: detail title: "#3 Crash on savestack trace attachedassign to bobrenameclose", server "Alpha" + land fetchIssues:open ; row-close 1 ; post me too ; (move failed: locator.fill: Timeout 2000ms exceeded.) ; select #4 ; land fetchComments:4 ; land fetchLabels:4 ; fail closeIssue:3 ; close ; remove-label 0 ; close ; rename Bravo ; land renameIssue:4,Bravo ; user bob ; add-label ui ; fail closeIssue:4 ; land addLabel:4,ui ; select #3 ; focus ; fail fetchLabels:3 ; row-assign 0 ; assign ; land + seed 11 step 46: detail title: "#3 Crash on savestack trace attachedassign to guestrenameclo", server "Bravo" + close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchIssues:open ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; post fixed? ; (move failed: locator.fill: Timeo diff --git a/eval/results/desk-2026-11-11/r10-regraded/vanilla-9.txt b/eval/results/desk-2026-11-11/r10-regraded/vanilla-9.txt new file mode 100644 index 00000000..2936dda0 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/vanilla-9.txt @@ -0,0 +1,15 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +29/38 passed +== fuzz +desk10-vanilla-9: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/stats.py b/eval/results/desk-2026-11-11/stats.py new file mode 100644 index 00000000..894bb5c8 --- /dev/null +++ b/eval/results/desk-2026-11-11/stats.py @@ -0,0 +1,36 @@ +import re, random, json +rows={} +for l in open('summary.txt'): + a,n,h,o,z=re.match(r'(\w+)-(\d) hidden/overlap=(\d+)/77 (\d+)/38\s+fuzz=(\d+)/20',l).groups() + rows[(a,int(n))]=dict(h=77-int(h),o=38-int(o),z=int(z)) +excl={} # fuzz holes traced to legal holds: (arm,n) -> count +def D(k): r=rows[k]; return r['h']+r['o']+r['z']-excl.get(k,0) +def perm(x,y,seed=11,N=100000): + random.seed(seed); obs=sum(y)/len(y)-sum(x)/len(x); pool=x+y; c=0 + for _ in range(N): + random.shuffle(pool); xx=pool[:len(x)]; yy=pool[len(x):] + if sum(yy)/len(yy)-sum(xx)/len(xx)>=obs-1e-12: c+=1 + return (c+1)/(N+1) +def boot(x,y,seed=11,N=10000): + random.seed(seed); d=[] + for _ in range(N): + xx=[random.choice(x) for _ in x]; yy=[random.choice(y) for _ in y] + d.append(sum(xx)/len(xx)-sum(yy)/len(yy)) + d.sort(); return d[int(.05*N)], d[int(.95*N)-1] +usd=json.load(open('usd.json')) +small=range(4,10); front=range(1,4); A=('rx','react','vanilla','foldkit') +for a in A: + for tier,rg in (('frontier',front),('small',small)): + v=[D((a,n)) for n in rg]; c=sum(1 for x in v if x==0) + hz=[sum(rows[(a,n)][k] for n in rg)/len(rg) for k in 'hoz'] + cost=[x[0] for x in usd[f'{a}-{tier}']]; m=sum(cost)/len(cost) + cpc=f"${sum(cost)/c:.3f}" if c else "undefined" + print(f" {a:8}{tier:9} mean {sum(v)/len(v):5.2f} per desk {v} h/o/z {hz[0]:.1f}/{hz[1]:.1f}/{hz[2]:.1f} correct {c} $/run {m:.3f} $/correct {cpc}") +S={a:[D((a,n)) for n in small] for a in A} +ps={'H11a rx holds if upper < 1: {hi<1}") +lo,hi=boot(S['rx'],F['react']); print(f" (H10c again) small rx - frontier react: {sum(S['rx'])/6-sum(F['react'])/3:.2f}, 90% CI [{lo:.2f}, {hi:.2f}] -> upper < 1: {hi<1}") diff --git a/eval/results/desk-2026-11-11/stats.txt b/eval/results/desk-2026-11-11/stats.txt new file mode 100644 index 00000000..7a7f261b --- /dev/null +++ b/eval/results/desk-2026-11-11/stats.txt @@ -0,0 +1,13 @@ + rx frontier mean 2.00 per desk [0, 5, 1] h/o/z 0.0/1.0/1.0 correct 1 $/run 2.048 $/correct $6.144 + rx small mean 3.83 per desk [3, 3, 4, 2, 9, 2] h/o/z 1.3/1.0/1.5 correct 0 $/run 0.092 $/correct undefined + react frontier mean 2.00 per desk [0, 2, 4] h/o/z 0.0/1.3/0.7 correct 1 $/run 1.384 $/correct $4.151 + react small mean 9.67 per desk [13, 2, 17, 14, 1, 11] h/o/z 0.2/7.2/2.3 correct 0 $/run 0.069 $/correct undefined + vanilla frontier mean 3.00 per desk [3, 3, 3] h/o/z 0.0/2.0/1.0 correct 0 $/run 1.265 $/correct undefined + vanilla small mean 14.50 per desk [12, 16, 11, 9, 20, 19] h/o/z 0.7/10.5/3.3 correct 0 $/run 0.077 $/correct undefined + foldkit frontier mean 2.67 per desk [4, 2, 2] h/o/z 0.0/2.0/0.7 correct 0 $/run 3.633 $/correct undefined + foldkit small mean 13.17 per desk [17, 13, 13, 12, 10, 14] h/o/z 0.2/10.8/2.2 correct 0 $/run 0.175 $/correct undefined + H11b rx holds if upper < 1: False + (H10c again) small rx - frontier react: 1.83, 90% CI [-0.33, 4.17] -> upper < 1: False diff --git a/eval/results/desk-2026-11-11/summary.txt b/eval/results/desk-2026-11-11/summary.txt new file mode 100644 index 00000000..fee9e073 --- /dev/null +++ b/eval/results/desk-2026-11-11/summary.txt @@ -0,0 +1,36 @@ +rx-1 hidden/overlap=77/77 38/38 fuzz=0/20 +rx-2 hidden/overlap=77/77 36/38 fuzz=3/20 +rx-3 hidden/overlap=77/77 37/38 fuzz=0/20 +rx-4 hidden/overlap=76/77 37/38 fuzz=1/20 +rx-5 hidden/overlap=77/77 37/38 fuzz=2/20 +rx-6 hidden/overlap=75/77 38/38 fuzz=2/20 +rx-7 hidden/overlap=76/77 38/38 fuzz=1/20 +rx-8 hidden/overlap=75/77 34/38 fuzz=3/20 +rx-9 hidden/overlap=75/77 38/38 fuzz=0/20 +react-1 hidden/overlap=77/77 38/38 fuzz=0/20 +react-2 hidden/overlap=77/77 36/38 fuzz=0/20 +react-3 hidden/overlap=77/77 36/38 fuzz=2/20 +react-4 hidden/overlap=77/77 27/38 fuzz=2/20 +react-5 hidden/overlap=76/77 37/38 fuzz=0/20 +react-6 hidden/overlap=77/77 26/38 fuzz=5/20 +react-7 hidden/overlap=77/77 28/38 fuzz=4/20 +react-8 hidden/overlap=77/77 37/38 fuzz=0/20 +react-9 hidden/overlap=77/77 30/38 fuzz=3/20 +vanilla-1 hidden/overlap=77/77 36/38 fuzz=1/20 +vanilla-2 hidden/overlap=77/77 36/38 fuzz=1/20 +vanilla-3 hidden/overlap=77/77 36/38 fuzz=1/20 +vanilla-4 hidden/overlap=77/77 29/38 fuzz=3/20 +vanilla-5 hidden/overlap=77/77 26/38 fuzz=4/20 +vanilla-6 hidden/overlap=77/77 29/38 fuzz=2/20 +vanilla-7 hidden/overlap=77/77 29/38 fuzz=0/20 +vanilla-8 hidden/overlap=75/77 27/38 fuzz=7/20 +vanilla-9 hidden/overlap=75/77 25/38 fuzz=4/20 +foldkit-1 hidden/overlap=77/77 36/38 fuzz=2/20 +foldkit-2 hidden/overlap=77/77 36/38 fuzz=0/20 +foldkit-3 hidden/overlap=77/77 36/38 fuzz=0/20 +foldkit-4 hidden/overlap=77/77 25/38 fuzz=4/20 +foldkit-5 hidden/overlap=77/77 27/38 fuzz=2/20 +foldkit-6 hidden/overlap=77/77 26/38 fuzz=1/20 +foldkit-7 hidden/overlap=77/77 27/38 fuzz=1/20 +foldkit-8 hidden/overlap=76/77 30/38 fuzz=1/20 +foldkit-9 hidden/overlap=77/77 28/38 fuzz=4/20 diff --git a/eval/results/desk-2026-11-11/usd.json b/eval/results/desk-2026-11-11/usd.json new file mode 100644 index 00000000..c27bada1 --- /dev/null +++ b/eval/results/desk-2026-11-11/usd.json @@ -0,0 +1 @@ +{"rx-frontier": [[2.1487032, 24, 537175.8, 1], [2.5559634, 26, 638990.85, 2], [1.4391163999999999, 14, 359779.1, 3]], "react-frontier": [[1.4005798, 10, 350144.95, 1], [1.25414, 12, 313535, 2], [1.4962373999999998, 16, 374059.35, 3]], "vanilla-frontier": [[1.2282464, 9, 307061.6, 1], [1.4704583999999998, 16, 367614.6, 2], [1.0970423999999999, 12, 274260.6, 3]], "foldkit-frontier": [[3.5695232, 39, 892380.8, 1], [4.010596, 39, 1002649, 2], [3.3185599999999997, 34, 829640, 3]], "rx-small": [[0.12365911, 21, 1236591.1, 4], [0.09165008, 15, 916500.8, 5], [0.07712176, 14, 771217.6, 6], [0.080014315, 12, 800143.15, 7], [0.064075775, 11, 640757.75, 8], [0.117098075, 20, 1170980.75, 9]], "react-small": [[0.088643315, 15, 886433.15, 4], [0.049387755, 10, 493877.55, 5], [0.068611995, 13, 686119.95, 6], [0.05683599499999999, 8, 568359.95, 7], [0.096263405, 16, 962634.05, 8], [0.05682778499999999, 9, 568277.85, 9]], "vanilla-small": [[0.07337236, 8, 733723.6, 4], [0.066924585, 9, 669245.85, 5], [0.09436494499999999, 13, 943649.45, 6], [0.06235414, 7, 623541.4, 7], [0.099328465, 16, 993284.65, 8], [0.068555105, 7, 685551.05, 9]], "foldkit-small": [[0.182765515, 33, 1827655.15, 4], [0.1660664, 32, 1660664, 5], [0.136690155, 29, 1366901.55, 6], [0.170539045, 41, 1705390.4500000002, 7], [0.23715972999999996, 59, 2371597.3, 8], [0.15544205, 37, 1554420.5, 9]]} \ No newline at end of file diff --git a/eval/tasks/desk9/brief-plain.md b/eval/tasks/desk9/brief-plain.md index d34fdeb3..bb82fb33 100644 --- a/eval/tasks/desk9/brief-plain.md +++ b/eval/tasks/desk9/brief-plain.md @@ -67,7 +67,8 @@ Elements the tests look for carry these `data-testid`s. Text in quotes must appe - The selected issue comes from `fetchIssue(id)`. While it loads: `loading #`. - An issue: `# ` and its body. - `null`: `detail error: issue #<id> not found`. A failure: `detail error: <message>`. - - A button `retry` (`data-testid="retry"`) calls `fetchIssue(id)` again and shows its answer. + - While an issue is selected, the detail always has a button `retry` (`data-testid="retry"`), whether it shows the + issue, `not found` or a failure. It calls `fetchIssue(id)` again and shows its answer. 5. **Assign.** While an issue is selected, the detail has a button `assign` (`data-testid="assign"`) reading `assign to <current user>`. Clicking it: - shows the current user as the issue's assignee in its list row straight away; diff --git a/eval/tasks/tracker/hidden-overlap.mjs b/eval/tasks/tracker/hidden-overlap.mjs index 39132959..98f5ae60 100644 --- a/eval/tasks/tracker/hidden-overlap.mjs +++ b/eval/tasks/tracker/hidden-overlap.mjs @@ -1,5 +1,6 @@ // Round 3 hidden tests: overlapping requests and writes. Never shown to the agent. import { D1, OPEN, selected } from "./fixtures.mjs"; +const AFTER = OPEN.map(i => i.id === 1 ? { ...i, assignee: "ann" } : i); // the server once #1 is assigned and #3 declined export default [ { name: "two issues assigned at once: declining one reverts only that row", async run(app) { @@ -9,8 +10,11 @@ export default [ await app.clickText("issue", "#1 "); await app.land("fetchIssue:1", D1); await app.click("assign"); await app.fail("assign:3,ann", "declined"); - await app.see("list", "#3 Crash on save (unassigned)", "#1 Login fails (ann)"); + // #3 may revert now, or stay "(ann)" until #1 lands too: a list held whole until both settle is a legal hold (.rx H-11, + // Solid D-8), not a defect. What must not happen is the decline taking #1 with it. + await app.see("list", "#1 Login fails (ann)"); await app.land("assign:1,ann", { id: 1, assignee: "ann" }); + await app.landIfAsked("fetchIssues:open", AFTER); // an app that re-asks the list after a write holds until it answers await app.see("list", "#3 Crash on save (unassigned)", "#1 Login fails (ann)"); } }, { name: "a list that loads while an assignment is out keeps the new assignee", async run(app) { diff --git a/ref/compile/rt-web.ts b/ref/compile/rt-web.ts index 849d001d..b3e13828 100644 --- a/ref/compile/rt-web.ts +++ b/ref/compile/rt-web.ts @@ -177,7 +177,7 @@ export function createApp(opts: { root: any; api?: Record<string, (...a: any[]) const [store, setStore] = (createOptimisticStore as any)((draft: any) => { const r = fn(draft); if (r instanceof Flight || r instanceof Promise) return (async () => assignDraft(draft, r instanceof Flight ? await r.first : await r))(); assignDraft(draft, r); }, key ? [] : {}, key ? { key } : undefined); const get = () => store; storeAccs.add(get); const plain = () => { throw new Error("a plain write to an optimistic store is not lowered yet"); }; - return [get, { set: plain, path: plain, guess: (v, keys = []) => setStore((d: any) => { if (keys.length === 0) { assignDraft(d, typeof v === "function" ? (v as (x: unknown) => unknown)(d) : v); return; } let o = d; for (const k of keys.slice(0, -1)) o = o[k]; const last = keys[keys.length - 1]; o[last] = typeof v === "function" ? (v as (x: unknown) => unknown)(o[last]) : v; }) }]; + return [get, { set: plain, path: plain, guess: (v, keys = []) => setStore((d: any) => { if (keys.length === 0) return typeof v === "function" ? (v as (x: unknown) => unknown)(d) : v; /* a whole guess is the setter\'s return, which Solid diffs shallowly against the view as guesses */ let o = d; for (const k of keys.slice(0, -1)) o = o[k]; const last = keys[keys.length - 1]; o[last] = typeof v === "function" ? (v as (x: unknown) => unknown)(o[last]) : v; }) }]; } if (mode.optimistic) { const [truth, setTruth] = createSignal(fn as any, opts as any); diff --git a/ref/compile/rt.ts b/ref/compile/rt.ts index ad40b5f5..333c7b64 100644 --- a/ref/compile/rt.ts +++ b/ref/compile/rt.ts @@ -187,7 +187,7 @@ export function createApp(opts: { api?: Record<string, (...a: any[]) => unknown> const [store, setStore] = (createOptimisticStore as any)((draft: any) => { const r = fn(draft); if (r instanceof Flight || r instanceof Promise) return (async () => assignDraft(draft, r instanceof Flight ? await r.first : await r))(); assignDraft(draft, r); }, key ? [] : {}, key ? { key } : undefined); const get = () => store; storeAccs.add(get); const plain = () => { throw new Error("a plain write to an optimistic store is not lowered yet"); }; - return [get, { set: plain, path: plain, guess: (v, keys = []) => setStore((d: any) => { if (keys.length === 0) { assignDraft(d, typeof v === "function" ? (v as (x: unknown) => unknown)(d) : v); return; } let o = d; for (const k of keys.slice(0, -1)) o = o[k]; const last = keys[keys.length - 1]; o[last] = typeof v === "function" ? (v as (x: unknown) => unknown)(o[last]) : v; }) }]; + return [get, { set: plain, path: plain, guess: (v, keys = []) => setStore((d: any) => { if (keys.length === 0) return typeof v === "function" ? (v as (x: unknown) => unknown)(d) : v; /* a whole guess is the setter\'s return, which Solid diffs shallowly against the view as guesses */ let o = d; for (const k of keys.slice(0, -1)) o = o[k]; const last = keys[keys.length - 1]; o[last] = typeof v === "function" ? (v as (x: unknown) => unknown)(o[last]) : v; }) }]; } if (mode.optimistic) { const [truth, setTruth] = createSignal(fn as any, opts as any); diff --git a/ref/oracle/lower.ts b/ref/oracle/lower.ts index b26eae1c..11d87b12 100644 --- a/ref/oracle/lower.ts +++ b/ref/oracle/lower.ts @@ -472,7 +472,7 @@ export class Oracle { const path = s.target.path.map(p => typeof p === "string" ? p : String(untrack(() => this.ev(p, scope, env)))); if (s.k === "optimistic") { if (!b.setGuess) throw new Error("OPTIMISTIC_NO_QUESTION"); - b.setGuess((d: any) => { if (path.length === 0) { assignDraft(d, typeof v === "function" ? (v as (x: unknown) => unknown)(d) : v); return; } let o = d; for (const k of path.slice(0, -1)) o = o[k]; const last = path[path.length - 1]; o[last] = typeof v === "function" ? (v as (x: unknown) => unknown)(o[last]) : v; }); // W-6 per key + b.setGuess((d: any) => { if (path.length === 0) return typeof v === "function" ? (v as (x: unknown) => unknown)(d) : v; /* a whole guess is the setter\'s return, which Solid diffs shallowly against the view as guesses */ let o = d; for (const k of path.slice(0, -1)) o = o[k]; const last = path[path.length - 1]; o[last] = typeof v === "function" ? (v as (x: unknown) => unknown)(o[last]) : v; }); // W-6 per key return; } if (path.length === 0) b.set(v);