From 5c31edd8d885936e534036aade04bb3d8d176dfe Mon Sep 17 00:00:00 2001 From: os-steve Date: Tue, 27 Oct 2026 10:40:13 +0000 Subject: [PATCH 1/2] fix(skills): make automation eval fixtures 5, 7, 8 agree with their own assertions Three ideal answers in flows-triggers-approvals.json contradicted the assertions that grade them, so a correct answer graded wrong: - eval 5 spelled the negated error-label string inside the ideal answer; it now says it in words (a bare error label routes nothing). - eval 7 never spelled the `type: 'position'` approver it requires and spelled the negated positions check; it now names `approvers: [{ type: 'position', value: 'finance_manager' }]` and says the negation in words. - eval 8 never spelled the `type: 'screen'` flow kind it requires. No fixture added, net 0 lines in skills/. The token ratchet row for the file moves to the new measurement (2724 -> 2734) after paying down 32 bytes of prose in evals 5 and 7. Claude-Session: https://claude.ai/code/session_01ADHkmT8qQiLPD3BRTdVxGH Co-authored-by: Claude --- scripts/check-skills-token-ratchet.mjs | 11 ++++++++++- .../evals/flows-triggers-approvals.json | 6 +++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/scripts/check-skills-token-ratchet.mjs b/scripts/check-skills-token-ratchet.mjs index 22e6424f4c8..74134bf99b9 100644 --- a/scripts/check-skills-token-ratchet.mjs +++ b/scripts/check-skills-token-ratchet.mjs @@ -829,7 +829,16 @@ export const CEILINGS = new Map([ // = 2724; headroom was 3 bytes, 0 tokens, so the ceiling moves +297, pinned // AT the measurement. Same maintainer ruling as the SKILL.md row above // (2026-10-27, tap A verbatim and untranslated): 「A:先削重复,余额按实测提高」. - ['skills/objectstack-automation/evals/flows-triggers-approvals.json', 2724], + // 10893 -> 10934 (#26515): no new fixture — three ideal answers rewritten + // to agree with their own assertions (eval 5 spelled the negated error-label + // string, eval 7 lacked the `type: 'position'` approver and spelled the + // negated positions check, eval 8 lacked `type: 'screen'`). Paid down first: + // evals 5 and 7 shortened by 25 and 7 bytes; the `approvers` line eval 7 + // owed costs more than that. +41 bytes -> ceil(10934 / 4) = 2734; headroom + // was 3 bytes, 0 tokens, so the ceiling moves +10, pinned AT the + // measurement. Same maintainer ruling as the SKILL.md row above (2026-10-27, + // tap A verbatim and untranslated): 「A:先削重复,余额按实测提高」. + ['skills/objectstack-automation/evals/flows-triggers-approvals.json', 2734], // 894 -> 1095 (#26179): one regression case for the field permission `when` // (the contract prompt: lock the legal terms for finance from approval on, // legal keeps the edit) — expected: `when` on the finance set's entries with diff --git a/skills/objectstack-automation/evals/flows-triggers-approvals.json b/skills/objectstack-automation/evals/flows-triggers-approvals.json index 793f4beaa3f..2c7246428fc 100644 --- a/skills/objectstack-automation/evals/flows-triggers-approvals.json +++ b/skills/objectstack-automation/evals/flows-triggers-approvals.json @@ -44,7 +44,7 @@ { "id": 5, "prompt": "Add a step that calls our `pricing.recalc` function to compute the discount and saves it on the quote, then posts the quote to our tax service over HTTP; if the tax call fails, route to a handler node instead of failing the run.", - "expected_output": "Emits a `script` node with `config.function: 'pricing.recalc'` (registered via `defineStack({ functions })`), `inputs`, and `outputVariable: 'recalc'`; the function is a pure compute step, so a following `update_record` persists it with `fields: { discount: { dialect: 'cel', source: 'recalc.discount' } }`. The `http` node sets `timeoutMs`. Failure routing is a `type: 'fault'` edge — a `label: 'error'` alone routes nothing.", + "expected_output": "Emits a `script` node with `config.function: 'pricing.recalc'` (registered via `defineStack({ functions })`), `inputs`, and `outputVariable: 'recalc'`; a pure compute step, so a following `update_record` persists it with `fields: { discount: { dialect: 'cel', source: 'recalc.discount' } }`. The `http` node sets `timeoutMs`. Failure routing is a `type: 'fault'` edge; a bare error label routes nothing.", "files": [], "assertions": { "must_contain": ["function:", "outputVariable", "type: 'fault'", "timeoutMs", "update_record", "dialect: 'cel'"], @@ -64,7 +64,7 @@ { "id": 7, "prompt": "An expense approval only holders of the `expense.approve_request` capability may start; route to the `finance_manager` position, and the submitter must never approve their own expense.", - "expected_output": "Flow-level `requiredPermissions: ['expense.approve_request']` (checked at every door that starts the flow, not a `current_user.positions` check inside it); the `approval` node sets `excludeSubmitter: true`, and its `onEmptyApprovers` is anything but `auto_approve`, which is refused beside it.", + "expected_output": "Flow-level `requiredPermissions: ['expense.approve_request']`, checked at every door that starts the flow (never a run-user positions check inside it); the `approval` node names `approvers: [{ type: 'position', value: 'finance_manager' }]`, sets `excludeSubmitter: true`, and its `onEmptyApprovers` is anything but `auto_approve`, refused beside it.", "files": [], "assertions": { "must_contain": ["requiredPermissions", "excludeSubmitter: true", "type: 'position'"], @@ -74,7 +74,7 @@ { "id": 8, "prompt": "A screen flow: the user uploads the signed contract PDF on the last step, then the flow creates the `contract` record with the file in its `signed_copy` field.", - "expected_output": "A `screen` field `{ name: 'signed_copy', type: 'file', required: true }`; the screen resumes with the uploaded `sys_file` id bound to `signed_copy`, and a `create_record` after the last screen writes `fields: { signed_copy: { dialect: 'cel', source: 'signed_copy' } }`, which claims the file for the record.", + "expected_output": "A `type: 'screen'` flow whose last `screen` carries `{ name: 'signed_copy', type: 'file', required: true }` and resumes with the uploaded `sys_file` id bound to `signed_copy`; a `create_record` after it writes `fields: { signed_copy: { dialect: 'cel', source: 'signed_copy' } }`, which claims the file for the record.", "files": [], "assertions": { "must_contain": ["type: 'screen'", "type: 'file'", "create_record", "signed_copy"], From 72cb4e28789a54b79c7d2a33bf158879b0ddbeb5 Mon Sep 17 00:00:00 2001 From: os-steve Date: Tue, 27 Oct 2026 11:17:31 +0000 Subject: [PATCH 2/2] chore(changeset): patch @objectstack/skills for the automation eval fixture fix The skills catalog ships as @objectstack/skills (sync-catalog.mjs copies skills/** into its dist), so the rewritten eval fixtures publish and take a patch changeset instead of skip-changeset. Claude-Session: https://claude.ai/code/session_01ADHkmT8qQiLPD3BRTdVxGH Co-authored-by: Claude --- .changeset/26515-automation-eval-fixtures.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .changeset/26515-automation-eval-fixtures.md diff --git a/.changeset/26515-automation-eval-fixtures.md b/.changeset/26515-automation-eval-fixtures.md new file mode 100644 index 00000000000..ee2aa17f305 --- /dev/null +++ b/.changeset/26515-automation-eval-fixtures.md @@ -0,0 +1,12 @@ +--- +'@objectstack/skills': patch +--- + +fix(skills): objectstack-automation eval fixtures 5, 7 and 8 agree with their own assertions + +Clause-②: no +Metadata-protocol: none + +- **Before.** Three ideal answers in `evals/flows-triggers-approvals.json` failed the assertions that grade them: eval 5 spelled the negated error-label string inside its answer, eval 7 never spelled the `type: 'position'` approver it requires and spelled the negated positions check, eval 8 never spelled the `type: 'screen'` flow kind it requires. Graded against those assertions, a correct answer scored wrong. +- **Now.** The three ideal answers say each negation in words, eval 7 names `approvers: [{ type: 'position', value: 'finance_manager' }]`, and eval 8 opens with the flow kind. No fixture added or removed; the assertions and the skill's teaching are unchanged. +- No schema, key or behaviour changes.