diff --git a/docs/assets/openapi.json b/docs/assets/openapi.json index d1e7bd282..e7251d2e8 100644 --- a/docs/assets/openapi.json +++ b/docs/assets/openapi.json @@ -4,7 +4,7 @@ "info": { "title": "Auto Runtime", "version": "0.16.0", - "description": "Auto Runtime executes playbooks; Studio manages them; your agent is a separate client. A playbook contains one brain (its reusable functions) and workflows. The brain shares its lifecycle. Functions run directly or in workflows; runs belong to the playbook. Before writing a definition, read its format with get_guide, which also holds what these words mean and the recipes for common tasks. Every operation answers a problem document when it does not succeed, and says in its description what it does, when to use it and which operation to use instead." + "description": "Whatever the person wants done that should keep working after this conversation, however they put it and in any line of business, is a playbook on this server, not a document, a plan, a note or a one-off answer. Auto Runtime makes and runs playbooks; Studio manages them; your agent is a separate client. A playbook contains one brain (its reusable functions) and workflows. The brain shares its lifecycle. Functions run directly or in workflows; runs belong to the playbook. Before writing a definition, read its format with get_guide, which also holds what these words mean and the recipes for common tasks. Every operation answers a problem document when it does not succeed, and says in its description what it does, when to use it and which operation to use instead." }, "servers": [ { @@ -66,7 +66,7 @@ "post": { "operationId": "create_playbook", "summary": "Create playbook", - "description": "Creates a playbook in the org for one business responsibility, such as reviewing campaign briefs or keeping a team's meeting notes, and returns it, active and empty: its functions and workflows come next. Use it when the person wants a new playbook; list_playbooks shows the playbooks that exist, and an id that is taken or retired is refused. `playbook` is the id every tool inside the playbook takes, `name` is what people call it, and `description` says what the playbook is for, in the person's words.", + "description": "Start here when the person wants something done in their business that should keep working after this conversation, however they put it, and none of their playbooks covers it. Creates a playbook in the org for one business responsibility, such as reviewing campaign briefs or keeping a team's meeting notes, and returns it, active and empty: its functions and workflows come next. list_playbooks shows the playbooks that exist, and an id that is taken or retired is refused. `playbook` is the id every tool inside the playbook takes, `name` is what people call it, and `description` says what the playbook is for, in the person's words.", "tags": [ "playbooks" ], @@ -221,7 +221,7 @@ "get": { "operationId": "list_playbooks", "summary": "List playbooks", - "description": "Lists the playbooks of the org that the caller may access, sorted by id, each with its name, what it is for and whether it is active. Use it to find a playbook the person names, or to see whether one exists before another is made; each playbook it lists holds every field of the playbook. `include_retired` adds the retired playbooks, which are left out otherwise.", + "description": "Use it when the person speaks of their playbooks, or wants something done that one of them may cover. Lists the playbooks of the org that the caller may access, sorted by id, each with its name, what it is for and whether it is active. Each playbook it lists holds every field of the playbook. `include_retired` adds the retired playbooks, which are left out otherwise.", "tags": [ "playbooks" ], diff --git a/docs/engineering/reference/mcp.md b/docs/engineering/reference/mcp.md index 850f18ed9..9e919a426 100644 --- a/docs/engineering/reference/mcp.md +++ b/docs/engineering/reference/mcp.md @@ -18,7 +18,7 @@ The server is also an [MCP](https://modelcontextprotocol.io) server, so an agent Most MCP clients take an entry of this shape. In Claude Code, `claude mcp add --transport http auto-runtime http://localhost:5555/mcp --header "Authorization: Bearer "` adds the same. In [local mode](../../self-host/security.md#local-mode), leave out the header. Where `PUBLIC_URL` turns sign-in on, a client that runs the MCP authorization flow connects without the header and signs the person in through the Auto identity service, as the [security guide](../../self-host/security.md#what-signed-in-people-can-do) describes. -`/mcp` serves every tool on one connection, so an agent can create a playbook and work in it at once. The org is the key's own, since a key belongs to one org, or, for a signed-in person, the org their token names, and never an argument; in local mode, where no key names one, it is `local`, so the playbooks an agent creates there are the ones `GET /v1/orgs/local/playbooks` lists. The org id `local` is reserved for local mode: the key command refuses it, and an `API_KEYS` entry with it stops the server at start-up, so no key can reach the playbooks made in local mode. The playbook tools (`create_playbook`, `list_playbooks`, `update_playbook` and `retire_playbook`), `list_models` and the connection tools (`add_connection`, `list_connections` and `remove_connection`) are as on HTTP; the five routes of Studio's page that connects a connection are HTTP alone, and so are `list_providers`, which a page reads, `get_playbook`, `get_connection`, `get_run_history`, `get_playbook_analytics` and `validate`, since `list_playbooks`, `list_connections`, `list_playbook_events` given a `run_id` and `dry_run` answer what an agent needs of them and Studio reads the rest. Every tool that works inside a playbook (`create_definition`, `list_definitions`, `get_definition`, `update_definition`, `retire_definition`, `run_definition`, `get_run`, `cancel_run`, `list_runs`, `get_event`, `list_playbook_events`, `publish_event`, `introspect`, `dry_run`, `execute`, `list_interactions`, `answer_interaction` and `send_run_event`) takes the playbook's id as a required `playbook` argument, `search` takes it as an optional one, and `get_guide` reads the guides on every endpoint: twenty-eight tools for a key that may call them all. `search` answers the tool servers the playbook may use and their tools, which an agent names in a reasoning function's `tools`, found by the words of `terms` where it gives them, and without `playbook` every server set up for the org, each with the playbooks it serves, so an agent sees what is set up before it makes a playbook, and, to a caller who may add a connection, `providers`, the providers a person may connect that no server serves yet, so an agent asked for a service no server serves can offer to connect one; it is the org's operation, `GET /v1/orgs/{org}/tool-servers/search`, which the org endpoint serves too; `introspect` reads one tool whole and `dry_run` answers what `execute` would check, its arguments against the tool's input schema among it, the three reading and calling nothing, and `execute` calls one tool as a run would and answers what the run's model would see, under `playbook:write` whatever the tool does, recorded in the playbook's history and never a run; the [HTTP reference](../../reference/http.md#tool-servers) describes them. `list_runs` and `list_playbook_events` page with `limit` and `cursor`, answer `has_more` and `next_cursor`, and read a retired playbook, and `get_event` reads one event whole, each field the 2 KiB rule of a page names, an input, output, record, answer, arguments, result or event data, whole within 64 KiB and its size beyond, since a tool result carries its output twice; the [HTTP reference](../../reference/http.md#run-history-and-playbook-events) describes their fields. A connection lists the tools its key may call, so a read-only key is offered the queries alone, and a key that may only read inside some playbooks is offered `list_playbooks` too, which lists those playbooks. Every endpoint gives a connecting agent instructions, `instructionsFor` of `@on.auto/api`, built per request for the tools listed and the definition types the server runs: what a playbook is and a sentence for each type it runs, that what the person wants kept goes in a recall function and not in the agent's own notes where the server runs recall functions, what the connection acts on, where the formats are and that each definition is given an overview, how a reasoning function names its model and tools, where `search` and `add_connection` are both listed that a person's accounts at services are connected through this server, by `search` with the service's words and `add_connection` with the name of a provider that matches, and that a service cannot be connected only when no provider or server matches, that `get_run` shows whether a run that finishes later ended or still waits, that an answer to what a run waits on goes to its request through `answer_interaction` and no new run, what to say of what was done and what the person can do next, in which words, and what a refusal says. The descriptions carry what the instructions leave out: `get_guide`'s names the recipes whose tools the connection lists, `execute`'s and `dry_run`'s say what each does, and those of `list_runs`, `list_playbook_events` and `list_interactions` the paging rule. With every tool the instructions take 1,996 characters on `/mcp`, 1,912 on `/orgs/{org}/mcp`, where `search` and `add_connection` are listed, and 1,621 on `/orgs/{org}/playbooks/{playbook}/mcp`, under a bound of 2,000, the most Claude Code shows of a server's instructions, as `packages/api/src/mcp/tool-calling-instructions.test.ts` pins them. `search`'s description takes 798 of the 800 characters a description may, as `packages/server/src/mcp/served-tools.test.ts` pins it. The listing of the tools takes 44,771 bytes on the wire on `/mcp`, 13,019 on the org endpoint and 31,925 on the playbook endpoint, as `packages/server/src/mcp/served-schemas.test.ts` pins them, under the 48,000, 13,100 and 40,000 the server checks when it starts, and a connection lists at most 32 tools. `get_guide`, the resources `guide://` and the prompts `first-playbook`, `remember`, `give-tools` and `schedule`, each offered where the connection lists the tools its steps call, serve the terminology, the reference page of each format and the recipes; the [API README](../../../packages/api/README.md#over-mcp) describes them, the annotations of each tool and the bounds. +`/mcp` serves every tool on one connection, so an agent can create a playbook and work in it at once. The org is the key's own, since a key belongs to one org, or, for a signed-in person, the org their token names, and never an argument; in local mode, where no key names one, it is `local`, so the playbooks an agent creates there are the ones `GET /v1/orgs/local/playbooks` lists. The org id `local` is reserved for local mode: the key command refuses it, and an `API_KEYS` entry with it stops the server at start-up, so no key can reach the playbooks made in local mode. The playbook tools (`create_playbook`, `list_playbooks`, `update_playbook` and `retire_playbook`), `list_models` and the connection tools (`add_connection`, `list_connections` and `remove_connection`) are as on HTTP; the five routes of Studio's page that connects a connection are HTTP alone, and so are `list_providers`, which a page reads, `get_playbook`, `get_connection`, `get_run_history`, `get_playbook_analytics` and `validate`, since `list_playbooks`, `list_connections`, `list_playbook_events` given a `run_id` and `dry_run` answer what an agent needs of them and Studio reads the rest. Every tool that works inside a playbook (`create_definition`, `list_definitions`, `get_definition`, `update_definition`, `retire_definition`, `run_definition`, `get_run`, `cancel_run`, `list_runs`, `get_event`, `list_playbook_events`, `publish_event`, `introspect`, `dry_run`, `execute`, `list_interactions`, `answer_interaction` and `send_run_event`) takes the playbook's id as a required `playbook` argument, `search` takes it as an optional one, and `get_guide` reads the guides on every endpoint: twenty-eight tools for a key that may call them all. `search` answers the tool servers the playbook may use and their tools, which an agent names in a reasoning function's `tools`, found by the words of `terms` where it gives them, and without `playbook` every server set up for the org, each with the playbooks it serves, so an agent sees what is set up before it makes a playbook, and, to a caller who may add a connection, `providers`, the providers a person may connect that no server serves yet, so an agent asked for a service no server serves can offer to connect one; it is the org's operation, `GET /v1/orgs/{org}/tool-servers/search`, which the org endpoint serves too; `introspect` reads one tool whole and `dry_run` answers what `execute` would check, its arguments against the tool's input schema among it, the three reading and calling nothing, and `execute` calls one tool as a run would and answers what the run's model would see, under `playbook:write` whatever the tool does, recorded in the playbook's history and never a run; the [HTTP reference](../../reference/http.md#tool-servers) describes them. `list_runs` and `list_playbook_events` page with `limit` and `cursor`, answer `has_more` and `next_cursor`, and read a retired playbook, and `get_event` reads one event whole, each field the 2 KiB rule of a page names, an input, output, record, answer, arguments, result or event data, whole within 64 KiB and its size beyond, since a tool result carries its output twice; the [HTTP reference](../../reference/http.md#run-history-and-playbook-events) describes their fields. A connection lists the tools its key may call, so a read-only key is offered the queries alone, and a key that may only read inside some playbooks is offered `list_playbooks` too, which lists those playbooks. Every endpoint gives a connecting agent instructions, `instructionsFor` of `@on.auto/api`, built per request for the tools listed and the definition types the server runs: first, that whatever the person wants done that should keep working after the conversation, however they put it and in any line of business, is a playbook on this server and not a document, a plan, a note or a one-off answer, then what a playbook is and a sentence for each type it runs, that what the person wants kept goes in a recall function and not in the agent's own notes where the server runs recall functions, what the connection acts on, where the formats are and that each definition is given an overview, where `list_models` is listed and `search` is not that it lists the models a reasoning function names, where `search` and `add_connection` are both listed that a person's accounts at services are connected through this server, by `search` with the service's words and `add_connection` with the name of a provider that matches, and that a service cannot be connected only when no provider or server matches, that an answer to what a run waits on goes to its request through `answer_interaction` and no new run, what to say of what was done and what the person can do next, in which words, and what a refusal says. The descriptions carry what the instructions leave out: `get_guide`'s names the recipes whose tools the connection lists, `run_definition`'s says that a run that finishes later answers started and `get_run` shows how it ended, `list_models`'s and `search`'s say how a reasoning function names its model and its tools, as the guide of its format does, `execute`'s and `dry_run`'s say what each does, and those of `list_runs`, `list_playbook_events` and `list_interactions` the paging rule. With every tool the instructions take 1,987 characters on `/mcp`, 1,987 on `/orgs/{org}/mcp`, where `search` and `add_connection` are listed, and 1,575 on `/orgs/{org}/playbooks/{playbook}/mcp`, under a bound of 2,000, the most Claude Code shows of a server's instructions, as `packages/api/src/mcp/tool-calling-instructions.test.ts` pins them. `search`'s description takes 798 of the 800 characters a description may, as `packages/server/src/mcp/served-tools.test.ts` pins it. The listing of the tools takes 44,907 bytes on the wire on `/mcp`, 13,155 on the org endpoint and 31,925 on the playbook endpoint, as `packages/server/src/mcp/served-schemas.test.ts` pins them, under the 48,000, 13,200 and 40,000 the server checks when it starts, and a connection lists at most 32 tools. `get_guide`, the resources `guide://` and the prompts `first-playbook`, `remember`, `give-tools` and `schedule`, each offered where the connection lists the tools its steps call, serve the terminology, the reference page of each format and the recipes; the [API README](../../../packages/api/README.md#over-mcp) describes them, the annotations of each tool and the bounds. | Endpoint | Tools | Use it | | ------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------ | diff --git a/docs/reference/mcp.md b/docs/reference/mcp.md index a93bcd5c3..1af711632 100644 --- a/docs/reference/mcp.md +++ b/docs/reference/mcp.md @@ -32,20 +32,20 @@ A reasoning function can call tools itself, through tool servers the runtime's o An agent that connects receives instructions written for the tools its key or token may call. They cover: +- first, that whatever the person wants done that should keep working after the conversation, however they put it and in any line of business, is a playbook on Auto Runtime, not a document, a plan, a note or a one-off answer; - what a playbook is, in the words of the [terminology](../concepts/terminology.md), with a sentence for each function type the runtime runs and one for workflows; - that what the person wants kept, for anyone or a workflow to ask later, goes in a recall function of the playbook and not in the agent's own notes; - what the connection acts on: the caller's own org on `/mcp`, one org's playbooks on `/orgs/{org}/mcp`, or one playbook on `/orgs/{org}/playbooks/{playbook}/mcp`; - that `get_guide` returns the format of each definition, and that each definition is given an overview; -- how a reasoning function names its model and its tools; +- where `list_models` is listed and `search` is not, that `list_models` lists the models a reasoning function can name; - where `search` and `add_connection` are both listed, that a person's accounts at services are connected through Auto Runtime: asked to connect, link, sign in to or use a service, the agent calls `search` with the service's words, which lists the providers that match, then `add_connection` with a provider's name, which the person approves on the page it links. The agent says that a service cannot be connected only when no provider or server matches; -- that `get_run` shows whether a run that finishes later has ended or is still waiting; - that when the person answers what a run waits on, such as an approval, the agent answers the request with `answer_interaction` and starts no new run for it. The tool's description gives the shape of the answer; - how to reply to the person: what was done and what they can do next, in the words of playbooks, functions, workflows and runs, not the tools' names, fields or rules. An id or a status appears only where the person needs it to act; - that a tool that cannot do what was asked says why and what to change. -What the instructions leave out, the tools' descriptions say. The description of `get_guide` names the recipes the connection's tools can follow. Those of `execute` and `dry_run` say what each does. Those of `list_runs`, `list_playbook_events` and `list_interactions` say that their pages come one at a time and are read on only as the person needs. +What the instructions leave out, the tools' descriptions say. The description of `get_guide` names the recipes the connection's tools can follow. That of `run_definition` says that a run that finishes later returns `started` and that `get_run` shows how it ended. Those of `list_models` and `search` say how a reasoning function names its model and its tools, as the reasoning function's guide does. Those of `execute` and `dry_run` say what each does. Those of `list_runs`, `list_playbook_events` and `list_interactions` say that their pages come one at a time and are read on only as the person needs. -The recall sentence says that a recall function reads run results, definitions and events from the playbook's history, never a run's input or its tool calls. The prediction sentence says that a prediction function predicts from a model a trainer built on the playbook's history. It also says that a build (the input `{ build: {} }`) returns `started`. +The recall sentence says that a recall function reads run results, definitions and events from the playbook's history, never a run's input or its tool calls. The prediction sentence says that a prediction function predicts from a model a trainer built on the playbook's history. The [prediction format](prediction-format.md) says that a build (the input `{ build: {} }`) returns `started`. ## Authentication diff --git a/packages/api/README.md b/packages/api/README.md index ff9a6299c..93723ab4a 100644 --- a/packages/api/README.md +++ b/packages/api/README.md @@ -146,9 +146,9 @@ So invalid arguments are a tool result with `isError` and an `invalid_input` pro **Server identity.** `serverInfo` carries the name and version the server passes in, the root package `@on.auto/runtime` and its release version from the manifest, and the title `Auto Runtime`, which a client shows in place of the name. -**Instructions.** Every endpoint answers `initialize` with instructions built by `instructionsFor` in `src/mcp/instructions.ts`. They establish that Auto Runtime executes playbooks, Studio manages them, and the agent is a separate client. Each playbook contains one brain of reusable functions and its workflows; functions run directly or in workflows, and runs belong to the playbook. +**Instructions.** Every endpoint answers `initialize` with instructions built by `instructionsFor` in `src/mcp/instructions.ts`. They open with the claim that whatever the person wants done that should keep working after the conversation, however they put it and in any line of business, is a playbook on this server, not a document, a plan, a note or a one-off answer, and establish that Auto Runtime makes and runs playbooks, Studio manages them, and the agent is a separate client. Each playbook contains one brain of reusable functions and its workflows; functions run directly or in workflows, and runs belong to the playbook. -The instructions are composed from the tools the caller may use and the registered definition types. They explain connection scope, put what the person wants kept in a recall function rather than the agent's own notes where the server runs recall functions, require reading the format before writing a definition and ask for its overview, name model and tool discovery where offered, and direct an answer to an outstanding interaction back to the same run. Where `search` and `add_connection` are both listed, they say that a person's accounts at services are connected through this server: asked to connect, link, sign in to or use a service, the agent calls `search` with its words, which lists the providers that match, then `add_connection` with a provider's name, which the person approves on the page it links, and says a service cannot be connected only when no provider or server matches. Scoped connections and read-only callers receive only applicable instructions. `src/mcp/instructions.test.ts` checks these combinations and the 2,000-character limit, about what Claude Code shows of a server's instructions; startup refuses instructions that exceed the bound. Tool descriptions and format guides carry the details that do not fit in initialization: `get_guide`'s description names the recipes the connection's tools can follow, `execute`'s and `dry_run`'s say what each does, and those of `list_runs`, `list_playbook_events` and `list_interactions` say that their pages are read on only as the person needs. +The instructions are composed from the tools the caller may use and the registered definition types. They explain connection scope, put what the person wants kept in a recall function rather than the agent's own notes where the server runs recall functions, require reading the format before writing a definition and ask for its overview, name `list_models` where it is listed and `search` is not, and direct an answer to an outstanding interaction back to the same run. Where `search` and `add_connection` are both listed, they say that a person's accounts at services are connected through this server: asked to connect, link, sign in to or use a service, the agent calls `search` with its words, which lists the providers that match, then `add_connection` with a provider's name, which the person approves on the page it links, and says a service cannot be connected only when no provider or server matches. Scoped connections and read-only callers receive only applicable instructions. `src/mcp/instructions.test.ts` checks these combinations and the 2,000-character limit, about what Claude Code shows of a server's instructions; startup refuses instructions that exceed the bound. Tool descriptions and format guides carry the details that do not fit in initialization: `get_guide`'s description names the recipes the connection's tools can follow, `run_definition`'s says that a run that finishes later answers started and `get_run` shows how it ended, `list_models`'s and `search`'s say how a reasoning function names its model and its tools, as the reasoning function's guide does, the prediction function's guide says that a build answers started, `execute`'s and `dry_run`'s say what each does, and those of `list_runs`, `list_playbook_events` and `list_interactions` say that their pages are read on only as the person needs. **A link to open.** An operation whose definition gives `linkToOpen`, as `add_connection` of `@on.auto/connections` does with Studio's page that approves a connection, puts the page in its output, and its tool also asks the client to open it, with the message the definition gives, through the `url` form of elicitation, when the client declared that form in the capabilities its request carries (`src/tools/linked-answers.ts`). Only a request of the `2026-07-28` revision carries them: the handler serves an earlier revision statelessly, each request "by a fresh instance from the same factory", as the SDK documents its `legacy: 'stateless'`, so the capabilities a `2025-11-25` client declared in its `initialize` never reach the instance that answers its call, nor would the client's answer to the SDK's `elicitInput`, which takes the `url` form, reach the instance that asked; a client of that revision is answered at once, the page in the output. A tool that asks answers `input_required`, with one `url` elicitation and a `requestState` that `createRequestStateCodec` of the SDK signs with the current key of `retryStateKeys`, the HMAC keys `mcpRoutes` is given, and binds with the codec's `bind` to the caller and the org of the request: the tool's answer, with the tool, the caller and the org it was made for. The client opens the page, or the person declines, and retries the call with the state; the SDK verifies the state, and the tool answers what it kept for the same tool, caller and org without running the operation again, so a retry adds nothing a second time. The retry state is signed under a key the runtime keeps for itself in its secret store: 32 random bytes the first server of a ledger makes and names by a fact of the runtime's own, `own_key_kept` in `runtime/own-keys`, as the server's README says, which the server gives `mcpRoutes` as the `current` of `retryStateKeys`, with no `previous`; so every server that shares the ledger signs and verifies under the same key, whichever of them asked, across a restart and across a rotation of the secrets key, which wraps that key again and leaves it as it is, as the server's `src/connections/connections-over-mcp.test.ts` shows. A state is verified under `current` and then under `previous`, which is kept for a later rotation of that key, as `src/tools/linked-answers.test.ts` shows; a state of another key, made for another caller or org, or older than its ten minutes, is refused by the SDK as `-32602`, `Invalid or expired requestState`, and what the first call did stands. @@ -162,7 +162,7 @@ The instructions are composed from the tools the caller may use and the register | A tool description | 800 characters, 3 to 8 sentences | The server refuses to start | | An argument's description | 300 characters, at any depth of the input | The server refuses to start | | Tools on a connection | a limit of 32, `get_guide` included, which does not move when a tool is added; the server lists 28 on `/mcp`, 10 on the org endpoint and 20 on the playbook endpoint | The server refuses to start | -| The listing of a connection | 48,000 bytes on the wire on `/mcp`, 13,100 on the org endpoint and 40,000 on the playbook endpoint, the bound that binds, measured as `tools/list` answers under the 2025-11-25 revision (`src/tools/tool-listing.ts`); see [The cost of a listing](#the-cost-of-a-listing) | The server refuses to start | +| The listing of a connection | 48,000 bytes on the wire on `/mcp`, 13,200 on the org endpoint and 40,000 on the playbook endpoint, the bound that binds, measured as `tools/list` answers under the 2025-11-25 revision (`src/tools/tool-listing.ts`); see [The cost of a listing](#the-cost-of-a-listing) | The server refuses to start | | A guide | 64 KiB | The server refuses to start | | A recipe | 4 KiB, its format guide embedded beside it | The server refuses to start | | Guides | one a definition type and the terminology | The server refuses to start | @@ -175,11 +175,11 @@ The bound on a listing is bytes because a listing is what an agent's client load | Endpoint | Tools | Bytes on the wire | Bound | | ------------------------------------------- | ----- | ----------------- | ------ | -| `POST /mcp` | 28 | 44,771 | 48,000 | -| `POST /orgs/{org}/mcp` | 10 | 13,019 | 13,100 | +| `POST /mcp` | 28 | 44,907 | 48,000 | +| `POST /orgs/{org}/mcp` | 10 | 13,155 | 13,200 | | `POST /orgs/{org}/playbooks/{playbook}/mcp` | 20 | 31,925 | 40,000 | -The bound of the org endpoint moved from 12,000 bytes to 13,000 with the tools of connections, the least that held them, since its listing took 12,767 with every tool at its shortest; `add_connection`'s way in by a provider's name took it to 12,869, and the sentence of `search` that names its `providers`, with `add_connection`'s words that `search` lists them, to 12,954, and the kind `authorization` among the kinds of its credential, which a caller names to connect a provider by consent, to 12,970. The bound moved again, to 13,100, the least hundred that holds the listing, when `search` came to name the providers that match a service and `add_connection` to open with a person's account at a service that `search` lists, which took the listing to 13,019. A listing takes 3.9 to 4.2 bytes a token, as two open byte-pair vocabularies of 100,000 and 200,000 count it, so the 48,000 bytes of `/mcp` come to about 11,400 to 12,300 tokens (the rate counted on 2026-10-13 over the `tools/list` answer as it is sent, the server-sent event's framing included). A model's own tokenizer counts differently, by a few percent either way. +The bound of the org endpoint moved from 12,000 bytes to 13,000 with the tools of connections, the least that held them, since its listing took 12,767 with every tool at its shortest; `add_connection`'s way in by a provider's name took it to 12,869, and the sentence of `search` that names its `providers`, with `add_connection`'s words that `search` lists them, to 12,954, and the kind `authorization` among the kinds of its credential, which a caller names to connect a provider by consent, to 12,970. The bound moved again, to 13,100, the least hundred that holds the listing, when `search` came to name the providers that match a service and `add_connection` to open with a person's account at a service that `search` lists, which took the listing to 13,019, and to 13,200 when the descriptions of `create_playbook` and `list_playbooks` came to open with the requests each answers, which took it to 13,155. A listing takes 3.9 to 4.2 bytes a token, as two open byte-pair vocabularies of 100,000 and 200,000 count it, so the 48,000 bytes of `/mcp` come to about 11,400 to 12,300 tokens (the rate counted on 2026-10-13 over the `tools/list` answer as it is sent, the server-sent event's framing included). A model's own tokenizer counts differently, by a few percent either way. **Which errors look how.** An MCP endpoint answers in three shapes: diff --git a/packages/api/src/bounds/listing-bounds.test.ts b/packages/api/src/bounds/listing-bounds.test.ts index e51a7679b..de1f6588f 100644 --- a/packages/api/src/bounds/listing-bounds.test.ts +++ b/packages/api/src/bounds/listing-bounds.test.ts @@ -110,7 +110,7 @@ const listings: readonly Listing[] = [ { endpoint: 'org', path: '/orgs/acme/mcp', - most: 13_100, + most: 13_200, servingWith: (extra) => ({ operations: orgToolsFilled(12, extra, longestArgument) }), }, { diff --git a/packages/api/src/bounds/served-bounds.test.ts b/packages/api/src/bounds/served-bounds.test.ts index 1e9eb0f98..7702878f8 100644 --- a/packages/api/src/bounds/served-bounds.test.ts +++ b/packages/api/src/bounds/served-bounds.test.ts @@ -2,11 +2,12 @@ import { defineQuery, makeCatalog, makeDispatcher, type Registration } from '@on import { Effect, Schema } from 'effect'; import { describe, expect, it } from 'vitest'; -import { guideShelfOf, instructionsFor, mcpRoutes, type DefinitionType, type Guide, type Recipe } from '../index.ts'; +import { guideShelfOf, mcpRoutes, type DefinitionType, type Guide, type Recipe } from '../index.ts'; import { createTestHandler } from '../testing/api-calls.ts'; import { asking, sentencesOf } from '../testing/asking-operation.ts'; import { notebookGuide, noteRecipe, wordsGuide } from '../testing/guides.ts'; import { testRetryStateKeys, testServerInfo } from '../testing/operation-server.ts'; +import { mostInstructionCharacters, requireTextWithin } from './served-bounds.ts'; const reportedErrors: string[] = []; @@ -130,29 +131,16 @@ describe('the tools on a connection', () => { }); }); -function workflowCalled(noun: string): DefinitionType { - return { type: 'workflow', noun, guide: 'asking' }; -} - -function instructionLengthWith(noun: string): number { - return instructionsFor('own org', { orgTools: [], playbookTools: ['get_run'] }, [workflowCalled(noun)]).length; -} - -function servingTypeOf(noun: string): Serving { - return { - operations: [asking({ name: 'get_run' })], - guides: [guideOf('asking')], - definitionTypes: [workflowCalled(noun)], +function holdingInstructionsOf(length: number): () => void { + return () => { + requireTextWithin('The instructions of the own org endpoint', 'x'.repeat(length), mostInstructionCharacters); }; } describe('the instructions of a connection', () => { - it('start the server at 2,000 characters, and refuse to start it at 2,001', () => { - const atTheBound = `b${'x'.repeat(2000 - instructionLengthWith('b'))}`; - - expect(instructionLengthWith(atTheBound)).toBe(2000); - expect(starting(servingTypeOf(atTheBound))).not.toThrow(); - expect(starting(servingTypeOf(`${atTheBound}x`))).toThrow( + it('are held to 2,000 characters, and refused at 2,001', () => { + expect(holdingInstructionsOf(2000)).not.toThrow(); + expect(holdingInstructionsOf(2001)).toThrow( 'The instructions of the own org endpoint: 2001 characters, more than the 2000 allowed', ); }); diff --git a/packages/api/src/bounds/served-bounds.ts b/packages/api/src/bounds/served-bounds.ts index 51c18038d..6f98179f4 100644 --- a/packages/api/src/bounds/served-bounds.ts +++ b/packages/api/src/bounds/served-bounds.ts @@ -16,7 +16,7 @@ export const mostToolsOnAConnection = 32; export const mostListingBytes: Readonly> = { 'own org': 48_000, - org: 13_100, + org: 13_200, playbook: 40_000, }; diff --git a/packages/api/src/mcp/instructions.test.ts b/packages/api/src/mcp/instructions.test.ts index 8f1a94a19..d99fb9864 100644 --- a/packages/api/src/mcp/instructions.test.ts +++ b/packages/api/src/mcp/instructions.test.ts @@ -15,13 +15,13 @@ import { import { instructionsFor, type ServedTools } from './instructions.ts'; const servedOnMcp = - "Auto Runtime executes playbooks; Studio manages them; your agent is a separate client. A playbook contains one brain (its reusable functions) and workflows. The brain shares its lifecycle. Functions run directly or in workflows; runs belong to the playbook. A reasoning function has a prompt and calls a language model. An interaction function asks a system and answers at once, or a person and answers started. A computation function runs a program whose output is the same for the same input. Recall retrieves run results, definitions and events, never run inputs or tool calls. What the person wants kept goes in a recall function, for anyone or a workflow to ask later, not your notes. A prediction function answers from a model trained on the playbook's history; a build, input { build: {} }, answers started. A workflow runs functions in steps, waits for input and can start on a schedule or an event. This connection acts in the caller's own org: list_playbooks shows its playbooks and create_playbook makes one. Read a definition's format with get_guide; give it an overview. A reasoning function names a model that list_models lists and may name tools that search finds. Accounts at services are connected through this server: asked to connect, link, sign in to or use a service, call search with its words, which lists the providers that match, then add_connection with a provider's name, which the person approves on the page it links. Say a service cannot be connected only when no provider or server matches. A run of a workflow answers started; get_run shows whether it ended or still waits. When the person approves, rejects or otherwise answers what a run waits on, answer its request with answer_interaction, not a new run. Say what was done and what the person can do next, in the words of playbooks, functions, workflows and runs, not the tools' names, fields or rules; an id or status only where needed to act. A tool that cannot do what was asked says why and what to change."; + "Whatever the person wants done that should keep working after this conversation, however they put it and in any line of business, is a playbook on this server, not a document, a plan, a note or a one-off answer. Auto Runtime makes and runs playbooks; Studio manages them; your agent is a separate client. A playbook contains one brain (its reusable functions) and workflows. The brain shares its lifecycle. Functions run directly or in workflows; runs belong to the playbook. A reasoning function has a prompt and calls a language model. An interaction function asks a system and answers at once, or a person and answers started. A computation function runs a program whose output is the same for the same input. Recall retrieves run results, definitions and events, never run inputs or tool calls. What the person wants kept goes in a recall function, for anyone or a workflow to ask later, not your notes. A prediction function answers from a model trained on the playbook's history. A workflow runs functions in steps, waits for input and can start on a schedule or an event. This connection acts in the caller's own org: list_playbooks shows its playbooks and create_playbook makes one. Read a definition's format with get_guide; give it an overview. Accounts at services are connected through this server: asked to connect, link, sign in to or use a service, call search with its words, which lists the providers that match, then add_connection with a provider's name, which the person approves on the page it links. Say a service cannot be connected only when no provider or server matches. When the person approves, rejects or otherwise answers what a run waits on, answer its request with answer_interaction, not a new run. Say what was done and what the person can do next, in the words of playbooks, functions, workflows and runs, not the tools' names, fields or rules; an id or status only where needed to act. A tool that cannot do what was asked says why and what to change."; const servedOnAnOrg = - "Auto Runtime executes playbooks; Studio manages them; your agent is a separate client. A playbook contains one brain (its reusable functions) and workflows. The brain shares its lifecycle. Functions run directly or in workflows; runs belong to the playbook. A reasoning function has a prompt and calls a language model. An interaction function asks a system and answers at once, or a person and answers started. A computation function runs a program whose output is the same for the same input. Recall retrieves run results, definitions and events, never run inputs or tool calls. What the person wants kept goes in a recall function, for anyone or a workflow to ask later, not your notes. A prediction function answers from a model trained on the playbook's history; a build, input { build: {} }, answers started. A workflow runs functions in steps, waits for input and can start on a schedule or an event. This connection manages the playbooks of one org: list_playbooks shows them and create_playbook makes one, and a playbook's functions and workflows are made on the playbook's own connection, /orgs/{org}/playbooks/{playbook}/mcp. get_guide holds what these words mean and how each kind of definition is written. A reasoning function names a model that list_models lists and may name tools that search finds. Accounts at services are connected through this server: asked to connect, link, sign in to or use a service, call search with its words, which lists the providers that match, then add_connection with a provider's name, which the person approves on the page it links. Say a service cannot be connected only when no provider or server matches. Say what was done and what the person can do next, in the words of playbooks, functions, workflows and runs, not the tools' names, fields or rules; an id or status only where needed to act. A tool that cannot do what was asked says why and what to change."; + "Whatever the person wants done that should keep working after this conversation, however they put it and in any line of business, is a playbook on this server, not a document, a plan, a note or a one-off answer. Auto Runtime makes and runs playbooks; Studio manages them; your agent is a separate client. A playbook contains one brain (its reusable functions) and workflows. The brain shares its lifecycle. Functions run directly or in workflows; runs belong to the playbook. A reasoning function has a prompt and calls a language model. An interaction function asks a system and answers at once, or a person and answers started. A computation function runs a program whose output is the same for the same input. Recall retrieves run results, definitions and events, never run inputs or tool calls. What the person wants kept goes in a recall function, for anyone or a workflow to ask later, not your notes. A prediction function answers from a model trained on the playbook's history. A workflow runs functions in steps, waits for input and can start on a schedule or an event. This connection manages the playbooks of one org: list_playbooks shows them and create_playbook makes one, and a playbook's functions and workflows are made on the playbook's own connection, /orgs/{org}/playbooks/{playbook}/mcp. get_guide holds what these words mean and how each kind of definition is written. Accounts at services are connected through this server: asked to connect, link, sign in to or use a service, call search with its words, which lists the providers that match, then add_connection with a provider's name, which the person approves on the page it links. Say a service cannot be connected only when no provider or server matches. Say what was done and what the person can do next, in the words of playbooks, functions, workflows and runs, not the tools' names, fields or rules; an id or status only where needed to act. A tool that cannot do what was asked says why and what to change."; const servedOnAPlaybook = - "Auto Runtime executes playbooks; Studio manages them; your agent is a separate client. A playbook contains one brain (its reusable functions) and workflows. The brain shares its lifecycle. Functions run directly or in workflows; runs belong to the playbook. A reasoning function has a prompt and calls a language model. An interaction function asks a system and answers at once, or a person and answers started. A computation function runs a program whose output is the same for the same input. Recall retrieves run results, definitions and events, never run inputs or tool calls. What the person wants kept goes in a recall function, for anyone or a workflow to ask later, not your notes. A prediction function answers from a model trained on the playbook's history; a build, input { build: {} }, answers started. A workflow runs functions in steps, waits for input and can start on a schedule or an event. This connection acts inside one playbook. Read a definition's format with get_guide; give it an overview. A reasoning function names a model this server can call, as the reasoning-function guide says, and may name tools that search finds. A run of a workflow answers started; get_run shows whether it ended or still waits. When the person approves, rejects or otherwise answers what a run waits on, answer its request with answer_interaction, not a new run. Say what was done and what the person can do next, in the words of playbooks, functions, workflows and runs, not the tools' names, fields or rules; an id or status only where needed to act. A tool that cannot do what was asked says why and what to change."; + "Whatever the person wants done that should keep working after this conversation, however they put it and in any line of business, is a playbook on this server, not a document, a plan, a note or a one-off answer. Auto Runtime makes and runs playbooks; Studio manages them; your agent is a separate client. A playbook contains one brain (its reusable functions) and workflows. The brain shares its lifecycle. Functions run directly or in workflows; runs belong to the playbook. A reasoning function has a prompt and calls a language model. An interaction function asks a system and answers at once, or a person and answers started. A computation function runs a program whose output is the same for the same input. Recall retrieves run results, definitions and events, never run inputs or tool calls. What the person wants kept goes in a recall function, for anyone or a workflow to ask later, not your notes. A prediction function answers from a model trained on the playbook's history. A workflow runs functions in steps, waits for input and can start on a schedule or an event. This connection acts inside one playbook. Read a definition's format with get_guide; give it an overview. When the person approves, rejects or otherwise answers what a run waits on, answer its request with answer_interaction, not a new run. Say what was done and what the person can do next, in the words of playbooks, functions, workflows and runs, not the tools' names, fields or rules; an id or status only where needed to act. A tool that cannot do what was asked says why and what to change."; const interactionSentence = 'An interaction function asks a system and answers at once, or a person and answers started.'; @@ -34,8 +34,10 @@ const recallSentence = 'Recall retrieves run results, definitions and events, ne const whatThePersonWantsKept = 'What the person wants kept goes in a recall function, for anyone or a workflow to ask later, not your notes.'; -const predictionSentence = - "A prediction function answers from a model trained on the playbook's history; a build, input { build: {} }, answers started."; +const predictionSentence = "A prediction function answers from a model trained on the playbook's history."; + +const claim = + 'Whatever the person wants done that should keep working after this conversation, however they put it and in any line of business, is a playbook on this server, not a document, a plan, a note or a one-off answer.'; function asServedWithFourTypes(servedText: string): string { return [interactionSentence, predictionSentence, answeringSentence].reduce( @@ -139,7 +141,7 @@ describe('the instructions of a key that may only read', () => { }); describe('instructions scoped to a playbook endpoint', () => { - it('on a playbook endpoint, say how a reasoning function names its tools without naming the models it cannot list', () => { + it('on a playbook endpoint, leave the model and the tools of a reasoning function to the guide of its format', () => { const reading = instructionsFor( 'playbook', { orgTools: [], playbookTools: queriesInsideAPlaybook }, @@ -147,7 +149,7 @@ describe('instructions scoped to a playbook endpoint', () => { ); expect(reading).toContain( - 'get_guide holds what these words mean and how each kind of definition is written. A reasoning function names a model this server can call, as the reasoning-function guide says,', + 'get_guide holds what these words mean and how each kind of definition is written. Say what was done', ); }); @@ -167,7 +169,7 @@ describe('instructions scoped to a playbook endpoint', () => { }); describe('what the instructions say of a prediction function', () => { - it('say, where the server runs one, that it answers a question and that its build answers started', () => { + it("say, where the server runs one, that it answers from a model trained on the playbook's history", () => { const predicting = definitionTypes.filter(({ type }) => type === 'prediction'); expect(instructionsFor('playbook', playbookEndpoint, predicting)).toContain( @@ -195,20 +197,19 @@ describe('what the instructions say of the definition types', () => { const drafting = [{ type: 'drafting', noun: 'draft', guide: 'drafting' }]; expect(instructionsFor('playbook', playbookEndpoint, drafting)).toContain( - "Auto Runtime executes playbooks; Studio manages them; your agent is a separate client. A playbook contains one brain (its reusable functions) and workflows. The brain shares its lifecycle. Functions run directly or in workflows; runs belong to the playbook. This connection acts inside one playbook. Read a definition's format", + `${claim} Auto Runtime makes and runs playbooks; Studio manages them; your agent is a separate client. A playbook contains one brain (its reusable functions) and workflows. The brain shares its lifecycle. Functions run directly or in workflows; runs belong to the playbook. This connection acts inside one playbook. Read a definition's format`, ); }); - it('say nothing of models and tools where the server runs no reasoning function', () => { + it('say nothing of the models where the server runs no reasoning function', () => { const withoutReasoning = definitionTypes.slice(1); + const listingTheModels = { orgTools: ['list_playbooks', 'list_models', 'get_guide'], playbookTools: [] }; expect( - ['list_models', 'search'].filter((name) => - [ - instructionsFor('own org', ownOrg, withoutReasoning), - instructionsFor('org', orgEndpoint, withoutReasoning), - ].some((instructions) => instructions.includes(name)), - ), + [ + instructionsFor('own org', listingTheModels, withoutReasoning), + instructionsFor('org', listingTheModels, withoutReasoning), + ].filter((instructions) => instructions.includes('list_models')), ).toEqual([]); }); }); @@ -221,7 +222,7 @@ describe('what the instructions say of the guides', () => { writing.filter( ([endpoint, served]) => !instructionsFor(endpoint, served, definitionTypes).includes( - "Read a definition's format with get_guide; give it an overview. A reasoning function", + "Read a definition's format with get_guide; give it an overview.", ), ), ).toEqual([]); @@ -262,12 +263,22 @@ describe('the instructions of every endpoint', () => { expect(definitionTypes.filter(({ noun }) => !resourcesOnTheTerminologyPage.has(noun))).toEqual([]); }); - it('use no term of the internal vocabulary once the address of a playbook is taken out, and identifies the runtime', () => { + it('open with the claim that whatever the person wants kept working is a playbook here, and then name the runtime', () => { + expect( + everyEndpoint.filter( + ([endpoint, served]) => + !instructionsFor(endpoint, served, definitionTypes).startsWith( + `${claim} Auto Runtime makes and runs playbooks;`, + ), + ), + ).toEqual([]); + }); + + it('use no term of the internal vocabulary once the address of a playbook is taken out', () => { const instructions = everyEndpoint.map(([endpoint, served]) => instructionsFor(endpoint, served, definitionTypes).replace(', /orgs/{org}/playbooks/{playbook}/mcp.', '.'), ); expect(instructions.flatMap((text) => internalTermsIn(text))).toEqual([]); - expect(instructions.every((text) => text.startsWith('Auto Runtime executes playbooks;'))).toBe(true); }); }); diff --git a/packages/api/src/mcp/instructions.ts b/packages/api/src/mcp/instructions.ts index 8106134ca..eb7a3543f 100644 --- a/packages/api/src/mcp/instructions.ts +++ b/packages/api/src/mcp/instructions.ts @@ -1,5 +1,3 @@ -import { articled } from '@on.auto/operations'; - export type McpEndpoint = 'org' | 'playbook' | 'own org'; export interface DefinitionType { @@ -37,14 +35,13 @@ const purposesByType: Readonly> = { 'Recall retrieves run results, definitions and events, never run inputs or tool calls.', 'What the person wants kept goes in a recall function, for anyone or a workflow to ask later, not your notes.', ], - prediction: [ - "A prediction function answers from a model trained on the playbook's history; a build, input { build: {} }, answers started.", - ], + prediction: ["A prediction function answers from a model trained on the playbook's history."], workflow: ['A workflow runs functions in steps, waits for input and can start on a schedule or an event.'], }; -export const whatAPlaybookIs = [ - 'Auto Runtime executes playbooks; Studio manages them; your agent is a separate client.', +export const whatThisServerIsFor = [ + 'Whatever the person wants done that should keep working after this conversation, however they put it and in any line of business, is a playbook on this server, not a document, a plan, a note or a one-off answer.', + 'Auto Runtime makes and runs playbooks; Studio manages them; your agent is a separate client.', 'A playbook contains one brain (its reusable functions) and workflows. The brain shares its lifecycle. Functions run directly or in workflows; runs belong to the playbook.', ]; @@ -90,18 +87,6 @@ const guides: Sentences = ({ listed }) => [ : 'get_guide holds what these words mean and how each kind of definition is written.', ]; -const modelsAndTools: Sentences = ({ reasoning, listed }) => { - if (reasoning === undefined || !listed('search')) { - return []; - } - const tools = 'may name tools that search finds'; - return [ - listed('list_models') - ? `A reasoning function names a model that list_models lists and ${tools}.` - : `A reasoning function names a model this server can call, as the ${reasoning.guide} guide says, and ${tools}.`, - ]; -}; - const connectingAService = [ "Accounts at services are connected through this server: asked to connect, link, sign in to or use a service, call search with its words, which lists the providers that match, then add_connection with a provider's name, which the person approves on the page it links.", 'Say a service cannot be connected only when no provider or server matches.', @@ -109,17 +94,6 @@ const connectingAService = [ const connections: Sentences = ({ listed }) => (listed('search') && listed('add_connection') ? connectingAService : []); -const typesThatFinishLater: ReadonlySet = new Set(['workflow']); - -const runsThatFinishLater: Sentences = ({ listed, definitionTypes }) => { - const finishingLater = definitionTypes - .filter(({ type }) => typesThatFinishLater.has(type)) - .map(({ noun }) => articled(noun)); - return listed('get_run') && finishingLater.length > 0 - ? [`A run of ${finishingLater.join(' or ')} answers started; get_run shows whether it ended or still waits.`] - : []; -}; - const answeringWhatRunsWaitOn: Sentences = ({ listed }) => listed('answer_interaction') ? [ @@ -134,9 +108,7 @@ const orientation: readonly Sentences[] = [ connection, modelsBeforeWriting, guides, - modelsAndTools, connections, - runsThatFinishLater, answeringWhatRunsWaitOn, closing, ]; @@ -165,5 +137,5 @@ export function instructionsFor( definitionTypes, reasoning: definitionTypes.find(({ type }) => type === reasoningFunctionType), }; - return [...whatAPlaybookIs, ...orientation.flatMap((sentences) => sentences(serving))].join(' '); + return [...whatThisServerIsFor, ...orientation.flatMap((sentences) => sentences(serving))].join(' '); } diff --git a/packages/api/src/mcp/tool-calling-instructions.test.ts b/packages/api/src/mcp/tool-calling-instructions.test.ts index 79265de86..82c2a2be5 100644 --- a/packages/api/src/mcp/tool-calling-instructions.test.ts +++ b/packages/api/src/mcp/tool-calling-instructions.test.ts @@ -12,10 +12,10 @@ import { import { instructionsFor } from './instructions.ts'; describe('the length of the instructions with every tool', () => { - it('take 1,996 characters on /mcp, 1,912 on the org endpoint and 1,621 on the playbook endpoint', () => { + it('take 1,987 characters on /mcp, 1,987 on the org endpoint and 1,575 on the playbook endpoint', () => { expect( everyEndpoint.map(([endpoint, served]) => instructionsFor(endpoint, served, definitionTypes).length), - ).toEqual([1912, 1621, 1996]); + ).toEqual([1987, 1575, 1987]); }); }); @@ -25,12 +25,12 @@ describe('what the instructions say of calling a tool', () => { playbookTools: queriesInsideAPlaybook, }; - it('end the sentence on tools at what search finds, leaving execute and dry_run to their own descriptions, and keep the closing sentence', () => { + it('leave the model and tools of a reasoning function, a run that finishes later, execute and dry_run to their tools and guides, and keep the closing sentence', () => { expect(instructionsFor('own org', ownOrg, definitionTypes)).toContain( - 'A reasoning function names a model that list_models lists and may name tools that search finds. A run of a workflow', + "Read a definition's format with get_guide; give it an overview. When the person approves", ); expect(instructionsFor('playbook', playbookEndpoint, definitionTypes)).toContain( - 'A reasoning function names a model this server can call, as the reasoning-function guide says, and may name tools that search finds. A run of a workflow', + "Read a definition's format with get_guide; give it an overview. When the person approves", ); expect( everyEndpoint.filter(([endpoint, served]) => @@ -55,7 +55,7 @@ describe('what the instructions say of calling a tool', () => { instructionsFor('playbook', { orgTools: [], playbookTools: queriesInsideAPlaybook }, definitionTypes), ]; - expect(lengths.map(({ length }) => length)).toEqual([1507, 1544, 1540, 1504]); + expect(lengths.map(({ length }) => length)).toEqual([1498, 1498, 1615, 1458]); }); }); @@ -65,12 +65,12 @@ describe('the instructions of an endpoint that lists search and add_connection', 'Say a service cannot be connected only when no provider or server matches.', ].join(' '); - it('say after the sentence on tools that a service is connected through search and add_connection, on /mcp and the org endpoint', () => { + it('say after the sentence on the guides that a service is connected through search and add_connection, on /mcp and the org endpoint', () => { expect(instructionsFor('own org', withConnections(ownOrg), definitionTypes)).toContain( - `may name tools that search finds. ${connecting} A run of a workflow`, + `give it an overview. ${connecting} When the person approves`, ); expect(instructionsFor('org', withConnections(orgEndpoint), definitionTypes)).toContain( - `may name tools that search finds. ${connecting} Say what was done`, + `how each kind of definition is written. ${connecting} Say what was done`, ); }); diff --git a/packages/api/src/openapi/document-description.ts b/packages/api/src/openapi/document-description.ts index 2e813cc15..fb4231f0d 100644 --- a/packages/api/src/openapi/document-description.ts +++ b/packages/api/src/openapi/document-description.ts @@ -1,8 +1,8 @@ import { guideToolName } from '../guides/guide-tool.ts'; -import { whatAPlaybookIs } from '../mcp/instructions.ts'; +import { whatThisServerIsFor } from '../mcp/instructions.ts'; export const documentDescription = [ - ...whatAPlaybookIs, + ...whatThisServerIsFor, `Before writing a definition, read its format with ${guideToolName}, which also holds what these words mean and the recipes for common tasks.`, 'Every operation answers a problem document when it does not succeed, and says in its description what it does, when to use it and which operation to use instead.', ].join(' '); diff --git a/packages/playbooks/src/operations/create-playbook.ts b/packages/playbooks/src/operations/create-playbook.ts index 22b0221a6..9f21a4c44 100644 --- a/packages/playbooks/src/operations/create-playbook.ts +++ b/packages/playbooks/src/operations/create-playbook.ts @@ -10,9 +10,10 @@ export const createPlaybook = defineCommand('org', { name: 'create_playbook', title: 'Create playbook', description: [ + 'Start here when the person wants something done in their business that should keep working after this conversation, however they put it, and none of their playbooks covers it.', "Creates a playbook in the org for one business responsibility, such as reviewing campaign briefs or keeping a team's meeting notes,", 'and returns it, active and empty: its functions and workflows come next.', - 'Use it when the person wants a new playbook; list_playbooks shows the playbooks that exist, and an id that is taken or retired is refused.', + 'list_playbooks shows the playbooks that exist, and an id that is taken or retired is refused.', "`playbook` is the id every tool inside the playbook takes, `name` is what people call it, and `description` says what the playbook is for, in the person's words.", ].join(' '), route: { method: 'POST', path: '/playbooks' }, diff --git a/packages/playbooks/src/operations/list-playbooks.ts b/packages/playbooks/src/operations/list-playbooks.ts index edcc469e4..4752a1dd8 100644 --- a/packages/playbooks/src/operations/list-playbooks.ts +++ b/packages/playbooks/src/operations/list-playbooks.ts @@ -11,8 +11,9 @@ export const listPlaybooks = defineQuery('org', { name: 'list_playbooks', title: 'List playbooks', description: [ + 'Use it when the person speaks of their playbooks, or wants something done that one of them may cover.', 'Lists the playbooks of the org that the caller may access, sorted by id, each with its name, what it is for and whether it is active.', - 'Use it to find a playbook the person names, or to see whether one exists before another is made; each playbook it lists holds every field of the playbook.', + 'Each playbook it lists holds every field of the playbook.', '`include_retired` adds the retired playbooks, which are left out otherwise.', ].join(' '), route: { method: 'GET', path: '/playbooks' }, diff --git a/packages/server/src/mcp/answering-what-runs-wait-on.test.ts b/packages/server/src/mcp/answering-what-runs-wait-on.test.ts index 8767f0c14..1f365d6e7 100644 --- a/packages/server/src/mcp/answering-what-runs-wait-on.test.ts +++ b/packages/server/src/mcp/answering-what-runs-wait-on.test.ts @@ -147,17 +147,21 @@ async function approvedThenNextMonth(session: McpSession) { } describe('episode 7: the person approves, in the chat, a draft a workflow sent through a tool', () => { - it('finds in the instructions, answer_interaction and the guide that the answer goes to the waiting request, and nothing that says to wait in a loop', async () => { + it('finds in the instructions, answer_interaction and the guide that the answer goes to the waiting request, and nowhere that it waits in a loop', async () => { const { instructions } = meetings.surfaces; const guide = await meetings.onMcp((session) => session.callTool('get_guide', { guide: 'interaction-function' })); expect(sentenceNaming(instructions, 'answer_interaction')).toBe( 'When the person approves, rejects or otherwise answers what a run waits on, answer its request with answer_interaction, not a new run.', ); - expect(sentenceNaming(instructions, 'get_run')).toBe( - 'A run of a workflow answers started; get_run shows whether it ended or still waits.', + expect(descriptionIn(meetings.surfaces, 'run_definition')).toContain( + 'a run that finishes later answers started with a run_id unless it ends before its first wait, and get_run shows how it ended.', ); - expect(instructions).not.toMatch(/\bpoll|\buntil (?:it ends|its status changes)/iu); + expect( + [instructions, descriptionIn(meetings.surfaces, 'run_definition')].filter((words) => + /\bpoll|\buntil (?:it ends|its status changes)/iu.test(words), + ), + ).toEqual([]); expect(descriptionIn(meetings.surfaces, 'answer_interaction')).toContain( 'Use it when the person approves, rejects, revises or otherwise answers a request list_interactions shows, wherever it reached them; a new run asks again and answers nothing,', ); diff --git a/packages/server/src/mcp/mcp-endpoint-workflows.test.ts b/packages/server/src/mcp/mcp-endpoint-workflows.test.ts index 604a7c6cd..161bb6f1b 100644 --- a/packages/server/src/mcp/mcp-endpoint-workflows.test.ts +++ b/packages/server/src/mcp/mcp-endpoint-workflows.test.ts @@ -1,4 +1,4 @@ -import { toolNamesIn, withMcpSession, type McpSession, type ToolResult } from '@on.auto/api/testing'; +import { listedTools, toolNamesIn, withMcpSession, type McpSession, type ToolResult } from '@on.auto/api/testing'; import { answers, textResult } from '@on.auto/reasoning/testing'; import { afterEach, describe, expect, it } from 'vitest'; @@ -73,18 +73,16 @@ async function welcomedAfterReply(session: McpSession): Promise { } describe('/mcp with workflows', () => { - it('lists twenty-eight tools, and its instructions say how to see whether a workflow run ended or still waits', async () => { + it('lists twenty-eight tools, and the description of run_definition says how to see whether a workflow run ended', async () => { server = await servingWorkflows([]); - const served = await onMcp(async (session) => ({ - tools: toolNamesIn(await session.listTools()), - instructions: session.instructions, - })); + const listing = await onMcp((session) => session.listTools()); + const tools = toolNamesIn(listing); - expect(served.tools).toHaveLength(28); - expect(served.tools.slice(-2)).toEqual(['send_run_event', 'get_guide']); - expect(served.instructions).toContain( - 'A run of a workflow answers started; get_run shows whether it ended or still waits.', + expect(tools).toHaveLength(28); + expect(tools.slice(-2)).toEqual(['send_run_event', 'get_guide']); + expect(listedTools(listing).find(({ name }) => name === 'run_definition')?.description).toContain( + 'a run that finishes later answers started with a run_id unless it ends before its first wait, and get_run shows how it ended.', ); }); diff --git a/packages/server/src/mcp/served-schemas.test.ts b/packages/server/src/mcp/served-schemas.test.ts index 4557f9b4b..2a5669bfb 100644 --- a/packages/server/src/mcp/served-schemas.test.ts +++ b/packages/server/src/mcp/served-schemas.test.ts @@ -87,8 +87,8 @@ interface Endpoint { } const endpoints: readonly Endpoint[] = [ - { path: '/mcp', tools: 28, bytes: 44_771, mostBytes: mostListingBytes['own org'] }, - { path: '/orgs/local/mcp', tools: 10, bytes: 13_019, mostBytes: mostListingBytes.org }, + { path: '/mcp', tools: 28, bytes: 44_907, mostBytes: mostListingBytes['own org'] }, + { path: '/orgs/local/mcp', tools: 10, bytes: 13_155, mostBytes: mostListingBytes.org }, { path: '/orgs/local/playbooks/alpha/mcp', tools: 20, bytes: 31_925, mostBytes: mostListingBytes.playbook }, ]; diff --git a/packages/server/src/mcp/speaking-to-agents.test.ts b/packages/server/src/mcp/speaking-to-agents.test.ts index 11fa5449f..dc0bbfd1c 100644 --- a/packages/server/src/mcp/speaking-to-agents.test.ts +++ b/packages/server/src/mcp/speaking-to-agents.test.ts @@ -55,7 +55,7 @@ describe('episode 1: asked "what is a playbook?"', () => { const answer = sentencesOf(surfaces.instructions).slice(0, 3).join(' '); expect(answer).toBe( - 'Auto Runtime executes playbooks; Studio manages them; your agent is a separate client. A playbook contains one brain (its reusable functions) and workflows. The brain shares its lifecycle.', + 'Whatever the person wants done that should keep working after this conversation, however they put it and in any line of business, is a playbook on this server, not a document, a plan, a note or a one-off answer. Auto Runtime makes and runs playbooks; Studio manages them; your agent is a separate client. A playbook contains one brain (its reusable functions) and workflows.', ); expect(internalTermsIn(answer)).toEqual([]); expect(answer).not.toMatch(/[a-z]+_[a-z_]+|\bid\b/u); @@ -67,7 +67,7 @@ describe('episode 2: asked to create a playbook for meetings', () => { const created = await onMcp((session) => session.callTool('create_playbook', meetingsPlaybook)); expect(sentencesOf(descriptionOf('create_playbook')).filter((sentence) => /\bid\b/u.test(sentence))).toEqual([ - 'Use it when the person wants a new playbook; list_playbooks shows the playbooks that exist, and an id that is taken or retired is refused.', + 'list_playbooks shows the playbooks that exist, and an id that is taken or retired is refused.', "`playbook` is the id every tool inside the playbook takes, `name` is what people call it, and `description` says what the playbook is for, in the person's words.", ]); expect(descriptionOf('create_playbook')).not.toMatch(/lowercase|never changes|immutable|reused/u); @@ -78,11 +78,11 @@ describe('episode 2: asked to create a playbook for meetings', () => { }); describe('episode 3: asked to use Slack', () => { - it('finds search in the instructions, and its answer names each tool server', async () => { + it('finds in the description of search the tools of the tool servers, and its answer names each tool server', async () => { const listed = await onMcp((session) => session.callTool('search', { playbook: 'meetings' })); - expect(sentenceNaming(surfaces.instructions, 'search finds')).toBe( - 'A reasoning function names a model that list_models lists and may name tools that search finds.', + expect(descriptionOf('search')).toContain( + "Searches the tools of the tool servers this playbook's functions may use, named in a function's tools as server/tool or server/* and called by execute.", ); expect(plainTextIn(listed)).toMatch(/^This playbook's functions may use 2 tool servers\. /u); expect(plainTextIn(listed)).toContain('“slack” offers'); @@ -244,12 +244,10 @@ async function lookedAtOverMcp(session: McpSession) { } describe('episode 8: asked what a gateway-style tool server offers, and then to post to #random', () => { - it('finds search in the instructions and what execute answers in its description, finds the search tool by its words, calls it, and makes no function to look', async () => { + it('finds what search and execute do in their descriptions, finds the search tool by its words, calls it, and makes no function to look', async () => { const seen = await onMcp(lookedAtOverMcp); - expect(sentenceNaming(surfaces.instructions, 'search finds')).toBe( - 'A reasoning function names a model that list_models lists and may name tools that search finds.', - ); + expect(descriptionOf('search')).toContain('Searches the tools of the tool servers'); expect(descriptionOf('execute')).toContain( "as a run of a reasoning function would call it, and answers what that run's model would see.", );